preserve tmp mountpoint on reset

.dockerignore

@@ -0,0 +1,17 @@
+target/
+.git/
+.gitignore
+perf.*
+**/* copy.excalidraw
+ui/rpki-explorer/node_modules/
+ui/rpki-explorer/dist/
+ui/rpki-explorer/playwright-report/
+ui/rpki-explorer/test-results/
+ui/rpki-explorer/.vite/
+deploy/arm64-compose/.env
+target/
+*.profraw
+*.profdata
+*.tar
+*.tar.gz
+*.zip

.gitignore

@@ -1,3 +1,9 @@
 target/
 Cargo.lock
 perf.*
+specs/* copy.excalidraw
+ui/rpki-explorer/node_modules/
+ui/rpki-explorer/dist/
+ui/rpki-explorer/playwright-report/
+ui/rpki-explorer/test-results/
+ui/rpki-explorer/.vite/

Cargo.toml

@@ -21,14 +21,15 @@ ring = "0.17.14"
 x509-parser = { version = "0.18.0", features = ["verify"] }
 url = "2.5.8"
 serde = { version = "1.0.218", features = ["derive"] }
-serde_json = "1.0.140"
+serde_json = { version = "1.0.140", features = ["raw_value"] }
 toml = "0.8.20"
 rocksdb = { version = "0.22.0", optional = true, default-features = false, features = ["lz4"] }
 serde_cbor = "0.11.2"
+memmap2 = "0.9.10"
 roxmltree = "0.20.0"
 quick-xml = "0.37.2"
 uuid = { version = "1.7.0", features = ["v4"] }
-reqwest = { version = "0.12.12", default-features = false, features = ["blocking", "rustls-tls"] }
+reqwest = { version = "0.12.12", default-features = false, features = ["blocking", "rustls-tls", "gzip", "brotli", "deflate"] }
 pprof = { version = "0.14.1", optional = true, features = ["flamegraph", "prost-codec"] }
 flate2 = { version = "1.0.35", optional = true }
 tempfile = "3.16.0"

README.md

@@ -26,3 +26,38 @@ cargo install cargo-llvm-cov --locked
 cargo llvm-cov --fail-under-lines 90
 ```
 
+默认会复用现有插桩产物，不会先 clean。需要强制全量重编译时：
+
+```
+COVERAGE_FORCE_CLEAN=1 ./scripts/coverage.sh
+```
+
+说明：
+- 默认行为适合本地重复确认覆盖率，避免每次都重编译整套插桩目标；
+- 默认还会设置 `RPKI_SKIP_HEAVY_SCRIPT_REPLAY_TESTS=1`，跳过会拉起 shell replay pipeline 的重型集成测试，避免 coverage 期间额外触发 `target/release` 构建；
+- 默认还会设置 `RPKI_SKIP_HEAVY_BLACKBOX_TESTS=1`，跳过更慢的 blackbox CLI / CIR record 脚本测试，进一步降低日常 coverage 成本；
+- 默认还会设置 `RPKI_SKIP_HEAVY_CRYPTO_TESTS=1`，跳过需要大量 OpenSSL 生成证书/CRL 的重型密码学测试，进一步压缩日常 coverage 时长；
+- 如需把这批脚本回放测试也纳入 coverage，可显式关闭该开关：
+
+```
+RPKI_SKIP_HEAVY_SCRIPT_REPLAY_TESTS=0 ./scripts/coverage.sh
+```
+
+如需连同第二批 blackbox 测试一起跑：
+
+```
+RPKI_SKIP_HEAVY_BLACKBOX_TESTS=0 ./scripts/coverage.sh
+```
+
+如需连同重型 OpenSSL 证书路径测试一起跑：
+
+```
+RPKI_SKIP_HEAVY_CRYPTO_TESTS=0 ./scripts/coverage.sh
+```
+
+- replay 脚本现在也支持通过环境变量注入现成二进制，避免找不到二进制时自动 `cargo build --release`：
+  - `RPKI_BIN`
+  - `CIR_MATERIALIZE_BIN`
+  - `CIR_EXTRACT_INPUTS_BIN`
+  - `CCR_TO_COMPARE_VIEWS_BIN`
+- `COVERAGE_FORCE_CLEAN=1` 适合需要完全从零重建插桩目标时使用。

deploy/arm64-compose/.env.example

@@ -0,0 +1,87 @@
+# ours RP ARM64 Docker Compose 示例配置
+# 复制为 .env 后再启动：
+#   cp .env.example .env
+
+# 业务镜像名称；需要先通过 docker load 导入远端 Docker。
+RPKI_IMAGE=ours-rp-runtime-arm64:dev
+
+# 核心 RP 镜像强制以 ARM64 运行；远端233为 x86_64，需要 binfmt/qemu 支持。
+RPKI_PLATFORM=linux/arm64
+
+# 验收默认只跑 APNIC；all5 可改成 afrinic,apnic,arin,lacnic,ripe。
+RIRS=apnic
+
+# 固定跑两轮：首轮 snapshot，第二轮 delta。
+# 负数表示持续运行；ARM64 QEMU 首版验收不建议默认持续运行。
+MAX_RUNS=2
+
+# 两轮之间间隔秒数。
+INTERVAL_SECS=0
+
+# 保留最近多少个 run 目录。
+RETAIN_RUNS=5
+
+# TA/TAL 输入模式：
+# - file-with-ta：完全使用镜像内 fixture；
+# - file-live-ta：snapshot 轮先完成 live TA 获取，delta 轮后台刷新 TA；
+# - url：从 TAL URL 拉取。
+TAL_INPUT_MODE=file-live-ta
+
+# rsync 默认按 module-root 批量拉取，和当前 ours RP soak 默认优化配置一致。
+RSYNC_SCOPE=module-root
+
+# 容器里不需要杀宿主机上的其他 RP 进程；远端宿主机如有竞争进程，请在宿主机侧处理。
+DISABLE_COMPETING_RPS=0
+
+# 运行态数据目录。容器内固定路径，外部由 compose volume 保存。
+RUN_ROOT=/var/lib/ours-rp
+DB_DIR=/var/lib/ours-rp/state/db
+RSYNC_MIRROR_ROOT=/var/lib/ours-rp/state/rsync-mirror
+
+# 每轮结束后清理 daemon 临时目录。
+# Docker Compose 中 tmp 是独立 volume 挂载点，不能删除挂载点本身；默认关闭。
+CLEAN_TMP_AFTER_RUN=0
+
+# 报告使用 compact JSON，降低写盘体积。
+OUTPUT_COMPACT_REPORT=1
+
+# 复用 rsync mirror，避免 delta 每轮从零拉取。
+ALLOW_RSYNC_MIRROR_REUSE=1
+
+# 前一轮失败时，新一轮从 snapshot 恢复。
+FAILURE_SNAPSHOT_RESET=1
+
+# QEMU 验收优先降低额外统计开销；需要精确 DB 统计时改为 3 或其他正整数。
+DB_STATS_EXACT_EVERY=0
+
+# 开启当前主线使用的验证缓存与请求预取能力。
+ENABLE_CHILD_CERTIFICATE_VALIDATION_CACHE=1
+RPKI_ANALYZE=1
+RPKI_EXTRA_ARGS="--enable-transport-request-prefetch --enable-publication-point-validation-cache --enable-roa-validation-cache --parallel-max-repo-sync-workers-global 4 --parallel-phase2-object-workers 4 --memory-trim-after-validation"
+
+# 进度日志阈值。QEMU 下执行较慢，阈值不宜过低。
+RPKI_PROGRESS_LOG=1
+RPKI_PROGRESS_SLOW_SECS=20
+RPKI_PROGRESS_STAGE_FRESH_SLOW_MS=2000
+RPKI_PROGRESS_PP_CONTROL_SLOW_MS=200
+RPKI_PROGRESS_PP_CACHE_SLOW_MS=100
+RPKI_PROGRESS_CONTROL_LOOP_SLOW_MS=2000
+
+# live TA 刷新超时。
+LIVE_TA_REFRESH_CONNECT_TIMEOUT_SECS=15
+LIVE_TA_REFRESH_MAX_TIME_SECS=120
+
+# file-live-ta 在 snapshot 轮先完成 live TA 获取再启动子进程，避免首轮使用旧 fixture TA。
+LIVE_TA_REFRESH_BEFORE_SNAPSHOT=1
+
+# metrics sidecar 配置。
+METRICS_INSTANCE=remote233-arm64-qemu
+METRICS_PORT=9556
+METRICS_POLL_SECS=10
+
+# Prometheus / Grafana 配置。
+PROMETHEUS_PORT=9090
+PROMETHEUS_RETENTION=7d
+GRAFANA_PORT=3000
+GRAFANA_ADMIN_USER=admin
+GRAFANA_ADMIN_PASSWORD=admin

deploy/arm64-compose/docker-compose.yml

@@ -0,0 +1,94 @@
+services:
+  ours-rp-soak:
+    image: ${RPKI_IMAGE:-ours-rp-runtime-arm64:dev}
+    platform: ${RPKI_PLATFORM:-linux/arm64}
+    container_name: ours-rp-arm64-soak
+    env_file:
+      - ./.env
+    environment:
+      PACKAGE_ROOT: /opt/ours-rp
+      ENV_FILE: /opt/ours-rp/.env
+      RUN_ROOT: /var/lib/ours-rp
+      BIN_DIR: /opt/ours-rp/bin
+      FIXTURE_DIR: /opt/ours-rp/fixtures
+    volumes:
+      - ./.env:/opt/ours-rp/.env:ro
+      - rpki-state:/var/lib/ours-rp/state
+      - rpki-runs:/var/lib/ours-rp/runs
+      - rpki-logs:/var/lib/ours-rp/logs
+      - rpki-tmp:/var/lib/ours-rp/tmp
+    restart: "no"
+    profiles:
+      - core
+
+  artifact-metrics:
+    image: ${RPKI_IMAGE:-ours-rp-runtime-arm64:dev}
+    platform: ${RPKI_PLATFORM:-linux/arm64}
+    container_name: ours-rp-arm64-artifact-metrics
+    env_file:
+      - ./.env
+    command:
+      - /opt/ours-rp/bin/rpki_artifact_metrics
+      - --run-root
+      - /var/lib/ours-rp
+      - --listen
+      - 0.0.0.0:9556
+      - --poll-secs
+      - ${METRICS_POLL_SECS:-10}
+      - --instance
+      - ${METRICS_INSTANCE:-remote233-arm64-qemu}
+    ports:
+      - "${METRICS_PORT:-9556}:9556"
+    volumes:
+      - rpki-state:/var/lib/ours-rp/state:ro
+      - rpki-runs:/var/lib/ours-rp/runs:ro
+      - rpki-logs:/var/lib/ours-rp/logs:ro
+    restart: unless-stopped
+    profiles:
+      - sidecar
+
+  prometheus:
+    image: ${PROMETHEUS_IMAGE:-prom/prometheus:v2.55.1}
+    container_name: ours-rp-arm64-prometheus
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.path=/prometheus
+      - --storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-7d}
+      - --web.enable-lifecycle
+    depends_on:
+      - artifact-metrics
+    ports:
+      - "${PROMETHEUS_PORT:-9090}:9090"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus-data:/prometheus
+    restart: unless-stopped
+    profiles:
+      - monitor
+
+  grafana:
+    image: ${GRAFANA_IMAGE:-grafana/grafana:11.3.1}
+    container_name: ours-rp-arm64-grafana
+    depends_on:
+      - prometheus
+    ports:
+      - "${GRAFANA_PORT:-3000}:3000"
+    environment:
+      GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
+      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin}
+      GF_USERS_ALLOW_SIGN_UP: "false"
+    volumes:
+      - grafana-data:/var/lib/grafana
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
+    restart: unless-stopped
+    profiles:
+      - monitor
+
+volumes:
+  rpki-state:
+  rpki-runs:
+  rpki-logs:
+  rpki-tmp:
+  prometheus-data:
+  grafana-data:

deploy/arm64-compose/grafana/dashboards/ours-rp-repo-sync.json

@@ -0,0 +1,761 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_publication_points",
+          "legendFormat": "publication points",
+          "refId": "A"
+        }
+      ],
+      "title": "Publication Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 6,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_ok\"}",
+          "legendFormat": "rrdp ok",
+          "refId": "A"
+        }
+      ],
+      "title": "RRDP OK Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 0
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "sum(ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_ok\"}) or vector(0)",
+          "legendFormat": "fallback",
+          "refId": "A"
+        }
+      ],
+      "title": "Rsync Fallback Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_repo_terminal_state_count{terminal_state=\"failed_no_cache\"}",
+          "legendFormat": "failed no cache",
+          "refId": "A"
+        }
+      ],
+      "title": "Failed No Cache Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 4
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"repo_sync_total\"}",
+          "legendFormat": "repo sync total",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"rrdp_download_total\"}",
+          "legendFormat": "rrdp download",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"rsync_download_total\"}",
+          "legendFormat": "rsync download",
+          "refId": "C"
+        }
+      ],
+      "title": "Repo Sync Download Durations",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 12,
+        "w": 12,
+        "h": 8
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repo_sync_phase_count",
+          "legendFormat": "{{phase}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Repo Sync Phase Counts",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 12,
+        "w": 12,
+        "h": 8
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_ok\"}",
+          "legendFormat": "rrdp failed, rsync ok",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_failed\"}",
+          "legendFormat": "rrdp failed, rsync failed",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_repo_terminal_state_count{terminal_state=\"failed_no_cache\"}",
+          "legendFormat": "failed no cache",
+          "refId": "C"
+        },
+        {
+          "expr": "ours_rp_tree_instances{state=\"failed\"}",
+          "legendFormat": "tree failed",
+          "refId": "D"
+        }
+      ],
+      "title": "Repo Failure / Fallback Counts",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 20,
+        "w": 12,
+        "h": 8
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repo_sync_phase_duration_seconds_total{phase=\"rrdp_failed_rsync_ok\"}",
+          "legendFormat": "rsync fallback duration",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_repo_sync_phase_duration_seconds_total{phase=\"rrdp_failed_rsync_failed\"}",
+          "legendFormat": "failed duration",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_repo_terminal_state_duration_seconds_total{terminal_state=\"failed_no_cache\"}",
+          "legendFormat": "failed no cache duration",
+          "refId": "C"
+        }
+      ],
+      "title": "Repo Failure / Fallback Durations",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 29,
+        "w": 12,
+        "h": 9
+      },
+      "id": 9,
+      "options": {
+        "showHeader": true,
+        "cellHeight": "sm",
+        "footer": {
+          "show": false,
+          "reducer": [
+            "sum"
+          ],
+          "countRows": false,
+          "fields": ""
+        }
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_rrdp_rsync_failed_repository_duration_seconds",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "RRDP + Rsync Failed Repositories",
+      "type": "table",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "job": true,
+              "terminal_state": true,
+              "rank": true,
+              "transport": true,
+              "__name__": true,
+              "publication_points": true,
+              "instance": true,
+              "repo_id": true,
+              "pp_id": true,
+              "exported_instance": true,
+              "rp": true,
+              "source": true
+            },
+            "indexByName": {
+              "Time": 0,
+              "host": 1,
+              "phase": 2,
+              "uri": 3,
+              "Value": 4
+            },
+            "renameByName": {
+              "Value": "duration"
+            }
+          }
+        }
+      ]
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 29,
+        "w": 12,
+        "h": 9
+      },
+      "id": 11,
+      "options": {
+        "showHeader": true,
+        "cellHeight": "sm",
+        "footer": {
+          "show": false,
+          "reducer": [
+            "sum"
+          ],
+          "countRows": false,
+          "fields": ""
+        }
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "topk(20, ours_rp_top_repository_sync_duration_seconds)",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Top 20 Repositories by Sync Duration",
+      "type": "table",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "job": true,
+              "terminal_state": true,
+              "__name__": true,
+              "publication_points": true,
+              "instance": true,
+              "repo_id": true,
+              "phase": true,
+              "pp_id": true,
+              "exported_instance": true,
+              "rp": true,
+              "source": true
+            },
+            "indexByName": {
+              "Time": 0,
+              "host": 1,
+              "rank": 2,
+              "transport": 3,
+              "uri": 4,
+              "Value": 5
+            },
+            "renameByName": {
+              "Value": "value"
+            }
+          }
+        }
+      ]
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 38,
+        "w": 24,
+        "h": 9
+      },
+      "id": 10,
+      "options": {
+        "showHeader": true,
+        "cellHeight": "sm",
+        "footer": {
+          "show": false,
+          "reducer": [
+            "sum"
+          ],
+          "countRows": false,
+          "fields": ""
+        }
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "topk(20, ours_rp_top_publication_point_object_count)",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Top Publication Points by Objects",
+      "type": "table",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "job": true,
+              "__name__": true,
+              "publication_points": true,
+              "instance": true,
+              "repo_id": true,
+              "phase": true,
+              "pp_id": true,
+              "exported_instance": true,
+              "rp": true,
+              "source": true
+            },
+            "indexByName": {
+              "Time": 0,
+              "host": 1,
+              "rank": 2,
+              "terminal_state": 3,
+              "transport": 4,
+              "uri": 5,
+              "Value": 6
+            },
+            "renameByName": {}
+          }
+        }
+      ]
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "description": "Per-repository sync success in the latest successful run; 1 means successful, 0 means failed or failed_no_cache.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bool"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 47
+      },
+      "id": 12,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repository_sync_success",
+          "legendFormat": "{{host}} {{repo_id}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Repository Sync Success by Repo",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "description": "Per-repository total sync duration aggregated from publication point repo_sync_duration_ms.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 55
+      },
+      "id": 13,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repository_sync_duration_seconds{stat=\"sum\"}",
+          "legendFormat": "{{host}} {{repo_id}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Repository Sync Duration by Repo",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "description": "Per-repository downloaded bytes attributed from report.json downloads events.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 63
+      },
+      "id": 14,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repository_download_bytes",
+          "legendFormat": "{{host}} {{repo_id}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Repository Download Bytes by Repo",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 40,
+  "tags": [
+    "ours-rp",
+    "rpki",
+    "soak",
+    "repo-sync"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Ours RP Repo Sync",
+  "uid": "ours-rp-repo-sync",
+  "version": 3,
+  "weekStart": ""
+}

deploy/arm64-compose/grafana/dashboards/ours-rp-soak-overview.json

@@ -0,0 +1,875 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_cir_trust_anchors",
+          "legendFormat": "RIRs",
+          "refId": "A"
+        }
+      ],
+      "title": "Current Run RIRs",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 6,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_run_duration_seconds",
+          "legendFormat": "wall",
+          "refId": "A"
+        }
+      ],
+      "title": "Latest Wall Time",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_run_max_rss_bytes",
+          "legendFormat": "rss",
+          "refId": "A"
+        }
+      ],
+      "title": "Latest Max RSS",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 18,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_publication_points",
+          "legendFormat": "publication points",
+          "refId": "A"
+        }
+      ],
+      "title": "Publication Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 4,
+        "w": 6,
+        "h": 4
+      },
+      "id": 9,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_run_sequence",
+          "legendFormat": "seq",
+          "refId": "A"
+        }
+      ],
+      "title": "Latest Run Sequence",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 2,
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 90
+              },
+              {
+                "color": "green",
+                "value": 98
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 6,
+        "y": 4,
+        "w": 6,
+        "h": 4
+      },
+      "id": 10,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "100 * sum by (job, instance, exported_instance) (ours_rp_repo_terminal_state_count{terminal_state=\"publication_point_cache\"}) / sum by (job, instance, exported_instance) (ours_rp_publication_points)",
+          "legendFormat": "PP cache hit ratio",
+          "refId": "A"
+        }
+      ],
+      "title": "Latest PP Cache Hit Ratio",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 4,
+        "w": 6,
+        "h": 4
+      },
+      "id": 11,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_vrps{kind=\"total\"}",
+          "legendFormat": "VRPs raw",
+          "refId": "A"
+        }
+      ],
+      "title": "VRPs",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 18,
+        "y": 4,
+        "w": 6,
+        "h": 4
+      },
+      "id": 12,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_vaps",
+          "legendFormat": "VAPs",
+          "refId": "A"
+        }
+      ],
+      "title": "VAPs",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 8,
+        "w": 12,
+        "h": 8
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_run_duration_seconds",
+          "legendFormat": "wall",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_stage_duration_seconds{stage=\"validation\"}",
+          "legendFormat": "validation",
+          "refId": "B"
+        }
+      ],
+      "title": "Run / Validation Duration",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 8,
+        "w": 12,
+        "h": 8
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_vrps{kind=\"total\"}",
+          "legendFormat": "VRPs raw",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_vrps{kind=\"unique\"}",
+          "legendFormat": "VRPs unique",
+          "refId": "D"
+        },
+        {
+          "expr": "ours_rp_vaps",
+          "legendFormat": "VAPs",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_cir_objects",
+          "legendFormat": "CIR objects",
+          "refId": "C"
+        }
+      ],
+      "title": "Output and Input Sizes",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 16,
+        "w": 12,
+        "h": 8
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_large_publication_points",
+          "legendFormat": "> {{object_count_gt}} objects",
+          "refId": "A"
+        }
+      ],
+      "title": "Large Publication Points by Object Count",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 16,
+        "w": 12,
+        "h": 8
+      },
+      "id": 13,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"validation\"}",
+          "legendFormat": "validation",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"report_write\"}",
+          "legendFormat": "report write",
+          "refId": "E"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"ccr_write\"}",
+          "legendFormat": "ccr write",
+          "refId": "F"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"cir_write\"}",
+          "legendFormat": "cir write",
+          "refId": "G"
+        }
+      ],
+      "title": "Output Stage Durations",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "decimals": 2
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 24,
+        "w": 12,
+        "h": 8
+      },
+      "id": 14,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_run_max_rss_bytes",
+          "legendFormat": "Max RSS",
+          "refId": "A"
+        }
+      ],
+      "title": "Max RSS Over Time",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "decimals": 2,
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 90
+              },
+              {
+                "color": "green",
+                "value": 98
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 24,
+        "w": 12,
+        "h": 8
+      },
+      "id": 17,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "min",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "100 * sum by (job, instance, exported_instance) (ours_rp_repo_terminal_state_count{terminal_state=\"publication_point_cache\"}) / sum by (job, instance, exported_instance) (ours_rp_publication_points)",
+          "legendFormat": "PP cache hit ratio",
+          "refId": "A"
+        }
+      ],
+      "title": "PP Cache Hit Ratio",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "decimals": 2
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 32,
+        "w": 24,
+        "h": 8
+      },
+      "id": 15,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_state_db_size_bytes",
+          "legendFormat": "{{db}}",
+          "refId": "A"
+        }
+      ],
+      "title": "State DB Size Over Time",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 40,
+        "w": 24,
+        "h": 8
+      },
+      "id": 16,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_state_db_files",
+          "legendFormat": "{{db}}",
+          "refId": "A"
+        }
+      ],
+      "title": "State DB File Count Over Time",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0,
+          "min": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 48,
+        "w": 24,
+        "h": 8
+      },
+      "id": 18,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "sum by (job, instance, exported_instance) (ours_rp_repo_terminal_state_count{terminal_state=\"fresh\"})",
+          "legendFormat": "fresh pp",
+          "refId": "A"
+        },
+        {
+          "expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"roa\"})",
+          "legendFormat": "fresh roa",
+          "refId": "B"
+        },
+        {
+          "expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"manifest\"})",
+          "legendFormat": "fresh mft",
+          "refId": "C"
+        },
+        {
+          "expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"certificate\"})",
+          "legendFormat": "fresh crt",
+          "refId": "D"
+        },
+        {
+          "expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"crl\"})",
+          "legendFormat": "fresh crl",
+          "refId": "E"
+        }
+      ],
+      "title": "Fresh PP / Object Counts by Run",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 40,
+  "tags": [
+    "ours-rp",
+    "rpki",
+    "soak"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Ours RP Soak Overview",
+  "uid": "ours-rp-soak-overview",
+  "version": 4,
+  "weekStart": ""
+}

deploy/arm64-compose/grafana/provisioning/dashboards/dashboard.yml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+providers:
+  - name: ours-rp-arm64
+    orgId: 1
+    folder: Ours RP ARM64
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 10
+    allowUiUpdates: true
+    options:
+      path: /var/lib/grafana/dashboards

deploy/arm64-compose/grafana/provisioning/datasources/prometheus.yml

@@ -0,0 +1,10 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    uid: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: true

deploy/arm64-compose/prometheus.yml

@@ -0,0 +1,13 @@
+global:
+  scrape_interval: 5s
+  evaluation_interval: 5s
+
+scrape_configs:
+  - job_name: ours-rp-artifact-metrics
+    metrics_path: /metrics
+    static_configs:
+      - targets:
+          - artifact-metrics:9556
+        labels:
+          rp: ours-rp
+          source: arm64-compose-artifact-sidecar

deploy/arm64-installer/.env.example

@@ -0,0 +1,83 @@
+# ours RP ARM64 installer configuration
+# 中文说明见 docs/README.zh-CN.md。English guide: docs/README.en.md
+
+# Compose project name.
+COMPOSE_PROJECT_NAME=ours-rp-arm64
+
+# Runtime image loaded from images/*.tar.gz by install.sh.
+RPKI_IMAGE=ours-rp-runtime-arm64:dev
+RPKI_PLATFORM=linux/arm64
+
+# Restart policy for the soak container. Production default keeps the daemon alive.
+# For finite acceptance tests such as MAX_RUNS=3, set SOAK_RESTART_POLICY=no to avoid an extra restarted run.
+SOAK_RESTART_POLICY=unless-stopped
+
+# Host-side persistent data directory. All state/runs/logs/monitoring data are bind-mounted here.
+HOST_DATA_DIR=/var/lib/ours-rp-arm64
+
+# RIR list. Options: afrinic,apnic,arin,lacnic,ripe
+RIRS=afrinic,apnic,arin,lacnic,ripe
+
+# Negative MAX_RUNS means keep running forever. Default production interval is 10 minutes.
+MAX_RUNS=-1
+INTERVAL_SECS=600
+RETAIN_RUNS=100
+
+# TAL/TA input mode:
+# file-with-ta: use packaged fixture TAL + TA only.
+# file-live-ta: use packaged fixture TAL; snapshot waits for live TA refresh, delta refreshes TA in background.
+# url: pass TAL URL to child process.
+TAL_INPUT_MODE=file-live-ta
+LIVE_TA_REFRESH_BEFORE_SNAPSHOT=1
+LIVE_TA_REFRESH_CONNECT_TIMEOUT_SECS=15
+LIVE_TA_REFRESH_MAX_TIME_SECS=120
+
+# Sync and runtime behavior.
+RSYNC_SCOPE=module-root
+DISABLE_COMPETING_RPS=0
+RUN_ROOT=/var/lib/ours-rp
+DB_DIR=/var/lib/ours-rp/state/db
+RSYNC_MIRROR_ROOT=/var/lib/ours-rp/state/rsync-mirror
+CLEAN_TMP_AFTER_RUN=0
+OUTPUT_COMPACT_REPORT=1
+ALLOW_RSYNC_MIRROR_REUSE=1
+FAILURE_SNAPSHOT_RESET=1
+# Periodic snapshot reset of active state DB.
+# 0: keep existing behavior.
+# 1: after one successful snapshot, allow at most N successful delta runs;
+#    the next run is forced to snapshot and active state/db is rebuilt from empty.
+PERIODIC_SNAPSHOT_RESET=0
+PERIODIC_SNAPSHOT_MAX_DELTAS=100
+DB_STATS_EXACT_EVERY=0
+
+# Validation and performance options aligned with current optimized soak defaults.
+ENABLE_CHILD_CERTIFICATE_VALIDATION_CACHE=1
+RPKI_ANALYZE=1
+RPKI_EXTRA_ARGS="--enable-transport-request-prefetch --enable-publication-point-validation-cache --enable-roa-validation-cache --parallel-max-repo-sync-workers-global 4 --parallel-phase2-object-workers 4 --memory-trim-after-validation"
+
+# Progress logs.
+RPKI_PROGRESS_LOG=1
+RPKI_PROGRESS_SLOW_SECS=20
+RPKI_PROGRESS_STAGE_FRESH_SLOW_MS=2000
+RPKI_PROGRESS_PP_CONTROL_SLOW_MS=200
+RPKI_PROGRESS_PP_CACHE_SLOW_MS=100
+RPKI_PROGRESS_CONTROL_LOOP_SLOW_MS=2000
+
+# Metrics sidecar.
+METRICS_INSTANCE=arm64-installer
+METRICS_PORT=9556
+METRICS_POLL_SECS=10
+
+# Prometheus / Grafana.
+# Monitor images are packaged as ARM64 docker-save archives and loaded by install.sh.
+MONITOR_PLATFORM=linux/arm64
+PROMETHEUS_IMAGE=prom/prometheus:v2.55.1
+GRAFANA_IMAGE=grafana/grafana:11.3.1
+PROMETHEUS_PORT=9090
+PROMETHEUS_RETENTION=7d
+GRAFANA_PORT=3000
+GRAFANA_ADMIN_USER=admin
+GRAFANA_ADMIN_PASSWORD=admin
+
+# First snapshot waiting timeout used by start.sh.
+FIRST_RUN_WAIT_TIMEOUT_SECS=7200

deploy/arm64-installer/cleanup.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=scripts/common.sh
+source "$SCRIPT_DIR/scripts/common.sh"
+
+DRY_RUN=1
+KEEP_RUNS=""
+usage() {
+  cat <<'USAGE'
+Usage: ./cleanup.sh [--execute] [--keep-runs N]
+
+By default this is a dry-run. It removes old run_* directories beyond KEEP_RUNS
+and clears tmp contents.
+USAGE
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --execute)
+      DRY_RUN=0
+      shift
+      ;;
+    --keep-runs)
+      KEEP_RUNS="$2"
+      shift 2
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      die "unknown option: $1"
+      ;;
+  esac
+done
+
+load_env
+keep="${KEEP_RUNS:-${RETAIN_RUNS:-100}}"
+mapfile -t runs < <(find "$HOST_DATA_DIR/runs" -maxdepth 1 -type d -name 'run_*' 2>/dev/null | sort)
+delete_count=$(( ${#runs[@]} - keep ))
+if (( delete_count > 0 )); then
+  for ((i=0; i<delete_count; i++)); do
+    if [[ "$DRY_RUN" == "1" ]]; then
+      echo "DRY-RUN rm -rf ${runs[$i]}"
+    else
+      rm -rf "${runs[$i]}"
+    fi
+  done
+fi
+if [[ "$DRY_RUN" == "1" ]]; then
+  echo "DRY-RUN rm -rf $HOST_DATA_DIR/tmp/*"
+else
+  find "$HOST_DATA_DIR/tmp" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
+fi
+df -h "$HOST_DATA_DIR" 2>/dev/null || true

deploy/arm64-installer/compose/docker-compose.yml

@@ -0,0 +1,90 @@
+services:
+  ours-rp-soak:
+    image: ${RPKI_IMAGE:-ours-rp-runtime-arm64:dev}
+    platform: ${RPKI_PLATFORM:-linux/arm64}
+    container_name: ${COMPOSE_PROJECT_NAME:-ours-rp-arm64}-soak
+    env_file:
+      - ../.env
+    environment:
+      PACKAGE_ROOT: /opt/ours-rp
+      ENV_FILE: /opt/ours-rp/.env
+      RUN_ROOT: /var/lib/ours-rp
+      BIN_DIR: /opt/ours-rp/bin
+      FIXTURE_DIR: /opt/ours-rp/fixtures
+    volumes:
+      - ../.env:/opt/ours-rp/.env:ro
+      - ${HOST_DATA_DIR:-/var/lib/ours-rp-arm64}/state:/var/lib/ours-rp/state
+      - ${HOST_DATA_DIR:-/var/lib/ours-rp-arm64}/runs:/var/lib/ours-rp/runs
+      - ${HOST_DATA_DIR:-/var/lib/ours-rp-arm64}/logs:/var/lib/ours-rp/logs
+      - ${HOST_DATA_DIR:-/var/lib/ours-rp-arm64}/tmp:/var/lib/ours-rp/tmp
+    restart: ${SOAK_RESTART_POLICY:-unless-stopped}
+    profiles:
+      - core
+
+  artifact-metrics:
+    image: ${RPKI_IMAGE:-ours-rp-runtime-arm64:dev}
+    platform: ${RPKI_PLATFORM:-linux/arm64}
+    container_name: ${COMPOSE_PROJECT_NAME:-ours-rp-arm64}-artifact-metrics
+    env_file:
+      - ../.env
+    command:
+      - /opt/ours-rp/bin/rpki_artifact_metrics
+      - --run-root
+      - /var/lib/ours-rp
+      - --listen
+      - 0.0.0.0:9556
+      - --poll-secs
+      - ${METRICS_POLL_SECS:-10}
+      - --instance
+      - ${METRICS_INSTANCE:-arm64-installer}
+    ports:
+      - "${METRICS_PORT:-9556}:9556"
+    volumes:
+      - ${HOST_DATA_DIR:-/var/lib/ours-rp-arm64}/state:/var/lib/ours-rp/state:ro
+      - ${HOST_DATA_DIR:-/var/lib/ours-rp-arm64}/runs:/var/lib/ours-rp/runs:ro
+      - ${HOST_DATA_DIR:-/var/lib/ours-rp-arm64}/logs:/var/lib/ours-rp/logs:ro
+    restart: unless-stopped
+    profiles:
+      - sidecar
+
+  prometheus:
+    image: ${PROMETHEUS_IMAGE:-prom/prometheus:v2.55.1}
+    platform: ${MONITOR_PLATFORM:-linux/arm64}
+    container_name: ${COMPOSE_PROJECT_NAME:-ours-rp-arm64}-prometheus
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.path=/prometheus
+      - --storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-7d}
+      - --web.enable-lifecycle
+    depends_on:
+      - artifact-metrics
+    user: "0:0"
+    ports:
+      - "${PROMETHEUS_PORT:-9090}:9090"
+    volumes:
+      - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ${HOST_DATA_DIR:-/var/lib/ours-rp-arm64}/prometheus:/prometheus
+    restart: unless-stopped
+    profiles:
+      - monitor
+
+  grafana:
+    image: ${GRAFANA_IMAGE:-grafana/grafana:11.3.1}
+    platform: ${MONITOR_PLATFORM:-linux/arm64}
+    container_name: ${COMPOSE_PROJECT_NAME:-ours-rp-arm64}-grafana
+    depends_on:
+      - prometheus
+    user: "0:0"
+    ports:
+      - "${GRAFANA_PORT:-3000}:3000"
+    environment:
+      GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
+      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin}
+      GF_USERS_ALLOW_SIGN_UP: "false"
+    volumes:
+      - ${HOST_DATA_DIR:-/var/lib/ours-rp-arm64}/grafana:/var/lib/grafana
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
+    restart: unless-stopped
+    profiles:
+      - monitor

deploy/arm64-installer/compose/grafana/dashboards/ours-rp-repo-sync.json

@@ -0,0 +1,761 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_publication_points",
+          "legendFormat": "publication points",
+          "refId": "A"
+        }
+      ],
+      "title": "Publication Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 6,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_ok\"}",
+          "legendFormat": "rrdp ok",
+          "refId": "A"
+        }
+      ],
+      "title": "RRDP OK Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 0
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "sum(ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_ok\"}) or vector(0)",
+          "legendFormat": "fallback",
+          "refId": "A"
+        }
+      ],
+      "title": "Rsync Fallback Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_repo_terminal_state_count{terminal_state=\"failed_no_cache\"}",
+          "legendFormat": "failed no cache",
+          "refId": "A"
+        }
+      ],
+      "title": "Failed No Cache Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 4
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"repo_sync_total\"}",
+          "legendFormat": "repo sync total",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"rrdp_download_total\"}",
+          "legendFormat": "rrdp download",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"rsync_download_total\"}",
+          "legendFormat": "rsync download",
+          "refId": "C"
+        }
+      ],
+      "title": "Repo Sync Download Durations",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 12,
+        "w": 12,
+        "h": 8
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repo_sync_phase_count",
+          "legendFormat": "{{phase}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Repo Sync Phase Counts",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 12,
+        "w": 12,
+        "h": 8
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_ok\"}",
+          "legendFormat": "rrdp failed, rsync ok",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_failed\"}",
+          "legendFormat": "rrdp failed, rsync failed",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_repo_terminal_state_count{terminal_state=\"failed_no_cache\"}",
+          "legendFormat": "failed no cache",
+          "refId": "C"
+        },
+        {
+          "expr": "ours_rp_tree_instances{state=\"failed\"}",
+          "legendFormat": "tree failed",
+          "refId": "D"
+        }
+      ],
+      "title": "Repo Failure / Fallback Counts",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 20,
+        "w": 12,
+        "h": 8
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repo_sync_phase_duration_seconds_total{phase=\"rrdp_failed_rsync_ok\"}",
+          "legendFormat": "rsync fallback duration",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_repo_sync_phase_duration_seconds_total{phase=\"rrdp_failed_rsync_failed\"}",
+          "legendFormat": "failed duration",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_repo_terminal_state_duration_seconds_total{terminal_state=\"failed_no_cache\"}",
+          "legendFormat": "failed no cache duration",
+          "refId": "C"
+        }
+      ],
+      "title": "Repo Failure / Fallback Durations",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 29,
+        "w": 12,
+        "h": 9
+      },
+      "id": 9,
+      "options": {
+        "showHeader": true,
+        "cellHeight": "sm",
+        "footer": {
+          "show": false,
+          "reducer": [
+            "sum"
+          ],
+          "countRows": false,
+          "fields": ""
+        }
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_rrdp_rsync_failed_repository_duration_seconds",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "RRDP + Rsync Failed Repositories",
+      "type": "table",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "job": true,
+              "terminal_state": true,
+              "rank": true,
+              "transport": true,
+              "__name__": true,
+              "publication_points": true,
+              "instance": true,
+              "repo_id": true,
+              "pp_id": true,
+              "exported_instance": true,
+              "rp": true,
+              "source": true
+            },
+            "indexByName": {
+              "Time": 0,
+              "host": 1,
+              "phase": 2,
+              "uri": 3,
+              "Value": 4
+            },
+            "renameByName": {
+              "Value": "duration"
+            }
+          }
+        }
+      ]
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 29,
+        "w": 12,
+        "h": 9
+      },
+      "id": 11,
+      "options": {
+        "showHeader": true,
+        "cellHeight": "sm",
+        "footer": {
+          "show": false,
+          "reducer": [
+            "sum"
+          ],
+          "countRows": false,
+          "fields": ""
+        }
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "topk(20, ours_rp_top_repository_sync_duration_seconds)",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Top 20 Repositories by Sync Duration",
+      "type": "table",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "job": true,
+              "terminal_state": true,
+              "__name__": true,
+              "publication_points": true,
+              "instance": true,
+              "repo_id": true,
+              "phase": true,
+              "pp_id": true,
+              "exported_instance": true,
+              "rp": true,
+              "source": true
+            },
+            "indexByName": {
+              "Time": 0,
+              "host": 1,
+              "rank": 2,
+              "transport": 3,
+              "uri": 4,
+              "Value": 5
+            },
+            "renameByName": {
+              "Value": "value"
+            }
+          }
+        }
+      ]
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 38,
+        "w": 24,
+        "h": 9
+      },
+      "id": 10,
+      "options": {
+        "showHeader": true,
+        "cellHeight": "sm",
+        "footer": {
+          "show": false,
+          "reducer": [
+            "sum"
+          ],
+          "countRows": false,
+          "fields": ""
+        }
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "topk(20, ours_rp_top_publication_point_object_count)",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Top Publication Points by Objects",
+      "type": "table",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "job": true,
+              "__name__": true,
+              "publication_points": true,
+              "instance": true,
+              "repo_id": true,
+              "phase": true,
+              "pp_id": true,
+              "exported_instance": true,
+              "rp": true,
+              "source": true
+            },
+            "indexByName": {
+              "Time": 0,
+              "host": 1,
+              "rank": 2,
+              "terminal_state": 3,
+              "transport": 4,
+              "uri": 5,
+              "Value": 6
+            },
+            "renameByName": {}
+          }
+        }
+      ]
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "description": "Per-repository sync success in the latest successful run; 1 means successful, 0 means failed or failed_no_cache.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bool"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 47
+      },
+      "id": 12,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repository_sync_success",
+          "legendFormat": "{{host}} {{repo_id}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Repository Sync Success by Repo",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "description": "Per-repository total sync duration aggregated from publication point repo_sync_duration_ms.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 55
+      },
+      "id": 13,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repository_sync_duration_seconds{stat=\"sum\"}",
+          "legendFormat": "{{host}} {{repo_id}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Repository Sync Duration by Repo",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "description": "Per-repository downloaded bytes attributed from report.json downloads events.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 63
+      },
+      "id": 14,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repository_download_bytes",
+          "legendFormat": "{{host}} {{repo_id}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Repository Download Bytes by Repo",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 40,
+  "tags": [
+    "ours-rp",
+    "rpki",
+    "soak",
+    "repo-sync"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Ours RP Repo Sync",
+  "uid": "ours-rp-repo-sync",
+  "version": 3,
+  "weekStart": ""
+}

deploy/arm64-installer/compose/grafana/dashboards/ours-rp-soak-overview.json

@@ -0,0 +1,875 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_cir_trust_anchors",
+          "legendFormat": "RIRs",
+          "refId": "A"
+        }
+      ],
+      "title": "Current Run RIRs",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 6,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_run_duration_seconds",
+          "legendFormat": "wall",
+          "refId": "A"
+        }
+      ],
+      "title": "Latest Wall Time",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_run_max_rss_bytes",
+          "legendFormat": "rss",
+          "refId": "A"
+        }
+      ],
+      "title": "Latest Max RSS",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 18,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_publication_points",
+          "legendFormat": "publication points",
+          "refId": "A"
+        }
+      ],
+      "title": "Publication Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 4,
+        "w": 6,
+        "h": 4
+      },
+      "id": 9,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_run_sequence",
+          "legendFormat": "seq",
+          "refId": "A"
+        }
+      ],
+      "title": "Latest Run Sequence",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 2,
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 90
+              },
+              {
+                "color": "green",
+                "value": 98
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 6,
+        "y": 4,
+        "w": 6,
+        "h": 4
+      },
+      "id": 10,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "100 * sum by (job, instance, exported_instance) (ours_rp_repo_terminal_state_count{terminal_state=\"publication_point_cache\"}) / sum by (job, instance, exported_instance) (ours_rp_publication_points)",
+          "legendFormat": "PP cache hit ratio",
+          "refId": "A"
+        }
+      ],
+      "title": "Latest PP Cache Hit Ratio",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 4,
+        "w": 6,
+        "h": 4
+      },
+      "id": 11,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_vrps{kind=\"total\"}",
+          "legendFormat": "VRPs raw",
+          "refId": "A"
+        }
+      ],
+      "title": "VRPs",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 18,
+        "y": 4,
+        "w": 6,
+        "h": 4
+      },
+      "id": 12,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_vaps",
+          "legendFormat": "VAPs",
+          "refId": "A"
+        }
+      ],
+      "title": "VAPs",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 8,
+        "w": 12,
+        "h": 8
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_run_duration_seconds",
+          "legendFormat": "wall",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_stage_duration_seconds{stage=\"validation\"}",
+          "legendFormat": "validation",
+          "refId": "B"
+        }
+      ],
+      "title": "Run / Validation Duration",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 8,
+        "w": 12,
+        "h": 8
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_vrps{kind=\"total\"}",
+          "legendFormat": "VRPs raw",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_vrps{kind=\"unique\"}",
+          "legendFormat": "VRPs unique",
+          "refId": "D"
+        },
+        {
+          "expr": "ours_rp_vaps",
+          "legendFormat": "VAPs",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_cir_objects",
+          "legendFormat": "CIR objects",
+          "refId": "C"
+        }
+      ],
+      "title": "Output and Input Sizes",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 16,
+        "w": 12,
+        "h": 8
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_large_publication_points",
+          "legendFormat": "> {{object_count_gt}} objects",
+          "refId": "A"
+        }
+      ],
+      "title": "Large Publication Points by Object Count",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 16,
+        "w": 12,
+        "h": 8
+      },
+      "id": 13,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"validation\"}",
+          "legendFormat": "validation",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"report_write\"}",
+          "legendFormat": "report write",
+          "refId": "E"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"ccr_write\"}",
+          "legendFormat": "ccr write",
+          "refId": "F"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"cir_write\"}",
+          "legendFormat": "cir write",
+          "refId": "G"
+        }
+      ],
+      "title": "Output Stage Durations",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "decimals": 2
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 24,
+        "w": 12,
+        "h": 8
+      },
+      "id": 14,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_run_max_rss_bytes",
+          "legendFormat": "Max RSS",
+          "refId": "A"
+        }
+      ],
+      "title": "Max RSS Over Time",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "decimals": 2,
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 90
+              },
+              {
+                "color": "green",
+                "value": 98
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 24,
+        "w": 12,
+        "h": 8
+      },
+      "id": 17,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "min",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "100 * sum by (job, instance, exported_instance) (ours_rp_repo_terminal_state_count{terminal_state=\"publication_point_cache\"}) / sum by (job, instance, exported_instance) (ours_rp_publication_points)",
+          "legendFormat": "PP cache hit ratio",
+          "refId": "A"
+        }
+      ],
+      "title": "PP Cache Hit Ratio",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "decimals": 2
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 32,
+        "w": 24,
+        "h": 8
+      },
+      "id": 15,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_state_db_size_bytes",
+          "legendFormat": "{{db}}",
+          "refId": "A"
+        }
+      ],
+      "title": "State DB Size Over Time",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 40,
+        "w": 24,
+        "h": 8
+      },
+      "id": 16,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_state_db_files",
+          "legendFormat": "{{db}}",
+          "refId": "A"
+        }
+      ],
+      "title": "State DB File Count Over Time",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0,
+          "min": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 48,
+        "w": 24,
+        "h": 8
+      },
+      "id": 18,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "sum by (job, instance, exported_instance) (ours_rp_repo_terminal_state_count{terminal_state=\"fresh\"})",
+          "legendFormat": "fresh pp",
+          "refId": "A"
+        },
+        {
+          "expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"roa\"})",
+          "legendFormat": "fresh roa",
+          "refId": "B"
+        },
+        {
+          "expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"manifest\"})",
+          "legendFormat": "fresh mft",
+          "refId": "C"
+        },
+        {
+          "expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"certificate\"})",
+          "legendFormat": "fresh crt",
+          "refId": "D"
+        },
+        {
+          "expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"crl\"})",
+          "legendFormat": "fresh crl",
+          "refId": "E"
+        }
+      ],
+      "title": "Fresh PP / Object Counts by Run",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 40,
+  "tags": [
+    "ours-rp",
+    "rpki",
+    "soak"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Ours RP Soak Overview",
+  "uid": "ours-rp-soak-overview",
+  "version": 4,
+  "weekStart": ""
+}

deploy/arm64-installer/compose/grafana/provisioning/dashboards/dashboard.yml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+providers:
+  - name: ours-rp-arm64
+    orgId: 1
+    folder: Ours RP ARM64
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 10
+    allowUiUpdates: true
+    options:
+      path: /var/lib/grafana/dashboards

deploy/arm64-installer/compose/grafana/provisioning/datasources/prometheus.yml

@@ -0,0 +1,10 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    uid: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: true

deploy/arm64-installer/compose/prometheus.yml

@@ -0,0 +1,13 @@
+global:
+  scrape_interval: 5s
+  evaluation_interval: 5s
+
+scrape_configs:
+  - job_name: ours-rp-artifact-metrics
+    metrics_path: /metrics
+    static_configs:
+      - targets:
+          - artifact-metrics:9556
+        labels:
+          rp: ours-rp
+          source: arm64-compose-artifact-sidecar

deploy/arm64-installer/docs/README.en.md

@@ -0,0 +1,122 @@
+# ours RP ARM64 Installer Guide
+
+## Goal
+
+This package deploys ours RP on a `linux/arm64` server with Docker Compose and continuously runs all five RIR validation.
+
+The package includes the ours RP ARM64 runtime image, Prometheus ARM64 image and Grafana ARM64 image, so deployment does not pull application images on the target host. Runtime state, run artifacts, logs, Prometheus data and Grafana data are persisted through host bind mounts.
+
+## Quick Start
+
+```bash
+tar -xzf ours-rp-arm64-installer-*.tar.gz
+cd ours-rp-arm64-installer-*
+
+./install.sh
+cp .env.example .env   # install.sh creates .env automatically if missing
+vim .env
+./start.sh
+./status.sh
+```
+
+Defaults:
+
+- `RIRS=afrinic,apnic,arin,lacnic,ripe`
+- `MAX_RUNS=-1`
+- `INTERVAL_SECS=600`
+- `TAL_INPUT_MODE=file-live-ta`
+- `LIVE_TA_REFRESH_BEFORE_SNAPSHOT=1`
+- `PERIODIC_SNAPSHOT_RESET=0`
+- `PERIODIC_SNAPSHOT_MAX_DELTAS=100`
+- `HOST_DATA_DIR=/var/lib/ours-rp-arm64`
+- `SOAK_RESTART_POLICY=unless-stopped`
+- `MONITOR_PLATFORM=linux/arm64`
+
+## First Start Semantics
+
+If there is no successful run under `HOST_DATA_DIR/runs`, `start.sh` starts the core `ours-rp-soak` service first and waits for the first snapshot to succeed before starting metrics, Prometheus and Grafana.
+
+The first snapshot refreshes live TA certificates before starting the RP process.
+
+## Ports
+
+Default ports:
+
+- metrics: `http://<host>:9556/metrics`
+- Prometheus: `http://<host>:9090`
+- Grafana: `http://<host>:3000`
+
+Grafana credentials come from `.env`:
+
+```bash
+GRAFANA_ADMIN_USER=admin
+GRAFANA_ADMIN_PASSWORD=admin
+```
+
+Change the password and restrict public access for production deployments.
+
+## Data Directory
+
+Default host directory:
+
+```text
+/var/lib/ours-rp-arm64/
+  state/
+  runs/
+  logs/
+  tmp/
+  prometheus/
+  grafana/
+```
+
+Each `runs/run_XXXX/` directory contains `report.json`, `result.ccr`, `input.cir`, `vrps.csv`, `vaps.csv`, `stage-timing.json`, logs and metadata.
+
+## Periodic Snapshot Reset
+
+New knobs:
+
+```bash
+PERIODIC_SNAPSHOT_RESET=0
+PERIODIC_SNAPSHOT_MAX_DELTAS=100
+```
+
+Semantics:
+
+- disabled by default, keeping previous behavior unchanged;
+- when enabled, one successful snapshot is followed by at most `N` successful delta runs;
+- after the threshold is reached, the next run is forced to snapshot;
+- before that forced snapshot, only the active `state/db` is reset, while `runs/`, `logs/`, `state/rsync-mirror`, `.env`, and Prometheus/Grafana data are preserved;
+- after a successful forced snapshot, the old DB staging is deleted so disk usage does not keep growing elsewhere.
+
+Check the latest `run-meta.json` for:
+
+- `sync_mode`
+- `snapshot_reason`
+- `periodic_snapshot_delta_count`
+- `periodic_snapshot_forced`
+- `reset_db_cleanup_status`
+
+## Common Commands
+
+```bash
+./status.sh
+./logs.sh ours-rp-soak --tail 200
+./restart.sh
+./stop.sh
+./cleanup.sh --keep-runs 100 --execute
+./uninstall.sh
+```
+
+`uninstall.sh` keeps data by default. Use the following only when you really want to delete `HOST_DATA_DIR`:
+
+```bash
+./uninstall.sh --purge-data
+```
+
+For finite acceptance tests, for example `MAX_RUNS=3`, also set:
+
+```bash
+SOAK_RESTART_POLICY=no
+```
+
+Otherwise Compose `unless-stopped` will restart the container after it exits successfully.

deploy/arm64-installer/docs/README.zh-CN.md

@@ -0,0 +1,124 @@
+# ours RP ARM64 安装包使用说明
+
+## 目标
+
+本安装包用于在 `linux/arm64` 服务器上通过 Docker Compose 部署 ours RP，并持续运行 all5 RIR 同步验证任务。
+
+安装包内置 ours RP ARM64 runtime、Prometheus ARM64、Grafana ARM64 镜像，部署时不需要现场拉取应用镜像。运行产物、状态数据库、日志、Prometheus 和 Grafana 数据均通过宿主机目录挂载保存。
+
+## 快速开始
+
+```bash
+tar -xzf ours-rp-arm64-installer-*.tar.gz
+cd ours-rp-arm64-installer-*
+
+./install.sh
+cp .env.example .env   # 如 install.sh 已自动创建，可直接编辑现有 .env
+vim .env
+./start.sh
+./status.sh
+```
+
+默认配置：
+
+- `RIRS=afrinic,apnic,arin,lacnic,ripe`
+- `MAX_RUNS=-1`
+- `INTERVAL_SECS=600`
+- `TAL_INPUT_MODE=file-live-ta`
+- `LIVE_TA_REFRESH_BEFORE_SNAPSHOT=1`
+- `PERIODIC_SNAPSHOT_RESET=0`
+- `PERIODIC_SNAPSHOT_MAX_DELTAS=100`
+- `HOST_DATA_DIR=/var/lib/ours-rp-arm64`
+- `SOAK_RESTART_POLICY=unless-stopped`
+- `MONITOR_PLATFORM=linux/arm64`
+
+## 首次启动语义
+
+如果 `HOST_DATA_DIR/runs` 下没有成功 run，`start.sh` 会先启动核心 `ours-rp-soak`，等待第一轮 snapshot 成功后再启动 metrics、Prometheus 和 Grafana。
+
+第一轮 snapshot 会先拉取 live TA，避免 clean state 使用旧 fixture TA。
+
+## 访问端口
+
+默认端口：
+
+- metrics: `http://<host>:9556/metrics`
+- Prometheus: `http://<host>:9090`
+- Grafana: `http://<host>:3000`
+
+Grafana 默认账号密码来自 `.env`：
+
+```bash
+GRAFANA_ADMIN_USER=admin
+GRAFANA_ADMIN_PASSWORD=admin
+```
+
+生产部署时应修改密码并限制外部访问。
+
+## 数据目录
+
+默认宿主机目录：
+
+```text
+/var/lib/ours-rp-arm64/
+  state/
+  runs/
+  logs/
+  tmp/
+  prometheus/
+  grafana/
+```
+
+`runs/run_XXXX/` 中包含每轮 `report.json`、`result.ccr`、`input.cir`、`vrps.csv`、`vaps.csv`、`stage-timing.json`、日志和元数据。
+
+## 定期 snapshot reset
+
+新增配置：
+
+```bash
+PERIODIC_SNAPSHOT_RESET=0
+PERIODIC_SNAPSHOT_MAX_DELTAS=100
+```
+
+语义：
+
+- 默认关闭，行为与旧版本一致；
+- 开启后，一次成功 snapshot 后最多连续执行 `N` 个成功 delta；
+- 达到阈值后，下一轮强制跑 snapshot；
+- 强制 snapshot 前只重置 active `state/db`，保留 `runs/`、`logs/`、`state/rsync-mirror`、`.env`、Prometheus/Grafana 数据；
+- 强制 snapshot 成功后旧 DB staging 会被删除，避免磁盘只是换目录继续增长。
+
+可通过最新 `run-meta.json` 中的以下字段确认：
+
+- `sync_mode`
+- `snapshot_reason`
+- `periodic_snapshot_delta_count`
+- `periodic_snapshot_forced`
+- `reset_db_cleanup_status`
+
+## 常用命令
+
+```bash
+./status.sh
+./logs.sh ours-rp-soak --tail 200
+./restart.sh
+./stop.sh
+./cleanup.sh --keep-runs 100 --execute
+./uninstall.sh
+```
+
+如果做有限轮次验收，例如 `MAX_RUNS=3`，建议同时设置：
+
+```bash
+SOAK_RESTART_POLICY=no
+```
+
+否则 Compose 的 `unless-stopped` 策略会在容器正常退出后再次拉起下一轮。
+
+`uninstall.sh` 默认不删除数据。只有显式执行：
+
+```bash
+./uninstall.sh --purge-data
+```
+
+才会删除 `HOST_DATA_DIR`。

deploy/arm64-installer/docs/operations.en.md

@@ -0,0 +1,100 @@
+# Operations Guide
+
+## Install
+
+```bash
+./install.sh
+```
+
+The installer is idempotent:
+
+- existing `.env` is kept;
+- existing Docker/Compose installation is reused;
+- repeated loading of packaged ours RP, Prometheus and Grafana ARM64 images is safe;
+- existing data directory is reused.
+
+## Start
+
+```bash
+./start.sh
+```
+
+Start without waiting for the first snapshot:
+
+```bash
+./start.sh --no-wait-first-run
+```
+
+## Stop and Restart
+
+```bash
+./stop.sh
+./restart.sh
+```
+
+## Status Checks
+
+```bash
+./status.sh
+./self-check.sh
+```
+
+Important checks:
+
+- Docker/Compose availability;
+- runtime, Prometheus and Grafana images exist;
+- `HOST_DATA_DIR` is writable;
+- Compose config is valid;
+- latest run status;
+- metrics, Prometheus and Grafana endpoints.
+
+`status.sh` also prints:
+
+- `periodic_snapshot_reset`
+- `periodic_snapshot_max_deltas`
+
+## Upgrade
+
+Extract the new package into a new directory and explicitly reuse the existing `.env` through the upgrade script:
+
+```bash
+./upgrade.sh --reuse-env-from /path/to/old-installer/.env
+```
+
+If the new package directory already has a `.env`, the upgrade script keeps it.
+
+Upgrade does not delete:
+
+- `runs/`
+- `logs/`
+- `state/rsync-mirror`
+- runtime configuration referenced by `.env`
+- Prometheus / Grafana data
+
+To validate periodic forced snapshot behavior, temporarily set:
+
+```bash
+PERIODIC_SNAPSHOT_RESET=1
+PERIODIC_SNAPSHOT_MAX_DELTAS=2
+```
+
+Then confirm the latest `run-meta.json` contains:
+
+```bash
+snapshot_reason=periodic_snapshot_delta_limit
+```
+
+After validation, restore:
+
+```bash
+PERIODIC_SNAPSHOT_MAX_DELTAS=100
+```
+
+## Cleanup
+
+```bash
+./cleanup.sh --keep-runs 100
+./cleanup.sh --keep-runs 100 --execute
+```
+
+Cleanup is dry-run by default. Add `--execute` after reviewing the output.

deploy/arm64-installer/docs/operations.zh-CN.md

@@ -0,0 +1,100 @@
+# 运维手册
+
+## 安装
+
+```bash
+./install.sh
+```
+
+安装脚本是幂等的：
+
+- 已有 `.env` 不覆盖；
+- 已安装 Docker/Compose 则跳过；
+- 包内 ours RP、Prometheus、Grafana ARM64 镜像重复加载是安全的；
+- 数据目录已存在则复用。
+
+## 启动
+
+```bash
+./start.sh
+```
+
+如需后台启动后不等待首轮 snapshot：
+
+```bash
+./start.sh --no-wait-first-run
+```
+
+## 停止和重启
+
+```bash
+./stop.sh
+./restart.sh
+```
+
+## 状态检查
+
+```bash
+./status.sh
+./self-check.sh
+```
+
+重点检查项：
+
+- Docker/Compose 可用；
+- runtime、Prometheus、Grafana 镜像存在；
+- `HOST_DATA_DIR` 可写；
+- Compose 配置合法；
+- 最新 run 状态；
+- metrics、Prometheus、Grafana endpoint。
+
+`status.sh` 还会显示：
+
+- `periodic_snapshot_reset`
+- `periodic_snapshot_max_deltas`
+
+## 升级
+
+把新安装包解压到新目录后，推荐通过升级脚本显式复用旧 `.env`：
+
+```bash
+./upgrade.sh --reuse-env-from /path/to/old-installer/.env
+```
+
+如果新目录已经存在 `.env`，升级脚本会保留它，不覆盖。
+
+升级不会删除以下数据：
+
+- `runs/`
+- `logs/`
+- `state/rsync-mirror`
+- `.env` 对应的运行配置
+- Prometheus / Grafana 数据
+
+验证定期 forced snapshot 时，可临时设置：
+
+```bash
+PERIODIC_SNAPSHOT_RESET=1
+PERIODIC_SNAPSHOT_MAX_DELTAS=2
+```
+
+然后检查最新 `run-meta.json` 应出现：
+
+```bash
+snapshot_reason=periodic_snapshot_delta_limit
+```
+
+验证完成后恢复：
+
+```bash
+PERIODIC_SNAPSHOT_MAX_DELTAS=100
+```
+
+## 清理
+
+```bash
+./cleanup.sh --keep-runs 100
+./cleanup.sh --keep-runs 100 --execute
+```
+
+默认 dry-run，确认后加 `--execute`。

deploy/arm64-installer/docs/troubleshooting.en.md

@@ -0,0 +1,83 @@
+# Troubleshooting
+
+## Docker or Compose Is Unavailable
+
+Run:
+
+```bash
+docker version
+docker compose version
+```
+
+If missing, run:
+
+```bash
+./install.sh
+```
+
+## ARM64 Image Cannot Run
+
+Running ARM64 images on x86_64 requires binfmt/qemu:
+
+```bash
+docker run --rm --privileged tonistiigi/binfmt --install arm64
+docker run --rm --platform linux/arm64 debian:bookworm-slim uname -m
+```
+
+Expected output: `aarch64`.
+
+## First Snapshot Times Out
+
+All-five snapshot can be slow, especially under QEMU. Increase timeout:
+
+```bash
+./start.sh --timeout-secs 14400
+```
+
+## Output Counts Are Too Low
+
+Check:
+
+```bash
+grep LIVE_TA_REFRESH_BEFORE_SNAPSHOT .env
+ls -l /var/lib/ours-rp-arm64/state/live-ta
+tail -100 /var/lib/ours-rp-arm64/logs/live-ta-refresh-*.log
+```
+
+In `file-live-ta` mode, snapshot should wait until live TA refresh succeeds.
+
+## Grafana Login Fails
+
+Check `.env`:
+
+```bash
+GRAFANA_ADMIN_USER=admin
+GRAFANA_ADMIN_PASSWORD=admin
+```
+
+If Grafana has already started, changing `.env` may not reset the existing Grafana database. Stop services and back up/clean `${HOST_DATA_DIR}/grafana` if needed.
+
+## A Finite Acceptance Test Starts an Extra Run
+
+If `.env` sets a finite `MAX_RUNS=3` while `SOAK_RESTART_POLICY=unless-stopped`, Docker Compose restarts the soak container after it exits successfully.
+
+For finite tests, set:
+
+```bash
+SOAK_RESTART_POLICY=no
+```
+
+## How to Confirm a Periodic Forced Snapshot
+
+Check the latest run metadata:
+
+```bash
+latest="$(find ${HOST_DATA_DIR}/runs -maxdepth 1 -type d -name 'run_*' | sort | tail -1)"
+jq '{run_id,sync_mode,snapshot_reason,periodic_snapshot_delta_count,periodic_snapshot_forced,reset_db_cleanup_status}' "$latest/run-meta.json"
+```
+
+For a threshold-triggered reset you should see:
+
+- `sync_mode: "snapshot"`
+- `snapshot_reason: "periodic_snapshot_delta_limit"`
+- `periodic_snapshot_forced: true`

deploy/arm64-installer/docs/troubleshooting.zh-CN.md

@@ -0,0 +1,83 @@
+# 故障排查
+
+## Docker 或 Compose 不可用
+
+执行：
+
+```bash
+docker version
+docker compose version
+```
+
+如果缺失，重新执行：
+
+```bash
+./install.sh
+```
+
+## ARM64 镜像无法运行
+
+在 x86_64 机器上运行 ARM64 镜像需要 binfmt/qemu：
+
+```bash
+docker run --rm --privileged tonistiigi/binfmt --install arm64
+docker run --rm --platform linux/arm64 debian:bookworm-slim uname -m
+```
+
+预期输出 `aarch64`。
+
+## 首轮 snapshot 超时
+
+all5 snapshot 可能很慢，尤其在 QEMU 环境。可以提高超时：
+
+```bash
+./start.sh --timeout-secs 14400
+```
+
+## 产物数量异常偏低
+
+检查：
+
+```bash
+grep LIVE_TA_REFRESH_BEFORE_SNAPSHOT .env
+ls -l /var/lib/ours-rp-arm64/state/live-ta
+tail -100 /var/lib/ours-rp-arm64/logs/live-ta-refresh-*.log
+```
+
+`file-live-ta` 模式下，snapshot 应等待 live TA 成功刷新。
+
+## Grafana 无法登录
+
+确认 `.env` 中：
+
+```bash
+GRAFANA_ADMIN_USER=admin
+GRAFANA_ADMIN_PASSWORD=admin
+```
+
+如果曾经启动过 Grafana，修改 `.env` 不一定重置已有 Grafana 数据库账号。可以停止服务后按需备份并清理 `${HOST_DATA_DIR}/grafana`。
+
+## 有限轮次验收后又多跑了一轮
+
+如果 `.env` 中设置了 `MAX_RUNS=3` 这类有限轮次，同时 `SOAK_RESTART_POLICY=unless-stopped`，Docker Compose 会在 soak 容器正常退出后重新启动容器。
+
+有限验收建议设置：
+
+```bash
+SOAK_RESTART_POLICY=no
+```
+
+## 如何确认触发了定期 forced snapshot
+
+检查最新 run metadata：
+
+```bash
+latest="$(find ${HOST_DATA_DIR}/runs -maxdepth 1 -type d -name 'run_*' | sort | tail -1)"
+jq '{run_id,sync_mode,snapshot_reason,periodic_snapshot_delta_count,periodic_snapshot_forced,reset_db_cleanup_status}' "$latest/run-meta.json"
+```
+
+阈值触发时应看到：
+
+- `sync_mode: "snapshot"`
+- `snapshot_reason: "periodic_snapshot_delta_limit"`
+- `periodic_snapshot_forced: true`

deploy/arm64-installer/install.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=scripts/common.sh
+source "$SCRIPT_DIR/scripts/common.sh"
+
+usage() {
+  cat <<'USAGE'
+Usage: ./install.sh [--skip-dep-install]
+
+Install or update the ours RP ARM64 compose package idempotently.
+USAGE
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --skip-dep-install)
+      export SKIP_DEP_INSTALL=1
+      shift
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      die "unknown option: $1"
+      ;;
+  esac
+done
+
+load_env
+install_docker_if_missing
+require_cmd curl
+require_cmd jq
+require_cmd rsync
+require_cmd gzip
+require_cmd tar
+create_data_dirs
+load_installer_images
+ensure_binfmt_if_needed
+verify_runtime_image
+verify_monitor_images
+compose_cmd --profile core --profile sidecar --profile monitor config >/tmp/ours-rp-arm64-compose-config.yml
+"$SCRIPT_DIR/self-check.sh" --quick
+log "install complete"

deploy/arm64-installer/logs.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=scripts/common.sh
+source "$SCRIPT_DIR/scripts/common.sh"
+load_env
+compose_cmd --profile core --profile sidecar --profile monitor logs "$@"

deploy/arm64-installer/restart.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+"$SCRIPT_DIR/stop.sh" || true
+"$SCRIPT_DIR/start.sh" "$@"

deploy/arm64-installer/scripts/common.sh

@@ -0,0 +1,265 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+INSTALLER_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+ENV_FILE="${ENV_FILE:-$INSTALLER_ROOT/.env}"
+ENV_EXAMPLE="$INSTALLER_ROOT/.env.example"
+COMPOSE_FILE="$INSTALLER_ROOT/compose/docker-compose.yml"
+
+log() {
+  printf '[ours-rp-installer] %s\n' "$*"
+}
+
+warn() {
+  printf '[ours-rp-installer][WARN] %s\n' "$*" >&2
+}
+
+die() {
+  printf '[ours-rp-installer][ERROR] %s\n' "$*" >&2
+  exit 1
+}
+
+require_cmd() {
+  command -v "$1" >/dev/null 2>&1 || die "missing command: $1"
+}
+
+load_env() {
+  if [[ ! -f "$ENV_FILE" ]]; then
+    [[ -f "$ENV_EXAMPLE" ]] || die "missing $ENV_EXAMPLE"
+    cp "$ENV_EXAMPLE" "$ENV_FILE"
+    log "created .env from .env.example"
+  fi
+  set -a
+  # shellcheck disable=SC1090
+  source "$ENV_FILE"
+  set +a
+  HOST_DATA_DIR="${HOST_DATA_DIR:-/var/lib/ours-rp-arm64}"
+  COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-ours-rp-arm64}"
+  RPKI_IMAGE="${RPKI_IMAGE:-ours-rp-runtime-arm64:dev}"
+  RPKI_PLATFORM="${RPKI_PLATFORM:-linux/arm64}"
+  MONITOR_PLATFORM="${MONITOR_PLATFORM:-linux/arm64}"
+  PROMETHEUS_IMAGE="${PROMETHEUS_IMAGE:-prom/prometheus:v2.55.1}"
+  GRAFANA_IMAGE="${GRAFANA_IMAGE:-grafana/grafana:11.3.1}"
+  FIRST_RUN_WAIT_TIMEOUT_SECS="${FIRST_RUN_WAIT_TIMEOUT_SECS:-7200}"
+}
+
+compose_cmd() {
+  docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" -p "${COMPOSE_PROJECT_NAME:-ours-rp-arm64}" "$@"
+}
+
+create_data_dirs() {
+  load_env
+  mkdir -p \
+    "$HOST_DATA_DIR/state" \
+    "$HOST_DATA_DIR/runs" \
+    "$HOST_DATA_DIR/logs" \
+    "$HOST_DATA_DIR/tmp" \
+    "$HOST_DATA_DIR/prometheus" \
+    "$HOST_DATA_DIR/grafana"
+  chmod 755 "$HOST_DATA_DIR" "$HOST_DATA_DIR/state" "$HOST_DATA_DIR/runs" "$HOST_DATA_DIR/logs" "$HOST_DATA_DIR/tmp" || true
+  chmod 777 "$HOST_DATA_DIR/prometheus" "$HOST_DATA_DIR/grafana" || true
+}
+
+latest_run_dir() {
+  load_env
+  find "$HOST_DATA_DIR/runs" -maxdepth 1 -mindepth 1 -type d -name 'run_*' 2>/dev/null | sort | tail -1
+}
+
+latest_success_run_dir() {
+  load_env
+  find "$HOST_DATA_DIR/runs" -maxdepth 2 -type f -path '*/run-summary.json' 2>/dev/null \
+    | while read -r summary; do
+        if jq -e '.status == "success"' "$summary" >/dev/null 2>&1; then
+          dirname "$summary"
+        fi
+      done | sort | tail -1
+}
+
+has_success_run() {
+  [[ -n "$(latest_success_run_dir)" ]]
+}
+
+print_run_summary() {
+  local run_dir="$1"
+  local summary="$run_dir/run-summary.json"
+  local meta="$run_dir/run-meta.json"
+  local timing="$run_dir/stage-timing.json"
+  local process_time="$run_dir/process-time.txt"
+  local vrps_file="$run_dir/vrps.csv"
+  local vaps_file="$run_dir/vaps.csv"
+  local status="unknown"
+  local sync_mode="unknown"
+  local wall_ms="null"
+  local validation_ms="null"
+  local repo_sync_ms="null"
+  local max_rss_kb="null"
+  local publication_points="null"
+  local vrps="null"
+  local vaps="null"
+  local warnings="null"
+  [[ -f "$summary" ]] || {
+    warn "missing run-summary.json in $run_dir"
+    return 1
+  }
+  status="$(jq -r '.status // "unknown"' "$summary" 2>/dev/null || echo unknown)"
+  wall_ms="$(jq -r '.wallMs // .wall_ms // "null"' "$summary" 2>/dev/null || echo null)"
+  warnings="$(jq -r '.warningCount // .warnings // "null"' "$summary" 2>/dev/null || echo null)"
+  if [[ -f "$meta" ]]; then
+    sync_mode="$(jq -r '.sync_mode // .syncMode // "unknown"' "$meta" 2>/dev/null || echo unknown)"
+    status="$(jq -r --arg fallback "$status" '.status // $fallback' "$meta" 2>/dev/null || echo "$status")"
+  fi
+  if [[ -f "$timing" ]]; then
+    validation_ms="$(jq -r '.validation_ms // "null"' "$timing" 2>/dev/null || echo null)"
+    repo_sync_ms="$(jq -r '.repo_sync_ms_total // "null"' "$timing" 2>/dev/null || echo null)"
+    publication_points="$(jq -r '.publication_points // "null"' "$timing" 2>/dev/null || echo null)"
+  fi
+  if [[ -f "$process_time" ]]; then
+    max_rss_kb="$(awk -F': ' '/Maximum resident set size/ {print $2; found=1} END {if (!found) print "null"}' "$process_time")"
+  fi
+  if [[ -f "$vrps_file" ]]; then
+    vrps="$(( $(wc -l < "$vrps_file") > 0 ? $(wc -l < "$vrps_file") - 1 : 0 ))"
+  fi
+  if [[ -f "$vaps_file" ]]; then
+    vaps="$(( $(wc -l < "$vaps_file") > 0 ? $(wc -l < "$vaps_file") - 1 : 0 ))"
+  fi
+  jq -n \
+    --arg run "$(basename "$run_dir")" \
+    --arg status "$status" \
+    --arg syncMode "$sync_mode" \
+    --argjson wallMs "$wall_ms" \
+    --argjson validationMs "$validation_ms" \
+    --argjson repoSyncMs "$repo_sync_ms" \
+    --argjson maxRssKb "$max_rss_kb" \
+    --argjson vrps "$vrps" \
+    --argjson vaps "$vaps" \
+    --argjson publicationPoints "$publication_points" \
+    --argjson warnings "$warnings" \
+    '{run:$run,status:$status,syncMode:$syncMode,wallMs:$wallMs,validationMs:$validationMs,repoSyncMs:$repoSyncMs,maxRssKb:$maxRssKb,vrps:$vrps,vaps:$vaps,publicationPoints:$publicationPoints,warnings:$warnings}'
+}
+
+wait_for_new_success_run() {
+  local before_latest="$1"
+  local timeout_secs="$2"
+  local start_epoch now run_dir summary meta status meta_status
+  start_epoch="$(date +%s)"
+  while true; do
+    run_dir="$(latest_run_dir || true)"
+    if [[ -n "$run_dir" && "$run_dir" != "$before_latest" ]]; then
+      summary="$run_dir/run-summary.json"
+      meta="$run_dir/run-meta.json"
+      if [[ -f "$summary" ]]; then
+        status="$(jq -r '.status // "unknown"' "$summary" 2>/dev/null || echo unknown)"
+        if [[ "$status" == "success" ]]; then
+          meta_status="unknown"
+          if [[ -f "$meta" ]]; then
+            meta_status="$(jq -r '.status // "unknown"' "$meta" 2>/dev/null || echo unknown)"
+          fi
+          if [[ "$meta_status" == "success" ]]; then
+            print_run_summary "$run_dir" || true
+            return 0
+          fi
+        fi
+        if [[ "$status" == "failed" || "$status" == "error" ]]; then
+          print_run_summary "$run_dir" || true
+          die "run failed: $run_dir"
+        fi
+      fi
+    fi
+    now="$(date +%s)"
+    if (( now - start_epoch > timeout_secs )); then
+      die "timed out waiting for first successful run after ${timeout_secs}s"
+    fi
+    sleep 10
+  done
+}
+
+docker_compose_available() {
+  docker compose version >/dev/null 2>&1
+}
+
+install_docker_if_missing() {
+  if command -v docker >/dev/null 2>&1 && docker_compose_available && command -v jq >/dev/null 2>&1 && command -v rsync >/dev/null 2>&1 && command -v curl >/dev/null 2>&1; then
+    log "docker and docker compose are already installed"
+    return 0
+  fi
+  if [[ "${SKIP_DEP_INSTALL:-0}" == "1" ]]; then
+    die "docker/docker compose missing and SKIP_DEP_INSTALL=1"
+  fi
+  if ! command -v apt-get >/dev/null 2>&1; then
+    die "docker/docker compose missing; automatic install currently supports apt-get only"
+  fi
+  log "installing missing runtime packages via apt"
+  apt-get update
+  DEBIAN_FRONTEND=noninteractive apt-get install -y ca-certificates curl jq rsync gzip tar docker.io
+  if ! docker_compose_available; then
+    if apt-cache show docker-compose-v2 >/dev/null 2>&1; then
+      DEBIAN_FRONTEND=noninteractive apt-get install -y docker-compose-v2
+    elif apt-cache show docker-compose-plugin >/dev/null 2>&1; then
+      DEBIAN_FRONTEND=noninteractive apt-get install -y docker-compose-plugin
+    elif apt-cache show docker-compose >/dev/null 2>&1; then
+      DEBIAN_FRONTEND=noninteractive apt-get install -y docker-compose
+    fi
+  fi
+  systemctl enable --now docker >/dev/null 2>&1 || true
+  docker_compose_available || die "docker compose is still unavailable after install"
+}
+
+load_installer_images() {
+  require_cmd docker
+  shopt -s nullglob
+  local image
+  local found=0
+  for image in "$INSTALLER_ROOT"/images/*.tar "$INSTALLER_ROOT"/images/*.tar.gz; do
+    found=1
+    log "loading docker image: $image"
+    if [[ "$image" == *.gz ]]; then
+      gzip -dc "$image" | docker load
+    else
+      docker load -i "$image"
+    fi
+  done
+  shopt -u nullglob
+  (( found == 1 )) || warn "no image tar found under $INSTALLER_ROOT/images"
+}
+
+ensure_binfmt_if_needed() {
+  require_cmd docker
+  load_env
+  local host_arch
+  host_arch="$(uname -m)"
+  if [[ "$RPKI_PLATFORM" == "linux/arm64" && "$host_arch" != "aarch64" && "$host_arch" != "arm64" ]]; then
+    log "host arch is $host_arch; ensuring binfmt/qemu for arm64"
+    docker run --rm --privileged tonistiigi/binfmt --install arm64
+  fi
+}
+
+verify_runtime_image() {
+  load_env
+  require_cmd docker
+  log "verifying runtime image $RPKI_IMAGE on $RPKI_PLATFORM"
+  docker image inspect "$RPKI_IMAGE" >/dev/null
+  docker run --rm --platform "$RPKI_PLATFORM" "$RPKI_IMAGE" /opt/ours-rp/bin/rpki --help >/tmp/ours-rp-arm64-rpki-help.txt
+  head -5 /tmp/ours-rp-arm64-rpki-help.txt || true
+}
+
+verify_image_platform() {
+  local image="$1"
+  local expected_platform="$2"
+  local role="$3"
+  local actual_platform
+  docker image inspect "$image" >/dev/null
+  actual_platform="$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image" 2>/dev/null || echo unknown)"
+  [[ "$actual_platform" == "$expected_platform" ]] || die "$role image platform mismatch: image=$image expected=$expected_platform actual=$actual_platform"
+}
+
+verify_monitor_images() {
+  load_env
+  require_cmd docker
+  verify_image_platform "$PROMETHEUS_IMAGE" "$MONITOR_PLATFORM" "prometheus"
+  verify_image_platform "$GRAFANA_IMAGE" "$MONITOR_PLATFORM" "grafana"
+}
+
+endpoint_ok() {
+  local url="$1"
+  curl -fsS --max-time 5 "$url" >/dev/null 2>&1
+}

deploy/arm64-installer/self-check.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=scripts/common.sh
+source "$SCRIPT_DIR/scripts/common.sh"
+
+QUICK=0
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --quick)
+      QUICK=1
+      shift
+      ;;
+    -h|--help)
+      echo "Usage: ./self-check.sh [--quick]"
+      exit 0
+      ;;
+    *)
+      die "unknown option: $1"
+      ;;
+  esac
+done
+
+load_env
+require_cmd docker
+require_cmd jq
+docker compose version >/dev/null
+[[ -f "$COMPOSE_FILE" ]] || die "missing compose file"
+[[ -f "$ENV_FILE" ]] || die "missing .env"
+create_data_dirs
+[[ -w "$HOST_DATA_DIR" ]] || die "data dir is not writable: $HOST_DATA_DIR"
+compose_cmd --profile core --profile sidecar --profile monitor config >/dev/null
+verify_image_platform "$RPKI_IMAGE" "$RPKI_PLATFORM" "runtime"
+verify_monitor_images
+if [[ "$QUICK" == "0" ]]; then
+  verify_runtime_image
+fi
+log "self-check ok"

deploy/arm64-installer/start.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=scripts/common.sh
+source "$SCRIPT_DIR/scripts/common.sh"
+
+WAIT_FIRST_RUN=1
+TIMEOUT_SECS=""
+
+usage() {
+  cat <<'USAGE'
+Usage: ./start.sh [--no-wait-first-run] [--timeout-secs N]
+
+Start ours RP. If no successful run exists, wait for the first snapshot to succeed
+before starting metrics, Prometheus and Grafana.
+USAGE
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --no-wait-first-run)
+      WAIT_FIRST_RUN=0
+      shift
+      ;;
+    --timeout-secs)
+      TIMEOUT_SECS="$2"
+      shift 2
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      die "unknown option: $1"
+      ;;
+  esac
+done
+
+load_env
+create_data_dirs
+timeout_secs="${TIMEOUT_SECS:-$FIRST_RUN_WAIT_TIMEOUT_SECS}"
+before_latest="$(latest_run_dir || true)"
+had_success=0
+if has_success_run; then
+  had_success=1
+fi
+
+log "starting core soak service"
+compose_cmd --profile core up -d ours-rp-soak
+
+if [[ "$had_success" == "0" && "$WAIT_FIRST_RUN" == "1" ]]; then
+  log "no previous successful run found; waiting for first run timeout=${timeout_secs}s"
+  wait_for_new_success_run "$before_latest" "$timeout_secs"
+fi
+
+log "starting metrics and monitor services"
+compose_cmd --profile sidecar --profile monitor up -d artifact-metrics prometheus grafana
+"$SCRIPT_DIR/status.sh" --brief || true

deploy/arm64-installer/status.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=scripts/common.sh
+source "$SCRIPT_DIR/scripts/common.sh"
+
+BRIEF=0
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --brief)
+      BRIEF=1
+      shift
+      ;;
+    -h|--help)
+      echo "Usage: ./status.sh [--brief]"
+      exit 0
+      ;;
+    *)
+      die "unknown option: $1"
+      ;;
+  esac
+done
+
+load_env
+echo "installer_root=$INSTALLER_ROOT"
+echo "host_data_dir=$HOST_DATA_DIR"
+echo "image=$RPKI_IMAGE"
+echo "platform=$RPKI_PLATFORM"
+echo "rirs=${RIRS:-}"
+echo "max_runs=${MAX_RUNS:-}"
+echo "interval_secs=${INTERVAL_SECS:-}"
+echo "periodic_snapshot_reset=${PERIODIC_SNAPSHOT_RESET:-0}"
+echo "periodic_snapshot_max_deltas=${PERIODIC_SNAPSHOT_MAX_DELTAS:-100}"
+echo
+if command -v docker >/dev/null 2>&1; then
+  docker version --format 'docker={{.Server.Version}}' 2>/dev/null || echo "docker=unavailable"
+  docker compose version 2>/dev/null || true
+  compose_cmd --profile core --profile sidecar --profile monitor ps || true
+else
+  echo "docker=missing"
+fi
+echo
+df -h "$HOST_DATA_DIR" 2>/dev/null || true
+echo
+latest="$(latest_run_dir || true)"
+if [[ -n "$latest" ]]; then
+  echo "latest_run=$latest"
+  print_run_summary "$latest" || true
+else
+  echo "latest_run=none"
+fi
+if [[ "$BRIEF" == "0" ]]; then
+  echo
+  endpoint_ok "http://127.0.0.1:${METRICS_PORT:-9556}/metrics" && echo "metrics=ok" || echo "metrics=unavailable"
+  endpoint_ok "http://127.0.0.1:${PROMETHEUS_PORT:-9090}/-/ready" && echo "prometheus=ok" || echo "prometheus=unavailable"
+  endpoint_ok "http://127.0.0.1:${GRAFANA_PORT:-3000}/api/health" && echo "grafana=ok" || echo "grafana=unavailable"
+fi

deploy/arm64-installer/stop.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=scripts/common.sh
+source "$SCRIPT_DIR/scripts/common.sh"
+load_env
+compose_cmd --profile core --profile sidecar --profile monitor stop "$@"

deploy/arm64-installer/uninstall.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=scripts/common.sh
+source "$SCRIPT_DIR/scripts/common.sh"
+
+PURGE_DATA=0
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --purge-data)
+      PURGE_DATA=1
+      shift
+      ;;
+    -h|--help)
+      echo "Usage: ./uninstall.sh [--purge-data]"
+      exit 0
+      ;;
+    *)
+      die "unknown option: $1"
+      ;;
+  esac
+done
+
+load_env
+compose_cmd --profile core --profile sidecar --profile monitor down --remove-orphans || true
+if [[ "$PURGE_DATA" == "1" ]]; then
+  [[ "$HOST_DATA_DIR" == "/" || -z "$HOST_DATA_DIR" ]] && die "refuse to purge unsafe HOST_DATA_DIR=$HOST_DATA_DIR"
+  rm -rf "$HOST_DATA_DIR"
+  log "purged data dir $HOST_DATA_DIR"
+else
+  log "containers removed; data kept at $HOST_DATA_DIR"
+fi

deploy/arm64-installer/upgrade.sh

@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=scripts/common.sh
+source "$SCRIPT_DIR/scripts/common.sh"
+
+REUSE_ENV_FROM=""
+UPDATE_PACKAGE_IMAGE=1
+
+usage() {
+  cat <<'USAGE'
+Usage: ./upgrade.sh [--reuse-env-from /path/to/.env] [--keep-reused-image]
+
+By default, --reuse-env-from creates the new .env from this package's
+.env.example first, then overlays existing user settings from the old .env.
+Image tags are intentionally kept from the new package so the upgraded service
+actually runs the new packaged runtime and monitor images. Use
+--keep-reused-image only when you intentionally want to keep previous image tags.
+USAGE
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --reuse-env-from)
+      REUSE_ENV_FROM="$2"
+      shift 2
+      ;;
+    --keep-reused-image)
+      UPDATE_PACKAGE_IMAGE=0
+      shift
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      die "unknown option: $1"
+      ;;
+  esac
+done
+
+env_get_key() {
+  local env_path="$1"
+  local key="$2"
+  awk -F= -v key="$key" '$1 == key {sub(/^[^=]*=/, ""); print; exit}' "$env_path"
+}
+
+env_set_key() {
+  local env_path="$1"
+  local key="$2"
+  local value="$3"
+  local tmp_env
+  if grep -q "^${key}=" "$env_path"; then
+    tmp_env="$(mktemp)"
+    awk -v key="$key" -v value="$value" '
+      BEGIN { done=0 }
+      $0 ~ "^" key "=" { print key "=" value; done=1; next }
+      { print }
+      END { if (!done) print key "=" value }
+    ' "$env_path" > "$tmp_env"
+    mv "$tmp_env" "$env_path"
+  else
+    printf '%s=%s\n' "$key" "$value" >> "$env_path"
+  fi
+}
+
+overlay_reused_env() {
+  local source_env="$1"
+  local target_env="$2"
+  local key
+  local value
+  while IFS='=' read -r key _; do
+    [[ -n "$key" ]] || continue
+    [[ "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || continue
+    case "$key" in
+      RPKI_IMAGE|PROMETHEUS_IMAGE|GRAFANA_IMAGE)
+        [[ "$UPDATE_PACKAGE_IMAGE" == "0" ]] || continue
+        ;;
+    esac
+    value="$(env_get_key "$source_env" "$key")"
+    env_set_key "$target_env" "$key" "$value"
+  done < <(grep -E '^[A-Za-z_][A-Za-z0-9_]*=' "$source_env" || true)
+}
+
+if [[ -n "$REUSE_ENV_FROM" ]]; then
+  [[ -f "$REUSE_ENV_FROM" ]] || die "missing reuse env file: $REUSE_ENV_FROM"
+  if [[ ! -f "$ENV_FILE" ]]; then
+    [[ -f "$ENV_EXAMPLE" ]] || die "missing $ENV_EXAMPLE"
+    cp "$ENV_EXAMPLE" "$ENV_FILE"
+    overlay_reused_env "$REUSE_ENV_FROM" "$ENV_FILE"
+    log "created new package env from .env.example and overlaid user settings from $REUSE_ENV_FROM"
+  else
+    log "keeping existing env at $ENV_FILE; reuse source ignored: $REUSE_ENV_FROM"
+  fi
+fi
+
+load_env
+create_data_dirs
+install_docker_if_missing
+load_installer_images
+ensure_binfmt_if_needed
+verify_runtime_image
+verify_monitor_images
+compose_cmd --profile core --profile sidecar --profile monitor up -d --force-recreate
+"$SCRIPT_DIR/status.sh" --brief || true

docker/ours-rp-runtime.Dockerfile

@@ -0,0 +1,94 @@
+ARG BUILDER_IMAGE=ours-rp-base-rust-amd64:1-bookworm
+ARG RUNTIME_IMAGE=ours-rp-base-debian-arm64:bookworm-slim
+
+FROM --platform=$BUILDPLATFORM ${BUILDER_IMAGE} AS builder
+
+WORKDIR /src
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+      build-essential \
+      ca-certificates \
+      clang \
+      cmake \
+      g++-aarch64-linux-gnu \
+      gcc-aarch64-linux-gnu \
+      git \
+      libclang-dev \
+      libc6-dev-arm64-cross \
+      make \
+      perl \
+      pkg-config \
+      python3 \
+    && rm -rf /var/lib/apt/lists/* \
+    && rustup target add aarch64-unknown-linux-gnu
+
+ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc \
+    CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc \
+    CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ \
+    AR_aarch64_unknown_linux_gnu=aarch64-linux-gnu-ar \
+    CARGO_BUILD_TARGET=aarch64-unknown-linux-gnu \
+    PKG_CONFIG_ALLOW_CROSS=1
+
+COPY . .
+
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+    --mount=type=cache,target=/usr/local/cargo/git \
+    --mount=type=cache,target=/src/target \
+    cargo build --release --target aarch64-unknown-linux-gnu \
+      --bin rpki \
+      --bin rpki_daemon \
+      --bin db_stats \
+      --bin rpki_artifact_metrics \
+    && mkdir -p /build-out/bin \
+    && cp \
+      target/aarch64-unknown-linux-gnu/release/rpki \
+      target/aarch64-unknown-linux-gnu/release/rpki_daemon \
+      target/aarch64-unknown-linux-gnu/release/db_stats \
+      target/aarch64-unknown-linux-gnu/release/rpki_artifact_metrics \
+      /build-out/bin/
+
+FROM --platform=$TARGETPLATFORM ${RUNTIME_IMAGE} AS runtime
+
+LABEL org.opencontainers.image.title="ours-rp-runtime" \
+      org.opencontainers.image.description="Ours RP runtime image for ARM64 Docker Compose deployment"
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+      bash \
+      ca-certificates \
+      coreutils \
+      curl \
+      findutils \
+      iputils-ping \
+      jq \
+      procps \
+      python3 \
+      rsync \
+      time \
+      tzdata \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /opt/ours-rp
+
+COPY --from=builder /build-out/bin/ /opt/ours-rp/bin/
+COPY scripts/soak/run_soak.sh /opt/ours-rp/run_soak.sh
+COPY scripts/soak/portable-soak.env.example /opt/ours-rp/portable-soak.env.example
+COPY tests/fixtures/tal/ /opt/ours-rp/fixtures/tal/
+COPY tests/fixtures/ta/ /opt/ours-rp/fixtures/ta/
+COPY fixtures/live_20260619/tal/ /opt/ours-rp/fixtures/live_20260619/tal/
+COPY fixtures/live_20260619/ta/ /opt/ours-rp/fixtures/live_20260619/ta/
+
+RUN chmod +x /opt/ours-rp/run_soak.sh /opt/ours-rp/bin/* \
+    && mkdir -p /var/lib/ours-rp/state /var/lib/ours-rp/runs /var/lib/ours-rp/logs /var/lib/ours-rp/tmp
+
+ENV PACKAGE_ROOT=/opt/ours-rp \
+    ENV_FILE=/opt/ours-rp/.env \
+    RUN_ROOT=/var/lib/ours-rp \
+    BIN_DIR=/opt/ours-rp/bin \
+    FIXTURE_DIR=/opt/ours-rp/fixtures \
+    RUST_BACKTRACE=1
+
+VOLUME ["/var/lib/ours-rp"]
+
+CMD ["/opt/ours-rp/run_soak.sh"]

fixtures/live_20260619/manifest.json

@@ -0,0 +1,80 @@
+{
+  "created_at_utc": "2026-06-19T08:08:03Z",
+  "items": [
+    {
+      "rir": "afrinic",
+      "ta_bytes": 1216,
+      "ta_download": "200 1216 1.351147",
+      "ta_elapsed_s": 1.227,
+      "ta_path": "rpki_2/rpki/fixtures/live_20260619/ta/afrinic-ta.cer",
+      "ta_sha256": "43a26fd28bafb9398e5b2ab19e036b450bd04f4973a7f5ad151cebdee0edac36",
+      "ta_uri": "https://rpki.afrinic.net/repository/AfriNIC.cer",
+      "tal_bytes": 496,
+      "tal_download": "200 496 1.361326",
+      "tal_elapsed_s": 1.238,
+      "tal_path": "rpki_2/rpki/fixtures/live_20260619/tal/afrinic.tal",
+      "tal_sha256": "2838ef30ea27ce5705abf5f5adb131d8c35b1f50858338a2f3c84bb207c2fa35",
+      "tal_url": "https://rpki.afrinic.net/tal/afrinic.tal"
+    },
+    {
+      "rir": "apnic",
+      "ta_bytes": 1222,
+      "ta_download": "200 1222 1.012321",
+      "ta_elapsed_s": 0.921,
+      "ta_path": "rpki_2/rpki/fixtures/live_20260619/ta/apnic-ta.cer",
+      "ta_sha256": "2014230ad49b2777ac2bde0948ddfa4b8f207114c549e26d755de88c3593e3af",
+      "ta_uri": "https://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer",
+      "tal_bytes": 532,
+      "tal_download": "200 466 1.007155",
+      "tal_elapsed_s": 0.917,
+      "tal_path": "rpki_2/rpki/fixtures/live_20260619/tal/apnic.tal",
+      "tal_sha256": "472e551f7c551c2e999e582b7c9437d3bee4900fe53afff62aeb28d4940ade94",
+      "tal_url": "https://tal.apnic.net/apnic.tal"
+    },
+    {
+      "rir": "arin",
+      "ta_bytes": 1143,
+      "ta_download": "200 1143 0.816714",
+      "ta_elapsed_s": 0.743,
+      "ta_path": "rpki_2/rpki/fixtures/live_20260619/ta/arin-ta.cer",
+      "ta_sha256": "5b3c2f6f04abd19261084487a43c156f778b0b8926d63801d48c7a93ed349492",
+      "ta_uri": "https://rrdp.arin.net/arin-rpki-ta.cer",
+      "tal_bytes": 1258,
+      "tal_download": "200 1258 0.848581",
+      "tal_elapsed_s": 0.774,
+      "tal_path": "rpki_2/rpki/fixtures/live_20260619/tal/arin.tal",
+      "tal_sha256": "1f8bdb03bcc30a3b8e11fd9a87102fba250c22137a3c8baa9c81b139cb412639",
+      "tal_url": "https://www.arin.net/resources/manage/rpki/arin.tal"
+    },
+    {
+      "rir": "lacnic",
+      "ta_bytes": 1166,
+      "ta_download": "200 1166 1.025273",
+      "ta_elapsed_s": 0.932,
+      "ta_path": "rpki_2/rpki/fixtures/live_20260619/ta/lacnic-ta.cer",
+      "ta_sha256": "f44bc51008fd6998de7597b72a79b07bf3ebcb0f14daa7c7c022c0e9d66e0ad0",
+      "ta_uri": "https://rrdp.lacnic.net/ta/rta-lacnic-rpki.cer",
+      "tal_bytes": 502,
+      "tal_download": "200 502 1.729122",
+      "tal_elapsed_s": 1.565,
+      "tal_path": "rpki_2/rpki/fixtures/live_20260619/tal/lacnic.tal",
+      "tal_sha256": "d44bb9394ab009c8b53e5efebf2a1c9450bab61a27efe00de5a3e4587a3a2f6a",
+      "tal_url": "https://www.lacnic.net/innovaportal/file/4983/1/lacnic.tal"
+    },
+    {
+      "rir": "ripe",
+      "ta_bytes": 1036,
+      "ta_download": "200 1036 1.060992",
+      "ta_elapsed_s": 4.536,
+      "ta_path": "rpki_2/rpki/fixtures/live_20260619/ta/ripe-ncc-ta.cer",
+      "ta_sha256": "3e3f7e4efc8d0cea03d9cc1fde6e168b45c26d7b3272e14abd8da4871886e539",
+      "ta_uri": "https://rpki.ripe.net/ta/ripe-ncc-ta.cer",
+      "tal_bytes": 482,
+      "tal_download": "200 482 1.092082",
+      "tal_elapsed_s": 0.992,
+      "tal_path": "rpki_2/rpki/fixtures/live_20260619/tal/ripe-ncc.tal",
+      "tal_sha256": "59ca27ef93f23682749fcefe7c6d70fbc723343549ff9e4d3996acaff79817fb",
+      "tal_url": "https://tal.rpki.ripe.net/ripe-ncc.tal"
+    }
+  ]
+}

fixtures/live_20260619/tal/afrinic.tal

@@ -0,0 +1,10 @@
+rsync://rpki.afrinic.net/repository/AfriNIC.cer
+https://rpki.afrinic.net/repository/AfriNIC.cer
+
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxsAqAhWIO+ON2Ef9oRDM
+pKxv+AfmSLIdLWJtjrvUyDxJPBjgR+kVrOHUeTaujygFUp49tuN5H2C1rUuQavTH
+vve6xNF5fU3OkTcqEzMOZy+ctkbde2SRMVdvbO22+TH9gNhKDc9l7Vu01qU4LeJH
+k3X0f5uu5346YrGAOSv6AaYBXVgXxa0s9ZvgqFpim50pReQe/WI3QwFKNgpPzfQL
+6Y7fDPYdYaVOXPXSKtx7P4s4KLA/ZWmRL/bobw/i2fFviAGhDrjqqqum+/9w1hEl
+L/vqihVnV18saKTnLvkItA/Bf5i11Yhw2K7qv573YWxyuqCknO/iYLTR1DToBZcZ
+UQIDAQAB

fixtures/live_20260619/tal/apnic-rfc7730-https.tal

@@ -0,0 +1,10 @@
+https://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer
+rsync://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer
+
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx9RWSL61YAAYumEiU8z8
+qH2ETVIL01ilxZlzIL9JYSORMN5Cmtf8V2JblIealSqgOTGjvSjEsiV73s67zYQI
+7C/iSOb96uf3/s86NqbxDiFQGN8qG7RNcdgVuUlAidl8WxvLNI8VhqbAB5uSg/Mr
+LeSOvXRja041VptAxIhcGzDMvlAJRwkrYK/Mo8P4E2rSQgwqCgae0ebY1CsJ3Cjf
+i67C1nw7oXqJJovvXJ4apGmEv8az23OLC6Ki54Ul/E6xk227BFttqFV3YMtKx42H
+cCcDVZZy01n7JjzvO8ccaXmHIgR7utnqhBRNNq5Xc5ZhbkrUsNtiJmrZzVlgU6Ou
+0wIDAQAB

fixtures/live_20260619/tal/apnic.tal

@@ -0,0 +1,10 @@
+https://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer
+rsync://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer
+
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx9RWSL61YAAYumEiU8z8
+qH2ETVIL01ilxZlzIL9JYSORMN5Cmtf8V2JblIealSqgOTGjvSjEsiV73s67zYQI
+7C/iSOb96uf3/s86NqbxDiFQGN8qG7RNcdgVuUlAidl8WxvLNI8VhqbAB5uSg/Mr
+LeSOvXRja041VptAxIhcGzDMvlAJRwkrYK/Mo8P4E2rSQgwqCgae0ebY1CsJ3Cjf
+i67C1nw7oXqJJovvXJ4apGmEv8az23OLC6Ki54Ul/E6xk227BFttqFV3YMtKx42H
+cCcDVZZy01n7JjzvO8ccaXmHIgR7utnqhBRNNq5Xc5ZhbkrUsNtiJmrZzVlgU6Ou
+0wIDAQAB

fixtures/live_20260619/tal/arin.tal

@@ -0,0 +1,19 @@
+# THIS TRUST ANCHOR LOCATOR IS PROVIDED BY THE AMERICAN REGISTRY FOR
+# INTERNET NUMBERS (ARIN) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL ARIN BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS PUBLIC KEY, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+rsync://rpki.arin.net/repository/arin-rpki-ta.cer
+https://rrdp.arin.net/arin-rpki-ta.cer
+
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3lZPjbHvMRV5sDDqfLc/685th5FnreHMJjg8
+pEZUbG8Y8TQxSBsDebbsDpl3Ov3Cj1WtdrJ3CIfQODCPrrJdOBSrMATeUbPC+JlNf2SRP3UB+VJFgtTj
+0RN8cEYIuhBW5t6AxQbHhdNQH+A1F/OJdw0q9da2U29Lx85nfFxvnC1EpK9CbLJS4m37+RlpNbT1cba+
+b+loXpx0Qcb1C4UpJCGDy7uNf5w6/+l7RpATAHqqsX4qCtwwDYlbHzp2xk9owF3mkCxzl0HwncO+sEHH
+eaL3OjtwdIGrRGeHi2Mpt+mvWHhtQqVG+51MHTyg+nIjWFKKGx1Q9+KDx4wJStwveQIDAQAB

fixtures/live_20260619/tal/lacnic.tal

@@ -0,0 +1,4 @@
+https://rrdp.lacnic.net/ta/rta-lacnic-rpki.cer
+rsync://repository.lacnic.net/rpki/lacnic/rta-lacnic-rpki.cer
+
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqZEzhYK0+PtDOPfub/KRc3MeWx3neXx4/wbnJWGbNAtbYqXg3uU5J4HFzPgk/VIppgSKAhlO0H60DRP48by9gr5/yDHu2KXhOmnMg46sYsUIpfgtBS9+VtrqWziJfb+pkGtuOWeTnj6zBmBNZKK+5AlMCW1WPhrylIcB+XSZx8tk9GS/3SMQ+YfMVwwAyYjsex14Uzto4GjONALE5oh1M3+glRQduD6vzSwOD+WahMbc9vCOTED+2McLHRKgNaQf0YJ9a1jG9oJIvDkKXEqdfqDRktwyoD74cV57bW3tBAexB7GglITbInyQAsmdngtfg2LUMrcROHHP86QPZINjDQIDAQAB

fixtures/live_20260619/tal/ripe-ncc.tal

@@ -0,0 +1,10 @@
+https://rpki.ripe.net/ta/ripe-ncc-ta.cer
+rsync://rpki.ripe.net/ta/ripe-ncc-ta.cer
+
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0URYSGqUz2myBsOzeW1j
+Q6NsxNvlLMyhWknvnl8NiBCs/T/S2XuNKQNZ+wBZxIgPPV2pFBFeQAvoH/WK83Hw
+A26V2siwm/MY2nKZ+Olw+wlpzlZ1p3Ipj2eNcKrmit8BwBC8xImzuCGaV0jkRB0G
+Z0hoH6Ml03umLprRsn6v0xOP0+l6Qc1ZHMFVFb385IQ7FQQTcVIxrdeMsoyJq9eM
+kE6DoclHhF/NlSllXubASQ9KUWqJ0+Ot3QCXr4LXECMfkpkVR2TZT+v5v658bHVs
+6ZxRD1b6Uk1uQKAyHUbn/tXvP8lrjAibGzVsXDT2L0x4Edx+QdixPgOji3gBMyL2
+VwIDAQAB

monitor/README.md

@@ -0,0 +1,220 @@
+# Ours RP Prometheus / Grafana Monitor
+
+本目录提供本地开发监控栈，用于采集 `rpki_artifact_metrics` 暴露的 ours RP soak 指标。
+
+## 前置条件
+
+1. Docker + Docker Compose v2；
+2. 宿主机已启动 `rpki_artifact_metrics`，并监听 Docker 网桥可访问的地址，例如 `0.0.0.0:9556`；
+3. Prometheus 容器通过 `host.docker.internal:9556` 访问宿主 sidecar。
+
+Linux Docker 下 compose 已配置：
+
+```yaml
+extra_hosts:
+  - host.docker.internal:host-gateway
+```
+
+## 启动
+
+```bash
+cd rpki_2/rpki/monitor
+docker compose up -d
+```
+
+默认镜像使用官方 Docker Hub 镜像：
+
+```text
+prom/prometheus:v2.55.1
+grafana/grafana:11.3.1
+```
+
+如需切到其它镜像源：
+
+```bash
+PROMETHEUS_IMAGE=<mirror>/prom/prometheus:v2.55.1 \
+GRAFANA_IMAGE=<mirror>/grafana/grafana:11.3.1 \
+docker compose up -d
+```
+
+默认端口：
+
+- Prometheus: <http://localhost:9090>
+- Grafana: <http://localhost:3000>
+- Grafana 默认账号密码：`admin` / `admin`
+
+如端口冲突：
+
+```bash
+PROMETHEUS_PORT=19090 GRAFANA_PORT=13000 docker compose up -d
+```
+
+Prometheus 默认保留 7 天数据；可通过 `PROMETHEUS_RETENTION` 覆盖：
+
+```bash
+PROMETHEUS_RETENTION=7d docker compose up -d
+```
+
+## 长期稳定性测试
+
+portable soak package 内置 `run_24h_soak_with_metrics.sh`，用于连续运行 ours RP、启动 metrics sidecar、启动本监控栈，并每小时生成报告：
+
+```bash
+cd /path/to/portable-soak
+SOAK_DURATION_SECS=0 \
+HOURLY_REPORT_INTERVAL_SECS=3600 \
+SOAK_RETAIN_RUNS=100 \
+CLEAN_TMP_AFTER_RUN=1 \
+PROMETHEUS_RETENTION=7d \
+STOP_MONITOR_STACK_ON_EXIT=0 \
+FEISHU_WEBHOOK_SCRIPT=/home/yuyr/.codex/skills/user/feishu-webhook/scripts/send_feishu_text.py \
+./run_24h_soak_with_metrics.sh
+```
+
+`SOAK_DURATION_SECS=0` 表示持续运行不自动停止；如需 24 小时自然停止，可设置为 `86400`，脚本会等当前 run 完成后退出，不会直接 kill 半轮验证。
+
+关键产物：
+
+- `runs/run_xxxx/`：最近 100 个 run 原始产物；
+- `hourly_reports/hour_*.md`：小时级报告；
+- `hourly_reports/hourly_summary.jsonl`：小时级结构化汇总；
+- `incident_runs/run_xxxx/`：异常 run 固化副本；
+- `logs/metrics.*`、`logs/24h-soak.*`、`logs/hourly-reporter.*`：运行日志。
+
+短周期联调可把 `SOAK_DURATION_SECS` 和 `HOURLY_REPORT_INTERVAL_SECS` 调小，并设置 `FEISHU_DRY_RUN=1` 避免真实飞书通知。
+
+## 停止
+
+```bash
+cd rpki_2/rpki/monitor
+docker compose down
+```
+
+保留数据 volume。若要清理数据：
+
+```bash
+docker compose down -v
+```
+
+## 典型本地联调命令
+
+先启动 APNIC soak 和 metrics sidecar，例如：
+
+```bash
+# soak .env 关键配置
+MAX_RUNS=-1
+RIRS=apnic
+RETAIN_RUNS=5
+INTERVAL_SECS=0
+
+# metrics sidecar
+rpki_artifact_metrics \
+  --run-root /path/to/portable-soak \
+  --listen 0.0.0.0:9556 \
+  --poll-secs 5 \
+  --instance local-apnic-continuous
+```
+
+再启动监控栈：
+
+```bash
+cd rpki_2/rpki/monitor
+docker compose up -d
+```
+
+## 验证
+
+Prometheus target：
+
+```bash
+curl -s 'http://localhost:9090/api/v1/targets' | python3 -m json.tool
+```
+
+Prometheus query：
+
+```bash
+curl -G 'http://localhost:9090/api/v1/query' \
+  --data-urlencode 'query=up{job="ours-rp-artifact-metrics"}'
+
+curl -G 'http://localhost:9090/api/v1/query' \
+  --data-urlencode 'query=ours_rp_run_completed_total{status="success"}'
+```
+
+Grafana health：
+
+```bash
+curl -s http://localhost:3000/api/health | python3 -m json.tool
+```
+
+Grafana dashboard：
+
+- 打开 <http://localhost:3000/d/ours-rp-soak-overview/ours-rp-soak-overview>
+
+## 主要指标
+
+- `ours_rp_metrics_service_up`
+- `ours_rp_run_completed_total`
+- `ours_rp_run_duration_seconds`
+- `ours_rp_run_max_rss_bytes`
+- `ours_rp_vrps{kind="total|unique"}`：`total` 为去重前 VRP 条目数，`unique` 按 `(ASN, IP Prefix, Max Length)` 去重。
+- `ours_rp_vaps`
+- `ours_rp_publication_points`
+- `ours_rp_repo_sync_phase_count`
+- `ours_rp_large_publication_points{object_count_gt="10|50|100|..."}`
+- `ours_rp_cir_objects`
+- `ours_rp_ccr_state_items`
+
+## Inter-RP 持续对比监控
+
+`rpki_inter_rp_metrics` 用于汇总三方 RP 的最新产物：
+
+- ours RP：读取当前 portable soak 的 `runs/run_xxxx/run-summary.json`、`result.ccr`、CSV 产物；
+- Routinator：读取远端200同步来的 `routinator/latest/run-meta.json`、`vrps.csv`、`vaps.csv`；
+- rpki-client 9.8：读取远端200同步来的 `rpki-client/latest/run-meta.json`、`vrps.csv`、`vaps.csv`、`result.ccr`。
+
+远端231 启动 sidecar 示例：
+
+```bash
+rpki_inter_rp_metrics \
+  --ours-run-root /root/rpki_20260608_2_feature062_24h_20260608T075547Z/portable-soak \
+  --peer-root /root/inter-rp-aggregator/synced-from-200 \
+  --listen 0.0.0.0:9557 \
+  --poll-secs 30 \
+  --instance remote231-inter-rp
+```
+
+Prometheus 已新增 `ours-rp-inter-rp-metrics` scrape job，默认访问 `host.docker.internal:9557`。
+
+远端200 runner 与远端231同步脚本位于：
+
+```text
+scripts/inter_rp/run_remote200_rp_loops.sh
+scripts/inter_rp/run_single_rp_with_rss.sh
+scripts/inter_rp/sync_remote200_to_231.sh
+scripts/inter_rp/run_inter_rp_metrics_sidecar.sh
+scripts/inter_rp/inter-rp.env.example
+```
+
+如需从本机独立开关远端200上的 Routinator 或 rpki-client，使用：
+
+```bash
+scripts/inter_rp/control_remote200_rp.sh status all
+scripts/inter_rp/control_remote200_rp.sh stop routinator
+scripts/inter_rp/control_remote200_rp.sh start routinator
+scripts/inter_rp/control_remote200_rp.sh restart rpki-client
+```
+
+默认远端为 `root@43.110.128.200`，可通过 `REMOTE_HOST=...` 覆盖；脚本只管理指定 RP 的 loop 和当前子进程，不会自动影响另一个 RP。
+
+关键指标：
+
+- `inter_rp_run_wall_seconds{rp="ours-rp|routinator|rpki-client"}`
+- `inter_rp_run_max_rss_bytes{rp="...",kind="aggregate_peak"}`
+- `inter_rp_vrps{rp="..."}`：按 `(ASN, IP Prefix, Max Length)` 去重。
+- `inter_rp_vaps{rp="..."}`：按 `(Customer ASN, Providers)` 去重，Routinator 使用 `--enable-aspa` JSON 输出转换，rpki-client 使用 `-j` JSON 输出转换。
+- `inter_rp_ccr_digest_match{left="ours-rp",right="rpki-client",state="overall|mfts|vrps|vaps|tas|rks"}`
+- `inter_rp_sync_age_seconds`
+
+Grafana dashboard：
+
+- <http://localhost:3000/d/ours-rp-inter-rp/ours-rp-inter-rp>

monitor/docker-compose.yml

@@ -0,0 +1,38 @@
+services:
+  prometheus:
+    image: ${PROMETHEUS_IMAGE:-prom/prometheus:v2.55.1}
+    container_name: ours-rp-prometheus
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.path=/prometheus
+      - --storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-7d}
+      - --web.enable-lifecycle
+    extra_hosts:
+      - host.docker.internal:host-gateway
+    ports:
+      - "${PROMETHEUS_PORT:-9090}:9090"
+    volumes:
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus-data:/prometheus
+    restart: unless-stopped
+
+  grafana:
+    image: ${GRAFANA_IMAGE:-grafana/grafana:11.3.1}
+    container_name: ours-rp-grafana
+    depends_on:
+      - prometheus
+    ports:
+      - "${GRAFANA_PORT:-3000}:3000"
+    environment:
+      GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
+      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:-admin}
+      GF_USERS_ALLOW_SIGN_UP: "false"
+    volumes:
+      - grafana-data:/var/lib/grafana
+      - ./grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./grafana/dashboards:/var/lib/grafana/dashboards:ro
+    restart: unless-stopped
+
+volumes:
+  prometheus-data:
+  grafana-data:

monitor/grafana/dashboards/ours-rp-inter-rp.json

@@ -0,0 +1,826 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "id": 1,
+      "title": "Ours Only Repo Count",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "expr": "max(inter_rp_repo_sync_overlap_total{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"only_ours\"})",
+          "legendFormat": "only ours",
+          "refId": "A",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "id": 2,
+      "title": "Routinator Only Repo Count",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 6,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "expr": "max(inter_rp_repo_sync_overlap_total{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"only_routinator\"})",
+          "legendFormat": "only routinator",
+          "refId": "A",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "id": 3,
+      "title": "Ours vs Routinator VAP Diff",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "expr": "max(inter_rp_vaps_diff{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\"})",
+          "legendFormat": "vap diff",
+          "refId": "A",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "id": 4,
+      "title": "Ours vs Routinator VRP Diff",
+      "type": "stat",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "targets": [
+        {
+          "expr": "max(inter_rp_vrps_diff{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\"})",
+          "legendFormat": "vrp diff",
+          "refId": "A",
+          "instant": true
+        }
+      ]
+    },
+    {
+      "id": 5,
+      "title": "Wall Time by RP",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 4
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_run_wall_seconds{exported_instance=~\".*inter-rp\",rp=~\"ours-rp|routinator\"}",
+          "legendFormat": "{{rp}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 6,
+      "title": "Max RSS Aggregate Peak by RP",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 4
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_run_max_rss_bytes{exported_instance=~\".*inter-rp\",kind=\"aggregate_peak\",rp=~\"ours-rp|routinator\"}",
+          "legendFormat": "{{rp}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 7,
+      "title": "VRPs by RP (unique ASN/Prefix/MaxLen)",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 12
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_vrps{exported_instance=~\".*inter-rp\",rp=~\"ours-rp|routinator\"}",
+          "legendFormat": "{{rp}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 8,
+      "title": "VAPs / ASPAs by RP (unique Customer/Providers)",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 12
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_vaps{exported_instance=~\".*inter-rp\",rp=~\"ours-rp|routinator\"}",
+          "legendFormat": "{{rp}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 9,
+      "title": "Latest RP Runs",
+      "type": "table",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 20
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "showHeader": true,
+        "sortBy": []
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_run_seq{exported_instance=~\".*inter-rp\",rp=~\"ours-rp|routinator\"}",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "{{rp}} seq",
+          "refId": "A"
+        },
+        {
+          "expr": "inter_rp_run_success{exported_instance=~\".*inter-rp\",rp=~\"ours-rp|routinator\"}",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "{{rp}} success",
+          "refId": "B"
+        },
+        {
+          "expr": "inter_rp_run_wall_seconds{exported_instance=~\".*inter-rp\",rp=~\"ours-rp|routinator\"}",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "{{rp}} wall",
+          "refId": "C"
+        }
+      ]
+    },
+    {
+      "id": 10,
+      "title": "Output Count Diffs (unique)",
+      "type": "table",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 20
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "showHeader": true,
+        "sortBy": []
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_vrps_diff{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\"}",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "vrps ours-rp-routinator",
+          "refId": "A"
+        },
+        {
+          "expr": "inter_rp_vaps_diff{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\"}",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "vaps ours-rp-routinator",
+          "refId": "B"
+        }
+      ]
+    },
+    {
+      "id": 15,
+      "title": "VRP Diff Trend by Class",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 28
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "min": 0,
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_vrps_diff_by_class{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"total\"}",
+          "legendFormat": "total diff",
+          "refId": "A"
+        },
+        {
+          "expr": "inter_rp_vrps_diff_by_class{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"only_ours\"}",
+          "legendFormat": "only ours",
+          "refId": "B"
+        },
+        {
+          "expr": "inter_rp_vrps_diff_by_class{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"only_routinator\"}",
+          "legendFormat": "only routinator",
+          "refId": "C"
+        }
+      ]
+    },
+    {
+      "id": 16,
+      "title": "VAP Diff Trend by Class",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 28
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "min": 0,
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_vaps_diff_by_class{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"total\"}",
+          "legendFormat": "total diff",
+          "refId": "A"
+        },
+        {
+          "expr": "inter_rp_vaps_diff_by_class{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"only_ours\"}",
+          "legendFormat": "only ours",
+          "refId": "B"
+        },
+        {
+          "expr": "inter_rp_vaps_diff_by_class{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"only_routinator\"}",
+          "legendFormat": "only routinator",
+          "refId": "C"
+        }
+      ]
+    },
+    {
+      "id": 11,
+      "title": "Artifact Age by RP",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 36
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "min": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_artifact_age_seconds{exported_instance=~\".*inter-rp\",rp=~\"ours-rp|routinator\"}",
+          "legendFormat": "{{rp}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 12,
+      "title": "Repo Sync Availability by RP",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 44
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "min": 0,
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_repo_sync_total{exported_instance=~\".*inter-rp\",state=~\"available|failed\",rp=~\"ours-rp|routinator\"}",
+          "legendFormat": "{{rp}} {{state}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 13,
+      "title": "Repo Sync Overlap Classes",
+      "type": "timeseries",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 44
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "min": 0,
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_repo_sync_overlap_total{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\"}",
+          "legendFormat": "{{class}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "id": 14,
+      "title": "Repo Sync Diff URIs",
+      "type": "table",
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "gridPos": {
+        "h": 9,
+        "w": 24,
+        "x": 0,
+        "y": 52
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "uri"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 760
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "class"
+            },
+            "properties": [
+              {
+                "id": "custom.width",
+                "value": 140
+              }
+            ]
+          },
+          {
+            "matcher": {
+              "id": "byRegexp",
+              "options": "^(mft|crl|crt|roa|aspa)$"
+            },
+            "properties": [
+              {
+                "id": "custom.align",
+                "value": "right"
+              },
+              {
+                "id": "custom.width",
+                "value": 80
+              }
+            ]
+          }
+        ]
+      },
+      "options": {
+        "showHeader": true,
+        "sortBy": []
+      },
+      "targets": [
+        {
+          "expr": "inter_rp_repo_sync_diff_info{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\"}",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "{{class}} #{{rank}}",
+          "refId": "A"
+        }
+      ],
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Value": true,
+              "__name__": true,
+              "exported_instance": true,
+              "instance": true,
+              "job": true,
+              "left": true,
+              "right": true,
+              "rank": true,
+              "routinator_duration": true
+            },
+            "indexByName": {
+              "class": 0,
+              "uri": 1,
+              "mft": 2,
+              "crl": 3,
+              "crt": 4,
+              "roa": 5,
+              "aspa": 6
+            },
+            "renameByName": {
+              "class": "class",
+              "uri": "uri",
+              "mft": "mft",
+              "crl": "crl",
+              "crt": "crt",
+              "roa": "roa",
+              "aspa": "aspa"
+            }
+          }
+        }
+      ]
+    }
+  ],
+  "refresh": "10s",
+  "schemaVersion": 40,
+  "tags": [
+    "rpki",
+    "inter-rp",
+    "routinator"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timezone": "browser",
+  "title": "Ours RP vs Routinator",
+  "uid": "ours-rp-inter-rp",
+  "version": 4
+}

monitor/grafana/dashboards/ours-rp-repo-sync.json

@@ -0,0 +1,761 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 0,
+        "y": 0
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_publication_points",
+          "legendFormat": "publication points",
+          "refId": "A"
+        }
+      ],
+      "title": "Publication Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 6,
+        "y": 0
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_ok\"}",
+          "legendFormat": "rrdp ok",
+          "refId": "A"
+        }
+      ],
+      "title": "RRDP OK Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 12,
+        "y": 0
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "sum(ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_ok\"}) or vector(0)",
+          "legendFormat": "fallback",
+          "refId": "A"
+        }
+      ],
+      "title": "Rsync Fallback Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 6,
+        "x": 18,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_repo_terminal_state_count{terminal_state=\"failed_no_cache\"}",
+          "legendFormat": "failed no cache",
+          "refId": "A"
+        }
+      ],
+      "title": "Failed No Cache Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 4
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"repo_sync_total\"}",
+          "legendFormat": "repo sync total",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"rrdp_download_total\"}",
+          "legendFormat": "rrdp download",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"rsync_download_total\"}",
+          "legendFormat": "rsync download",
+          "refId": "C"
+        }
+      ],
+      "title": "Repo Sync Download Durations",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 12,
+        "w": 12,
+        "h": 8
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repo_sync_phase_count",
+          "legendFormat": "{{phase}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Repo Sync Phase Counts",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 12,
+        "w": 12,
+        "h": 8
+      },
+      "id": 7,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_ok\"}",
+          "legendFormat": "rrdp failed, rsync ok",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_failed\"}",
+          "legendFormat": "rrdp failed, rsync failed",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_repo_terminal_state_count{terminal_state=\"failed_no_cache\"}",
+          "legendFormat": "failed no cache",
+          "refId": "C"
+        },
+        {
+          "expr": "ours_rp_tree_instances{state=\"failed\"}",
+          "legendFormat": "tree failed",
+          "refId": "D"
+        }
+      ],
+      "title": "Repo Failure / Fallback Counts",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 20,
+        "w": 12,
+        "h": 8
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repo_sync_phase_duration_seconds_total{phase=\"rrdp_failed_rsync_ok\"}",
+          "legendFormat": "rsync fallback duration",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_repo_sync_phase_duration_seconds_total{phase=\"rrdp_failed_rsync_failed\"}",
+          "legendFormat": "failed duration",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_repo_terminal_state_duration_seconds_total{terminal_state=\"failed_no_cache\"}",
+          "legendFormat": "failed no cache duration",
+          "refId": "C"
+        }
+      ],
+      "title": "Repo Failure / Fallback Durations",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 29,
+        "w": 12,
+        "h": 9
+      },
+      "id": 9,
+      "options": {
+        "showHeader": true,
+        "cellHeight": "sm",
+        "footer": {
+          "show": false,
+          "reducer": [
+            "sum"
+          ],
+          "countRows": false,
+          "fields": ""
+        }
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_rrdp_rsync_failed_repository_duration_seconds",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "RRDP + Rsync Failed Repositories",
+      "type": "table",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "job": true,
+              "terminal_state": true,
+              "rank": true,
+              "transport": true,
+              "__name__": true,
+              "publication_points": true,
+              "instance": true,
+              "repo_id": true,
+              "pp_id": true,
+              "exported_instance": true,
+              "rp": true,
+              "source": true
+            },
+            "indexByName": {
+              "Time": 0,
+              "host": 1,
+              "phase": 2,
+              "uri": 3,
+              "Value": 4
+            },
+            "renameByName": {
+              "Value": "duration"
+            }
+          }
+        }
+      ]
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 29,
+        "w": 12,
+        "h": 9
+      },
+      "id": 11,
+      "options": {
+        "showHeader": true,
+        "cellHeight": "sm",
+        "footer": {
+          "show": false,
+          "reducer": [
+            "sum"
+          ],
+          "countRows": false,
+          "fields": ""
+        }
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "topk(20, ours_rp_top_repository_sync_duration_seconds)",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Top 20 Repositories by Sync Duration",
+      "type": "table",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "job": true,
+              "terminal_state": true,
+              "__name__": true,
+              "publication_points": true,
+              "instance": true,
+              "repo_id": true,
+              "phase": true,
+              "pp_id": true,
+              "exported_instance": true,
+              "rp": true,
+              "source": true
+            },
+            "indexByName": {
+              "Time": 0,
+              "host": 1,
+              "rank": 2,
+              "transport": 3,
+              "uri": 4,
+              "Value": 5
+            },
+            "renameByName": {
+              "Value": "value"
+            }
+          }
+        }
+      ]
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 38,
+        "w": 24,
+        "h": 9
+      },
+      "id": 10,
+      "options": {
+        "showHeader": true,
+        "cellHeight": "sm",
+        "footer": {
+          "show": false,
+          "reducer": [
+            "sum"
+          ],
+          "countRows": false,
+          "fields": ""
+        }
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "topk(20, ours_rp_top_publication_point_object_count)",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Top Publication Points by Objects",
+      "type": "table",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "job": true,
+              "__name__": true,
+              "publication_points": true,
+              "instance": true,
+              "repo_id": true,
+              "phase": true,
+              "pp_id": true,
+              "exported_instance": true,
+              "rp": true,
+              "source": true
+            },
+            "indexByName": {
+              "Time": 0,
+              "host": 1,
+              "rank": 2,
+              "terminal_state": 3,
+              "transport": 4,
+              "uri": 5,
+              "Value": 6
+            },
+            "renameByName": {}
+          }
+        }
+      ]
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "description": "Per-repository sync success in the latest successful run; 1 means successful, 0 means failed or failed_no_cache.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bool"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 47
+      },
+      "id": 12,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repository_sync_success",
+          "legendFormat": "{{host}} {{repo_id}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Repository Sync Success by Repo",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "description": "Per-repository total sync duration aggregated from publication point repo_sync_duration_ms.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 55
+      },
+      "id": 13,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repository_sync_duration_seconds{stat=\"sum\"}",
+          "legendFormat": "{{host}} {{repo_id}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Repository Sync Duration by Repo",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "description": "Per-repository downloaded bytes attributed from report.json downloads events.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 63
+      },
+      "id": 14,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_repository_download_bytes",
+          "legendFormat": "{{host}} {{repo_id}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Repository Download Bytes by Repo",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 40,
+  "tags": [
+    "ours-rp",
+    "rpki",
+    "soak",
+    "repo-sync"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Ours RP Repo Sync",
+  "uid": "ours-rp-repo-sync",
+  "version": 3,
+  "weekStart": ""
+}

monitor/grafana/dashboards/ours-rp-soak-overview.json

@@ -0,0 +1,875 @@
+{
+  "annotations": {
+    "list": []
+  },
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": null,
+  "links": [],
+  "panels": [
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_cir_trust_anchors",
+          "legendFormat": "RIRs",
+          "refId": "A"
+        }
+      ],
+      "title": "Current Run RIRs",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 6,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_run_duration_seconds",
+          "legendFormat": "wall",
+          "refId": "A"
+        }
+      ],
+      "title": "Latest Wall Time",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_run_max_rss_bytes",
+          "legendFormat": "rss",
+          "refId": "A"
+        }
+      ],
+      "title": "Latest Max RSS",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 18,
+        "y": 0,
+        "w": 6,
+        "h": 4
+      },
+      "id": 4,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_publication_points",
+          "legendFormat": "publication points",
+          "refId": "A"
+        }
+      ],
+      "title": "Publication Points",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 4,
+        "w": 6,
+        "h": 4
+      },
+      "id": 9,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_run_sequence",
+          "legendFormat": "seq",
+          "refId": "A"
+        }
+      ],
+      "title": "Latest Run Sequence",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 2,
+          "unit": "percent",
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 90
+              },
+              {
+                "color": "green",
+                "value": 98
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 6,
+        "y": 4,
+        "w": 6,
+        "h": 4
+      },
+      "id": 10,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "100 * sum by (job, instance, exported_instance) (ours_rp_repo_terminal_state_count{terminal_state=\"publication_point_cache\"}) / sum by (job, instance, exported_instance) (ours_rp_publication_points)",
+          "legendFormat": "PP cache hit ratio",
+          "refId": "A"
+        }
+      ],
+      "title": "Latest PP Cache Hit Ratio",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 4,
+        "w": 6,
+        "h": 4
+      },
+      "id": 11,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_vrps{kind=\"total\"}",
+          "legendFormat": "VRPs raw",
+          "refId": "A"
+        }
+      ],
+      "title": "VRPs",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 18,
+        "y": 4,
+        "w": 6,
+        "h": 4
+      },
+      "id": 12,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto",
+        "wideLayout": true
+      },
+      "pluginVersion": "11.3.1",
+      "targets": [
+        {
+          "expr": "ours_rp_vaps",
+          "legendFormat": "VAPs",
+          "refId": "A"
+        }
+      ],
+      "title": "VAPs",
+      "type": "stat"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 8,
+        "w": 12,
+        "h": 8
+      },
+      "id": 5,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_run_duration_seconds",
+          "legendFormat": "wall",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_stage_duration_seconds{stage=\"validation\"}",
+          "legendFormat": "validation",
+          "refId": "B"
+        }
+      ],
+      "title": "Run / Validation Duration",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 8,
+        "w": 12,
+        "h": 8
+      },
+      "id": 6,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_vrps{kind=\"total\"}",
+          "legendFormat": "VRPs raw",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_vrps{kind=\"unique\"}",
+          "legendFormat": "VRPs unique",
+          "refId": "D"
+        },
+        {
+          "expr": "ours_rp_vaps",
+          "legendFormat": "VAPs",
+          "refId": "B"
+        },
+        {
+          "expr": "ours_rp_cir_objects",
+          "legendFormat": "CIR objects",
+          "refId": "C"
+        }
+      ],
+      "title": "Output and Input Sizes",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "decimals": 0,
+          "unit": "none"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 16,
+        "w": 12,
+        "h": 8
+      },
+      "id": 8,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_large_publication_points",
+          "legendFormat": "> {{object_count_gt}} objects",
+          "refId": "A"
+        }
+      ],
+      "title": "Large Publication Points by Object Count",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 16,
+        "w": 12,
+        "h": 8
+      },
+      "id": 13,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"validation\"}",
+          "legendFormat": "validation",
+          "refId": "A"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"report_write\"}",
+          "legendFormat": "report write",
+          "refId": "E"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"ccr_write\"}",
+          "legendFormat": "ccr write",
+          "refId": "F"
+        },
+        {
+          "expr": "ours_rp_run_stage_duration_seconds{stage=\"cir_write\"}",
+          "legendFormat": "cir write",
+          "refId": "G"
+        }
+      ],
+      "title": "Output Stage Durations",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "decimals": 2
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 24,
+        "w": 12,
+        "h": 8
+      },
+      "id": 14,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_run_max_rss_bytes",
+          "legendFormat": "Max RSS",
+          "refId": "A"
+        }
+      ],
+      "title": "Max RSS Over Time",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percent",
+          "decimals": 2,
+          "min": 0,
+          "max": 100,
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "red",
+                "value": null
+              },
+              {
+                "color": "orange",
+                "value": 90
+              },
+              {
+                "color": "green",
+                "value": 98
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 24,
+        "w": 12,
+        "h": 8
+      },
+      "id": 17,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "min",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "100 * sum by (job, instance, exported_instance) (ours_rp_repo_terminal_state_count{terminal_state=\"publication_point_cache\"}) / sum by (job, instance, exported_instance) (ours_rp_publication_points)",
+          "legendFormat": "PP cache hit ratio",
+          "refId": "A"
+        }
+      ],
+      "title": "PP Cache Hit Ratio",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "bytes",
+          "decimals": 2
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 32,
+        "w": 24,
+        "h": 8
+      },
+      "id": 15,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_state_db_size_bytes",
+          "legendFormat": "{{db}}",
+          "refId": "A"
+        }
+      ],
+      "title": "State DB Size Over Time",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 40,
+        "w": 24,
+        "h": 8
+      },
+      "id": 16,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "ours_rp_state_db_files",
+          "legendFormat": "{{db}}",
+          "refId": "A"
+        }
+      ],
+      "title": "State DB File Count Over Time",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "Prometheus"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "none",
+          "decimals": 0,
+          "min": 0
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 48,
+        "w": 24,
+        "h": 8
+      },
+      "id": 18,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "expr": "sum by (job, instance, exported_instance) (ours_rp_repo_terminal_state_count{terminal_state=\"fresh\"})",
+          "legendFormat": "fresh pp",
+          "refId": "A"
+        },
+        {
+          "expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"roa\"})",
+          "legendFormat": "fresh roa",
+          "refId": "B"
+        },
+        {
+          "expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"manifest\"})",
+          "legendFormat": "fresh mft",
+          "refId": "C"
+        },
+        {
+          "expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"certificate\"})",
+          "legendFormat": "fresh crt",
+          "refId": "D"
+        },
+        {
+          "expr": "sum by (job, instance, exported_instance) (ours_rp_cir_objects_by_source_type{exported_source=\"fresh\",object_type=\"crl\"})",
+          "legendFormat": "fresh crl",
+          "refId": "E"
+        }
+      ],
+      "title": "Fresh PP / Object Counts by Run",
+      "type": "timeseries"
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 40,
+  "tags": [
+    "ours-rp",
+    "rpki",
+    "soak"
+  ],
+  "templating": {
+    "list": []
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Ours RP Soak Overview",
+  "uid": "ours-rp-soak-overview",
+  "version": 4,
+  "weekStart": ""
+}

monitor/grafana/provisioning/dashboards/dashboard.yml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+providers:
+  - name: ours-rp
+    orgId: 1
+    folder: Ours RP
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 10
+    allowUiUpdates: true
+    options:
+      path: /var/lib/grafana/dashboards

monitor/grafana/provisioning/datasources/prometheus.yml

@@ -0,0 +1,10 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    uid: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: true

monitor/prometheus/prometheus.yml

@@ -0,0 +1,20 @@
+global:
+  scrape_interval: 5s
+  evaluation_interval: 5s
+
+scrape_configs:
+  - job_name: ours-rp-artifact-metrics
+    metrics_path: /metrics
+    static_configs:
+      - targets:
+          - host.docker.internal:9556
+        labels:
+          rp: ours-rp
+          source: artifact-sidecar
+  - job_name: ours-rp-inter-rp-metrics
+    metrics_path: /metrics
+    static_configs:
+      - targets:
+          - host.docker.internal:9557
+        labels:
+          source: inter-rp-sidecar

scripts/cir/run_cir_drop_sequence.sh

@@ -34,11 +34,7 @@ from pathlib import Path
 sequence_root = Path(sys.argv[1]).resolve()
 drop_bin = sys.argv[2]
 sequence = json.loads((sequence_root / "sequence.json").read_text(encoding="utf-8"))
-static_root = sequence_root / sequence["staticRoot"] if "staticRoot" in sequence else None
-raw_store_db = sequence_root / sequence["rawStoreDbPath"] if "rawStoreDbPath" in sequence else None
-backend_count = sum(x is not None for x in (static_root, raw_store_db))
-if backend_count != 1:
-    raise SystemExit("sequence must set exactly one of staticRoot or rawStoreDbPath")
+repo_bytes_db = sequence_root / sequence["repoBytesDbPath"]
 
 summaries = []
 for step in sequence["steps"]:
@@ -58,10 +54,7 @@ for step in sequence["steps"]:
         "--md-out",
         str(out_dir / "drop.md"),
     ]
-    if static_root is not None:
-        cmd.extend(["--static-root", str(static_root)])
-    else:
-        cmd.extend(["--raw-store-db", str(raw_store_db)])
+    cmd.extend(["--repo-bytes-db", str(repo_bytes_db)])
     proc = subprocess.run(cmd, capture_output=True, text=True)
     if proc.returncode != 0:
         raise SystemExit(

scripts/cir/run_cir_record_full_delta.sh

@@ -92,7 +92,8 @@ if [[ -z "$DELTA_VALIDATION_TIME" ]]; then
 fi
 
 rm -rf "$OUT_DIR"
-mkdir -p "$OUT_DIR/full" "$OUT_DIR/delta-001" "$OUT_DIR/static"
+mkdir -p "$OUT_DIR/full" "$OUT_DIR/delta-001"
+REPO_BYTES_DB="$OUT_DIR/repo-bytes.db"
 
 FULL_DB="$OUT_DIR/full/db"
 DELTA_DB="$OUT_DIR/delta-001/db"
@@ -110,7 +111,7 @@ DELTA_DB="$OUT_DIR/delta-001/db"
   --report-json "$OUT_DIR/full/report.json" \
   --cir-enable \
   --cir-out "$OUT_DIR/full/input.cir" \
-  --cir-static-root "$OUT_DIR/static" \
+  --repo-bytes-db "$REPO_BYTES_DB" \
   --cir-tal-uri "$CIR_TAL_URI" \
   >"$OUT_DIR/full/run.stdout.log" 2>"$OUT_DIR/full/run.stderr.log"
 
@@ -130,7 +131,7 @@ DELTA_DB="$OUT_DIR/delta-001/db"
   --report-json "$OUT_DIR/delta-001/report.json" \
   --cir-enable \
   --cir-out "$OUT_DIR/delta-001/input.cir" \
-  --cir-static-root "$OUT_DIR/static" \
+  --repo-bytes-db "$REPO_BYTES_DB" \
   --cir-tal-uri "$CIR_TAL_URI" \
   >"$OUT_DIR/delta-001/run.stdout.log" 2>"$OUT_DIR/delta-001/run.stderr.log"
 
@@ -143,13 +144,12 @@ from pathlib import Path
 out = Path(sys.argv[1])
 base_validation_time = sys.argv[2]
 delta_validation_time = sys.argv[3]
-static_files = sum(1 for _ in (out / "static").rglob("*") if _.is_file())
 summary = {
     "version": 1,
     "kind": "cir_pair",
     "baseValidationTime": base_validation_time,
     "deltaValidationTime": delta_validation_time,
-    "staticRoot": "static",
+    "repoBytesDbPath": "repo-bytes.db",
     "steps": [
         {
             "kind": "full",
@@ -165,7 +165,7 @@ summary = {
             "previous": "full",
         },
     ],
-    "staticFileCount": static_files,
+    "repoBytesDbExists": (out / "repo-bytes.db").exists(),
 }
 (out / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
 PY

scripts/cir/run_cir_record_sequence_multi_rir_offline.sh

@@ -116,13 +116,13 @@ for rir in rirs:
         "rir": rir,
         "root": str(root),
         "stepCount": len(seq["steps"]),
-        "staticFileCount": summ["staticFileCount"],
+        "repoBytesDbExists": summ.get("repoBytesDbExists", False),
     })
 summary = {"version": 1, "rirs": items}
 summary_json.write_text(json.dumps(summary, indent=2), encoding="utf-8")
 lines = ["# Multi-RIR Offline CIR Sequence Summary", ""]
 for item in items:
-    lines.append(f"- `{item['rir']}`: `stepCount={item['stepCount']}` `staticFileCount={item['staticFileCount']}` `root={item['root']}`")
+    lines.append(f"- `{item['rir']}`: `stepCount={item['stepCount']}` `repoBytesDbExists={item['repoBytesDbExists']}` `root={item['root']}`")
 summary_md.write_text("\n".join(lines) + "\n", encoding="utf-8")
 PY
 

scripts/cir/run_cir_record_sequence_offline.sh

@@ -97,7 +97,8 @@ if [[ -z "$DELTA_VALIDATION_TIME" ]]; then
 fi
 
 rm -rf "$OUT_DIR"
-mkdir -p "$OUT_DIR/static" "$OUT_DIR/full"
+mkdir -p "$OUT_DIR/full"
+REPO_BYTES_DB="$OUT_DIR/repo-bytes.db"
 
 run_step() {
   local kind="$1"
@@ -114,7 +115,7 @@ run_step() {
     --report-json "$step_dir/report.json" \
     --cir-enable \
     --cir-out "$step_dir/input.cir" \
-    --cir-static-root "$OUT_DIR/static" \
+    --repo-bytes-db "$REPO_BYTES_DB" \
     --cir-tal-uri "$CIR_TAL_URI"
   )
   if [[ "$FULL_REPO" -ne 1 ]]; then
@@ -186,7 +187,7 @@ for idx in range(1, delta_count + 1):
 summary = {
     "version": 1,
     "kind": "cir_sequence_offline",
-    "staticRoot": "static",
+    "repoBytesDbPath": "repo-bytes.db",
     "steps": steps,
 }
 (out / "sequence.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
@@ -195,7 +196,8 @@ summary = {
         {
             "version": 1,
             "stepCount": len(steps),
-            "staticFileCount": sum(1 for p in (out / "static").rglob("*") if p.is_file()),
+            "repoBytesDbPath": "repo-bytes.db",
+            "repoBytesDbExists": (out / "repo-bytes.db").exists(),
         },
         indent=2,
     ),

scripts/cir/run_cir_record_sequence_remote.sh

@@ -87,6 +87,7 @@ fi
 mkdir -p "$OUT"
 DB="$OUT/work-db"
 RAW_STORE_DB="$OUT/raw-store.db"
+REPO_BYTES_DB="$OUT/repo-bytes.db"
 ROWS="$OUT/.sequence_rows.tsv"
 : > "$ROWS"
 
@@ -140,6 +141,7 @@ PY
     target/release/rpki
     --db "$DB"
     --raw-store-db "$RAW_STORE_DB"
+    --repo-bytes-db "$REPO_BYTES_DB"
     --tal-path "$TAL_REL"
     --ta-path "$TA_REL"
     --ccr-out "$ccr_out"
@@ -216,7 +218,7 @@ for idx, row in enumerate(rows):
     })
 
 (out / "sequence.json").write_text(
-    json.dumps({"version": 1, "rawStoreDbPath": "raw-store.db", "steps": steps}, indent=2),
+    json.dumps({"version": 1, "repoBytesDbPath": "repo-bytes.db", "steps": steps}, indent=2),
     encoding="utf-8",
 )
 

scripts/cir/run_cir_record_sequence_ta_only_multi_rir.sh

@@ -56,16 +56,17 @@ IFS=',' read -r -a ITEMS <<< "$RIRS"
 for rir in "${ITEMS[@]}"; do
   read -r tal_rel ta_rel < <(case_paths "$rir")
   rir_root="$OUT_ROOT/$rir"
-  mkdir -p "$rir_root/full" "$rir_root/static"
+  mkdir -p "$rir_root/full"
+  repo_bytes_db="$rir_root/repo-bytes.db"
   "$HELPER_BIN" \
     --tal-path "$ROOT_DIR/$tal_rel" \
     --ta-path "$ROOT_DIR/$ta_rel" \
     --tal-uri "https://example.test/$rir.tal" \
     --validation-time "2026-04-09T00:00:00Z" \
     --cir-out "$rir_root/full/input.cir" \
-    --static-root "$rir_root/static"
+    --repo-bytes-db "$repo_bytes_db"
   "$EXTRACT_BIN" --cir "$rir_root/full/input.cir" --tals-dir "$rir_root/.tmp/tals" --meta-json "$rir_root/.tmp/meta.json"
-  "$MATERIALIZE_BIN" --cir "$rir_root/full/input.cir" --static-root "$rir_root/static" --mirror-root "$rir_root/.tmp/mirror"
+  "$MATERIALIZE_BIN" --cir "$rir_root/full/input.cir" --repo-bytes-db "$repo_bytes_db" --mirror-root "$rir_root/.tmp/mirror"
   FIRST_TAL="$(python3 - <<'PY' "$rir_root/.tmp/meta.json"
 import json,sys
 print(json.load(open(sys.argv[1]))["talFiles"][0]["path"])
@@ -99,7 +100,7 @@ for i in range(1, delta_count + 1):
     step = f"delta-{i:03d}"
     steps.append({"stepId":step,"kind":"delta","validationTime":"2026-04-09T00:00:00Z","cirPath":f"{step}/input.cir","ccrPath":f"{step}/result.ccr","reportPath":f"{step}/report.json","previousStepId":prev})
     prev = step
-(root/"sequence.json").write_text(json.dumps({"version":1,"staticRoot":"static","steps":steps}, indent=2), encoding="utf-8")
+(root/"sequence.json").write_text(json.dumps({"version":1,"repoBytesDbPath":"repo-bytes.db","steps":steps}, indent=2), encoding="utf-8")
 (root/"summary.json").write_text(json.dumps({"version":1,"stepCount":len(steps)}, indent=2), encoding="utf-8")
 PY
 done

scripts/cir/run_cir_replay_matrix.sh

@@ -6,7 +6,7 @@ usage() {
 Usage:
   ./scripts/cir/run_cir_replay_matrix.sh \
     --cir <path> \
-    --static-root <path> \
+    --repo-bytes-db <path> \
     --out-dir <path> \
     --reference-ccr <path> \
     --rpki-client-build-dir <path> \
@@ -21,7 +21,7 @@ EOF
 ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 
 CIR=""
-STATIC_ROOT=""
+REPO_BYTES_DB=""
 OUT_DIR=""
 REFERENCE_CCR=""
 RPKI_CLIENT_BUILD_DIR=""
@@ -37,7 +37,7 @@ RPKI_CLIENT_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_rpki_client.sh"
 while [[ $# -gt 0 ]]; do
   case "$1" in
     --cir) CIR="$2"; shift 2 ;;
-    --static-root) STATIC_ROOT="$2"; shift 2 ;;
+    --repo-bytes-db) REPO_BYTES_DB="$2"; shift 2 ;;
     --out-dir) OUT_DIR="$2"; shift 2 ;;
     --reference-ccr) REFERENCE_CCR="$2"; shift 2 ;;
     --rpki-client-build-dir) RPKI_CLIENT_BUILD_DIR="$2"; shift 2 ;;
@@ -51,7 +51,7 @@ while [[ $# -gt 0 ]]; do
   esac
 done
 
-[[ -n "$CIR" && -n "$STATIC_ROOT" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" && -n "$RPKI_CLIENT_BUILD_DIR" ]] || {
+[[ -n "$CIR" && -n "$REPO_BYTES_DB" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" && -n "$RPKI_CLIENT_BUILD_DIR" ]] || {
   usage >&2
   exit 2
 }
@@ -102,7 +102,7 @@ mkdir -p "$OURS_OUT" "$ROUTINATOR_OUT" "$RPKI_CLIENT_OUT"
 ours_cmd=(
   "$OURS_SCRIPT"
   --cir "$CIR"
-  --static-root "$STATIC_ROOT"
+  --repo-bytes-db "$REPO_BYTES_DB"
   --out-dir "$OURS_OUT"
   --reference-ccr "$REFERENCE_CCR"
   --rpki-bin "$RPKI_BIN"
@@ -111,7 +111,7 @@ ours_cmd=(
 routinator_cmd=(
   "$ROUTINATOR_SCRIPT"
   --cir "$CIR"
-  --static-root "$STATIC_ROOT"
+  --repo-bytes-db "$REPO_BYTES_DB"
   --out-dir "$ROUTINATOR_OUT"
   --reference-ccr "$REFERENCE_CCR"
   --routinator-root "$ROUTINATOR_ROOT"
@@ -121,7 +121,7 @@ routinator_cmd=(
 rpki_client_cmd=(
   "$RPKI_CLIENT_SCRIPT"
   --cir "$CIR"
-  --static-root "$STATIC_ROOT"
+  --repo-bytes-db "$REPO_BYTES_DB"
   --out-dir "$RPKI_CLIENT_OUT"
   --reference-ccr "$REFERENCE_CCR"
   --build-dir "$RPKI_CLIENT_BUILD_DIR"
@@ -159,7 +159,7 @@ DETAIL_MD="$OUT_DIR/detail.md"
 
 python3 - <<'PY' \
   "$CIR" \
-  "$STATIC_ROOT" \
+  "$REPO_BYTES_DB" \
   "$REFERENCE_CCR" \
   "$OURS_OUT" \
   "$ROUTINATOR_OUT" \
@@ -171,7 +171,7 @@ import json
 import sys
 from pathlib import Path
 
-cir_path, static_root, reference_ccr, ours_out, routinator_out, rpki_client_out, summary_json, summary_md, detail_md = sys.argv[1:]
+cir_path, repo_bytes_db, reference_ccr, ours_out, routinator_out, rpki_client_out, summary_json, summary_md, detail_md = sys.argv[1:]
 
 participants = []
 all_match = True
@@ -194,6 +194,9 @@ for name, out_dir in [
         "summaryPath": str(out / "compare-summary.json"),
         "exitCode": timing["exitCode"],
         "durationMs": timing["durationMs"],
+        "compareMode": compare.get("compareMode"),
+        "talCount": compare.get("talCount"),
+        "talPaths": compare.get("talPaths", []),
         "vrps": vrps,
         "vaps": vaps,
         "match": bool(vrps.get("match")) and bool(vaps.get("match")) and timing["exitCode"] == 0,
@@ -204,7 +207,7 @@ for name, out_dir in [
 
 summary = {
     "cirPath": cir_path,
-    "staticRoot": static_root,
+    "repoBytesDb": repo_bytes_db,
     "referenceCcr": reference_ccr,
     "participants": participants,
     "allMatch": all_match,
@@ -215,22 +218,24 @@ lines = [
     "# CIR Replay Matrix Summary",
     "",
     f"- `cir`: `{cir_path}`",
-    f"- `static_root`: `{static_root}`",
+    f"- `repo_bytes_db`: `{repo_bytes_db}`",
     f"- `reference_ccr`: `{reference_ccr}`",
     f"- `all_match`: `{all_match}`",
     "",
-    "| Participant | Exit | Duration (ms) | VRP actual/ref | VRP match | VAP actual/ref | VAP match | Log |",
-    "| --- | ---: | ---: | --- | --- | --- | --- | --- |",
+    "| Participant | Exit | Duration (ms) | TALs | Compare mode | VRP actual/ref | VRP match | VAP actual/ref | VAP match | Log |",
+    "| --- | ---: | ---: | ---: | --- | --- | --- | --- | --- | --- |",
 ]
 for participant in participants:
     vrps = participant["vrps"] or {}
     vaps = participant["vaps"] or {}
     log_path = participant["logPaths"][0] if participant["logPaths"] else ""
     lines.append(
-        "| {name} | {exit_code} | {duration_ms} | {vrp_actual}/{vrp_ref} | {vrp_match} | {vap_actual}/{vap_ref} | {vap_match} | `{log_path}` |".format(
+        "| {name} | {exit_code} | {duration_ms} | {tal_count} | {compare_mode} | {vrp_actual}/{vrp_ref} | {vrp_match} | {vap_actual}/{vap_ref} | {vap_match} | `{log_path}` |".format(
             name=participant["name"],
             exit_code=participant["exitCode"],
             duration_ms=participant["durationMs"],
+            tal_count=participant.get("talCount") if participant.get("talCount") is not None else "-",
+            compare_mode=participant.get("compareMode") or "-",
             vrp_actual=vrps.get("actual", "-"),
             vrp_ref=vrps.get("reference", "-"),
             vrp_match=vrps.get("match", False),
@@ -258,6 +263,8 @@ for participant in participants:
         f"- `mirror_path`: `{participant['mirrorPath']}`",
         f"- `summary_path`: `{participant['summaryPath']}`",
         f"- `timing_path`: `{participant['timingPath']}`",
+        f"- `compare_mode`: `{participant.get('compareMode')}`",
+        f"- `tal_count`: `{participant.get('talCount')}`",
         f"- `log_paths`: `{', '.join(participant['logPaths'])}`",
         f"- `vrps`: `actual={vrps.get('actual', '-')}` `reference={vrps.get('reference', '-')}` `match={vrps.get('match', False)}`",
         f"- `vaps`: `actual={vaps.get('actual', '-')}` `reference={vaps.get('reference', '-')}` `match={vaps.get('match', False)}`",

scripts/cir/run_cir_replay_ours.sh

@@ -6,10 +6,15 @@ usage() {
 Usage:
   ./scripts/cir/run_cir_replay_ours.sh \
     --cir <path> \
-    [--static-root <path> | --raw-store-db <path>] \
+    --repo-bytes-db <path> \
     --out-dir <path> \
     --reference-ccr <path> \
     [--keep-db] \
+    [--write-actual-ccr] \
+    [--write-report-json] \
+    [--report-json-compact] \
+    [--phase2-object-workers <n>] \
+    [--phase2-worker-queue-capacity <n>] \
     [--rpki-bin <path>] \
     [--real-rsync-bin <path>]
 EOF
@@ -18,27 +23,34 @@ EOF
 ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 
 CIR=""
-STATIC_ROOT=""
-POOL_DB=""
-RAW_STORE_DB=""
+REPO_BYTES_DB=""
 OUT_DIR=""
 REFERENCE_CCR=""
 KEEP_DB=0
-RPKI_BIN="$ROOT_DIR/target/release/rpki"
-CIR_MATERIALIZE_BIN="$ROOT_DIR/target/release/cir_materialize"
-CIR_EXTRACT_INPUTS_BIN="$ROOT_DIR/target/release/cir_extract_inputs"
-CCR_TO_COMPARE_VIEWS_BIN="$ROOT_DIR/target/release/ccr_to_compare_views"
+WRITE_ACTUAL_CCR=0
+WRITE_REPORT_JSON=0
+REPORT_JSON_COMPACT=0
+PHASE2_OBJECT_WORKERS="${CIR_REPLAY_PHASE2_OBJECT_WORKERS:-4}"
+PHASE2_WORKER_QUEUE_CAPACITY="${CIR_REPLAY_PHASE2_WORKER_QUEUE_CAPACITY:-64}"
+RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
+CIR_MATERIALIZE_BIN="${CIR_MATERIALIZE_BIN:-$ROOT_DIR/target/release/cir_materialize}"
+CIR_EXTRACT_INPUTS_BIN="${CIR_EXTRACT_INPUTS_BIN:-$ROOT_DIR/target/release/cir_extract_inputs}"
+CCR_TO_COMPARE_VIEWS_BIN="${CCR_TO_COMPARE_VIEWS_BIN:-$ROOT_DIR/target/release/ccr_to_compare_views}"
 REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
 WRAPPER="$ROOT_DIR/scripts/cir/cir-rsync-wrapper"
 
 while [[ $# -gt 0 ]]; do
   case "$1" in
     --cir) CIR="$2"; shift 2 ;;
-    --static-root) STATIC_ROOT="$2"; shift 2 ;;
-    --raw-store-db) RAW_STORE_DB="$2"; shift 2 ;;
+    --repo-bytes-db) REPO_BYTES_DB="$2"; shift 2 ;;
     --out-dir) OUT_DIR="$2"; shift 2 ;;
     --reference-ccr) REFERENCE_CCR="$2"; shift 2 ;;
     --keep-db) KEEP_DB=1; shift ;;
+    --write-actual-ccr) WRITE_ACTUAL_CCR=1; shift ;;
+    --write-report-json) WRITE_REPORT_JSON=1; shift ;;
+    --report-json-compact) WRITE_REPORT_JSON=1; REPORT_JSON_COMPACT=1; shift ;;
+    --phase2-object-workers) PHASE2_OBJECT_WORKERS="$2"; shift 2 ;;
+    --phase2-worker-queue-capacity) PHASE2_WORKER_QUEUE_CAPACITY="$2"; shift 2 ;;
     --rpki-bin) RPKI_BIN="$2"; shift 2 ;;
     --real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
     -h|--help) usage; exit 0 ;;
@@ -46,18 +58,20 @@ while [[ $# -gt 0 ]]; do
   esac
 done
 
-[[ -n "$CIR" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" ]] || {
+[[ -n "$CIR" && -n "$REPO_BYTES_DB" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" ]] || {
   usage >&2
   exit 2
 }
-backend_count=0
-[[ -n "$STATIC_ROOT" ]] && backend_count=$((backend_count+1))
-[[ -n "$RAW_STORE_DB" ]] && backend_count=$((backend_count+1))
-[[ "$backend_count" -eq 1 ]] || { usage >&2; exit 2; }
 
 mkdir -p "$OUT_DIR"
 
+needs_build=0
 if [[ ! -x "$RPKI_BIN" || ! -x "$CIR_MATERIALIZE_BIN" || ! -x "$CIR_EXTRACT_INPUTS_BIN" || ! -x "$CCR_TO_COMPARE_VIEWS_BIN" ]]; then
+  needs_build=1
+elif [[ "$RPKI_BIN" == "$ROOT_DIR/target/release/rpki" ]] && find "$ROOT_DIR/src" "$ROOT_DIR/Cargo.toml" -newer "$RPKI_BIN" -print -quit | grep -q .; then
+  needs_build=1
+fi
+if [[ "$needs_build" -eq 1 ]]; then
   (
     cd "$ROOT_DIR"
     cargo build --release --bin rpki --bin cir_materialize --bin cir_extract_inputs --bin ccr_to_compare_views
@@ -69,6 +83,8 @@ TALS_DIR="$TMP_ROOT/tals"
 META_JSON="$TMP_ROOT/meta.json"
 MIRROR_ROOT="$TMP_ROOT/mirror"
 DB_DIR="$TMP_ROOT/work-db"
+REPLAY_RAW_STORE_DB="$TMP_ROOT/replay-raw-store.db"
+REPLAY_REPO_BYTES_DB="$TMP_ROOT/replay-repo-bytes.db"
 ACTUAL_CCR="$OUT_DIR/actual.ccr"
 ACTUAL_REPORT="$OUT_DIR/report.json"
 ACTUAL_VRPS="$OUT_DIR/actual-vrps.csv"
@@ -82,12 +98,7 @@ rm -rf "$TMP_ROOT"
 mkdir -p "$TMP_ROOT"
 
 "$CIR_EXTRACT_INPUTS_BIN" --cir "$CIR" --tals-dir "$TALS_DIR" --meta-json "$META_JSON"
-materialize_cmd=("$CIR_MATERIALIZE_BIN" --cir "$CIR" --mirror-root "$MIRROR_ROOT")
-if [[ -n "$STATIC_ROOT" ]]; then
-  materialize_cmd+=(--static-root "$STATIC_ROOT")
-else
-  materialize_cmd+=(--raw-store-db "$RAW_STORE_DB")
-fi
+materialize_cmd=("$CIR_MATERIALIZE_BIN" --cir "$CIR" --repo-bytes-db "$REPO_BYTES_DB" --mirror-root "$MIRROR_ROOT")
 if [[ "$KEEP_DB" -eq 1 ]]; then
   materialize_cmd+=(--keep-db)
 fi
@@ -98,11 +109,16 @@ import json,sys
 print(json.load(open(sys.argv[1]))["validationTime"])
 PY
 )"
-FIRST_TAL="$(python3 - <<'PY' "$META_JSON"
-import json,sys
-print(json.load(open(sys.argv[1]))["talFiles"][0]["path"])
+mapfile -t TAL_PATHS < <(python3 - <<'PY' "$META_JSON"
+import json, sys
+for item in json.load(open(sys.argv[1], encoding="utf-8"))["talFiles"]:
+    print(item["path"])
 PY
-)"
+)
+TAL_ARGS=()
+for tal_path in "${TAL_PATHS[@]}"; do
+  TAL_ARGS+=(--tal-path "$tal_path")
+done
 
 export CIR_MIRROR_ROOT="$(python3 - <<'PY' "$MIRROR_ROOT"
 from pathlib import Path
@@ -113,43 +129,117 @@ PY
 export REAL_RSYNC_BIN="$REAL_RSYNC_BIN"
 export CIR_LOCAL_LINK_MODE=1
 
+REPORT_JSON_ARGS=(--skip-report-build)
+VCIR_ARGS=(--skip-vcir-persist)
+if [[ "$WRITE_REPORT_JSON" -eq 1 ]]; then
+  REPORT_JSON_ARGS=(--report-json "$ACTUAL_REPORT")
+  if [[ "$REPORT_JSON_COMPACT" -eq 1 ]]; then
+    REPORT_JSON_ARGS+=(--report-json-compact)
+  fi
+fi
+CCR_ARGS=()
+if [[ "$WRITE_ACTUAL_CCR" -eq 1 ]]; then
+  CCR_ARGS=(--ccr-out "$ACTUAL_CCR")
+fi
+
 "$RPKI_BIN" \
   --db "$DB_DIR" \
-  --tal-path "$FIRST_TAL" \
+  --raw-store-db "$REPLAY_RAW_STORE_DB" \
+  --repo-bytes-db "$REPLAY_REPO_BYTES_DB" \
+  "${TAL_ARGS[@]}" \
+  --parallel-phase2-object-workers "$PHASE2_OBJECT_WORKERS" \
+  --parallel-phase2-worker-queue-capacity "$PHASE2_WORKER_QUEUE_CAPACITY" \
   --disable-rrdp \
   --rsync-command "$WRAPPER" \
   --validation-time "$VALIDATION_TIME" \
-  --ccr-out "$ACTUAL_CCR" \
-  --report-json "$ACTUAL_REPORT" \
+  "${CCR_ARGS[@]}" \
+  --vrps-csv-out "$ACTUAL_VRPS" \
+  --vaps-csv-out "$ACTUAL_VAPS" \
+  --compare-view-trust-anchor unknown \
+  "${VCIR_ARGS[@]}" \
+  "${REPORT_JSON_ARGS[@]}" \
   >"$RUN_LOG" 2>&1
 
-"$CCR_TO_COMPARE_VIEWS_BIN" --ccr "$ACTUAL_CCR" --vrps-out "$ACTUAL_VRPS" --vaps-out "$ACTUAL_VAPS" --trust-anchor unknown
+sort_compare_csv() {
+  local path="$1"
+  local tmp="${path}.sorted.tmp"
+  {
+    head -n 1 "$path"
+    tail -n +2 "$path" | LC_ALL=C sort -u
+  } >"$tmp"
+  mv "$tmp" "$path"
+}
+
+sort_compare_csv "$ACTUAL_VRPS"
+sort_compare_csv "$ACTUAL_VAPS"
 "$CCR_TO_COMPARE_VIEWS_BIN" --ccr "$REFERENCE_CCR" --vrps-out "$REF_VRPS" --vaps-out "$REF_VAPS" --trust-anchor unknown
 
-python3 - <<'PY' "$ACTUAL_VRPS" "$REF_VRPS" "$ACTUAL_VAPS" "$REF_VAPS" "$COMPARE_JSON"
+python3 - <<'PY' "$ACTUAL_VRPS" "$REF_VRPS" "$ACTUAL_VAPS" "$REF_VAPS" "$COMPARE_JSON" "$META_JSON" "$WRITE_REPORT_JSON" "$WRITE_ACTUAL_CCR" "$PHASE2_OBJECT_WORKERS" "$PHASE2_WORKER_QUEUE_CAPACITY"
 import csv, json, sys
-def rows(path):
-    with open(path, newline="") as f:
-        return list(csv.reader(f))[1:]
-actual_vrps = {tuple(r) for r in rows(sys.argv[1])}
-ref_vrps = {tuple(r) for r in rows(sys.argv[2])}
-actual_vaps = {tuple(r) for r in rows(sys.argv[3])}
-ref_vaps = {tuple(r) for r in rows(sys.argv[4])}
+
+def next_row(reader):
+    try:
+        return tuple(next(reader))
+    except StopIteration:
+        return None
+
+def compare_sorted_csv(actual_path, ref_path):
+    actual_count = 0
+    ref_count = 0
+    only_actual_count = 0
+    only_ref_count = 0
+    only_actual_sample = []
+    only_ref_sample = []
+    with open(actual_path, newline="") as actual_file, open(ref_path, newline="") as ref_file:
+        actual_reader = csv.reader(actual_file)
+        ref_reader = csv.reader(ref_file)
+        next(actual_reader, None)
+        next(ref_reader, None)
+        actual = next_row(actual_reader)
+        ref = next_row(ref_reader)
+        while actual is not None or ref is not None:
+            if ref is None or (actual is not None and actual < ref):
+                actual_count += 1
+                only_actual_count += 1
+                if len(only_actual_sample) < 20:
+                    only_actual_sample.append(list(actual))
+                actual = next_row(actual_reader)
+            elif actual is None or ref < actual:
+                ref_count += 1
+                only_ref_count += 1
+                if len(only_ref_sample) < 20:
+                    only_ref_sample.append(list(ref))
+                ref = next_row(ref_reader)
+            else:
+                actual_count += 1
+                ref_count += 1
+                actual = next_row(actual_reader)
+                ref = next_row(ref_reader)
+    return {
+        "actual": actual_count,
+        "reference": ref_count,
+        "only_in_actual": only_actual_sample,
+        "only_in_reference": only_ref_sample,
+        "only_in_actual_count": only_actual_count,
+        "only_in_reference_count": only_ref_count,
+        "match": only_actual_count == 0 and only_ref_count == 0,
+    }
+
+vrps = compare_sorted_csv(sys.argv[1], sys.argv[2])
+vaps = compare_sorted_csv(sys.argv[3], sys.argv[4])
+meta = json.load(open(sys.argv[6], encoding="utf-8"))
 summary = {
-  "vrps": {
-    "actual": len(actual_vrps),
-    "reference": len(ref_vrps),
-    "only_in_actual": sorted(actual_vrps - ref_vrps)[:20],
-    "only_in_reference": sorted(ref_vrps - actual_vrps)[:20],
-    "match": actual_vrps == ref_vrps,
+  "compareMode": "trust-anchor-agnostic",
+  "talCount": len(meta["talFiles"]),
+  "talPaths": [item["path"] for item in meta["talFiles"]],
+  "actualCcrWritten": sys.argv[8] == "1",
+  "reportJsonWritten": sys.argv[7] == "1",
+  "replayParallelism": {
+    "phase2ObjectWorkers": int(sys.argv[9]),
+    "phase2WorkerQueueCapacity": int(sys.argv[10]),
   },
-  "vaps": {
-    "actual": len(actual_vaps),
-    "reference": len(ref_vaps),
-    "only_in_actual": sorted(actual_vaps - ref_vaps)[:20],
-    "only_in_reference": sorted(ref_vaps - actual_vaps)[:20],
-    "match": actual_vaps == ref_vaps,
-  }
+  "vrps": vrps,
+  "vaps": vaps,
 }
 with open(sys.argv[5], "w") as f:
     json.dump(summary, f, indent=2)

scripts/cir/run_cir_replay_routinator.sh

@@ -6,7 +6,7 @@ usage() {
 Usage:
   ./scripts/cir/run_cir_replay_routinator.sh \
     --cir <path> \
-    [--static-root <path> | --raw-store-db <path>] \
+    --repo-bytes-db <path> \
     --out-dir <path> \
     --reference-ccr <path> \
     [--keep-db] \
@@ -20,18 +20,16 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 RPKI_DEV_ROOT="${RPKI_DEV_ROOT:-$ROOT_DIR}"
 
 CIR=""
-STATIC_ROOT=""
-POOL_DB=""
-RAW_STORE_DB=""
+REPO_BYTES_DB=""
 OUT_DIR=""
 REFERENCE_CCR=""
 KEEP_DB=0
 ROUTINATOR_ROOT="${ROUTINATOR_ROOT:-/home/yuyr/dev/rust_playground/routinator}"
 ROUTINATOR_BIN="${ROUTINATOR_BIN:-$ROUTINATOR_ROOT/target/debug/routinator}"
 REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
-CIR_MATERIALIZE_BIN="$ROOT_DIR/target/release/cir_materialize"
-CIR_EXTRACT_INPUTS_BIN="$ROOT_DIR/target/release/cir_extract_inputs"
-CCR_TO_COMPARE_VIEWS_BIN="$ROOT_DIR/target/release/ccr_to_compare_views"
+CIR_MATERIALIZE_BIN="${CIR_MATERIALIZE_BIN:-$ROOT_DIR/target/release/cir_materialize}"
+CIR_EXTRACT_INPUTS_BIN="${CIR_EXTRACT_INPUTS_BIN:-$ROOT_DIR/target/release/cir_extract_inputs}"
+CCR_TO_COMPARE_VIEWS_BIN="${CCR_TO_COMPARE_VIEWS_BIN:-$ROOT_DIR/target/release/ccr_to_compare_views}"
 WRAPPER="$ROOT_DIR/scripts/cir/cir-rsync-wrapper"
 JSON_TO_VAPS="$ROOT_DIR/scripts/cir/json_to_vaps_csv.py"
 FAKETIME_LIB="${FAKETIME_LIB:-$ROOT_DIR/target/tools/faketime_pkg/extracted/libfaketime/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1}"
@@ -39,8 +37,7 @@ FAKETIME_LIB="${FAKETIME_LIB:-$ROOT_DIR/target/tools/faketime_pkg/extracted/libf
 while [[ $# -gt 0 ]]; do
   case "$1" in
     --cir) CIR="$2"; shift 2 ;;
-    --static-root) STATIC_ROOT="$2"; shift 2 ;;
-    --raw-store-db) RAW_STORE_DB="$2"; shift 2 ;;
+    --repo-bytes-db) REPO_BYTES_DB="$2"; shift 2 ;;
     --out-dir) OUT_DIR="$2"; shift 2 ;;
     --reference-ccr) REFERENCE_CCR="$2"; shift 2 ;;
     --keep-db) KEEP_DB=1; shift ;;
@@ -52,14 +49,14 @@ while [[ $# -gt 0 ]]; do
   esac
 done
 
-[[ -n "$CIR" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" ]] || {
+[[ -n "$CIR" && -n "$REPO_BYTES_DB" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" ]] || {
   usage >&2
   exit 2
 }
-backend_count=0
-[[ -n "$STATIC_ROOT" ]] && backend_count=$((backend_count+1))
-[[ -n "$RAW_STORE_DB" ]] && backend_count=$((backend_count+1))
-[[ "$backend_count" -eq 1 ]] || { usage >&2; exit 2; }
+if [[ ! -x "$ROUTINATOR_BIN" ]]; then
+  echo "routinator binary not executable: $ROUTINATOR_BIN" >&2
+  exit 2
+fi
 
 mkdir -p "$OUT_DIR"
 if [[ ! -x "$CIR_MATERIALIZE_BIN" || ! -x "$CIR_EXTRACT_INPUTS_BIN" || ! -x "$CCR_TO_COMPARE_VIEWS_BIN" ]]; then
@@ -102,12 +99,7 @@ for tal in Path(sys.argv[1]).glob("*.tal"):
             seen_sep = True
     tal.write_text("\n".join(rsync_uris) + "\n\n" + "\n".join(base64_lines) + "\n", encoding="utf-8")
 PY
-materialize_cmd=("$CIR_MATERIALIZE_BIN" --cir "$CIR" --mirror-root "$MIRROR_ROOT")
-if [[ -n "$STATIC_ROOT" ]]; then
-  materialize_cmd+=(--static-root "$STATIC_ROOT")
-else
-  materialize_cmd+=(--raw-store-db "$RAW_STORE_DB")
-fi
+materialize_cmd=("$CIR_MATERIALIZE_BIN" --cir "$CIR" --repo-bytes-db "$REPO_BYTES_DB" --mirror-root "$MIRROR_ROOT")
 if [[ "$KEEP_DB" -eq 1 ]]; then
   materialize_cmd+=(--keep-db)
 fi
@@ -118,12 +110,13 @@ import json,sys
 print(json.load(open(sys.argv[1]))["validationTime"])
 PY
 )"
-FIRST_TAL="$(python3 - <<'PY' "$META_JSON"
-import json,sys
-print(json.load(open(sys.argv[1]))["talFiles"][0]["path"])
+mapfile -t TAL_PATHS < <(python3 - <<'PY' "$META_JSON"
+import json, sys
+for item in json.load(open(sys.argv[1], encoding="utf-8"))["talFiles"]:
+    print(item["path"])
 PY
-)"
-TA_NAME="$(basename "$FIRST_TAL" .tal)"
+)
+COMPARE_TRUST_ANCHOR="unknown"
 FAKE_EPOCH="$(python3 - <<'PY' "$VALIDATION_TIME"
 from datetime import datetime, timezone
 import sys
@@ -183,9 +176,30 @@ env \
   vrps --noupdate --format json -o "$ACTUAL_VAPS_JSON" >>"$RUN_LOG" 2>&1
 
 python3 "$JSON_TO_VAPS" --input "$ACTUAL_VAPS_JSON" --csv-out "$ACTUAL_VAPS"
-"$CCR_TO_COMPARE_VIEWS_BIN" --ccr "$REFERENCE_CCR" --vrps-out "$REF_VRPS" --vaps-out "$REF_VAPS" --trust-anchor "$TA_NAME"
 
-python3 - <<'PY' "$ACTUAL_VRPS" "$REF_VRPS" "$ACTUAL_VAPS" "$REF_VAPS" "$SUMMARY_JSON"
+normalize_trust_anchor_csv() {
+  python3 - <<'PY' "$1" "$2"
+import csv
+import sys
+from pathlib import Path
+
+path = Path(sys.argv[1])
+trust_anchor = sys.argv[2]
+rows = list(csv.reader(path.open(newline="", encoding="utf-8")))
+if rows:
+    for row in rows[1:]:
+        if row:
+            row[-1] = trust_anchor
+with path.open("w", newline="", encoding="utf-8") as fh:
+    csv.writer(fh).writerows(rows)
+PY
+}
+
+normalize_trust_anchor_csv "$ACTUAL_VRPS" "$COMPARE_TRUST_ANCHOR"
+normalize_trust_anchor_csv "$ACTUAL_VAPS" "$COMPARE_TRUST_ANCHOR"
+"$CCR_TO_COMPARE_VIEWS_BIN" --ccr "$REFERENCE_CCR" --vrps-out "$REF_VRPS" --vaps-out "$REF_VAPS" --trust-anchor "$COMPARE_TRUST_ANCHOR"
+
+python3 - <<'PY' "$ACTUAL_VRPS" "$REF_VRPS" "$ACTUAL_VAPS" "$REF_VAPS" "$SUMMARY_JSON" "$META_JSON"
 import csv, json, sys
 def rows(path):
     with open(path, newline="") as f:
@@ -194,7 +208,11 @@ actual_vrps = {tuple(r) for r in rows(sys.argv[1])}
 ref_vrps = {tuple(r) for r in rows(sys.argv[2])}
 actual_vaps = {tuple(r) for r in rows(sys.argv[3])}
 ref_vaps = {tuple(r) for r in rows(sys.argv[4])}
+meta = json.load(open(sys.argv[6], encoding="utf-8"))
 summary = {
+  "compareMode": "trust-anchor-agnostic",
+  "talCount": len(meta["talFiles"]),
+  "talPaths": [item["path"] for item in meta["talFiles"]],
   "vrps": {
     "actual": len(actual_vrps),
     "reference": len(ref_vrps),

scripts/cir/run_cir_replay_rpki_client.sh

@@ -6,10 +6,10 @@ usage() {
 Usage:
   ./scripts/cir/run_cir_replay_rpki_client.sh \
     --cir <path> \
-    [--static-root <path> | --raw-store-db <path>] \
+    --repo-bytes-db <path> \
     --out-dir <path> \
     --reference-ccr <path> \
-    --build-dir <path> \
+    [--build-dir <path> | --rpki-client-bin <path>] \
     [--keep-db] \
     [--real-rsync-bin <path>]
 EOF
@@ -17,27 +17,26 @@ EOF
 
 ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 CIR=""
-STATIC_ROOT=""
-POOL_DB=""
-RAW_STORE_DB=""
+REPO_BYTES_DB=""
 OUT_DIR=""
 REFERENCE_CCR=""
 BUILD_DIR=""
+RPKI_CLIENT_BIN=""
 KEEP_DB=0
 REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
-CIR_MATERIALIZE_BIN="$ROOT_DIR/target/release/cir_materialize"
-CIR_EXTRACT_INPUTS_BIN="$ROOT_DIR/target/release/cir_extract_inputs"
-CCR_TO_COMPARE_VIEWS_BIN="$ROOT_DIR/target/release/ccr_to_compare_views"
+CIR_MATERIALIZE_BIN="${CIR_MATERIALIZE_BIN:-$ROOT_DIR/target/release/cir_materialize}"
+CIR_EXTRACT_INPUTS_BIN="${CIR_EXTRACT_INPUTS_BIN:-$ROOT_DIR/target/release/cir_extract_inputs}"
+CCR_TO_COMPARE_VIEWS_BIN="${CCR_TO_COMPARE_VIEWS_BIN:-$ROOT_DIR/target/release/ccr_to_compare_views}"
 WRAPPER="$ROOT_DIR/scripts/cir/cir-rsync-wrapper"
 
 while [[ $# -gt 0 ]]; do
   case "$1" in
     --cir) CIR="$2"; shift 2 ;;
-    --static-root) STATIC_ROOT="$2"; shift 2 ;;
-    --raw-store-db) RAW_STORE_DB="$2"; shift 2 ;;
+    --repo-bytes-db) REPO_BYTES_DB="$2"; shift 2 ;;
     --out-dir) OUT_DIR="$2"; shift 2 ;;
     --reference-ccr) REFERENCE_CCR="$2"; shift 2 ;;
     --build-dir) BUILD_DIR="$2"; shift 2 ;;
+    --rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
     --keep-db) KEEP_DB=1; shift ;;
     --real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
     -h|--help) usage; exit 0 ;;
@@ -45,14 +44,21 @@ while [[ $# -gt 0 ]]; do
   esac
 done
 
-[[ -n "$CIR" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" && -n "$BUILD_DIR" ]] || {
+[[ -n "$CIR" && -n "$REPO_BYTES_DB" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" ]] || {
   usage >&2
   exit 2
 }
-backend_count=0
-[[ -n "$STATIC_ROOT" ]] && backend_count=$((backend_count+1))
-[[ -n "$RAW_STORE_DB" ]] && backend_count=$((backend_count+1))
-[[ "$backend_count" -eq 1 ]] || { usage >&2; exit 2; }
+if [[ -z "$BUILD_DIR" && -z "$RPKI_CLIENT_BIN" ]]; then
+  usage >&2
+  exit 2
+fi
+if [[ -z "$RPKI_CLIENT_BIN" ]]; then
+  RPKI_CLIENT_BIN="$BUILD_DIR/src/rpki-client"
+fi
+if [[ ! -x "$RPKI_CLIENT_BIN" ]]; then
+  echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2
+  exit 2
+fi
 
 mkdir -p "$OUT_DIR"
 if [[ ! -x "$CIR_MATERIALIZE_BIN" || ! -x "$CIR_EXTRACT_INPUTS_BIN" || ! -x "$CCR_TO_COMPARE_VIEWS_BIN" ]]; then
@@ -97,12 +103,7 @@ for tal in Path(sys.argv[1]).glob("*.tal"):
             seen_sep = True
     tal.write_text("\n".join(rsync_uris) + "\n\n" + "\n".join(base64_lines) + "\n", encoding="utf-8")
 PY
-materialize_cmd=("$CIR_MATERIALIZE_BIN" --cir "$CIR" --mirror-root "$MIRROR_ROOT")
-if [[ -n "$STATIC_ROOT" ]]; then
-  materialize_cmd+=(--static-root "$STATIC_ROOT")
-else
-  materialize_cmd+=(--raw-store-db "$RAW_STORE_DB")
-fi
+materialize_cmd=("$CIR_MATERIALIZE_BIN" --cir "$CIR" --repo-bytes-db "$REPO_BYTES_DB" --mirror-root "$MIRROR_ROOT")
 if [[ "$KEEP_DB" -eq 1 ]]; then
   materialize_cmd+=(--keep-db)
 fi
@@ -116,12 +117,17 @@ dt = datetime.fromisoformat(vt.replace("Z", "+00:00")).astimezone(timezone.utc)
 print(int(dt.timestamp()))
 PY
 )"
-FIRST_TAL="$(python3 - <<'PY' "$META_JSON"
-import json,sys
-print(json.load(open(sys.argv[1]))["talFiles"][0]["path"])
+mapfile -t TAL_PATHS < <(python3 - <<'PY' "$META_JSON"
+import json, sys
+for item in json.load(open(sys.argv[1], encoding="utf-8"))["talFiles"]:
+    print(item["path"])
 PY
-)"
-TA_NAME="$(basename "$FIRST_TAL" .tal)"
+)
+CLIENT_TAL_ARGS=()
+for tal_path in "${TAL_PATHS[@]}"; do
+  CLIENT_TAL_ARGS+=(-t "$tal_path")
+done
+COMPARE_TRUST_ANCHOR="unknown"
 
 export CIR_MIRROR_ROOT="$(python3 - <<'PY' "$MIRROR_ROOT"
 from pathlib import Path
@@ -133,19 +139,62 @@ export REAL_RSYNC_BIN="$REAL_RSYNC_BIN"
 export CIR_LOCAL_LINK_MODE=1
 
 mkdir -p "$CACHE_DIR" "$OUT_CCR_DIR"
-"$BUILD_DIR/src/rpki-client" \
+chmod -R 0777 "$TMP_ROOT"
+"$RPKI_CLIENT_BIN" \
   -R \
   -e "$WRAPPER" \
   -P "$VALIDATION_EPOCH" \
-  -t "$FIRST_TAL" \
+  "${CLIENT_TAL_ARGS[@]}" \
   -d "$CACHE_DIR" \
   "$OUT_CCR_DIR" >"$RUN_LOG" 2>&1
 
-"$CCR_TO_COMPARE_VIEWS_BIN" \
-  --ccr "$OUT_CCR_DIR/rpki.ccr" \
-  --vrps-out "$ACTUAL_VRPS" \
-  --vaps-out "$ACTUAL_VAPS" \
-  --trust-anchor "$TA_NAME"
+if [[ -f "$OUT_CCR_DIR/rpki.ccr" ]]; then
+  "$CCR_TO_COMPARE_VIEWS_BIN" \
+    --ccr "$OUT_CCR_DIR/rpki.ccr" \
+    --vrps-out "$ACTUAL_VRPS" \
+    --vaps-out "$ACTUAL_VAPS" \
+    --trust-anchor "$COMPARE_TRUST_ANCHOR"
+else
+  python3 - <<'PY' "$OUT_CCR_DIR/json" "$ACTUAL_VRPS" "$ACTUAL_VAPS" "$COMPARE_TRUST_ANCHOR"
+import csv
+import json
+import sys
+from pathlib import Path
+
+json_path = Path(sys.argv[1])
+vrps_out = Path(sys.argv[2])
+vaps_out = Path(sys.argv[3])
+compare_ta = sys.argv[4]
+
+if not json_path.is_file():
+    raise SystemExit(f"rpki-client output has neither rpki.ccr nor json: {json_path}")
+
+data = json.loads(json_path.read_text(encoding="utf-8"))
+
+vrps_out.parent.mkdir(parents=True, exist_ok=True)
+with vrps_out.open("w", newline="", encoding="utf-8") as fh:
+    writer = csv.writer(fh)
+    writer.writerow(["ASN", "IP Prefix", "Max Length", "Trust Anchor"])
+    for roa in data.get("roas", []):
+        writer.writerow([
+            f"AS{roa['asn']}",
+            roa["prefix"],
+            str(roa["maxLength"]),
+            compare_ta,
+        ])
+
+with vaps_out.open("w", newline="", encoding="utf-8") as fh:
+    writer = csv.writer(fh)
+    writer.writerow(["Customer ASN", "Providers", "Trust Anchor"])
+    for aspa in data.get("aspas", []):
+        providers = ";".join(f"AS{item}" for item in sorted(aspa.get("providers", [])))
+        writer.writerow([
+            f"AS{aspa['customer_asid']}",
+            providers,
+            compare_ta,
+        ])
+PY
+fi
 
 python3 - <<'PY' "$ACTUAL_VRPS" "$ACTUAL_VAPS" "$ACTUAL_VRPS_META" "$ACTUAL_VAPS_META"
 import csv, json, sys
@@ -157,9 +206,9 @@ json.dump({"count": count_rows(sys.argv[1])}, open(sys.argv[3], "w"), indent=2)
 json.dump({"count": count_rows(sys.argv[2])}, open(sys.argv[4], "w"), indent=2)
 PY
 
-"$CCR_TO_COMPARE_VIEWS_BIN" --ccr "$REFERENCE_CCR" --vrps-out "$REF_VRPS" --vaps-out "$REF_VAPS" --trust-anchor "$TA_NAME"
+"$CCR_TO_COMPARE_VIEWS_BIN" --ccr "$REFERENCE_CCR" --vrps-out "$REF_VRPS" --vaps-out "$REF_VAPS" --trust-anchor "$COMPARE_TRUST_ANCHOR"
 
-python3 - <<'PY' "$ACTUAL_VRPS" "$REF_VRPS" "$ACTUAL_VAPS" "$REF_VAPS" "$SUMMARY_JSON"
+python3 - <<'PY' "$ACTUAL_VRPS" "$REF_VRPS" "$ACTUAL_VAPS" "$REF_VAPS" "$SUMMARY_JSON" "$META_JSON"
 import csv, json, sys
 def rows(path):
     with open(path, newline="") as f:
@@ -168,7 +217,11 @@ actual_vrps = {tuple(r) for r in rows(sys.argv[1])}
 ref_vrps = {tuple(r) for r in rows(sys.argv[2])}
 actual_vaps = {tuple(r) for r in rows(sys.argv[3])}
 ref_vaps = {tuple(r) for r in rows(sys.argv[4])}
+meta = json.load(open(sys.argv[6], encoding="utf-8"))
 summary = {
+  "compareMode": "trust-anchor-agnostic",
+  "talCount": len(meta["talFiles"]),
+  "talPaths": [item["path"] for item in meta["talFiles"]],
   "vrps": {
     "actual": len(actual_vrps),
     "reference": len(ref_vrps),

scripts/cir/run_cir_replay_sequence_ours.sh

@@ -55,11 +55,7 @@ rpki_bin = sys.argv[6]
 real_rsync_bin = sys.argv[7]
 
 sequence = json.loads((sequence_root / "sequence.json").read_text(encoding="utf-8"))
-static_root = sequence_root / sequence["staticRoot"] if "staticRoot" in sequence else None
-raw_store_db = sequence_root / sequence["rawStoreDbPath"] if "rawStoreDbPath" in sequence else None
-backend_count = sum(x is not None for x in (static_root, raw_store_db))
-if backend_count != 1:
-    raise SystemExit("sequence must set exactly one of staticRoot or rawStoreDbPath")
+repo_bytes_db = sequence_root / sequence["repoBytesDbPath"]
 steps = sequence["steps"]
 
 results = []
@@ -81,10 +77,7 @@ for step in steps:
         "--real-rsync-bin",
         real_rsync_bin,
     ]
-    if static_root is not None:
-        cmd.extend(["--static-root", str(static_root)])
-    else:
-        cmd.extend(["--raw-store-db", str(raw_store_db)])
+    cmd.extend(["--repo-bytes-db", str(repo_bytes_db)])
     proc = subprocess.run(cmd, capture_output=True, text=True)
     if proc.returncode != 0:
         raise SystemExit(
@@ -99,6 +92,9 @@ for step in steps:
         "outDir": str(out_dir),
         "comparePath": str(out_dir / "compare-summary.json"),
         "timingPath": str(out_dir / "timing.json"),
+        "compareMode": compare.get("compareMode"),
+        "talCount": compare.get("talCount"),
+        "talPaths": compare.get("talPaths", []),
         "compare": compare,
         "timing": timing,
         "match": bool(compare["vrps"]["match"]) and bool(compare["vaps"]["match"]),
@@ -124,16 +120,18 @@ lines = [
     f"- `step_count`: `{len(results)}`",
     f"- `all_match`: `{all_match}`",
     "",
-    "| Step | Kind | VRP actual/ref | VRP match | VAP actual/ref | VAP match | Duration (ms) |",
-    "| --- | --- | --- | --- | --- | --- | ---: |",
+    "| Step | Kind | TALs | Compare mode | VRP actual/ref | VRP match | VAP actual/ref | VAP match | Duration (ms) |",
+    "| --- | --- | ---: | --- | --- | --- | --- | --- | ---: |",
 ]
 for item in results:
     compare = item["compare"]
     timing = item.get("timing") or {}
     lines.append(
-        "| {step} | {kind} | {va}/{vr} | {vm} | {aa}/{ar} | {am} | {dur} |".format(
+        "| {step} | {kind} | {tal_count} | {compare_mode} | {va}/{vr} | {vm} | {aa}/{ar} | {am} | {dur} |".format(
             step=item["stepId"],
             kind=item["kind"],
+            tal_count=item.get("talCount") if item.get("talCount") is not None else "-",
+            compare_mode=item.get("compareMode") or "-",
             va=compare["vrps"]["actual"],
             vr=compare["vrps"]["reference"],
             vm=compare["vrps"]["match"],

scripts/cir/run_cir_replay_sequence_routinator.sh

@@ -57,11 +57,7 @@ routinator_bin = sys.argv[6]
 real_rsync_bin = sys.argv[7]
 
 sequence = json.loads((sequence_root / "sequence.json").read_text(encoding="utf-8"))
-static_root = sequence_root / sequence["staticRoot"] if "staticRoot" in sequence else None
-raw_store_db = sequence_root / sequence["rawStoreDbPath"] if "rawStoreDbPath" in sequence else None
-backend_count = sum(x is not None for x in (static_root, raw_store_db))
-if backend_count != 1:
-    raise SystemExit("sequence must set exactly one of staticRoot or rawStoreDbPath")
+repo_bytes_db = sequence_root / sequence["repoBytesDbPath"]
 steps = sequence["steps"]
 results = []
 all_match = True
@@ -85,10 +81,7 @@ for step in steps:
         "--real-rsync-bin",
         real_rsync_bin,
     ]
-    if static_root is not None:
-        cmd.extend(["--static-root", str(static_root)])
-    else:
-        cmd.extend(["--raw-store-db", str(raw_store_db)])
+    cmd.extend(["--repo-bytes-db", str(repo_bytes_db)])
     proc = subprocess.run(cmd, capture_output=True, text=True)
     if proc.returncode != 0:
         raise SystemExit(
@@ -104,6 +97,9 @@ for step in steps:
             "validationTime": step["validationTime"],
             "outDir": str(out_dir),
             "comparePath": str(out_dir / "compare-summary.json"),
+            "compareMode": compare.get("compareMode"),
+            "talCount": compare.get("talCount"),
+            "talPaths": compare.get("talPaths", []),
             "match": match,
             "compare": compare,
         }
@@ -125,7 +121,25 @@ lines = [
     f"- `step_count`: `{len(results)}`",
     f"- `all_match`: `{all_match}`",
     "",
+    "| Step | Kind | TALs | Compare mode | VRP actual/ref | VRP match | VAP actual/ref | VAP match |",
+    "| --- | --- | ---: | --- | --- | --- | --- | --- |",
 ]
+for item in results:
+    compare = item["compare"]
+    lines.append(
+        "| {step} | {kind} | {tal_count} | {compare_mode} | {va}/{vr} | {vm} | {aa}/{ar} | {am} |".format(
+            step=item["stepId"],
+            kind=item["kind"],
+            tal_count=item.get("talCount") if item.get("talCount") is not None else "-",
+            compare_mode=item.get("compareMode") or "-",
+            va=compare["vrps"]["actual"],
+            vr=compare["vrps"]["reference"],
+            vm=compare["vrps"]["match"],
+            aa=compare["vaps"]["actual"],
+            ar=compare["vaps"]["reference"],
+            am=compare["vaps"]["match"],
+        )
+    )
 summary_md.write_text("\n".join(lines), encoding="utf-8")
 PY
 

scripts/cir/run_cir_replay_sequence_rpki_client.sh

@@ -6,7 +6,7 @@ usage() {
 Usage:
   ./scripts/cir/run_cir_replay_sequence_rpki_client.sh \
     --sequence-root <path> \
-    --build-dir <path> \
+    [--build-dir <path> | --rpki-client-bin <path>] \
     [--real-rsync-bin <path>]
 EOF
 }
@@ -14,6 +14,7 @@ EOF
 ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 SEQUENCE_ROOT=""
 BUILD_DIR=""
+RPKI_CLIENT_BIN=""
 REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
 STEP_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_rpki_client.sh"
 
@@ -21,13 +22,14 @@ while [[ $# -gt 0 ]]; do
   case "$1" in
     --sequence-root) SEQUENCE_ROOT="$2"; shift 2 ;;
     --build-dir) BUILD_DIR="$2"; shift 2 ;;
+    --rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
     --real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
     -h|--help) usage; exit 0 ;;
     *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
   esac
 done
 
-[[ -n "$SEQUENCE_ROOT" && -n "$BUILD_DIR" ]] || { usage >&2; exit 2; }
+[[ -n "$SEQUENCE_ROOT" && ( -n "$BUILD_DIR" || -n "$RPKI_CLIENT_BIN" ) ]] || { usage >&2; exit 2; }
 
 SEQUENCE_ROOT="$(python3 - <<'PY' "$SEQUENCE_ROOT"
 from pathlib import Path
@@ -39,7 +41,7 @@ PY
 SUMMARY_JSON="$SEQUENCE_ROOT/sequence-summary-rpki-client.json"
 SUMMARY_MD="$SEQUENCE_ROOT/sequence-summary-rpki-client.md"
 
-python3 - <<'PY' "$SEQUENCE_ROOT" "$SUMMARY_JSON" "$SUMMARY_MD" "$STEP_SCRIPT" "$BUILD_DIR" "$REAL_RSYNC_BIN"
+python3 - <<'PY' "$SEQUENCE_ROOT" "$SUMMARY_JSON" "$SUMMARY_MD" "$STEP_SCRIPT" "$BUILD_DIR" "$RPKI_CLIENT_BIN" "$REAL_RSYNC_BIN"
 import json
 import subprocess
 import sys
@@ -50,14 +52,11 @@ summary_json = Path(sys.argv[2])
 summary_md = Path(sys.argv[3])
 step_script = Path(sys.argv[4])
 build_dir = sys.argv[5]
-real_rsync_bin = sys.argv[6]
+rpki_client_bin = sys.argv[6]
+real_rsync_bin = sys.argv[7]
 
 sequence = json.loads((sequence_root / "sequence.json").read_text(encoding="utf-8"))
-static_root = sequence_root / sequence["staticRoot"] if "staticRoot" in sequence else None
-raw_store_db = sequence_root / sequence["rawStoreDbPath"] if "rawStoreDbPath" in sequence else None
-backend_count = sum(x is not None for x in (static_root, raw_store_db))
-if backend_count != 1:
-    raise SystemExit("sequence must set exactly one of staticRoot or rawStoreDbPath")
+repo_bytes_db = sequence_root / sequence["repoBytesDbPath"]
 steps = sequence["steps"]
 results = []
 all_match = True
@@ -74,15 +73,14 @@ for step in steps:
         str(out_dir),
         "--reference-ccr",
         str(sequence_root / step["ccrPath"]),
-        "--build-dir",
-        build_dir,
         "--real-rsync-bin",
         real_rsync_bin,
     ]
-    if static_root is not None:
-        cmd.extend(["--static-root", str(static_root)])
+    if rpki_client_bin:
+        cmd.extend(["--rpki-client-bin", rpki_client_bin])
     else:
-        cmd.extend(["--raw-store-db", str(raw_store_db)])
+        cmd.extend(["--build-dir", build_dir])
+    cmd.extend(["--repo-bytes-db", str(repo_bytes_db)])
     proc = subprocess.run(cmd, capture_output=True, text=True)
     if proc.returncode != 0:
         raise SystemExit(
@@ -98,6 +96,9 @@ for step in steps:
             "validationTime": step["validationTime"],
             "outDir": str(out_dir),
             "comparePath": str(out_dir / "compare-summary.json"),
+            "compareMode": compare.get("compareMode"),
+            "talCount": compare.get("talCount"),
+            "talPaths": compare.get("talPaths", []),
             "match": match,
             "compare": compare,
         }
@@ -119,7 +120,25 @@ lines = [
     f"- `step_count`: `{len(results)}`",
     f"- `all_match`: `{all_match}`",
     "",
+    "| Step | Kind | TALs | Compare mode | VRP actual/ref | VRP match | VAP actual/ref | VAP match |",
+    "| --- | --- | ---: | --- | --- | --- | --- | --- |",
 ]
+for item in results:
+    compare = item["compare"]
+    lines.append(
+        "| {step} | {kind} | {tal_count} | {compare_mode} | {va}/{vr} | {vm} | {aa}/{ar} | {am} |".format(
+            step=item["stepId"],
+            kind=item["kind"],
+            tal_count=item.get("talCount") if item.get("talCount") is not None else "-",
+            compare_mode=item.get("compareMode") or "-",
+            va=compare["vrps"]["actual"],
+            vr=compare["vrps"]["reference"],
+            vm=compare["vrps"]["match"],
+            aa=compare["vaps"]["actual"],
+            ar=compare["vaps"]["reference"],
+            am=compare["vaps"]["match"],
+        )
+    )
 summary_md.write_text("\n".join(lines), encoding="utf-8")
 PY
 

scripts/compare/run_perf_compare_quick_remote.sh

@@ -0,0 +1,502 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/compare/run_perf_compare_quick_remote.sh \
+    --run-root <path> \
+    --remote-root <path> \
+    [--rir-set <mixed2|all5>] \
+    [--ssh-target <user@host>] \
+    [--rpki-client-bin <path>] \
+    [--libtls-path <path>] \
+    [--rp-run-mode <serial|parallel>] \
+    [--copy-rpki-client-cache] \
+    [--probe-rpki-client-cache] \
+    [--ours-extra-args '<args>'] \
+    [--dry-run]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+first_existing_executable() {
+  local fallback="$1"
+  shift
+  local candidate
+  for candidate in "$@"; do
+    if [[ -x "$candidate" ]]; then
+      printf '%s' "$candidate"
+      return
+    fi
+  done
+  printf '%s' "$fallback"
+}
+
+first_existing_file() {
+  local fallback="$1"
+  shift
+  local candidate
+  for candidate in "$@"; do
+    if [[ -f "$candidate" ]]; then
+      printf '%s' "$candidate"
+      return
+    fi
+  done
+  printf '%s' "$fallback"
+}
+
+RUN_ROOT=""
+REMOTE_ROOT=""
+SSH_TARGET="${SSH_TARGET:-root@47.251.56.108}"
+RPKI_CLIENT_BIN="${RPKI_CLIENT_BIN:-$(first_existing_executable \
+  "/home/yuyr/dev/rpki-client-9.7/build-m5/src/rpki-client" \
+  "$ROOT_DIR/../../.cache/rpki-client-9.7-build/bin/rpki-client" \
+  "$ROOT_DIR/../../.cache/rpki-client-9.7-build/src/rpki-client-9.7/src/rpki-client" \
+  "$ROOT_DIR/../../.cache/rpki-client-remote9.0/rpki-client" \
+  "/home/yuyr/dev/rpki-client-9.7/build-m5/src/rpki-client")}"
+LIBTLS_PATH="${LIBTLS_PATH:-$(first_existing_file \
+  "/home/yuyr/dev/rpki-client-9.7/.deps/libtls/root/usr/lib/x86_64-linux-gnu/libtls.so.28.0.0" \
+  "$ROOT_DIR/../../.cache/rpki-client-9.7-build/runlib/libtls.so.28" \
+  "$ROOT_DIR/../../.cache/rpki-client-9.7-build/sysroot/usr/lib/x86_64-linux-gnu/libtls.so.28.0.0" \
+  "$ROOT_DIR/../../.cache/rpki-client-remote9.0/libtls.so.28" \
+  "/home/yuyr/dev/rpki-client-9.7/.deps/libtls/root/usr/lib/x86_64-linux-gnu/libtls.so.28.0.0")}"
+RP_RUN_MODE="${RP_RUN_MODE:-serial}"
+RIR_SET="${RIR_SET:-mixed2}"
+OURS_EXTRA_ARGS="${OURS_EXTRA_ARGS:-}"
+COPY_RPKI_CLIENT_CACHE="${COPY_RPKI_CLIENT_CACHE:-0}"
+PROBE_RPKI_CLIENT_CACHE="${PROBE_RPKI_CLIENT_CACHE:-0}"
+DRY_RUN=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --run-root) RUN_ROOT="$2"; shift 2 ;;
+    --remote-root) REMOTE_ROOT="$2"; shift 2 ;;
+    --rir-set) RIR_SET="$2"; shift 2 ;;
+    --ssh-target) SSH_TARGET="$2"; shift 2 ;;
+    --rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
+    --libtls-path) LIBTLS_PATH="$2"; shift 2 ;;
+    --rp-run-mode) RP_RUN_MODE="$2"; shift 2 ;;
+    --copy-rpki-client-cache) COPY_RPKI_CLIENT_CACHE=1; shift ;;
+    --probe-rpki-client-cache) PROBE_RPKI_CLIENT_CACHE=1; shift ;;
+    --ours-extra-args) OURS_EXTRA_ARGS="$2"; shift 2 ;;
+    --dry-run) DRY_RUN=1; shift ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$RUN_ROOT" && -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
+[[ "$RP_RUN_MODE" == "serial" || "$RP_RUN_MODE" == "parallel" ]] || { echo "invalid --rp-run-mode: $RP_RUN_MODE" >&2; usage; exit 2; }
+[[ "$RIR_SET" == "mixed2" || "$RIR_SET" == "all5" ]] || { echo "invalid --rir-set: $RIR_SET" >&2; usage; exit 2; }
+[[ "$DRY_RUN" -eq 1 || -x "$RPKI_CLIENT_BIN" ]] || { echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2; exit 2; }
+[[ "$DRY_RUN" -eq 1 || -f "$LIBTLS_PATH" ]] || { echo "libtls not found: $LIBTLS_PATH" >&2; exit 2; }
+
+RUN_ROOT="$(python3 - <<'PY' "$RUN_ROOT"
+from pathlib import Path
+import sys
+print(Path(sys.argv[1]).resolve())
+PY
+)"
+
+mkdir -p "$RUN_ROOT/steps/step-001/ours" "$RUN_ROOT/steps/step-001/rpki-client" "$RUN_ROOT/steps/step-001/compare"
+mkdir -p "$RUN_ROOT/steps/step-002/ours" "$RUN_ROOT/steps/step-002/rpki-client" "$RUN_ROOT/steps/step-002/compare"
+
+tal_path_for_rir() {
+  case "$1" in
+    afrinic) printf '%s' "$ROOT_DIR/tests/fixtures/tal/afrinic.tal" ;;
+    apnic) printf '%s' "$ROOT_DIR/tests/fixtures/tal/apnic-rfc7730-https.tal" ;;
+    arin) printf '%s' "$ROOT_DIR/tests/fixtures/tal/arin.tal" ;;
+    lacnic) printf '%s' "$ROOT_DIR/tests/fixtures/tal/lacnic.tal" ;;
+    ripe) printf '%s' "$ROOT_DIR/tests/fixtures/tal/ripe-ncc.tal" ;;
+    *) echo "unknown rir: $1" >&2; exit 2 ;;
+  esac
+}
+
+ta_path_for_rir() {
+  case "$1" in
+    afrinic) printf '%s' "$ROOT_DIR/tests/fixtures/ta/afrinic-ta.cer" ;;
+    apnic) printf '%s' "$ROOT_DIR/tests/fixtures/ta/apnic-ta.cer" ;;
+    arin) printf '%s' "$ROOT_DIR/tests/fixtures/ta/arin-ta.cer" ;;
+    lacnic) printf '%s' "$ROOT_DIR/tests/fixtures/ta/lacnic-ta.cer" ;;
+    ripe) printf '%s' "$ROOT_DIR/tests/fixtures/ta/ripe-ncc-ta.cer" ;;
+    *) echo "unknown rir: $1" >&2; exit 2 ;;
+  esac
+}
+
+case "$RIR_SET" in
+  mixed2)
+    RIRS=(apnic arin)
+    SCOPE_LABEL="APNIC+ARIN mixed release two-step synchronized compare"
+    ;;
+  all5)
+    RIRS=(afrinic apnic arin lacnic ripe)
+    SCOPE_LABEL="all-five-RIR mixed release two-step synchronized compare"
+    ;;
+esac
+
+COPY_FILES=()
+for rir in "${RIRS[@]}"; do
+  COPY_FILES+=("$(tal_path_for_rir "$rir")" "$(ta_path_for_rir "$rir")")
+done
+
+if [[ "$DRY_RUN" -eq 1 ]]; then
+  cat <<EOF2
+workflow_name=性能对比测试快速版
+scope=$SCOPE_LABEL
+rir_set=$RIR_SET
+rirs=${RIRS[*]}
+run_root=$RUN_ROOT
+remote_root=$REMOTE_ROOT
+ssh_target=$SSH_TARGET
+rp_run_mode=$RP_RUN_MODE
+ours_extra_args=$OURS_EXTRA_ARGS
+EOF2
+  exit 0
+fi
+
+cleanup_remote() {
+  if [[ "${KEEP_REMOTE:-0}" != "1" ]]; then
+    ssh "$SSH_TARGET" "rm -rf '$REMOTE_ROOT'" >/dev/null 2>&1 || true
+  fi
+}
+trap cleanup_remote EXIT
+
+(
+  cd "$ROOT_DIR"
+  cargo build --release --bin rpki --bin ccr_to_compare_views --bin ccr_state_compare --bin cir_state_compare --bin cir_probe_rpki_client_cache
+)
+
+ssh "$SSH_TARGET" "set -e; systemctl disable --now rpki-client.timer >/dev/null 2>&1 || true; systemctl stop rpki-client.service >/dev/null 2>&1 || true; pkill -f '[/]rpki-client([[:space:]]|$)' >/dev/null 2>&1 || true; pkill -f '[/]routinator([[:space:]]|$)' >/dev/null 2>&1 || true; id -u _rpki-client >/dev/null 2>&1 || useradd -r -M -s /usr/sbin/nologin _rpki-client || true; rm -rf '$REMOTE_ROOT'; mkdir -p '$REMOTE_ROOT/bin' '$REMOTE_ROOT/lib' '$REMOTE_ROOT/state/ours' '$REMOTE_ROOT/state/rpki-client' '$REMOTE_ROOT/steps/step-001/ours' '$REMOTE_ROOT/steps/step-001/rpki-client' '$REMOTE_ROOT/steps/step-002/ours' '$REMOTE_ROOT/steps/step-002/rpki-client'"
+scp "$ROOT_DIR/target/release/rpki" "${COPY_FILES[@]}" "$SSH_TARGET:$REMOTE_ROOT/"
+if [[ "$PROBE_RPKI_CLIENT_CACHE" == "1" ]]; then
+  scp "$ROOT_DIR/target/release/cir_probe_rpki_client_cache" "$SSH_TARGET:$REMOTE_ROOT/bin/"
+fi
+scp "$RPKI_CLIENT_BIN" "$SSH_TARGET:$REMOTE_ROOT/bin/rpki-client"
+scp "$LIBTLS_PATH" "$SSH_TARGET:$REMOTE_ROOT/lib/libtls.so.28"
+printf '%s' "$OURS_EXTRA_ARGS" | ssh "$SSH_TARGET" "cat > '$REMOTE_ROOT/ours-extra-args.txt'"
+printf '%s' "$RP_RUN_MODE" | ssh "$SSH_TARGET" "cat > '$REMOTE_ROOT/rp-run-mode.txt'"
+printf '%s' "$RIR_SET" | ssh "$SSH_TARGET" "cat > '$REMOTE_ROOT/rir-set.txt'"
+
+run_step() {
+  local step_id="$1"
+  local kind="$2"
+  local local_step="$RUN_ROOT/steps/$step_id"
+
+  ssh "$SSH_TARGET" bash -s -- "$REMOTE_ROOT" "$step_id" "$kind" <<'EOS'
+set -euo pipefail
+REMOTE_ROOT="$1"
+STEP_ID="$2"
+KIND="$3"
+
+cd "$REMOTE_ROOT"
+mkdir -p "steps/$STEP_ID/ours" "steps/$STEP_ID/rpki-client"
+touch rpki-client-skiplist
+chmod 0644 rpki-client-skiplist
+OURS_EXTRA_ARGS="$(cat ours-extra-args.txt)"
+RP_RUN_MODE="$(cat rp-run-mode.txt)"
+RIR_SET="$(cat rir-set.txt)"
+OURS_EXTRA_ARGV=()
+if [[ -n "$OURS_EXTRA_ARGS" ]]; then
+  # shellcheck disable=SC2206
+  OURS_EXTRA_ARGV=($OURS_EXTRA_ARGS)
+fi
+
+case "$RIR_SET" in
+  mixed2) RIRS=(apnic arin) ;;
+  all5) RIRS=(afrinic apnic arin lacnic ripe) ;;
+  *) echo "invalid rir set: $RIR_SET" >&2; exit 2 ;;
+esac
+
+tal_file_for_rir() {
+  case "$1" in
+    afrinic) printf '%s' "afrinic.tal" ;;
+    apnic) printf '%s' "apnic-rfc7730-https.tal" ;;
+    arin) printf '%s' "arin.tal" ;;
+    lacnic) printf '%s' "lacnic.tal" ;;
+    ripe) printf '%s' "ripe-ncc.tal" ;;
+    *) echo "unknown rir: $1" >&2; exit 2 ;;
+  esac
+}
+
+tal_uri_for_rir() {
+  case "$1" in
+    afrinic) printf '%s' "https://rpki.afrinic.net/repository/AfriNIC.cer" ;;
+    apnic) printf '%s' "https://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer" ;;
+    arin) printf '%s' "https://rrdp.arin.net/arin-rpki-ta.cer" ;;
+    lacnic) printf '%s' "https://rrdp.lacnic.net/ta/rta-lacnic-rpki.cer" ;;
+    ripe) printf '%s' "https://rpki.ripe.net/ta/ripe-ncc-ta.cer" ;;
+    *) echo "unknown rir: $1" >&2; exit 2 ;;
+  esac
+}
+
+ta_file_for_rir() {
+  case "$1" in
+    afrinic) printf '%s' "afrinic-ta.cer" ;;
+    apnic) printf '%s' "apnic-ta.cer" ;;
+    arin) printf '%s' "arin-ta.cer" ;;
+    lacnic) printf '%s' "lacnic-ta.cer" ;;
+    ripe) printf '%s' "ripe-ncc-ta.cer" ;;
+    *) echo "unknown rir: $1" >&2; exit 2 ;;
+  esac
+}
+
+refresh_ta_file_for_rir() {
+  local rir="$1"
+  local uri
+  local file
+  uri="$(tal_uri_for_rir "$rir")"
+  file="$(ta_file_for_rir "$rir")"
+  python3 - <<'PY' "$uri" "$file"
+import sys
+import urllib.request
+uri, path = sys.argv[1:]
+request = urllib.request.Request(uri, headers={"User-Agent": "rpki-dev/compare-fast-path"})
+with urllib.request.urlopen(request, timeout=30) as response:
+    data = response.read()
+if not data:
+    raise SystemExit(f"empty TA certificate response: {uri}")
+with open(path, "wb") as output:
+    output.write(data)
+PY
+}
+
+for rir in "${RIRS[@]}"; do
+  refresh_ta_file_for_rir "$rir"
+done
+
+OURS_TAL_ARGS=()
+CLIENT_TAL_ARGS=()
+OURS_CIR_TAL_ARGS=()
+for rir in "${RIRS[@]}"; do
+  tal_file="$(tal_file_for_rir "$rir")"
+  ta_file="$(ta_file_for_rir "$rir")"
+  tal_uri="$(tal_uri_for_rir "$rir")"
+  OURS_TAL_ARGS+=(--tal-path "$tal_file" --ta-path "$ta_file")
+  OURS_CIR_TAL_ARGS+=(--cir-tal-uri "$tal_uri")
+  CLIENT_TAL_ARGS+=(-t "../../$tal_file")
+done
+
+if [[ "$KIND" == "snapshot" ]]; then
+  rm -rf state/ours/work-db state/ours/raw-store.db state/ours/repo-bytes.db state/rpki-client/cache state/rpki-client/out state/rpki-client/ta state/rpki-client/.ta
+fi
+mkdir -p state/ours/work-db state/ours/raw-store.db state/ours/repo-bytes.db state/rpki-client/cache state/rpki-client/out state/rpki-client/ta state/rpki-client/.ta
+chmod 0777 state/ours/work-db state/ours/raw-store.db state/ours/repo-bytes.db
+chmod -R 0777 state/rpki-client
+touch state/rpki-client/rpki-client-skiplist
+chmod 0644 state/rpki-client/rpki-client-skiplist
+
+START_EPOCH="$(python3 - <<'PY'
+import time
+print(time.time() + 3.0)
+PY
+)"
+
+run_ours() {
+  python3 - <<'PY' "$START_EPOCH"
+import sys, time
+x = float(sys.argv[1])
+d = x - time.time()
+if d > 0:
+    time.sleep(d)
+PY
+  started_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+  set +e
+  env RPKI_PROGRESS_LOG=1 RPKI_PROGRESS_SLOW_SECS=0 ./rpki \
+    --db state/ours/work-db \
+    --raw-store-db state/ours/raw-store.db \
+    --repo-bytes-db state/ours/repo-bytes.db \
+    "${OURS_TAL_ARGS[@]}" \
+    "${OURS_EXTRA_ARGV[@]}" \
+    --ccr-out "steps/$STEP_ID/ours/result.ccr" \
+    --cir-enable \
+    --cir-out "steps/$STEP_ID/ours/result.cir" \
+    "${OURS_CIR_TAL_ARGS[@]}" \
+    --report-json "steps/$STEP_ID/ours/report.json" \
+    > "steps/$STEP_ID/ours/run.log" 2>&1
+  exit_code=$?
+  set -e
+  finished_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+  python3 - <<'PY' "steps/$STEP_ID/ours/round-result.json" "$STEP_ID" "$KIND" "$started_ms" "$finished_ms" "$exit_code"
+import json, sys
+path, step_id, kind, started_ms, finished_ms, exit_code = sys.argv[1:]
+json.dump(
+    {
+        "stepId": step_id,
+        "kind": kind,
+        "durationMs": int(finished_ms) - int(started_ms),
+        "exitCode": int(exit_code),
+    },
+    open(path, "w"),
+    indent=2,
+)
+PY
+}
+
+run_client() {
+  cd state/rpki-client
+  python3 - <<'PY' "$START_EPOCH"
+import sys, time
+x = float(sys.argv[1])
+d = x - time.time()
+if d > 0:
+    time.sleep(d)
+PY
+  started_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+  set +e
+  LD_LIBRARY_PATH="$REMOTE_ROOT/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" "$REMOTE_ROOT/bin/rpki-client" \
+    -vv \
+    -S rpki-client-skiplist \
+    "${CLIENT_TAL_ARGS[@]}" \
+    -d cache out \
+    > "$REMOTE_ROOT/steps/$STEP_ID/rpki-client/run.log" 2>&1
+  exit_code=$?
+  set -e
+  cp out/rpki.ccr "$REMOTE_ROOT/steps/$STEP_ID/rpki-client/result.ccr" 2>/dev/null || true
+  cp out/rpki.cir "$REMOTE_ROOT/steps/$STEP_ID/rpki-client/result.cir" 2>/dev/null || true
+  cp out/openbgpd "$REMOTE_ROOT/steps/$STEP_ID/rpki-client/openbgpd" 2>/dev/null || true
+  finished_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+  python3 - <<'PY' "$REMOTE_ROOT/steps/$STEP_ID/rpki-client/round-result.json" "$STEP_ID" "$KIND" "$started_ms" "$finished_ms" "$exit_code"
+import json, sys
+path, step_id, kind, started_ms, finished_ms, exit_code = sys.argv[1:]
+json.dump(
+    {
+        "stepId": step_id,
+        "kind": kind,
+        "durationMs": int(finished_ms) - int(started_ms),
+        "exitCode": int(exit_code),
+    },
+    open(path, "w"),
+    indent=2,
+)
+PY
+}
+
+if [[ "$RP_RUN_MODE" == "parallel" ]]; then
+  run_ours &
+  OURS_PID=$!
+  run_client &
+  CLIENT_PID=$!
+  wait "$OURS_PID"
+  wait "$CLIENT_PID"
+else
+  run_ours
+  run_client
+fi
+EOS
+
+  for rel in result.ccr result.cir round-result.json run.log stage-timing.json; do
+    scp -C "$SSH_TARGET:$REMOTE_ROOT/steps/$step_id/ours/$rel" "$local_step/ours/"
+  done
+  for rel in result.ccr result.cir round-result.json run.log openbgpd; do
+    scp -C "$SSH_TARGET:$REMOTE_ROOT/steps/$step_id/rpki-client/$rel" "$local_step/rpki-client/" || true
+  done
+  if [[ "$COPY_RPKI_CLIENT_CACHE" == "1" ]]; then
+    mkdir -p "$local_step/rpki-client/cache"
+    rsync -a --delete "$SSH_TARGET:$REMOTE_ROOT/state/rpki-client/cache/" "$local_step/rpki-client/cache/"
+  fi
+
+  if [[ -f "$local_step/ours/result.cir" && -f "$local_step/rpki-client/result.cir" ]]; then
+    "$ROOT_DIR/scripts/periodic/compare_ccr_cir_round.sh" \
+      --ours-ccr "$local_step/ours/result.ccr" \
+      --rpki-client-ccr "$local_step/rpki-client/result.ccr" \
+      --ours-cir "$local_step/ours/result.cir" \
+      --rpki-client-cir "$local_step/rpki-client/result.cir" \
+      --out-dir "$local_step/compare" \
+      --trust-anchor unknown >/dev/null
+    if [[ "$PROBE_RPKI_CLIENT_CACHE" == "1" ]]; then
+      ssh "$SSH_TARGET" "set -e; mkdir -p '$REMOTE_ROOT/steps/$step_id/compare/cir'; '$REMOTE_ROOT/bin/cir_probe_rpki_client_cache' --ours-cir '$REMOTE_ROOT/steps/$step_id/ours/result.cir' --rpki-client-cir '$REMOTE_ROOT/steps/$step_id/rpki-client/result.cir' --cache-root '$REMOTE_ROOT/state/rpki-client/cache' --rpki-client-log '$REMOTE_ROOT/steps/$step_id/rpki-client/run.log' --out-json '$REMOTE_ROOT/steps/$step_id/compare/cir/rpki-client-cache-probe.json' --sample-limit 50 >/dev/null"
+      scp -C "$SSH_TARGET:$REMOTE_ROOT/steps/$step_id/compare/cir/rpki-client-cache-probe.json" "$local_step/compare/cir/"
+    fi
+    if [[ "$COPY_RPKI_CLIENT_CACHE" == "1" ]]; then
+      "$ROOT_DIR/target/release/cir_probe_rpki_client_cache" \
+        --ours-cir "$local_step/ours/result.cir" \
+        --rpki-client-cir "$local_step/rpki-client/result.cir" \
+        --cache-root "$local_step/rpki-client/cache" \
+        --rpki-client-log "$local_step/rpki-client/run.log" \
+        --out-json "$local_step/compare/cir/rpki-client-cache-probe.json" \
+        --sample-limit 50 >/dev/null
+    fi
+  else
+    "$ROOT_DIR/scripts/periodic/compare_ccr_round.sh" \
+      --ours-ccr "$local_step/ours/result.ccr" \
+      --rpki-client-ccr "$local_step/rpki-client/result.ccr" \
+      --out-dir "$local_step/compare" \
+      --trust-anchor unknown >/dev/null
+  fi
+
+  python3 - <<'PY' "$local_step/ours/round-result.json" "$local_step/rpki-client/round-result.json" "$local_step/ours/stage-timing.json" "$local_step/compare/summary.json" "$local_step/compare/compare-summary.json" "$local_step/step-summary.json" "$OURS_EXTRA_ARGS"
+import json, sys
+ours = json.load(open(sys.argv[1]))
+client = json.load(open(sys.argv[2]))
+stage = json.load(open(sys.argv[3]))
+compare_path = sys.argv[4] if __import__('pathlib').Path(sys.argv[4]).exists() else sys.argv[5]
+compare = json.load(open(compare_path))
+ours_extra_args = sys.argv[7]
+json.dump(
+    {
+        "stepId": ours["stepId"],
+        "kind": ours["kind"],
+        "oursExtraArgs": ours_extra_args,
+        "oursDurationMs": ours["durationMs"],
+        "rpkiClientDurationMs": client["durationMs"],
+        "oursExitCode": ours["exitCode"],
+        "rpkiClientExitCode": client["exitCode"],
+        "oursTotalMs": stage["total_ms"],
+        "oursRepoSyncMsTotal": stage["repo_sync_ms_total"],
+        "oursPublicationPointRepoSyncMsTotal": stage.get("publication_point_repo_sync_ms_total"),
+        "oursDownloadEventCount": stage.get("download_event_count"),
+        "oursRrdpDownloadMsTotal": stage.get("rrdp_download_ms_total"),
+        "oursRsyncDownloadMsTotal": stage.get("rsync_download_ms_total"),
+        "oursDownloadBytesTotal": stage.get("download_bytes_total"),
+        "oursVrps": compare["vrps"]["ours"],
+        "rpkiClientVrps": compare["vrps"]["rpkiClient"],
+        "oursVaps": compare["vaps"]["ours"],
+        "rpkiClientVaps": compare["vaps"]["rpkiClient"],
+        "vrpMatch": compare["vrps"]["match"],
+        "vapMatch": compare["vaps"]["match"],
+        "allMatch": compare["allMatch"],
+        "onlyInOurs": len(compare["vrps"]["onlyInOurs"]),
+        "onlyInRpkiClient": len(compare["vrps"]["onlyInRpkiClient"]),
+    },
+    open(sys.argv[6], "w"),
+    indent=2,
+)
+PY
+}
+
+run_step step-001 snapshot
+run_step step-002 delta
+
+python3 - <<'PY' "$RUN_ROOT/steps/step-001/step-summary.json" "$RUN_ROOT/steps/step-002/step-summary.json" "$RUN_ROOT/summary.json" "$RP_RUN_MODE" "$OURS_EXTRA_ARGS" "$RIR_SET" "$SCOPE_LABEL" "${RIRS[@]}"
+import json, sys
+steps = [json.load(open(p)) for p in sys.argv[1:3]]
+summary = {
+    "workflowName": "性能对比测试快速版",
+    "scope": sys.argv[7],
+    "rpRunMode": sys.argv[4],
+    "oursExtraArgs": sys.argv[5],
+    "rirSet": sys.argv[6],
+    "rirs": sys.argv[8:],
+    "steps": steps,
+}
+json.dump(summary, open(sys.argv[3], "w"), indent=2, ensure_ascii=False)
+print(json.dumps(summary, indent=2, ensure_ascii=False))
+PY

scripts/coverage.sh

@@ -5,6 +5,19 @@ set -euo pipefail
 #   rustup component add llvm-tools-preview
 #   cargo install cargo-llvm-cov --locked
 
+# Optional:
+#   COVERAGE_FORCE_CLEAN=1  Force `cargo llvm-cov clean --workspace` before the run.
+#                           Default behavior is to reuse existing llvm-cov build artifacts.
+#   RPKI_SKIP_HEAVY_SCRIPT_REPLAY_TESTS=1  Skip replay/matrix integration tests that
+#                           spawn shell pipelines and can trigger separate release builds.
+#                           coverage.sh enables this by default.
+#   RPKI_SKIP_HEAVY_BLACKBOX_TESTS=1  Skip slower blackbox CLI/script integration tests
+#                           that provide low incremental coverage per wall-clock second.
+#                           coverage.sh enables this by default.
+#   RPKI_SKIP_HEAVY_CRYPTO_TESTS=1  Skip slower OpenSSL-heavy certificate generation tests
+#                           that provide low incremental coverage per wall-clock second.
+#                           coverage.sh enables this by default.
+
 run_out="$(mktemp)"
 text_out="$(mktemp)"
 html_out="$(mktemp)"
@@ -14,13 +27,22 @@ cleanup() {
 }
 trap cleanup EXIT
 
-IGNORE_REGEX='src/bin/replay_bundle_capture\.rs|src/bin/replay_bundle_capture_delta\.rs|src/bin/replay_bundle_capture_sequence\.rs|src/bin/replay_bundle_record\.rs|src/bin/replay_bundle_refresh_sequence_outputs\.rs|src/bin/measure_sequence_replay\.rs|src/bin/repository_view_stats\.rs|src/bin/trace_arin_missing_vrps\.rs|src/bin/db_stats\.rs|src/bin/rrdp_state_dump\.rs|src/bin/ccr_dump\.rs|src/bin/ccr_verify\.rs|src/bin/ccr_to_routinator_csv\.rs|src/bin/ccr_to_compare_views\.rs|src/bin/cir_materialize\.rs|src/bin/cir_extract_inputs\.rs|src/bin/cir_drop_report\.rs|src/bin/cir_ta_only_fixture\.rs|src/bundle/live_capture\.rs|src/bundle/record_io\.rs|src/bundle/compare_view\.rs|src/progress_log\.rs|src/cli\.rs|src/validation/run_tree_from_tal\.rs|src/validation/from_tal\.rs|src/sync/store_projection\.rs|src/cir/materialize\.rs'
+IGNORE_REGEX='repository_view_stats\.rs|db_stats\.rs|rrdp_state_dump\.rs|ccr_dump\.rs|ccr_verify\.rs|ccr_to_routinator_csv\.rs|ccr_to_compare_views\.rs|cir_materialize\.rs|cir_extract_inputs\.rs|cir_drop_report\.rs|cir_ta_only_fixture\.rs|cir_dump_reject_list\.rs|rpki_object_parse\.rs|rpki_query_indexer\.rs|rpki_query_service\.rs|triage_ccr_cir_pair\.rs|rpki_artifact_metrics|rpki_inter_rp_metrics|rpki_daemon\.rs|sequence_triage_ccr_cir|ccr_state_compare\.rs|cir_state_compare\.rs|cir_probe_rpki_client_cache\.rs|ccr/compare_view\.rs|progress_log\.rs|cli\.rs|validation/run_tree_from_tal\.rs|validation/tree_parallel\.rs|validation/tree_runner|validation/from_tal\.rs|sync/store_projection\.rs|sync/repo\.rs|sync/rrdp|(^|/)storage(/|\.rs$)|cir/materialize\.rs'
 
 # Preserve colored output even though we post-process output by running under a pseudo-TTY.
 # We run tests only once, then generate both CLI text + HTML reports without rerunning tests.
 set +e
 
-cargo llvm-cov clean --workspace >/dev/null 2>&1
+if [ "${COVERAGE_FORCE_CLEAN:-0}" = "1" ]; then
+  cargo llvm-cov clean --workspace >/dev/null 2>&1
+  echo "coverage mode: clean build (COVERAGE_FORCE_CLEAN=1)"
+else
+  echo "coverage mode: reuse existing llvm-cov artifacts (default)"
+fi
+
+export RPKI_SKIP_HEAVY_SCRIPT_REPLAY_TESTS="${RPKI_SKIP_HEAVY_SCRIPT_REPLAY_TESTS:-1}"
+export RPKI_SKIP_HEAVY_BLACKBOX_TESTS="${RPKI_SKIP_HEAVY_BLACKBOX_TESTS:-1}"
+export RPKI_SKIP_HEAVY_CRYPTO_TESTS="${RPKI_SKIP_HEAVY_CRYPTO_TESTS:-1}"
 
 # 1) Run tests once to collect coverage data (no report).
 script -q -e -c "CARGO_TERM_COLOR=always cargo llvm-cov --no-report" "$run_out" >/dev/null 2>&1

scripts/docker/build_arm64_installer_package.sh

@@ -0,0 +1,216 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+IMAGE_TAG="${IMAGE_TAG:-ours-rp-runtime-arm64:dev}"
+IMAGE_TAR="${IMAGE_TAR:-}"
+PROMETHEUS_IMAGE="${PROMETHEUS_IMAGE:-prom/prometheus:v2.55.1}"
+PROMETHEUS_IMAGE_TAR="${PROMETHEUS_IMAGE_TAR:-}"
+GRAFANA_IMAGE="${GRAFANA_IMAGE:-grafana/grafana:11.3.1}"
+GRAFANA_IMAGE_TAR="${GRAFANA_IMAGE_TAR:-}"
+OUT_DIR="${OUT_DIR:-$REPO_ROOT/target/arm64-installer}"
+PACKAGE_PREFIX="${PACKAGE_PREFIX:-ours-rp-arm64-installer}"
+TEMPLATE_DIR="${TEMPLATE_DIR:-$REPO_ROOT/deploy/arm64-installer}"
+MONITOR_PLATFORM="${MONITOR_PLATFORM:-linux/arm64}"
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  scripts/docker/build_arm64_installer_package.sh [options]
+
+Options:
+  --image <tag>       Runtime image tag recorded in package manifest.
+  --image-tar <path>  Existing docker save tar/tar.gz to include.
+  --prometheus-image <tag>
+                      Prometheus image tag to record and package.
+  --prometheus-image-tar <path>
+                      Existing Prometheus docker save tar/tar.gz to include.
+  --grafana-image <tag>
+                      Grafana image tag to record and package.
+  --grafana-image-tar <path>
+                      Existing Grafana docker save tar/tar.gz to include.
+  --out-dir <path>    Output directory.
+  --prefix <name>     Package directory/tar prefix.
+  -h, --help          Show help.
+
+If --image-tar is omitted, the script uses the newest
+target/arm64-docker/*.tar.gz file.
+USAGE
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --image)
+      IMAGE_TAG="$2"
+      shift 2
+      ;;
+    --image-tar)
+      IMAGE_TAR="$2"
+      shift 2
+      ;;
+    --prometheus-image)
+      PROMETHEUS_IMAGE="$2"
+      shift 2
+      ;;
+    --prometheus-image-tar)
+      PROMETHEUS_IMAGE_TAR="$2"
+      shift 2
+      ;;
+    --grafana-image)
+      GRAFANA_IMAGE="$2"
+      shift 2
+      ;;
+    --grafana-image-tar)
+      GRAFANA_IMAGE_TAR="$2"
+      shift 2
+      ;;
+    --out-dir)
+      OUT_DIR="$2"
+      shift 2
+      ;;
+    --prefix)
+      PACKAGE_PREFIX="$2"
+      shift 2
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      echo "unknown option: $1" >&2
+      usage >&2
+      exit 2
+      ;;
+  esac
+done
+
+[[ -d "$TEMPLATE_DIR" ]] || {
+  echo "missing template dir: $TEMPLATE_DIR" >&2
+  exit 2
+}
+
+if [[ -z "$IMAGE_TAR" ]]; then
+  IMAGE_TAR="$(find "$REPO_ROOT/target/arm64-docker" -maxdepth 1 -type f \( -name '*.tar.gz' -o -name '*.tar' \) -printf '%T@ %p\n' 2>/dev/null | sort -nr | awk 'NR==1 {print $2}')"
+fi
+
+[[ -n "$IMAGE_TAR" && -f "$IMAGE_TAR" ]] || {
+  cat >&2 <<EOF
+missing runtime image tar.
+
+Build one first, for example:
+  scripts/docker/build_arm64_runtime_image.sh --image $IMAGE_TAG
+EOF
+  exit 2
+}
+
+safe_tag_name() {
+  printf '%s' "$1" | tr '/:' '--'
+}
+
+save_image_if_needed() {
+  local image="$1"
+  local existing_tar="$2"
+  local out_dir="$3"
+  local role="$4"
+  if [[ -n "$existing_tar" ]]; then
+    [[ -f "$existing_tar" ]] || {
+      echo "missing $role image tar: $existing_tar" >&2
+      exit 2
+    }
+    printf '%s\n' "$existing_tar"
+    return 0
+  fi
+  if ! docker image inspect "$image" >/dev/null 2>&1; then
+    cat >&2 <<EOF
+missing local $role image: $image
+
+Prepare it before building the installer package, for example:
+  docker pull --platform $MONITOR_PLATFORM $image
+EOF
+    exit 2
+  fi
+  local actual_platform
+  actual_platform="$(docker image inspect --format '{{.Os}}/{{.Architecture}}' "$image" 2>/dev/null || echo unknown)"
+  if [[ "$actual_platform" != "$MONITOR_PLATFORM" ]]; then
+    cat >&2 <<EOF
+wrong platform for $role image: $image
+  expected: $MONITOR_PLATFORM
+  actual:   $actual_platform
+
+Pull the ARM64 variant explicitly:
+  docker pull --platform $MONITOR_PLATFORM $image
+EOF
+    exit 2
+  fi
+  local tar_path="$out_dir/$(safe_tag_name "$image").tar.gz"
+  echo "saving $role image to $tar_path" >&2
+  docker save "$image" | gzip -c > "$tar_path"
+  printf '%s\n' "$tar_path"
+}
+
+mkdir -p "$OUT_DIR"
+commit="$(git -C "$REPO_ROOT" rev-parse --short HEAD 2>/dev/null || echo unknown)"
+timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
+package_name="${PACKAGE_PREFIX}-${timestamp}-${commit}"
+stage="$OUT_DIR/$package_name"
+tar_path="$OUT_DIR/$package_name.tar.gz"
+
+rm -rf "$stage"
+rsync -a --delete "$TEMPLATE_DIR"/ "$stage"/
+mkdir -p "$stage/images"
+cp "$IMAGE_TAR" "$stage/images/"
+
+monitor_image_stage="$OUT_DIR/.monitor-images-$timestamp"
+rm -rf "$monitor_image_stage"
+mkdir -p "$monitor_image_stage"
+prometheus_tar="$(save_image_if_needed "$PROMETHEUS_IMAGE" "$PROMETHEUS_IMAGE_TAR" "$monitor_image_stage" "prometheus")"
+grafana_tar="$(save_image_if_needed "$GRAFANA_IMAGE" "$GRAFANA_IMAGE_TAR" "$monitor_image_stage" "grafana")"
+cp "$prometheus_tar" "$stage/images/"
+cp "$grafana_tar" "$stage/images/"
+
+if [[ -f "$stage/.env.example" ]]; then
+  tmp_env="$stage/.env.example.tmp"
+  awk -v image="$IMAGE_TAG" -v prometheus="$PROMETHEUS_IMAGE" -v grafana="$GRAFANA_IMAGE" -v monitor_platform="$MONITOR_PLATFORM" '
+    BEGIN { done=0 }
+    /^RPKI_IMAGE=/ { print "RPKI_IMAGE=" image; done=1; next }
+    /^PROMETHEUS_IMAGE=/ { print "PROMETHEUS_IMAGE=" prometheus; next }
+    /^GRAFANA_IMAGE=/ { print "GRAFANA_IMAGE=" grafana; next }
+    /^MONITOR_PLATFORM=/ { print "MONITOR_PLATFORM=" monitor_platform; next }
+    { print }
+    END { if (!done) print "RPKI_IMAGE=" image }
+  ' "$stage/.env.example" > "$tmp_env"
+  mv "$tmp_env" "$stage/.env.example"
+fi
+
+cat > "$stage/PACKAGE-MANIFEST.env" <<EOF
+package_name=$package_name
+created_at_utc=$timestamp
+git_commit=$commit
+git_status_count=$(git -C "$REPO_ROOT" status --short 2>/dev/null | wc -l | tr -d ' ')
+image_tag=$IMAGE_TAG
+image_tar=$(basename "$IMAGE_TAR")
+image_tar_size_bytes=$(wc -c < "$IMAGE_TAR")
+prometheus_image=$PROMETHEUS_IMAGE
+prometheus_image_tar=$(basename "$prometheus_tar")
+prometheus_image_tar_size_bytes=$(wc -c < "$prometheus_tar")
+grafana_image=$GRAFANA_IMAGE
+grafana_image_tar=$(basename "$grafana_tar")
+grafana_image_tar_size_bytes=$(wc -c < "$grafana_tar")
+target_platform=linux/arm64
+monitor_platform=$MONITOR_PLATFORM
+EOF
+
+chmod +x "$stage"/*.sh "$stage/scripts"/*.sh
+tar -C "$OUT_DIR" -czf "$tar_path" "$package_name"
+rm -rf "$monitor_image_stage"
+
+{
+  echo "package=$tar_path"
+  echo "package_dir=$stage"
+  echo "package_size_bytes=$(wc -c < "$tar_path")"
+  echo "manifest=$stage/PACKAGE-MANIFEST.env"
+} > "$OUT_DIR/$package_name.summary.env"
+
+echo "package built: $tar_path"

scripts/docker/build_arm64_runtime_image.sh

@@ -0,0 +1,185 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+IMAGE_TAG="${IMAGE_TAG:-ours-rp-runtime-arm64:dev}"
+BUILDER_IMAGE="${BUILDER_IMAGE:-ours-rp-base-rust-amd64:1-bookworm}"
+RUNTIME_IMAGE="${RUNTIME_IMAGE:-ours-rp-base-debian-arm64:bookworm-slim}"
+OUT_DIR="${OUT_DIR:-$REPO_ROOT/target/arm64-docker}"
+DOCKERFILE="${DOCKERFILE:-$REPO_ROOT/docker/ours-rp-runtime.Dockerfile}"
+BUILDER_NAME="${BUILDER_NAME:-default}"
+INSTALL_BINFMT="${INSTALL_BINFMT:-1}"
+SAVE_IMAGE="${SAVE_IMAGE:-1}"
+LOAD_IMAGE="${LOAD_IMAGE:-1}"
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  scripts/docker/build_arm64_runtime_image.sh [options]
+
+Options:
+  --image <tag>       Docker image tag (default: ours-rp-runtime-arm64:dev)
+  --out-dir <path>    Directory for docker save tar.gz (default: target/arm64-docker)
+  --dockerfile <path> Dockerfile path
+  --builder <name>    buildx builder name
+  --no-binfmt         Do not install binfmt/qemu
+  --no-save           Build image but do not docker save it
+  --no-load           Use buildx output tar instead of --load
+  -h, --help          Show this help
+USAGE
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --image)
+      IMAGE_TAG="$2"
+      shift 2
+      ;;
+    --out-dir)
+      OUT_DIR="$2"
+      shift 2
+      ;;
+    --dockerfile)
+      DOCKERFILE="$2"
+      shift 2
+      ;;
+    --builder)
+      BUILDER_NAME="$2"
+      shift 2
+      ;;
+    --no-binfmt)
+      INSTALL_BINFMT=0
+      shift
+      ;;
+    --no-save)
+      SAVE_IMAGE=0
+      shift
+      ;;
+    --no-load)
+      LOAD_IMAGE=0
+      shift
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      echo "unknown option: $1" >&2
+      usage >&2
+      exit 2
+      ;;
+  esac
+done
+
+require_command() {
+  command -v "$1" >/dev/null 2>&1 || {
+    echo "missing required command: $1" >&2
+    exit 2
+  }
+}
+
+safe_tag_name() {
+  printf '%s' "$1" | tr '/:' '--'
+}
+
+require_command docker
+mkdir -p "$OUT_DIR"
+
+require_docker_image() {
+  local image="$1"
+  if ! docker image inspect "$image" >/dev/null 2>&1; then
+    cat >&2 <<EOF
+missing required local Docker image: $image
+
+This script intentionally builds with local base image tags to avoid
+multi-architecture tag ambiguity during cross builds.
+
+Prepare the default base tags with:
+  docker pull --platform linux/amd64 rust:1-bookworm
+  docker tag rust:1-bookworm ours-rp-base-rust-amd64:1-bookworm
+  docker pull --platform linux/arm64 debian:bookworm-slim
+  docker tag debian:bookworm-slim ours-rp-base-debian-arm64:bookworm-slim
+EOF
+    exit 2
+  fi
+}
+
+require_docker_image "$BUILDER_IMAGE"
+require_docker_image "$RUNTIME_IMAGE"
+
+if [[ "$INSTALL_BINFMT" == "1" ]]; then
+  echo "installing binfmt/qemu for arm64"
+  docker run --rm --privileged tonistiigi/binfmt --install arm64
+fi
+
+if [[ "$BUILDER_NAME" != "default" ]] && ! docker buildx inspect "$BUILDER_NAME" >/dev/null 2>&1; then
+  docker buildx create --name "$BUILDER_NAME" --driver docker-container --use >/dev/null
+else
+  docker buildx use "$BUILDER_NAME" >/dev/null
+fi
+docker buildx inspect --bootstrap >/dev/null
+
+metadata_path="$OUT_DIR/$(safe_tag_name "$IMAGE_TAG").build-metadata.json"
+tar_path="$OUT_DIR/$(safe_tag_name "$IMAGE_TAG").tar.gz"
+build_log="$OUT_DIR/$(safe_tag_name "$IMAGE_TAG").build.log"
+
+echo "building linux/arm64 image: $IMAGE_TAG"
+echo "repo: $REPO_ROOT"
+echo "dockerfile: $DOCKERFILE"
+echo "builder_image: $BUILDER_IMAGE"
+echo "runtime_image: $RUNTIME_IMAGE"
+start_epoch="$(date +%s)"
+
+if [[ "$LOAD_IMAGE" == "1" ]]; then
+  docker buildx build \
+    --platform linux/arm64 \
+    --builder "$BUILDER_NAME" \
+    --load \
+    --build-arg BUILDKIT_INLINE_CACHE=1 \
+    --build-arg "BUILDER_IMAGE=$BUILDER_IMAGE" \
+    --build-arg "RUNTIME_IMAGE=$RUNTIME_IMAGE" \
+    --metadata-file "$metadata_path" \
+    -t "$IMAGE_TAG" \
+    -f "$DOCKERFILE" \
+    "$REPO_ROOT" 2>&1 | tee "$build_log"
+else
+  raw_tar_path="$OUT_DIR/$(safe_tag_name "$IMAGE_TAG").tar"
+  docker buildx build \
+    --platform linux/arm64 \
+    --builder "$BUILDER_NAME" \
+    --output "type=docker,dest=$raw_tar_path" \
+    --build-arg BUILDKIT_INLINE_CACHE=1 \
+    --build-arg "BUILDER_IMAGE=$BUILDER_IMAGE" \
+    --build-arg "RUNTIME_IMAGE=$RUNTIME_IMAGE" \
+    --metadata-file "$metadata_path" \
+    -t "$IMAGE_TAG" \
+    -f "$DOCKERFILE" \
+    "$REPO_ROOT" 2>&1 | tee "$build_log"
+  gzip -f "$raw_tar_path"
+  tar_path="${raw_tar_path}.gz"
+fi
+
+elapsed_secs=$(( $(date +%s) - start_epoch ))
+
+if [[ "$SAVE_IMAGE" == "1" && "$LOAD_IMAGE" == "1" ]]; then
+  echo "saving image to $tar_path"
+  docker save "$IMAGE_TAG" | gzip -c > "$tar_path"
+fi
+
+{
+  echo "image=$IMAGE_TAG"
+  echo "platform=linux/arm64"
+  echo "builder_image=$BUILDER_IMAGE"
+  echo "runtime_image=$RUNTIME_IMAGE"
+  echo "elapsed_secs=$elapsed_secs"
+  echo "metadata=$metadata_path"
+  echo "tar=$tar_path"
+  echo "tar_size_bytes=$(wc -c < "$tar_path" 2>/dev/null || echo 0)"
+  echo "git_commit=$(git -C "$REPO_ROOT" rev-parse --short HEAD 2>/dev/null || echo unknown)"
+  echo "git_status_count=$(git -C "$REPO_ROOT" status --short 2>/dev/null | wc -l | tr -d ' ')"
+  echo "built_at_utc=$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+} > "$OUT_DIR/$(safe_tag_name "$IMAGE_TAG").build-summary.env"
+
+echo "build complete: elapsed=${elapsed_secs}s tar=$tar_path"

scripts/docker/deploy_remote233_arm64_compose.sh

@@ -0,0 +1,159 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+REMOTE_HOST="${REMOTE_HOST:-root@47.77.204.233}"
+REMOTE_ROOT="${REMOTE_ROOT:-/root/ours-rp-arm64-compose}"
+IMAGE_TAG="${IMAGE_TAG:-ours-rp-runtime-arm64:dev}"
+IMAGE_TAR="${IMAGE_TAR:-$REPO_ROOT/target/arm64-docker/ours-rp-runtime-arm64-dev.tar.gz}"
+EXECUTE="${EXECUTE:-0}"
+INSTALL_DOCKER="${INSTALL_DOCKER:-1}"
+INSTALL_BINFMT="${INSTALL_BINFMT:-1}"
+START_CORE="${START_CORE:-0}"
+START_SIDECARS="${START_SIDECARS:-0}"
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  scripts/docker/deploy_remote233_arm64_compose.sh [options]
+
+Default is dry-run. Pass --execute to modify the remote host.
+
+Options:
+  --execute                 Actually install/copy/load/start on remote
+  --remote <ssh-host>       SSH host (default: root@47.77.204.233)
+  --remote-root <path>      Remote compose root
+  --image <tag>             Image tag loaded by docker load
+  --image-tar <path>        Local docker save tar.gz
+  --no-install-docker       Skip Docker installation
+  --no-binfmt               Skip binfmt/qemu installation
+  --start-core              Start ours-rp-soak after deploy
+  --start-sidecars          Start artifact metrics, Prometheus and Grafana after deploy
+  -h, --help                Show this help
+USAGE
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --execute)
+      EXECUTE=1
+      shift
+      ;;
+    --remote)
+      REMOTE_HOST="$2"
+      shift 2
+      ;;
+    --remote-root)
+      REMOTE_ROOT="$2"
+      shift 2
+      ;;
+    --image)
+      IMAGE_TAG="$2"
+      shift 2
+      ;;
+    --image-tar)
+      IMAGE_TAR="$2"
+      shift 2
+      ;;
+    --no-install-docker)
+      INSTALL_DOCKER=0
+      shift
+      ;;
+    --no-binfmt)
+      INSTALL_BINFMT=0
+      shift
+      ;;
+    --start-core)
+      START_CORE=1
+      shift
+      ;;
+    --start-sidecars)
+      START_SIDECARS=1
+      shift
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      echo "unknown option: $1" >&2
+      usage >&2
+      exit 2
+      ;;
+  esac
+done
+
+require_command() {
+  command -v "$1" >/dev/null 2>&1 || {
+    echo "missing required command: $1" >&2
+    exit 2
+  }
+}
+
+run_or_echo() {
+  if [[ "$EXECUTE" == "1" ]]; then
+    "$@"
+  else
+    printf 'DRY-RUN:'
+    printf ' %q' "$@"
+    printf '\n'
+  fi
+}
+
+remote_run() {
+  if [[ "$EXECUTE" == "1" ]]; then
+    ssh "$REMOTE_HOST" "$@"
+  else
+    printf 'DRY-RUN: ssh %q %q\n' "$REMOTE_HOST" "$*"
+  fi
+}
+
+require_command ssh
+require_command rsync
+
+[[ -f "$IMAGE_TAR" ]] || {
+  echo "missing image tar: $IMAGE_TAR" >&2
+  exit 2
+}
+
+compose_src="$REPO_ROOT/deploy/arm64-compose/"
+[[ -f "$compose_src/docker-compose.yml" ]] || {
+  echo "missing compose source: $compose_src" >&2
+  exit 2
+}
+
+echo "remote=$REMOTE_HOST"
+echo "remote_root=$REMOTE_ROOT"
+echo "image=$IMAGE_TAG"
+echo "image_tar=$IMAGE_TAR"
+
+if [[ "$INSTALL_DOCKER" == "1" ]]; then
+  remote_run "if ! command -v docker >/dev/null 2>&1; then apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y docker.io; fi; if ! docker compose version >/dev/null 2>&1; then if apt-cache show docker-compose-plugin >/dev/null 2>&1; then DEBIAN_FRONTEND=noninteractive apt-get install -y docker-compose-plugin; elif apt-cache show docker-compose-v2 >/dev/null 2>&1; then DEBIAN_FRONTEND=noninteractive apt-get install -y docker-compose-v2; elif apt-cache show docker-compose >/dev/null 2>&1; then DEBIAN_FRONTEND=noninteractive apt-get install -y docker-compose; else echo 'no docker compose package found' >&2; exit 2; fi; fi; systemctl enable --now docker || true"
+fi
+
+remote_run "mkdir -p '$REMOTE_ROOT/images'"
+run_or_echo rsync -a --delete "$compose_src" "$REMOTE_HOST:$REMOTE_ROOT/"
+run_or_echo rsync -a "$IMAGE_TAR" "$REMOTE_HOST:$REMOTE_ROOT/images/"
+
+remote_tar="$REMOTE_ROOT/images/$(basename "$IMAGE_TAR")"
+remote_run "cd '$REMOTE_ROOT' && test -f .env || cp .env.example .env"
+remote_run "gunzip -c '$remote_tar' | docker load"
+
+if [[ "$INSTALL_BINFMT" == "1" ]]; then
+  remote_run "docker run --rm --privileged tonistiigi/binfmt --install arm64"
+fi
+
+remote_run "docker run --rm --platform linux/arm64 '$IMAGE_TAG' uname -m"
+remote_run "docker run --rm --platform linux/arm64 '$IMAGE_TAG' /opt/ours-rp/bin/rpki --help >/tmp/ours-rp-arm64-help.txt && head -5 /tmp/ours-rp-arm64-help.txt"
+
+if [[ "$START_CORE" == "1" ]]; then
+  remote_run "cd '$REMOTE_ROOT' && docker compose --profile core up -d ours-rp-soak"
+fi
+
+if [[ "$START_SIDECARS" == "1" ]]; then
+  remote_run "cd '$REMOTE_ROOT' && docker compose --profile sidecar --profile monitor up -d artifact-metrics prometheus grafana"
+fi
+
+remote_run "cd '$REMOTE_ROOT' && docker compose ps"

scripts/experiments/feature035/experiments.json

@@ -0,0 +1,41 @@
+{
+  "schemaVersion": 1,
+  "defaultRirs": ["afrinic", "apnic", "arin", "lacnic", "ripe"],
+  "experiments": [
+    {
+      "id": "sync-ours-rsync-only",
+      "left": { "rpKind": "ours", "mode": "standard", "protocol": "rrdp+rsync", "rsyncScope": "module-root" },
+      "right": { "rpKind": "ours", "mode": "standard", "protocol": "rsync-only", "rsyncScope": "module-root" }
+    },
+    {
+      "id": "sync-rpki-client-rsync-only",
+      "left": { "rpKind": "rpki-client", "mode": "standard", "protocol": "rrdp+rsync" },
+      "right": { "rpKind": "rpki-client", "mode": "standard", "protocol": "rsync-only" }
+    },
+    {
+      "id": "strict-name",
+      "left": { "rpKind": "ours", "mode": "standard", "protocol": "rrdp+rsync" },
+      "right": { "rpKind": "ours", "mode": "strict-name", "protocol": "rrdp+rsync" }
+    },
+    {
+      "id": "strict-cms-der",
+      "left": { "rpKind": "ours", "mode": "standard", "protocol": "rrdp+rsync" },
+      "right": { "rpKind": "ours", "mode": "strict-cms-der", "protocol": "rrdp+rsync" }
+    },
+    {
+      "id": "strict-signed-attrs",
+      "left": { "rpKind": "ours", "mode": "standard", "protocol": "rrdp+rsync" },
+      "right": { "rpKind": "ours", "mode": "strict-signed-attrs", "protocol": "rrdp+rsync" }
+    },
+    {
+      "id": "strict-all",
+      "left": { "rpKind": "ours", "mode": "standard", "protocol": "rrdp+rsync" },
+      "right": { "rpKind": "ours", "mode": "strict-all", "protocol": "rrdp+rsync" }
+    },
+    {
+      "id": "rp-implementation-standard",
+      "left": { "rpKind": "ours", "mode": "standard", "protocol": "rrdp+rsync" },
+      "right": { "rpKind": "rpki-client", "mode": "standard", "protocol": "rrdp+rsync" }
+    }
+  ]
+}

scripts/experiments/feature035/feature035_bundle.py

@@ -0,0 +1,431 @@
+#!/usr/bin/env python3
+import argparse
+import hashlib
+import json
+import os
+import platform
+import subprocess
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+
+RIR_FIXTURES = {
+    "afrinic": {
+        "tal": "tal/afrinic.tal",
+        "ta": "ta/afrinic-ta.cer",
+    },
+    "apnic": {
+        "tal": "tal/apnic-rfc7730-https.tal",
+        "ta": "ta/apnic-ta.cer",
+    },
+    "arin": {
+        "tal": "tal/arin.tal",
+        "ta": "ta/arin-ta.cer",
+    },
+    "lacnic": {
+        "tal": "tal/lacnic.tal",
+        "ta": "ta/lacnic-ta.cer",
+    },
+    "ripe": {
+        "tal": "tal/ripe-ncc.tal",
+        "ta": "ta/ripe-ncc-ta.cer",
+    },
+}
+
+
+def utc_now() -> str:
+    return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def sha256_file(path: Path) -> str:
+    hasher = hashlib.sha256()
+    with path.open("rb") as file:
+        for chunk in iter(lambda: file.read(1024 * 1024), b""):
+            hasher.update(chunk)
+    return hasher.hexdigest()
+
+
+def read_tal_uris(path: Path) -> list[str]:
+    uris: list[str] = []
+    with path.open("r", encoding="utf-8") as file:
+        for line in file:
+            item = line.strip()
+            if not item or item.startswith("#"):
+                if uris:
+                    break
+                continue
+            if item.startswith(("rsync://", "https://", "http://")):
+                uris.append(item)
+                continue
+            if uris:
+                break
+    return uris
+
+
+def first_uri(uris: list[str], prefixes: tuple[str, ...]) -> str | None:
+    for uri in uris:
+        if uri.startswith(prefixes):
+            return uri
+    return None
+
+
+def parse_rirs(raw: str) -> list[str]:
+    rirs = [item.strip().lower() for item in raw.split(",") if item.strip()]
+    if not rirs:
+        raise SystemExit("RIR list must not be empty")
+    invalid = [item for item in rirs if item not in RIR_FIXTURES]
+    if invalid:
+        raise SystemExit(
+            f"invalid RIR(s): {','.join(invalid)}; allowed: {','.join(RIR_FIXTURES)}"
+        )
+    return rirs
+
+
+def rel_or_abs(path: Path, root: Path | None) -> str:
+    path = path.resolve()
+    if root is not None:
+        try:
+            return path.relative_to(root.resolve()).as_posix()
+        except ValueError:
+            pass
+    return path.as_posix()
+
+
+def git_commit(repo_root: Path) -> str | None:
+    try:
+        return subprocess.check_output(
+            ["git", "-C", str(repo_root), "rev-parse", "--short", "HEAD"],
+            text=True,
+            stderr=subprocess.DEVNULL,
+        ).strip()
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        return None
+
+
+def write_json(path: Path, value: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(value, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
+
+
+def build_fixture_proof(
+    fixture_dir: Path,
+    rirs: list[str],
+    repo_root: Path | None,
+    ta_online_fetch_observed: bool,
+) -> dict[str, Any]:
+    trust_anchors = []
+    for rir in rirs:
+        mapping = RIR_FIXTURES[rir]
+        tal_path = fixture_dir / mapping["tal"]
+        ta_path = fixture_dir / mapping["ta"]
+        if not tal_path.is_file():
+            raise SystemExit(f"missing TAL fixture for {rir}: {tal_path}")
+        if not ta_path.is_file():
+            raise SystemExit(f"missing TA fixture for {rir}: {ta_path}")
+        tal_uris = read_tal_uris(tal_path)
+        trust_anchors.append(
+            {
+                "rir": rir,
+                "talPath": rel_or_abs(tal_path, repo_root),
+                "taPath": rel_or_abs(ta_path, repo_root),
+                "talUri": first_uri(tal_uris, ("https://", "http://")),
+                "taRsyncUri": first_uri(tal_uris, ("rsync://",)),
+                "talSha256": sha256_file(tal_path),
+                "taCertificateSha256": sha256_file(ta_path),
+                "talBytes": tal_path.stat().st_size,
+                "taCertificateBytes": ta_path.stat().st_size,
+                "taFixturePinned": not ta_online_fetch_observed,
+                "taOnlineFetchObserved": ta_online_fetch_observed,
+            }
+        )
+    return {
+        "schemaVersion": 1,
+        "generatedBy": "feature035-experiment-driver",
+        "generatedAtUtc": utc_now(),
+        "fixtureDir": rel_or_abs(fixture_dir, repo_root),
+        "all5": set(rirs) == set(RIR_FIXTURES),
+        "rirs": rirs,
+        "trustAnchors": trust_anchors,
+    }
+
+
+def parse_csv(raw: str) -> list[str]:
+    if not raw:
+        return []
+    return [item.strip() for item in raw.split(",") if item.strip()]
+
+
+def optional_path(raw: str | None, repo_root: Path | None) -> str | None:
+    if raw is None:
+        return None
+    return rel_or_abs(Path(raw), repo_root)
+
+
+def build_run_meta(args: argparse.Namespace) -> dict[str, Any]:
+    rirs = parse_rirs(args.rirs)
+    repo_root = Path(args.repo_root).resolve() if args.repo_root else None
+    argv = json.loads(args.argv_json) if args.argv_json else []
+    env_whitelist = json.loads(args.env_json) if args.env_json else {}
+    fixture_proof_summary = (
+        json.loads(args.fixture_proof_summary_json)
+        if args.fixture_proof_summary_json
+        else None
+    )
+    fixture_proof = None
+    if args.fixture_proof:
+        fixture_proof_path = Path(args.fixture_proof)
+        if fixture_proof_path.is_file():
+            fixture_proof = json.loads(fixture_proof_path.read_text(encoding="utf-8"))
+    if not isinstance(argv, list):
+        raise SystemExit("--argv-json must decode to a JSON array")
+    if not isinstance(env_whitelist, dict):
+        raise SystemExit("--env-json must decode to a JSON object")
+    if fixture_proof_summary is not None and not isinstance(fixture_proof_summary, dict):
+        raise SystemExit("--fixture-proof-summary-json must decode to a JSON object")
+
+    return {
+        "schemaVersion": 1,
+        "generatedBy": "feature035-experiment-driver",
+        "generatedAtUtc": utc_now(),
+        "experimentId": args.experiment_id,
+        "side": args.side,
+        "sideLabel": args.side_label,
+        "step": args.step,
+        "runId": args.run_id,
+        "liveRun": not args.replay_used,
+        "replayUsed": args.replay_used,
+        "rp": {
+            "kind": args.rp_kind,
+            "binary": args.rp_binary,
+            "version": args.rp_version,
+            "gitCommit": args.rp_git_commit,
+            "mode": args.rp_mode,
+            "protocolMode": args.protocol_mode,
+            "strictPolicies": parse_csv(args.strict_policies),
+        },
+        "scope": {
+            "rirs": rirs,
+            "all5": set(rirs) == set(RIR_FIXTURES),
+        },
+        "command": {
+            "argv": argv,
+            "cwd": args.cwd,
+            "envWhitelist": env_whitelist,
+        },
+        "state": {
+            "resetBeforeRun": args.reset_before_run,
+            "stateRoot": args.state_root,
+            "db": args.db,
+            "repoBytesDb": args.repo_bytes_db,
+            "rawStoreDb": args.raw_store_db,
+            "rsyncMirrorRoot": args.rsync_mirror_root,
+            "cacheRoot": args.cache_root,
+        },
+        "artifacts": {
+            "ccr": optional_path(args.ccr, repo_root),
+            "cir": optional_path(args.cir, repo_root),
+            "runMeta": optional_path(str(args.out), repo_root),
+            "fixtureProof": optional_path(args.fixture_proof, repo_root),
+            "reportJson": optional_path(args.report_json, repo_root),
+            "stageTimingJson": optional_path(args.stage_timing_json, repo_root),
+            "stdoutLog": optional_path(args.stdout_log, repo_root),
+            "stderrLog": optional_path(args.stderr_log, repo_root),
+            "processTime": optional_path(args.process_time, repo_root),
+            "vrpsCsv": optional_path(args.vrps_csv, repo_root),
+            "vapsCsv": optional_path(args.vaps_csv, repo_root),
+        },
+        "fixtureProof": fixture_proof,
+        "fixtureProofSummary": fixture_proof_summary,
+        "metrics": {
+            "exitCode": args.exit_code,
+            "wallMs": args.wall_ms,
+            "maxRssKb": args.max_rss_kb,
+            "vrps": args.vrps,
+            "vaps": args.vaps,
+            "publicationPoints": args.publication_points,
+            "warnings": args.warnings,
+            "cirObjectCount": args.cir_object_count,
+            "cirRejectCount": args.cir_reject_count,
+            "cirTrustAnchorCount": args.cir_trust_anchor_count,
+            "ccrStateDigest": args.ccr_state_digest,
+        },
+        "environment": {
+            "host": args.host or platform.node(),
+            "platform": args.platform or platform.platform(),
+        },
+    }
+
+
+def command_fixture_proof(args: argparse.Namespace) -> None:
+    repo_root = Path(args.repo_root).resolve() if args.repo_root else None
+    proof = build_fixture_proof(
+        fixture_dir=Path(args.fixture_dir),
+        rirs=parse_rirs(args.rirs),
+        repo_root=repo_root,
+        ta_online_fetch_observed=args.ta_online_fetch_observed,
+    )
+    write_json(Path(args.out), proof)
+
+
+def command_run_meta(args: argparse.Namespace) -> None:
+    write_json(Path(args.out), build_run_meta(args))
+
+
+def command_dry_run_bundle(args: argparse.Namespace) -> None:
+    out_dir = Path(args.out_dir)
+    repo_root = Path(args.repo_root).resolve() if args.repo_root else Path.cwd().resolve()
+    fixture_proof = out_dir / "fixture-proof.json"
+    command_fixture_proof(
+        argparse.Namespace(
+            fixture_dir=args.fixture_dir,
+            rirs=args.rirs,
+            repo_root=str(repo_root),
+            out=str(fixture_proof),
+            ta_online_fetch_observed=False,
+        )
+    )
+    for side, side_label in (("left", "A"), ("right", "B")):
+        run_dir = out_dir / side_label / "snapshot"
+        meta_args = argparse.Namespace(
+            out=run_dir / "run-meta.json",
+            repo_root=str(repo_root),
+            experiment_id=args.experiment_id,
+            side=side,
+            side_label=side_label,
+            step="snapshot",
+            run_id=f"{side_label}-snapshot-dry-run",
+            replay_used=False,
+            rp_kind="ours" if side_label == "A" else "rpki-client",
+            rp_binary=f"bin/{'rpki' if side_label == 'A' else 'rpki-client'}",
+            rp_version="dry-run",
+            rp_git_commit=git_commit(repo_root),
+            rp_mode="standard",
+            protocol_mode="rrdp+rsync",
+            strict_policies="",
+            rirs=args.rirs,
+            argv_json=json.dumps(["dry-run"]),
+            env_json=json.dumps({"RPKI_PROGRESS_LOG": "1"}),
+            cwd=str(out_dir),
+            reset_before_run=True,
+            state_root=str(out_dir / side_label / "state"),
+            db=str(out_dir / side_label / "state" / "work-db"),
+            repo_bytes_db=str(out_dir / side_label / "state" / "repo-bytes.db"),
+            raw_store_db=str(out_dir / side_label / "state" / "raw-store.db"),
+            rsync_mirror_root=str(out_dir / side_label / "state" / "rsync-mirror"),
+            cache_root=str(out_dir / side_label / "state" / "cache"),
+            ccr=str(run_dir / "result.ccr"),
+            cir=str(run_dir / "result.cir"),
+            fixture_proof=str(fixture_proof),
+            report_json=str(run_dir / "report.json"),
+            stage_timing_json=str(run_dir / "stage-timing.json"),
+            stdout_log=str(run_dir / "stdout.log"),
+            stderr_log=str(run_dir / "stderr.log"),
+            process_time=str(run_dir / "process-time.txt"),
+            vrps_csv=str(run_dir / "vrps.csv"),
+            vaps_csv=str(run_dir / "vaps.csv"),
+            exit_code=0,
+            wall_ms=0,
+            max_rss_kb=0,
+            vrps=0,
+            vaps=0,
+            publication_points=0,
+            warnings=0,
+            cir_object_count=0,
+            cir_reject_count=0,
+            cir_trust_anchor_count=len(parse_rirs(args.rirs)),
+            ccr_state_digest=None,
+            fixture_proof_summary_json=json.dumps(
+                {
+                    "taFixturePinned": True,
+                    "taOnlineFetchObserved": False,
+                    "trustAnchorCount": len(parse_rirs(args.rirs)),
+                }
+            ),
+        )
+        command_run_meta(meta_args)
+
+
+def add_run_meta_args(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument("--out", required=True)
+    parser.add_argument("--repo-root")
+    parser.add_argument("--experiment-id", required=True)
+    parser.add_argument("--side", choices=["left", "right"], required=True)
+    parser.add_argument("--side-label", choices=["A", "B"], required=True)
+    parser.add_argument("--step", choices=["snapshot", "delta"], required=True)
+    parser.add_argument("--run-id", required=True)
+    parser.add_argument("--replay-used", action="store_true")
+    parser.add_argument("--rp-kind", required=True)
+    parser.add_argument("--rp-binary", required=True)
+    parser.add_argument("--rp-version")
+    parser.add_argument("--rp-git-commit")
+    parser.add_argument("--rp-mode", default="standard")
+    parser.add_argument("--protocol-mode", default="rrdp+rsync")
+    parser.add_argument("--strict-policies", default="")
+    parser.add_argument("--rirs", default="afrinic,apnic,arin,lacnic,ripe")
+    parser.add_argument("--argv-json")
+    parser.add_argument("--env-json")
+    parser.add_argument("--cwd", default=os.getcwd())
+    parser.add_argument("--reset-before-run", action="store_true")
+    parser.add_argument("--state-root")
+    parser.add_argument("--db")
+    parser.add_argument("--repo-bytes-db")
+    parser.add_argument("--raw-store-db")
+    parser.add_argument("--rsync-mirror-root")
+    parser.add_argument("--cache-root")
+    parser.add_argument("--ccr")
+    parser.add_argument("--cir")
+    parser.add_argument("--fixture-proof")
+    parser.add_argument("--fixture-proof-summary-json")
+    parser.add_argument("--report-json")
+    parser.add_argument("--stage-timing-json")
+    parser.add_argument("--stdout-log")
+    parser.add_argument("--stderr-log")
+    parser.add_argument("--process-time")
+    parser.add_argument("--vrps-csv")
+    parser.add_argument("--vaps-csv")
+    parser.add_argument("--exit-code", type=int)
+    parser.add_argument("--wall-ms", type=int)
+    parser.add_argument("--max-rss-kb", type=int)
+    parser.add_argument("--vrps", type=int)
+    parser.add_argument("--vaps", type=int)
+    parser.add_argument("--publication-points", type=int)
+    parser.add_argument("--warnings", type=int)
+    parser.add_argument("--cir-object-count", type=int)
+    parser.add_argument("--cir-reject-count", type=int)
+    parser.add_argument("--cir-trust-anchor-count", type=int)
+    parser.add_argument("--ccr-state-digest")
+    parser.add_argument("--host")
+    parser.add_argument("--platform")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Feature #035 CCR/CIR experiment bundle helpers")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    fixture = subparsers.add_parser("fixture-proof")
+    fixture.add_argument("--fixture-dir", default="tests/fixtures")
+    fixture.add_argument("--rirs", default="afrinic,apnic,arin,lacnic,ripe")
+    fixture.add_argument("--repo-root")
+    fixture.add_argument("--out", required=True)
+    fixture.add_argument("--ta-online-fetch-observed", action="store_true")
+    fixture.set_defaults(func=command_fixture_proof)
+
+    run_meta = subparsers.add_parser("run-meta")
+    add_run_meta_args(run_meta)
+    run_meta.set_defaults(func=command_run_meta)
+
+    dry_run = subparsers.add_parser("dry-run-bundle")
+    dry_run.add_argument("--out-dir", required=True)
+    dry_run.add_argument("--repo-root", default=".")
+    dry_run.add_argument("--fixture-dir", default="tests/fixtures")
+    dry_run.add_argument("--rirs", default="afrinic,apnic,arin,lacnic,ripe")
+    dry_run.add_argument("--experiment-id", default="m2-dry-run")
+    dry_run.set_defaults(func=command_dry_run_bundle)
+
+    args = parser.parse_args()
+    args.func(args)
+
+
+if __name__ == "__main__":
+    main()

scripts/experiments/feature035/fixture-manifest.json

@@ -0,0 +1,10 @@
+{
+  "schemaVersion": 1,
+  "rirs": {
+    "afrinic": { "tal": "tal/afrinic.tal", "ta": "ta/afrinic-ta.cer" },
+    "apnic": { "tal": "tal/apnic-rfc7730-https.tal", "ta": "ta/apnic-ta.cer" },
+    "arin": { "tal": "tal/arin.tal", "ta": "ta/arin-ta.cer" },
+    "lacnic": { "tal": "tal/lacnic.tal", "ta": "ta/lacnic-ta.cer" },
+    "ripe": { "tal": "tal/ripe-ncc.tal", "ta": "ta/ripe-ncc-ta.cer" }
+  }
+}

scripts/experiments/feature043/run_sequence_triage_experiment.py

@@ -0,0 +1,939 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+from concurrent.futures import ThreadPoolExecutor, as_completed
+import hashlib
+import json
+import os
+import shlex
+import subprocess
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+REPO_ROOT = SCRIPT_DIR.parents[2]
+DEV_ROOT = REPO_ROOT.parents[1]
+FEATURE035_DIR = REPO_ROOT / "scripts" / "experiments" / "feature035"
+FIXTURE_MANIFEST_PATH = FEATURE035_DIR / "fixture-manifest.json"
+PORTABLE_ROOT = DEV_ROOT / "rpki-client-portable"
+CACHED_CIR_RPKI_CLIENT = DEV_ROOT / ".cache" / "rpki-client-9.8-cir" / "rpki-client"
+CACHED_CIR_LIBTLS = DEV_ROOT / ".cache" / "rpki-client-9.8-cir" / "libtls.so.28"
+
+DEFAULT_RIRS = ["afrinic", "apnic", "arin", "lacnic", "ripe"]
+
+
+def run_local(argv: list[str], *, cwd: Path | None = None, capture: bool = False, check: bool = True) -> subprocess.CompletedProcess[str]:
+    result = subprocess.run(argv, cwd=str(cwd) if cwd else None, text=True, capture_output=capture, check=False)
+    if check and result.returncode != 0:
+        raise SystemExit(
+            f"command failed ({result.returncode}): {' '.join(shlex.quote(x) for x in argv)}\n"
+            f"stdout:\n{result.stdout or ''}\nstderr:\n{result.stderr or ''}"
+        )
+    return result
+
+
+def ssh_script(target: str, script: str, *, capture: bool = False, check: bool = True) -> subprocess.CompletedProcess[str]:
+    result = subprocess.run(["ssh", target, "bash", "-s"], input=script, text=True, capture_output=capture, check=False)
+    if check and result.returncode != 0:
+        raise SystemExit(f"remote script failed ({result.returncode}) on {target}\n{result.stdout}\n{result.stderr}")
+    return result
+
+
+def rsync_to_remote(target: str, source: Path, destination: str | Path) -> None:
+    run_local(["rsync", "-a", str(source), f"{target}:{destination}"])
+
+
+def rsync_dir_to_remote(target: str, source: Path, destination: str | Path) -> None:
+    run_local(["rsync", "-a", f"{source}/", f"{target}:{destination}/"])
+
+
+def rsync_from_remote(target: str, source: str | Path, destination: Path) -> None:
+    destination.mkdir(parents=True, exist_ok=True)
+    run_local(["rsync", "-a", f"{target}:{source}/", f"{destination}/"])
+
+
+def rsync_run_artifacts_from_remote(target: str, source: str | Path, destination: Path) -> None:
+    destination.mkdir(parents=True, exist_ok=True)
+    rsync_base = ["rsync", "-az", "--ignore-missing-args", "--partial", "--partial-dir=.rsync-partial"]
+    for name in [
+        "result.ccr",
+        "result.cir",
+        "report.json",
+        "vrps.csv",
+        "vaps.csv",
+        "process-time.txt",
+        "remote-run-meta.json",
+        "exit-code.txt",
+        "started-at.txt",
+        "finished-at.txt",
+        "stdout.log",
+        "stderr.log",
+    ]:
+        run_local([*rsync_base, f"{target}:{source}/{name}", f"{destination}/"])
+
+
+def rsync_remote_analysis_from_remote(target: str, remote_exp_root: str | Path, local_exp_root: Path) -> None:
+    local_exp_root.mkdir(parents=True, exist_ok=True)
+    rsync_base = ["rsync", "-az", "--ignore-missing-args", "--partial", "--partial-dir=.rsync-partial"]
+    for name in [
+        "left-sequence.jsonl",
+        "right-sequence.jsonl",
+        "run-progress.json",
+        "sequence-triage-time.txt",
+        "sequence-triage/sequence-triage.json",
+    ]:
+        destination = local_exp_root / Path(name).parent
+        destination.mkdir(parents=True, exist_ok=True)
+        run_local([*rsync_base, f"{target}:{remote_exp_root}/{name}", f"{destination}/"])
+
+
+def same_remote_location(
+    left_target: str,
+    left_root: str | Path,
+    right_target: str,
+    right_root: str | Path,
+) -> bool:
+    return left_target == right_target and str(left_root) == str(right_root)
+
+
+def load_json(path: Path) -> Any:
+    with path.open("r", encoding="utf-8") as handle:
+        return json.load(handle)
+
+
+def write_json(path: Path, value: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8") as handle:
+        json.dump(value, handle, indent=2, sort_keys=True, ensure_ascii=False)
+        handle.write("\n")
+
+
+def append_jsonl(path: Path, value: Any) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("a", encoding="utf-8") as handle:
+        handle.write(json.dumps(value, sort_keys=True, ensure_ascii=False))
+        handle.write("\n")
+
+
+def utc_stamp() -> str:
+    return time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())
+
+
+def rfc3339_now() -> str:
+    return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+
+
+def fixture_manifest() -> dict[str, Any]:
+    return load_json(FIXTURE_MANIFEST_PATH)
+
+
+def fixture_name(rir: str, kind: str) -> str:
+    return Path(fixture_manifest()["rirs"][rir][kind]).name
+
+
+def fixture_desc(rir: str) -> str:
+    return {
+        "afrinic": "afrinic",
+        "apnic": "apnic-rfc7730-https",
+        "arin": "arin",
+        "lacnic": "lacnic",
+        "ripe": "ripe-ncc",
+    }[rir]
+
+
+def cir_tal_uri_for_rir(rir: str) -> str:
+    return {
+        "afrinic": "https://rpki.afrinic.net/tal/afrinic.tal",
+        "apnic": "https://rpki.apnic.net/tal/apnic-rfc7730-https.tal",
+        "arin": "https://www.arin.net/resources/manage/rpki/arin.tal",
+        "lacnic": "https://www.lacnic.net/innovaportal/file/4983/1/lacnic.tal",
+        "ripe": "https://tal.rpki.ripe.net/ripe-ncc.tal",
+    }[rir]
+
+
+def sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def parse_elapsed_to_ms(raw: str) -> int:
+    raw = raw.strip()
+    if not raw:
+        return 0
+    if "-" in raw:
+        days, raw = raw.split("-", 1)
+    else:
+        days = "0"
+    parts = raw.split(":")
+    if len(parts) == 3:
+        hours, minutes, seconds = parts
+    elif len(parts) == 2:
+        hours = "0"
+        minutes, seconds = parts
+    else:
+        hours = "0"
+        minutes = "0"
+        seconds = parts[0]
+    return int(round((int(days) * 86400 + int(hours) * 3600 + int(minutes) * 60 + float(seconds)) * 1000))
+
+
+def parse_time_file(path: Path) -> dict[str, Any]:
+    data: dict[str, Any] = {}
+    if not path.is_file():
+        return data
+    for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
+        if "Elapsed (wall clock) time" in line:
+            elapsed = line.rsplit(":", 1)[1] if "):" not in line else line.rsplit("):", 1)[1]
+            data["wallMs"] = parse_elapsed_to_ms(elapsed)
+        elif "Maximum resident set size" in line:
+            try:
+                data["maxRssKb"] = int(line.rsplit(":", 1)[1].strip())
+            except ValueError:
+                pass
+    return data
+
+
+def rpki_client_bin_path() -> Path:
+    primary = PORTABLE_ROOT / "src" / "rpki-client"
+    for candidate in (primary, CACHED_CIR_RPKI_CLIENT):
+        if not candidate.is_file():
+            continue
+        smoke = run_local([str(candidate), "-T", "invalid"], capture=True, check=False)
+        if "--ta-fixture requires <tal>:<path>" in (smoke.stderr + smoke.stdout):
+            return candidate
+    raise SystemExit("rpki-client binary lacks CIR/TA fixture support; checkout feature/cir-output-for-rp-compare or restore .cache/rpki-client-9.8-cir/rpki-client")
+
+
+def detect_libtls_path(rpki_client_bin: Path) -> Path:
+    if CACHED_CIR_LIBTLS.is_file():
+        return CACHED_CIR_LIBTLS
+    ldd = run_local(["ldd", str(rpki_client_bin)], capture=True)
+    for line in ldd.stdout.splitlines():
+        if "libtls.so.28" not in line or "=>" not in line:
+            continue
+        candidate = Path(line.split("=>", 1)[1].strip().split(" ", 1)[0])
+        if candidate.is_file():
+            return candidate
+    fallback = DEV_ROOT / ".cache" / "rpki-client-9.8-cir" / "libtls.so.28"
+    if fallback.is_file():
+        return fallback
+    raise SystemExit("unable to locate libtls.so.28 for rpki-client")
+
+
+def build_tool_binaries() -> None:
+    run_local([
+        "cargo", "build", "--release",
+        "--bin", "rpki",
+        "--bin", "sequence_triage_ccr_cir",
+        "--bin", "cir_dump_reject_list",
+    ], cwd=REPO_ROOT)
+    _ = rpki_client_bin_path()
+
+
+def validate_remote_disk(ssh_target: str) -> None:
+    script = r'''
+set -euo pipefail
+df -h /data / || true
+python3 - <<'PY'
+import shutil
+for path in ['/data', '/']:
+    try:
+        usage = shutil.disk_usage(path)
+    except FileNotFoundError:
+        continue
+    used = usage.used / usage.total if usage.total else 0
+    print(f'{path} used={used:.2%}')
+    if used >= 0.90:
+        raise SystemExit(f'{path} disk usage >= 90%; cleanup required before all5 sequence experiment')
+PY
+'''
+    ssh_script(ssh_target, script)
+
+
+def prepare_remote(ssh_target: str, remote_root: Path, needs_rpki_client: bool) -> None:
+    validate_remote_disk(ssh_target)
+    preflight = (
+        "set -euo pipefail; "
+        "systemctl disable --now rpki-client.timer >/dev/null 2>&1 || true; "
+        "systemctl stop rpki-client.service >/dev/null 2>&1 || true; "
+        "pkill -x rpki-client >/dev/null 2>&1 || true; "
+        "pkill -x routinator >/dev/null 2>&1 || true; "
+        f"mkdir -p {shlex.quote(str(remote_root / 'bin'))} {shlex.quote(str(remote_root / 'lib'))} "
+        f"{shlex.quote(str(remote_root / 'fixtures' / 'tal'))} {shlex.quote(str(remote_root / 'fixtures' / 'ta'))} "
+        f"{shlex.quote(str(remote_root / 'experiments'))}; "
+        f"df -h /data / > {shlex.quote(str(remote_root / 'df-before.txt'))} 2>&1 || true; "
+        f"free -h > {shlex.quote(str(remote_root / 'free-before.txt'))} 2>&1 || true"
+    )
+    ssh_script(ssh_target, preflight)
+    rsync_dir_to_remote(ssh_target, REPO_ROOT / "tests" / "fixtures" / "tal", remote_root / "fixtures" / "tal")
+    rsync_dir_to_remote(ssh_target, REPO_ROOT / "tests" / "fixtures" / "ta", remote_root / "fixtures" / "ta")
+    rsync_to_remote(ssh_target, REPO_ROOT / "target" / "release" / "rpki", remote_root / "bin" / "rpki")
+    rsync_to_remote(ssh_target, REPO_ROOT / "target" / "release" / "sequence_triage_ccr_cir", remote_root / "bin" / "sequence_triage_ccr_cir")
+    rsync_to_remote(ssh_target, REPO_ROOT / "target" / "release" / "cir_dump_reject_list", remote_root / "bin" / "cir_dump_reject_list")
+    if needs_rpki_client:
+        rpki_client_bin = rpki_client_bin_path()
+        rsync_to_remote(ssh_target, rpki_client_bin, remote_root / "bin" / "rpki-client")
+        rsync_to_remote(ssh_target, detect_libtls_path(rpki_client_bin), remote_root / "lib" / "libtls.so.28")
+
+
+def prepare_remote_once(
+    prepared: dict[tuple[str, str], bool],
+    ssh_target: str,
+    remote_root: Path,
+    needs_rpki_client: bool,
+) -> None:
+    key = (ssh_target, str(remote_root))
+    already_has_rpki_client = prepared.get(key)
+    if already_has_rpki_client is not None and (already_has_rpki_client or not needs_rpki_client):
+        return
+    prepare_remote(ssh_target, remote_root, needs_rpki_client)
+    prepared[key] = bool(already_has_rpki_client or needs_rpki_client)
+
+
+def side_config(name: str) -> dict[str, Any]:
+    if name == "ours-standard":
+        return {"rpKind": "ours", "mode": "standard", "protocol": "rrdp+rsync", "rsyncScope": "module-root"}
+    if name == "ours-strict-all":
+        return {
+            "rpKind": "ours",
+            "mode": "strict-all",
+            "protocol": "rrdp+rsync",
+            "rsyncScope": "module-root",
+            "strictPolicies": "name,cms-der,signed-attrs",
+        }
+    if name == "rpki-client-standard":
+        return {"rpKind": "rpki-client", "mode": "standard", "protocol": "rrdp+rsync"}
+    raise SystemExit(f"unknown side config: {name}")
+
+
+def parse_rirs(raw: str) -> list[str]:
+    rirs = [item.strip() for item in raw.split(",") if item.strip()]
+    if not rirs:
+        raise SystemExit("--rirs must contain at least one RIR")
+    seen: set[str] = set()
+    invalid: list[str] = []
+    duplicate: list[str] = []
+    for rir in rirs:
+        if rir not in DEFAULT_RIRS:
+            invalid.append(rir)
+        if rir in seen:
+            duplicate.append(rir)
+        seen.add(rir)
+    if invalid:
+        raise SystemExit(f"unsupported RIR(s): {','.join(invalid)}; valid values: {','.join(DEFAULT_RIRS)}")
+    if duplicate:
+        raise SystemExit(f"duplicate RIR(s): {','.join(duplicate)}")
+    return rirs
+
+
+def build_remote_command(remote_root: Path, side_name: str, side: dict[str, Any], side_label: str, seq: int, rirs: list[str]) -> tuple[Path, str]:
+    run_dir = remote_root / "experiments" / "sequence" / side_label / f"run_{seq:04d}"
+    state_dir = remote_root / "experiments" / "sequence" / side_label / "state" / side["rpKind"]
+    sync_mode = "snapshot" if seq == 1 else "delta"
+    ensure = [f"mkdir -p {shlex.quote(str(run_dir))}", f"chmod 0777 {shlex.quote(str(run_dir))}"]
+    if side["rpKind"] == "ours":
+        if seq == 1:
+            ensure.append(f"rm -rf {shlex.quote(str(state_dir))}")
+        ensure.extend([
+            f"mkdir -p {shlex.quote(str(state_dir / 'work-db'))} {shlex.quote(str(state_dir / 'rsync-mirror'))}",
+            f"chmod -R 0777 {shlex.quote(str(state_dir.parent))}",
+        ])
+        argv = [
+            str(remote_root / "bin" / "rpki"),
+            "--db", str(state_dir / "work-db"),
+            "--raw-store-db", str(state_dir / "raw-store.db"),
+            "--repo-bytes-db", str(state_dir / "repo-bytes.db"),
+            "--rsync-scope", side.get("rsyncScope", "module-root"),
+        ]
+        if side.get("strictPolicies"):
+            argv.extend(["--strict", str(side["strictPolicies"])])
+        for rir in rirs:
+            argv.extend(["--tal-path", str(remote_root / "fixtures" / "tal" / fixture_name(rir, "tal"))])
+            argv.extend(["--ta-path", str(remote_root / "fixtures" / "ta" / fixture_name(rir, "ta"))])
+        argv.extend(["--report-json", str(run_dir / "report.json"), "--report-json-compact"])
+        argv.extend(["--ccr-out", str(run_dir / "result.ccr"), "--cir-enable", "--cir-out", str(run_dir / "result.cir")])
+        for rir in rirs:
+            argv.extend(["--cir-tal-uri", cir_tal_uri_for_rir(rir)])
+        argv.extend(["--vrps-csv-out", str(run_dir / "vrps.csv"), "--vaps-csv-out", str(run_dir / "vaps.csv")])
+        prefix = "env RPKI_PROGRESS_LOG=1 RPKI_PROGRESS_SLOW_SECS=10 /usr/bin/time"
+    else:
+        if seq == 1:
+            ensure.append(f"rm -rf {shlex.quote(str(state_dir))}")
+        ensure.extend([
+            f"mkdir -p {shlex.quote(str(state_dir / 'cache' / 'fixtures'))}",
+            f"touch {shlex.quote(str(state_dir / 'rpki-client-skiplist'))}",
+            f"chmod -R 0777 {shlex.quote(str(state_dir.parent))}",
+        ])
+        for rir in rirs:
+            ensure.append(
+                f"cp -f {shlex.quote(str(remote_root / 'fixtures' / 'ta' / fixture_name(rir, 'ta')))} "
+                f"{shlex.quote(str(state_dir / 'cache' / 'fixtures' / fixture_name(rir, 'ta')))}"
+            )
+        argv = [str(remote_root / "bin" / "rpki-client"), "-vv", "-S", str(state_dir / "rpki-client-skiplist")]
+        for rir in rirs:
+            argv.extend(["-t", str(remote_root / "fixtures" / "tal" / fixture_name(rir, "tal"))])
+            argv.extend(["-T", f"{fixture_desc(rir)}:{state_dir / 'cache' / 'fixtures' / fixture_name(rir, 'ta')}"])
+        argv.extend(["-d", str(state_dir / "cache"), str(run_dir)])
+        prefix = f"env LD_LIBRARY_PATH={shlex.quote(str(remote_root / 'lib'))} /usr/bin/time"
+    command = (
+        "set -euo pipefail; "
+        + "; ".join(ensure)
+        + "; date -u +%Y-%m-%dT%H:%M:%SZ > " + shlex.quote(str(run_dir / "started-at.txt"))
+        + "; set +e; "
+        + prefix + " -v -o " + shlex.quote(str(run_dir / "process-time.txt"))
+        + " -- " + shlex.join(argv)
+        + " > " + shlex.quote(str(run_dir / "stdout.log"))
+        + " 2> " + shlex.quote(str(run_dir / "stderr.log"))
+        + "; ec=$?; set -e; printf '%s\n' \"$ec\" > " + shlex.quote(str(run_dir / "exit-code.txt"))
+        + "; date -u +%Y-%m-%dT%H:%M:%SZ > " + shlex.quote(str(run_dir / "finished-at.txt"))
+        + "; true"
+    )
+    if side["rpKind"] == "rpki-client":
+        command += (
+            f"; [ -f {shlex.quote(str(run_dir / 'json'))} ] && cp -f {shlex.quote(str(run_dir / 'json'))} {shlex.quote(str(run_dir / 'report.json'))} || true"
+            f"; [ -f {shlex.quote(str(run_dir / 'rpki.ccr'))} ] && cp -f {shlex.quote(str(run_dir / 'rpki.ccr'))} {shlex.quote(str(run_dir / 'result.ccr'))} || true"
+            f"; [ -f {shlex.quote(str(run_dir / 'rpki.cir'))} ] && cp -f {shlex.quote(str(run_dir / 'rpki.cir'))} {shlex.quote(str(run_dir / 'result.cir'))} || true"
+        )
+    command += (
+        f"; python3 - <<'REMOTE_META' {shlex.quote(str(run_dir))} {shlex.quote(side_name)} {shlex.quote(side_label)} {seq} {shlex.quote(sync_mode)}\n"
+        "import json, pathlib, sys\n"
+        "run_dir=pathlib.Path(sys.argv[1]); side_name=sys.argv[2]; side_label=sys.argv[3]; seq=int(sys.argv[4]); sync_mode=sys.argv[5]\n"
+        "def read(p):\n    return p.read_text().strip() if p.exists() else None\n"
+        "def counts_from_report():\n"
+        "    p=run_dir/'report.json'\n"
+        "    if not p.exists():\n"
+        "        return {}\n"
+        "    try:\n"
+        "        report=json.load(open(p))\n"
+        "    except Exception:\n"
+        "        return {}\n"
+        "    meta=report.get('metadata') if isinstance(report, dict) else None\n"
+        "    if isinstance(meta, dict):\n"
+        "        return {'vrps': int(meta.get('vrps') or 0), 'vaps': int(meta.get('vaps') or meta.get('aspas') or 0), 'publicationPoints': int(meta.get('repositories') or 0), 'warnings': 0}\n"
+        "    pps=report.get('publication_points', []) if isinstance(report, dict) else []\n"
+        "    tree=report.get('tree', {}) if isinstance(report, dict) else {}\n"
+        "    pp_warnings=sum(len(pp.get('warnings', [])) for pp in pps if isinstance(pp, dict)) if isinstance(pps, list) else 0\n"
+        "    return {'vrps': len(report.get('vrps', [])), 'vaps': len(report.get('aspas', [])), 'publicationPoints': len(pps) if isinstance(pps, list) else 0, 'warnings': len(tree.get('warnings', [])) + pp_warnings if isinstance(tree, dict) else pp_warnings}\n"
+        "meta={'sideName':side_name,'sideLabel':side_label,'seq':seq,'syncMode':sync_mode,'startedAt':read(run_dir/'started-at.txt'),'finishedAt':read(run_dir/'finished-at.txt'),'exitCode':int(read(run_dir/'exit-code.txt') or '1'),'counts':counts_from_report()}\n"
+        "json.dump(meta, open(run_dir/'remote-run-meta.json','w'), indent=2, sort_keys=True); print()\n"
+        "REMOTE_META"
+    )
+    return run_dir, command
+
+
+def run_remote_sample(ssh_target: str, remote_root: Path, side_name: str, side: dict[str, Any], side_label: str, seq: int, rirs: list[str]) -> Path:
+    run_dir, command = build_remote_command(remote_root, side_name, side, side_label, seq, rirs)
+    ssh_script(ssh_target, command)
+    return run_dir
+
+
+def append_remote_sequence_item(
+    ssh_target: str,
+    remote_root: Path,
+    side_name: str,
+    side_label: str,
+    seq: int,
+    run_dir: Path,
+    schedule_mode: str,
+) -> dict[str, Any]:
+    remote_exp_root = remote_root / "experiments" / "sequence"
+    seq_path = remote_exp_root / ("left-sequence.jsonl" if side_label == "A" else "right-sequence.jsonl")
+    side_value = "left" if side_label == "A" else "right"
+    script = f"""
+set -euo pipefail
+python3 - <<'REMOTE_SEQUENCE_ITEM' {shlex.quote(str(remote_exp_root))} {shlex.quote(str(seq_path))} {shlex.quote(str(run_dir))} {shlex.quote(str(remote_root / 'bin' / 'cir_dump_reject_list'))} {shlex.quote(side_name)} {shlex.quote(side_label)} {seq} {shlex.quote(side_value)} {shlex.quote(schedule_mode)}
+import hashlib, json, os, pathlib, subprocess, sys
+exp_root=pathlib.Path(sys.argv[1])
+seq_path=pathlib.Path(sys.argv[2])
+run_dir=pathlib.Path(sys.argv[3])
+cir_dump=pathlib.Path(sys.argv[4])
+side_name=sys.argv[5]
+side_label=sys.argv[6]
+seq=int(sys.argv[7])
+side_value=sys.argv[8]
+schedule_mode=sys.argv[9]
+def read(p):
+    return p.read_text().strip() if p.exists() else None
+def sha256_file(p):
+    h=hashlib.sha256()
+    with open(p, 'rb') as f:
+        for chunk in iter(lambda: f.read(1024 * 1024), b''):
+            h.update(chunk)
+    return h.hexdigest()
+def parse_elapsed_to_ms(raw):
+    raw=raw.strip()
+    if not raw:
+        return 0
+    if '-' in raw:
+        days, raw=raw.split('-', 1)
+    else:
+        days='0'
+    parts=raw.split(':')
+    if len(parts) == 3:
+        hours, minutes, seconds=parts
+    elif len(parts) == 2:
+        hours='0'; minutes, seconds=parts
+    else:
+        hours='0'; minutes='0'; seconds=parts[0]
+    return int(round((int(days)*86400 + int(hours)*3600 + int(minutes)*60 + float(seconds))*1000))
+def parse_time_file(p):
+    data={{}}
+    if not p.exists():
+        return data
+    for line in p.read_text(errors='replace').splitlines():
+        if 'Elapsed (wall clock) time' in line:
+            elapsed=line.rsplit('):', 1)[1] if '):' in line else line.rsplit(':', 1)[1]
+            data['wallMs']=parse_elapsed_to_ms(elapsed)
+        elif 'Maximum resident set size' in line:
+            try:
+                data['maxRssKb']=int(line.rsplit(':', 1)[1].strip())
+            except ValueError:
+                pass
+    return data
+def cir_counts(cir):
+    values={{}}
+    if not cir.exists():
+        return {{}}
+    result=subprocess.run([str(cir_dump), '--cir', str(cir), '--limit', '0'], text=True, capture_output=True, check=True)
+    for line in result.stdout.splitlines():
+        if '=' not in line:
+            continue
+        key, value=line.split('=', 1)
+        if key in ('object_count', 'trust_anchor_count', 'reject_count'):
+            values[key]=int(value)
+    return {{'cirObjectCount': values.get('object_count', 0), 'cirTrustAnchorCount': values.get('trust_anchor_count', 0), 'cirRejectCount': values.get('reject_count', 0)}}
+meta=json.load(open(run_dir/'remote-run-meta.json'))
+time_info=parse_time_file(run_dir/'process-time.txt')
+counts=dict(meta.get('counts') or {{}})
+ccr=run_dir/'result.ccr'
+cir=run_dir/'result.cir'
+if meta.get('exitCode') != 0:
+    raise SystemExit(f"remote run failed: exitCode={{meta.get('exitCode')}} runDir={{run_dir}}")
+missing=[str(path) for path in (ccr, cir) if not path.exists()]
+if missing:
+    raise SystemExit(f"remote run missing required artifact(s): {{missing}}")
+counts.update(cir_counts(cir))
+item={{
+    'schemaVersion': 1,
+    'rpId': side_name,
+    'side': side_value,
+    'seq': seq,
+    'runId': f'{{side_label}}-{{seq:04d}}',
+    'syncMode': 'snapshot' if seq == 1 else 'delta',
+    'status': 'success' if meta.get('exitCode') == 0 else 'failed',
+    'startTime': meta.get('startedAt'),
+    'finishTime': meta.get('finishedAt'),
+    'validationTime': None,
+    'ccrPath': os.path.relpath(ccr, exp_root),
+    'cirPath': os.path.relpath(cir, exp_root),
+    'ccrSha256': sha256_file(ccr) if ccr.exists() else None,
+    'cirSha256': sha256_file(cir) if cir.exists() else None,
+    'wallMs': time_info.get('wallMs'),
+    'maxRssKb': time_info.get('maxRssKb'),
+    'vrps': counts.get('vrps'),
+    'vaps': counts.get('vaps'),
+    'publicationPoints': counts.get('publicationPoints'),
+    'cirObjectCount': counts.get('cirObjectCount'),
+    'cirRejectCount': counts.get('cirRejectCount'),
+    'cirTrustAnchorCount': counts.get('cirTrustAnchorCount'),
+    'scheduleMode': schedule_mode,
+}}
+seq_path.parent.mkdir(parents=True, exist_ok=True)
+with open(seq_path, 'a', encoding='utf-8') as handle:
+    handle.write(json.dumps(item, sort_keys=True, ensure_ascii=False) + '\\n')
+print(json.dumps(item, sort_keys=True, ensure_ascii=False))
+REMOTE_SEQUENCE_ITEM
+"""
+    result = ssh_script(ssh_target, script, capture=True)
+    lines = [line for line in result.stdout.splitlines() if line.strip()]
+    if not lines:
+        raise SystemExit("remote sequence item append produced no output")
+    return json.loads(lines[-1])
+
+
+def cleanup_remote_run_nonessential(ssh_target: str, run_dir: Path) -> None:
+    keep = {
+        "result.ccr",
+        "result.cir",
+        "process-time.txt",
+        "remote-run-meta.json",
+        "exit-code.txt",
+        "started-at.txt",
+        "finished-at.txt",
+        "stdout.log",
+        "stderr.log",
+    }
+    keep_json = json.dumps(sorted(keep), ensure_ascii=False)
+    script = f"""
+set -euo pipefail
+python3 - <<'REMOTE_CLEAN' {shlex.quote(str(run_dir))} {shlex.quote(keep_json)}
+import json, pathlib, sys
+run_dir=pathlib.Path(sys.argv[1])
+keep=set(json.loads(sys.argv[2]))
+removed=0
+if run_dir.exists():
+    for path in run_dir.iterdir():
+        if path.name in keep or path.is_dir():
+            continue
+        try:
+            removed += path.stat().st_size
+            path.unlink()
+        except FileNotFoundError:
+            pass
+print(f'cleaned_nonessential_bytes={{removed}}')
+REMOTE_CLEAN
+"""
+    ssh_script(ssh_target, script)
+
+
+def cir_counts(cir_path: Path) -> dict[str, int]:
+    result = run_local([str(REPO_ROOT / "target" / "release" / "cir_dump_reject_list"), "--cir", str(cir_path), "--limit", "0"], capture=True)
+    values: dict[str, int] = {}
+    for line in result.stdout.splitlines():
+        if "=" not in line:
+            continue
+        key, value = line.split("=", 1)
+        if key in {"object_count", "trust_anchor_count", "reject_count"}:
+            values[key] = int(value)
+    return {
+        "cirObjectCount": values.get("object_count", 0),
+        "cirTrustAnchorCount": values.get("trust_anchor_count", 0),
+        "cirRejectCount": values.get("reject_count", 0),
+    }
+
+
+def report_counts(path: Path, rp_kind: str) -> dict[str, int]:
+    if not path.is_file():
+        return {}
+    report = load_json(path)
+    if rp_kind == "rpki-client":
+        meta = report.get("metadata", {})
+        return {
+            "vrps": int(meta.get("vrps", 0)),
+            "vaps": int(meta.get("vaps", 0) or meta.get("aspas", 0) or 0),
+            "publicationPoints": int(meta.get("repositories", 0)),
+            "warnings": 0,
+        }
+    pps = report.get("publication_points", [])
+    tree = report.get("tree", {})
+    return {
+        "vrps": len(report.get("vrps", [])),
+        "vaps": len(report.get("aspas", [])),
+        "publicationPoints": len(pps),
+        "warnings": len(tree.get("warnings", [])) + sum(len(pp.get("warnings", [])) for pp in pps if isinstance(pp, dict)),
+    }
+
+
+def build_sequence_item(local_root: Path, side_name: str, side_label: str, side: dict[str, Any], seq: int, run_dir: Path) -> dict[str, Any]:
+    ccr = run_dir / "result.ccr"
+    cir = run_dir / "result.cir"
+    meta = load_json(run_dir / "remote-run-meta.json")
+    time_info = parse_time_file(run_dir / "process-time.txt")
+    counts = dict(meta.get("counts") or {})
+    if not counts:
+        counts = report_counts(run_dir / "report.json", side["rpKind"])
+    counts.update(cir_counts(cir))
+    return {
+        "schemaVersion": 1,
+        "rpId": side_name,
+        "side": "left" if side_label == "A" else "right",
+        "seq": seq,
+        "runId": f"{side_label}-{seq:04d}",
+        "syncMode": "snapshot" if seq == 1 else "delta",
+        "status": "success" if meta.get("exitCode") == 0 else "failed",
+        "startTime": meta.get("startedAt"),
+        "finishTime": meta.get("finishedAt"),
+        "validationTime": None,
+        "ccrPath": ccr.relative_to(local_root).as_posix(),
+        "cirPath": cir.relative_to(local_root).as_posix(),
+        "ccrSha256": sha256_file(ccr),
+        "cirSha256": sha256_file(cir),
+        "wallMs": time_info.get("wallMs"),
+        "maxRssKb": time_info.get("maxRssKb"),
+        "vrps": counts.get("vrps"),
+        "vaps": counts.get("vaps"),
+        "publicationPoints": counts.get("publicationPoints"),
+        "cirObjectCount": counts.get("cirObjectCount"),
+        "cirRejectCount": counts.get("cirRejectCount"),
+        "cirTrustAnchorCount": counts.get("cirTrustAnchorCount"),
+    }
+
+
+def run_sequence_triage(local_exp_root: Path, args: argparse.Namespace) -> None:
+    compare_dir = local_exp_root / "sequence-triage"
+    run_local([
+        str(REPO_ROOT / "target" / "release" / "sequence_triage_ccr_cir"),
+        "--left-sequence", str(local_exp_root / "left-sequence.jsonl"),
+        "--right-sequence", str(local_exp_root / "right-sequence.jsonl"),
+        "--out-dir", str(compare_dir),
+        "--align-window-runs", str(args.align_window_runs),
+        "--align-window-secs", str(args.align_window_secs),
+        "--sample-limit", str(args.sample_limit),
+        "--timeline-sample-limit", str(args.timeline_sample_limit),
+    ])
+
+
+def run_sequence_triage_remote(ssh_target: str, remote_root: Path, args: argparse.Namespace) -> None:
+    remote_exp_root = remote_root / "experiments" / "sequence"
+    compare_dir = remote_exp_root / "sequence-triage"
+    time_path = remote_exp_root / "sequence-triage-time.txt"
+    command = " ".join(
+        shlex.quote(item)
+        for item in [
+            str(remote_root / "bin" / "sequence_triage_ccr_cir"),
+            "--left-sequence", str(remote_exp_root / "left-sequence.jsonl"),
+            "--right-sequence", str(remote_exp_root / "right-sequence.jsonl"),
+            "--out-dir", str(compare_dir),
+            "--align-window-runs", str(args.align_window_runs),
+            "--align-window-secs", str(args.align_window_secs),
+            "--sample-limit", str(args.sample_limit),
+            "--timeline-sample-limit", str(args.timeline_sample_limit),
+        ]
+    )
+    ssh_script(
+        ssh_target,
+        "set -euo pipefail; "
+        f"rm -rf {shlex.quote(str(compare_dir))} {shlex.quote(str(time_path))}; "
+        f"/usr/bin/time -v -o {shlex.quote(str(time_path))} -- {command}",
+    )
+
+
+def sync_side_to_analysis_remote(
+    source_ssh_target: str,
+    source_remote_root: Path,
+    analysis_ssh_target: str,
+    analysis_remote_root: Path,
+    side_label: str,
+) -> None:
+    source_exp_root = source_remote_root / "experiments" / "sequence"
+    analysis_exp_root = analysis_remote_root / "experiments" / "sequence"
+    sequence_name = "left-sequence.jsonl" if side_label == "A" else "right-sequence.jsonl"
+    if same_remote_location(source_ssh_target, source_exp_root, analysis_ssh_target, analysis_exp_root):
+        return
+    side_dir = source_exp_root / side_label
+    script = (
+        "set -euo pipefail; "
+        f"ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new {shlex.quote(analysis_ssh_target)} "
+        f"{shlex.quote('mkdir -p ' + shlex.quote(str(analysis_exp_root / side_label)))}; "
+        f"rsync -az --delete {shlex.quote(str(side_dir))}/ {shlex.quote(analysis_ssh_target)}:{shlex.quote(str(analysis_exp_root / side_label))}/; "
+        f"rsync -az {shlex.quote(str(source_exp_root / sequence_name))} "
+        f"{shlex.quote(analysis_ssh_target)}:{shlex.quote(str(analysis_exp_root / sequence_name))}"
+    )
+    ssh_script(source_ssh_target, script)
+
+
+def run_side_sequence(
+    args: argparse.Namespace,
+    ssh_target: str,
+    remote_root: Path,
+    local_exp_root: Path,
+    side_label: str,
+    side_name: str,
+    side: dict[str, Any],
+    seq_path: Path,
+    rirs: list[str],
+) -> list[dict[str, Any]]:
+    side_progress: list[dict[str, Any]] = []
+    for seq in range(1, args.samples_per_side + 1):
+        side_progress.append(
+            run_one_side_sample(args, ssh_target, remote_root, local_exp_root, side_label, side_name, side, seq_path, seq, rirs)
+        )
+    return side_progress
+
+
+def run_one_side_sample(
+    args: argparse.Namespace,
+    ssh_target: str,
+    remote_root: Path,
+    local_exp_root: Path,
+    side_label: str,
+    side_name: str,
+    side: dict[str, Any],
+    seq_path: Path,
+    seq: int,
+    rirs: list[str],
+) -> dict[str, Any]:
+    rir_label = ",".join(rirs)
+    print(
+        f"[run] {side_label} {side_name} seq={seq} rirs={rir_label} schedule={args.schedule_mode}",
+        flush=True,
+    )
+    remote_run_dir = run_remote_sample(ssh_target, remote_root, side_name, side, side_label, seq, rirs)
+    if args.remote_triage:
+        item = append_remote_sequence_item(
+            ssh_target,
+            remote_root,
+            side_name,
+            side_label,
+            seq,
+            remote_run_dir,
+            args.schedule_mode,
+        )
+        if args.cleanup_run_nonessential:
+            cleanup_remote_run_nonessential(ssh_target, remote_run_dir)
+    else:
+        local_run_dir = local_exp_root / side_label / f"run_{seq:04d}"
+        rsync_run_artifacts_from_remote(ssh_target, remote_run_dir, local_run_dir)
+        item = build_sequence_item(local_exp_root, side_name, side_label, side, seq, local_run_dir)
+        item["scheduleMode"] = args.schedule_mode
+    append_jsonl(seq_path, item)
+    print(
+        f"[done] {side_label} seq={seq} wallMs={item.get('wallMs')} vrps={item.get('vrps')} vaps={item.get('vaps')} objects={item.get('cirObjectCount')} rejects={item.get('cirRejectCount')}",
+        flush=True,
+    )
+    return item
+
+
+def run_experiment(args: argparse.Namespace) -> None:
+    if not args.skip_build:
+        build_tool_binaries()
+    rirs = parse_rirs(args.rirs)
+    left = side_config(args.left)
+    right = side_config(args.right)
+    run_root = Path(args.run_root).resolve()
+    remote_root = Path(args.remote_root)
+    left_ssh_target = args.left_ssh_target or args.ssh_target
+    right_ssh_target = args.right_ssh_target or args.ssh_target
+    analysis_ssh_target = args.analysis_ssh_target or left_ssh_target
+    left_remote_root = Path(args.left_remote_root or args.remote_root)
+    right_remote_root = Path(args.right_remote_root or args.remote_root)
+    analysis_remote_root = Path(args.analysis_remote_root or args.remote_root)
+    run_root.mkdir(parents=True, exist_ok=True)
+    write_json(run_root / "experiment-config.json", {
+        "schemaVersion": 1,
+        "generatedAtUtc": utc_stamp(),
+        "left": args.left,
+        "right": args.right,
+        "samplesPerSide": args.samples_per_side,
+        "rirs": rirs,
+        "scheduleMode": args.schedule_mode,
+        "remoteRoot": str(remote_root),
+        "leftSshTarget": left_ssh_target,
+        "rightSshTarget": right_ssh_target,
+        "analysisSshTarget": analysis_ssh_target,
+        "leftRemoteRoot": str(left_remote_root),
+        "rightRemoteRoot": str(right_remote_root),
+        "analysisRemoteRoot": str(analysis_remote_root),
+        "sshTarget": args.ssh_target,
+    })
+    if args.dry_run:
+        print(json.dumps(load_json(run_root / "experiment-config.json"), indent=2, ensure_ascii=False))
+        return
+    if args.triage_only:
+        run_sequence_triage(run_root / "experiments" / "sequence", args)
+        print(json.dumps({
+            "runRoot": str(run_root),
+            "triage": str(run_root / "experiments" / "sequence" / "sequence-triage" / "sequence-triage.json"),
+        }, indent=2))
+        return
+    prepared_remotes: dict[tuple[str, str], bool] = {}
+    prepare_remote_once(prepared_remotes, left_ssh_target, left_remote_root, needs_rpki_client=(left["rpKind"] == "rpki-client"))
+    prepare_remote_once(prepared_remotes, right_ssh_target, right_remote_root, needs_rpki_client=(right["rpKind"] == "rpki-client"))
+    prepare_remote_once(prepared_remotes, analysis_ssh_target, analysis_remote_root, needs_rpki_client=False)
+    local_exp_root = run_root / "experiments" / "sequence"
+    left_seq_path = local_exp_root / "left-sequence.jsonl"
+    right_seq_path = local_exp_root / "right-sequence.jsonl"
+    left_seq_path.unlink(missing_ok=True)
+    right_seq_path.unlink(missing_ok=True)
+    progress: list[dict[str, Any]] = []
+    if args.schedule_mode == "interleaved":
+        for seq in range(1, args.samples_per_side + 1):
+            for side_label, ssh_target, side_remote_root, side_name, side, seq_path in [
+                ("A", left_ssh_target, left_remote_root, args.left, left, left_seq_path),
+                ("B", right_ssh_target, right_remote_root, args.right, right, right_seq_path),
+            ]:
+                progress.append(
+                    run_one_side_sample(
+                        args,
+                        ssh_target,
+                        side_remote_root,
+                        local_exp_root,
+                        side_label,
+                        side_name,
+                        side,
+                        seq_path,
+                        seq,
+                        rirs,
+                    )
+                )
+    else:
+        with ThreadPoolExecutor(max_workers=2) as executor:
+            futures = [
+                executor.submit(run_side_sequence, args, left_ssh_target, left_remote_root, local_exp_root, "A", args.left, left, left_seq_path, rirs),
+                executor.submit(run_side_sequence, args, right_ssh_target, right_remote_root, local_exp_root, "B", args.right, right, right_seq_path, rirs),
+            ]
+            for future in as_completed(futures):
+                progress.extend(future.result())
+    progress.sort(key=lambda item: (str(item.get("side")), int(item.get("seq") or 0)))
+    write_json(local_exp_root / "run-progress.json", progress)
+    if args.remote_triage:
+        sync_side_to_analysis_remote(left_ssh_target, left_remote_root, analysis_ssh_target, analysis_remote_root, "A")
+        sync_side_to_analysis_remote(right_ssh_target, right_remote_root, analysis_ssh_target, analysis_remote_root, "B")
+        remote_exp_root = analysis_remote_root / "experiments" / "sequence"
+        remote_progress = json.dumps(progress, sort_keys=True, ensure_ascii=False)
+        ssh_script(
+            analysis_ssh_target,
+            "set -euo pipefail; "
+            f"cat > {shlex.quote(str(remote_exp_root / 'run-progress.json'))} <<'REMOTE_PROGRESS_JSON'\n"
+            f"{remote_progress}\n"
+            "REMOTE_PROGRESS_JSON\n",
+        )
+        run_sequence_triage_remote(analysis_ssh_target, analysis_remote_root, args)
+        if args.fetch_remote_analysis:
+            rsync_remote_analysis_from_remote(analysis_ssh_target, remote_exp_root, local_exp_root)
+    else:
+        run_sequence_triage(local_exp_root, args)
+    ssh_script(analysis_ssh_target, f"df -h /data / > {shlex.quote(str(analysis_remote_root / 'df-after.txt'))} 2>&1 || true; free -h > {shlex.quote(str(analysis_remote_root / 'free-after.txt'))} 2>&1 || true")
+    compare_dir = local_exp_root / "sequence-triage"
+    remote_compare_dir = analysis_remote_root / "experiments" / "sequence" / "sequence-triage"
+    print(json.dumps({
+        "runRoot": str(run_root),
+        "remoteRoot": str(remote_root),
+        "leftRemoteRoot": str(left_remote_root),
+        "rightRemoteRoot": str(right_remote_root),
+        "analysisRemoteRoot": str(analysis_remote_root),
+        "triage": str(compare_dir / "sequence-triage.json") if not args.remote_triage or args.fetch_remote_analysis else None,
+        "remoteTriage": str(remote_compare_dir / "sequence-triage.json") if args.remote_triage else None,
+    }, indent=2))
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Feature #043 all5 sequence triage experiment driver")
+    parser.add_argument("--run-root", required=True)
+    parser.add_argument("--remote-root", required=True)
+    parser.add_argument("--ssh-target", default=os.environ.get("SSH_TARGET", "root@47.251.56.108"))
+    parser.add_argument("--left-ssh-target")
+    parser.add_argument("--right-ssh-target")
+    parser.add_argument("--analysis-ssh-target")
+    parser.add_argument("--left-remote-root")
+    parser.add_argument("--right-remote-root")
+    parser.add_argument("--analysis-remote-root")
+    parser.add_argument("--left", default="ours-standard")
+    parser.add_argument("--right", default="rpki-client-standard")
+    parser.add_argument("--samples-per-side", type=int, default=3)
+    parser.add_argument("--rirs", default=",".join(DEFAULT_RIRS))
+    parser.add_argument("--schedule-mode", choices=["interleaved", "parallel"], default="interleaved")
+    parser.add_argument("--align-window-runs", type=int, default=2)
+    parser.add_argument("--align-window-secs", type=int, default=1800)
+    parser.add_argument("--sample-limit", type=int, default=200)
+    parser.add_argument("--timeline-sample-limit", type=int, default=0)
+    parser.add_argument("--dry-run", action="store_true")
+    parser.add_argument("--skip-build", action="store_true", help="reuse existing release binaries")
+    parser.add_argument("--triage-only", action="store_true", help="only rerun local sequence triage for an existing run root")
+    parser.add_argument("--remote-triage", action="store_true", help="keep CIR/CCR on remote, write sequence JSONL remotely, and run triage on remote")
+    parser.add_argument("--fetch-remote-analysis", action="store_true", help="when --remote-triage is set, fetch only small sequence/triage JSON outputs; never fetch CIR/CCR")
+    parser.add_argument("--cleanup-run-nonessential", action="store_true", help="after each successful remote sequence item, remove report/log/CSV files and keep only CIR/CCR/timing/meta")
+    args = parser.parse_args()
+    if args.samples_per_side < 2:
+        raise SystemExit("--samples-per-side must be >= 2")
+    run_experiment(args)
+
+
+if __name__ == "__main__":
+    main()

scripts/inter_rp/control_remote200_rp.sh

@@ -0,0 +1,265 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REMOTE_HOST="${REMOTE_HOST:-root@43.110.128.200}"
+REMOTE_CONFIG="${REMOTE_CONFIG:-/root/inter-rp-runners/inter-rp.env}"
+REMOTE_SCRIPTS_DIR="${REMOTE_SCRIPTS_DIR:-/root/inter-rp-runners/scripts}"
+DEFAULT_RETAIN_RUNS="${RETAIN_RUNS:-20}"
+DEFAULT_RSS_SAMPLE_MS="${RSS_SAMPLE_MS:-500}"
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  control_remote200_rp.sh status [rpki-client|routinator|all]
+  control_remote200_rp.sh start  <rpki-client|routinator>
+  control_remote200_rp.sh stop   <rpki-client|routinator|all>
+  control_remote200_rp.sh restart <rpki-client|routinator>
+
+Environment overrides:
+  REMOTE_HOST=root@43.110.128.200
+  REMOTE_CONFIG=/root/inter-rp-runners/inter-rp.env
+  REMOTE_SCRIPTS_DIR=/root/inter-rp-runners/scripts
+  RETAIN_RUNS=20
+  RSS_SAMPLE_MS=500
+
+Notes:
+  - start uses the remote run_single_rp_with_rss.sh loop and runs until stopped.
+  - stop only kills the selected RP loop branch and its current child processes.
+  - rpki-client and routinator are managed independently.
+USAGE
+}
+
+ACTION="${1:-status}"
+RP="${2:-all}"
+
+case "$ACTION" in
+  status|start|stop|restart) ;;
+  -h|--help|help)
+    usage
+    exit 0
+    ;;
+  *)
+    echo "unknown action: $ACTION" >&2
+    usage >&2
+    exit 2
+    ;;
+esac
+
+case "$RP" in
+  rpki-client|routinator|all) ;;
+  *)
+    echo "unknown RP: $RP" >&2
+    usage >&2
+    exit 2
+    ;;
+esac
+
+if [[ "$ACTION" == "start" || "$ACTION" == "restart" ]] && [[ "$RP" == "all" ]]; then
+  echo "start/restart requires one RP: rpki-client or routinator" >&2
+  exit 2
+fi
+
+ssh "$REMOTE_HOST" \
+  "REMOTE_CONFIG='$REMOTE_CONFIG' REMOTE_SCRIPTS_DIR='$REMOTE_SCRIPTS_DIR' RETAIN_RUNS='$DEFAULT_RETAIN_RUNS' RSS_SAMPLE_MS='$DEFAULT_RSS_SAMPLE_MS' ACTION='$ACTION' RP='$RP' bash -s" <<'REMOTE'
+set -euo pipefail
+
+load_config() {
+  if [[ -f "$REMOTE_CONFIG" ]]; then
+    # shellcheck disable=SC1090
+    source "$REMOTE_CONFIG"
+  fi
+  INTER_RP_ROOT="${INTER_RP_ROOT:-/var/lib/inter-rp-runners}"
+  RETAIN_RUNS="${RETAIN_RUNS:-20}"
+  RSS_SAMPLE_MS="${RSS_SAMPLE_MS:-500}"
+  ROUTINATOR_RUN_COMMAND="${ROUTINATOR_RUN_COMMAND:-/root/inter-rp-runners/scripts/run_routinator_once.sh}"
+  RPKI_CLIENT_RUN_COMMAND="${RPKI_CLIENT_RUN_COMMAND:-/root/inter-rp-runners/scripts/run_rpki_client_official_98_once.sh}"
+}
+
+rp_root_name() {
+  case "$1" in
+    routinator) echo "routinator" ;;
+    rpki-client) echo "rpki-client" ;;
+  esac
+}
+
+rp_pid_file() {
+  echo "$INTER_RP_ROOT/$(rp_root_name "$1").loop.pid"
+}
+
+rp_log_file() {
+  echo "$INTER_RP_ROOT/$(rp_root_name "$1").loop.log"
+}
+
+rp_command() {
+  case "$1" in
+    routinator) echo "$ROUTINATOR_RUN_COMMAND" ;;
+    rpki-client) echo "$RPKI_CLIENT_RUN_COMMAND" ;;
+  esac
+}
+
+rp_binary_pattern() {
+  case "$1" in
+    routinator) echo "/root/inter-rp-runners/bin/routinator|[[:space:]]routinator[[:space:]]" ;;
+    rpki-client) echo "rpki-client-official-9\\.8|[[:space:]]rpki-client[[:space:]]" ;;
+  esac
+}
+
+loop_pattern() {
+  local rp="$1"
+  echo "run_single_rp_with_rss\\.sh --rp $rp "
+}
+
+is_running_pid() {
+  local pid="${1:-}"
+  [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null
+}
+
+current_loop_pids() {
+  local rp="$1"
+  pgrep -af "$(loop_pattern "$rp")" 2>/dev/null | awk '{print $1}' || true
+}
+
+status_one() {
+  local rp="$1"
+  local pid_file
+  pid_file="$(rp_pid_file "$rp")"
+  local pid=""
+  [[ -f "$pid_file" ]] && pid="$(cat "$pid_file" 2>/dev/null || true)"
+  echo "== $rp =="
+  if is_running_pid "$pid"; then
+    echo "loop: running pid=$pid"
+  else
+    local detected
+    detected="$(current_loop_pids "$rp" | paste -sd, -)"
+    if [[ -n "$detected" ]]; then
+      echo "loop: running detected_pids=$detected"
+    else
+      echo "loop: stopped"
+    fi
+  fi
+  ps -eo pid,ppid,stat,etime,%cpu,%mem,rss,cmd --sort=-%cpu \
+    | grep -E "$(loop_pattern "$rp")|$(rp_binary_pattern "$rp")" \
+    | grep -v grep \
+    | head -20 || true
+  local latest="$INTER_RP_ROOT/$(rp_root_name "$rp")/latest/run-meta.json"
+  if [[ -f "$latest" ]]; then
+    python3 - "$latest" <<'PY'
+import json, sys
+path = sys.argv[1]
+with open(path, "r", encoding="utf-8") as handle:
+    meta = json.load(handle)
+print("latest:", "runSeq=", meta.get("runSeq"), "success=", meta.get("success"), "wallMs=", meta.get("wallMs"), "vrps=", meta.get("counts", {}).get("vrps"), "vaps=", meta.get("counts", {}).get("vaps"), "rssKb=", meta.get("maxRssKb", {}).get("aggregatePeak"))
+PY
+  else
+    echo "latest: none"
+  fi
+}
+
+start_one() {
+  local rp="$1"
+  local pid_file log_file root_name run_command
+  pid_file="$(rp_pid_file "$rp")"
+  log_file="$(rp_log_file "$rp")"
+  root_name="$(rp_root_name "$rp")"
+  run_command="$(rp_command "$rp")"
+  mkdir -p "$INTER_RP_ROOT/$root_name" "$INTER_RP_ROOT"
+  local existing=""
+  if [[ -f "$pid_file" ]]; then
+    existing="$(cat "$pid_file" 2>/dev/null || true)"
+  fi
+  if is_running_pid "$existing"; then
+    echo "$rp already running pid=$existing"
+    return 0
+  fi
+  local detected
+  detected="$(current_loop_pids "$rp" | head -1)"
+  if [[ -n "$detected" ]] && is_running_pid "$detected"; then
+    echo "$detected" >"$pid_file"
+    echo "$rp already running detected pid=$detected"
+    return 0
+  fi
+  nohup bash -lc '
+    set -euo pipefail
+    while true; do
+      "'"$REMOTE_SCRIPTS_DIR"'/run_single_rp_with_rss.sh" \
+        --rp "'"$rp"'" \
+        --root "'"$INTER_RP_ROOT/$root_name"'" \
+        --command "'"$run_command"'" \
+        --retain-runs "'"$RETAIN_RUNS"'" \
+        --sample-ms "'"$RSS_SAMPLE_MS"'" || true
+    done
+  ' >"$log_file" 2>&1 &
+  local pid="$!"
+  echo "$pid" >"$pid_file"
+  echo "$rp started pid=$pid log=$log_file"
+}
+
+stop_one() {
+  local rp="$1"
+  local candidates=()
+  local pid_file pid
+  pid_file="$(rp_pid_file "$rp")"
+  if [[ -f "$pid_file" ]]; then
+    pid="$(cat "$pid_file" 2>/dev/null || true)"
+    [[ -n "$pid" ]] && candidates+=("$pid")
+  fi
+  while read -r pid; do
+    [[ -n "$pid" ]] && candidates+=("$pid")
+  done < <(current_loop_pids "$rp")
+  while read -r pid; do
+    [[ -n "$pid" ]] && candidates+=("$pid")
+  done < <(pgrep -af "$(rp_binary_pattern "$rp")" 2>/dev/null | awk '{print $1}' || true)
+
+  if ((${#candidates[@]} == 0)); then
+    echo "$rp already stopped"
+    rm -f "$pid_file"
+    return 0
+  fi
+
+  local unique_pids
+  unique_pids="$(printf '%s\n' "${candidates[@]}" | awk '!seen[$0]++' | tr '\n' ' ')"
+  echo "stopping $rp pids=$unique_pids"
+  for pid in $unique_pids; do
+    kill -TERM "$pid" 2>/dev/null || true
+  done
+  sleep 3
+  for pid in $unique_pids; do
+    kill -KILL "$pid" 2>/dev/null || true
+  done
+  rm -f "$pid_file"
+  echo "$rp stopped"
+}
+
+load_config
+
+case "$ACTION:$RP" in
+  status:all)
+    date -Is
+    uptime
+    status_one routinator
+    status_one rpki-client
+    ;;
+  status:*)
+    date -Is
+    uptime
+    status_one "$RP"
+    ;;
+  start:*)
+    start_one "$RP"
+    status_one "$RP"
+    ;;
+  stop:all)
+    stop_one routinator
+    stop_one rpki-client
+    ;;
+  stop:*)
+    stop_one "$RP"
+    status_one "$RP"
+    ;;
+  restart:*)
+    stop_one "$RP"
+    start_one "$RP"
+    status_one "$RP"
+    ;;
+esac
+REMOTE

scripts/inter_rp/inter-rp.env.example

@@ -0,0 +1,26 @@
+# 远端200 runner 配置示例。
+# 该脚本只负责任务调度、RSS采样、run目录和run-meta.json；具体RP命令由下面两个变量提供。
+
+# 远端200上的运行根目录。
+# 建议把运行态数据放到 /var/lib，避免 rpki-client 降权后无法读写 /root。
+INTER_RP_ROOT=/var/lib/inter-rp-runners
+
+# -1 表示持续运行；正整数表示每个RP运行多少轮。
+MAX_RUNS=-1
+
+# 每轮结束后的等待秒数。
+RUN_INTERVAL_SECS=0
+
+# 每个RP保留最近多少轮。
+RETAIN_RUNS=20
+
+# RSS采样间隔，毫秒。
+RSS_SAMPLE_MS=500
+
+# Routinator 命令模板。命令运行时会自动导出 RUN_DIR/RP_ROOT/RUN_SEQ/RUN_ID/RP_NAME。
+# 命令必须把 vrps.csv/vaps.csv 写入 $RUN_DIR；Routinator 使用 --enable-aspa + JSON 输出转换出 VAP CSV。
+ROUTINATOR_RUN_COMMAND=/root/inter-rp-runners/scripts/run_routinator_once.sh
+
+# 官方 rpki-client 9.8 命令模板。
+# rpki-client 会降权运行，cache 和 output 必须预先放在可写目录；命令需复制 output/csv、output/rpki.ccr，并从 output/json 转换出 vaps.csv。
+RPKI_CLIENT_RUN_COMMAND=/root/inter-rp-runners/scripts/run_rpki_client_official_98_once.sh

scripts/inter_rp/inter_rp_ours_routinator_exporter.py

@@ -0,0 +1,472 @@
+#!/usr/bin/env python3
+import csv
+import datetime as dt
+import json
+import os
+import re
+import socket
+import sys
+import threading
+import time
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+RUN_ROOT = Path(os.environ.get("OURS_RUN_ROOT", "/root/ours-rp-continuous/portable-soak"))
+PEER_ROOT = Path(os.environ.get("PEER_ROOT", "/root/ours-rp-continuous/portable-soak/inter-rp-peers"))
+INSTANCE = os.environ.get("INTER_RP_INSTANCE", "remote200-inter-rp")
+LISTEN = os.environ.get("INTER_RP_LISTEN", "0.0.0.0:9557")
+SCAN_TTL = float(os.environ.get("INTER_RP_SCAN_TTL_SECONDS", "10"))
+
+_cache_lock = threading.Lock()
+_cache = {"deadline": 0.0, "metrics": "", "status": {}}
+_count_cache = {}
+_set_cache = {}
+
+def unix_now():
+    return time.time()
+
+def parse_rfc3339(value):
+    if not value:
+        return None
+    text = str(value).replace("Z", "+00:00")
+    try:
+        return dt.datetime.fromisoformat(text).timestamp()
+    except Exception:
+        return None
+
+def read_json(path):
+    with open(path, "r", encoding="utf-8") as handle:
+        return json.load(handle)
+
+def latest_ours_run():
+    runs = RUN_ROOT / "runs"
+    candidates = sorted(p for p in runs.glob("run_*") if (p / "run-summary.json").exists())
+    return candidates[-1] if candidates else None
+
+def unique_csv_set(path, cols):
+    if not path.exists():
+        return None
+    stat = path.stat()
+    key = (str(path), stat.st_mtime_ns, stat.st_size, cols)
+    if key in _set_cache:
+        return _set_cache[key]
+    seen = set()
+    with open(path, "r", encoding="utf-8", newline="") as handle:
+        reader = csv.reader(handle)
+        first = True
+        for row in reader:
+            if first:
+                first = False
+                continue
+            if not row:
+                continue
+            if len(row) < cols:
+                continue
+            seen.add(tuple(cell.strip() for cell in row[:cols]))
+    _set_cache.clear()
+    _set_cache[key] = seen
+    return seen
+
+def count_unique_csv(path, cols):
+    values = unique_csv_set(path, cols)
+    if values is None:
+        return None
+    return len(values)
+
+def metric_line(name, labels, value):
+    label_text = ",".join(f'{k}="{str(v).replace(chr(92), chr(92)+chr(92)).replace(chr(34), chr(92)+chr(34))}"' for k, v in labels.items())
+    return f"{name}{{{label_text}}} {value}\n"
+
+def bool_num(value):
+    return 1 if value else 0
+
+def sample_ours(now, errors):
+    sample = {"rp": "ours-rp", "present": False, "success": False, "max": {}, "errors": 0}
+    run_dir = latest_ours_run()
+    if run_dir is None:
+        sample["errors"] += 1
+        errors.append("ours-rp: no run-summary.json")
+        return sample
+    sample["present"] = True
+    summary_path = run_dir / "run-summary.json"
+    try:
+        summary = read_json(summary_path)
+    except Exception as exc:
+        sample["errors"] += 1
+        errors.append(f"ours-rp: read {summary_path}: {exc}")
+        return sample
+    sample["run_id"] = summary.get("runId") or run_dir.name
+    sample["run_seq"] = summary.get("runSeq") or int(run_dir.name.split("_")[-1])
+    sample["success"] = summary.get("status") == "success" and int(summary.get("exitCode", 0)) == 0
+    if summary.get("wallMs") is not None:
+        sample["wall"] = float(summary["wallMs"]) / 1000.0
+    finished = parse_rfc3339(summary.get("finishedAtRfc3339Utc"))
+    if finished is not None:
+        sample["finished"] = finished
+        sample["age"] = max(0.0, now - finished)
+    rss = summary.get("processMetrics", {}).get("maxRssKb")
+    if rss is not None:
+        sample["max"]["parent"] = int(rss) * 1024
+        sample["max"]["aggregate_peak"] = int(rss) * 1024
+    sample["vrp_set"] = unique_csv_set(run_dir / "vrps.csv", 3)
+    sample["vap_set"] = unique_csv_set(run_dir / "vaps.csv", 2)
+    sample["vrps"] = len(sample["vrp_set"]) if sample["vrp_set"] is not None else None
+    sample["vaps"] = len(sample["vap_set"]) if sample["vap_set"] is not None else None
+    return sample
+
+def sample_routinator(now, errors):
+    rp = "routinator"
+    latest = PEER_ROOT / rp / "latest"
+    sample = {"rp": rp, "present": False, "success": False, "max": {}, "errors": 0}
+    if not latest.exists():
+        sample["errors"] += 1
+        errors.append(f"routinator: missing latest directory: {latest}")
+        return sample
+    sample["present"] = True
+    meta_path = latest / "run-meta.json"
+    try:
+        meta = read_json(meta_path)
+    except Exception as exc:
+        sample["errors"] += 1
+        errors.append(f"routinator: read {meta_path}: {exc}")
+        return sample
+    sample["run_id"] = meta.get("runId")
+    sample["run_seq"] = meta.get("runSeq")
+    sample["success"] = bool(meta.get("success"))
+    if meta.get("wallMs") is not None:
+        sample["wall"] = float(meta["wallMs"]) / 1000.0
+    finished = parse_rfc3339(meta.get("finishedAtRfc3339Utc"))
+    if finished is not None:
+        sample["finished"] = finished
+        sample["age"] = max(0.0, now - finished)
+    max_rss = meta.get("maxRssKb", {})
+    for label, key in [("parent", "parent"), ("child_max", "childMax"), ("aggregate_peak", "aggregatePeak")]:
+        if max_rss.get(key) is not None:
+            sample["max"][label] = int(max_rss[key]) * 1024
+    sample["vrp_set"] = unique_csv_set(latest / "vrps.csv", 3)
+    sample["vap_set"] = unique_csv_set(latest / "vaps.csv", 2)
+    sample["vrps"] = len(sample["vrp_set"]) if sample["vrp_set"] is not None else None
+    sample["vaps"] = len(sample["vap_set"]) if sample["vap_set"] is not None else None
+    return sample
+
+
+def parse_prom_labels(label_text):
+    labels = {}
+    for key, value in re.findall(r'(\w+)="((?:[^"\\]|\\.)*)"', label_text):
+        labels[key] = value.replace(r'\"', '"').replace(r'\\', '\\')
+    return labels
+
+def parse_prometheus_samples(text, metric_name):
+    pattern = re.compile(r'^' + re.escape(metric_name) + r'\{([^}]*)\}\s+([-+0-9.eE]+)\s*$', re.MULTILINE)
+    for match in pattern.finditer(text):
+        try:
+            value = float(match.group(2))
+        except ValueError:
+            continue
+        yield parse_prom_labels(match.group(1)), value
+
+def canonical_object_type(value):
+    value = (value or "").strip().lower()
+    if value in ("manifest", "mft"):
+        return "mft"
+    if value == "crl":
+        return "crl"
+    if value in ("certificate", "cert", "ca_cert", "router_cert", "ee_cert", "crt"):
+        return "crt"
+    if value == "roa":
+        return "roa"
+    if value == "aspa":
+        return "aspa"
+    return None
+
+def empty_object_counts():
+    return {"mft": 0, "crl": 0, "crt": 0, "roa": 0, "aspa": 0}
+
+def add_object_count(counts_by_uri, uri, object_type, value):
+    canonical = canonical_object_type(object_type)
+    if not uri or canonical is None:
+        return
+    counts = counts_by_uri.setdefault(uri, empty_object_counts())
+    counts[canonical] = counts.get(canonical, 0) + int(value)
+
+def object_count_labels(counts_by_uri, uri):
+    counts = counts_by_uri.get(uri, empty_object_counts())
+    return {key: str(int(counts.get(key, 0))) for key in ["mft", "crl", "crt", "roa", "aspa"]}
+
+def load_ours_repo_sets(errors):
+    try:
+        import urllib.request
+        text = urllib.request.urlopen("http://127.0.0.1:9556/metrics", timeout=10).read().decode("utf-8", "replace")
+    except Exception as exc:
+        errors.append(f"ours-rp repo metrics: {exc}")
+        return {"total": set(), "available": set(), "failed": set(), "info": {}, "object_counts": {}}
+    info = {}
+    states = {}
+    object_counts = {}
+    for labels, value in parse_prometheus_samples(text, "ours_rp_repository_info"):
+        uri = labels.get("uri")
+        if uri:
+            info[uri] = labels
+    for labels, value in parse_prometheus_samples(text, "ours_rp_repository_terminal_state_publication_points"):
+        uri = None
+        repo_id = labels.get("repo_id")
+        for candidate_uri, candidate in info.items():
+            if candidate.get("repo_id") == repo_id:
+                uri = candidate_uri
+                break
+        if not uri:
+            continue
+        states.setdefault(uri, {})[labels.get("terminal_state", "unknown")] = value
+    for labels, value in parse_prometheus_samples(text, "ours_rp_repository_objects_by_type"):
+        uri = labels.get("uri")
+        if not uri:
+            repo_id = labels.get("repo_id")
+            for candidate_uri, candidate in info.items():
+                if candidate.get("repo_id") == repo_id:
+                    uri = candidate_uri
+                    break
+        add_object_count(object_counts, uri, labels.get("object_type"), value)
+    total = set(info)
+    available = set()
+    failed = set()
+    for uri in total:
+        repo_states = states.get(uri, {})
+        non_failed = sum(value for state, value in repo_states.items() if state != "failed_no_cache")
+        if non_failed > 0:
+            available.add(uri)
+        else:
+            failed.add(uri)
+    return {"total": total, "available": available, "failed": failed, "info": info, "object_counts": object_counts}
+
+def load_routinator_repo_sets(errors):
+    path = PEER_ROOT / "routinator" / "routinator-metrics.prom"
+    if not path.exists():
+        errors.append(f"routinator repo metrics: missing {path}")
+        return {"total": set(), "success": set(), "failed": set(), "duration": {}, "object_counts": {}}
+    try:
+        text = path.read_text(encoding="utf-8", errors="replace")
+    except Exception as exc:
+        errors.append(f"routinator repo metrics: read {path}: {exc}")
+        return {"total": set(), "success": set(), "failed": set(), "duration": {}, "object_counts": {}}
+    total = set()
+    success = set()
+    failed = set()
+    duration = {}
+    object_counts = {}
+    publication_point_states = {}
+    for metric in ["routinator_rrdp_status", "routinator_rsync_status"]:
+        for labels, value in parse_prometheus_samples(text, metric):
+            uri = labels.get("uri")
+            if not uri:
+                continue
+            total.add(uri)
+            ok = value in (0.0, 200.0, 304.0)
+            if ok:
+                success.add(uri)
+            else:
+                failed.add(uri)
+    for labels, value in parse_prometheus_samples(text, "routinator_repository_publication_points_total"):
+        uri = labels.get("uri")
+        state = labels.get("state", "unknown")
+        if not uri:
+            continue
+        total.add(uri)
+        publication_point_states.setdefault(uri, {})[state] = publication_point_states.setdefault(uri, {}).get(state, 0.0) + value
+    for uri, states in publication_point_states.items():
+        valid_count = states.get("valid", 0.0)
+        non_valid_count = sum(value for state, value in states.items() if state != "valid")
+        if valid_count > 0:
+            success.add(uri)
+        elif non_valid_count > 0:
+            failed.add(uri)
+    for metric in ["routinator_rrdp_duration", "routinator_rsync_duration"]:
+        for labels, value in parse_prometheus_samples(text, metric):
+            uri = labels.get("uri")
+            if uri:
+                duration[uri] = max(duration.get(uri, 0.0), value)
+    for labels, value in parse_prometheus_samples(text, "routinator_repository_objects_total"):
+        add_object_count(object_counts, labels.get("uri"), labels.get("type"), value)
+    failed = failed - success
+    return {"total": total, "success": success, "failed": failed, "duration": duration, "object_counts": object_counts}
+
+def emit_repo_diff_metrics(out, errors):
+    ours = load_ours_repo_sets(errors)
+    routinator = load_routinator_repo_sets(errors)
+    ours_ok = ours["available"]
+    rout_ok = routinator["success"]
+    only_ours = sorted(ours_ok - rout_ok)
+    only_routinator = sorted(rout_ok - ours_ok)
+    both = ours_ok & rout_ok
+    neither = (ours["total"] | routinator["total"]) - (ours_ok | rout_ok)
+    base = {"instance": INSTANCE, "left": "ours-rp", "right": "routinator"}
+    out.append(metric_line("inter_rp_repo_sync_total", {"instance": INSTANCE, "rp": "ours-rp", "state": "total"}, len(ours["total"])))
+    out.append(metric_line("inter_rp_repo_sync_total", {"instance": INSTANCE, "rp": "ours-rp", "state": "available"}, len(ours_ok)))
+    out.append(metric_line("inter_rp_repo_sync_total", {"instance": INSTANCE, "rp": "ours-rp", "state": "failed"}, len(ours["failed"])))
+    out.append(metric_line("inter_rp_repo_sync_total", {"instance": INSTANCE, "rp": "routinator", "state": "total"}, len(routinator["total"])))
+    out.append(metric_line("inter_rp_repo_sync_total", {"instance": INSTANCE, "rp": "routinator", "state": "available"}, len(rout_ok)))
+    out.append(metric_line("inter_rp_repo_sync_total", {"instance": INSTANCE, "rp": "routinator", "state": "failed"}, len(routinator["failed"])))
+    out.append(metric_line("inter_rp_repo_sync_overlap_total", {**base, "class": "both_available"}, len(both)))
+    out.append(metric_line("inter_rp_repo_sync_overlap_total", {**base, "class": "only_ours"}, len(only_ours)))
+    out.append(metric_line("inter_rp_repo_sync_overlap_total", {**base, "class": "only_routinator"}, len(only_routinator)))
+    out.append(metric_line("inter_rp_repo_sync_overlap_total", {**base, "class": "neither_available"}, len(neither)))
+    for diff_class, uris, counts_by_uri in [
+        ("only_ours", only_ours, ours["object_counts"]),
+        ("only_routinator", only_routinator, routinator["object_counts"]),
+    ]:
+        for rank, uri in enumerate(uris[:50], start=1):
+            labels = {
+                **base,
+                "class": diff_class,
+                "rank": rank,
+                "uri": uri,
+                **object_count_labels(counts_by_uri, uri),
+            }
+            if uri in routinator["duration"]:
+                labels["routinator_duration"] = f"{routinator['duration'][uri]:.3f}"
+            out.append(metric_line("inter_rp_repo_sync_diff_info", labels, 1))
+    return {
+        "oursTotal": len(ours["total"]),
+        "oursAvailable": len(ours_ok),
+        "routinatorTotal": len(routinator["total"]),
+        "routinatorAvailable": len(rout_ok),
+        "onlyOurs": only_ours[:50],
+        "onlyRoutinator": only_routinator[:50],
+        "bothAvailable": len(both),
+        "neitherAvailable": len(neither),
+    }
+
+def sync_metrics(now):
+    path = PEER_ROOT / "sync-status.json"
+    if not path.exists():
+        return {"present": False, "success": False, "message": f"missing {path}"}
+    try:
+        value = read_json(path)
+    except Exception as exc:
+        return {"present": True, "success": False, "message": str(exc)}
+    ts = parse_rfc3339(value.get("lastSyncAtRfc3339Utc"))
+    return {
+        "present": True,
+        "success": bool(value.get("success")),
+        "timestamp": ts,
+        "age": max(0.0, now - ts) if ts is not None else None,
+        "remote": value.get("remoteHost", ""),
+        "message": value.get("message", ""),
+    }
+
+def build_metrics():
+    now = unix_now()
+    errors = []
+    start = time.time()
+    samples = [sample_ours(now, errors), sample_routinator(now, errors)]
+    sync = sync_metrics(now)
+    if not sync.get("success"):
+        errors.append("sync: " + str(sync.get("message", "failed")))
+    out = []
+    out.append(metric_line("inter_rp_service_up", {"instance": INSTANCE}, 1))
+    out.append(metric_line("inter_rp_service_last_scan_timestamp_seconds", {"instance": INSTANCE}, now))
+    out.append(metric_line("inter_rp_service_last_scan_duration_seconds", {"instance": INSTANCE}, time.time() - start))
+    out.append(metric_line("inter_rp_service_last_reload_success", {"instance": INSTANCE}, bool_num(len(errors) == 0)))
+    out.append(metric_line("inter_rp_parse_errors", {"instance": INSTANCE}, len(errors)))
+    out.append(metric_line("inter_rp_sync_present", {"instance": INSTANCE}, bool_num(sync.get("present"))))
+    out.append(metric_line("inter_rp_sync_last_success", {"instance": INSTANCE}, bool_num(sync.get("success"))))
+    if sync.get("age") is not None:
+        out.append(metric_line("inter_rp_sync_age_seconds", {"instance": INSTANCE}, sync["age"]))
+    if sync.get("timestamp") is not None:
+        out.append(metric_line("inter_rp_sync_last_timestamp_seconds", {"instance": INSTANCE}, sync["timestamp"]))
+    by_rp = {s["rp"]: s for s in samples}
+    for s in samples:
+        labels = {"instance": INSTANCE, "rp": s["rp"]}
+        out.append(metric_line("inter_rp_run_present", labels, bool_num(s.get("present"))))
+        out.append(metric_line("inter_rp_run_success", labels, bool_num(s.get("success"))))
+        out.append(metric_line("inter_rp_sample_parse_errors", labels, s.get("errors", 0)))
+        if s.get("run_seq") is not None:
+            out.append(metric_line("inter_rp_run_seq", labels, s["run_seq"]))
+        if s.get("wall") is not None:
+            out.append(metric_line("inter_rp_run_wall_seconds", labels, s["wall"]))
+        if s.get("age") is not None:
+            out.append(metric_line("inter_rp_artifact_age_seconds", labels, s["age"]))
+        if s.get("vrps") is not None:
+            out.append(metric_line("inter_rp_vrps", labels, s["vrps"]))
+        if s.get("vaps") is not None:
+            out.append(metric_line("inter_rp_vaps", labels, s["vaps"]))
+        for kind, value in s.get("max", {}).items():
+            labels2 = dict(labels)
+            labels2["kind"] = kind
+            out.append(metric_line("inter_rp_run_max_rss_bytes", labels2, value))
+    ours, rout = by_rp.get("ours-rp", {}), by_rp.get("routinator", {})
+    if ours.get("vrp_set") is not None and rout.get("vrp_set") is not None:
+        only_ours = ours["vrp_set"] - rout["vrp_set"]
+        only_rout = rout["vrp_set"] - ours["vrp_set"]
+        total_diff = len(only_ours) + len(only_rout)
+        labels = {"instance": INSTANCE, "left": "ours-rp", "right": "routinator"}
+        out.append(metric_line("inter_rp_vrps_diff", labels, total_diff))
+        out.append(metric_line("inter_rp_vrps_diff_by_class", {**labels, "class": "total"}, total_diff))
+        out.append(metric_line("inter_rp_vrps_diff_by_class", {**labels, "class": "only_ours"}, len(only_ours)))
+        out.append(metric_line("inter_rp_vrps_diff_by_class", {**labels, "class": "only_routinator"}, len(only_rout)))
+    elif ours.get("vrps") is not None and rout.get("vrps") is not None:
+        labels = {"instance": INSTANCE, "left": "ours-rp", "right": "routinator"}
+        total_diff = abs(int(ours["vrps"]) - int(rout["vrps"]))
+        out.append(metric_line("inter_rp_vrps_diff", labels, total_diff))
+        out.append(metric_line("inter_rp_vrps_diff_by_class", {**labels, "class": "total"}, total_diff))
+    if ours.get("vap_set") is not None and rout.get("vap_set") is not None:
+        only_ours = ours["vap_set"] - rout["vap_set"]
+        only_rout = rout["vap_set"] - ours["vap_set"]
+        total_diff = len(only_ours) + len(only_rout)
+        labels = {"instance": INSTANCE, "left": "ours-rp", "right": "routinator"}
+        out.append(metric_line("inter_rp_vaps_diff", labels, total_diff))
+        out.append(metric_line("inter_rp_vaps_diff_by_class", {**labels, "class": "total"}, total_diff))
+        out.append(metric_line("inter_rp_vaps_diff_by_class", {**labels, "class": "only_ours"}, len(only_ours)))
+        out.append(metric_line("inter_rp_vaps_diff_by_class", {**labels, "class": "only_routinator"}, len(only_rout)))
+    elif ours.get("vaps") is not None and rout.get("vaps") is not None:
+        labels = {"instance": INSTANCE, "left": "ours-rp", "right": "routinator"}
+        total_diff = abs(int(ours["vaps"]) - int(rout["vaps"]))
+        out.append(metric_line("inter_rp_vaps_diff", labels, total_diff))
+        out.append(metric_line("inter_rp_vaps_diff_by_class", {**labels, "class": "total"}, total_diff))
+    repo_diff = emit_repo_diff_metrics(out, errors)
+    return "".join(out), {"errors": errors, "samples": samples, "sync": sync, "repoDiff": repo_diff}
+
+def get_metrics():
+    now = time.time()
+    with _cache_lock:
+        if _cache["metrics"] and _cache["deadline"] > now:
+            return _cache["metrics"]
+        metrics, status = build_metrics()
+        _cache["metrics"] = metrics
+        _cache["status"] = status
+        _cache["deadline"] = now + SCAN_TTL
+        return metrics
+
+class Handler(BaseHTTPRequestHandler):
+    def do_GET(self):
+        if self.path == "/metrics":
+            body = get_metrics().encode("utf-8")
+            self.send_response(200)
+            self.send_header("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+            self.send_header("Content-Length", str(len(body)))
+            self.end_headers()
+            self.wfile.write(body)
+            return
+        if self.path == "/status":
+            with _cache_lock:
+                if not _cache["status"] or _cache["deadline"] <= time.time():
+                    build_metrics()
+                body = json.dumps(_cache["status"], indent=2).encode("utf-8")
+            self.send_response(200)
+            self.send_header("Content-Type", "application/json")
+            self.send_header("Content-Length", str(len(body)))
+            self.end_headers()
+            self.wfile.write(body)
+            return
+        self.send_response(404)
+        self.end_headers()
+    def log_message(self, fmt, *args):
+        return
+
+def main():
+    host, port = LISTEN.rsplit(":", 1)
+    server = ThreadingHTTPServer((host, int(port)), Handler)
+    print(f"inter-rp ours+routinator exporter listen={LISTEN} instance={INSTANCE}", flush=True)
+    server.serve_forever()
+
+if __name__ == "__main__":
+    main()

scripts/inter_rp/run_inter_rp_metrics_sidecar.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+CONFIG_FILE="${INTER_RP_METRICS_CONFIG:-./inter-rp-metrics.env}"
+if [[ -f "$CONFIG_FILE" ]]; then
+  # shellcheck disable=SC1090
+  source "$CONFIG_FILE"
+fi
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+INTER_RP_METRICS_EXPORTER="${INTER_RP_METRICS_EXPORTER:-ours-routinator}"
+RPKI_INTER_RP_METRICS_BIN="${RPKI_INTER_RP_METRICS_BIN:-./bin/rpki_inter_rp_metrics}"
+OURS_ROUTINATOR_EXPORTER="${OURS_ROUTINATOR_EXPORTER:-$SCRIPT_DIR/inter_rp_ours_routinator_exporter.py}"
+OURS_RUN_ROOT="${OURS_RUN_ROOT:?OURS_RUN_ROOT is required}"
+PEER_ROOT="${PEER_ROOT:-/root/inter-rp-aggregator/synced-from-200}"
+LISTEN="${INTER_RP_METRICS_LISTEN:-0.0.0.0:9557}"
+POLL_SECS="${INTER_RP_METRICS_POLL_SECS:-30}"
+SCAN_TTL_SECONDS="${INTER_RP_SCAN_TTL_SECONDS:-20}"
+INSTANCE="${INTER_RP_METRICS_INSTANCE:-remote231-inter-rp}"
+LOG_DIR="${INTER_RP_METRICS_LOG_DIR:-./logs}"
+
+mkdir -p "$LOG_DIR"
+case "$INTER_RP_METRICS_EXPORTER" in
+  ours-routinator)
+    exec env \
+      OURS_RUN_ROOT="$OURS_RUN_ROOT" \
+      PEER_ROOT="$PEER_ROOT" \
+      INTER_RP_INSTANCE="$INSTANCE" \
+      INTER_RP_LISTEN="$LISTEN" \
+      INTER_RP_SCAN_TTL_SECONDS="$SCAN_TTL_SECONDS" \
+      "$OURS_ROUTINATOR_EXPORTER" \
+      >>"$LOG_DIR/inter-rp-metrics.log" 2>&1
+    ;;
+  rust-generic)
+    exec "$RPKI_INTER_RP_METRICS_BIN" \
+      --ours-run-root "$OURS_RUN_ROOT" \
+      --peer-root "$PEER_ROOT" \
+      --listen "$LISTEN" \
+      --poll-secs "$POLL_SECS" \
+      --instance "$INSTANCE" \
+      >>"$LOG_DIR/inter-rp-metrics.log" 2>&1
+    ;;
+  *)
+    echo "unknown INTER_RP_METRICS_EXPORTER: $INTER_RP_METRICS_EXPORTER" >&2
+    exit 2
+    ;;
+esac

scripts/inter_rp/run_remote200_rp_loops.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+CONFIG_FILE="${INTER_RP_CONFIG:-./inter-rp.env}"
+if [[ -f "$CONFIG_FILE" ]]; then
+  # shellcheck disable=SC1090
+  source "$CONFIG_FILE"
+fi
+
+INTER_RP_ROOT="${INTER_RP_ROOT:-/root/inter-rp-runners}"
+MAX_RUNS="${MAX_RUNS:--1}"
+RUN_INTERVAL_SECS="${RUN_INTERVAL_SECS:-0}"
+RETAIN_RUNS="${RETAIN_RUNS:-20}"
+RSS_SAMPLE_MS="${RSS_SAMPLE_MS:-500}"
+ROUTINATOR_RUN_COMMAND="${ROUTINATOR_RUN_COMMAND:-}"
+RPKI_CLIENT_RUN_COMMAND="${RPKI_CLIENT_RUN_COMMAND:-}"
+
+if [[ -z "$ROUTINATOR_RUN_COMMAND" || -z "$RPKI_CLIENT_RUN_COMMAND" ]]; then
+  cat >&2 <<'MSG'
+ROUTINATOR_RUN_COMMAND and RPKI_CLIENT_RUN_COMMAND are required.
+Each command runs with RUN_DIR, RP_ROOT, RUN_SEQ, RUN_ID and RP_NAME exported.
+MSG
+  exit 2
+fi
+
+mkdir -p "$INTER_RP_ROOT"
+
+run_loop() {
+  local rp_name="$1"
+  local run_command="$2"
+  local rp_root="$INTER_RP_ROOT/$rp_name"
+  local completed="0"
+  mkdir -p "$rp_root"
+  while true; do
+    if [[ "$MAX_RUNS" =~ ^[0-9]+$ ]] && (( completed >= MAX_RUNS )); then
+      break
+    fi
+    "$SCRIPT_DIR/run_single_rp_with_rss.sh" \
+      --rp "$rp_name" \
+      --root "$rp_root" \
+      --command "$run_command" \
+      --retain-runs "$RETAIN_RUNS" \
+      --sample-ms "$RSS_SAMPLE_MS" || true
+    completed=$((completed + 1))
+    if (( RUN_INTERVAL_SECS > 0 )); then
+      sleep "$RUN_INTERVAL_SECS"
+    fi
+  done
+}
+
+run_loop routinator "$ROUTINATOR_RUN_COMMAND" >"$INTER_RP_ROOT/routinator.loop.log" 2>&1 &
+ROUTINATOR_LOOP_PID=$!
+run_loop rpki-client "$RPKI_CLIENT_RUN_COMMAND" >"$INTER_RP_ROOT/rpki-client.loop.log" 2>&1 &
+RPKI_CLIENT_LOOP_PID=$!
+
+cat >"$INTER_RP_ROOT/runner-pids.env" <<PIDS
+ROUTINATOR_LOOP_PID=$ROUTINATOR_LOOP_PID
+RPKI_CLIENT_LOOP_PID=$RPKI_CLIENT_LOOP_PID
+PIDS
+
+wait "$ROUTINATOR_LOOP_PID" "$RPKI_CLIENT_LOOP_PID"

scripts/inter_rp/run_routinator_once.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+: "${RUN_DIR:?RUN_DIR is required}"
+: "${RP_ROOT:?RP_ROOT is required}"
+
+timeout 1800 /root/inter-rp-runners/bin/routinator \
+  --repository-dir "$RP_ROOT/repository" \
+  --no-rir-tals \
+  --extra-tals-dir /var/lib/inter-rp-runners/fixtures/tal \
+  --enable-aspa \
+  vrps --format json --output "$RUN_DIR/routinator-output.json"
+
+python3 - "$RUN_DIR/routinator-output.json" "$RUN_DIR/vrps.csv" "$RUN_DIR/vaps.csv" <<'PY'
+import csv
+import json
+import sys
+
+json_path, vrps_path, vaps_path = sys.argv[1:4]
+
+with open(json_path, "r", encoding="utf-8") as handle:
+    payload = json.load(handle)
+
+def asn_text(value):
+    if isinstance(value, int):
+        return f"AS{value}"
+    text = str(value)
+    return text if text.startswith("AS") else f"AS{text}"
+
+def asn_sort_key(value):
+    text = asn_text(value)
+    try:
+        return int(text[2:])
+    except ValueError:
+        return text
+
+with open(vrps_path, "w", encoding="utf-8", newline="") as handle:
+    writer = csv.writer(handle)
+    writer.writerow(["ASN", "IP Prefix", "Max Length", "Trust Anchor"])
+    for row in payload.get("roas", []):
+        writer.writerow([
+            asn_text(row.get("asn", "")),
+            row.get("prefix", ""),
+            row.get("maxLength", row.get("max_length", "")),
+            row.get("ta", row.get("trust_anchor", "")),
+        ])
+
+with open(vaps_path, "w", encoding="utf-8", newline="") as handle:
+    writer = csv.writer(handle)
+    writer.writerow(["Customer ASN", "Providers", "Trust Anchor"])
+    for row in payload.get("aspas", payload.get("aspa", [])):
+        providers = sorted({asn_text(provider) for provider in row.get("providers", [])}, key=asn_sort_key)
+        writer.writerow([
+            asn_text(row.get("customer", row.get("customer_asid", ""))),
+            ";".join(providers),
+            row.get("ta", row.get("trust_anchor", "unknown")),
+        ])
+PY
+
+if [[ "${KEEP_INTER_RP_RAW_JSON:-0}" != "1" ]]; then
+  rm -f "$RUN_DIR/routinator-output.json"
+fi

scripts/inter_rp/run_rpki_client_official_98_once.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+: "${RUN_DIR:?RUN_DIR is required}"
+: "${RP_ROOT:?RP_ROOT is required}"
+
+install -d -o nobody -g nogroup "$RP_ROOT/cache" "$RUN_DIR/output"
+chown nobody:nogroup "$RUN_DIR"
+
+timeout 1800 /root/inter-rp-runners/bin/rpki-client-official-9.8 \
+  -c \
+  -j \
+  -t /var/lib/inter-rp-runners/fixtures/tal/afrinic.tal \
+  -t /var/lib/inter-rp-runners/fixtures/tal/apnic-rfc7730-https.tal \
+  -t /var/lib/inter-rp-runners/fixtures/tal/arin.tal \
+  -t /var/lib/inter-rp-runners/fixtures/tal/lacnic.tal \
+  -t /var/lib/inter-rp-runners/fixtures/tal/ripe-ncc.tal \
+  -d "$RP_ROOT/cache" \
+  "$RUN_DIR/output"
+
+cp "$RUN_DIR/output/csv" "$RUN_DIR/vrps.csv"
+cp "$RUN_DIR/output/rpki.ccr" "$RUN_DIR/result.ccr"
+
+python3 - "$RUN_DIR/output/json" "$RUN_DIR/vaps.csv" <<'PY'
+import csv
+import json
+import sys
+
+json_path, vaps_path = sys.argv[1:3]
+
+with open(json_path, "r", encoding="utf-8") as handle:
+    payload = json.load(handle)
+
+def asn_text(value):
+    if isinstance(value, int):
+        return f"AS{value}"
+    text = str(value)
+    return text if text.startswith("AS") else f"AS{text}"
+
+def asn_sort_key(value):
+    text = asn_text(value)
+    try:
+        return int(text[2:])
+    except ValueError:
+        return text
+
+with open(vaps_path, "w", encoding="utf-8", newline="") as handle:
+    writer = csv.writer(handle)
+    writer.writerow(["Customer ASN", "Providers", "Trust Anchor"])
+    for row in payload.get("aspas", []):
+        providers = sorted({asn_text(provider) for provider in row.get("providers", [])}, key=asn_sort_key)
+        writer.writerow([
+            asn_text(row.get("customer_asid", row.get("customer", ""))),
+            ";".join(providers),
+            row.get("ta", row.get("trust_anchor", "unknown")),
+        ])
+PY
+
+rm -rf "$RUN_DIR/output"

scripts/inter_rp/run_single_rp_with_rss.sh

@@ -0,0 +1,266 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'USAGE'
+Usage: run_single_rp_with_rss.sh --rp <name> --root <path> --command <shell-command> [--retain-runs <n>] [--sample-ms <n>]
+
+The command runs with RUN_DIR, RP_ROOT, RUN_SEQ, RUN_ID, and RP_NAME exported.
+It must write artifacts into RUN_DIR. The wrapper writes run-meta.json and atomically updates latest.
+USAGE
+}
+
+RP_NAME=""
+RP_ROOT=""
+RUN_COMMAND=""
+RETAIN_RUNS="${RETAIN_RUNS:-20}"
+SAMPLE_MS="${RSS_SAMPLE_MS:-500}"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --rp)
+      RP_NAME="$2"
+      shift 2
+      ;;
+    --root)
+      RP_ROOT="$2"
+      shift 2
+      ;;
+    --command)
+      RUN_COMMAND="$2"
+      shift 2
+      ;;
+    --retain-runs)
+      RETAIN_RUNS="$2"
+      shift 2
+      ;;
+    --sample-ms)
+      SAMPLE_MS="$2"
+      shift 2
+      ;;
+    -h|--help)
+      usage
+      exit 0
+      ;;
+    *)
+      echo "unknown argument: $1" >&2
+      usage >&2
+      exit 2
+      ;;
+  esac
+done
+
+if [[ -z "$RP_NAME" || -z "$RP_ROOT" || -z "$RUN_COMMAND" ]]; then
+  usage >&2
+  exit 2
+fi
+
+mkdir -p "$RP_ROOT/runs" "$RP_ROOT/logs"
+
+next_seq() {
+  local max_seq="0"
+  local run_path
+  shopt -s nullglob
+  for run_path in "$RP_ROOT"/runs/run_*; do
+    local run_name="${run_path##*/}"
+    local seq="${run_name#run_}"
+    if [[ "$seq" =~ ^[0-9]+$ ]] && (( 10#$seq > max_seq )); then
+      max_seq=$((10#$seq))
+    fi
+  done
+  shopt -u nullglob
+  printf '%06d' $((max_seq + 1))
+}
+
+rss_kb_for_pid() {
+  local pid="$1"
+  awk '/^VmRSS:/ {print $2; found=1} END {if (!found) print 0}' "/proc/$pid/status" 2>/dev/null || echo 0
+}
+
+collect_related_pids() {
+  local root_pid="$1"
+  local process_group="$2"
+  local queue=("$root_pid")
+  local pid
+  declare -A seen=()
+
+  if [[ -n "$process_group" ]]; then
+    while read -r pid; do
+      [[ -n "$pid" ]] && queue+=("$pid")
+    done < <(pgrep -g "$process_group" 2>/dev/null || true)
+  fi
+
+  while ((${#queue[@]} > 0)); do
+    pid="${queue[0]}"
+    queue=("${queue[@]:1}")
+    [[ -z "$pid" || -n "${seen[$pid]:-}" ]] && continue
+    seen[$pid]=1
+    echo "$pid"
+    while read -r child_pid; do
+      [[ -n "$child_pid" ]] && queue+=("$child_pid")
+    done < <(pgrep -P "$pid" 2>/dev/null || true)
+  done
+}
+
+sum_related_rss() {
+  local process_group="$1"
+  local parent_pid="$2"
+  local total_rss="0"
+  local child_max_rss="0"
+  local pid
+  while read -r pid; do
+    [[ -z "$pid" ]] && continue
+    local rss
+    rss="$(rss_kb_for_pid "$pid")"
+    total_rss=$((total_rss + rss))
+    if [[ "$pid" != "$parent_pid" ]] && (( rss > child_max_rss )); then
+      child_max_rss="$rss"
+    fi
+  done < <(collect_related_pids "$parent_pid" "$process_group")
+  printf '%s %s
+' "$total_rss" "$child_max_rss"
+}
+
+count_csv_rows() {
+  local path="$1"
+  if [[ ! -f "$path" ]]; then
+    echo 0
+    return
+  fi
+  awk 'BEGIN {count=0} /^[[:space:]]*$/ {next} /^#/ {next} NR==1 {next} {count++} END {print count}' "$path"
+}
+
+RUN_SEQ="$(next_seq)"
+RUN_ID="run_${RUN_SEQ}"
+RUN_DIR="$RP_ROOT/runs/$RUN_ID"
+mkdir -p "$RUN_DIR"
+
+STARTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+START_EPOCH_MS="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+
+export RP_NAME RP_ROOT RUN_SEQ RUN_ID RUN_DIR RUN_COMMAND
+set +e
+setsid bash -lc "$RUN_COMMAND" >"$RUN_DIR/stdout.log" 2>"$RUN_DIR/stderr.log" &
+CHILD_PID=$!
+set -e
+sleep 0.05
+PROCESS_GROUP="$(ps -o pgid= -p "$CHILD_PID" 2>/dev/null | tr -d '[:space:]' || true)"
+if [[ -z "$PROCESS_GROUP" ]]; then
+  PROCESS_GROUP="$CHILD_PID"
+fi
+
+PARENT_MAX_RSS_KB="0"
+CHILD_MAX_RSS_KB="0"
+AGGREGATE_PEAK_RSS_KB="0"
+SAMPLE_INTERVAL_SECONDS="$(python3 - <<PY
+print(max(float($SAMPLE_MS) / 1000.0, 0.05))
+PY
+)"
+
+while kill -0 "$CHILD_PID" 2>/dev/null; do
+  parent_rss="$(rss_kb_for_pid "$CHILD_PID")"
+  read -r aggregate_rss child_rss < <(sum_related_rss "$PROCESS_GROUP" "$CHILD_PID")
+  if (( parent_rss > PARENT_MAX_RSS_KB )); then
+    PARENT_MAX_RSS_KB="$parent_rss"
+  fi
+  if (( child_rss > CHILD_MAX_RSS_KB )); then
+    CHILD_MAX_RSS_KB="$child_rss"
+  fi
+  if (( aggregate_rss > AGGREGATE_PEAK_RSS_KB )); then
+    AGGREGATE_PEAK_RSS_KB="$aggregate_rss"
+  fi
+  sleep "$SAMPLE_INTERVAL_SECONDS"
+done
+
+set +e
+wait "$CHILD_PID"
+EXIT_CODE=$?
+set -e
+
+FINISHED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+END_EPOCH_MS="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+WALL_MS=$((END_EPOCH_MS - START_EPOCH_MS))
+SUCCESS=false
+if [[ "$EXIT_CODE" == "0" ]]; then
+  SUCCESS=true
+fi
+
+VRPS_COUNT="$(count_csv_rows "$RUN_DIR/vrps.csv")"
+VAPS_COUNT="$(count_csv_rows "$RUN_DIR/vaps.csv")"
+
+export STARTED_AT FINISHED_AT WALL_MS EXIT_CODE SUCCESS
+export PARENT_MAX_RSS_KB CHILD_MAX_RSS_KB AGGREGATE_PEAK_RSS_KB SAMPLE_MS
+export VRPS_COUNT VAPS_COUNT
+export CCR_ARTIFACT_PATH=""
+export VRPS_ARTIFACT_PATH=""
+export VAPS_ARTIFACT_PATH=""
+if [[ -f "$RUN_DIR/result.ccr" ]]; then
+  CCR_ARTIFACT_PATH="result.ccr"
+fi
+if [[ -f "$RUN_DIR/vrps.csv" ]]; then
+  VRPS_ARTIFACT_PATH="vrps.csv"
+fi
+if [[ -f "$RUN_DIR/vaps.csv" ]]; then
+  VAPS_ARTIFACT_PATH="vaps.csv"
+fi
+
+python3 - <<'PY' >"$RUN_DIR/run-meta.json"
+import os
+import json, socket
+def optional(name):
+    value = os.environ.get(name, "")
+    return value if value else None
+meta = {
+    "schemaVersion": 1,
+    "rp": os.environ["RP_NAME"],
+    "runSeq": int(os.environ["RUN_SEQ"]),
+    "runId": os.environ["RUN_ID"],
+    "host": socket.gethostname(),
+    "command": os.environ["RUN_COMMAND"],
+    "startedAtRfc3339Utc": os.environ["STARTED_AT"],
+    "finishedAtRfc3339Utc": os.environ["FINISHED_AT"],
+    "wallMs": int(os.environ["WALL_MS"]),
+    "exitCode": int(os.environ["EXIT_CODE"]),
+    "success": os.environ["SUCCESS"] == "true",
+    "maxRssKb": {
+        "parent": int(os.environ["PARENT_MAX_RSS_KB"]),
+        "childMax": int(os.environ["CHILD_MAX_RSS_KB"]),
+        "aggregatePeak": int(os.environ["AGGREGATE_PEAK_RSS_KB"]),
+        "sampleIntervalMs": int(os.environ["SAMPLE_MS"]),
+    },
+    "artifacts": {
+        "vrpsCsv": optional("VRPS_ARTIFACT_PATH"),
+        "vapsCsv": optional("VAPS_ARTIFACT_PATH"),
+        "ccr": optional("CCR_ARTIFACT_PATH"),
+        "stdout": "stdout.log",
+        "stderr": "stderr.log",
+    },
+    "counts": {
+        "vrps": int(os.environ["VRPS_COUNT"]),
+        "vaps": int(os.environ["VAPS_COUNT"]),
+    },
+}
+print(json.dumps(meta, indent=2, ensure_ascii=False))
+PY
+
+ln -sfn "runs/$RUN_ID" "$RP_ROOT/latest.tmp"
+mv -Tf "$RP_ROOT/latest.tmp" "$RP_ROOT/latest"
+
+echo "$RUN_ID $RP_NAME exit=$EXIT_CODE wall_ms=$WALL_MS vrps=$VRPS_COUNT vaps=$VAPS_COUNT rss_kb=$AGGREGATE_PEAK_RSS_KB"
+
+if [[ "$RETAIN_RUNS" =~ ^[0-9]+$ ]] && (( RETAIN_RUNS > 0 )); then
+  mapfile -t old_runs < <(find "$RP_ROOT/runs" -maxdepth 1 -type d -name 'run_*' -printf '%f\n' | sort | head -n -"$RETAIN_RUNS" || true)
+  for old_run in "${old_runs[@]}"; do
+    rm -rf "$RP_ROOT/runs/$old_run"
+  done
+fi
+
+exit "$EXIT_CODE"

scripts/inter_rp/sync_remote200_to_231.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+CONFIG_FILE="${INTER_RP_SYNC_CONFIG:-./inter-rp-sync.env}"
+if [[ -f "$CONFIG_FILE" ]]; then
+  # shellcheck disable=SC1090
+  source "$CONFIG_FILE"
+fi
+
+REMOTE200="${REMOTE200:-root@43.110.128.200}"
+REMOTE200_ROOT="${REMOTE200_ROOT:-/root/inter-rp-runners}"
+PEER_ROOT="${PEER_ROOT:-/root/inter-rp-aggregator/synced-from-200}"
+SYNC_INTERVAL_SECS="${SYNC_INTERVAL_SECS:-60}"
+MAX_SYNCS="${MAX_SYNCS:--1}"
+RPS="${RPS:-routinator rpki-client}"
+
+mkdir -p "$PEER_ROOT"
+
+write_status() {
+  local success="$1"
+  local message="$2"
+  export SYNC_SUCCESS="$success"
+  export SYNC_MESSAGE="$message"
+  export SYNC_REMOTE200="$REMOTE200"
+  export SYNC_REMOTE200_ROOT="$REMOTE200_ROOT"
+  python3 - <<'PY' >"$PEER_ROOT/sync-status.json"
+import json, socket, datetime
+print(json.dumps({
+    "schemaVersion": 1,
+    "success": __import__("os").environ["SYNC_SUCCESS"] == "true",
+    "lastSyncAtRfc3339Utc": datetime.datetime.now(datetime.UTC).replace(microsecond=0).isoformat().replace('+00:00', 'Z'),
+    "remoteHost": __import__("os").environ["SYNC_REMOTE200"],
+    "remoteRoot": __import__("os").environ["SYNC_REMOTE200_ROOT"],
+    "localHost": socket.gethostname(),
+    "message": __import__("os").environ["SYNC_MESSAGE"],
+}, indent=2))
+PY
+}
+
+sync_once() {
+  local rp_name
+  local temp_root="$PEER_ROOT/.sync-tmp-$$"
+  rm -rf "$temp_root"
+  mkdir -p "$temp_root"
+  for rp_name in $RPS; do
+    mkdir -p "$temp_root/$rp_name"
+    if ! rsync -aL --delete \
+      --include='run-meta.json' \
+      --include='result.ccr' \
+      --include='vrps.csv' \
+      --include='vaps.csv' \
+      --include='stdout.log' \
+      --include='stderr.log' \
+      --exclude='*' \
+      "$REMOTE200:$REMOTE200_ROOT/$rp_name/latest/" "$temp_root/$rp_name/latest/"; then
+      echo "sync failed for $rp_name; clearing stale local latest" >&2
+      rm -rf "$PEER_ROOT/$rp_name/latest" "$PEER_ROOT/$rp_name/latest.next" "$PEER_ROOT/$rp_name/latest.prev"
+      return 1
+    fi
+    if [[ ! -d "$temp_root/$rp_name/latest" ]]; then
+      echo "missing synced latest directory for $rp_name" >&2
+      rm -rf "$PEER_ROOT/$rp_name/latest" "$PEER_ROOT/$rp_name/latest.next" "$PEER_ROOT/$rp_name/latest.prev"
+      return 1
+    fi
+    rm -rf "$PEER_ROOT/$rp_name/latest.next" "$PEER_ROOT/$rp_name/latest.prev"
+    mkdir -p "$PEER_ROOT/$rp_name"
+    mv "$temp_root/$rp_name/latest" "$PEER_ROOT/$rp_name/latest.next"
+    if [[ -e "$PEER_ROOT/$rp_name/latest" ]]; then
+      mv "$PEER_ROOT/$rp_name/latest" "$PEER_ROOT/$rp_name/latest.prev"
+    fi
+    mv "$PEER_ROOT/$rp_name/latest.next" "$PEER_ROOT/$rp_name/latest"
+    rm -rf "$PEER_ROOT/$rp_name/latest.prev"
+  done
+  rm -rf "$temp_root"
+}
+
+completed="0"
+while true; do
+  if sync_once; then
+    write_status true "ok"
+  else
+    rm -rf "$PEER_ROOT/.sync-tmp-$$"
+    write_status false "rsync failed"
+  fi
+  completed=$((completed + 1))
+  if [[ "$MAX_SYNCS" =~ ^[0-9]+$ ]] && (( completed >= MAX_SYNCS )); then
+    break
+  fi
+  sleep "$SYNC_INTERVAL_SECS"
+done

scripts/inter_rp/sync_routinator_from_remote231.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+set -euo pipefail
+REMOTE231="${REMOTE231:-root@47.251.127.231}"
+REMOTE_ROOT="${REMOTE_ROOT:-/var/lib/inter-rp-runners}"
+PEER_ROOT="${PEER_ROOT:-/root/ours-rp-continuous/portable-soak/inter-rp-peers}"
+SYNC_INTERVAL_SECS="${SYNC_INTERVAL_SECS:-60}"
+MAX_SYNCS="${MAX_SYNCS:--1}"
+LOG_PREFIX="[inter-rp-sync]"
+mkdir -p "$PEER_ROOT/routinator"
+write_status() {
+  local success="$1"
+  local message="$2"
+  env SYNC_SUCCESS="$success" SYNC_MESSAGE="$message" SYNC_REMOTE="$REMOTE231" SYNC_REMOTE_ROOT="$REMOTE_ROOT" python3 - "$PEER_ROOT/sync-status.json" <<'PY'
+import datetime, json, os, socket, sys
+path = sys.argv[1]
+payload = {
+    "schemaVersion": 1,
+    "success": os.environ["SYNC_SUCCESS"] == "true",
+    "lastSyncAtRfc3339Utc": datetime.datetime.now(datetime.UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
+    "remoteHost": os.environ["SYNC_REMOTE"],
+    "remoteRoot": os.environ["SYNC_REMOTE_ROOT"],
+    "localHost": socket.gethostname(),
+    "message": os.environ["SYNC_MESSAGE"],
+}
+with open(path, "w", encoding="utf-8") as handle:
+    json.dump(payload, handle, indent=2)
+    handle.write("\n")
+PY
+}
+sync_once() {
+  local tmp="$PEER_ROOT/.sync-routinator-$$"
+  rm -rf "$tmp"
+  mkdir -p "$tmp"
+  if ! rsync -aL --delete \
+    --include='run-meta.json' \
+    --include='result.ccr' \
+    --include='vrps.csv' \
+    --include='vaps.csv' \
+    --include='stdout.log' \
+    --include='stderr.log' \
+    --exclude='*' \
+    "$REMOTE231:$REMOTE_ROOT/routinator/latest/" "$tmp/latest/"; then
+    rm -rf "$tmp"
+    return 1
+  fi
+  if [[ ! -f "$tmp/latest/run-meta.json" ]]; then
+    rm -rf "$tmp"
+    return 2
+  fi
+  if ! ssh -o BatchMode=yes -o ConnectTimeout=8 "$REMOTE231"     "curl -fsS --max-time 20 http://127.0.0.1:9558/metrics" >"$tmp/routinator-metrics.prom"; then
+    rm -rf "$tmp"
+    return 3
+  fi
+  rm -rf "$PEER_ROOT/routinator/latest.next" "$PEER_ROOT/routinator/latest.prev" "$PEER_ROOT/routinator/routinator-metrics.prom.next"
+  mv "$tmp/routinator-metrics.prom" "$PEER_ROOT/routinator/routinator-metrics.prom.next"
+  mv "$PEER_ROOT/routinator/routinator-metrics.prom.next" "$PEER_ROOT/routinator/routinator-metrics.prom"
+  mv "$tmp/latest" "$PEER_ROOT/routinator/latest.next"
+  if [[ -e "$PEER_ROOT/routinator/latest" ]]; then
+    mv "$PEER_ROOT/routinator/latest" "$PEER_ROOT/routinator/latest.prev"
+  fi
+  mv "$PEER_ROOT/routinator/latest.next" "$PEER_ROOT/routinator/latest"
+  rm -rf "$PEER_ROOT/routinator/latest.prev" "$tmp"
+}
+completed=0
+while true; do
+  if sync_once; then
+    echo "$LOG_PREFIX $(date -u +%Y-%m-%dT%H:%M:%SZ) ok"
+    write_status true "ok"
+  else
+    code=$?
+    echo "$LOG_PREFIX $(date -u +%Y-%m-%dT%H:%M:%SZ) sync failed code=$code" >&2
+    write_status false "routinator rsync failed code=$code"
+  fi
+  completed=$((completed + 1))
+  if [[ "$MAX_SYNCS" =~ ^[0-9]+$ ]] && (( completed >= MAX_SYNCS )); then
+    break
+  fi
+  sleep "$SYNC_INTERVAL_SECS"
+done

scripts/local_repo_replay/build_local_repo_replay_package.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  build_local_repo_replay_package.sh --out <path> [--tar-gz]
+
+Build a standalone local repository tree replay package for Routinator and
+rpki-client. The package does not include repository data and does not include
+materialize tooling.
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+SRC_DIR="$ROOT_DIR/scripts/local_repo_replay"
+OUT=""
+TAR_GZ=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --out) OUT="$2"; shift 2 ;;
+    --tar-gz) TAR_GZ=1; shift ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+[[ -n "$OUT" ]] || { usage >&2; exit 2; }
+
+if [[ "$TAR_GZ" -eq 1 ]]; then
+  PACKAGE_DIR="$(mktemp -d)"
+  TARGET_DIR="$PACKAGE_DIR/local-repo-replay-package"
+else
+  TARGET_DIR="$OUT"
+  rm -rf "$TARGET_DIR"
+fi
+
+mkdir -p "$TARGET_DIR/scripts" "$TARGET_DIR/docs" "$TARGET_DIR/examples"
+
+install -m 0755 "$SRC_DIR/run_routinator_from_local_tree.sh" "$TARGET_DIR/scripts/"
+install -m 0755 "$SRC_DIR/run_rpki_client_from_local_tree.sh" "$TARGET_DIR/scripts/"
+install -m 0755 "$SRC_DIR/run_dual_local_tree_replay.sh" "$TARGET_DIR/scripts/"
+install -m 0755 "$SRC_DIR/prepare_tals.py" "$TARGET_DIR/scripts/"
+install -m 0755 "$SRC_DIR/normalize_rp_outputs.py" "$TARGET_DIR/scripts/"
+install -m 0755 "$SRC_DIR/compare_normalized_sets.py" "$TARGET_DIR/scripts/"
+install -m 0755 "$SRC_DIR/summarize_replay.py" "$TARGET_DIR/scripts/"
+install -m 0755 "$ROOT_DIR/scripts/cir/cir-rsync-wrapper" "$TARGET_DIR/scripts/"
+install -m 0755 "$ROOT_DIR/scripts/cir/cir-local-link-sync.py" "$TARGET_DIR/scripts/"
+install -m 0644 "$SRC_DIR/templates/README.md" "$TARGET_DIR/README.md"
+install -m 0644 "$SRC_DIR/templates/docs/input_tree_requirements.md" "$TARGET_DIR/docs/"
+install -m 0644 "$SRC_DIR/templates/docs/offline_replay_limits.md" "$TARGET_DIR/docs/"
+install -m 0644 "$SRC_DIR/templates/docs/output_files.md" "$TARGET_DIR/docs/"
+install -m 0755 "$SRC_DIR/templates/examples/routinator_example.sh" "$TARGET_DIR/examples/"
+install -m 0755 "$SRC_DIR/templates/examples/rpki_client_example.sh" "$TARGET_DIR/examples/"
+install -m 0755 "$SRC_DIR/templates/examples/dual_compare_example.sh" "$TARGET_DIR/examples/"
+cat > "$TARGET_DIR/env.example" <<'EOF'
+# 本地目录树 replay 示例配置。目录树由使用者提前准备，不包含在 package 中。
+TAL_DIR=/data/replay/tals
+MIRROR_ROOT=/data/replay/mirror
+ROUTINATOR_BIN=/opt/routinator/target/release/routinator
+RPKI_CLIENT_BIN=/opt/rpki-client/src/rpki-client
+RPKI_CLIENT_CACHE_DIR=/data/replay/work/rpki-client-cache
+VALIDATION_TIME=2026-05-23T00:00:00Z
+EOF
+
+if grep -R -E 'cir_materialize|repo-bytes\\.db|\\.cir' "$TARGET_DIR/scripts" >/dev/null; then
+  echo "package contains forbidden materialize/repo-bytes implementation references" >&2
+  exit 1
+fi
+
+if [[ "$TAR_GZ" -eq 1 ]]; then
+  mkdir -p "$(dirname "$OUT")"
+  tar -C "$PACKAGE_DIR" -czf "$OUT" local-repo-replay-package
+  rm -rf "$PACKAGE_DIR"
+  echo "$OUT"
+else
+  echo "$TARGET_DIR"
+fi

scripts/local_repo_replay/compare_normalized_sets.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+
+def read_set(path: Path) -> set[str]:
+    if not path.is_file():
+        return set()
+    return {line.strip() for line in path.read_text(encoding="utf-8", errors="replace").splitlines() if line.strip()}
+
+
+def compare(left: set[str], right: set[str]) -> dict[str, object]:
+    union = left | right
+    inter = left & right
+    return {
+        "left": len(left),
+        "right": len(right),
+        "intersection": len(inter),
+        "onlyLeft": len(left - right),
+        "onlyRight": len(right - left),
+        "jaccard": round(len(inter) / len(union), 8) if union else 1.0,
+        "onlyLeftSamples": sorted(left - right)[:20],
+        "onlyRightSamples": sorted(right - left)[:20],
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--left", type=Path, required=True)
+    parser.add_argument("--right", type=Path, required=True)
+    parser.add_argument("--left-name", default="left")
+    parser.add_argument("--right-name", default="right")
+    parser.add_argument("--out", type=Path, required=True)
+    args = parser.parse_args()
+    result = {
+        "leftName": args.left_name,
+        "rightName": args.right_name,
+        "vrps": compare(read_set(args.left / "vrps.normalized.txt"), read_set(args.right / "vrps.normalized.txt")),
+        "vaps": compare(read_set(args.left / "vaps.normalized.txt"), read_set(args.right / "vaps.normalized.txt")),
+    }
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    print(args.out)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/local_repo_replay/normalize_rp_outputs.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+from pathlib import Path
+from typing import Any
+
+
+def normalize_asn(value: Any) -> str:
+    text = str(value).strip().upper()
+    if text.startswith("AS"):
+        text = text[2:]
+    return f"AS{int(text)}"
+
+
+def write_set(path: Path, rows: set[str]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text("\n".join(sorted(rows)) + ("\n" if rows else ""), encoding="utf-8")
+
+
+def load_json(path: Path) -> Any:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def normalize_vrps_from_json(path: Path) -> set[str]:
+    data = load_json(path)
+    rows: set[str] = set()
+    objects: list[dict[str, Any]] = []
+    if isinstance(data, dict):
+        for key in ("roas", "routeOrigins", "valid_roas"):
+            value = data.get(key)
+            if isinstance(value, list):
+                objects.extend(item for item in value if isinstance(item, dict))
+    elif isinstance(data, list):
+        objects = [item for item in data if isinstance(item, dict)]
+    for item in objects:
+        asn = item.get("asn") or item.get("asID") or item.get("origin")
+        prefix = item.get("prefix")
+        max_length = item.get("maxLength") or item.get("max_length") or item.get("maxlen") or item.get("maxLengthPrefix")
+        if asn is None or prefix is None or max_length is None:
+            continue
+        rows.add(f"{normalize_asn(asn)}|{prefix}|{int(max_length)}")
+    return rows
+
+
+def normalize_vaps_from_json(path: Path) -> set[str]:
+    data = load_json(path)
+    rows: set[str] = set()
+    objects: list[dict[str, Any]] = []
+    if isinstance(data, dict):
+        for key in ("aspas", "aspaAssertions", "vaps"):
+            value = data.get(key)
+            if isinstance(value, list):
+                objects.extend(item for item in value if isinstance(item, dict))
+    elif isinstance(data, list):
+        objects = [item for item in data if isinstance(item, dict)]
+    for item in objects:
+        customer = (
+            item.get("customer")
+            or item.get("customer_asid")
+            or item.get("customerASID")
+            or item.get("customerAsid")
+            or item.get("customerAsn")
+        )
+        providers = item.get("providers") or item.get("provider_asns") or item.get("providerASNs") or []
+        if customer is None:
+            continue
+        provider_asns = [normalize_asn(provider) for provider in providers]
+        rows.add(f"{normalize_asn(customer)}|{','.join(sorted(set(provider_asns), key=lambda value: int(value[2:])))}")
+    return rows
+
+
+def normalize_vrps_from_csv(path: Path) -> set[str]:
+    rows: set[str] = set()
+    with path.open(newline="", encoding="utf-8", errors="replace") as handle:
+        for row in csv.DictReader(handle):
+            asn = row.get("ASN") or row.get("asn") or row.get("AS")
+            prefix = row.get("IP Prefix") or row.get("prefix") or row.get("Prefix")
+            max_length = row.get("Max Length") or row.get("maxLength") or row.get("max_length")
+            if asn and prefix and max_length:
+                rows.add(f"{normalize_asn(asn)}|{prefix}|{int(max_length)}")
+    return rows
+
+
+def normalize_routinator(input_path: Path) -> tuple[set[str], set[str]]:
+    return normalize_vrps_from_json(input_path), normalize_vaps_from_json(input_path)
+
+
+def normalize_rpki_client(input_path: Path) -> tuple[set[str], set[str]]:
+    json_path = input_path / "json" if input_path.is_dir() else input_path
+    csv_path = input_path / "csv" if input_path.is_dir() else input_path
+    vrps: set[str] = set()
+    vaps: set[str] = set()
+    if json_path.is_file():
+        vrps |= normalize_vrps_from_json(json_path)
+        vaps |= normalize_vaps_from_json(json_path)
+    if csv_path.is_file():
+        vrps |= normalize_vrps_from_csv(csv_path)
+    return vrps, vaps
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--kind", choices=["routinator", "rpki-client"], required=True)
+    parser.add_argument("--input", type=Path, required=True)
+    parser.add_argument("--vrps-out", type=Path, required=True)
+    parser.add_argument("--vaps-out", type=Path, required=True)
+    args = parser.parse_args()
+
+    if args.kind == "routinator":
+        vrps, vaps = normalize_routinator(args.input)
+    else:
+        vrps, vaps = normalize_rpki_client(args.input)
+    write_set(args.vrps_out, vrps)
+    write_set(args.vaps_out, vaps)
+    print(json.dumps({"vrps": len(vrps), "vaps": len(vaps)}, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/local_repo_replay/prepare_tals.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+
+def split_tal(text: str) -> tuple[list[str], list[str]]:
+    lines = text.splitlines()
+    for index, line in enumerate(lines):
+        if line.strip() == "":
+            return lines[:index], lines[index + 1 :]
+    return lines, []
+
+
+def rsync_only_tal(source: Path) -> tuple[str, dict[str, object]]:
+    text = source.read_text(encoding="utf-8", errors="replace")
+    uri_lines, key_lines = split_tal(text)
+    uris = [
+        line.strip()
+        for line in uri_lines
+        if line.strip() and not line.lstrip().startswith("#")
+    ]
+    rsync_uris = [
+        uri for uri in uris
+        if uri.lower().startswith("rsync://")
+    ]
+    if not rsync_uris:
+        return text if text.endswith("\n") else text + "\n", {
+            "file": source.name,
+            "mode": "unchanged_no_rsync_uri",
+            "uris": len(uris),
+            "rsyncUris": 0,
+        }
+    out = "\n".join(rsync_uris) + "\n\n" + "\n".join(key_lines).strip() + "\n"
+    return out, {
+        "file": source.name,
+        "mode": "rsync_only",
+        "uris": len(uris),
+        "rsyncUris": len(rsync_uris),
+    }
+
+
+def collect_tals(tal_dir: Path | None, tals: list[Path]) -> list[Path]:
+    paths: list[Path] = []
+    if tal_dir is not None:
+        paths.extend(sorted(tal_dir.glob("*.tal")))
+    paths.extend(tals)
+    unique: dict[str, Path] = {}
+    for path in paths:
+        unique[path.name] = path
+    return [unique[name] for name in sorted(unique)]
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Prepare TALs for local rsync-tree replay.")
+    parser.add_argument("--tal-dir", type=Path)
+    parser.add_argument("--tal", type=Path, action="append", default=[])
+    parser.add_argument("--out-dir", type=Path, required=True)
+    parser.add_argument("--summary", type=Path)
+    args = parser.parse_args()
+
+    sources = collect_tals(args.tal_dir, args.tal)
+    if not sources:
+        raise SystemExit("no TAL files provided")
+
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    rows = []
+    for source in sources:
+        if not source.is_file():
+            raise SystemExit(f"TAL file not found: {source}")
+        content, row = rsync_only_tal(source)
+        (args.out_dir / source.name).write_text(content, encoding="utf-8")
+        rows.append(row)
+
+    if args.summary is not None:
+        args.summary.parent.mkdir(parents=True, exist_ok=True)
+        args.summary.write_text(json.dumps(rows, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/local_repo_replay/run_dual_local_tree_replay.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  run_dual_local_tree_replay.sh \
+    --routinator-bin <path> --routinator-mirror-root <dir> \
+    --rpki-client-bin <path> --rpki-client-mirror-root <dir> \
+    --tal-dir <dir> --out-dir <dir> [--validation-time <RFC3339>]
+EOF
+}
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROUTINATOR_BIN=""
+ROUTINATOR_MIRROR_ROOT=""
+RPKI_CLIENT_BIN=""
+RPKI_CLIENT_MIRROR_ROOT=""
+RPKI_CLIENT_CACHE_DIR=""
+TAL_DIR=""
+OUT_DIR=""
+VALIDATION_TIME=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --routinator-bin) ROUTINATOR_BIN="$2"; shift 2 ;;
+    --routinator-mirror-root) ROUTINATOR_MIRROR_ROOT="$2"; shift 2 ;;
+    --rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
+    --rpki-client-mirror-root) RPKI_CLIENT_MIRROR_ROOT="$2"; shift 2 ;;
+    --rpki-client-cache-dir) RPKI_CLIENT_CACHE_DIR="$2"; shift 2 ;;
+    --tal-dir) TAL_DIR="$2"; shift 2 ;;
+    --out-dir) OUT_DIR="$2"; shift 2 ;;
+    --validation-time) VALIDATION_TIME="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+if [[ -z "$RPKI_CLIENT_MIRROR_ROOT" ]]; then
+  RPKI_CLIENT_MIRROR_ROOT="$ROUTINATOR_MIRROR_ROOT"
+fi
+[[ -n "$ROUTINATOR_BIN" && -n "$ROUTINATOR_MIRROR_ROOT" && -n "$RPKI_CLIENT_BIN" && -n "$RPKI_CLIENT_MIRROR_ROOT" && -n "$TAL_DIR" && -n "$OUT_DIR" ]] || { usage >&2; exit 2; }
+
+mkdir -p "$OUT_DIR"
+TIME_ARGS=()
+if [[ -n "$VALIDATION_TIME" ]]; then
+  TIME_ARGS=(--validation-time "$VALIDATION_TIME")
+fi
+CACHE_ARGS=()
+if [[ -n "$RPKI_CLIENT_CACHE_DIR" ]]; then
+  CACHE_ARGS=(--cache-dir "$RPKI_CLIENT_CACHE_DIR")
+fi
+
+"$SCRIPT_DIR/run_routinator_from_local_tree.sh" \
+  --routinator-bin "$ROUTINATOR_BIN" \
+  --mirror-root "$ROUTINATOR_MIRROR_ROOT" \
+  --tal-dir "$TAL_DIR" \
+  --out-dir "$OUT_DIR/routinator" \
+  --enable-aspa \
+  "${TIME_ARGS[@]}"
+
+"$SCRIPT_DIR/run_rpki_client_from_local_tree.sh" \
+  --rpki-client-bin "$RPKI_CLIENT_BIN" \
+  --mirror-root "$RPKI_CLIENT_MIRROR_ROOT" \
+  --tal-dir "$TAL_DIR" \
+  --out-dir "$OUT_DIR/rpki-client" \
+  "${CACHE_ARGS[@]}" \
+  "${TIME_ARGS[@]}"
+
+python3 "$SCRIPT_DIR/compare_normalized_sets.py" \
+  --left "$OUT_DIR/routinator" \
+  --right "$OUT_DIR/rpki-client" \
+  --left-name routinator \
+  --right-name rpki-client \
+  --out "$OUT_DIR/compare-summary.json"
+echo "done: $OUT_DIR"

scripts/local_repo_replay/run_routinator_from_local_tree.sh

@@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  run_routinator_from_local_tree.sh \
+    --routinator-bin <path> \
+    --mirror-root <local-rsync-mirror-root> \
+    --tal-dir <dir> | --tal <file>... \
+    --out-dir <path> \
+    [--validation-time <RFC3339>] \
+    [--real-rsync-bin <path>] \
+    [--enable-aspa]
+
+The input mirror is prepared by the caller. This script does not generate it.
+Pass --validation-time only if FAKETIME_LIB points to a working libfaketime
+library; otherwise Routinator will validate at wall-clock time.
+Example:
+  export FAKETIME_LIB=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1
+EOF
+}
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROUTINATOR_BIN=""
+MIRROR_ROOT=""
+OUT_DIR=""
+VALIDATION_TIME=""
+REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
+ENABLE_ASPA=0
+TAL_DIR=""
+TALS=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --routinator-bin) ROUTINATOR_BIN="$2"; shift 2 ;;
+    --mirror-root) MIRROR_ROOT="$2"; shift 2 ;;
+    --tal-dir) TAL_DIR="$2"; shift 2 ;;
+    --tal) TALS+=("$2"); shift 2 ;;
+    --out-dir) OUT_DIR="$2"; shift 2 ;;
+    --validation-time) VALIDATION_TIME="$2"; shift 2 ;;
+    --real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
+    --enable-aspa) ENABLE_ASPA=1; shift ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+[[ -n "$ROUTINATOR_BIN" && -n "$MIRROR_ROOT" && -n "$OUT_DIR" ]] || { usage >&2; exit 2; }
+[[ -x "$ROUTINATOR_BIN" ]] || { echo "routinator binary not executable: $ROUTINATOR_BIN" >&2; exit 2; }
+[[ -d "$MIRROR_ROOT" ]] || { echo "mirror root not found: $MIRROR_ROOT" >&2; exit 2; }
+if [[ -z "$TAL_DIR" && "${#TALS[@]}" -eq 0 ]]; then
+  echo "provide --tal-dir or at least one --tal" >&2
+  exit 2
+fi
+
+mkdir -p "$OUT_DIR"
+WORK_DIR="$OUT_DIR/work"
+REPO_DIR="$WORK_DIR/repository"
+EXTRA_TALS="$WORK_DIR/tals"
+CONFIG_FILE="$WORK_DIR/routinator.conf"
+rm -rf "$WORK_DIR"
+mkdir -p "$REPO_DIR" "$EXTRA_TALS"
+
+PREPARE_TAL_ARGS=(--out-dir "$EXTRA_TALS" --summary "$OUT_DIR/prepared-tals.json")
+if [[ -n "$TAL_DIR" ]]; then
+  PREPARE_TAL_ARGS+=(--tal-dir "$TAL_DIR")
+fi
+for tal in "${TALS[@]}"; do
+  PREPARE_TAL_ARGS+=(--tal "$tal")
+done
+python3 "$SCRIPT_DIR/prepare_tals.py" "${PREPARE_TAL_ARGS[@]}"
+
+cat > "$CONFIG_FILE" <<EOF
+repository-dir = "$REPO_DIR"
+no-rir-tals = true
+extra-tals-dir = "$EXTRA_TALS"
+disable-rrdp = true
+rrdp-fallback = "never"
+rsync-command = "$SCRIPT_DIR/cir-rsync-wrapper"
+EOF
+
+export CIR_MIRROR_ROOT="$(cd "$MIRROR_ROOT" && pwd)"
+export REAL_RSYNC_BIN="$REAL_RSYNC_BIN"
+export CIR_LOCAL_LINK_MODE=1
+export LOCAL_REPO_REPLAY_VALIDATION_TIME="$VALIDATION_TIME"
+export LOCAL_REPO_REPLAY_OUT_DIR="$OUT_DIR"
+
+ARGS=(
+  "$ROUTINATOR_BIN"
+  --config "$CONFIG_FILE"
+  --repository-dir "$REPO_DIR"
+  --disable-rrdp
+  --rrdp-fallback never
+  --rsync-command "$SCRIPT_DIR/cir-rsync-wrapper"
+  --no-rir-tals
+  --extra-tals-dir "$EXTRA_TALS"
+)
+if [[ "$ENABLE_ASPA" -eq 1 ]]; then
+  echo 'enable-aspa = true' >> "$CONFIG_FILE"
+  ARGS+=(--enable-aspa)
+fi
+if [[ -n "$VALIDATION_TIME" && -z "${FAKETIME_LIB:-}" ]]; then
+  echo "warning: --validation-time is ignored for Routinator because FAKETIME_LIB is not set" >&2
+fi
+VRP_ARGS=(vrps -o "$OUT_DIR/vrps.csv")
+JSON_ARGS=(vrps -f jsonext -o "$OUT_DIR/routinator.json")
+
+/usr/bin/time -v -o "$OUT_DIR/process-time.txt" bash -c '
+  set -euo pipefail
+  if [[ -n "$LOCAL_REPO_REPLAY_VALIDATION_TIME" && -n "${FAKETIME_LIB:-}" ]]; then
+    faketime_value="$(python3 - <<'"'"'PY'"'"' "$LOCAL_REPO_REPLAY_VALIDATION_TIME"
+from datetime import datetime, timezone
+import sys
+dt = datetime.fromisoformat(sys.argv[1].replace("Z", "+00:00")).astimezone(timezone.utc)
+print("@" + dt.strftime("%Y-%m-%d %H:%M:%S"))
+PY
+)"
+    export LD_PRELOAD="$FAKETIME_LIB"
+    export TZ=UTC
+    unset FAKETIME_FMT
+    export FAKETIME="$faketime_value"
+    export FAKETIME_DONT_FAKE_MONOTONIC=1
+  fi
+  "$@" update --complete >"$LOCAL_REPO_REPLAY_OUT_DIR/update.log" 2>&1 || true
+  "$@" vrps -o "$LOCAL_REPO_REPLAY_OUT_DIR/vrps.csv" >"$LOCAL_REPO_REPLAY_OUT_DIR/vrps.log" 2>&1
+  "$@" vrps -f jsonext -o "$LOCAL_REPO_REPLAY_OUT_DIR/routinator.json" >"$LOCAL_REPO_REPLAY_OUT_DIR/json.log" 2>&1
+' local_repo_replay "${ARGS[@]}"
+python3 "$SCRIPT_DIR/normalize_rp_outputs.py" --kind routinator --input "$OUT_DIR/routinator.json" --vrps-out "$OUT_DIR/vrps.normalized.txt" --vaps-out "$OUT_DIR/vaps.normalized.txt"
+python3 "$SCRIPT_DIR/summarize_replay.py" --rp routinator --out-dir "$OUT_DIR" --summary "$OUT_DIR/summary.json"
+echo "done: $OUT_DIR"

scripts/local_repo_replay/run_rpki_client_from_local_tree.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  run_rpki_client_from_local_tree.sh \
+    --rpki-client-bin <path> \
+    --mirror-root <local-rsync-mirror-root> \
+    --tal-dir <dir> | --tal <file>... \
+    --out-dir <path> \
+    [--cache-dir <work-cache-dir>] \
+    [--validation-time <RFC3339>] \
+    [--real-rsync-bin <path>] \
+    [--parser-workers <n>]
+
+The input mirror is prepared by the caller. This script disables RRDP, points
+rpki-client's rsync command at a local URI mapper, and fetches only from the
+local filesystem tree.
+EOF
+}
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+RPKI_CLIENT_BIN=""
+CACHE_DIR=""
+MIRROR_ROOT=""
+OUT_DIR=""
+TAL_DIR=""
+VALIDATION_TIME=""
+PARSER_WORKERS="${PARSER_WORKERS:-4}"
+REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
+TALS=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
+    --mirror-root) MIRROR_ROOT="$2"; shift 2 ;;
+    --cache-dir) CACHE_DIR="$2"; shift 2 ;;
+    --tal-dir) TAL_DIR="$2"; shift 2 ;;
+    --tal) TALS+=("$2"); shift 2 ;;
+    --out-dir) OUT_DIR="$2"; shift 2 ;;
+    --validation-time) VALIDATION_TIME="$2"; shift 2 ;;
+    --real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
+    --parser-workers) PARSER_WORKERS="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+[[ -n "$RPKI_CLIENT_BIN" && -n "$MIRROR_ROOT" && -n "$OUT_DIR" ]] || { usage >&2; exit 2; }
+[[ -x "$RPKI_CLIENT_BIN" ]] || { echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2; exit 2; }
+[[ -d "$MIRROR_ROOT" ]] || { echo "mirror root not found: $MIRROR_ROOT" >&2; exit 2; }
+if [[ -z "$TAL_DIR" && "${#TALS[@]}" -eq 0 ]]; then
+  echo "provide --tal-dir or at least one --tal" >&2
+  exit 2
+fi
+
+mkdir -p "$OUT_DIR"
+if [[ -z "$CACHE_DIR" ]]; then
+  CACHE_DIR="$OUT_DIR/cache"
+fi
+mkdir -p "$CACHE_DIR" "$OUT_DIR/out"
+chmod a+rwx "$OUT_DIR" "$CACHE_DIR" "$OUT_DIR/out"
+WORK_DIR="$OUT_DIR/work"
+PREPARED_TALS="$WORK_DIR/tals"
+rm -rf "$WORK_DIR"
+mkdir -p "$PREPARED_TALS"
+PREPARE_TAL_ARGS=(--out-dir "$PREPARED_TALS" --summary "$OUT_DIR/prepared-tals.json")
+if [[ -n "$TAL_DIR" ]]; then
+  PREPARE_TAL_ARGS+=(--tal-dir "$TAL_DIR")
+fi
+for tal in "${TALS[@]}"; do
+  PREPARE_TAL_ARGS+=(--tal "$tal")
+done
+python3 "$SCRIPT_DIR/prepare_tals.py" "${PREPARE_TAL_ARGS[@]}"
+
+CLIENT_TAL_ARGS=()
+while IFS= read -r tal; do
+  CLIENT_TAL_ARGS+=(-t "$tal")
+done < <(find "$PREPARED_TALS" -maxdepth 1 -type f -name '*.tal' | sort)
+
+TIME_ARGS=()
+if [[ -n "$VALIDATION_TIME" ]]; then
+  epoch="$(python3 - <<'PY' "$VALIDATION_TIME"
+from datetime import datetime, timezone
+import sys
+print(int(datetime.fromisoformat(sys.argv[1].replace("Z", "+00:00")).astimezone(timezone.utc).timestamp()))
+PY
+)"
+  TIME_ARGS=(-P "$epoch")
+fi
+
+export CIR_MIRROR_ROOT="$(cd "$MIRROR_ROOT" && pwd)"
+export REAL_RSYNC_BIN="$REAL_RSYNC_BIN"
+export CIR_LOCAL_LINK_MODE=1
+
+/usr/bin/time -v -o "$OUT_DIR/process-time.txt" \
+  "$RPKI_CLIENT_BIN" \
+  -R -e "$SCRIPT_DIR/cir-rsync-wrapper" -j -c -p "$PARSER_WORKERS" \
+  "${TIME_ARGS[@]}" \
+  "${CLIENT_TAL_ARGS[@]}" \
+  -d "$CACHE_DIR" \
+  "$OUT_DIR/out" >"$OUT_DIR/stdout.log" 2>"$OUT_DIR/stderr.log"
+
+python3 "$SCRIPT_DIR/normalize_rp_outputs.py" --kind rpki-client --input "$OUT_DIR/out" --vrps-out "$OUT_DIR/vrps.normalized.txt" --vaps-out "$OUT_DIR/vaps.normalized.txt"
+python3 "$SCRIPT_DIR/summarize_replay.py" --rp rpki-client --out-dir "$OUT_DIR" --summary "$OUT_DIR/summary.json"
+echo "done: $OUT_DIR"

scripts/local_repo_replay/summarize_replay.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from pathlib import Path
+
+
+def count_lines(path: Path) -> int:
+    if not path.is_file():
+        return 0
+    return sum(1 for line in path.read_text(encoding="utf-8", errors="replace").splitlines() if line.strip())
+
+
+def parse_time(path: Path) -> dict[str, object]:
+    if not path.is_file():
+        return {}
+    text = path.read_text(encoding="utf-8", errors="replace")
+    elapsed = re.search(r"Elapsed \(wall clock\) time .*: (.+)", text)
+    rss = re.search(r"Maximum resident set size \(kbytes\): (\d+)", text)
+    return {
+        "elapsed": elapsed.group(1).strip() if elapsed else "",
+        "maxRssKb": int(rss.group(1)) if rss else 0,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--rp", required=True)
+    parser.add_argument("--out-dir", type=Path, required=True)
+    parser.add_argument("--summary", type=Path, required=True)
+    args = parser.parse_args()
+    summary = {
+        "rp": args.rp,
+        "vrps": count_lines(args.out_dir / "vrps.normalized.txt"),
+        "vaps": count_lines(args.out_dir / "vaps.normalized.txt"),
+        "time": parse_time(args.out_dir / "process-time.txt"),
+        "artifacts": sorted(path.name for path in args.out_dir.iterdir() if path.is_file()),
+    }
+    args.summary.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    print(args.summary)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/local_repo_replay/templates/README.md

@@ -0,0 +1,118 @@
+# Local Repository Tree Replay Package
+
+This package replays already prepared local RPKI repository trees with
+Routinator and rpki-client.
+
+It is intentionally independent from CIR:
+
+- it does not read `.cir`;
+- it does not read `repo-bytes.db`;
+- it does not call `cir_materialize`;
+- it does not generate a local repository tree.
+
+The caller must prepare the local repository/cache tree before running these
+scripts.
+
+## Contents
+
+```text
+local-repo-replay-package/
+  scripts/
+    run_routinator_from_local_tree.sh
+    run_rpki_client_from_local_tree.sh
+    run_dual_local_tree_replay.sh
+    prepare_tals.py
+    cir-rsync-wrapper
+    cir-local-link-sync.py
+    normalize_rp_outputs.py
+    compare_normalized_sets.py
+    summarize_replay.py
+  docs/
+    input_tree_requirements.md
+    offline_replay_limits.md
+    output_files.md
+  examples/
+    routinator_example.sh
+    rpki_client_example.sh
+    dual_compare_example.sh
+  env.example
+```
+
+## Routinator replay
+
+```bash
+./scripts/run_routinator_from_local_tree.sh \
+  --routinator-bin /opt/routinator/target/release/routinator \
+  --mirror-root /data/replay/mirror \
+  --tal-dir /data/replay/tals \
+  --out-dir /data/replay/out/routinator \
+  --enable-aspa
+```
+
+The script uses `--disable-rrdp`, `--rsync-command ./scripts/cir-rsync-wrapper`,
+and the local mirror root to satisfy rsync fetches from the local filesystem.
+The wrapper name is historical; in this package it is only a generic
+`rsync://` to local-path mapper.
+
+If `--validation-time` is needed for Routinator, set `FAKETIME_LIB` to a working
+libfaketime shared library. Otherwise Routinator validates at wall-clock time.
+
+On Ubuntu, install and use faketime like this:
+
+```bash
+sudo apt-get install -y libfaketime
+export FAKETIME_LIB=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1
+./scripts/run_routinator_from_local_tree.sh \
+  --routinator-bin /opt/routinator/target/release/routinator \
+  --mirror-root /data/replay/mirror \
+  --tal-dir /data/replay/tals \
+  --out-dir /data/replay/out/routinator \
+  --validation-time 2026-05-14T06:48:00Z \
+  --enable-aspa
+```
+
+Without `FAKETIME_LIB`, old local trees can produce empty or smaller output
+because Routinator validates manifests and CRLs against current wall-clock time.
+
+## rpki-client replay
+
+```bash
+./scripts/run_rpki_client_from_local_tree.sh \
+  --rpki-client-bin /opt/rpki-client/src/rpki-client \
+  --mirror-root /data/replay/mirror \
+  --tal-dir /data/replay/tals \
+  --out-dir /data/replay/out/rpki-client \
+  --parser-workers 4
+```
+
+The script uses `rpki-client -R -e ./scripts/cir-rsync-wrapper` so RRDP is
+disabled and rsync fetches are served from the local mirror. `--cache-dir` is an
+optional working cache directory used by rpki-client during this local replay.
+
+## Dual replay
+
+```bash
+./scripts/run_dual_local_tree_replay.sh \
+  --routinator-bin /opt/routinator/target/release/routinator \
+  --routinator-mirror-root /data/replay/mirror \
+  --rpki-client-bin /opt/rpki-client/src/rpki-client \
+  --rpki-client-mirror-root /data/replay/mirror \
+  --tal-dir /data/replay/tals \
+  --out-dir /data/replay/out/dual
+```
+
+If `--validation-time` is passed to dual replay, remember to export
+`FAKETIME_LIB` first so Routinator and rpki-client use the same logical
+validation time.
+
+## Outputs
+
+Each run writes normalized output:
+
+- `vrps.normalized.txt`
+- `vaps.normalized.txt`
+- `summary.json`
+- raw RP output and logs
+- `process-time.txt`
+
+See `docs/output_files.md`.

scripts/local_repo_replay/templates/docs/input_tree_requirements.md

@@ -0,0 +1,39 @@
+# Input Tree Requirements
+
+The input tree is not part of this package. The caller must prepare it before
+running replay.
+
+## Mirror root
+
+The mirror root must map rsync URIs to local paths:
+
+```text
+rsync://rpki.example.net/repo/a/b/c.roa
+=> <mirror-root>/rpki.example.net/repo/a/b/c.roa
+```
+
+The tree must contain all objects needed by the selected TALs: TA certificates,
+manifests, CRLs, ROAs, ASPAs, router certs, and child CA certificates.
+
+Both Routinator and rpki-client scripts consume this same mirror root through a
+local rsync command wrapper.
+
+## rpki-client working cache
+
+For rpki-client replay, `--cache-dir` is only rpki-client's working cache
+directory for this local run. It is not the input dataset. The authoritative
+input is `--mirror-root`.
+
+## TALs
+
+Provide either `--tal-dir <dir>` or repeated `--tal <file>`.
+
+The scripts prepare a replay-local TAL copy that prefers `rsync://` TA
+certificate URIs. This prevents a TAL with an HTTPS URI listed first from
+escaping to the network during local replay. The TAL set should match the local
+tree. Mixing a tree from one run with different TALs may produce meaningless
+differences.
+
+## No generation
+
+This package does not generate the tree and does not repair missing objects.