20260605 移除audit rule index和DB trace

scripts/coverage.sh

@@ -27,7 +27,7 @@ cleanup() {
 }
 trap cleanup EXIT
 
-IGNORE_REGEX='src/bin/repository_view_stats\.rs|src/bin/trace_arin_missing_vrps\.rs|src/bin/db_stats\.rs|src/bin/rrdp_state_dump\.rs|src/bin/ccr_dump\.rs|src/bin/ccr_verify\.rs|src/bin/ccr_to_routinator_csv\.rs|src/bin/ccr_to_compare_views\.rs|src/bin/cir_materialize\.rs|src/bin/cir_extract_inputs\.rs|src/bin/cir_drop_report\.rs|src/bin/cir_ta_only_fixture\.rs|src/bin/cir_dump_reject_list\.rs|src/bin/rpki_object_parse\.rs|src/bin/triage_ccr_cir_pair\.rs|src/bin/rpki_artifact_metrics\.rs|src/bin/rpki_daemon\.rs|src/bin/sequence_triage_ccr_cir\.rs|src/tools/rpki_artifact_metrics\.rs|src/ccr/compare_view\.rs|src/progress_log\.rs|src/cli\.rs|src/validation/run_tree_from_tal\.rs|src/validation/tree_parallel\.rs|src/validation/tree_runner\.rs|src/validation/from_tal\.rs|src/sync/store_projection\.rs|src/sync/repo\.rs|src/sync/rrdp\.rs|src/storage\.rs|src/cir/materialize\.rs'
+IGNORE_REGEX='src/bin/repository_view_stats\.rs|src/bin/db_stats\.rs|src/bin/rrdp_state_dump\.rs|src/bin/ccr_dump\.rs|src/bin/ccr_verify\.rs|src/bin/ccr_to_routinator_csv\.rs|src/bin/ccr_to_compare_views\.rs|src/bin/cir_materialize\.rs|src/bin/cir_extract_inputs\.rs|src/bin/cir_drop_report\.rs|src/bin/cir_ta_only_fixture\.rs|src/bin/cir_dump_reject_list\.rs|src/bin/rpki_object_parse\.rs|src/bin/triage_ccr_cir_pair\.rs|src/bin/rpki_artifact_metrics\.rs|src/bin/rpki_daemon\.rs|src/bin/sequence_triage_ccr_cir\.rs|src/tools/rpki_artifact_metrics\.rs|src/ccr/compare_view\.rs|src/progress_log\.rs|src/cli\.rs|src/validation/run_tree_from_tal\.rs|src/validation/tree_parallel\.rs|src/validation/tree_runner\.rs|src/validation/from_tal\.rs|src/sync/store_projection\.rs|src/sync/repo\.rs|src/sync/rrdp\.rs|src/storage\.rs|src/cir/materialize\.rs'
 
 # Preserve colored output even though we post-process output by running under a pseudo-TTY.
 # We run tests only once, then generate both CLI text + HTML reports without rerunning tests.

scripts/experiments/feature043/run_sequence_triage_experiment.py

@@ -75,6 +75,30 @@ def rsync_run_artifacts_from_remote(target: str, source: str | Path, destination
         run_local([*rsync_base, f"{target}:{source}/{name}", f"{destination}/"])
 
 
+def rsync_remote_analysis_from_remote(target: str, remote_exp_root: str | Path, local_exp_root: Path) -> None:
+    local_exp_root.mkdir(parents=True, exist_ok=True)
+    rsync_base = ["rsync", "-az", "--ignore-missing-args", "--partial", "--partial-dir=.rsync-partial"]
+    for name in [
+        "left-sequence.jsonl",
+        "right-sequence.jsonl",
+        "run-progress.json",
+        "sequence-triage-time.txt",
+        "sequence-triage/sequence-triage.json",
+    ]:
+        destination = local_exp_root / Path(name).parent
+        destination.mkdir(parents=True, exist_ok=True)
+        run_local([*rsync_base, f"{target}:{remote_exp_root}/{name}", f"{destination}/"])
+
+
+def same_remote_location(
+    left_target: str,
+    left_root: str | Path,
+    right_target: str,
+    right_root: str | Path,
+) -> bool:
+    return left_target == right_target and str(left_root) == str(right_root)
+
+
 def load_json(path: Path) -> Any:
     with path.open("r", encoding="utf-8") as handle:
         return json.load(handle)
@@ -258,9 +282,31 @@ def prepare_remote(ssh_target: str, remote_root: Path, needs_rpki_client: bool)
         rsync_to_remote(ssh_target, detect_libtls_path(rpki_client_bin), remote_root / "lib" / "libtls.so.28")
 
 
+def prepare_remote_once(
+    prepared: dict[tuple[str, str], bool],
+    ssh_target: str,
+    remote_root: Path,
+    needs_rpki_client: bool,
+) -> None:
+    key = (ssh_target, str(remote_root))
+    already_has_rpki_client = prepared.get(key)
+    if already_has_rpki_client is not None and (already_has_rpki_client or not needs_rpki_client):
+        return
+    prepare_remote(ssh_target, remote_root, needs_rpki_client)
+    prepared[key] = bool(already_has_rpki_client or needs_rpki_client)
+
+
 def side_config(name: str) -> dict[str, Any]:
     if name == "ours-standard":
         return {"rpKind": "ours", "mode": "standard", "protocol": "rrdp+rsync", "rsyncScope": "module-root"}
+    if name == "ours-strict-all":
+        return {
+            "rpKind": "ours",
+            "mode": "strict-all",
+            "protocol": "rrdp+rsync",
+            "rsyncScope": "module-root",
+            "strictPolicies": "name,cms-der,signed-attrs",
+        }
     if name == "rpki-client-standard":
         return {"rpKind": "rpki-client", "mode": "standard", "protocol": "rrdp+rsync"}
     raise SystemExit(f"unknown side config: {name}")
@@ -305,6 +351,8 @@ def build_remote_command(remote_root: Path, side_name: str, side: dict[str, Any]
             "--repo-bytes-db", str(state_dir / "repo-bytes.db"),
             "--rsync-scope", side.get("rsyncScope", "module-root"),
         ]
+        if side.get("strictPolicies"):
+            argv.extend(["--strict", str(side["strictPolicies"])])
         for rir in rirs:
             argv.extend(["--tal-path", str(remote_root / "fixtures" / "tal" / fixture_name(rir, "tal"))])
             argv.extend(["--ta-path", str(remote_root / "fixtures" / "ta" / fixture_name(rir, "ta"))])
@@ -357,7 +405,22 @@ def build_remote_command(remote_root: Path, side_name: str, side: dict[str, Any]
         "import json, pathlib, sys\n"
         "run_dir=pathlib.Path(sys.argv[1]); side_name=sys.argv[2]; side_label=sys.argv[3]; seq=int(sys.argv[4]); sync_mode=sys.argv[5]\n"
         "def read(p):\n    return p.read_text().strip() if p.exists() else None\n"
-        "meta={'sideName':side_name,'sideLabel':side_label,'seq':seq,'syncMode':sync_mode,'startedAt':read(run_dir/'started-at.txt'),'finishedAt':read(run_dir/'finished-at.txt'),'exitCode':int(read(run_dir/'exit-code.txt') or '1')}\n"
+        "def counts_from_report():\n"
+        "    p=run_dir/'report.json'\n"
+        "    if not p.exists():\n"
+        "        return {}\n"
+        "    try:\n"
+        "        report=json.load(open(p))\n"
+        "    except Exception:\n"
+        "        return {}\n"
+        "    meta=report.get('metadata') if isinstance(report, dict) else None\n"
+        "    if isinstance(meta, dict):\n"
+        "        return {'vrps': int(meta.get('vrps') or 0), 'vaps': int(meta.get('vaps') or meta.get('aspas') or 0), 'publicationPoints': int(meta.get('repositories') or 0), 'warnings': 0}\n"
+        "    pps=report.get('publication_points', []) if isinstance(report, dict) else []\n"
+        "    tree=report.get('tree', {}) if isinstance(report, dict) else {}\n"
+        "    pp_warnings=sum(len(pp.get('warnings', [])) for pp in pps if isinstance(pp, dict)) if isinstance(pps, list) else 0\n"
+        "    return {'vrps': len(report.get('vrps', [])), 'vaps': len(report.get('aspas', [])), 'publicationPoints': len(pps) if isinstance(pps, list) else 0, 'warnings': len(tree.get('warnings', [])) + pp_warnings if isinstance(tree, dict) else pp_warnings}\n"
+        "meta={'sideName':side_name,'sideLabel':side_label,'seq':seq,'syncMode':sync_mode,'startedAt':read(run_dir/'started-at.txt'),'finishedAt':read(run_dir/'finished-at.txt'),'exitCode':int(read(run_dir/'exit-code.txt') or '1'),'counts':counts_from_report()}\n"
         "json.dump(meta, open(run_dir/'remote-run-meta.json','w'), indent=2, sort_keys=True); print()\n"
         "REMOTE_META"
     )
@@ -370,6 +433,165 @@ def run_remote_sample(ssh_target: str, remote_root: Path, side_name: str, side:
     return run_dir
 
 
+def append_remote_sequence_item(
+    ssh_target: str,
+    remote_root: Path,
+    side_name: str,
+    side_label: str,
+    seq: int,
+    run_dir: Path,
+    schedule_mode: str,
+) -> dict[str, Any]:
+    remote_exp_root = remote_root / "experiments" / "sequence"
+    seq_path = remote_exp_root / ("left-sequence.jsonl" if side_label == "A" else "right-sequence.jsonl")
+    side_value = "left" if side_label == "A" else "right"
+    script = f"""
+set -euo pipefail
+python3 - <<'REMOTE_SEQUENCE_ITEM' {shlex.quote(str(remote_exp_root))} {shlex.quote(str(seq_path))} {shlex.quote(str(run_dir))} {shlex.quote(str(remote_root / 'bin' / 'cir_dump_reject_list'))} {shlex.quote(side_name)} {shlex.quote(side_label)} {seq} {shlex.quote(side_value)} {shlex.quote(schedule_mode)}
+import hashlib, json, os, pathlib, subprocess, sys
+exp_root=pathlib.Path(sys.argv[1])
+seq_path=pathlib.Path(sys.argv[2])
+run_dir=pathlib.Path(sys.argv[3])
+cir_dump=pathlib.Path(sys.argv[4])
+side_name=sys.argv[5]
+side_label=sys.argv[6]
+seq=int(sys.argv[7])
+side_value=sys.argv[8]
+schedule_mode=sys.argv[9]
+def read(p):
+    return p.read_text().strip() if p.exists() else None
+def sha256_file(p):
+    h=hashlib.sha256()
+    with open(p, 'rb') as f:
+        for chunk in iter(lambda: f.read(1024 * 1024), b''):
+            h.update(chunk)
+    return h.hexdigest()
+def parse_elapsed_to_ms(raw):
+    raw=raw.strip()
+    if not raw:
+        return 0
+    if '-' in raw:
+        days, raw=raw.split('-', 1)
+    else:
+        days='0'
+    parts=raw.split(':')
+    if len(parts) == 3:
+        hours, minutes, seconds=parts
+    elif len(parts) == 2:
+        hours='0'; minutes, seconds=parts
+    else:
+        hours='0'; minutes='0'; seconds=parts[0]
+    return int(round((int(days)*86400 + int(hours)*3600 + int(minutes)*60 + float(seconds))*1000))
+def parse_time_file(p):
+    data={{}}
+    if not p.exists():
+        return data
+    for line in p.read_text(errors='replace').splitlines():
+        if 'Elapsed (wall clock) time' in line:
+            elapsed=line.rsplit('):', 1)[1] if '):' in line else line.rsplit(':', 1)[1]
+            data['wallMs']=parse_elapsed_to_ms(elapsed)
+        elif 'Maximum resident set size' in line:
+            try:
+                data['maxRssKb']=int(line.rsplit(':', 1)[1].strip())
+            except ValueError:
+                pass
+    return data
+def cir_counts(cir):
+    values={{}}
+    if not cir.exists():
+        return {{}}
+    result=subprocess.run([str(cir_dump), '--cir', str(cir), '--limit', '0'], text=True, capture_output=True, check=True)
+    for line in result.stdout.splitlines():
+        if '=' not in line:
+            continue
+        key, value=line.split('=', 1)
+        if key in ('object_count', 'trust_anchor_count', 'reject_count'):
+            values[key]=int(value)
+    return {{'cirObjectCount': values.get('object_count', 0), 'cirTrustAnchorCount': values.get('trust_anchor_count', 0), 'cirRejectCount': values.get('reject_count', 0)}}
+meta=json.load(open(run_dir/'remote-run-meta.json'))
+time_info=parse_time_file(run_dir/'process-time.txt')
+counts=dict(meta.get('counts') or {{}})
+ccr=run_dir/'result.ccr'
+cir=run_dir/'result.cir'
+if meta.get('exitCode') != 0:
+    raise SystemExit(f"remote run failed: exitCode={{meta.get('exitCode')}} runDir={{run_dir}}")
+missing=[str(path) for path in (ccr, cir) if not path.exists()]
+if missing:
+    raise SystemExit(f"remote run missing required artifact(s): {{missing}}")
+counts.update(cir_counts(cir))
+item={{
+    'schemaVersion': 1,
+    'rpId': side_name,
+    'side': side_value,
+    'seq': seq,
+    'runId': f'{{side_label}}-{{seq:04d}}',
+    'syncMode': 'snapshot' if seq == 1 else 'delta',
+    'status': 'success' if meta.get('exitCode') == 0 else 'failed',
+    'startTime': meta.get('startedAt'),
+    'finishTime': meta.get('finishedAt'),
+    'validationTime': None,
+    'ccrPath': os.path.relpath(ccr, exp_root),
+    'cirPath': os.path.relpath(cir, exp_root),
+    'ccrSha256': sha256_file(ccr) if ccr.exists() else None,
+    'cirSha256': sha256_file(cir) if cir.exists() else None,
+    'wallMs': time_info.get('wallMs'),
+    'maxRssKb': time_info.get('maxRssKb'),
+    'vrps': counts.get('vrps'),
+    'vaps': counts.get('vaps'),
+    'publicationPoints': counts.get('publicationPoints'),
+    'cirObjectCount': counts.get('cirObjectCount'),
+    'cirRejectCount': counts.get('cirRejectCount'),
+    'cirTrustAnchorCount': counts.get('cirTrustAnchorCount'),
+    'scheduleMode': schedule_mode,
+}}
+seq_path.parent.mkdir(parents=True, exist_ok=True)
+with open(seq_path, 'a', encoding='utf-8') as handle:
+    handle.write(json.dumps(item, sort_keys=True, ensure_ascii=False) + '\\n')
+print(json.dumps(item, sort_keys=True, ensure_ascii=False))
+REMOTE_SEQUENCE_ITEM
+"""
+    result = ssh_script(ssh_target, script, capture=True)
+    lines = [line for line in result.stdout.splitlines() if line.strip()]
+    if not lines:
+        raise SystemExit("remote sequence item append produced no output")
+    return json.loads(lines[-1])
+
+
+def cleanup_remote_run_nonessential(ssh_target: str, run_dir: Path) -> None:
+    keep = {
+        "result.ccr",
+        "result.cir",
+        "process-time.txt",
+        "remote-run-meta.json",
+        "exit-code.txt",
+        "started-at.txt",
+        "finished-at.txt",
+        "stdout.log",
+        "stderr.log",
+    }
+    keep_json = json.dumps(sorted(keep), ensure_ascii=False)
+    script = f"""
+set -euo pipefail
+python3 - <<'REMOTE_CLEAN' {shlex.quote(str(run_dir))} {shlex.quote(keep_json)}
+import json, pathlib, sys
+run_dir=pathlib.Path(sys.argv[1])
+keep=set(json.loads(sys.argv[2]))
+removed=0
+if run_dir.exists():
+    for path in run_dir.iterdir():
+        if path.name in keep or path.is_dir():
+            continue
+        try:
+            removed += path.stat().st_size
+            path.unlink()
+        except FileNotFoundError:
+            pass
+print(f'cleaned_nonessential_bytes={{removed}}')
+REMOTE_CLEAN
+"""
+    ssh_script(ssh_target, script)
+
+
 def cir_counts(cir_path: Path) -> dict[str, int]:
     result = run_local([str(REPO_ROOT / "target" / "release" / "cir_dump_reject_list"), "--cir", str(cir_path), "--limit", "0"], capture=True)
     values: dict[str, int] = {}
@@ -413,6 +635,8 @@ def build_sequence_item(local_root: Path, side_name: str, side_label: str, side:
     cir = run_dir / "result.cir"
     meta = load_json(run_dir / "remote-run-meta.json")
     time_info = parse_time_file(run_dir / "process-time.txt")
+    counts = dict(meta.get("counts") or {})
+    if not counts:
         counts = report_counts(run_dir / "report.json", side["rpKind"])
     counts.update(cir_counts(cir))
     return {
@@ -455,8 +679,58 @@ def run_sequence_triage(local_exp_root: Path, args: argparse.Namespace) -> None:
     ])
 
 
+def run_sequence_triage_remote(ssh_target: str, remote_root: Path, args: argparse.Namespace) -> None:
+    remote_exp_root = remote_root / "experiments" / "sequence"
+    compare_dir = remote_exp_root / "sequence-triage"
+    time_path = remote_exp_root / "sequence-triage-time.txt"
+    command = " ".join(
+        shlex.quote(item)
+        for item in [
+            str(remote_root / "bin" / "sequence_triage_ccr_cir"),
+            "--left-sequence", str(remote_exp_root / "left-sequence.jsonl"),
+            "--right-sequence", str(remote_exp_root / "right-sequence.jsonl"),
+            "--out-dir", str(compare_dir),
+            "--align-window-runs", str(args.align_window_runs),
+            "--align-window-secs", str(args.align_window_secs),
+            "--sample-limit", str(args.sample_limit),
+            "--timeline-sample-limit", str(args.timeline_sample_limit),
+        ]
+    )
+    ssh_script(
+        ssh_target,
+        "set -euo pipefail; "
+        f"rm -rf {shlex.quote(str(compare_dir))} {shlex.quote(str(time_path))}; "
+        f"/usr/bin/time -v -o {shlex.quote(str(time_path))} -- {command}",
+    )
+
+
+def sync_side_to_analysis_remote(
+    source_ssh_target: str,
+    source_remote_root: Path,
+    analysis_ssh_target: str,
+    analysis_remote_root: Path,
+    side_label: str,
+) -> None:
+    source_exp_root = source_remote_root / "experiments" / "sequence"
+    analysis_exp_root = analysis_remote_root / "experiments" / "sequence"
+    sequence_name = "left-sequence.jsonl" if side_label == "A" else "right-sequence.jsonl"
+    if same_remote_location(source_ssh_target, source_exp_root, analysis_ssh_target, analysis_exp_root):
+        return
+    side_dir = source_exp_root / side_label
+    script = (
+        "set -euo pipefail; "
+        f"ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new {shlex.quote(analysis_ssh_target)} "
+        f"{shlex.quote('mkdir -p ' + shlex.quote(str(analysis_exp_root / side_label)))}; "
+        f"rsync -az --delete {shlex.quote(str(side_dir))}/ {shlex.quote(analysis_ssh_target)}:{shlex.quote(str(analysis_exp_root / side_label))}/; "
+        f"rsync -az {shlex.quote(str(source_exp_root / sequence_name))} "
+        f"{shlex.quote(analysis_ssh_target)}:{shlex.quote(str(analysis_exp_root / sequence_name))}"
+    )
+    ssh_script(source_ssh_target, script)
+
+
 def run_side_sequence(
     args: argparse.Namespace,
+    ssh_target: str,
     remote_root: Path,
     local_exp_root: Path,
     side_label: str,
@@ -468,13 +742,14 @@ def run_side_sequence(
     side_progress: list[dict[str, Any]] = []
     for seq in range(1, args.samples_per_side + 1):
         side_progress.append(
-            run_one_side_sample(args, remote_root, local_exp_root, side_label, side_name, side, seq_path, seq, rirs)
+            run_one_side_sample(args, ssh_target, remote_root, local_exp_root, side_label, side_name, side, seq_path, seq, rirs)
         )
     return side_progress
 
 
 def run_one_side_sample(
     args: argparse.Namespace,
+    ssh_target: str,
     remote_root: Path,
     local_exp_root: Path,
     side_label: str,
@@ -489,9 +764,22 @@ def run_one_side_sample(
         f"[run] {side_label} {side_name} seq={seq} rirs={rir_label} schedule={args.schedule_mode}",
         flush=True,
     )
-    remote_run_dir = run_remote_sample(args.ssh_target, remote_root, side_name, side, side_label, seq, rirs)
+    remote_run_dir = run_remote_sample(ssh_target, remote_root, side_name, side, side_label, seq, rirs)
+    if args.remote_triage:
+        item = append_remote_sequence_item(
+            ssh_target,
+            remote_root,
+            side_name,
+            side_label,
+            seq,
+            remote_run_dir,
+            args.schedule_mode,
+        )
+        if args.cleanup_run_nonessential:
+            cleanup_remote_run_nonessential(ssh_target, remote_run_dir)
+    else:
         local_run_dir = local_exp_root / side_label / f"run_{seq:04d}"
-    rsync_run_artifacts_from_remote(args.ssh_target, remote_run_dir, local_run_dir)
+        rsync_run_artifacts_from_remote(ssh_target, remote_run_dir, local_run_dir)
         item = build_sequence_item(local_exp_root, side_name, side_label, side, seq, local_run_dir)
         item["scheduleMode"] = args.schedule_mode
     append_jsonl(seq_path, item)
@@ -510,6 +798,12 @@ def run_experiment(args: argparse.Namespace) -> None:
     right = side_config(args.right)
     run_root = Path(args.run_root).resolve()
     remote_root = Path(args.remote_root)
+    left_ssh_target = args.left_ssh_target or args.ssh_target
+    right_ssh_target = args.right_ssh_target or args.ssh_target
+    analysis_ssh_target = args.analysis_ssh_target or left_ssh_target
+    left_remote_root = Path(args.left_remote_root or args.remote_root)
+    right_remote_root = Path(args.right_remote_root or args.remote_root)
+    analysis_remote_root = Path(args.analysis_remote_root or args.remote_root)
     run_root.mkdir(parents=True, exist_ok=True)
     write_json(run_root / "experiment-config.json", {
         "schemaVersion": 1,
@@ -520,6 +814,12 @@ def run_experiment(args: argparse.Namespace) -> None:
         "rirs": rirs,
         "scheduleMode": args.schedule_mode,
         "remoteRoot": str(remote_root),
+        "leftSshTarget": left_ssh_target,
+        "rightSshTarget": right_ssh_target,
+        "analysisSshTarget": analysis_ssh_target,
+        "leftRemoteRoot": str(left_remote_root),
+        "rightRemoteRoot": str(right_remote_root),
+        "analysisRemoteRoot": str(analysis_remote_root),
         "sshTarget": args.ssh_target,
     })
     if args.dry_run:
@@ -532,7 +832,10 @@ def run_experiment(args: argparse.Namespace) -> None:
             "triage": str(run_root / "experiments" / "sequence" / "sequence-triage" / "sequence-triage.json"),
         }, indent=2))
         return
-    prepare_remote(args.ssh_target, remote_root, needs_rpki_client=(left["rpKind"] == "rpki-client" or right["rpKind"] == "rpki-client"))
+    prepared_remotes: dict[tuple[str, str], bool] = {}
+    prepare_remote_once(prepared_remotes, left_ssh_target, left_remote_root, needs_rpki_client=(left["rpKind"] == "rpki-client"))
+    prepare_remote_once(prepared_remotes, right_ssh_target, right_remote_root, needs_rpki_client=(right["rpKind"] == "rpki-client"))
+    prepare_remote_once(prepared_remotes, analysis_ssh_target, analysis_remote_root, needs_rpki_client=False)
     local_exp_root = run_root / "experiments" / "sequence"
     left_seq_path = local_exp_root / "left-sequence.jsonl"
     right_seq_path = local_exp_root / "right-sequence.jsonl"
@@ -541,27 +844,63 @@ def run_experiment(args: argparse.Namespace) -> None:
     progress: list[dict[str, Any]] = []
     if args.schedule_mode == "interleaved":
         for seq in range(1, args.samples_per_side + 1):
-            for side_label, side_name, side, seq_path in [
-                ("A", args.left, left, left_seq_path),
-                ("B", args.right, right, right_seq_path),
+            for side_label, ssh_target, side_remote_root, side_name, side, seq_path in [
+                ("A", left_ssh_target, left_remote_root, args.left, left, left_seq_path),
+                ("B", right_ssh_target, right_remote_root, args.right, right, right_seq_path),
             ]:
                 progress.append(
-                    run_one_side_sample(args, remote_root, local_exp_root, side_label, side_name, side, seq_path, seq, rirs)
+                    run_one_side_sample(
+                        args,
+                        ssh_target,
+                        side_remote_root,
+                        local_exp_root,
+                        side_label,
+                        side_name,
+                        side,
+                        seq_path,
+                        seq,
+                        rirs,
+                    )
                 )
     else:
         with ThreadPoolExecutor(max_workers=2) as executor:
             futures = [
-                executor.submit(run_side_sequence, args, remote_root, local_exp_root, "A", args.left, left, left_seq_path, rirs),
-                executor.submit(run_side_sequence, args, remote_root, local_exp_root, "B", args.right, right, right_seq_path, rirs),
+                executor.submit(run_side_sequence, args, left_ssh_target, left_remote_root, local_exp_root, "A", args.left, left, left_seq_path, rirs),
+                executor.submit(run_side_sequence, args, right_ssh_target, right_remote_root, local_exp_root, "B", args.right, right, right_seq_path, rirs),
             ]
             for future in as_completed(futures):
                 progress.extend(future.result())
     progress.sort(key=lambda item: (str(item.get("side")), int(item.get("seq") or 0)))
     write_json(local_exp_root / "run-progress.json", progress)
+    if args.remote_triage:
+        sync_side_to_analysis_remote(left_ssh_target, left_remote_root, analysis_ssh_target, analysis_remote_root, "A")
+        sync_side_to_analysis_remote(right_ssh_target, right_remote_root, analysis_ssh_target, analysis_remote_root, "B")
+        remote_exp_root = analysis_remote_root / "experiments" / "sequence"
+        remote_progress = json.dumps(progress, sort_keys=True, ensure_ascii=False)
+        ssh_script(
+            analysis_ssh_target,
+            "set -euo pipefail; "
+            f"cat > {shlex.quote(str(remote_exp_root / 'run-progress.json'))} <<'REMOTE_PROGRESS_JSON'\n"
+            f"{remote_progress}\n"
+            "REMOTE_PROGRESS_JSON\n",
+        )
+        run_sequence_triage_remote(analysis_ssh_target, analysis_remote_root, args)
+        if args.fetch_remote_analysis:
+            rsync_remote_analysis_from_remote(analysis_ssh_target, remote_exp_root, local_exp_root)
+    else:
         run_sequence_triage(local_exp_root, args)
-    ssh_script(args.ssh_target, f"df -h /data / > {shlex.quote(str(remote_root / 'df-after.txt'))} 2>&1 || true; free -h > {shlex.quote(str(remote_root / 'free-after.txt'))} 2>&1 || true")
+    ssh_script(analysis_ssh_target, f"df -h /data / > {shlex.quote(str(analysis_remote_root / 'df-after.txt'))} 2>&1 || true; free -h > {shlex.quote(str(analysis_remote_root / 'free-after.txt'))} 2>&1 || true")
     compare_dir = local_exp_root / "sequence-triage"
-    print(json.dumps({"runRoot": str(run_root), "remoteRoot": str(remote_root), "triage": str(compare_dir / "sequence-triage.json")}, indent=2))
+    remote_compare_dir = analysis_remote_root / "experiments" / "sequence" / "sequence-triage"
+    print(json.dumps({
+        "runRoot": str(run_root),
+        "remoteRoot": str(remote_root),
+        "leftRemoteRoot": str(left_remote_root),
+        "rightRemoteRoot": str(right_remote_root),
+        "analysisRemoteRoot": str(analysis_remote_root),
+        "triage": str(compare_dir / "sequence-triage.json") if not args.remote_triage or args.fetch_remote_analysis else None,
+        "remoteTriage": str(remote_compare_dir / "sequence-triage.json") if args.remote_triage else None,
+    }, indent=2))
 
 
 def main() -> None:
@@ -569,6 +908,12 @@ def main() -> None:
     parser.add_argument("--run-root", required=True)
     parser.add_argument("--remote-root", required=True)
     parser.add_argument("--ssh-target", default=os.environ.get("SSH_TARGET", "root@47.251.56.108"))
+    parser.add_argument("--left-ssh-target")
+    parser.add_argument("--right-ssh-target")
+    parser.add_argument("--analysis-ssh-target")
+    parser.add_argument("--left-remote-root")
+    parser.add_argument("--right-remote-root")
+    parser.add_argument("--analysis-remote-root")
     parser.add_argument("--left", default="ours-standard")
     parser.add_argument("--right", default="rpki-client-standard")
     parser.add_argument("--samples-per-side", type=int, default=3)
@@ -581,6 +926,9 @@ def main() -> None:
     parser.add_argument("--dry-run", action="store_true")
     parser.add_argument("--skip-build", action="store_true", help="reuse existing release binaries")
     parser.add_argument("--triage-only", action="store_true", help="only rerun local sequence triage for an existing run root")
+    parser.add_argument("--remote-triage", action="store_true", help="keep CIR/CCR on remote, write sequence JSONL remotely, and run triage on remote")
+    parser.add_argument("--fetch-remote-analysis", action="store_true", help="when --remote-triage is set, fetch only small sequence/triage JSON outputs; never fetch CIR/CCR")
+    parser.add_argument("--cleanup-run-nonessential", action="store_true", help="after each successful remote sequence item, remove report/log/CSV files and keep only CIR/CCR/timing/meta")
     args = parser.parse_args()
     if args.samples_per_side < 2:
         raise SystemExit("--samples-per-side must be >= 2")

scripts/manual_sync/delta_sync.sh

@@ -389,7 +389,6 @@ def fmt_db_stats(db: dict) -> str:
         "repository_view",
         "raw_by_hash",
         "vcir",
-        "audit_rule_index",
         "rrdp_source",
         "rrdp_source_member",
         "rrdp_uri_owner",

src/audit_trace.rs

@@ -1,893 +0,0 @@
-use crate::data_model::aspa::AspaObject;
-use crate::data_model::manifest::ManifestObject;
-use crate::data_model::roa::RoaObject;
-use crate::storage::{
-    AuditRuleIndexEntry, AuditRuleKind, RawByHashEntry, RocksStore, ValidatedCaInstanceResult,
-    VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus, VcirLocalOutput,
-    VcirOutputType,
-};
-use serde::Serialize;
-use std::collections::HashSet;
-
-#[derive(Debug, thiserror::Error)]
-pub enum AuditTraceError {
-    #[error("storage error: {0}")]
-    Storage(#[from] crate::storage::StorageError),
-
-    #[error("audit rule index points to missing VCIR: {manifest_rsync_uri}")]
-    MissingVcir { manifest_rsync_uri: String },
-
-    #[error(
-        "audit rule index points to missing local output: rule_hash={rule_hash}, output_id={output_id}, manifest={manifest_rsync_uri}"
-    )]
-    MissingLocalOutput {
-        rule_hash: String,
-        output_id: String,
-        manifest_rsync_uri: String,
-    },
-
-    #[error("detected VCIR parent cycle at {manifest_rsync_uri}")]
-    ParentCycle { manifest_rsync_uri: String },
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
-pub struct AuditTraceRawRef {
-    pub sha256_hex: String,
-    pub raw_present: bool,
-    #[serde(skip_serializing_if = "Vec::is_empty", default)]
-    pub origin_uris: Vec<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub object_type: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub byte_len: Option<usize>,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
-pub struct AuditTraceArtifact {
-    pub artifact_role: VcirArtifactRole,
-    pub artifact_kind: VcirArtifactKind,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub uri: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub object_type: Option<String>,
-    pub validation_status: VcirArtifactValidationStatus,
-    pub raw: AuditTraceRawRef,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
-pub struct AuditTraceChainNode {
-    pub manifest_rsync_uri: String,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub parent_manifest_rsync_uri: Option<String>,
-    pub tal_id: String,
-    pub ca_subject_name: String,
-    pub ca_ski: String,
-    pub issuer_ski: String,
-    pub current_manifest_rsync_uri: String,
-    pub current_crl_rsync_uri: String,
-    pub last_successful_validation_time_rfc3339_utc: String,
-    pub local_output_count: usize,
-    pub child_count: usize,
-    pub related_artifacts: Vec<AuditTraceArtifact>,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
-pub struct AuditTraceResolvedOutput {
-    pub output_id: String,
-    pub output_type: VcirOutputType,
-    pub rule_hash: String,
-    pub source_object_uri: String,
-    pub source_object_type: String,
-    pub source_object_hash: String,
-    pub source_ee_cert_hash: String,
-    pub item_effective_until_rfc3339_utc: String,
-    pub payload_json: String,
-    pub validation_path_hint: Vec<String>,
-}
-
-#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
-pub struct AuditRuleTrace {
-    pub rule: AuditRuleIndexEntry,
-    pub resolved_output: AuditTraceResolvedOutput,
-    pub source_object_raw: AuditTraceRawRef,
-    pub source_ee_cert_raw: AuditTraceRawRef,
-    pub chain_leaf_to_root: Vec<AuditTraceChainNode>,
-}
-
-pub fn trace_rule_to_root(
-    store: &RocksStore,
-    kind: AuditRuleKind,
-    rule_hash: &str,
-) -> Result<Option<AuditRuleTrace>, AuditTraceError> {
-    let Some(rule) = store.get_audit_rule_index_entry(kind, rule_hash)? else {
-        return Ok(None);
-    };
-
-    let Some(leaf_vcir) = store.get_vcir(&rule.manifest_rsync_uri)? else {
-        return Err(AuditTraceError::MissingVcir {
-            manifest_rsync_uri: rule.manifest_rsync_uri.clone(),
-        });
-    };
-
-    let Some(local_output) = leaf_vcir
-        .local_outputs
-        .iter()
-        .find(|output| output.output_id == rule.output_id && output.rule_hash == rule.rule_hash)
-        .or_else(|| {
-            leaf_vcir
-                .local_outputs
-                .iter()
-                .find(|output| output.rule_hash == rule.rule_hash)
-        })
-        .cloned()
-    else {
-        return Err(AuditTraceError::MissingLocalOutput {
-            rule_hash: rule.rule_hash.clone(),
-            output_id: rule.output_id.clone(),
-            manifest_rsync_uri: rule.manifest_rsync_uri.clone(),
-        });
-    };
-
-    let chain = trace_vcir_chain_to_root(store, &leaf_vcir.manifest_rsync_uri)?
-        .expect("leaf VCIR already loaded must exist");
-
-    Ok(Some(AuditRuleTrace {
-        rule,
-        resolved_output: resolved_output_from_local(&local_output),
-        source_object_raw: resolve_raw_ref(
-            store,
-            &local_output.source_object_hash,
-            Some(&local_output.source_object_uri),
-            Some(local_output.source_object_type.as_str()),
-        )?,
-        source_ee_cert_raw: resolve_source_ee_cert_raw_ref(store, &local_output)?,
-        chain_leaf_to_root: chain,
-    }))
-}
-
-pub fn trace_vcir_chain_to_root(
-    store: &RocksStore,
-    manifest_rsync_uri: &str,
-) -> Result<Option<Vec<AuditTraceChainNode>>, AuditTraceError> {
-    let Some(mut current) = store.get_vcir(manifest_rsync_uri)? else {
-        return Ok(None);
-    };
-
-    let mut seen = HashSet::new();
-    let mut chain = Vec::new();
-    loop {
-        if !seen.insert(current.manifest_rsync_uri.clone()) {
-            return Err(AuditTraceError::ParentCycle {
-                manifest_rsync_uri: current.manifest_rsync_uri,
-            });
-        }
-        let parent = current.parent_manifest_rsync_uri.clone();
-        chain.push(trace_chain_node(store, &current)?);
-        let Some(parent_manifest_rsync_uri) = parent else {
-            break;
-        };
-        let Some(parent_vcir) = store.get_vcir(&parent_manifest_rsync_uri)? else {
-            return Err(AuditTraceError::MissingVcir {
-                manifest_rsync_uri: parent_manifest_rsync_uri,
-            });
-        };
-        current = parent_vcir;
-    }
-
-    Ok(Some(chain))
-}
-
-fn trace_chain_node(
-    store: &RocksStore,
-    vcir: &ValidatedCaInstanceResult,
-) -> Result<AuditTraceChainNode, AuditTraceError> {
-    let mut related_artifacts = Vec::with_capacity(vcir.related_artifacts.len());
-    for artifact in &vcir.related_artifacts {
-        related_artifacts.push(AuditTraceArtifact {
-            artifact_role: artifact.artifact_role,
-            artifact_kind: artifact.artifact_kind,
-            uri: artifact.uri.clone(),
-            object_type: artifact.object_type.clone(),
-            validation_status: artifact.validation_status,
-            raw: resolve_raw_ref(
-                store,
-                &artifact.sha256,
-                artifact.uri.as_deref(),
-                artifact.object_type.as_deref(),
-            )?,
-        });
-    }
-
-    Ok(AuditTraceChainNode {
-        manifest_rsync_uri: vcir.manifest_rsync_uri.clone(),
-        parent_manifest_rsync_uri: vcir.parent_manifest_rsync_uri.clone(),
-        tal_id: vcir.tal_id.clone(),
-        ca_subject_name: vcir.ca_subject_name.clone(),
-        ca_ski: vcir.ca_ski.clone(),
-        issuer_ski: vcir.issuer_ski.clone(),
-        current_manifest_rsync_uri: vcir.current_manifest_rsync_uri.clone(),
-        current_crl_rsync_uri: vcir.current_crl_rsync_uri.clone(),
-        last_successful_validation_time_rfc3339_utc: vcir
-            .last_successful_validation_time
-            .rfc3339_utc
-            .clone(),
-        local_output_count: vcir.local_outputs.len(),
-        child_count: vcir.child_entries.len(),
-        related_artifacts,
-    })
-}
-
-fn resolved_output_from_local(local: &VcirLocalOutput) -> AuditTraceResolvedOutput {
-    AuditTraceResolvedOutput {
-        output_id: local.output_id.clone(),
-        output_type: local.output_type,
-        rule_hash: local.rule_hash.clone(),
-        source_object_uri: local.source_object_uri.clone(),
-        source_object_type: local.source_object_type.clone(),
-        source_object_hash: local.source_object_hash.clone(),
-        source_ee_cert_hash: local.source_ee_cert_hash.clone(),
-        item_effective_until_rfc3339_utc: local.item_effective_until.rfc3339_utc.clone(),
-        payload_json: local.payload_json.clone(),
-        validation_path_hint: local.validation_path_hint.clone(),
-    }
-}
-
-fn resolve_raw_ref(
-    store: &RocksStore,
-    sha256_hex: &str,
-    fallback_uri: Option<&str>,
-    fallback_object_type: Option<&str>,
-) -> Result<AuditTraceRawRef, AuditTraceError> {
-    let raw = store.get_raw_by_hash_entry(sha256_hex)?;
-    if raw.is_some() {
-        return Ok(raw_ref_from_entry(sha256_hex, raw.as_ref()));
-    }
-
-    let blob = store.get_blob_bytes(sha256_hex)?;
-    match blob {
-        Some(bytes) => Ok(AuditTraceRawRef {
-            sha256_hex: sha256_hex.to_string(),
-            raw_present: true,
-            origin_uris: fallback_uri
-                .map(|uri| vec![uri.to_string()])
-                .unwrap_or_default(),
-            object_type: fallback_object_type.map(str::to_string),
-            byte_len: Some(bytes.len()),
-        }),
-        None => Ok(raw_ref_from_entry(sha256_hex, None)),
-    }
-}
-
-fn resolve_source_ee_cert_raw_ref(
-    store: &RocksStore,
-    local: &VcirLocalOutput,
-) -> Result<AuditTraceRawRef, AuditTraceError> {
-    let raw = store.get_raw_by_hash_entry(&local.source_ee_cert_hash)?;
-    if raw.is_some() {
-        return Ok(raw_ref_from_entry(&local.source_ee_cert_hash, raw.as_ref()));
-    }
-
-    let source_bytes = store.get_blob_bytes(&local.source_object_hash)?;
-    let Some(source_bytes) = source_bytes else {
-        return Ok(raw_ref_from_entry(&local.source_ee_cert_hash, None));
-    };
-
-    let derived = match local.source_object_type.as_str() {
-        "roa" => RoaObject::decode_der(&source_bytes).ok().and_then(|roa| {
-            roa.signed_object
-                .signed_data
-                .certificates
-                .first()
-                .map(|cert| cert.raw_der.to_vec())
-        }),
-        "aspa" => AspaObject::decode_der(&source_bytes).ok().and_then(|aspa| {
-            aspa.signed_object
-                .signed_data
-                .certificates
-                .first()
-                .map(|cert| cert.raw_der.to_vec())
-        }),
-        "mft" => ManifestObject::decode_der(&source_bytes)
-            .ok()
-            .and_then(|manifest| {
-                manifest
-                    .signed_object
-                    .signed_data
-                    .certificates
-                    .first()
-                    .map(|cert| cert.raw_der.to_vec())
-            }),
-        "router_key" => Some(source_bytes),
-        _ => None,
-    };
-
-    let Some(ee_der) = derived else {
-        return Ok(raw_ref_from_entry(&local.source_ee_cert_hash, None));
-    };
-    if crate::audit::sha256_hex(ee_der.as_slice()) != local.source_ee_cert_hash {
-        return Ok(raw_ref_from_entry(&local.source_ee_cert_hash, None));
-    }
-
-    Ok(AuditTraceRawRef {
-        sha256_hex: local.source_ee_cert_hash.clone(),
-        raw_present: true,
-        origin_uris: Vec::new(),
-        object_type: Some("cer".to_string()),
-        byte_len: Some(ee_der.len()),
-    })
-}
-
-fn raw_ref_from_entry(sha256_hex: &str, entry: Option<&RawByHashEntry>) -> AuditTraceRawRef {
-    match entry {
-        Some(entry) => AuditTraceRawRef {
-            sha256_hex: sha256_hex.to_string(),
-            raw_present: true,
-            origin_uris: entry.origin_uris.clone(),
-            object_type: entry.object_type.clone(),
-            byte_len: Some(entry.bytes.len()),
-        },
-        None => AuditTraceRawRef {
-            sha256_hex: sha256_hex.to_string(),
-            raw_present: false,
-            origin_uris: Vec::new(),
-            object_type: None,
-            byte_len: None,
-        },
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::audit::sha256_hex;
-    use crate::data_model::roa::RoaObject;
-    use crate::storage::{
-        PackTime, ValidatedManifestMeta, VcirAuditSummary, VcirCcrManifestProjection,
-        VcirChildEntry, VcirInstanceGate, VcirRelatedArtifact, VcirSummary,
-    };
-    use base64::Engine as _;
-
-    fn sample_vcir(
-        manifest_rsync_uri: &str,
-        parent_manifest_rsync_uri: Option<&str>,
-        tal_id: &str,
-        local_output: Option<VcirLocalOutput>,
-        related_artifacts: Vec<VcirRelatedArtifact>,
-    ) -> ValidatedCaInstanceResult {
-        let now = time::OffsetDateTime::now_utc();
-        let next = PackTime::from_utc_offset_datetime(now + time::Duration::hours(1));
-        let local_outputs: Vec<VcirLocalOutput> = local_output.into_iter().collect();
-        let ccr_manifest_projection = VcirCcrManifestProjection {
-            manifest_rsync_uri: manifest_rsync_uri.to_string(),
-            manifest_sha256: vec![0x44; 32],
-            manifest_size: 2048,
-            manifest_ee_aki: vec![0x55; 20],
-            manifest_number_be: vec![1],
-            manifest_this_update: PackTime::from_utc_offset_datetime(now),
-            manifest_sia_locations_der: vec![vec![
-                0x30, 0x11, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x05, 0x86, 0x05,
-                b'r', b's', b'y', b'n', b'c',
-            ]],
-            subordinate_skis: vec![vec![0x33; 20]],
-        };
-        ValidatedCaInstanceResult {
-            manifest_rsync_uri: manifest_rsync_uri.to_string(),
-            parent_manifest_rsync_uri: parent_manifest_rsync_uri.map(str::to_string),
-            tal_id: tal_id.to_string(),
-            ca_subject_name: format!("CN={manifest_rsync_uri}"),
-            ca_ski: "11".repeat(20),
-            issuer_ski: "22".repeat(20),
-            last_successful_validation_time: PackTime::from_utc_offset_datetime(now),
-            current_manifest_rsync_uri: manifest_rsync_uri.to_string(),
-            current_crl_rsync_uri: manifest_rsync_uri.replace(".mft", ".crl"),
-            validated_manifest_meta: ValidatedManifestMeta {
-                validated_manifest_number: vec![1],
-                validated_manifest_this_update: PackTime::from_utc_offset_datetime(now),
-                validated_manifest_next_update: next.clone(),
-            },
-            ccr_manifest_projection,
-            instance_gate: VcirInstanceGate {
-                manifest_next_update: next.clone(),
-                current_crl_next_update: next.clone(),
-                self_ca_not_after: PackTime::from_utc_offset_datetime(
-                    now + time::Duration::hours(2),
-                ),
-                instance_effective_until: next,
-            },
-            child_entries: vec![VcirChildEntry {
-                child_manifest_rsync_uri: "rsync://example.test/child/child.mft".to_string(),
-                child_cert_rsync_uri: "rsync://example.test/parent/child.cer".to_string(),
-                child_cert_hash: sha256_hex(b"child-cert"),
-                child_ski: "33".repeat(20),
-                child_rsync_base_uri: "rsync://example.test/child/".to_string(),
-                child_publication_point_rsync_uri: "rsync://example.test/child/".to_string(),
-                child_rrdp_notification_uri: Some(
-                    "https://example.test/child/notify.xml".to_string(),
-                ),
-                child_effective_ip_resources: None,
-                child_effective_as_resources: None,
-                accepted_at_validation_time: PackTime::from_utc_offset_datetime(now),
-            }],
-            summary: VcirSummary {
-                local_vrp_count: local_outputs
-                    .iter()
-                    .filter(|output| output.output_type == VcirOutputType::Vrp)
-                    .count() as u32,
-                local_aspa_count: local_outputs
-                    .iter()
-                    .filter(|output| output.output_type == VcirOutputType::Aspa)
-                    .count() as u32,
-                local_router_key_count: local_outputs
-                    .iter()
-                    .filter(|output| output.output_type == VcirOutputType::RouterKey)
-                    .count() as u32,
-                child_count: 1,
-                accepted_object_count: related_artifacts.len() as u32,
-                rejected_object_count: 0,
-            },
-            local_outputs,
-            related_artifacts,
-            audit_summary: VcirAuditSummary {
-                failed_fetch_eligible: true,
-                last_failed_fetch_reason: None,
-                warning_count: 0,
-                audit_flags: Vec::new(),
-            },
-        }
-    }
-
-    fn sample_local_output(manifest_rsync_uri: &str) -> VcirLocalOutput {
-        let now = time::OffsetDateTime::now_utc();
-        VcirLocalOutput {
-            output_id: sha256_hex(b"vrp-output"),
-            output_type: VcirOutputType::Vrp,
-            item_effective_until: PackTime::from_utc_offset_datetime(
-                now + time::Duration::minutes(30),
-            ),
-            source_object_uri: "rsync://example.test/leaf/a.roa".to_string(),
-            source_object_type: "roa".to_string(),
-            source_object_hash: sha256_hex(b"roa-raw"),
-            source_ee_cert_hash: sha256_hex(b"roa-ee"),
-            payload_json:
-                serde_json::json!({"asn": 64496, "prefix": "203.0.113.0/24", "max_length": 24})
-                    .to_string(),
-            rule_hash: sha256_hex(b"roa-rule"),
-            validation_path_hint: vec![
-                manifest_rsync_uri.to_string(),
-                "rsync://example.test/leaf/a.roa".to_string(),
-                sha256_hex(b"roa-raw"),
-            ],
-        }
-    }
-
-    fn sample_artifacts(manifest_rsync_uri: &str, roa_hash: &str) -> Vec<VcirRelatedArtifact> {
-        vec![
-            VcirRelatedArtifact {
-                artifact_role: VcirArtifactRole::Manifest,
-                artifact_kind: VcirArtifactKind::Mft,
-                uri: Some(manifest_rsync_uri.to_string()),
-                sha256: sha256_hex(manifest_rsync_uri.as_bytes()),
-                object_type: Some("mft".to_string()),
-                validation_status: VcirArtifactValidationStatus::Accepted,
-            },
-            VcirRelatedArtifact {
-                artifact_role: VcirArtifactRole::CurrentCrl,
-                artifact_kind: VcirArtifactKind::Crl,
-                uri: Some(manifest_rsync_uri.replace(".mft", ".crl")),
-                sha256: sha256_hex(format!("{}-crl", manifest_rsync_uri).as_bytes()),
-                object_type: Some("crl".to_string()),
-                validation_status: VcirArtifactValidationStatus::Accepted,
-            },
-            VcirRelatedArtifact {
-                artifact_role: VcirArtifactRole::SignedObject,
-                artifact_kind: VcirArtifactKind::Roa,
-                uri: Some("rsync://example.test/leaf/a.roa".to_string()),
-                sha256: roa_hash.to_string(),
-                object_type: Some("roa".to_string()),
-                validation_status: VcirArtifactValidationStatus::Accepted,
-            },
-        ]
-    }
-
-    fn put_raw_evidence(store: &RocksStore, bytes: &[u8], uri: &str, object_type: &str) {
-        let mut entry = RawByHashEntry::from_bytes(sha256_hex(bytes), bytes.to_vec());
-        entry.origin_uris.push(uri.to_string());
-        entry.object_type = Some(object_type.to_string());
-        entry.encoding = Some("der".to_string());
-        store
-            .put_raw_by_hash_entry(&entry)
-            .expect("put raw evidence");
-    }
-
-    fn put_blob_only(store: &RocksStore, bytes: &[u8]) {
-        store
-            .put_blob_bytes_batch(&[(sha256_hex(bytes), bytes.to_vec())])
-            .expect("put blob bytes");
-    }
-
-    #[test]
-    fn trace_rule_to_root_returns_leaf_to_root_chain_and_evidence_refs() {
-        let store_dir = tempfile::tempdir().expect("store dir");
-        let store = RocksStore::open(store_dir.path()).expect("open rocksdb");
-
-        let root_manifest = "rsync://example.test/root/root.mft";
-        let leaf_manifest = "rsync://example.test/leaf/leaf.mft";
-        let local = sample_local_output(leaf_manifest);
-        let leaf_vcir = sample_vcir(
-            leaf_manifest,
-            Some(root_manifest),
-            "test-tal",
-            Some(local.clone()),
-            sample_artifacts(leaf_manifest, &local.source_object_hash),
-        );
-        let root_vcir = sample_vcir(
-            root_manifest,
-            None,
-            "test-tal",
-            None,
-            sample_artifacts(root_manifest, &sha256_hex(b"root-object")),
-        );
-        store.put_vcir(&leaf_vcir).expect("put leaf vcir");
-        store.put_vcir(&root_vcir).expect("put root vcir");
-
-        let rule_entry = AuditRuleIndexEntry {
-            kind: AuditRuleKind::Roa,
-            rule_hash: local.rule_hash.clone(),
-            manifest_rsync_uri: leaf_manifest.to_string(),
-            source_object_uri: local.source_object_uri.clone(),
-            source_object_hash: local.source_object_hash.clone(),
-            output_id: local.output_id.clone(),
-            item_effective_until: local.item_effective_until.clone(),
-        };
-        store
-            .put_audit_rule_index_entry(&rule_entry)
-            .expect("put rule index");
-
-        put_raw_evidence(&store, leaf_manifest.as_bytes(), leaf_manifest, "mft");
-        put_raw_evidence(
-            &store,
-            format!("{}-crl", leaf_manifest).as_bytes(),
-            &leaf_manifest.replace(".mft", ".crl"),
-            "crl",
-        );
-        put_raw_evidence(&store, b"roa-raw", &local.source_object_uri, "roa");
-        put_raw_evidence(&store, b"roa-ee", "rsync://example.test/leaf/a.ee", "cer");
-        put_raw_evidence(&store, root_manifest.as_bytes(), root_manifest, "mft");
-        put_raw_evidence(
-            &store,
-            format!("{}-crl", root_manifest).as_bytes(),
-            &root_manifest.replace(".mft", ".crl"),
-            "crl",
-        );
-
-        let trace = trace_rule_to_root(&store, AuditRuleKind::Roa, &local.rule_hash)
-            .expect("trace rule")
-            .expect("trace exists");
-
-        assert_eq!(trace.rule, rule_entry);
-        assert_eq!(trace.resolved_output.output_id, local.output_id);
-        assert_eq!(trace.chain_leaf_to_root.len(), 2);
-        assert_eq!(
-            trace.chain_leaf_to_root[0].manifest_rsync_uri,
-            leaf_manifest
-        );
-        assert_eq!(
-            trace.chain_leaf_to_root[1].manifest_rsync_uri,
-            root_manifest
-        );
-        assert_eq!(
-            trace.chain_leaf_to_root[0]
-                .parent_manifest_rsync_uri
-                .as_deref(),
-            Some(root_manifest)
-        );
-        assert!(trace.source_object_raw.raw_present);
-        assert!(trace.source_ee_cert_raw.raw_present);
-        assert!(
-            trace.chain_leaf_to_root[0]
-                .related_artifacts
-                .iter()
-                .any(|artifact| {
-                    artifact.uri.as_deref() == Some(leaf_manifest) && artifact.raw.raw_present
-                })
-        );
-    }
-
-    #[test]
-    fn trace_rule_to_root_supports_router_key_rules() {
-        let store_dir = tempfile::tempdir().expect("store dir");
-        let store = RocksStore::open(store_dir.path()).expect("open rocksdb");
-        let manifest = "rsync://example.test/router/leaf.mft";
-        let mut local = sample_local_output(manifest);
-        local.output_type = VcirOutputType::RouterKey;
-        local.source_object_uri = "rsync://example.test/router/router.cer".to_string();
-        local.source_object_type = "router_key".to_string();
-        local.payload_json = serde_json::json!({
-            "as_id": 64496,
-            "ski_hex": "11".repeat(20),
-            "spki_der_base64": base64::engine::general_purpose::STANDARD.encode([0x30u8, 0x00]),
-        })
-        .to_string();
-        let mut vcir = sample_vcir(
-            manifest,
-            None,
-            "test-tal",
-            Some(local),
-            sample_artifacts(manifest, &sha256_hex(b"router-object")),
-        );
-        vcir.local_outputs[0].output_type = VcirOutputType::RouterKey;
-        vcir.local_outputs[0].source_object_uri =
-            "rsync://example.test/router/router.cer".to_string();
-        vcir.local_outputs[0].source_object_type = "router_key".to_string();
-        vcir.local_outputs[0].payload_json = serde_json::json!({
-            "as_id": 64496,
-            "ski_hex": "11".repeat(20),
-            "spki_der_base64": base64::engine::general_purpose::STANDARD.encode([0x30u8, 0x00]),
-        })
-        .to_string();
-        vcir.summary.local_vrp_count = 0;
-        vcir.summary.local_router_key_count = 1;
-        store.put_vcir(&vcir).expect("put vcir");
-        let rule_entry = AuditRuleIndexEntry {
-            kind: AuditRuleKind::RouterKey,
-            rule_hash: vcir.local_outputs[0].rule_hash.clone(),
-            manifest_rsync_uri: manifest.to_string(),
-            source_object_uri: vcir.local_outputs[0].source_object_uri.clone(),
-            source_object_hash: vcir.local_outputs[0].source_object_hash.clone(),
-            output_id: vcir.local_outputs[0].output_id.clone(),
-            item_effective_until: vcir.local_outputs[0].item_effective_until.clone(),
-        };
-        store
-            .put_audit_rule_index_entry(&rule_entry)
-            .expect("put rule");
-        let trace = trace_rule_to_root(&store, AuditRuleKind::RouterKey, &rule_entry.rule_hash)
-            .expect("trace rule")
-            .expect("trace exists");
-        assert_eq!(trace.rule.kind, AuditRuleKind::RouterKey);
-        assert_eq!(trace.resolved_output.output_type, VcirOutputType::RouterKey);
-    }
-
-    #[test]
-    fn trace_rule_to_root_lazily_derives_source_ee_cert_when_raw_is_missing() {
-        let store_dir = tempfile::tempdir().expect("store dir");
-        let store = RocksStore::open(store_dir.path()).expect("open rocksdb");
-        let manifest = "rsync://example.test/leaf/leaf.mft";
-        let roa_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
-            .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142071.roa");
-        let roa_bytes = std::fs::read(&roa_path).expect("read ROA fixture");
-        let roa = RoaObject::decode_der(&roa_bytes).expect("decode ROA fixture");
-        let local = VcirLocalOutput {
-            output_id: sha256_hex(b"lazy-vrp-output"),
-            output_type: VcirOutputType::Vrp,
-            item_effective_until: PackTime::from_utc_offset_datetime(
-                time::OffsetDateTime::now_utc() + time::Duration::minutes(30),
-            ),
-            source_object_uri: "rsync://example.test/leaf/a.roa".to_string(),
-            source_object_type: "roa".to_string(),
-            source_object_hash: sha256_hex(&roa_bytes),
-            source_ee_cert_hash: sha256_hex(
-                roa.signed_object.signed_data.certificates[0]
-                    .raw_der
-                    .as_slice(),
-            ),
-            payload_json:
-                serde_json::json!({"asn": 64496, "prefix": "203.0.113.0/24", "max_length": 24})
-                    .to_string(),
-            rule_hash: sha256_hex(b"lazy-roa-rule"),
-            validation_path_hint: vec![manifest.to_string()],
-        };
-        let vcir = sample_vcir(
-            manifest,
-            None,
-            "test-tal",
-            Some(local.clone()),
-            sample_artifacts(manifest, &local.source_object_hash),
-        );
-        store.put_vcir(&vcir).expect("put vcir");
-        let rule_entry = AuditRuleIndexEntry {
-            kind: AuditRuleKind::Roa,
-            rule_hash: local.rule_hash.clone(),
-            manifest_rsync_uri: manifest.to_string(),
-            source_object_uri: local.source_object_uri.clone(),
-            source_object_hash: local.source_object_hash.clone(),
-            output_id: local.output_id.clone(),
-            item_effective_until: local.item_effective_until.clone(),
-        };
-        store
-            .put_audit_rule_index_entry(&rule_entry)
-            .expect("put rule index");
-
-        put_raw_evidence(&store, manifest.as_bytes(), manifest, "mft");
-        put_raw_evidence(
-            &store,
-            format!("{}-crl", manifest).as_bytes(),
-            &manifest.replace(".mft", ".crl"),
-            "crl",
-        );
-        put_raw_evidence(&store, &roa_bytes, &local.source_object_uri, "roa");
-
-        let trace = trace_rule_to_root(&store, AuditRuleKind::Roa, &local.rule_hash)
-            .expect("trace rule")
-            .expect("trace exists");
-        assert!(trace.source_object_raw.raw_present);
-        assert!(trace.source_ee_cert_raw.raw_present);
-        assert_eq!(trace.source_ee_cert_raw.object_type.as_deref(), Some("cer"));
-    }
-
-    #[test]
-    fn trace_rule_to_root_uses_blob_only_fallback_for_source_object_raw() {
-        let store_dir = tempfile::tempdir().expect("store dir");
-        let main_db = store_dir.path().join("main-db");
-        let raw_db = store_dir.path().join("raw-store.db");
-        let store =
-            RocksStore::open_with_external_raw_store(&main_db, &raw_db).expect("open rocksdb");
-        let manifest = "rsync://example.test/leaf/leaf.mft";
-        let roa_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
-            .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142071.roa");
-        let roa_bytes = std::fs::read(&roa_path).expect("read ROA fixture");
-        let roa = RoaObject::decode_der(&roa_bytes).expect("decode ROA fixture");
-        let local = VcirLocalOutput {
-            output_id: sha256_hex(b"blob-only-vrp-output"),
-            output_type: VcirOutputType::Vrp,
-            item_effective_until: PackTime::from_utc_offset_datetime(
-                time::OffsetDateTime::now_utc() + time::Duration::minutes(30),
-            ),
-            source_object_uri: "rsync://example.test/leaf/blob-only.roa".to_string(),
-            source_object_type: "roa".to_string(),
-            source_object_hash: sha256_hex(&roa_bytes),
-            source_ee_cert_hash: sha256_hex(
-                roa.signed_object.signed_data.certificates[0]
-                    .raw_der
-                    .as_slice(),
-            ),
-            payload_json:
-                serde_json::json!({"asn": 64496, "prefix": "203.0.113.0/24", "max_length": 24})
-                    .to_string(),
-            rule_hash: sha256_hex(b"blob-only-roa-rule"),
-            validation_path_hint: vec![manifest.to_string()],
-        };
-        let vcir = sample_vcir(
-            manifest,
-            None,
-            "test-tal",
-            Some(local.clone()),
-            sample_artifacts(manifest, &local.source_object_hash),
-        );
-        store.put_vcir(&vcir).expect("put vcir");
-        let rule_entry = AuditRuleIndexEntry {
-            kind: AuditRuleKind::Roa,
-            rule_hash: local.rule_hash.clone(),
-            manifest_rsync_uri: manifest.to_string(),
-            source_object_uri: local.source_object_uri.clone(),
-            source_object_hash: local.source_object_hash.clone(),
-            output_id: local.output_id.clone(),
-            item_effective_until: local.item_effective_until.clone(),
-        };
-        store
-            .put_audit_rule_index_entry(&rule_entry)
-            .expect("put rule index");
-
-        put_raw_evidence(&store, manifest.as_bytes(), manifest, "mft");
-        put_raw_evidence(
-            &store,
-            format!("{}-crl", manifest).as_bytes(),
-            &manifest.replace(".mft", ".crl"),
-            "crl",
-        );
-        put_blob_only(&store, &roa_bytes);
-
-        let trace = trace_rule_to_root(&store, AuditRuleKind::Roa, &local.rule_hash)
-            .expect("trace rule")
-            .expect("trace exists");
-        assert!(trace.source_object_raw.raw_present);
-        assert_eq!(
-            trace.source_object_raw.origin_uris,
-            vec![local.source_object_uri.clone()]
-        );
-        assert_eq!(trace.source_object_raw.object_type.as_deref(), Some("roa"));
-        assert_eq!(trace.source_object_raw.byte_len, Some(roa_bytes.len()));
-        assert!(trace.source_ee_cert_raw.raw_present);
-    }
-
-    #[test]
-    fn trace_rule_to_root_returns_none_for_missing_rule_index() {
-        let store_dir = tempfile::tempdir().expect("store dir");
-        let store = RocksStore::open(store_dir.path()).expect("open rocksdb");
-
-        assert!(
-            trace_rule_to_root(&store, AuditRuleKind::Roa, &sha256_hex(b"missing"))
-                .expect("missing trace ok")
-                .is_none()
-        );
-    }
-
-    #[test]
-    fn trace_rule_to_root_errors_when_index_points_to_missing_vcir() {
-        let store_dir = tempfile::tempdir().expect("store dir");
-        let store = RocksStore::open(store_dir.path()).expect("open rocksdb");
-        let rule_hash = sha256_hex(b"missing-vcir-rule");
-        store
-            .put_audit_rule_index_entry(&AuditRuleIndexEntry {
-                kind: AuditRuleKind::Roa,
-                rule_hash: rule_hash.clone(),
-                manifest_rsync_uri: "rsync://example.test/missing.mft".to_string(),
-                source_object_uri: "rsync://example.test/missing.roa".to_string(),
-                source_object_hash: sha256_hex(b"missing-source"),
-                output_id: sha256_hex(b"missing-output"),
-                item_effective_until: PackTime::from_utc_offset_datetime(
-                    time::OffsetDateTime::now_utc() + time::Duration::minutes(1),
-                ),
-            })
-            .expect("put rule index");
-
-        let err = trace_rule_to_root(&store, AuditRuleKind::Roa, &rule_hash).unwrap_err();
-        assert!(matches!(
-            err,
-            AuditTraceError::MissingVcir { manifest_rsync_uri }
-            if manifest_rsync_uri == "rsync://example.test/missing.mft"
-        ));
-    }
-
-    #[test]
-    fn trace_rule_to_root_errors_when_vcir_local_output_is_missing() {
-        let store_dir = tempfile::tempdir().expect("store dir");
-        let store = RocksStore::open(store_dir.path()).expect("open rocksdb");
-        let manifest = "rsync://example.test/leaf/leaf.mft";
-        let vcir = sample_vcir(
-            manifest,
-            None,
-            "test-tal",
-            None,
-            sample_artifacts(manifest, &sha256_hex(b"leaf-object")),
-        );
-        store.put_vcir(&vcir).expect("put vcir");
-        let rule_hash = sha256_hex(b"missing-output-rule");
-        store
-            .put_audit_rule_index_entry(&AuditRuleIndexEntry {
-                kind: AuditRuleKind::Roa,
-                rule_hash: rule_hash.clone(),
-                manifest_rsync_uri: manifest.to_string(),
-                source_object_uri: "rsync://example.test/leaf/a.roa".to_string(),
-                source_object_hash: sha256_hex(b"leaf-object"),
-                output_id: sha256_hex(b"missing-output"),
-                item_effective_until: PackTime::from_utc_offset_datetime(
-                    time::OffsetDateTime::now_utc() + time::Duration::minutes(1),
-                ),
-            })
-            .expect("put rule index");
-
-        let err = trace_rule_to_root(&store, AuditRuleKind::Roa, &rule_hash).unwrap_err();
-        assert!(matches!(err, AuditTraceError::MissingLocalOutput { .. }));
-    }
-
-    #[test]
-    fn trace_vcir_chain_to_root_detects_parent_cycle() {
-        let store_dir = tempfile::tempdir().expect("store dir");
-        let store = RocksStore::open(store_dir.path()).expect("open rocksdb");
-        let a_manifest = "rsync://example.test/a.mft";
-        let b_manifest = "rsync://example.test/b.mft";
-        let a_vcir = sample_vcir(
-            a_manifest,
-            Some(b_manifest),
-            "test-tal",
-            None,
-            sample_artifacts(a_manifest, &sha256_hex(b"a-object")),
-        );
-        let b_vcir = sample_vcir(
-            b_manifest,
-            Some(a_manifest),
-            "test-tal",
-            None,
-            sample_artifacts(b_manifest, &sha256_hex(b"b-object")),
-        );
-        store.put_vcir(&a_vcir).expect("put a");
-        store.put_vcir(&b_vcir).expect("put b");
-
-        let err = trace_vcir_chain_to_root(&store, a_manifest).unwrap_err();
-        assert!(matches!(
-            err,
-            AuditTraceError::ParentCycle { manifest_rsync_uri }
-            if manifest_rsync_uri == a_manifest
-        ));
-    }
-}

src/bin/db_stats.rs

@@ -4,9 +4,8 @@ use std::path::{Path, PathBuf};
 
 use rocksdb::{DB, IteratorMode, Options};
 use rpki::storage::{
-    ALL_COLUMN_FAMILY_NAMES, CF_AUDIT_RULE_INDEX, CF_MANIFEST_REPLAY_META, CF_RAW_BY_HASH,
-    CF_REPOSITORY_VIEW, CF_RRDP_SOURCE, CF_RRDP_SOURCE_MEMBER, CF_RRDP_URI_OWNER, CF_VCIR,
-    column_family_descriptors,
+    ALL_COLUMN_FAMILY_NAMES, CF_MANIFEST_REPLAY_META, CF_RAW_BY_HASH, CF_REPOSITORY_VIEW,
+    CF_RRDP_SOURCE, CF_RRDP_SOURCE_MEMBER, CF_RRDP_URI_OWNER, CF_VCIR, column_family_descriptors,
 };
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -99,7 +98,7 @@ Output:
 
 Output groups:
   - current_repository_view: repository_view + raw_by_hash
-  - current_validation_state: vcir + audit_rule_index
+  - current_validation_state: vcir + manifest_replay_meta
   - current_rrdp_state: rrdp_source + rrdp_source_member + rrdp_uri_owner
 "
     )
@@ -183,7 +182,7 @@ fn collect_db_file_stats(db_path: &Path) -> Result<DbFileStats, Box<dyn std::err
 fn cf_group(cf_name: &str) -> CfGroup {
     match cf_name {
         CF_REPOSITORY_VIEW | CF_RAW_BY_HASH => CfGroup::CurrentRepositoryView,
-        CF_VCIR | CF_MANIFEST_REPLAY_META | CF_AUDIT_RULE_INDEX => CfGroup::CurrentValidationState,
+        CF_VCIR | CF_MANIFEST_REPLAY_META => CfGroup::CurrentValidationState,
         CF_RRDP_SOURCE | CF_RRDP_SOURCE_MEMBER | CF_RRDP_URI_OWNER => CfGroup::CurrentRrdpState,
         _ => CfGroup::LegacyCompatibility,
     }
@@ -375,10 +374,6 @@ mod tests {
             cf_group(CF_MANIFEST_REPLAY_META),
             CfGroup::CurrentValidationState
         );
-        assert_eq!(
-            cf_group(CF_AUDIT_RULE_INDEX),
-            CfGroup::CurrentValidationState
-        );
         assert_eq!(cf_group(CF_RRDP_SOURCE), CfGroup::CurrentRrdpState);
         assert_eq!(cf_group(CF_RRDP_URI_OWNER), CfGroup::CurrentRrdpState);
         assert_eq!(cf_group("unknown_legacy"), CfGroup::LegacyCompatibility);
@@ -390,7 +385,7 @@ mod tests {
             (CF_REPOSITORY_VIEW, 5),
             (CF_RAW_BY_HASH, 7),
             (CF_VCIR, 11),
-            (CF_AUDIT_RULE_INDEX, 13),
+            (CF_MANIFEST_REPLAY_META, 13),
             (CF_RRDP_SOURCE_MEMBER, 19),
         ]);
 
@@ -420,7 +415,7 @@ mod tests {
             live_sst_size_bytes: 55,
             live_sst_files: 2,
         };
-        let audit = CfStats {
+        let replay_meta = CfStats {
             keys: 5,
             key_bytes: 50,
             value_bytes: 500,
@@ -433,7 +428,7 @@ mod tests {
         let grouped = summarize_cf_stats([
             (CF_REPOSITORY_VIEW, &repo),
             (CF_VCIR, &vcir),
-            (CF_AUDIT_RULE_INDEX, &audit),
+            (CF_MANIFEST_REPLAY_META, &replay_meta),
         ]);
 
         assert_eq!(grouped.get(&CfGroup::CurrentRepositoryView), Some(&repo));

src/bin/trace_arin_missing_vrps.rs

@@ -1,103 +0,0 @@
-use rpki::audit_trace::trace_rule_to_root;
-use rpki::storage::{AuditRuleKind, RocksStore, VcirOutputType};
-use serde_json::Value;
-use std::env;
-use std::path::Path;
-
-fn main() {
-    let args: Vec<String> = env::args().collect();
-    if args.len() < 3 {
-        eprintln!("usage: trace_arin_missing_vrps <db> <row> [<row> ...]");
-        std::process::exit(2);
-    }
-
-    let store = RocksStore::open(Path::new(&args[1])).expect("open db");
-    let vcirs = store.list_vcirs().expect("list vcirs");
-
-    for row in &args[2..] {
-        let parts: Vec<&str> = row.split(',').collect();
-        if parts.len() != 4 {
-            println!("ROW {row}");
-            println!("ERROR invalid compare row");
-            println!();
-            continue;
-        }
-        let asn: u32 = parts[0]
-            .trim_start_matches("AS")
-            .parse()
-            .expect("parse asn");
-        let prefix = parts[1].to_string();
-        let max_length: u8 = parts[2].parse().expect("parse max length");
-        let mut found = false;
-
-        println!("ROW {row}");
-        for vcir in &vcirs {
-            for output in &vcir.local_outputs {
-                if output.output_type != VcirOutputType::Vrp {
-                    continue;
-                }
-                let payload: Value = match serde_json::from_str(&output.payload_json) {
-                    Ok(value) => value,
-                    Err(_) => continue,
-                };
-                let payload_asn = payload
-                    .get("asn")
-                    .and_then(|v| v.as_u64())
-                    .map(|v| v as u32);
-                let payload_prefix = payload
-                    .get("prefix")
-                    .and_then(|v| v.as_str())
-                    .map(|v| v.to_string());
-                let payload_max = payload
-                    .get("max_length")
-                    .and_then(|v| v.as_u64())
-                    .map(|v| v as u8);
-                if payload_asn == Some(asn)
-                    && payload_prefix.as_ref() == Some(&prefix)
-                    && payload_max == Some(max_length)
-                {
-                    found = true;
-                    println!("manifest_rsync_uri={}", vcir.manifest_rsync_uri);
-                    println!("source_object_uri={}", output.source_object_uri);
-                    println!("source_object_hash={}", output.source_object_hash);
-                    println!("source_ee_cert_hash={}", output.source_ee_cert_hash);
-                    println!("rule_hash={}", output.rule_hash);
-                    println!("validation_path_hint={:?}", output.validation_path_hint);
-
-                    if let Some(trace) =
-                        trace_rule_to_root(&store, AuditRuleKind::Roa, &output.rule_hash)
-                            .expect("trace rule")
-                    {
-                        println!(
-                            "trace_leaf_manifest={}",
-                            trace
-                                .chain_leaf_to_root
-                                .first()
-                                .map(|node| node.manifest_rsync_uri.as_str())
-                                .unwrap_or("")
-                        );
-                        println!(
-                            "trace_source_object_uri={}",
-                            trace.resolved_output.source_object_uri
-                        );
-                        println!("trace_chain_len={}", trace.chain_leaf_to_root.len());
-                        for (idx, node) in trace.chain_leaf_to_root.iter().enumerate() {
-                            println!("chain[{idx}].manifest={}", node.manifest_rsync_uri);
-                            println!(
-                                "chain[{idx}].current_manifest={}",
-                                node.current_manifest_rsync_uri
-                            );
-                            println!("chain[{idx}].current_crl={}", node.current_crl_rsync_uri);
-                        }
-                    }
-                    println!();
-                }
-            }
-        }
-
-        if !found {
-            println!("NOT_FOUND");
-            println!();
-        }
-    }
-}

src/blob_store.rs

@@ -3,7 +3,10 @@ use std::sync::Arc;
 
 use rocksdb::{DB, Options, WriteBatch};
 
-use crate::storage::{RawByHashEntry, RocksStore, StorageError, StorageResult};
+use crate::storage::{
+    RawByHashEntry, RocksDbMemoryDbSnapshot, RocksStore, StorageError, StorageResult,
+    memory_db_snapshot_for_column_families,
+};
 
 const RAW_BY_HASH_KEY_PREFIX: &str = "rawbyhash:";
 const RAW_BLOB_KEY_PREFIX: &str = "rawblob:";
@@ -167,6 +170,10 @@ impl ExternalRawStoreDb {
     pub fn path(&self) -> &PathBuf {
         &self.path
     }
+
+    pub(crate) fn memory_snapshot(&self, label: impl Into<String>) -> RocksDbMemoryDbSnapshot {
+        memory_db_snapshot_for_column_families(label, self.db.as_ref(), None)
+    }
 }
 
 impl ExternalRepoBytesDb {
@@ -234,6 +241,10 @@ impl ExternalRepoBytesDb {
     pub fn path(&self) -> &PathBuf {
         &self.path
     }
+
+    pub(crate) fn memory_snapshot(&self, label: impl Into<String>) -> RocksDbMemoryDbSnapshot {
+        memory_db_snapshot_for_column_families(label, self.db.as_ref(), None)
+    }
 }
 
 impl RawObjectStore for RocksStore {

src/ccr/accumulator.rs

@@ -26,6 +26,21 @@ pub struct CcrManifestContribution {
     pub subordinate_skis: Vec<Vec<u8>>,
 }
 
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct CcrAccumulatorMemoryStats {
+    pub trust_anchor_count: u64,
+    pub manifest_count: u64,
+    pub estimated_heap_bytes: u64,
+    pub string_bytes: u64,
+    pub string_capacity_bytes: u64,
+    pub vec_payload_bytes: u64,
+    pub vec_capacity_bytes: u64,
+    pub locations_der_count: u64,
+    pub subordinate_ski_count: u64,
+    pub btree_key_capacity_bytes: u64,
+    pub btree_entry_shallow_bytes: u64,
+}
+
 impl CcrManifestContribution {
     fn from_projection(projection: &VcirCcrManifestProjection) -> Result<Self, String> {
         let this_update = projection
@@ -57,6 +72,36 @@ impl CcrManifestContribution {
             subordinates: self.subordinate_skis.clone(),
         }
     }
+
+    fn add_memory_stats(&self, stats: &mut CcrAccumulatorMemoryStats) {
+        stats.string_bytes += self.manifest_rsync_uri.len() as u64;
+        stats.string_capacity_bytes += self.manifest_rsync_uri.capacity() as u64;
+        stats.estimated_heap_bytes += self.manifest_rsync_uri.capacity() as u64;
+
+        add_vec_stats(&self.hash, stats);
+        add_vec_stats(&self.aki, stats);
+        add_vec_stats(&self.manifest_number_be, stats);
+        add_vec_of_vec_stats(&self.locations_der, stats);
+        add_vec_of_vec_stats(&self.subordinate_skis, stats);
+        stats.locations_der_count += self.locations_der.len() as u64;
+        stats.subordinate_ski_count += self.subordinate_skis.len() as u64;
+    }
+}
+
+fn add_vec_stats(value: &Vec<u8>, stats: &mut CcrAccumulatorMemoryStats) {
+    stats.vec_payload_bytes += value.len() as u64;
+    stats.vec_capacity_bytes += value.capacity() as u64;
+    stats.estimated_heap_bytes += value.capacity() as u64;
+}
+
+fn add_vec_of_vec_stats(values: &Vec<Vec<u8>>, stats: &mut CcrAccumulatorMemoryStats) {
+    let outer_capacity = values.capacity() * std::mem::size_of::<Vec<u8>>();
+    stats.vec_payload_bytes += (values.len() * std::mem::size_of::<Vec<u8>>()) as u64;
+    stats.vec_capacity_bytes += outer_capacity as u64;
+    stats.estimated_heap_bytes += outer_capacity as u64;
+    for value in values {
+        add_vec_stats(value, stats);
+    }
 }
 
 #[derive(Clone, Debug, PartialEq, Eq)]
@@ -138,6 +183,61 @@ impl CcrAccumulator {
     pub fn manifest_count(&self) -> usize {
         self.manifests_by_hash.len()
     }
+
+    pub fn memory_stats(&self) -> CcrAccumulatorMemoryStats {
+        let mut stats = CcrAccumulatorMemoryStats {
+            trust_anchor_count: self.trust_anchors.len() as u64,
+            manifest_count: self.manifests_by_hash.len() as u64,
+            ..CcrAccumulatorMemoryStats::default()
+        };
+        stats.estimated_heap_bytes +=
+            (self.trust_anchors.capacity() * std::mem::size_of::<TrustAnchor>()) as u64;
+        for trust_anchor in &self.trust_anchors {
+            add_vec_stats(&trust_anchor.tal.raw, &mut stats);
+            add_vec_of_string_stats(&trust_anchor.tal.comments, &mut stats);
+            stats.vec_payload_bytes +=
+                (trust_anchor.tal.ta_uris.len() * std::mem::size_of::<url::Url>()) as u64;
+            stats.vec_capacity_bytes +=
+                (trust_anchor.tal.ta_uris.capacity() * std::mem::size_of::<url::Url>()) as u64;
+            stats.estimated_heap_bytes +=
+                (trust_anchor.tal.ta_uris.capacity() * std::mem::size_of::<url::Url>()) as u64;
+            for uri in &trust_anchor.tal.ta_uris {
+                stats.string_bytes += uri.as_str().len() as u64;
+                stats.string_capacity_bytes += uri.as_str().len() as u64;
+                stats.estimated_heap_bytes += uri.as_str().len() as u64;
+            }
+            add_vec_stats(&trust_anchor.tal.subject_public_key_info_der, &mut stats);
+            add_vec_stats(&trust_anchor.ta_certificate.raw_der, &mut stats);
+            if let Some(uri) = &trust_anchor.resolved_ta_uri {
+                stats.string_bytes += uri.as_str().len() as u64;
+                stats.string_capacity_bytes += uri.as_str().len() as u64;
+                stats.estimated_heap_bytes += uri.as_str().len() as u64;
+            }
+        }
+
+        stats.btree_entry_shallow_bytes = (self.manifests_by_hash.len()
+            * (std::mem::size_of::<Vec<u8>>() + std::mem::size_of::<CcrManifestContribution>()))
+            as u64;
+        stats.estimated_heap_bytes += stats.btree_entry_shallow_bytes;
+        for (key, contribution) in &self.manifests_by_hash {
+            stats.btree_key_capacity_bytes += key.capacity() as u64;
+            stats.estimated_heap_bytes += key.capacity() as u64;
+            contribution.add_memory_stats(&mut stats);
+        }
+        stats
+    }
+}
+
+fn add_vec_of_string_stats(values: &Vec<String>, stats: &mut CcrAccumulatorMemoryStats) {
+    let outer_capacity = values.capacity() * std::mem::size_of::<String>();
+    stats.vec_payload_bytes += (values.len() * std::mem::size_of::<String>()) as u64;
+    stats.vec_capacity_bytes += outer_capacity as u64;
+    stats.estimated_heap_bytes += outer_capacity as u64;
+    for value in values {
+        stats.string_bytes += value.len() as u64;
+        stats.string_capacity_bytes += value.capacity() as u64;
+        stats.estimated_heap_bytes += value.capacity() as u64;
+    }
 }
 
 #[cfg(test)]

src/cli.rs

@@ -7,17 +7,23 @@ use crate::cir::{CirTrustAnchorBinding, export_cir_from_run_multi};
 use std::path::{Path, PathBuf};
 
 use crate::analysis::timing::{TimingHandle, TimingMeta, TimingMetaUpdate};
+use crate::audit::AuditRepoSyncStats;
+#[cfg(test)]
 use crate::audit::{
-    AspaOutput, AuditRepoSyncStats, AuditReportV2, AuditRunMeta, AuditWarning, TreeSummary,
-    VrpOutput, format_roa_ip_prefix,
+    AspaOutput, AuditReportV2, AuditRunMeta, AuditWarning, TreeSummary, VrpOutput,
+    format_roa_ip_prefix,
 };
 use crate::fetch::http::{BlockingHttpFetcher, HttpFetcherConfig};
 use crate::fetch::rsync::LocalDirRsyncFetcher;
 use crate::fetch::rsync_system::{RsyncScopePolicy, SystemRsyncConfig, SystemRsyncFetcher};
+use crate::memory_telemetry::{
+    MallocTrimProbe, MemoryTelemetryCheckpoint, MemoryTelemetrySummary, ObjectGraphMemoryMetric,
+    ObjectGraphMemorySection, ObjectGraphMemorySummary,
+};
 use crate::parallel::config::{ParallelPhase1Config, ParallelPhase2Config};
 use crate::parallel::types::TalInputSpec;
 use crate::policy::{Policy, StrictPolicy};
-use crate::storage::RocksStore;
+use crate::storage::{RocksStore, VcirStorageSummary};
 use crate::validation::run_tree_from_tal::{
     RunTreeFromTalAuditOutput, run_tree_from_multiple_tals_parallel_phase2_audit,
     run_tree_from_tal_and_ta_der_parallel_phase2_audit,
@@ -28,7 +34,11 @@ use crate::validation::run_tree_from_tal::{
     run_tree_from_tal_url_parallel_phase2_audit,
 };
 use crate::validation::tree::TreeRunConfig;
-use output::{ReportJsonFormat, run_compare_view_task, write_json, write_stage_timing};
+#[cfg(test)]
+use output::write_json;
+use output::{
+    ReportJsonFormat, run_compare_view_task, write_report_json_from_shared, write_stage_timing,
+};
 use serde::Serialize;
 use std::sync::Arc;
 
@@ -53,6 +63,35 @@ struct RunStageTiming {
     rrdp_download_ms_total: u64,
     rsync_download_ms_total: u64,
     download_bytes_total: u64,
+    vcir_storage_summary_ms: Option<u64>,
+    vcir_storage: Option<VcirStorageSummary>,
+    memory_telemetry: Option<MemoryTelemetrySummary>,
+}
+
+fn record_memory_checkpoint(
+    checkpoints: &mut Vec<MemoryTelemetryCheckpoint>,
+    label: &str,
+    total_started: &std::time::Instant,
+    store: &RocksStore,
+) {
+    checkpoints.push(MemoryTelemetryCheckpoint {
+        label: label.to_string(),
+        elapsed_ms: total_started.elapsed().as_millis() as u64,
+        process: crate::memory_telemetry::process_memory_snapshot(label),
+        rocksdb: store.memory_snapshot(),
+    });
+}
+
+fn memory_trim_probe_enabled() -> bool {
+    std::env::var("RPKI_MEMORY_TRIM_PROBE")
+        .map(|value| matches!(value.as_str(), "1" | "true" | "TRUE" | "yes" | "YES"))
+        .unwrap_or(false)
+}
+
+fn vcir_storage_summary_enabled() -> bool {
+    std::env::var("RPKI_VCIR_STORAGE_SUMMARY")
+        .map(|value| matches!(value.as_str(), "1" | "true" | "TRUE" | "yes" | "YES"))
+        .unwrap_or(false)
 }
 
 #[derive(Clone, Debug, PartialEq, Eq)]
@@ -92,6 +131,7 @@ pub struct CliArgs {
     pub payload_base_validation_time: Option<time::OffsetDateTime>,
     pub payload_delta_archive: Option<PathBuf>,
     pub payload_delta_locks: Option<PathBuf>,
+    pub memory_trim_after_validation: bool,
 
     pub rsync_local_dir: Option<PathBuf>,
     pub disable_rrdp: bool,
@@ -144,6 +184,7 @@ Options:
   --payload-base-validation-time <rfc3339> Validation time for the base bootstrap inside offline delta replay
   --payload-delta-archive <path>   Use local delta payload archive root (offline delta replay)
   --payload-delta-locks <path>     Use local locks-delta.json (offline delta replay)
+  --memory-trim-after-validation   Call malloc_trim(0) after validation/report memory checkpoints (Linux glibc only; default off)
 
   --tal-url <url>            TAL URL (repeatable; URL mode)
   --tal-path <path>          TAL file path (repeatable; file mode)
@@ -221,6 +262,7 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
     let mut payload_base_validation_time: Option<time::OffsetDateTime> = None;
     let mut payload_delta_archive: Option<PathBuf> = None;
     let mut payload_delta_locks: Option<PathBuf> = None;
+    let mut memory_trim_after_validation = false;
 
     let mut rsync_local_dir: Option<PathBuf> = None;
     let mut disable_rrdp: bool = false;
@@ -497,6 +539,9 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
                     .ok_or("--payload-delta-locks requires a value")?;
                 payload_delta_locks = Some(PathBuf::from(v));
             }
+            "--memory-trim-after-validation" => {
+                memory_trim_after_validation = true;
+            }
             "--rsync-local-dir" => {
                 i += 1;
                 let v = argv.get(i).ok_or("--rsync-local-dir requires a value")?;
@@ -841,6 +886,7 @@ pub fn parse_args(argv: &[String]) -> Result<CliArgs, String> {
         payload_base_validation_time,
         payload_delta_archive,
         payload_delta_locks,
+        memory_trim_after_validation,
         rsync_local_dir,
         disable_rrdp,
         rsync_command,
@@ -880,10 +926,12 @@ fn unique_rrdp_repos_from_publication_points(
     set.len()
 }
 
+#[cfg(test)]
 fn unique_rrdp_repos(report: &AuditReportV2) -> usize {
     unique_rrdp_repos_from_publication_points(&report.publication_points)
 }
 
+#[cfg(test)]
 fn print_summary(report: &AuditReportV2) {
     let rrdp_repos = unique_rrdp_repos(report);
     println!("RPKI stage2 serial run summary");
@@ -1014,6 +1062,459 @@ impl PostValidationShared {
     }
 }
 
+#[derive(Default)]
+struct ObjectGraphSectionBuilder {
+    name: String,
+    item_count: u64,
+    shallow_bytes: u64,
+    heap_bytes: u64,
+    string_count: u64,
+    string_bytes: u64,
+    string_capacity_bytes: u64,
+    vec_count: u64,
+    vec_heap_bytes: u64,
+    vec_capacity_bytes: u64,
+    details: Vec<ObjectGraphMemoryMetric>,
+}
+
+impl ObjectGraphSectionBuilder {
+    fn new(name: impl Into<String>) -> Self {
+        Self {
+            name: name.into(),
+            ..Self::default()
+        }
+    }
+
+    fn items(&mut self, count: usize, item_size: usize) {
+        self.item_count += count as u64;
+        self.shallow_bytes += (count as u64) * (item_size as u64);
+    }
+
+    fn heap_bytes(&mut self, value: usize) {
+        self.heap_bytes += value as u64;
+    }
+
+    fn string(&mut self, value: &str) {
+        self.string_count += 1;
+        self.string_bytes += value.len() as u64;
+        self.string_capacity_bytes += value.len() as u64;
+        self.heap_bytes += value.len() as u64;
+    }
+
+    fn owned_string(&mut self, value: &String) {
+        self.string_count += 1;
+        self.string_bytes += value.len() as u64;
+        self.string_capacity_bytes += value.capacity() as u64;
+        self.heap_bytes += value.capacity() as u64;
+    }
+
+    fn optional_string(&mut self, value: Option<&String>) {
+        if let Some(value) = value {
+            self.owned_string(value);
+        }
+    }
+
+    fn vec_header_with_capacity(&mut self, len: usize, capacity: usize, element_size: usize) {
+        self.vec_count += 1;
+        let payload_bytes = len * element_size;
+        let capacity_bytes = capacity * element_size;
+        self.vec_heap_bytes += payload_bytes as u64;
+        self.vec_capacity_bytes += capacity_bytes as u64;
+        self.heap_bytes += capacity_bytes as u64;
+    }
+
+    fn byte_vec_owned(&mut self, value: &Vec<u8>) {
+        self.vec_header_with_capacity(value.len(), value.capacity(), std::mem::size_of::<u8>());
+    }
+
+    fn string_vec_owned(&mut self, values: &Vec<String>) {
+        self.vec_header_with_capacity(
+            values.len(),
+            values.capacity(),
+            std::mem::size_of::<String>(),
+        );
+        for value in values {
+            self.owned_string(value);
+        }
+    }
+
+    fn metric(&mut self, name: impl Into<String>, value: u64) {
+        self.details.push(ObjectGraphMemoryMetric {
+            name: name.into(),
+            value,
+        });
+    }
+
+    fn finish(self) -> ObjectGraphMemorySection {
+        let estimated_bytes = self.shallow_bytes + self.heap_bytes;
+        ObjectGraphMemorySection {
+            name: self.name,
+            item_count: self.item_count,
+            shallow_bytes: self.shallow_bytes,
+            heap_bytes: self.heap_bytes,
+            estimated_bytes,
+            string_count: self.string_count,
+            string_bytes: self.string_bytes,
+            string_capacity_bytes: self.string_capacity_bytes,
+            vec_count: self.vec_count,
+            vec_heap_bytes: self.vec_heap_bytes,
+            vec_capacity_bytes: self.vec_capacity_bytes,
+            details: self.details,
+        }
+    }
+}
+
+fn estimate_shared_object_graph(shared: &PostValidationShared) -> ObjectGraphMemorySummary {
+    let mut sections = Vec::new();
+    sections.push(estimate_publication_points_graph(
+        shared.publication_points.as_ref(),
+    ));
+    sections.push(estimate_vrps_graph(shared.vrps.as_ref()));
+    sections.push(estimate_aspas_graph(shared.aspas.as_ref()));
+    sections.push(estimate_router_keys_graph(shared.router_keys.as_ref()));
+    sections.push(estimate_warnings_graph(
+        "tree_warnings",
+        shared.tree_warnings.as_ref(),
+    ));
+    sections.push(estimate_downloads_graph(shared.downloads.as_ref()));
+    sections.push(estimate_current_repo_objects_graph(
+        shared.current_repo_objects.as_ref(),
+    ));
+    sections.push(estimate_trust_anchor_graph(shared));
+    sections.push(estimate_ccr_accumulator_graph(
+        shared.ccr_accumulator.as_ref(),
+    ));
+
+    let total_estimated_bytes = sections
+        .iter()
+        .map(|section| section.estimated_bytes)
+        .sum::<u64>();
+    ObjectGraphMemorySummary {
+        captured_at_label: "after_validation".to_string(),
+        total_estimated_bytes,
+        sections,
+        notes: vec![
+            "Estimated bytes are Rust object graph approximations based on struct sizes and owned String/Vec payload lengths.".to_string(),
+            "The estimate intentionally excludes allocator metadata, fragmentation, freed-but-retained arenas, RocksDB C++ heap, and transient worker allocations.".to_string(),
+            "Large RSS minus this estimate points to allocator retention or structures not yet modeled by this telemetry.".to_string(),
+        ],
+    }
+}
+
+fn estimate_publication_points_graph(
+    publication_points: &[crate::audit::PublicationPointAudit],
+) -> ObjectGraphMemorySection {
+    let mut builder = ObjectGraphSectionBuilder::new("publication_points");
+    builder.items(
+        publication_points.len(),
+        std::mem::size_of::<crate::audit::PublicationPointAudit>(),
+    );
+    builder.metric("publication_point_count", publication_points.len() as u64);
+    let mut object_count = 0u64;
+    let mut pp_warning_count = 0u64;
+    let mut pp_discovered_from_count = 0u64;
+    let mut object_detail_count = 0u64;
+
+    for pp in publication_points {
+        builder.owned_string(&pp.rsync_base_uri);
+        builder.owned_string(&pp.manifest_rsync_uri);
+        builder.owned_string(&pp.publication_point_rsync_uri);
+        builder.optional_string(pp.rrdp_notification_uri.as_ref());
+        builder.owned_string(&pp.source);
+        builder.optional_string(pp.repo_sync_source.as_ref());
+        builder.optional_string(pp.repo_sync_phase.as_ref());
+        builder.optional_string(pp.repo_sync_error.as_ref());
+        builder.owned_string(&pp.repo_terminal_state);
+        builder.owned_string(&pp.this_update_rfc3339_utc);
+        builder.owned_string(&pp.next_update_rfc3339_utc);
+        builder.owned_string(&pp.verified_at_rfc3339_utc);
+
+        if let Some(discovered_from) = &pp.discovered_from {
+            pp_discovered_from_count += 1;
+            builder.heap_bytes(std::mem::size_of::<crate::audit::DiscoveredFrom>());
+            builder.owned_string(&discovered_from.parent_manifest_rsync_uri);
+            builder.owned_string(&discovered_from.child_ca_certificate_rsync_uri);
+            builder.owned_string(&discovered_from.child_ca_certificate_sha256_hex);
+        }
+
+        pp_warning_count += pp.warnings.len() as u64;
+        builder.vec_header_with_capacity(
+            pp.warnings.len(),
+            pp.warnings.capacity(),
+            std::mem::size_of::<crate::audit::AuditWarning>(),
+        );
+        for warning in &pp.warnings {
+            builder.owned_string(&warning.message);
+            builder.string_vec_owned(&warning.rfc_refs);
+            builder.optional_string(warning.context.as_ref());
+        }
+
+        object_count += pp.objects.len() as u64;
+        builder.vec_header_with_capacity(
+            pp.objects.len(),
+            pp.objects.capacity(),
+            std::mem::size_of::<crate::audit::ObjectAuditEntry>(),
+        );
+        for object in &pp.objects {
+            builder.owned_string(&object.rsync_uri);
+            builder.owned_string(&object.sha256_hex);
+            if object.detail.is_some() {
+                object_detail_count += 1;
+            }
+            builder.optional_string(object.detail.as_ref());
+        }
+    }
+
+    builder.metric("object_audit_entry_count", object_count);
+    builder.metric("publication_point_warning_count", pp_warning_count);
+    builder.metric(
+        "publication_point_discovered_from_count",
+        pp_discovered_from_count,
+    );
+    builder.metric("object_detail_count", object_detail_count);
+    builder.finish()
+}
+
+fn estimate_vrps_graph(vrps: &[crate::validation::objects::Vrp]) -> ObjectGraphMemorySection {
+    let mut builder = ObjectGraphSectionBuilder::new("vrps");
+    builder.items(
+        vrps.len(),
+        std::mem::size_of::<crate::validation::objects::Vrp>(),
+    );
+    builder.metric("vrp_count", vrps.len() as u64);
+    builder.finish()
+}
+
+fn estimate_aspas_graph(
+    aspas: &[crate::validation::objects::AspaAttestation],
+) -> ObjectGraphMemorySection {
+    let mut builder = ObjectGraphSectionBuilder::new("aspas");
+    builder.items(
+        aspas.len(),
+        std::mem::size_of::<crate::validation::objects::AspaAttestation>(),
+    );
+    let mut providers_total = 0u64;
+    for aspa in aspas {
+        providers_total += aspa.provider_as_ids.len() as u64;
+        builder.vec_header_with_capacity(
+            aspa.provider_as_ids.len(),
+            aspa.provider_as_ids.capacity(),
+            std::mem::size_of::<u32>(),
+        );
+    }
+    builder.metric("aspa_count", aspas.len() as u64);
+    builder.metric("provider_asn_count", providers_total);
+    builder.finish()
+}
+
+fn estimate_router_keys_graph(
+    router_keys: &[crate::validation::objects::RouterKeyPayload],
+) -> ObjectGraphMemorySection {
+    let mut builder = ObjectGraphSectionBuilder::new("router_keys");
+    builder.items(
+        router_keys.len(),
+        std::mem::size_of::<crate::validation::objects::RouterKeyPayload>(),
+    );
+    for router_key in router_keys {
+        builder.byte_vec_owned(&router_key.ski);
+        builder.byte_vec_owned(&router_key.spki_der);
+        builder.owned_string(&router_key.source_object_uri);
+        builder.owned_string(&router_key.source_object_hash);
+        builder.owned_string(&router_key.source_ee_cert_hash);
+    }
+    builder.metric("router_key_count", router_keys.len() as u64);
+    builder.finish()
+}
+
+fn estimate_warnings_graph(
+    name: &str,
+    warnings: &[crate::report::Warning],
+) -> ObjectGraphMemorySection {
+    let mut builder = ObjectGraphSectionBuilder::new(name);
+    builder.items(
+        warnings.len(),
+        std::mem::size_of::<crate::report::Warning>(),
+    );
+    for warning in warnings {
+        builder.owned_string(&warning.message);
+        builder.vec_header_with_capacity(
+            warning.rfc_refs.len(),
+            warning.rfc_refs.capacity(),
+            std::mem::size_of::<crate::report::RfcRef>(),
+        );
+        builder.optional_string(warning.context.as_ref());
+    }
+    builder.metric("warning_count", warnings.len() as u64);
+    builder.finish()
+}
+
+fn estimate_downloads_graph(
+    downloads: &[crate::audit::AuditDownloadEvent],
+) -> ObjectGraphMemorySection {
+    let mut builder = ObjectGraphSectionBuilder::new("downloads");
+    builder.items(
+        downloads.len(),
+        std::mem::size_of::<crate::audit::AuditDownloadEvent>(),
+    );
+    let mut error_count = 0u64;
+    let mut bytes_count = 0u64;
+    let mut objects_stat_count = 0u64;
+    for event in downloads {
+        builder.owned_string(&event.uri);
+        builder.owned_string(&event.started_at_rfc3339_utc);
+        builder.owned_string(&event.finished_at_rfc3339_utc);
+        if event.error.is_some() {
+            error_count += 1;
+        }
+        if event.bytes.is_some() {
+            bytes_count += 1;
+        }
+        if event.objects.is_some() {
+            objects_stat_count += 1;
+        }
+        builder.optional_string(event.error.as_ref());
+    }
+    builder.metric("download_event_count", downloads.len() as u64);
+    builder.metric("download_error_count", error_count);
+    builder.metric("download_bytes_field_count", bytes_count);
+    builder.metric("download_objects_stat_count", objects_stat_count);
+    builder.finish()
+}
+
+fn estimate_current_repo_objects_graph(
+    objects: &[crate::current_repo_index::CurrentRepoObject],
+) -> ObjectGraphMemorySection {
+    let mut builder = ObjectGraphSectionBuilder::new("current_repo_objects");
+    builder.items(
+        objects.len(),
+        std::mem::size_of::<crate::current_repo_index::CurrentRepoObject>(),
+    );
+    let mut object_type_count = 0u64;
+    for object in objects {
+        builder.owned_string(&object.rsync_uri);
+        builder.owned_string(&object.current_hash_hex);
+        builder.owned_string(&object.repository_source);
+        if object.object_type.is_some() {
+            object_type_count += 1;
+        }
+        builder.optional_string(object.object_type.as_ref());
+    }
+    builder.metric("current_repo_object_count", objects.len() as u64);
+    builder.metric("current_repo_object_type_count", object_type_count);
+    builder.finish()
+}
+
+fn estimate_trust_anchor_graph(shared: &PostValidationShared) -> ObjectGraphMemorySection {
+    let mut builder = ObjectGraphSectionBuilder::new("trust_anchors_and_tal_inputs");
+    builder.items(
+        1,
+        std::mem::size_of::<crate::validation::from_tal::DiscoveredRootCaInstance>(),
+    );
+    estimate_discovered_root(&mut builder, &shared.discovery);
+    builder.items(
+        shared.discoveries.len(),
+        std::mem::size_of::<crate::validation::from_tal::DiscoveredRootCaInstance>(),
+    );
+    for discovery in shared.discoveries.iter() {
+        estimate_discovered_root(&mut builder, discovery);
+    }
+    builder.items(
+        shared.successful_tal_inputs.len(),
+        std::mem::size_of::<TalInputSpec>(),
+    );
+    for tal_input in shared.successful_tal_inputs.iter() {
+        estimate_tal_input(&mut builder, tal_input);
+    }
+    builder.metric("discoveries_count", shared.discoveries.len() as u64);
+    builder.metric(
+        "successful_tal_inputs_count",
+        shared.successful_tal_inputs.len() as u64,
+    );
+    builder.finish()
+}
+
+fn estimate_discovered_root(
+    builder: &mut ObjectGraphSectionBuilder,
+    discovery: &crate::validation::from_tal::DiscoveredRootCaInstance,
+) {
+    builder.optional_string(discovery.tal_url.as_ref());
+    estimate_trust_anchor(builder, &discovery.trust_anchor);
+    builder.owned_string(&discovery.ca_instance.rsync_base_uri);
+    builder.owned_string(&discovery.ca_instance.manifest_rsync_uri);
+    builder.owned_string(&discovery.ca_instance.publication_point_rsync_uri);
+    builder.optional_string(discovery.ca_instance.rrdp_notification_uri.as_ref());
+}
+
+fn estimate_trust_anchor(
+    builder: &mut ObjectGraphSectionBuilder,
+    trust_anchor: &crate::data_model::ta::TrustAnchor,
+) {
+    builder.byte_vec_owned(&trust_anchor.tal.raw);
+    builder.string_vec_owned(&trust_anchor.tal.comments);
+    builder.vec_header_with_capacity(
+        trust_anchor.tal.ta_uris.len(),
+        trust_anchor.tal.ta_uris.capacity(),
+        std::mem::size_of::<url::Url>(),
+    );
+    for uri in &trust_anchor.tal.ta_uris {
+        builder.string(uri.as_str());
+    }
+    builder.byte_vec_owned(&trust_anchor.tal.subject_public_key_info_der);
+    builder.byte_vec_owned(&trust_anchor.ta_certificate.raw_der);
+    if let Some(uri) = &trust_anchor.resolved_ta_uri {
+        builder.string(uri.as_str());
+    }
+}
+
+fn estimate_tal_input(builder: &mut ObjectGraphSectionBuilder, tal_input: &TalInputSpec) {
+    builder.owned_string(&tal_input.tal_id);
+    builder.owned_string(&tal_input.rir_id);
+    match &tal_input.source {
+        crate::parallel::types::TalSource::Url(url) => builder.owned_string(url),
+        crate::parallel::types::TalSource::DerBytes {
+            tal_url,
+            tal_bytes,
+            ta_der,
+        } => {
+            builder.owned_string(tal_url);
+            builder.byte_vec_owned(tal_bytes);
+            builder.byte_vec_owned(ta_der);
+        }
+        crate::parallel::types::TalSource::FilePath(path) => {
+            builder.string(&path.to_string_lossy());
+        }
+        crate::parallel::types::TalSource::FilePathWithTa { tal_path, ta_path } => {
+            builder.string(&tal_path.to_string_lossy());
+            builder.string(&ta_path.to_string_lossy());
+        }
+    }
+}
+
+fn estimate_ccr_accumulator_graph(
+    accumulator: Option<&CcrAccumulator>,
+) -> ObjectGraphMemorySection {
+    let mut builder = ObjectGraphSectionBuilder::new("ccr_accumulator");
+    if let Some(accumulator) = accumulator {
+        builder.items(1, std::mem::size_of::<CcrAccumulator>());
+        let stats = accumulator.memory_stats();
+        builder.heap_bytes(stats.estimated_heap_bytes as usize);
+        builder.metric("trust_anchor_count", stats.trust_anchor_count);
+        builder.metric("manifest_count", stats.manifest_count);
+        builder.metric("string_bytes", stats.string_bytes);
+        builder.metric("string_capacity_bytes", stats.string_capacity_bytes);
+        builder.metric("vec_payload_bytes", stats.vec_payload_bytes);
+        builder.metric("vec_capacity_bytes", stats.vec_capacity_bytes);
+        builder.metric("locations_der_count", stats.locations_der_count);
+        builder.metric("subordinate_ski_count", stats.subordinate_ski_count);
+        builder.metric("btree_key_capacity_bytes", stats.btree_key_capacity_bytes);
+        builder.metric("btree_entry_shallow_bytes", stats.btree_entry_shallow_bytes);
+    } else {
+        builder.metric("manifest_count", 0);
+    }
+    builder.finish()
+}
+
+#[cfg(test)]
 fn build_report(
     policy: &Policy,
     validation_time: time::OffsetDateTime,
@@ -1072,7 +1573,6 @@ fn build_report(
 
 #[derive(Clone, Debug, PartialEq, Eq)]
 struct ReportTaskOutput {
-    report: Option<AuditReportV2>,
     report_build_ms: u64,
     report_write_ms: Option<u64>,
 }
@@ -1080,7 +1580,6 @@ struct ReportTaskOutput {
 impl ReportTaskOutput {
     fn skipped() -> Self {
         Self {
-            report: None,
             report_build_ms: 0,
             report_write_ms: None,
         }
@@ -1094,23 +1593,21 @@ fn run_report_task(
     report_json_path: Option<&Path>,
     report_json_format: ReportJsonFormat,
 ) -> Result<ReportTaskOutput, String> {
-    let report_started = std::time::Instant::now();
-    let report = build_report(policy, validation_time, shared);
-    let report_build_ms = report_started.elapsed().as_millis() as u64;
-
-    let report_write_ms = if let Some(path) = report_json_path {
-        let started = std::time::Instant::now();
-        write_json(path, &report, report_json_format)?;
-        Some(started.elapsed().as_millis() as u64)
-    } else {
-        None
-    };
-
+    if let Some(path) = report_json_path {
+        let timing = write_report_json_from_shared(
+            path,
+            policy,
+            validation_time,
+            shared,
+            report_json_format,
+        )?;
         Ok(ReportTaskOutput {
-        report: Some(report),
-        report_build_ms,
-        report_write_ms,
+            report_build_ms: timing.build_ms,
+            report_write_ms: Some(timing.write_ms),
         })
+    } else {
+        Ok(ReportTaskOutput::skipped())
+    }
 }
 
 #[derive(Clone, Debug, PartialEq, Eq)]
@@ -1443,6 +1940,15 @@ pub fn run(argv: &[String]) -> Result<(), String> {
     };
 
     let total_started = std::time::Instant::now();
+    let mut memory_checkpoints: Vec<MemoryTelemetryCheckpoint> = Vec::new();
+    let mut malloc_trim_probes: Vec<MallocTrimProbe> = Vec::new();
+    let enable_memory_trim_probe = memory_trim_probe_enabled() || args.memory_trim_after_validation;
+    record_memory_checkpoint(
+        &mut memory_checkpoints,
+        "after_store_open",
+        &total_started,
+        store.as_ref(),
+    );
     let validation_started = std::time::Instant::now();
     let collect_current_repo_objects = false;
     let out = if delta_replay_mode {
@@ -1607,6 +2113,34 @@ pub fn run(argv: &[String]) -> Result<(), String> {
 
     let validation_ms = validation_started.elapsed().as_millis() as u64;
     let shared = PostValidationShared::from_run_output(out);
+    let vcir_storage_summary_enabled = vcir_storage_summary_enabled();
+    let vcir_storage_summary_started = std::time::Instant::now();
+    let vcir_storage = if config.persist_vcir && vcir_storage_summary_enabled {
+        Some(
+            store
+                .summarize_vcir_storage()
+                .map_err(|e| format!("summarize VCIR storage failed: {e}"))?,
+        )
+    } else {
+        None
+    };
+    let vcir_storage_summary_ms = (config.persist_vcir && vcir_storage_summary_enabled)
+        .then(|| vcir_storage_summary_started.elapsed().as_millis() as u64);
+    record_memory_checkpoint(
+        &mut memory_checkpoints,
+        "after_validation",
+        &total_started,
+        store.as_ref(),
+    );
+    if enable_memory_trim_probe {
+        malloc_trim_probes.push(crate::memory_telemetry::malloc_trim_probe());
+        record_memory_checkpoint(
+            &mut memory_checkpoints,
+            "after_validation_malloc_trim",
+            &total_started,
+            store.as_ref(),
+        );
+    }
 
     if let Some((_out_dir, t)) = timing.as_ref() {
         t.record_count("instances_processed", shared.instances_processed as u64);
@@ -1705,7 +2239,21 @@ pub fn run(argv: &[String]) -> Result<(), String> {
     };
     let report_output = report_result?;
     let ccr_output = ccr_result?;
-    let report = report_output.report;
+    record_memory_checkpoint(
+        &mut memory_checkpoints,
+        "after_report_and_ccr",
+        &total_started,
+        store.as_ref(),
+    );
+    if enable_memory_trim_probe {
+        malloc_trim_probes.push(crate::memory_telemetry::malloc_trim_probe());
+        record_memory_checkpoint(
+            &mut memory_checkpoints,
+            "after_report_and_ccr_malloc_trim",
+            &total_started,
+            store.as_ref(),
+        );
+    }
     let report_build_ms = report_output.report_build_ms;
     let report_write_ms = report_output.report_write_ms;
     let ccr_build_ms = ccr_output.ccr_build_ms;
@@ -1723,6 +2271,12 @@ pub fn run(argv: &[String]) -> Result<(), String> {
     )?;
     let compare_view_build_ms = compare_view_output.build_ms;
     let compare_view_write_ms = compare_view_output.write_ms;
+    record_memory_checkpoint(
+        &mut memory_checkpoints,
+        "after_compare_view",
+        &total_started,
+        store.as_ref(),
+    );
 
     let mut cir_build_cir_ms = None;
     let mut cir_write_cir_ms = None;
@@ -1775,7 +2329,19 @@ pub fn run(argv: &[String]) -> Result<(), String> {
             summary.timing.write_cir_ms,
             summary.timing.total_ms
         );
+        record_memory_checkpoint(
+            &mut memory_checkpoints,
+            "after_cir",
+            &total_started,
+            store.as_ref(),
+        );
     }
+    record_memory_checkpoint(
+        &mut memory_checkpoints,
+        "before_stage_timing",
+        &total_started,
+        store.as_ref(),
+    );
     let stage_timing = RunStageTiming {
         validation_ms,
         report_build_ms,
@@ -1796,6 +2362,13 @@ pub fn run(argv: &[String]) -> Result<(), String> {
         rrdp_download_ms_total,
         rsync_download_ms_total,
         download_bytes_total,
+        vcir_storage_summary_ms,
+        vcir_storage,
+        memory_telemetry: Some(MemoryTelemetrySummary {
+            checkpoints: memory_checkpoints,
+            object_graph: Some(estimate_shared_object_graph(&shared)),
+            malloc_trim_probes,
+        }),
     };
     let stage_timing_anchor_path = args
         .report_json_path
@@ -1849,11 +2422,7 @@ pub fn run(argv: &[String]) -> Result<(), String> {
         eprintln!("analysis: wrote {}", pb_path.display());
     }
 
-    if let Some(report) = report.as_ref() {
-        print_summary(report);
-    } else {
     print_summary_from_shared(validation_time, &shared);
-    }
     Ok(())
 }
 

src/cli/output.rs

@@ -1,7 +1,10 @@
 use std::io::BufWriter;
 use std::path::Path;
 
-use crate::audit::AuditReportV2;
+use serde::Serialize;
+use serde::ser::SerializeSeq;
+
+use crate::audit::{AspaOutput, AuditRunMeta, AuditWarning, VrpOutput};
 use crate::ccr::canonical_vrp_prefix;
 
 use super::{PostValidationShared, RunStageTiming};
@@ -12,9 +15,9 @@ pub(super) enum ReportJsonFormat {
     Compact,
 }
 
-pub(super) fn write_json(
+pub(super) fn write_json<T: Serialize>(
     path: &Path,
-    report: &AuditReportV2,
+    report: &T,
     format: ReportJsonFormat,
 ) -> Result<(), String> {
     let f = std::fs::File::create(path)
@@ -28,6 +31,128 @@ pub(super) fn write_json(
     Ok(())
 }
 
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub(super) struct ReportJsonWriteTiming {
+    pub(super) build_ms: u64,
+    pub(super) write_ms: u64,
+}
+
+#[derive(Serialize)]
+struct BorrowedAuditReportV2<'a> {
+    format_version: u32,
+    meta: AuditRunMeta,
+    policy: &'a crate::policy::Policy,
+    tree: BorrowedTreeSummary<'a>,
+    publication_points: &'a [crate::audit::PublicationPointAudit],
+    vrps: VrpReportSequence<'a>,
+    aspas: AspaReportSequence<'a>,
+    downloads: &'a [crate::audit::AuditDownloadEvent],
+    download_stats: &'a crate::audit::AuditDownloadStats,
+    repo_sync_stats: crate::audit::AuditRepoSyncStats,
+}
+
+#[derive(Serialize)]
+struct BorrowedTreeSummary<'a> {
+    instances_processed: usize,
+    instances_failed: usize,
+    warnings: WarningReportSequence<'a>,
+}
+
+struct WarningReportSequence<'a>(&'a [crate::report::Warning]);
+
+impl Serialize for WarningReportSequence<'_> {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        let mut seq = serializer.serialize_seq(Some(self.0.len()))?;
+        for warning in self.0 {
+            seq.serialize_element(&AuditWarning::from(warning))?;
+        }
+        seq.end()
+    }
+}
+
+struct VrpReportSequence<'a>(&'a [crate::validation::objects::Vrp]);
+
+impl Serialize for VrpReportSequence<'_> {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        let mut seq = serializer.serialize_seq(Some(self.0.len()))?;
+        for vrp in self.0 {
+            seq.serialize_element(&VrpOutput {
+                asn: vrp.asn,
+                prefix: crate::audit::format_roa_ip_prefix(&vrp.prefix),
+                max_length: vrp.max_length,
+            })?;
+        }
+        seq.end()
+    }
+}
+
+struct AspaReportSequence<'a>(&'a [crate::validation::objects::AspaAttestation]);
+
+impl Serialize for AspaReportSequence<'_> {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        let mut seq = serializer.serialize_seq(Some(self.0.len()))?;
+        for aspa in self.0 {
+            seq.serialize_element(&AspaOutput {
+                customer_as_id: aspa.customer_as_id,
+                provider_as_ids: aspa.provider_as_ids.clone(),
+            })?;
+        }
+        seq.end()
+    }
+}
+
+pub(super) fn write_report_json_from_shared(
+    path: &Path,
+    policy: &crate::policy::Policy,
+    validation_time: time::OffsetDateTime,
+    shared: &PostValidationShared,
+    format: ReportJsonFormat,
+) -> Result<ReportJsonWriteTiming, String> {
+    use time::format_description::well_known::Rfc3339;
+
+    let build_started = std::time::Instant::now();
+    let validation_time_rfc3339_utc = validation_time
+        .to_offset(time::UtcOffset::UTC)
+        .format(&Rfc3339)
+        .expect("format validation_time");
+    let repo_sync_stats = super::build_repo_sync_stats(shared.publication_points.as_ref());
+    let report = BorrowedAuditReportV2 {
+        format_version: 2,
+        meta: AuditRunMeta {
+            validation_time_rfc3339_utc,
+        },
+        policy,
+        tree: BorrowedTreeSummary {
+            instances_processed: shared.instances_processed,
+            instances_failed: shared.instances_failed,
+            warnings: WarningReportSequence(shared.tree_warnings.as_ref()),
+        },
+        publication_points: shared.publication_points.as_ref(),
+        vrps: VrpReportSequence(shared.vrps.as_ref()),
+        aspas: AspaReportSequence(shared.aspas.as_ref()),
+        downloads: shared.downloads.as_ref(),
+        download_stats: &shared.download_stats,
+        repo_sync_stats,
+    };
+    let build_ms = build_started.elapsed().as_millis() as u64;
+
+    let write_started = std::time::Instant::now();
+    write_json(path, &report, format)?;
+    Ok(ReportJsonWriteTiming {
+        build_ms,
+        write_ms: write_started.elapsed().as_millis() as u64,
+    })
+}
+
 #[derive(Clone, Debug, PartialEq, Eq)]
 pub(super) struct CompareViewTaskOutput {
     pub(super) build_ms: Option<u64>,

src/cli/tests.rs

@@ -1,4 +1,12 @@
 use super::*;
+use crate::memory_telemetry::{
+    MemoryTelemetryCheckpoint, MemoryTelemetrySummary, ProcessMemorySnapshot,
+};
+use crate::storage::{
+    RocksDbMemorySnapshot, RocksDbMemoryTotals, VcirCcrProjectionSizeBreakdown,
+    VcirChildResourceSizeBreakdown, VcirCoreFieldSizeBreakdown, VcirFieldSizeBreakdown,
+    VcirStorageEntrySummary, VcirStorageSummary,
+};
 
 #[test]
 fn parse_help_returns_usage() {
@@ -9,6 +17,7 @@ fn parse_help_returns_usage() {
     assert!(err.contains("--rsync-mirror-root"), "{err}");
     assert!(err.contains("--rsync-scope"), "{err}");
     assert!(err.contains("--parallel-phase2-object-workers"), "{err}");
+    assert!(err.contains("--memory-trim-after-validation"), "{err}");
     assert!(!err.contains("--parallel-phase1"), "{err}");
     assert!(!err.contains("--parallel-phase2          "), "{err}");
 }
@@ -82,6 +91,37 @@ fn parse_accepts_ccr_out_path() {
     );
 }
 
+#[test]
+fn parse_accepts_memory_trim_after_validation() {
+    let argv = vec![
+        "rpki".to_string(),
+        "--db".to_string(),
+        "db".to_string(),
+        "--tal-path".to_string(),
+        "x.tal".to_string(),
+        "--ta-path".to_string(),
+        "x.cer".to_string(),
+        "--memory-trim-after-validation".to_string(),
+    ];
+    let args = parse_args(&argv).expect("parse args");
+    assert!(args.memory_trim_after_validation);
+}
+
+#[test]
+fn parse_disables_memory_trim_after_validation_by_default() {
+    let argv = vec![
+        "rpki".to_string(),
+        "--db".to_string(),
+        "db".to_string(),
+        "--tal-path".to_string(),
+        "x.tal".to_string(),
+        "--ta-path".to_string(),
+        "x.cer".to_string(),
+    ];
+    let args = parse_args(&argv).expect("parse args");
+    assert!(!args.memory_trim_after_validation);
+}
+
 #[test]
 fn parse_accepts_report_json_compact_when_report_json_is_set() {
     let argv = vec![
@@ -1414,13 +1454,14 @@ fn run_report_task_and_stage_timing_work() {
     )
     .expect("run report task");
 
-    let report = report_output.report.as_ref().expect("report built");
-    assert_eq!(report.vrps.len(), 2);
-    assert_eq!(report.aspas.len(), 1);
     assert!(report_output.report_write_ms.is_some());
 
     let report_json = std::fs::read_to_string(&report_path).expect("read report json");
     assert!(!report_json.contains('\n'), "{report_json}");
+    let report: serde_json::Value =
+        serde_json::from_str(&report_json).expect("parse compact report json");
+    assert_eq!(report["vrps"].as_array().unwrap().len(), 2);
+    assert_eq!(report["aspas"].as_array().unwrap().len(), 1);
 
     let stage_timing = RunStageTiming {
         validation_ms: 1,
@@ -1442,12 +1483,62 @@ fn run_report_task_and_stage_timing_work() {
         rrdp_download_ms_total: 13,
         rsync_download_ms_total: 14,
         download_bytes_total: 15,
+        vcir_storage_summary_ms: Some(16),
+        vcir_storage: Some(VcirStorageSummary {
+            entry_count: 2,
+            vcir_value_bytes: 100,
+            vcir_value_bytes_max: 60,
+            vcir_value_bytes_max_manifest_rsync_uri: Some(
+                "rsync://example.test/repo/max.mft".to_string(),
+            ),
+            core_fields: VcirCoreFieldSizeBreakdown {
+                manifest_rsync_uri_bytes: 10,
+                ..VcirCoreFieldSizeBreakdown::default()
+            },
+            ccr_projection: VcirCcrProjectionSizeBreakdown {
+                manifest_sha256_bytes: 32,
+                ..VcirCcrProjectionSizeBreakdown::default()
+            },
+            child_resources: VcirChildResourceSizeBreakdown {
+                effective_ip_resource_cbor_bytes: 12,
+                effective_as_resource_cbor_bytes: 6,
+            },
+            field_sizes: VcirFieldSizeBreakdown {
+                local_output_count: 1,
+                local_output_payload_json_bytes: 70,
+                local_output_payload_typed_body_bytes: 20,
+                ..VcirFieldSizeBreakdown::default()
+            },
+            local_output_old_projection_bytes: 80,
+            local_output_typed_projection_bytes: 30,
+            local_output_projection_saved_bytes: 50,
+            top_entries_by_vcir_value_bytes: vec![VcirStorageEntrySummary {
+                manifest_rsync_uri: "rsync://example.test/repo/max.mft".to_string(),
+                vcir_value_bytes: 60,
+                local_vrp_count: 1,
+                local_aspa_count: 0,
+                local_router_key_count: 0,
+                accepted_object_count: 1,
+                rejected_object_count: 0,
+                child_count: 0,
+                core_fields: VcirCoreFieldSizeBreakdown::default(),
+                ccr_projection: VcirCcrProjectionSizeBreakdown::default(),
+                child_resources: VcirChildResourceSizeBreakdown::default(),
+                field_sizes: VcirFieldSizeBreakdown::default(),
+                local_output_old_projection_bytes: 1,
+                local_output_typed_projection_bytes: 1,
+                local_output_projection_saved_bytes: 0,
+            }],
+        }),
+        memory_telemetry: None,
     };
     write_stage_timing(Some(&report_path), &stage_timing).expect("write stage timing");
     let stage_timing_json =
         std::fs::read_to_string(dir.path().join("stage-timing.json")).expect("read timing");
     assert!(stage_timing_json.contains("\"validation_ms\""));
     assert!(stage_timing_json.contains("\"ccr_build_ms\""));
+    assert!(stage_timing_json.contains("\"vcir_storage\""));
+    assert!(stage_timing_json.contains("\"local_output_projection_saved_bytes\""));
 
     let ccr_path = dir.path().join("result.ccr");
     write_stage_timing(Some(&ccr_path), &stage_timing).expect("write stage timing via ccr path");
@@ -1457,11 +1548,136 @@ fn run_report_task_and_stage_timing_work() {
     );
 
     let skipped = ReportTaskOutput::skipped();
-    assert!(skipped.report.is_none());
     assert_eq!(skipped.report_build_ms, 0);
     assert!(skipped.report_write_ms.is_none());
 }
 
+#[test]
+fn stage_timing_serializes_memory_telemetry() {
+    let dir = tempfile::tempdir().expect("tmpdir");
+    let report_path = dir.path().join("report.json");
+    let stage_timing = RunStageTiming {
+        validation_ms: 1,
+        report_build_ms: 2,
+        report_write_ms: None,
+        ccr_build_ms: None,
+        ccr_build_breakdown: None,
+        ccr_write_ms: None,
+        compare_view_build_ms: None,
+        compare_view_write_ms: None,
+        cir_build_cir_ms: None,
+        cir_write_cir_ms: None,
+        cir_total_ms: None,
+        total_ms: 3,
+        publication_points: 4,
+        repo_sync_ms_total: 5,
+        publication_point_repo_sync_ms_total: 6,
+        download_event_count: 7,
+        rrdp_download_ms_total: 8,
+        rsync_download_ms_total: 9,
+        download_bytes_total: 10,
+        vcir_storage_summary_ms: None,
+        vcir_storage: None,
+        memory_telemetry: Some(MemoryTelemetrySummary {
+            checkpoints: vec![MemoryTelemetryCheckpoint {
+                label: "after_validation".to_string(),
+                elapsed_ms: 11,
+                process: ProcessMemorySnapshot {
+                    label: "after_validation".to_string(),
+                    vm_rss_kb: Some(12),
+                    vm_size_kb: None,
+                    vm_data_kb: None,
+                    vm_swap_kb: None,
+                    rss_anon_kb: Some(13),
+                    rss_file_kb: None,
+                    rss_shmem_kb: None,
+                    threads: Some(14),
+                    fd_count: Some(15),
+                    smaps_rollup: None,
+                    smaps_mapping_summary: None,
+                    errors: Vec::new(),
+                },
+                rocksdb: RocksDbMemorySnapshot {
+                    databases: Vec::new(),
+                    totals: RocksDbMemoryTotals {
+                        cur_size_all_mem_tables: 16,
+                        size_all_mem_tables: 17,
+                        estimate_table_readers_mem: 18,
+                        block_cache_capacity: 19,
+                        block_cache_usage: 20,
+                        block_cache_pinned_usage: 21,
+                    },
+                },
+            }],
+            object_graph: None,
+            malloc_trim_probes: Vec::new(),
+        }),
+    };
+
+    write_stage_timing(Some(&report_path), &stage_timing).expect("write stage timing");
+    let value: serde_json::Value = serde_json::from_str(
+        &std::fs::read_to_string(dir.path().join("stage-timing.json")).unwrap(),
+    )
+    .expect("parse stage timing json");
+    let checkpoint = &value["memory_telemetry"]["checkpoints"][0];
+    assert_eq!(checkpoint["label"], "after_validation");
+    assert_eq!(checkpoint["process"]["vm_rss_kb"], 12);
+    assert_eq!(
+        checkpoint["rocksdb"]["totals"]["cur_size_all_mem_tables"],
+        16
+    );
+    assert!(
+        value["memory_telemetry"]
+            .as_object()
+            .expect("memory telemetry object")
+            .get("malloc_trim_probes")
+            .is_none()
+    );
+}
+
+#[test]
+fn shared_object_graph_estimate_counts_audit_and_outputs() {
+    let mut shared = synthetic_post_validation_shared();
+    let mut publication_points = shared
+        .publication_points
+        .iter()
+        .cloned()
+        .collect::<Vec<_>>();
+    publication_points[0].rsync_base_uri = "rsync://example.test/repo/".to_string();
+    publication_points[0].manifest_rsync_uri = "rsync://example.test/repo/a.mft".to_string();
+    publication_points[0].publication_point_rsync_uri = "rsync://example.test/repo/".to_string();
+    publication_points[0].objects = vec![crate::audit::ObjectAuditEntry {
+        rsync_uri: "rsync://example.test/repo/a.roa".to_string(),
+        sha256_hex: "11".repeat(32),
+        kind: crate::audit::AuditObjectKind::Roa,
+        result: crate::audit::AuditObjectResult::Ok,
+        detail: None,
+    }];
+    shared.publication_points = publication_points.into();
+
+    let graph = estimate_shared_object_graph(&shared);
+    let publication_points_section = graph
+        .sections
+        .iter()
+        .find(|section| section.name == "publication_points")
+        .expect("publication points section");
+    let object_count = publication_points_section
+        .details
+        .iter()
+        .find(|metric| metric.name == "object_audit_entry_count")
+        .expect("object count metric");
+    assert_eq!(object_count.value, 1);
+    assert!(publication_points_section.estimated_bytes > 0);
+
+    let vrps_section = graph
+        .sections
+        .iter()
+        .find(|section| section.name == "vrps")
+        .expect("vrps section");
+    assert_eq!(vrps_section.item_count, 2);
+    assert!(graph.total_estimated_bytes >= publication_points_section.estimated_bytes);
+}
+
 #[test]
 fn run_compare_view_task_writes_csv_from_shared_output() {
     let shared = synthetic_post_validation_shared();

src/data_model/roa.rs

@@ -217,7 +217,9 @@ impl RoaObject {
     }
 }
 
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
+#[derive(
+    Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize,
+)]
 pub enum RoaAfi {
     Ipv4,
     Ipv6,

src/lib.rs

@@ -9,8 +9,6 @@ pub mod audit;
 #[cfg(feature = "full")]
 pub mod audit_downloads;
 #[cfg(feature = "full")]
-pub mod audit_trace;
-#[cfg(feature = "full")]
 pub mod blob_store;
 #[cfg(feature = "full")]
 pub mod cli;
@@ -19,6 +17,8 @@ pub mod current_repo_index;
 #[cfg(feature = "full")]
 pub mod fetch;
 #[cfg(feature = "full")]
+pub mod memory_telemetry;
+#[cfg(feature = "full")]
 pub mod parallel;
 #[cfg(feature = "full")]
 pub mod policy;

src/memory_telemetry.rs

@@ -0,0 +1,348 @@
+use serde::Serialize;
+
+use crate::storage::RocksDbMemorySnapshot;
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+pub struct MallocTrimProbe {
+    pub supported: bool,
+    pub return_value: Option<i32>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+pub struct ProcessMemorySnapshot {
+    pub label: String,
+    pub vm_rss_kb: Option<u64>,
+    pub vm_size_kb: Option<u64>,
+    pub vm_data_kb: Option<u64>,
+    pub vm_swap_kb: Option<u64>,
+    pub rss_anon_kb: Option<u64>,
+    pub rss_file_kb: Option<u64>,
+    pub rss_shmem_kb: Option<u64>,
+    pub threads: Option<u64>,
+    pub fd_count: Option<u64>,
+    pub smaps_rollup: Option<SmapsRollupSnapshot>,
+    pub smaps_mapping_summary: Option<SmapsMappingSummary>,
+    pub errors: Vec<String>,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
+pub struct SmapsRollupSnapshot {
+    pub rss_kb: Option<u64>,
+    pub pss_kb: Option<u64>,
+    pub shared_clean_kb: Option<u64>,
+    pub shared_dirty_kb: Option<u64>,
+    pub private_clean_kb: Option<u64>,
+    pub private_dirty_kb: Option<u64>,
+    pub anonymous_kb: Option<u64>,
+    pub swap_kb: Option<u64>,
+    pub swap_pss_kb: Option<u64>,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
+pub struct SmapsMappingSummary {
+    pub heap: SmapsMappingCategory,
+    pub anonymous_mmap: SmapsMappingCategory,
+    pub file_backed: SmapsMappingCategory,
+    pub stack: SmapsMappingCategory,
+    pub special: SmapsMappingCategory,
+    pub total: SmapsMappingCategory,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
+pub struct SmapsMappingCategory {
+    pub mappings: u64,
+    pub size_kb: u64,
+    pub rss_kb: u64,
+    pub pss_kb: u64,
+    pub private_clean_kb: u64,
+    pub private_dirty_kb: u64,
+    pub anonymous_kb: u64,
+    pub largest_mapping_rss_kb: u64,
+    pub large_mapping_count_64m: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+pub struct MemoryTelemetryCheckpoint {
+    pub label: String,
+    pub elapsed_ms: u64,
+    pub process: ProcessMemorySnapshot,
+    pub rocksdb: RocksDbMemorySnapshot,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
+pub struct MemoryTelemetrySummary {
+    pub checkpoints: Vec<MemoryTelemetryCheckpoint>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub object_graph: Option<ObjectGraphMemorySummary>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub malloc_trim_probes: Vec<MallocTrimProbe>,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
+pub struct ObjectGraphMemorySummary {
+    pub captured_at_label: String,
+    pub total_estimated_bytes: u64,
+    pub sections: Vec<ObjectGraphMemorySection>,
+    pub notes: Vec<String>,
+}
+
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
+pub struct ObjectGraphMemorySection {
+    pub name: String,
+    pub item_count: u64,
+    pub shallow_bytes: u64,
+    pub heap_bytes: u64,
+    pub estimated_bytes: u64,
+    pub string_count: u64,
+    pub string_bytes: u64,
+    pub string_capacity_bytes: u64,
+    pub vec_count: u64,
+    pub vec_heap_bytes: u64,
+    pub vec_capacity_bytes: u64,
+    pub details: Vec<ObjectGraphMemoryMetric>,
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
+pub struct ObjectGraphMemoryMetric {
+    pub name: String,
+    pub value: u64,
+}
+
+pub fn process_memory_snapshot(label: impl Into<String>) -> ProcessMemorySnapshot {
+    let label = label.into();
+    let mut snapshot = ProcessMemorySnapshot {
+        label,
+        vm_rss_kb: None,
+        vm_size_kb: None,
+        vm_data_kb: None,
+        vm_swap_kb: None,
+        rss_anon_kb: None,
+        rss_file_kb: None,
+        rss_shmem_kb: None,
+        threads: None,
+        fd_count: current_fd_count(),
+        smaps_rollup: None,
+        smaps_mapping_summary: None,
+        errors: Vec::new(),
+    };
+
+    match std::fs::read_to_string("/proc/self/status") {
+        Ok(status) => parse_status(&status, &mut snapshot),
+        Err(err) => snapshot
+            .errors
+            .push(format!("read /proc/self/status failed: {err}")),
+    }
+
+    match std::fs::read_to_string("/proc/self/smaps_rollup") {
+        Ok(smaps) => snapshot.smaps_rollup = Some(parse_smaps_rollup(&smaps)),
+        Err(err) => snapshot
+            .errors
+            .push(format!("read /proc/self/smaps_rollup failed: {err}")),
+    }
+
+    match std::fs::read_to_string("/proc/self/smaps") {
+        Ok(smaps) => snapshot.smaps_mapping_summary = Some(parse_smaps_mapping_summary(&smaps)),
+        Err(err) => snapshot
+            .errors
+            .push(format!("read /proc/self/smaps failed: {err}")),
+    }
+
+    snapshot
+}
+
+pub fn malloc_trim_probe() -> MallocTrimProbe {
+    #[cfg(all(target_os = "linux", target_env = "gnu"))]
+    {
+        MallocTrimProbe {
+            supported: true,
+            return_value: Some(unsafe { malloc_trim(0) }),
+        }
+    }
+    #[cfg(not(all(target_os = "linux", target_env = "gnu")))]
+    {
+        MallocTrimProbe {
+            supported: false,
+            return_value: None,
+        }
+    }
+}
+
+#[cfg(all(target_os = "linux", target_env = "gnu"))]
+unsafe extern "C" {
+    fn malloc_trim(pad: usize) -> i32;
+}
+
+fn current_fd_count() -> Option<u64> {
+    std::fs::read_dir("/proc/self/fd")
+        .ok()
+        .map(|entries| entries.filter_map(Result::ok).count() as u64)
+}
+
+fn parse_status(status: &str, snapshot: &mut ProcessMemorySnapshot) {
+    for line in status.lines() {
+        let Some((key, value)) = line.split_once(':') else {
+            continue;
+        };
+        let parsed = parse_kb_or_plain_u64(value);
+        match key {
+            "VmRSS" => snapshot.vm_rss_kb = parsed,
+            "VmSize" => snapshot.vm_size_kb = parsed,
+            "VmData" => snapshot.vm_data_kb = parsed,
+            "VmSwap" => snapshot.vm_swap_kb = parsed,
+            "RssAnon" => snapshot.rss_anon_kb = parsed,
+            "RssFile" => snapshot.rss_file_kb = parsed,
+            "RssShmem" => snapshot.rss_shmem_kb = parsed,
+            "Threads" => snapshot.threads = parsed,
+            _ => {}
+        }
+    }
+}
+
+fn parse_smaps_rollup(smaps: &str) -> SmapsRollupSnapshot {
+    let mut snapshot = SmapsRollupSnapshot::default();
+    for line in smaps.lines() {
+        let Some((key, value)) = line.split_once(':') else {
+            continue;
+        };
+        let parsed = parse_kb_or_plain_u64(value);
+        match key {
+            "Rss" => snapshot.rss_kb = parsed,
+            "Pss" => snapshot.pss_kb = parsed,
+            "Shared_Clean" => snapshot.shared_clean_kb = parsed,
+            "Shared_Dirty" => snapshot.shared_dirty_kb = parsed,
+            "Private_Clean" => snapshot.private_clean_kb = parsed,
+            "Private_Dirty" => snapshot.private_dirty_kb = parsed,
+            "Anonymous" => snapshot.anonymous_kb = parsed,
+            "Swap" => snapshot.swap_kb = parsed,
+            "SwapPss" => snapshot.swap_pss_kb = parsed,
+            _ => {}
+        }
+    }
+    snapshot
+}
+
+fn parse_smaps_mapping_summary(smaps: &str) -> SmapsMappingSummary {
+    let mut summary = SmapsMappingSummary::default();
+    let mut current_path = String::new();
+    let mut current = SmapsMappingCategory::default();
+    let mut have_mapping = false;
+
+    for line in smaps.lines() {
+        if is_smaps_mapping_header(line) {
+            if have_mapping {
+                add_mapping(&mut summary, &current_path, &current);
+            }
+            current_path = smaps_header_path(line);
+            current = SmapsMappingCategory {
+                mappings: 1,
+                ..SmapsMappingCategory::default()
+            };
+            have_mapping = true;
+            continue;
+        }
+
+        if !have_mapping {
+            continue;
+        }
+
+        let Some((key, value)) = line.split_once(':') else {
+            continue;
+        };
+        let parsed = parse_kb_or_plain_u64(value).unwrap_or(0);
+        match key {
+            "Size" => current.size_kb = parsed,
+            "Rss" => current.rss_kb = parsed,
+            "Pss" => current.pss_kb = parsed,
+            "Private_Clean" => current.private_clean_kb = parsed,
+            "Private_Dirty" => current.private_dirty_kb = parsed,
+            "Anonymous" => current.anonymous_kb = parsed,
+            _ => {}
+        }
+    }
+
+    if have_mapping {
+        add_mapping(&mut summary, &current_path, &current);
+    }
+
+    summary
+}
+
+fn is_smaps_mapping_header(line: &str) -> bool {
+    let mut parts = line.split_whitespace();
+    let Some(range) = parts.next() else {
+        return false;
+    };
+    let Some(perms) = parts.next() else {
+        return false;
+    };
+    let Some((start, end)) = range.split_once('-') else {
+        return false;
+    };
+    !start.is_empty()
+        && !end.is_empty()
+        && start.as_bytes().iter().all(u8::is_ascii_hexdigit)
+        && end.as_bytes().iter().all(u8::is_ascii_hexdigit)
+        && perms.len() == 4
+        && perms
+            .as_bytes()
+            .iter()
+            .all(|b| matches!(b, b'r' | b'w' | b'x' | b's' | b'p' | b'-'))
+}
+
+fn smaps_header_path(line: &str) -> String {
+    line.split_whitespace()
+        .skip(5)
+        .collect::<Vec<_>>()
+        .join(" ")
+}
+
+fn add_mapping(summary: &mut SmapsMappingSummary, path: &str, mapping: &SmapsMappingCategory) {
+    add_category(&mut summary.total, mapping);
+    match mapping_category(path) {
+        MappingCategory::Heap => add_category(&mut summary.heap, mapping),
+        MappingCategory::AnonymousMmap => add_category(&mut summary.anonymous_mmap, mapping),
+        MappingCategory::FileBacked => add_category(&mut summary.file_backed, mapping),
+        MappingCategory::Stack => add_category(&mut summary.stack, mapping),
+        MappingCategory::Special => add_category(&mut summary.special, mapping),
+    }
+}
+
+fn add_category(target: &mut SmapsMappingCategory, source: &SmapsMappingCategory) {
+    target.mappings += source.mappings;
+    target.size_kb += source.size_kb;
+    target.rss_kb += source.rss_kb;
+    target.pss_kb += source.pss_kb;
+    target.private_clean_kb += source.private_clean_kb;
+    target.private_dirty_kb += source.private_dirty_kb;
+    target.anonymous_kb += source.anonymous_kb;
+    target.largest_mapping_rss_kb = target.largest_mapping_rss_kb.max(source.rss_kb);
+    if source.rss_kb >= 64 * 1024 {
+        target.large_mapping_count_64m += source.mappings;
+    }
+}
+
+enum MappingCategory {
+    Heap,
+    AnonymousMmap,
+    FileBacked,
+    Stack,
+    Special,
+}
+
+fn mapping_category(path: &str) -> MappingCategory {
+    if path == "[heap]" {
+        MappingCategory::Heap
+    } else if path.starts_with("[stack") {
+        MappingCategory::Stack
+    } else if path.is_empty() {
+        MappingCategory::AnonymousMmap
+    } else if path.starts_with('/') {
+        MappingCategory::FileBacked
+    } else {
+        MappingCategory::Special
+    }
+}
+
+fn parse_kb_or_plain_u64(value: &str) -> Option<u64> {
+    value.split_whitespace().next()?.parse::<u64>().ok()
+}

src/storage/config.rs

@@ -5,7 +5,6 @@ pub const CF_RAW_BY_HASH: &str = "raw_by_hash";
 pub const CF_RAW_BLOB: &str = "raw_blob";
 pub const CF_VCIR: &str = "vcir";
 pub const CF_MANIFEST_REPLAY_META: &str = "manifest_replay_meta";
-pub const CF_AUDIT_RULE_INDEX: &str = "audit_rule_index";
 pub const CF_RRDP_SOURCE: &str = "rrdp_source";
 pub const CF_RRDP_SOURCE_MEMBER: &str = "rrdp_source_member";
 pub const CF_RRDP_URI_OWNER: &str = "rrdp_uri_owner";
@@ -16,7 +15,6 @@ pub const ALL_COLUMN_FAMILY_NAMES: &[&str] = &[
     CF_RAW_BLOB,
     CF_VCIR,
     CF_MANIFEST_REPLAY_META,
-    CF_AUDIT_RULE_INDEX,
     CF_RRDP_SOURCE,
     CF_RRDP_SOURCE_MEMBER,
     CF_RRDP_URI_OWNER,
@@ -27,14 +25,12 @@ pub(super) const RAW_BY_HASH_KEY_PREFIX: &str = "rawbyhash:";
 pub(super) const RAW_BLOB_KEY_PREFIX: &str = "rawblob:";
 pub(super) const VCIR_KEY_PREFIX: &str = "vcir:";
 pub(super) const MANIFEST_REPLAY_META_KEY_PREFIX: &str = "manifest_replay_meta:";
-pub(super) const AUDIT_ROA_RULE_KEY_PREFIX: &str = "audit:roa_rule:";
-pub(super) const AUDIT_ASPA_RULE_KEY_PREFIX: &str = "audit:aspa_rule:";
-pub(super) const AUDIT_ROUTER_KEY_RULE_KEY_PREFIX: &str = "audit:router_key_rule:";
 pub(super) const RRDP_SOURCE_KEY_PREFIX: &str = "rrdp_source:";
 pub(super) const RRDP_SOURCE_MEMBER_KEY_PREFIX: &str = "rrdp_source_member:";
 pub(super) const RRDP_URI_OWNER_KEY_PREFIX: &str = "rrdp_uri_owner:";
 
 const WORK_DB_BLOB_MODE_ENV: &str = "RPKI_WORK_DB_BLOB_MODE";
+const WORK_DB_MEMORY_PROFILE_ENV: &str = "RPKI_WORK_DB_MEMORY_PROFILE";
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub(super) enum WorkDbBlobMode {
@@ -43,6 +39,12 @@ pub(super) enum WorkDbBlobMode {
     Lz4,
 }
 
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub(super) enum WorkDbMemoryProfile {
+    Default,
+    Compact,
+}
+
 pub(super) fn parse_work_db_blob_mode(raw: &str) -> Option<WorkDbBlobMode> {
     match raw.trim().to_ascii_lowercase().as_str() {
         "" | "default" => Some(default_work_db_blob_mode()),
@@ -74,8 +76,36 @@ pub(super) fn work_db_blob_mode_from_env() -> WorkDbBlobMode {
     }
 }
 
-pub(super) fn configure_work_db_options(opts: &mut Options, blob_mode: WorkDbBlobMode) {
+pub(super) fn parse_work_db_memory_profile(raw: &str) -> Option<WorkDbMemoryProfile> {
+    match raw.trim().to_ascii_lowercase().as_str() {
+        "" | "default" | "off" | "none" => Some(WorkDbMemoryProfile::Default),
+        "compact" | "low" | "low_memory" | "low-memory" => Some(WorkDbMemoryProfile::Compact),
+        _ => None,
+    }
+}
+
+pub(super) fn work_db_memory_profile_from_env() -> WorkDbMemoryProfile {
+    let Ok(raw) = std::env::var(WORK_DB_MEMORY_PROFILE_ENV) else {
+        return WorkDbMemoryProfile::Default;
+    };
+    match parse_work_db_memory_profile(&raw) {
+        Some(profile) => profile,
+        None => {
+            eprintln!(
+                "warning: unsupported {WORK_DB_MEMORY_PROFILE_ENV}={raw:?}; using default work-db memory profile"
+            );
+            WorkDbMemoryProfile::Default
+        }
+    }
+}
+
+pub(super) fn configure_work_db_options(
+    opts: &mut Options,
+    blob_mode: WorkDbBlobMode,
+    memory_profile: WorkDbMemoryProfile,
+) {
     opts.set_compression_type(DBCompressionType::Lz4);
+    apply_work_db_memory_profile(opts, memory_profile);
     match blob_mode {
         WorkDbBlobMode::Current => enable_blobdb_current(opts),
         WorkDbBlobMode::Disabled => {}
@@ -86,9 +116,9 @@ pub(super) fn configure_work_db_options(opts: &mut Options, blob_mode: WorkDbBlo
     }
 }
 
-pub(super) fn cf_opts(blob_mode: WorkDbBlobMode) -> Options {
+pub(super) fn cf_opts(blob_mode: WorkDbBlobMode, memory_profile: WorkDbMemoryProfile) -> Options {
     let mut opts = Options::default();
-    configure_work_db_options(&mut opts, blob_mode);
+    configure_work_db_options(&mut opts, blob_mode, memory_profile);
     opts
 }
 
@@ -99,12 +129,23 @@ pub fn column_family_descriptors() -> Vec<ColumnFamilyDescriptor> {
 pub(super) fn column_family_descriptors_for_blob_mode(
     blob_mode: WorkDbBlobMode,
 ) -> Vec<ColumnFamilyDescriptor> {
+    let memory_profile = work_db_memory_profile_from_env();
     ALL_COLUMN_FAMILY_NAMES
         .iter()
-        .map(|name| ColumnFamilyDescriptor::new(*name, cf_opts(blob_mode)))
+        .map(|name| ColumnFamilyDescriptor::new(*name, cf_opts(blob_mode, memory_profile)))
         .collect()
 }
 
+fn apply_work_db_memory_profile(opts: &mut Options, memory_profile: WorkDbMemoryProfile) {
+    match memory_profile {
+        WorkDbMemoryProfile::Default => {}
+        WorkDbMemoryProfile::Compact => {
+            opts.set_write_buffer_size(16 * 1024 * 1024);
+            opts.set_max_write_buffer_number(1);
+        }
+    }
+}
+
 pub(super) fn enable_blobdb_current(opts: &mut Options) {
     #[allow(unused_mut)]
     let mut _enabled = false;

src/storage/keys.rs

@@ -4,7 +4,7 @@ use crate::data_model::common::der_take_tlv;
 
 use super::config::*;
 use super::pack::PackTime;
-use super::{AuditRuleKind, StorageError, StorageResult, VcirOutputType};
+use super::{StorageError, StorageResult};
 
 pub(super) fn repository_view_key(rsync_uri: &str) -> String {
     format!("{REPOSITORY_VIEW_KEY_PREFIX}{rsync_uri}")
@@ -30,20 +30,6 @@ pub(super) fn manifest_replay_meta_key(manifest_rsync_uri: &str) -> String {
     format!("{MANIFEST_REPLAY_META_KEY_PREFIX}{manifest_rsync_uri}")
 }
 
-pub(super) fn audit_rule_kind_for_output_type(
-    output_type: VcirOutputType,
-) -> Option<AuditRuleKind> {
-    match output_type {
-        VcirOutputType::Vrp => Some(AuditRuleKind::Roa),
-        VcirOutputType::Aspa => Some(AuditRuleKind::Aspa),
-        VcirOutputType::RouterKey => Some(AuditRuleKind::RouterKey),
-    }
-}
-
-pub(super) fn audit_rule_key(kind: AuditRuleKind, rule_hash: &str) -> String {
-    format!("{}{rule_hash}", kind.key_prefix())
-}
-
 pub(super) fn rrdp_source_key(notify_uri: &str) -> String {
     format!("{RRDP_SOURCE_KEY_PREFIX}{notify_uri}")
 }

src/storage/tests.rs

@@ -10,6 +10,10 @@ fn sha256_hex(input: &[u8]) -> String {
     hex::encode(compute_sha256_32(input))
 }
 
+fn sha256_32(input: &[u8]) -> [u8; 32] {
+    compute_sha256_32(input)
+}
+
 #[test]
 fn parse_work_db_blob_mode_accepts_supported_values() {
     assert_eq!(default_work_db_blob_mode(), WorkDbBlobMode::Disabled);
@@ -41,6 +45,61 @@ fn parse_work_db_blob_mode_accepts_supported_values() {
     assert_eq!(parse_work_db_blob_mode("unexpected"), None);
 }
 
+#[test]
+fn parse_work_db_memory_profile_accepts_supported_values() {
+    assert_eq!(
+        parse_work_db_memory_profile("default"),
+        Some(WorkDbMemoryProfile::Default)
+    );
+    assert_eq!(
+        parse_work_db_memory_profile("none"),
+        Some(WorkDbMemoryProfile::Default)
+    );
+    assert_eq!(
+        parse_work_db_memory_profile("compact"),
+        Some(WorkDbMemoryProfile::Compact)
+    );
+    assert_eq!(
+        parse_work_db_memory_profile("low-memory"),
+        Some(WorkDbMemoryProfile::Compact)
+    );
+    assert_eq!(parse_work_db_memory_profile("unexpected"), None);
+}
+
+#[test]
+fn vcir_field_size_breakdown_counts_local_outputs_and_artifacts() {
+    let vcir = sample_vcir("rsync://example.test/repo/current.mft");
+    let breakdown = VcirFieldSizeBreakdown::from_vcir(&vcir);
+    assert_eq!(breakdown.local_output_count, 2);
+    assert_eq!(
+        breakdown.local_output_payload_json_bytes,
+        vcir.local_outputs
+            .iter()
+            .map(|output| output.payload_json().len() as u64)
+            .sum::<u64>()
+    );
+    assert_eq!(
+        breakdown.local_output_rule_hash_hex_bytes,
+        vcir.local_outputs.len() as u64 * 64
+    );
+    assert_eq!(breakdown.related_artifact_count, 2);
+    assert!(breakdown.related_artifact_uri_bytes > 0);
+    assert_eq!(breakdown.child_entry_count, 1);
+    assert!(breakdown.child_entry_uri_bytes > 0);
+    assert_eq!(
+        breakdown.local_output_old_projection_bytes(),
+        breakdown.local_output_source_type_bytes
+            + breakdown.local_output_source_hash_hex_bytes
+            + breakdown.local_output_source_ee_hash_hex_bytes
+            + breakdown.local_output_payload_json_bytes
+            + breakdown.local_output_rule_hash_hex_bytes
+    );
+    assert!(
+        breakdown.local_output_old_projection_bytes()
+            > breakdown.local_output_typed_projection_bytes()
+    );
+}
+
 fn sample_repository_view_entry(rsync_uri: &str, bytes: &[u8]) -> RepositoryViewEntry {
     RepositoryViewEntry {
         rsync_uri: rsync_uri.to_string(),
@@ -126,28 +185,37 @@ fn sample_vcir(manifest_rsync_uri: &str) -> ValidatedCaInstanceResult {
         }],
         local_outputs: vec![
             VcirLocalOutput {
-                output_id: "vrp-1".to_string(),
                 output_type: VcirOutputType::Vrp,
                 item_effective_until: pack_time(12),
                 source_object_uri: "rsync://example.test/repo/object.roa".to_string(),
-                source_object_type: "roa".to_string(),
-                source_object_hash: sha256_hex(&roa_bytes),
-                source_ee_cert_hash: sha256_hex(&ee_bytes),
-                payload_json: r#"{"asn":64496,"prefix":"203.0.113.0/24"}"#.to_string(),
-                rule_hash: sha256_hex(b"vrp-rule-1"),
-                validation_path_hint: vec![manifest_rsync_uri.to_string()],
+                source_object_type: VcirSourceObjectType::Roa,
+                source_object_hash: sha256_32(&roa_bytes),
+                source_ee_cert_hash: sha256_32(&ee_bytes),
+                payload: VcirLocalOutputPayload::Vrp {
+                    asn: 64496,
+                    afi: crate::data_model::roa::RoaAfi::Ipv4,
+                    prefix_len: 24,
+                    addr: {
+                        let mut addr = [0u8; 16];
+                        addr[..4].copy_from_slice(&[203, 0, 113, 0]);
+                        addr
+                    },
+                    max_length: 24,
+                },
+                rule_hash: sha256_32(b"vrp-rule-1"),
             },
             VcirLocalOutput {
-                output_id: "aspa-1".to_string(),
                 output_type: VcirOutputType::Aspa,
                 item_effective_until: pack_time(10),
                 source_object_uri: "rsync://example.test/repo/object.asa".to_string(),
-                source_object_type: "aspa".to_string(),
-                source_object_hash: sha256_hex(b"aspa-object"),
-                source_ee_cert_hash: sha256_hex(b"aspa-ee-cert"),
-                payload_json: r#"{"customer_as":64496,"providers":[64497]}"#.to_string(),
-                rule_hash: sha256_hex(b"aspa-rule-1"),
-                validation_path_hint: vec![manifest_rsync_uri.to_string()],
+                source_object_type: VcirSourceObjectType::Aspa,
+                source_object_hash: sha256_32(b"aspa-object"),
+                source_ee_cert_hash: sha256_32(b"aspa-ee-cert"),
+                payload: VcirLocalOutputPayload::Aspa {
+                    customer_as_id: 64496,
+                    provider_as_ids: vec![64497],
+                },
+                rule_hash: sha256_32(b"aspa-rule-1"),
             },
         ],
         related_artifacts: vec![
@@ -230,34 +298,6 @@ fn vcir_ccr_manifest_projection_validate_rejects_invalid_fields() {
     ));
 }
 
-fn sample_audit_rule_entry(kind: AuditRuleKind) -> AuditRuleIndexEntry {
-    AuditRuleIndexEntry {
-        kind,
-        rule_hash: sha256_hex(match kind {
-            AuditRuleKind::Roa => b"roa-index-rule",
-            AuditRuleKind::Aspa => b"aspa-index-rule",
-            AuditRuleKind::RouterKey => b"router-key-index-rule",
-        }),
-        manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(),
-        source_object_uri: match kind {
-            AuditRuleKind::Roa => "rsync://example.test/repo/object.roa".to_string(),
-            AuditRuleKind::Aspa => "rsync://example.test/repo/object.asa".to_string(),
-            AuditRuleKind::RouterKey => "rsync://example.test/repo/router.cer".to_string(),
-        },
-        source_object_hash: sha256_hex(match kind {
-            AuditRuleKind::Roa => b"roa-object",
-            AuditRuleKind::Aspa => b"aspa-object",
-            AuditRuleKind::RouterKey => b"router-key-object",
-        }),
-        output_id: match kind {
-            AuditRuleKind::Roa => "vrp-1".to_string(),
-            AuditRuleKind::Aspa => "aspa-1".to_string(),
-            AuditRuleKind::RouterKey => "router-key-1".to_string(),
-        },
-        item_effective_until: pack_time(12),
-    }
-}
-
 fn sample_rrdp_source_record(notify_uri: &str) -> RrdpSourceRecord {
     RrdpSourceRecord {
         notify_uri: notify_uri.to_string(),
@@ -464,6 +504,32 @@ fn repo_bytes_db_is_physically_separate_from_external_raw_store() {
     );
 }
 
+#[test]
+fn memory_snapshot_includes_work_db_and_external_stores() {
+    let td = tempfile::tempdir().expect("tempdir");
+    let store = RocksStore::open_with_external_stores(
+        &td.path().join("main-db"),
+        Some(&td.path().join("raw-store.db")),
+        Some(&td.path().join("repo-bytes.db")),
+    )
+    .expect("open store");
+
+    let snapshot = store.memory_snapshot();
+    let labels: Vec<&str> = snapshot
+        .databases
+        .iter()
+        .map(|db| db.label.as_str())
+        .collect();
+    assert_eq!(labels, vec!["work-db", "raw-store.db", "repo-bytes.db"]);
+    assert!(
+        snapshot.databases[0]
+            .column_families
+            .iter()
+            .any(|cf| cf.name == CF_REPOSITORY_VIEW)
+    );
+    serde_json::to_value(&snapshot).expect("serialize memory snapshot");
+}
+
 #[test]
 fn put_blob_bytes_batch_accepts_empty_batch_with_external_raw_store() {
     let td = tempfile::tempdir().expect("tempdir");
@@ -784,106 +850,111 @@ fn list_vcirs_returns_all_entries() {
 }
 
 #[test]
-fn audit_rule_index_roundtrip_for_roa_aspa_and_router_key() {
+fn summarize_vcir_storage_aggregates_values_and_field_sizes() {
     let td = tempfile::tempdir().expect("tempdir");
     let store = RocksStore::open(td.path()).expect("open rocksdb");
 
-    let roa = sample_audit_rule_entry(AuditRuleKind::Roa);
-    let aspa = sample_audit_rule_entry(AuditRuleKind::Aspa);
-    let router_key = sample_audit_rule_entry(AuditRuleKind::RouterKey);
-    store
-        .put_audit_rule_index_entry(&roa)
-        .expect("put roa audit rule entry");
-    store
-        .put_audit_rule_index_entry(&aspa)
-        .expect("put aspa audit rule entry");
-    store
-        .put_audit_rule_index_entry(&router_key)
-        .expect("put router key audit rule entry");
+    let vcir1 = sample_vcir("rsync://example.test/repo/a.mft");
+    let vcir2 = sample_vcir("rsync://example.test/repo/b.mft");
+    store.put_vcir(&vcir1).expect("put vcir1");
+    store.put_vcir(&vcir2).expect("put vcir2");
 
-    let got_roa = store
-        .get_audit_rule_index_entry(AuditRuleKind::Roa, &roa.rule_hash)
-        .expect("get roa audit rule entry")
-        .expect("roa entry exists");
-    let got_aspa = store
-        .get_audit_rule_index_entry(AuditRuleKind::Aspa, &aspa.rule_hash)
-        .expect("get aspa audit rule entry")
-        .expect("aspa entry exists");
-    let got_router_key = store
-        .get_audit_rule_index_entry(AuditRuleKind::RouterKey, &router_key.rule_hash)
-        .expect("get router key audit rule entry")
-        .expect("router key entry exists");
-    assert_eq!(got_roa, roa);
-    assert_eq!(got_aspa, aspa);
-    assert_eq!(got_router_key, router_key);
+    let summary = store.summarize_vcir_storage().expect("summarize vcirs");
+    let mut expected_fields = VcirFieldSizeBreakdown::default();
+    expected_fields.add_assign(&VcirFieldSizeBreakdown::from_vcir(&vcir1));
+    expected_fields.add_assign(&VcirFieldSizeBreakdown::from_vcir(&vcir2));
 
-    store
-        .delete_audit_rule_index_entry(AuditRuleKind::Roa, &roa.rule_hash)
-        .expect("delete roa audit rule entry");
+    assert_eq!(summary.entry_count, 2);
+    assert!(summary.vcir_value_bytes > 0);
+    assert!(summary.vcir_value_bytes_max > 0);
+    assert!(summary.vcir_value_bytes_max_manifest_rsync_uri.is_some());
+    assert_eq!(summary.top_entries_by_vcir_value_bytes.len(), 2);
     assert!(
-        store
-            .get_audit_rule_index_entry(AuditRuleKind::Roa, &roa.rule_hash)
-            .expect("get deleted roa audit rule entry")
-            .is_none()
+        summary.top_entries_by_vcir_value_bytes[0].vcir_value_bytes
+            >= summary.top_entries_by_vcir_value_bytes[1].vcir_value_bytes
+    );
+    assert_eq!(summary.field_sizes, expected_fields);
+    assert!(summary.core_fields.manifest_rsync_uri_bytes > 0);
+    assert!(summary.ccr_projection.manifest_sha256_bytes > 0);
+    assert!(summary.child_resources.effective_ip_resource_cbor_bytes > 0);
+    assert_eq!(
+        summary.local_output_old_projection_bytes,
+        expected_fields.local_output_old_projection_bytes()
+    );
+    assert_eq!(
+        summary.local_output_typed_projection_bytes,
+        expected_fields.local_output_typed_projection_bytes()
+    );
+    assert_eq!(
+        summary.local_output_projection_saved_bytes,
+        expected_fields.local_output_projection_saved_bytes()
     );
-
-    let mut invalid = sample_audit_rule_entry(AuditRuleKind::Roa);
-    invalid.rule_hash = "bad".to_string();
-    let err = store
-        .put_audit_rule_index_entry(&invalid)
-        .expect_err("invalid audit rule hash must fail");
-    assert!(err.to_string().contains("64-character"));
 }
 
 #[test]
-fn replace_vcir_and_audit_rule_indexes_replaces_previous_entries_in_one_step() {
+fn replace_vcir_and_manifest_replay_meta_replaces_current_entry() {
     let td = tempfile::tempdir().expect("tempdir");
     let store = RocksStore::open(td.path()).expect("open rocksdb");
 
     let mut previous = sample_vcir("rsync://example.test/repo/current.mft");
     previous.local_outputs = vec![VcirLocalOutput {
-        output_id: "old-output".to_string(),
         output_type: VcirOutputType::Vrp,
         item_effective_until: pack_time(10),
         source_object_uri: "rsync://example.test/repo/old.roa".to_string(),
-        source_object_type: "roa".to_string(),
-        source_object_hash: sha256_hex(b"old-roa"),
-        source_ee_cert_hash: sha256_hex(b"old-ee"),
-        payload_json: "{}".to_string(),
-        rule_hash: sha256_hex(b"old-rule"),
-        validation_path_hint: vec![previous.manifest_rsync_uri.clone()],
+        source_object_type: VcirSourceObjectType::Roa,
+        source_object_hash: sha256_32(b"old-roa"),
+        source_ee_cert_hash: sha256_32(b"old-ee"),
+        payload: VcirLocalOutputPayload::Vrp {
+            asn: 64496,
+            afi: crate::data_model::roa::RoaAfi::Ipv4,
+            prefix_len: 24,
+            addr: {
+                let mut addr = [0u8; 16];
+                addr[..4].copy_from_slice(&[203, 0, 113, 0]);
+                addr
+            },
+            max_length: 24,
+        },
+        rule_hash: sha256_32(b"old-rule"),
     }];
     previous.summary.local_vrp_count = 1;
     previous.summary.local_aspa_count = 0;
     previous.summary.local_router_key_count = 0;
-    store
-        .replace_vcir_and_audit_rule_indexes(None, &previous)
+    let previous_timing = store
+        .replace_vcir_and_manifest_replay_meta(&previous)
         .expect("store previous vcir");
-    assert!(
-        store
-            .get_audit_rule_index_entry(AuditRuleKind::Roa, &previous.local_outputs[0].rule_hash)
-            .expect("get old audit entry")
-            .is_some()
+    assert!(previous_timing.vcir_value_bytes > 0);
+    assert!(previous_timing.replay_meta_value_bytes > 0);
+    assert_eq!(
+        previous_timing.total_encoded_bytes,
+        previous_timing.vcir_value_bytes + previous_timing.replay_meta_value_bytes
     );
 
     let mut current = sample_vcir("rsync://example.test/repo/current.mft");
     current.local_outputs = vec![VcirLocalOutput {
-        output_id: "new-output".to_string(),
         output_type: VcirOutputType::Aspa,
         item_effective_until: pack_time(11),
         source_object_uri: "rsync://example.test/repo/new.asa".to_string(),
-        source_object_type: "aspa".to_string(),
-        source_object_hash: sha256_hex(b"new-aspa"),
-        source_ee_cert_hash: sha256_hex(b"new-ee"),
-        payload_json: "{}".to_string(),
-        rule_hash: sha256_hex(b"new-rule"),
-        validation_path_hint: vec![current.manifest_rsync_uri.clone()],
+        source_object_type: VcirSourceObjectType::Aspa,
+        source_object_hash: sha256_32(b"new-aspa"),
+        source_ee_cert_hash: sha256_32(b"new-ee"),
+        payload: VcirLocalOutputPayload::Aspa {
+            customer_as_id: 64496,
+            provider_as_ids: vec![64497],
+        },
+        rule_hash: sha256_32(b"new-rule"),
     }];
     current.summary.local_vrp_count = 0;
     current.summary.local_aspa_count = 1;
-    store
-        .replace_vcir_and_audit_rule_indexes(Some(&previous), &current)
-        .expect("replace vcir and audit indexes");
+    let current_timing = store
+        .replace_vcir_and_manifest_replay_meta(&current)
+        .expect("replace vcir and replay meta");
+    assert!(current_timing.vcir_value_bytes > 0);
+    assert!(current_timing.replay_meta_value_bytes > 0);
+    assert_eq!(
+        current_timing.total_encoded_bytes,
+        current_timing.vcir_value_bytes + current_timing.replay_meta_value_bytes
+    );
 
     let got = store
         .get_vcir(&current.manifest_rsync_uri)
@@ -902,18 +973,6 @@ fn replace_vcir_and_audit_rule_indexes_replaces_previous_entries_in_one_step() {
         replay_meta.manifest_sha256,
         current.ccr_manifest_projection.manifest_sha256
     );
-    assert!(
-        store
-            .get_audit_rule_index_entry(AuditRuleKind::Roa, &previous.local_outputs[0].rule_hash)
-            .expect("get deleted old audit entry")
-            .is_none()
-    );
-    assert!(
-        store
-            .get_audit_rule_index_entry(AuditRuleKind::Aspa, &current.local_outputs[0].rule_hash)
-            .expect("get new audit entry")
-            .is_some()
-    );
 }
 
 #[test]

src/tools/sequence_triage_ccr_cir.rs

@@ -7,11 +7,11 @@ mod output;
 mod sandwich;
 
 use args::{Args, parse_args};
-use churn::build_intra_rp_churn;
-use loader::load_sequence;
+use churn::build_intra_rp_churn_streaming;
+use loader::load_sequence_meta;
 use model::Side;
-use output::{build_output, write_json, write_markdown};
-use sandwich::build_sandwich_analysis;
+use output::{build_output_from_meta, write_json, write_markdown};
+use sandwich::build_sandwich_analysis_streaming;
 
 pub fn main_entry() -> Result<(), String> {
     real_main()
@@ -25,15 +25,15 @@ fn real_main() -> Result<(), String> {
 fn run(args: Args) -> Result<(), String> {
     std::fs::create_dir_all(&args.out_dir)
         .map_err(|e| format!("create out-dir failed: {}: {e}", args.out_dir.display()))?;
-    let left = load_sequence(&args.left_sequence, Side::Left)?;
-    let right = load_sequence(&args.right_sequence, Side::Right)?;
+    let left = load_sequence_meta(&args.left_sequence, Side::Left)?;
+    let right = load_sequence_meta(&args.right_sequence, Side::Right)?;
     if left.is_empty() || right.is_empty() {
         return Err("left and right sequences must both contain at least one sample".into());
     }
 
-    let sandwich = build_sandwich_analysis(&args, &left, &right);
-    let churn = build_intra_rp_churn(&left, &right);
-    let output = build_output(&args, &left, &right, &sandwich, &churn);
+    let sandwich = build_sandwich_analysis_streaming(&args, &left, &right)?;
+    let churn = build_intra_rp_churn_streaming(&left, &right)?;
+    let output = build_output_from_meta(&args, &left, &right, &sandwich, &churn);
     write_json(&args.out_dir.join("sequence-triage.json"), &output)?;
     write_markdown(&args.out_dir.join("sequence-triage.md"), &output)?;
     println!("{}", args.out_dir.display());

src/tools/sequence_triage_ccr_cir/churn.rs

@@ -1,71 +1,82 @@
 use std::collections::{BTreeMap, BTreeSet};
 
-use super::model::{ChurnRecord, ChurnSummaryRecord, IntraRpChurn, SequenceSample, Side};
+use super::loader::{load_sample_ccr_from_meta, load_sample_cir_from_meta};
+use super::model::{
+    ChurnRecord, ChurnSummaryRecord, IntraRpChurn, SequenceMeta, SequenceSample, Side,
+};
 
-pub(super) fn build_intra_rp_churn(
-    left: &[SequenceSample],
-    right: &[SequenceSample],
-) -> IntraRpChurn {
-    let left_records = build_side_records(Side::Left, left);
-    let right_records = build_side_records(Side::Right, right);
+pub(super) fn build_intra_rp_churn_streaming(
+    left: &[SequenceMeta],
+    right: &[SequenceMeta],
+) -> Result<IntraRpChurn, String> {
+    let left_records = build_side_records_streaming(Side::Left, left)?;
+    let right_records = build_side_records_streaming(Side::Right, right)?;
     let mut summary = summarize_records(&left_records);
     summary.extend(summarize_records(&right_records));
-    IntraRpChurn {
+    Ok(IntraRpChurn {
         left: left_records,
         right: right_records,
         summary,
-    }
+    })
 }
 
-fn build_side_records(side: Side, samples: &[SequenceSample]) -> Vec<ChurnRecord> {
+fn build_side_records_streaming(
+    side: Side,
+    samples: &[SequenceMeta],
+) -> Result<Vec<ChurnRecord>, String> {
     let mut records = Vec::new();
     for pair in samples.windows(2) {
-        let from = &pair[0];
-        let to = &pair[1];
+        let from_cir = load_sample_cir_from_meta(&pair[0])?;
+        let to_cir = load_sample_cir_from_meta(&pair[1])?;
         records.push(record_churn(
             side,
-            from,
-            to,
+            &from_cir,
+            &to_cir,
             "object",
-            &from.object_hashes,
-            &to.object_hashes,
+            &from_cir.object_hashes,
+            &to_cir.object_hashes,
         ));
-        let from_output = output_keys(from);
-        let to_output = output_keys(to);
         records.push(record_churn(
             side,
-            from,
-            to,
+            &from_cir,
+            &to_cir,
+            "reject",
+            &from_cir.rejects,
+            &to_cir.rejects,
+        ));
+        drop(from_cir);
+        drop(to_cir);
+
+        let from_ccr = load_sample_ccr_from_meta(&pair[0])?;
+        let to_ccr = load_sample_ccr_from_meta(&pair[1])?;
+        let from_output = output_keys(&from_ccr);
+        let to_output = output_keys(&to_ccr);
+        records.push(record_churn(
+            side,
+            &from_ccr,
+            &to_ccr,
             "output",
             &from_output,
             &to_output,
         ));
         records.push(record_churn(
             side,
-            from,
-            to,
+            &from_ccr,
+            &to_ccr,
             "vrp_output",
-            &from.vrps,
-            &to.vrps,
+            &from_ccr.vrps,
+            &to_ccr.vrps,
         ));
         records.push(record_churn(
             side,
-            from,
-            to,
+            &from_ccr,
+            &to_ccr,
             "vap_output",
-            &from.vaps,
-            &to.vaps,
-        ));
-        records.push(record_churn(
-            side,
-            from,
-            to,
-            "reject",
-            &from.rejects,
-            &to.rejects,
+            &from_ccr.vaps,
+            &to_ccr.vaps,
         ));
     }
-    records
+    Ok(records)
 }
 
 fn output_keys(sample: &SequenceSample) -> BTreeSet<String> {

src/tools/sequence_triage_ccr_cir/loader.rs

@@ -5,9 +5,9 @@ use crate::ccr::{decode_ccr_compare_views, decode_content_info};
 use crate::cir::decode_cir;
 
 use super::io::{object_hash_key, parse_rfc3339, read_file, resolve_path};
-use super::model::{SequenceItemRaw, SequenceSample, Side};
+use super::model::{SequenceItemRaw, SequenceMeta, SequenceSample, Side};
 
-pub(super) fn load_sequence(path: &Path, side: Side) -> Result<Vec<SequenceSample>, String> {
+pub(super) fn load_sequence_meta(path: &Path, side: Side) -> Result<Vec<SequenceMeta>, String> {
     let base_dir = path.parent().unwrap_or_else(|| Path::new("."));
     let text = std::fs::read_to_string(path)
         .map_err(|e| format!("read sequence failed: {}: {e}", path.display()))?;
@@ -25,6 +25,128 @@ pub(super) fn load_sequence(path: &Path, side: Side) -> Result<Vec<SequenceSampl
                 line_index + 1
             )
         })?;
+        validate_raw(path, line_index, &raw, side, &mut seen_seq)?;
+        let cir_path = resolve_path(base_dir, &raw.cir_path);
+        let ccr_path = resolve_path(base_dir, &raw.ccr_path);
+        let cir = decode_cir(&read_file(&cir_path)?).map_err(|e| {
+            format!(
+                "decode CIR metadata failed for sample {} ({}): {e}",
+                raw.run_id,
+                cir_path.display()
+            )
+        })?;
+        let _sequence_validation_time = raw
+            .validation_time
+            .as_deref()
+            .map(parse_rfc3339)
+            .transpose()?;
+        let validation_time = cir.validation_time;
+        samples.push(SequenceMeta {
+            cir_object_count: raw.cir_object_count.or(Some(cir.objects.len() as u64)),
+            cir_reject_count: raw
+                .cir_reject_count
+                .or(Some(cir.rejected_objects.len() as u64)),
+            cir_trust_anchor_count: raw
+                .cir_trust_anchor_count
+                .or(Some(cir.trust_anchors.len() as u64)),
+            raw,
+            validation_time,
+            ccr_path,
+            cir_path,
+        });
+    }
+    samples.sort_by(|left, right| {
+        left.validation_time
+            .cmp(&right.validation_time)
+            .then_with(|| left.raw.seq.cmp(&right.raw.seq))
+            .then_with(|| left.raw.run_id.cmp(&right.raw.run_id))
+    });
+    Ok(samples)
+}
+
+pub(super) fn load_sample_cir_from_meta(meta: &SequenceMeta) -> Result<SequenceSample, String> {
+    load_sample_parts(meta, true, false)
+}
+
+pub(super) fn load_sample_ccr_from_meta(meta: &SequenceMeta) -> Result<SequenceSample, String> {
+    load_sample_parts(meta, false, true)
+}
+
+fn load_sample_parts(
+    meta: &SequenceMeta,
+    include_cir: bool,
+    include_ccr: bool,
+) -> Result<SequenceSample, String> {
+    let mut objects = BTreeMap::new();
+    let mut object_hashes = BTreeSet::new();
+    let mut rejects = BTreeSet::new();
+    if include_cir {
+        let cir = decode_cir(&read_file(&meta.cir_path)?).map_err(|e| {
+            format!(
+                "decode CIR failed for sample {} ({}): {e}",
+                meta.raw.run_id,
+                meta.cir_path.display()
+            )
+        })?;
+        objects = cir
+            .objects
+            .iter()
+            .map(|item| (item.rsync_uri.clone(), hex::encode(&item.sha256)))
+            .collect::<BTreeMap<_, _>>();
+        object_hashes = objects
+            .iter()
+            .map(|(uri, hash)| object_hash_key(uri, hash))
+            .collect::<BTreeSet<_>>();
+        rejects = cir
+            .rejected_objects
+            .iter()
+            .map(|item| item.object_uri.clone())
+            .collect::<BTreeSet<_>>();
+    }
+    let mut vrps = BTreeSet::new();
+    let mut vaps = BTreeSet::new();
+    if include_ccr {
+        let ccr = decode_content_info(&read_file(&meta.ccr_path)?).map_err(|e| {
+            format!(
+                "decode CCR failed for sample {} ({}): {e}",
+                meta.raw.run_id,
+                meta.ccr_path.display()
+            )
+        })?;
+        let (decoded_vrps, decoded_vaps) = decode_ccr_compare_views(&ccr).map_err(|e| {
+            format!(
+                "decode CCR compare views failed for sample {} ({}): {e}",
+                meta.raw.run_id,
+                meta.ccr_path.display()
+            )
+        })?;
+        vrps = decoded_vrps
+            .into_iter()
+            .map(|row| format!("{}|{}|{}", row.asn, row.ip_prefix, row.max_length))
+            .collect::<BTreeSet<_>>();
+        vaps = decoded_vaps
+            .into_iter()
+            .map(|row| format!("{}|{}", row.customer_asn, row.providers))
+            .collect::<BTreeSet<_>>();
+    }
+    Ok(SequenceSample {
+        raw: meta.raw.clone(),
+        validation_time: meta.validation_time,
+        objects,
+        object_hashes,
+        rejects,
+        vrps,
+        vaps,
+    })
+}
+
+fn validate_raw(
+    path: &Path,
+    line_index: usize,
+    raw: &SequenceItemRaw,
+    side: Side,
+    seen_seq: &mut BTreeSet<u32>,
+) -> Result<(), String> {
     if raw.schema_version.unwrap_or(1) != 1 {
         return Err(format!(
             "unsupported sequence item schemaVersion in {}:{}",
@@ -43,102 +165,16 @@ pub(super) fn load_sequence(path: &Path, side: Side) -> Result<Vec<SequenceSampl
             raw.run_id
         ));
     }
-        let cir_path = resolve_path(base_dir, &raw.cir_path);
-        let ccr_path = resolve_path(base_dir, &raw.ccr_path);
-        let cir = decode_cir(&read_file(&cir_path)?).map_err(|e| {
-            format!(
-                "decode CIR failed for sample {} ({}): {e}",
-                raw.run_id,
-                cir_path.display()
-            )
-        })?;
-        let ccr = decode_content_info(&read_file(&ccr_path)?).map_err(|e| {
-            format!(
-                "decode CCR failed for sample {} ({}): {e}",
-                raw.run_id,
-                ccr_path.display()
-            )
-        })?;
-        let _sequence_validation_time = raw
-            .validation_time
-            .as_deref()
-            .map(parse_rfc3339)
-            .transpose()?;
-        let validation_time = cir.validation_time;
-        let objects = cir
-            .objects
-            .iter()
-            .map(|item| (item.rsync_uri.clone(), hex::encode(&item.sha256)))
-            .collect::<BTreeMap<_, _>>();
-        let object_uris = objects.keys().cloned().collect::<BTreeSet<_>>();
-        let object_hashes = objects
-            .iter()
-            .map(|(uri, hash)| object_hash_key(uri, hash))
-            .collect::<BTreeSet<_>>();
-        let rejects = cir
-            .rejected_objects
-            .iter()
-            .map(|item| item.object_uri.clone())
-            .collect::<BTreeSet<_>>();
-        let trust_anchors = cir
-            .trust_anchors
-            .iter()
-            .map(|item| {
-                format!(
-                    "{}|{}|{}|{}",
-                    item.ta_rsync_uri,
-                    item.tal_uri,
-                    hex::encode(crate::cir::sha256(&item.tal_bytes)),
-                    hex::encode(&item.ta_certificate_sha256)
-                )
-            })
-            .collect::<BTreeSet<_>>();
-        let (vrps, vaps) = decode_ccr_compare_views(&ccr).map_err(|e| {
-            format!(
-                "decode CCR compare views failed for sample {} ({}): {e}",
-                raw.run_id,
-                ccr_path.display()
-            )
-        })?;
-        let vrps = vrps
-            .into_iter()
-            .map(|row| format!("{}|{}|{}", row.asn, row.ip_prefix, row.max_length))
-            .collect::<BTreeSet<_>>();
-        let vaps = vaps
-            .into_iter()
-            .map(|row| format!("{}|{}", row.customer_asn, row.providers))
-            .collect::<BTreeSet<_>>();
-        samples.push(SequenceSample {
-            raw,
-            validation_time,
-            ccr_path,
-            cir_path,
-            objects,
-            object_uris,
-            object_hashes,
-            rejects,
-            trust_anchors,
-            vrps,
-            vaps,
-        });
-    }
-    samples.sort_by(|left, right| {
-        left.validation_time
-            .cmp(&right.validation_time)
-            .then_with(|| left.raw.seq.cmp(&right.raw.seq))
-            .then_with(|| left.raw.run_id.cmp(&right.raw.run_id))
-    });
-    if samples.iter().any(|sample| {
-        sample
-            .raw
+    if raw
         .side
         .as_deref()
         .is_some_and(|item| item != side.as_str())
-    }) {
+    {
         return Err(format!(
-            "sequence side field does not match expected side: {}",
-            side.as_str()
+            "sequence side field does not match expected side in {}:{}",
+            path.display(),
+            line_index + 1
         ));
     }
-    Ok(samples)
+    Ok(())
 }

src/tools/sequence_triage_ccr_cir/model.rs

@@ -25,23 +25,34 @@ pub(super) struct SequenceItemRaw {
     pub(super) max_rss_kb: Option<u64>,
     pub(super) vrps: Option<u64>,
     pub(super) vaps: Option<u64>,
+    pub(super) publication_points: Option<u64>,
+    pub(super) cir_object_count: Option<u64>,
+    pub(super) cir_reject_count: Option<u64>,
+    pub(super) cir_trust_anchor_count: Option<u64>,
 }
 
 #[derive(Clone, Debug)]
 pub(super) struct SequenceSample {
     pub(super) raw: SequenceItemRaw,
     pub(super) validation_time: OffsetDateTime,
-    pub(super) ccr_path: PathBuf,
-    pub(super) cir_path: PathBuf,
     pub(super) objects: BTreeMap<String, String>,
-    pub(super) object_uris: BTreeSet<String>,
     pub(super) object_hashes: BTreeSet<String>,
     pub(super) rejects: BTreeSet<String>,
-    pub(super) trust_anchors: BTreeSet<String>,
     pub(super) vrps: BTreeSet<String>,
     pub(super) vaps: BTreeSet<String>,
 }
 
+#[derive(Clone, Debug)]
+pub(super) struct SequenceMeta {
+    pub(super) raw: SequenceItemRaw,
+    pub(super) validation_time: OffsetDateTime,
+    pub(super) ccr_path: PathBuf,
+    pub(super) cir_path: PathBuf,
+    pub(super) cir_object_count: Option<u64>,
+    pub(super) cir_reject_count: Option<u64>,
+    pub(super) cir_trust_anchor_count: Option<u64>,
+}
+
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord)]
 pub(super) enum Side {
     #[default]

src/tools/sequence_triage_ccr_cir/output.rs

@@ -7,13 +7,13 @@ use super::args::Args;
 use super::io::{format_time, path_string};
 use super::model::{
     ChurnRecord, ChurnSummaryRecord, IntraRpChurn, SandwichAnalysis, SandwichGroupStats,
-    SandwichHeatmapRow, SandwichRecord, SequenceSample,
+    SandwichHeatmapRow, SandwichRecord, SequenceMeta,
 };
 
-pub(super) fn build_output(
+pub(super) fn build_output_from_meta(
     args: &Args,
-    left: &[SequenceSample],
-    right: &[SequenceSample],
+    left: &[SequenceMeta],
+    right: &[SequenceMeta],
     sandwich: &SandwichAnalysis,
     churn: &IntraRpChurn,
 ) -> Value {
@@ -30,6 +30,7 @@ pub(super) fn build_output(
             "warmupSamples": args.warmup_samples,
             "cooldownSamples": args.cooldown_samples,
             "timelineSampleLimit": if args.timeline_sample_limit == 0 { args.sample_limit } else { args.timeline_sample_limit },
+            "executionMode": "streaming-window",
             "sequenceSemantics": {
                 "timeSource": "cir.validation_time",
                 "sourceSampleOrder": "each side is sorted by CIR validation_time, with seq/runId used only as stable tie breakers",
@@ -37,11 +38,11 @@ pub(super) fn build_output(
                 "peerWindow": "all peer samples with source_start.time < peer.time < source_end.time are checked independently",
             },
         },
-        "left": sequence_summary(left),
-        "right": sequence_summary(right),
+        "left": sequence_meta_summary(left),
+        "right": sequence_meta_summary(right),
         "sandwich": {
             "strictTimeWindow": true,
-            "method": "For each side sorted by CIR validation_time, use two adjacent source samples as a stable interval. If source_start.time < peer.time < source_end.time and the source value is identical at both interval endpoints, every peer sample in the interval is expected to contain the same value.",
+            "method": "Streaming mode: first build sample timing metadata, then process each sandwich triple by loading only source_start, peer, and source_end CIR/CCR artifacts.",
             "totals": {
                 "occurrences": sandwich.total_occurrences,
                 "uniqueKeys": sandwich.unique_keys.len(),
@@ -59,7 +60,7 @@ pub(super) fn build_output(
             }
         },
         "intraRpChurn": {
-            "method": "For each side, compare adjacent samples independently. Object keys use object_uri|sha256, output keys use typed VRP/VAP canonical keys, and reject keys use rejected object URI without reason.",
+            "method": "Streaming mode: compare adjacent samples by loading only the two CIR or CCR artifacts needed for the current adjacent pair.",
             "left": churn_records_to_json(&churn.left),
             "right": churn_records_to_json(&churn.right),
             "summary": churn_summary_to_json(&churn.summary),
@@ -77,7 +78,7 @@ pub(super) fn build_output(
     })
 }
 
-fn sequence_summary(samples: &[SequenceSample]) -> Value {
+fn sequence_meta_summary(samples: &[SequenceMeta]) -> Value {
     json!({
         "sampleCount": samples.len(),
         "rpIds": samples.iter().map(|sample| sample.raw.rp_id.clone()).collect::<BTreeSet<_>>(),
@@ -99,12 +100,13 @@ fn sequence_summary(samples: &[SequenceSample]) -> Value {
             "cirSha256": sample.raw.cir_sha256,
             "wallMs": sample.raw.wall_ms,
             "maxRssKb": sample.raw.max_rss_kb,
-            "vrps": sample.raw.vrps.or(Some(sample.vrps.len() as u64)),
-            "vaps": sample.raw.vaps.or(Some(sample.vaps.len() as u64)),
-            "objectCount": sample.object_uris.len(),
-            "objectHashCount": sample.object_hashes.len(),
-            "rejectCount": sample.rejects.len(),
-            "trustAnchorCount": sample.trust_anchors.len(),
+            "vrps": sample.raw.vrps,
+            "vaps": sample.raw.vaps,
+            "publicationPoints": sample.raw.publication_points,
+            "objectCount": sample.cir_object_count,
+            "objectHashCount": sample.cir_object_count,
+            "rejectCount": sample.cir_reject_count,
+            "trustAnchorCount": sample.cir_trust_anchor_count,
         })).collect::<Vec<_>>(),
     })
 }

src/tools/sequence_triage_ccr_cir/sandwich.rs

@@ -2,103 +2,145 @@ use std::collections::BTreeSet;
 
 use super::args::Args;
 use super::io::format_time;
+use super::loader::{load_sample_ccr_from_meta, load_sample_cir_from_meta};
 use super::model::{
-    SandwichAnalysis, SandwichGroupStats, SandwichHeatmapRow, SandwichRecord, SequenceSample, Side,
+    SandwichAnalysis, SandwichGroupStats, SandwichHeatmapRow, SandwichRecord, SequenceMeta,
+    SequenceSample, Side,
 };
 
-pub(super) fn build_sandwich_analysis(
+pub(super) fn build_sandwich_analysis_streaming(
     args: &Args,
-    left: &[SequenceSample],
-    right: &[SequenceSample],
-) -> SandwichAnalysis {
+    left: &[SequenceMeta],
+    right: &[SequenceMeta],
+) -> Result<SandwichAnalysis, String> {
     let mut analysis = SandwichAnalysis::default();
-    analyze_sandwich_objects(&mut analysis, Side::Left, left, right, args);
-    analyze_sandwich_objects(&mut analysis, Side::Right, right, left, args);
-    analyze_sandwich_sets(
-        &mut analysis,
-        "reject_uri",
-        "PEER_MISSING_STABLE_REJECT",
-        Side::Left,
-        left,
-        right,
-        |sample| &sample.rejects,
-        args,
-    );
-    analyze_sandwich_sets(
-        &mut analysis,
-        "reject_uri",
-        "PEER_MISSING_STABLE_REJECT",
-        Side::Right,
-        right,
-        left,
-        |sample| &sample.rejects,
-        args,
-    );
-    analyze_sandwich_sets(
-        &mut analysis,
-        "vrp_output",
-        "PEER_MISSING_STABLE_OUTPUT",
-        Side::Left,
-        left,
-        right,
-        |sample| &sample.vrps,
-        args,
-    );
-    analyze_sandwich_sets(
-        &mut analysis,
-        "vrp_output",
-        "PEER_MISSING_STABLE_OUTPUT",
-        Side::Right,
-        right,
-        left,
-        |sample| &sample.vrps,
-        args,
-    );
-    analyze_sandwich_sets(
-        &mut analysis,
-        "vap_output",
-        "PEER_MISSING_STABLE_OUTPUT",
-        Side::Left,
-        left,
-        right,
-        |sample| &sample.vaps,
-        args,
-    );
-    analyze_sandwich_sets(
-        &mut analysis,
-        "vap_output",
-        "PEER_MISSING_STABLE_OUTPUT",
-        Side::Right,
-        right,
-        left,
-        |sample| &sample.vaps,
-        args,
-    );
-    analysis
+    analyze_sandwich_cir_streaming(&mut analysis, Side::Left, left, right, args)?;
+    analyze_sandwich_cir_streaming(&mut analysis, Side::Right, right, left, args)?;
+    analyze_sandwich_ccr_streaming(&mut analysis, Side::Left, left, right, args)?;
+    analyze_sandwich_ccr_streaming(&mut analysis, Side::Right, right, left, args)?;
+    Ok(analysis)
 }
 
-fn analyze_sandwich_objects(
+fn analyze_sandwich_cir_streaming(
     analysis: &mut SandwichAnalysis,
     source_side: Side,
-    source: &[SequenceSample],
-    peer: &[SequenceSample],
+    source: &[SequenceMeta],
+    peer: &[SequenceMeta],
+    args: &Args,
+) -> Result<(), String> {
+    for pair in source.windows(2) {
+        let source_start_meta = &pair[0];
+        let source_end_meta = &pair[1];
+        if source_start_meta.validation_time >= source_end_meta.validation_time {
+            continue;
+        }
+        let peer_indices = peer_meta_indices_between(peer, source_start_meta, source_end_meta);
+        if peer_indices.is_empty() {
+            continue;
+        }
+        let source_start = load_sample_cir_from_meta(source_start_meta)?;
+        let source_end = load_sample_cir_from_meta(source_end_meta)?;
+        for peer_index in peer_indices {
+            let peer_sample = load_sample_cir_from_meta(&peer[peer_index])?;
+            analyze_loaded_sandwich_objects(
+                analysis,
+                source_side,
+                &source_start,
+                &source_end,
+                &peer_sample,
+                args,
+            );
+            analyze_loaded_sandwich_set(
+                analysis,
+                "reject_uri",
+                "PEER_MISSING_STABLE_REJECT",
+                source_side,
+                &source_start,
+                &source_end,
+                &peer_sample,
+                |sample| &sample.rejects,
+                args,
+            );
+        }
+    }
+    Ok(())
+}
+
+fn analyze_sandwich_ccr_streaming(
+    analysis: &mut SandwichAnalysis,
+    source_side: Side,
+    source: &[SequenceMeta],
+    peer: &[SequenceMeta],
+    args: &Args,
+) -> Result<(), String> {
+    for pair in source.windows(2) {
+        let source_start_meta = &pair[0];
+        let source_end_meta = &pair[1];
+        if source_start_meta.validation_time >= source_end_meta.validation_time {
+            continue;
+        }
+        let peer_indices = peer_meta_indices_between(peer, source_start_meta, source_end_meta);
+        if peer_indices.is_empty() {
+            continue;
+        }
+        let source_start = load_sample_ccr_from_meta(source_start_meta)?;
+        let source_end = load_sample_ccr_from_meta(source_end_meta)?;
+        for peer_index in peer_indices {
+            let peer_sample = load_sample_ccr_from_meta(&peer[peer_index])?;
+            analyze_loaded_sandwich_set(
+                analysis,
+                "vrp_output",
+                "PEER_MISSING_STABLE_OUTPUT",
+                source_side,
+                &source_start,
+                &source_end,
+                &peer_sample,
+                |sample| &sample.vrps,
+                args,
+            );
+            analyze_loaded_sandwich_set(
+                analysis,
+                "vap_output",
+                "PEER_MISSING_STABLE_OUTPUT",
+                source_side,
+                &source_start,
+                &source_end,
+                &peer_sample,
+                |sample| &sample.vaps,
+                args,
+            );
+        }
+    }
+    Ok(())
+}
+
+fn peer_meta_indices_between(
+    peer: &[SequenceMeta],
+    source_start: &SequenceMeta,
+    source_end: &SequenceMeta,
+) -> Vec<usize> {
+    peer.iter()
+        .enumerate()
+        .filter_map(|(index, sample)| {
+            (source_start.validation_time < sample.validation_time
+                && sample.validation_time < source_end.validation_time)
+                .then_some(index)
+        })
+        .collect()
+}
+
+fn analyze_loaded_sandwich_objects(
+    analysis: &mut SandwichAnalysis,
+    source_side: Side,
+    source_start: &SequenceSample,
+    source_end: &SequenceSample,
+    peer_sample: &SequenceSample,
     args: &Args,
 ) {
-    for pair in source.windows(2) {
-        let source_start = &pair[0];
-        let source_end = &pair[1];
-        if source_start.validation_time >= source_end.validation_time {
-            continue;
-        }
-        let peers = peer_samples_between(peer, source_start, source_end);
-        if peers.is_empty() {
-            continue;
-        }
     for (uri, source_hash) in &source_start.objects {
         if source_end.objects.get(uri) != Some(source_hash) {
             continue;
         }
-            for peer_sample in &peers {
         match peer_sample.objects.get(uri) {
             Some(peer_hash) if peer_hash == source_hash => {}
             Some(peer_hash) => analysis.add(
@@ -136,39 +178,25 @@ fn analyze_sandwich_objects(
         }
     }
 }
-    }
-}
 
-fn analyze_sandwich_sets<F>(
+fn analyze_loaded_sandwich_set<F>(
     analysis: &mut SandwichAnalysis,
     set_type: &'static str,
     classification: &'static str,
     source_side: Side,
-    source: &[SequenceSample],
-    peer: &[SequenceSample],
+    source_start: &SequenceSample,
+    source_end: &SequenceSample,
+    peer_sample: &SequenceSample,
     extract: F,
     args: &Args,
 ) where
     F: for<'a> Fn(&'a SequenceSample) -> &'a BTreeSet<String>,
 {
-    for pair in source.windows(2) {
-        let source_start = &pair[0];
-        let source_end = &pair[1];
-        if source_start.validation_time >= source_end.validation_time {
-            continue;
-        }
-        let peers = peer_samples_between(peer, source_start, source_end);
-        if peers.is_empty() {
-            continue;
-        }
     let start_set = extract(source_start);
     let end_set = extract(source_end);
+    let peer_set = extract(peer_sample);
     for key in start_set {
-            if !end_set.contains(key) {
-                continue;
-            }
-            for peer_sample in &peers {
-                if extract(peer_sample).contains(key) {
+        if !end_set.contains(key) || peer_set.contains(key) {
             continue;
         }
         analysis.add(
@@ -189,21 +217,6 @@ fn analyze_sandwich_sets<F>(
         );
     }
 }
-    }
-}
-
-fn peer_samples_between<'a>(
-    peer: &'a [SequenceSample],
-    source_start: &SequenceSample,
-    source_end: &SequenceSample,
-) -> Vec<&'a SequenceSample> {
-    peer.iter()
-        .filter(|sample| {
-            source_start.validation_time < sample.validation_time
-                && sample.validation_time < source_end.validation_time
-        })
-        .collect()
-}
 
 #[allow(clippy::too_many_arguments)]
 fn sandwich_record(
@@ -239,7 +252,7 @@ fn sandwich_record(
 }
 
 impl SandwichAnalysis {
-    fn add(&mut self, class: &'static str, record: SandwichRecord, sample_limit: usize) {
+    pub(super) fn add(&mut self, class: &'static str, record: SandwichRecord, sample_limit: usize) {
         self.total_occurrences += 1;
         self.unique_keys.insert(sandwich_unique_key(&record));
         *self.by_set_type.entry(record.set_type).or_default() += 1;

src/validation/objects.rs

@@ -14,7 +14,10 @@ use crate::parallel::object_worker::{
 };
 use crate::policy::{Policy, SignedObjectFailurePolicy};
 use crate::report::{RfcRef, Warning};
-use crate::storage::{PackFile, PackTime, VcirLocalOutput, VcirOutputType};
+use crate::storage::{
+    PackFile, PackTime, VcirLocalOutput, VcirLocalOutputPayload, VcirOutputType,
+    VcirSourceObjectType,
+};
 use crate::validation::cert_path::{CertPathError, validate_signed_object_ee_cert_path_fast};
 use crate::validation::manifest::PublicationPointData;
 use crate::validation::publication_point::PublicationPointSnapshot;
@@ -28,6 +31,13 @@ const RFC_CRLDP: &[RfcRef] = &[RfcRef("RFC 6487 §4.8.6")];
 const RFC_CRLDP_AND_LOCKED_PACK: &[RfcRef] =
     &[RfcRef("RFC 6487 §4.8.6"), RfcRef("RFC 9286 §4.2.1")];
 
+fn sha256_hex_to_32(hex_value: &str) -> [u8; 32] {
+    let bytes = hex::decode(hex_value).expect("internal sha256 hex should decode");
+    let mut out = [0u8; 32];
+    out.copy_from_slice(&bytes);
+    out
+}
+
 fn decode_resource_certificate_with_policy(
     der: &[u8],
     policy: &Policy,
@@ -132,8 +142,6 @@ pub(crate) struct RoaTaskResult {
     pub(crate) worker_index: usize,
     pub(crate) queue_wait_ms: u64,
     pub(crate) worker_ms: u64,
-    pub(crate) rsync_uri: String,
-    pub(crate) sha256_hex: String,
     pub(crate) outcome: Result<RoaTaskOk, ObjectValidateError>,
 }
 
@@ -418,8 +426,8 @@ pub fn process_publication_point_for_issuer_with_options<P: PublicationPointData
                         local_outputs_cache.extend(ok.local_outputs);
                     }
                     audit.push(ObjectAuditEntry {
-                        rsync_uri: result.rsync_uri,
-                        sha256_hex: result.sha256_hex,
+                        rsync_uri: file.rsync_uri.clone(),
+                        sha256_hex: sha256_hex_from_32(&file.sha256),
                         kind: AuditObjectKind::Roa,
                         result: AuditObjectResult::Ok,
                         detail: None,
@@ -428,8 +436,8 @@ pub fn process_publication_point_for_issuer_with_options<P: PublicationPointData
                 Err(e) => match policy.signed_object_failure_policy {
                     SignedObjectFailurePolicy::DropObject => {
                         audit.push(ObjectAuditEntry {
-                            rsync_uri: result.rsync_uri.clone(),
-                            sha256_hex: result.sha256_hex,
+                            rsync_uri: file.rsync_uri.clone(),
+                            sha256_hex: sha256_hex_from_32(&file.sha256),
                             kind: AuditObjectKind::Roa,
                             result: AuditObjectResult::Error,
                             detail: Some(e.to_string()),
@@ -437,24 +445,21 @@ pub fn process_publication_point_for_issuer_with_options<P: PublicationPointData
                         let mut refs = vec![RfcRef("RFC 6488 §3"), RfcRef("RFC 9582 §4-§5")];
                         refs.extend_from_slice(extra_rfc_refs_for_crl_selection(&e));
                         warnings.push(
-                            Warning::new(format!(
-                                "dropping invalid ROA: {}: {e}",
-                                result.rsync_uri
-                            ))
+                            Warning::new(format!("dropping invalid ROA: {}: {e}", file.rsync_uri))
                                 .with_rfc_refs(&refs)
-                            .with_context(&result.rsync_uri),
+                                .with_context(&file.rsync_uri),
                         )
                     }
                     SignedObjectFailurePolicy::DropPublicationPoint => {
                         stats.publication_point_dropped = true;
                         audit.push(ObjectAuditEntry {
-                            rsync_uri: result.rsync_uri.clone(),
-                            sha256_hex: result.sha256_hex,
+                            rsync_uri: file.rsync_uri.clone(),
+                            sha256_hex: sha256_hex_from_32(&file.sha256),
                             kind: AuditObjectKind::Roa,
                             result: AuditObjectResult::Error,
                             detail: Some(e.to_string()),
                         });
-                        for f in locked_files.iter().skip(result.index + 1) {
+                        for f in locked_files.iter().skip(idx + 1) {
                             if f.rsync_uri.ends_with(".roa") {
                                 audit.push(ObjectAuditEntry {
                                     rsync_uri: f.rsync_uri.clone(),
@@ -484,7 +489,7 @@ pub fn process_publication_point_for_issuer_with_options<P: PublicationPointData
                         warnings.push(
                             Warning::new(format!(
                                 "dropping publication point due to invalid ROA: {}: {e}",
-                                result.rsync_uri
+                                file.rsync_uri
                             ))
                             .with_rfc_refs(&refs)
                             .with_context(manifest_rsync_uri),
@@ -786,19 +791,24 @@ pub fn process_publication_point_for_issuer_parallel_roa_with_pool_options<
 }
 
 #[derive(Clone)]
-pub(crate) struct OwnedRoaTask {
-    pub(crate) publication_point_id: u64,
-    index: usize,
-    file: PackFile,
-    manifest_rsync_uri: String,
+pub(crate) struct RoaTaskShared {
+    locked_files: Arc<[PackFile]>,
+    manifest_rsync_uri: Arc<str>,
     issuer_ca_der: Arc<[u8]>,
     issuer_ca: Arc<ResourceCertificate>,
     issuer_spki_der: Arc<[u8]>,
-    issuer_ca_rsync_uri: Option<String>,
+    issuer_ca_rsync_uri: Option<Arc<str>>,
     crl_cache: Arc<Mutex<std::collections::HashMap<String, CachedIssuerCrl>>>,
     issuer_resources_index: Arc<IssuerResourcesIndex>,
-    issuer_effective_ip: Option<crate::data_model::rc::IpResourceSet>,
-    issuer_effective_as: Option<crate::data_model::rc::AsResourceSet>,
+    issuer_effective_ip: Option<Arc<crate::data_model::rc::IpResourceSet>>,
+    issuer_effective_as: Option<Arc<crate::data_model::rc::AsResourceSet>>,
+}
+
+#[derive(Clone)]
+pub(crate) struct OwnedRoaTask {
+    pub(crate) publication_point_id: u64,
+    index: usize,
+    shared: Arc<RoaTaskShared>,
     validation_time: time::OffsetDateTime,
     collect_vcir_local_outputs: bool,
     strict_cms_der: bool,
@@ -862,8 +872,13 @@ fn validate_owned_roa_task(worker_index: usize, task: OwnedRoaTask) -> RoaTaskRe
         .map(|submitted_at| worker_started.saturating_duration_since(submitted_at))
         .map(|duration| duration.as_millis() as u64)
         .unwrap_or(0);
-    let sha256_hex = sha256_hex_from_32(&task.file.sha256);
-    let issuer_spki = match SubjectPublicKeyInfo::from_der(task.issuer_spki_der.as_ref()) {
+    let shared = task.shared.as_ref();
+    let file = task
+        .shared
+        .locked_files
+        .get(task.index)
+        .expect("ROA task index must reference locked file");
+    let issuer_spki = match SubjectPublicKeyInfo::from_der(shared.issuer_spki_der.as_ref()) {
         Ok((rem, spki)) if rem.is_empty() => spki,
         Ok((rem, _)) => {
             return RoaTaskResult {
@@ -872,8 +887,6 @@ fn validate_owned_roa_task(worker_index: usize, task: OwnedRoaTask) -> RoaTaskRe
                 worker_index,
                 queue_wait_ms,
                 worker_ms: worker_started.elapsed().as_millis() as u64,
-                rsync_uri: task.file.rsync_uri,
-                sha256_hex,
                 outcome: Err(ObjectValidateError::CertPath(
                     CertPathError::IssuerSpkiTrailingBytes(rem.len()),
                 )),
@@ -886,8 +899,6 @@ fn validate_owned_roa_task(worker_index: usize, task: OwnedRoaTask) -> RoaTaskRe
                 worker_index,
                 queue_wait_ms,
                 worker_ms: worker_started.elapsed().as_millis() as u64,
-                rsync_uri: task.file.rsync_uri,
-                sha256_hex,
                 outcome: Err(ObjectValidateError::CertPath(
                     CertPathError::IssuerSpkiParse(e.to_string()),
                 )),
@@ -895,16 +906,16 @@ fn validate_owned_roa_task(worker_index: usize, task: OwnedRoaTask) -> RoaTaskRe
         }
     };
     let outcome = process_roa_with_issuer_parallel_cached(
-        &task.file,
-        task.manifest_rsync_uri.as_str(),
-        task.issuer_ca_der.as_ref(),
-        task.issuer_ca.as_ref(),
+        file,
+        shared.manifest_rsync_uri.as_ref(),
+        shared.issuer_ca_der.as_ref(),
+        shared.issuer_ca.as_ref(),
         &issuer_spki,
-        task.issuer_ca_rsync_uri.as_deref(),
-        task.crl_cache.as_ref(),
-        task.issuer_resources_index.as_ref(),
-        task.issuer_effective_ip.as_ref(),
-        task.issuer_effective_as.as_ref(),
+        shared.issuer_ca_rsync_uri.as_deref(),
+        shared.crl_cache.as_ref(),
+        shared.issuer_resources_index.as_ref(),
+        shared.issuer_effective_ip.as_deref(),
+        shared.issuer_effective_as.as_deref(),
         task.validation_time,
         None,
         task.collect_vcir_local_outputs,
@@ -922,8 +933,6 @@ fn validate_owned_roa_task(worker_index: usize, task: OwnedRoaTask) -> RoaTaskRe
         worker_index,
         queue_wait_ms,
         worker_ms: worker_started.elapsed().as_millis() as u64,
-        rsync_uri: task.file.rsync_uri,
-        sha256_hex,
         outcome,
     }
 }
@@ -935,16 +944,7 @@ pub(crate) enum ParallelObjectsPrepare {
 
 pub(crate) struct ParallelObjectsStage {
     pub(crate) publication_point_id: u64,
-    pub(crate) locked_files: Vec<PackFile>,
-    pub(crate) manifest_rsync_uri: String,
-    issuer_ca_der: Arc<[u8]>,
-    issuer_ca: Arc<ResourceCertificate>,
-    issuer_spki_der: Arc<[u8]>,
-    issuer_ca_rsync_uri: Option<String>,
-    crl_cache: Arc<Mutex<std::collections::HashMap<String, CachedIssuerCrl>>>,
-    issuer_resources_index: Arc<IssuerResourcesIndex>,
-    issuer_effective_ip: Option<crate::data_model::rc::IpResourceSet>,
-    issuer_effective_as: Option<crate::data_model::rc::AsResourceSet>,
+    shared: Arc<RoaTaskShared>,
     validation_time: time::OffsetDateTime,
     collect_vcir_local_outputs: bool,
     strict_cms_der: bool,
@@ -955,31 +955,41 @@ pub(crate) struct ParallelObjectsStage {
 }
 
 impl ParallelObjectsStage {
+    #[cfg(test)]
     pub(crate) fn build_roa_tasks(&self) -> Vec<OwnedRoaTask> {
-        self.locked_files
+        let mut tasks = Vec::with_capacity(self.roa_task_count());
+        self.extend_roa_tasks(|task| tasks.push(task));
+        tasks
+    }
+
+    pub(crate) fn append_roa_tasks_to(
+        &self,
+        pending: &mut std::collections::VecDeque<OwnedRoaTask>,
+    ) {
+        self.extend_roa_tasks(|task| pending.push_back(task));
+    }
+
+    fn extend_roa_tasks<F>(&self, mut push: F)
+    where
+        F: FnMut(OwnedRoaTask),
+    {
+        let shared = self.shared.clone();
+        self.locked_files()
             .iter()
             .enumerate()
             .filter(|(_, file)| file.rsync_uri.ends_with(".roa"))
-            .map(|(index, file)| OwnedRoaTask {
+            .for_each(|(index, _)| {
+                push(OwnedRoaTask {
                     publication_point_id: self.publication_point_id,
                     index,
-                file: file.clone(),
-                manifest_rsync_uri: self.manifest_rsync_uri.clone(),
-                issuer_ca_der: self.issuer_ca_der.clone(),
-                issuer_ca: self.issuer_ca.clone(),
-                issuer_spki_der: self.issuer_spki_der.clone(),
-                issuer_ca_rsync_uri: self.issuer_ca_rsync_uri.clone(),
-                crl_cache: self.crl_cache.clone(),
-                issuer_resources_index: self.issuer_resources_index.clone(),
-                issuer_effective_ip: self.issuer_effective_ip.clone(),
-                issuer_effective_as: self.issuer_effective_as.clone(),
+                    shared: shared.clone(),
                     validation_time: self.validation_time,
                     collect_vcir_local_outputs: self.collect_vcir_local_outputs,
                     strict_cms_der: self.strict_cms_der,
                     strict_name: self.strict_name,
                     submitted_at: None,
-            })
-            .collect()
+                });
+            });
     }
 
     pub(crate) fn roa_task_count(&self) -> usize {
@@ -991,7 +1001,11 @@ impl ParallelObjectsStage {
     }
 
     pub(crate) fn locked_file_count(&self) -> usize {
-        self.locked_files.len()
+        self.shared.locked_files.len()
+    }
+
+    fn locked_files(&self) -> &[PackFile] {
+        self.shared.locked_files.as_ref()
     }
 }
 
@@ -1212,19 +1226,21 @@ pub(crate) fn prepare_publication_point_for_parallel_roa<P: PublicationPointData
 
     ParallelObjectsPrepare::Staged(ParallelObjectsStage {
         publication_point_id,
-        locked_files: locked_files.to_vec(),
-        manifest_rsync_uri: manifest_rsync_uri.to_string(),
+        shared: Arc::new(RoaTaskShared {
+            locked_files: Arc::<[PackFile]>::from(locked_files.to_vec()),
+            manifest_rsync_uri: Arc::<str>::from(manifest_rsync_uri),
             issuer_ca_der: Arc::<[u8]>::from(issuer_ca_der.to_vec()),
             issuer_spki_der: Arc::<[u8]>::from(issuer_ca.tbs.subject_public_key_info.clone()),
             issuer_ca: Arc::new(issuer_ca),
-        issuer_ca_rsync_uri: issuer_ca_rsync_uri.map(ToString::to_string),
+            issuer_ca_rsync_uri: issuer_ca_rsync_uri.map(Arc::<str>::from),
             crl_cache: Arc::new(Mutex::new(crl_cache)),
             issuer_resources_index: Arc::new(build_issuer_resources_index(
                 issuer_effective_ip,
                 issuer_effective_as,
             )),
-        issuer_effective_ip: issuer_effective_ip.cloned(),
-        issuer_effective_as: issuer_effective_as.cloned(),
+            issuer_effective_ip: issuer_effective_ip.cloned().map(Arc::new),
+            issuer_effective_as: issuer_effective_as.cloned().map(Arc::new),
+        }),
         validation_time,
         collect_vcir_local_outputs,
         strict_cms_der: policy.strict.cms_der,
@@ -1241,18 +1257,20 @@ pub(crate) fn reduce_parallel_roa_stage(
     timing: Option<&TimingHandle>,
 ) -> Result<ObjectsOutput, String> {
     roa_results.sort_by_key(|result| result.index);
-    let mut roa_by_index = roa_results
-        .into_iter()
-        .map(|result| (result.index, result))
-        .collect::<std::collections::HashMap<_, _>>();
-    let mut aspa_crl_cache = stage
+    let mut roa_results = roa_results.into_iter().peekable();
+    let shared = stage.shared.clone();
+    let mut aspa_crl_cache = shared
         .crl_cache
         .lock()
         .expect("parallel ROA CRL cache lock")
         .clone();
-    let issuer_spki = SubjectPublicKeyInfo::from_der(stage.issuer_spki_der.as_ref())
+    let issuer_spki = SubjectPublicKeyInfo::from_der(shared.issuer_spki_der.as_ref())
         .map_err(|e| e.to_string())?
         .1;
+    let collect_vcir_local_outputs = stage.collect_vcir_local_outputs;
+    let validation_time = stage.validation_time;
+    let strict_cms_der = stage.strict_cms_der;
+    let strict_name = stage.strict_name;
     let mut stats = stage.stats;
     let mut warnings = stage.warnings;
     let mut audit = stage.audit;
@@ -1260,21 +1278,32 @@ pub(crate) fn reduce_parallel_roa_stage(
     let mut aspas: Vec<AspaAttestation> = Vec::new();
     let mut local_outputs_cache: Vec<VcirLocalOutput> = Vec::new();
 
-    for (idx, file) in stage.locked_files.iter().enumerate() {
+    for (idx, file) in shared.locked_files.iter().enumerate() {
         if file.rsync_uri.ends_with(".roa") {
-            let result = roa_by_index
-                .remove(&idx)
-                .ok_or_else(|| format!("missing ROA task result for {}", file.rsync_uri))?;
+            let result = match roa_results.peek() {
+                Some(result) if result.index == idx => roa_results
+                    .next()
+                    .expect("peeked ROA task result must be present"),
+                Some(result) => {
+                    return Err(format!(
+                        "unexpected ROA task result index {} while reducing {} at index {}",
+                        result.index, file.rsync_uri, idx
+                    ));
+                }
+                None => {
+                    return Err(format!("missing ROA task result for {}", file.rsync_uri));
+                }
+            };
             match result.outcome {
                 Ok(mut ok) => {
                     stats.roa_ok += 1;
                     vrps.append(&mut ok.vrps);
-                    if stage.collect_vcir_local_outputs {
+                    if collect_vcir_local_outputs {
                         local_outputs_cache.extend(ok.local_outputs);
                     }
                     audit.push(ObjectAuditEntry {
-                        rsync_uri: result.rsync_uri,
-                        sha256_hex: result.sha256_hex,
+                        rsync_uri: file.rsync_uri.clone(),
+                        sha256_hex: sha256_hex_from_32(&file.sha256),
                         kind: AuditObjectKind::Roa,
                         result: AuditObjectResult::Ok,
                         detail: None,
@@ -1282,8 +1311,8 @@ pub(crate) fn reduce_parallel_roa_stage(
                 }
                 Err(e) => {
                     audit.push(ObjectAuditEntry {
-                        rsync_uri: result.rsync_uri.clone(),
-                        sha256_hex: result.sha256_hex,
+                        rsync_uri: file.rsync_uri.clone(),
+                        sha256_hex: sha256_hex_from_32(&file.sha256),
                         kind: AuditObjectKind::Roa,
                         result: AuditObjectResult::Error,
                         detail: Some(e.to_string()),
@@ -1291,9 +1320,9 @@ pub(crate) fn reduce_parallel_roa_stage(
                     let mut refs = vec![RfcRef("RFC 6488 §3"), RfcRef("RFC 9582 §4-§5")];
                     refs.extend_from_slice(extra_rfc_refs_for_crl_selection(&e));
                     warnings.push(
-                        Warning::new(format!("dropping invalid ROA: {}: {e}", result.rsync_uri))
+                        Warning::new(format!("dropping invalid ROA: {}: {e}", file.rsync_uri))
                             .with_rfc_refs(&refs)
-                            .with_context(&result.rsync_uri),
+                            .with_context(&file.rsync_uri),
                     )
                 }
             }
@@ -1301,20 +1330,20 @@ pub(crate) fn reduce_parallel_roa_stage(
             let _t = timing.as_ref().map(|t| t.span_phase("objects_aspa_total"));
             match process_aspa_with_issuer(
                 file,
-                &stage.manifest_rsync_uri,
-                stage.issuer_ca_der.as_ref(),
-                stage.issuer_ca.as_ref(),
+                shared.manifest_rsync_uri.as_ref(),
+                shared.issuer_ca_der.as_ref(),
+                shared.issuer_ca.as_ref(),
                 &issuer_spki,
-                stage.issuer_ca_rsync_uri.as_deref(),
+                shared.issuer_ca_rsync_uri.as_deref(),
                 &mut aspa_crl_cache,
-                stage.issuer_resources_index.as_ref(),
-                stage.issuer_effective_ip.as_ref(),
-                stage.issuer_effective_as.as_ref(),
-                stage.validation_time,
+                shared.issuer_resources_index.as_ref(),
+                shared.issuer_effective_ip.as_deref(),
+                shared.issuer_effective_as.as_deref(),
+                validation_time,
                 timing,
-                stage.collect_vcir_local_outputs,
-                stage.strict_cms_der,
-                stage.strict_name,
+                collect_vcir_local_outputs,
+                strict_cms_der,
+                strict_name,
             ) {
                 Ok((att, local_output)) => {
                     stats.aspa_ok += 1;
@@ -1349,6 +1378,12 @@ pub(crate) fn reduce_parallel_roa_stage(
             }
         }
     }
+    if let Some(result) = roa_results.next() {
+        return Err(format!(
+            "unexpected trailing ROA task result at index {}",
+            result.index
+        ));
+    }
 
     Ok(ObjectsOutput {
         vrps,
@@ -1388,8 +1423,9 @@ fn process_publication_point_for_issuer_parallel_roa_inner<P: PublicationPointDa
         ParallelObjectsPrepare::Staged(stage) => stage,
     };
 
-    let mut pending = std::collections::VecDeque::from(stage.build_roa_tasks());
     let roa_task_count = stage.roa_task_count();
+    let mut pending = std::collections::VecDeque::with_capacity(roa_task_count);
+    stage.append_roa_tasks_to(&mut pending);
     let mut worker_pool = pool
         .pool
         .lock()
@@ -1537,7 +1573,6 @@ pub(crate) fn validate_roa_task_serial(
     strict_cms_der: bool,
     strict_name: bool,
 ) -> RoaTaskResult {
-    let sha256_hex = sha256_hex_from_32(&task.file.sha256);
     let outcome = process_roa_with_issuer(
         task.file,
         manifest_rsync_uri,
@@ -1566,15 +1601,13 @@ pub(crate) fn validate_roa_task_serial(
         worker_index: 0,
         queue_wait_ms: 0,
         worker_ms: 0,
-        rsync_uri: task.file.rsync_uri.clone(),
-        sha256_hex,
         outcome,
     }
 }
 
 fn process_roa_with_issuer(
     file: &PackFile,
-    manifest_rsync_uri: &str,
+    _manifest_rsync_uri: &str,
     issuer_ca_der: &[u8],
     issuer_ca: &ResourceCertificate,
     issuer_spki: &SubjectPublicKeyInfo<'_>,
@@ -1659,12 +1692,6 @@ fn process_roa_with_issuer(
         .iter()
         .map(|vrp| {
             let prefix = vrp_prefix_to_string(vrp);
-            let payload_json = serde_json::json!({
-                "asn": vrp.asn,
-                "prefix": prefix,
-                "max_length": vrp.max_length,
-            })
-            .to_string();
             let rule_hash = crate::audit::sha256_hex(
                 format!(
                     "roa-rule:{}:{}:{}:{}",
@@ -1673,20 +1700,20 @@ fn process_roa_with_issuer(
                 .as_bytes(),
             );
             VcirLocalOutput {
-                output_id: rule_hash.clone(),
                 output_type: VcirOutputType::Vrp,
                 item_effective_until: item_effective_until.clone(),
                 source_object_uri: file.rsync_uri.clone(),
-                source_object_type: "roa".to_string(),
-                source_object_hash: source_object_hash.clone(),
-                source_ee_cert_hash: source_ee_cert_hash.clone(),
-                payload_json,
-                rule_hash,
-                validation_path_hint: vec![
-                    manifest_rsync_uri.to_string(),
-                    file.rsync_uri.clone(),
-                    source_object_hash.clone(),
-                ],
+                source_object_type: VcirSourceObjectType::Roa,
+                source_object_hash: file.sha256,
+                source_ee_cert_hash: sha256_hex_to_32(&source_ee_cert_hash),
+                payload: VcirLocalOutputPayload::Vrp {
+                    asn: vrp.asn,
+                    afi: vrp.prefix.afi,
+                    prefix_len: vrp.prefix.prefix_len,
+                    addr: vrp.prefix.addr,
+                    max_length: vrp.max_length,
+                },
+                rule_hash: sha256_hex_to_32(&rule_hash),
             }
         })
         .collect();
@@ -1696,7 +1723,7 @@ fn process_roa_with_issuer(
 
 fn process_roa_with_issuer_parallel_cached(
     file: &PackFile,
-    manifest_rsync_uri: &str,
+    _manifest_rsync_uri: &str,
     issuer_ca_der: &[u8],
     issuer_ca: &ResourceCertificate,
     issuer_spki: &SubjectPublicKeyInfo<'_>,
@@ -1787,12 +1814,6 @@ fn process_roa_with_issuer_parallel_cached(
         .iter()
         .map(|vrp| {
             let prefix = vrp_prefix_to_string(vrp);
-            let payload_json = serde_json::json!({
-                "asn": vrp.asn,
-                "prefix": prefix,
-                "max_length": vrp.max_length,
-            })
-            .to_string();
             let rule_hash = crate::audit::sha256_hex(
                 format!(
                     "roa-rule:{}:{}:{}:{}",
@@ -1801,20 +1822,20 @@ fn process_roa_with_issuer_parallel_cached(
                 .as_bytes(),
             );
             VcirLocalOutput {
-                output_id: rule_hash.clone(),
                 output_type: VcirOutputType::Vrp,
                 item_effective_until: item_effective_until.clone(),
                 source_object_uri: file.rsync_uri.clone(),
-                source_object_type: "roa".to_string(),
-                source_object_hash: source_object_hash.clone(),
-                source_ee_cert_hash: source_ee_cert_hash.clone(),
-                payload_json,
-                rule_hash,
-                validation_path_hint: vec![
-                    manifest_rsync_uri.to_string(),
-                    file.rsync_uri.clone(),
-                    source_object_hash.clone(),
-                ],
+                source_object_type: VcirSourceObjectType::Roa,
+                source_object_hash: file.sha256,
+                source_ee_cert_hash: sha256_hex_to_32(&source_ee_cert_hash),
+                payload: VcirLocalOutputPayload::Vrp {
+                    asn: vrp.asn,
+                    afi: vrp.prefix.afi,
+                    prefix_len: vrp.prefix.prefix_len,
+                    addr: vrp.prefix.addr,
+                    max_length: vrp.max_length,
+                },
+                rule_hash: sha256_hex_to_32(&rule_hash),
             }
         })
         .collect();
@@ -1824,7 +1845,7 @@ fn process_roa_with_issuer_parallel_cached(
 
 fn process_aspa_with_issuer(
     file: &PackFile,
-    manifest_rsync_uri: &str,
+    _manifest_rsync_uri: &str,
     issuer_ca_der: &[u8],
     issuer_ca: &ResourceCertificate,
     issuer_spki: &SubjectPublicKeyInfo<'_>,
@@ -1922,24 +1943,17 @@ fn process_aspa_with_issuer(
         .as_bytes(),
     );
     let local_output = VcirLocalOutput {
-        output_id: rule_hash.clone(),
         output_type: VcirOutputType::Aspa,
         item_effective_until,
         source_object_uri: file.rsync_uri.clone(),
-        source_object_type: "aspa".to_string(),
-        source_object_hash: source_object_hash.clone(),
-        source_ee_cert_hash,
-        payload_json: serde_json::json!({
-            "customer_as_id": attestation.customer_as_id,
-            "provider_as_ids": attestation.provider_as_ids,
-        })
-        .to_string(),
-        rule_hash,
-        validation_path_hint: vec![
-            manifest_rsync_uri.to_string(),
-            file.rsync_uri.clone(),
-            source_object_hash,
-        ],
+        source_object_type: VcirSourceObjectType::Aspa,
+        source_object_hash: file.sha256,
+        source_ee_cert_hash: sha256_hex_to_32(&source_ee_cert_hash),
+        payload: VcirLocalOutputPayload::Aspa {
+            customer_as_id: attestation.customer_as_id,
+            provider_as_ids: attestation.provider_as_ids.clone(),
+        },
+        rule_hash: sha256_hex_to_32(&rule_hash),
     };
 
     Ok((attestation, Some(local_output)))
@@ -2965,6 +2979,54 @@ mod tests {
         assert_eq!(roa_afi_to_string(RoaAfi::Ipv6), "ipv6");
     }
 
+    #[test]
+    fn parallel_stage_roa_tasks_share_stage_owned_payloads() {
+        let stage = ParallelObjectsStage {
+            publication_point_id: 7,
+            shared: Arc::new(RoaTaskShared {
+                locked_files: Arc::<[PackFile]>::from(vec![
+                    PackFile::from_bytes_with_sha256(
+                        "rsync://example.test/repo/a.roa",
+                        vec![1, 2, 3],
+                        [1u8; 32],
+                    ),
+                    PackFile::from_bytes_with_sha256(
+                        "rsync://example.test/repo/b.roa",
+                        vec![4, 5, 6],
+                        [2u8; 32],
+                    ),
+                ]),
+                manifest_rsync_uri: Arc::<str>::from("rsync://example.test/repo/manifest.mft"),
+                issuer_ca_der: Arc::from([0x01u8].as_slice()),
+                issuer_ca: Arc::new(
+                    ResourceCertificate::decode_der(&fixture_bytes(
+                        "tests/fixtures/ta/apnic-ta.cer",
+                    ))
+                    .expect("decode fixture CA certificate"),
+                ),
+                issuer_spki_der: Arc::from([0x02u8].as_slice()),
+                issuer_ca_rsync_uri: Some(Arc::<str>::from("rsync://example.test/repo/ca.cer")),
+                crl_cache: Arc::new(Mutex::new(HashMap::new())),
+                issuer_resources_index: Arc::new(IssuerResourcesIndex::default()),
+                issuer_effective_ip: None,
+                issuer_effective_as: None,
+            }),
+            validation_time: OffsetDateTime::now_utc(),
+            collect_vcir_local_outputs: false,
+            strict_cms_der: false,
+            strict_name: false,
+            warnings: Vec::new(),
+            stats: ObjectsStats {
+                roa_total: 2,
+                ..ObjectsStats::default()
+            },
+            audit: Vec::new(),
+        };
+        let tasks = stage.build_roa_tasks();
+        assert_eq!(tasks.len(), 2);
+        assert!(Arc::ptr_eq(&tasks[0].shared, &tasks[1].shared));
+    }
+
     #[test]
     fn strict_name_manifest_decode_failure_drops_publication_point() {
         let publication_point = PublicationPointSnapshot {

src/validation/tree_parallel.rs

@@ -55,8 +55,37 @@ struct InflightPublicationPoint {
 }
 
 struct FinishedPublicationPoint {
-    node: QueuedCaInstance,
-    result: Result<PublicationPointRunResult, String>,
+    node: FinishedPublicationPointNode,
+    result: FinishedPublicationPointResult,
+}
+
+#[derive(Clone, Debug)]
+struct FinishedPublicationPointNode {
+    id: u64,
+    parent_id: Option<u64>,
+    discovered_from: Option<DiscoveredFrom>,
+    manifest_rsync_uri: String,
+}
+
+impl FinishedPublicationPointNode {
+    fn from_queued(node: QueuedCaInstance) -> Self {
+        Self {
+            id: node.id,
+            parent_id: node.parent_id,
+            discovered_from: node.discovered_from,
+            manifest_rsync_uri: node.handle.manifest_rsync_uri,
+        }
+    }
+}
+
+#[derive(Debug)]
+enum FinishedPublicationPointResult {
+    Ok {
+        warnings: Vec<Warning>,
+        objects: ObjectsOutput,
+        audit: PublicationPointAudit,
+    },
+    Err(String),
 }
 
 struct FinalizeTask {
@@ -310,24 +339,28 @@ fn elapsed_ms(started: Instant) -> u64 {
 fn compact_phase2_finished_result(
     mut result: PublicationPointRunResult,
     compact_audit: bool,
-) -> PublicationPointRunResult {
-    // Phase2 only needs warnings, objects, audit, and traversal metadata after finalize.
-    // Dropping the snapshot here avoids retaining manifest/files/raw-byte caches until run end.
-    result.snapshot = None;
+) -> FinishedPublicationPointResult {
+    result.objects.audit.clear();
+    result.objects.local_outputs_cache.clear();
     if compact_audit {
         result.audit.objects.clear();
         result.audit.warnings.clear();
-        result.objects.audit.clear();
-        result.discovered_children.clear();
     }
-    result
+    FinishedPublicationPointResult::Ok {
+        warnings: result.warnings,
+        objects: result.objects,
+        audit: result.audit,
+    }
 }
 
 fn compact_phase2_finished_result_result(
     result: Result<PublicationPointRunResult, String>,
     compact_audit: bool,
-) -> Result<PublicationPointRunResult, String> {
-    result.map(|result| compact_phase2_finished_result(result, compact_audit))
+) -> FinishedPublicationPointResult {
+    match result {
+        Ok(result) => compact_phase2_finished_result(result, compact_audit),
+        Err(err) => FinishedPublicationPointResult::Err(err),
+    }
 }
 
 pub fn run_tree_parallel_phase2_audit_multi_root(
@@ -712,8 +745,8 @@ fn start_queued_ca_instances(
             }
             Err(err) => {
                 finished.push(FinishedPublicationPoint {
-                    node,
-                    result: Err(err),
+                    node: FinishedPublicationPointNode::from_queued(node),
+                    result: FinishedPublicationPointResult::Err(err),
                 });
             }
         }
@@ -781,7 +814,7 @@ fn stage_ready_publication_point(
                 metrics.child_enqueue_ms = elapsed_ms(child_enqueue_started);
             }
             finished.push(FinishedPublicationPoint {
-                node: ready.node,
+                node: FinishedPublicationPointNode::from_queued(ready.node),
                 result: compact_phase2_finished_result_result(fallback, compact_audit),
             });
             metrics.total_ms = elapsed_ms(publication_point_started);
@@ -897,13 +930,10 @@ fn stage_ready_publication_point(
             metrics.locked_files = objects_stage.locked_file_count();
             metrics.aspa_objects = objects_stage.aspa_task_count();
             let build_tasks_started = Instant::now();
-            let tasks = objects_stage.build_roa_tasks();
+            objects_stage.append_roa_tasks_to(pending_roa_dispatch);
             metrics.build_roa_tasks_ms = elapsed_ms(build_tasks_started);
             let task_count = objects_stage.roa_task_count();
             metrics.roa_tasks = task_count;
-            for task in tasks {
-                pending_roa_dispatch.push_back(task);
-            }
             if task_count == 0 {
                 metrics.zero_task_count = 1;
                 match reduce_parallel_roa_stage(objects_stage, Vec::new(), runner.timing.as_ref()) {
@@ -929,8 +959,8 @@ fn stage_ready_publication_point(
                         );
                     }
                     Err(err) => finished.push(FinishedPublicationPoint {
-                        node: ready.node,
-                        result: Err(err),
+                        node: FinishedPublicationPointNode::from_queued(ready.node),
+                        result: FinishedPublicationPointResult::Err(err),
                     }),
                 }
             } else {
@@ -1022,7 +1052,7 @@ fn finalize_ready_objects(
         )
         .map(|out| out.result);
     finished.push(FinishedPublicationPoint {
-        node,
+        node: FinishedPublicationPointNode::from_queued(node),
         result: compact_phase2_finished_result_result(result, compact_audit),
     });
 }
@@ -1400,7 +1430,7 @@ fn finalize_publication_point_state(
                 elapsed_ms(finalize_started),
             )
         }
-        Err(err) => (Err(err), 0),
+        Err(err) => (FinishedPublicationPointResult::Err(err), 0),
     };
     let finalize_worker_ms = elapsed_ms(finalize_worker_started);
     crate::progress_log::emit(
@@ -1438,7 +1468,10 @@ fn finalize_publication_point_state(
         }),
     );
     FinalizeWorkerResult {
-        finished: FinishedPublicationPoint { node, result },
+        finished: FinishedPublicationPoint {
+            node: FinishedPublicationPointNode::from_queued(node),
+            result,
+        },
         metrics: FinalizePublicationPointMetrics {
             reduce_ms,
             finalize_ms,
@@ -1541,25 +1574,29 @@ fn build_tree_output(mut finished: Vec<FinishedPublicationPoint>) -> TreeRunAudi
 
     for item in finished {
         match item.result {
-            Ok(result) => {
+            FinishedPublicationPointResult::Ok {
+                warnings: result_warnings,
+                objects,
+                audit,
+            } => {
                 instances_processed += 1;
-                warnings.extend(result.warnings);
-                warnings.extend(result.objects.warnings);
-                vrps.extend(result.objects.vrps);
-                aspas.extend(result.objects.aspas);
-                router_keys.extend(result.objects.router_keys);
+                warnings.extend(result_warnings);
+                warnings.extend(objects.warnings);
+                vrps.extend(objects.vrps);
+                aspas.extend(objects.aspas);
+                router_keys.extend(objects.router_keys);
 
-                let mut audit: PublicationPointAudit = result.audit;
+                let mut audit: PublicationPointAudit = audit;
                 audit.node_id = Some(item.node.id);
                 audit.parent_node_id = item.node.parent_id;
                 audit.discovered_from = item.node.discovered_from;
                 publication_points.push(audit);
             }
-            Err(err) => {
+            FinishedPublicationPointResult::Err(err) => {
                 instances_failed += 1;
                 warnings.push(
                     Warning::new(format!("publication point failed: {err}"))
-                        .with_context(&item.node.handle.manifest_rsync_uri),
+                        .with_context(&item.node.manifest_rsync_uri),
                 );
             }
         }
@@ -1588,7 +1625,10 @@ pub fn run_tree_parallel_phase2_audit(
 
 #[cfg(test)]
 mod tests {
-    use super::{compact_phase2_finished_result, compact_phase2_finished_result_result};
+    use super::{
+        FinishedPublicationPointResult, compact_phase2_finished_result,
+        compact_phase2_finished_result_result,
+    };
     use crate::audit::PublicationPointAudit;
     use crate::storage::PackTime;
     use crate::validation::manifest::PublicationPointSource;
@@ -1638,9 +1678,12 @@ mod tests {
     #[test]
     fn compact_phase2_finished_result_drops_snapshot() {
         let result = compact_phase2_finished_result(sample_result(), false);
-        assert!(result.snapshot.is_none());
-        assert_eq!(result.source, PublicationPointSource::Fresh);
-        assert!(result.discovered_children.is_empty());
+        match result {
+            FinishedPublicationPointResult::Ok { warnings, .. } => {
+                assert!(warnings.is_empty());
+            }
+            FinishedPublicationPointResult::Err(err) => panic!("unexpected error: {err}"),
+        }
     }
 
     #[test]
@@ -1666,15 +1709,21 @@ mod tests {
             detail: None,
         });
         let result = compact_phase2_finished_result(sample, true);
-        assert!(result.audit.objects.is_empty());
-        assert!(result.audit.warnings.is_empty());
-        assert!(result.objects.audit.is_empty());
+        match result {
+            FinishedPublicationPointResult::Ok { objects, audit, .. } => {
+                assert!(audit.objects.is_empty());
+                assert!(audit.warnings.is_empty());
+                assert!(objects.audit.is_empty());
+            }
+            FinishedPublicationPointResult::Err(err) => panic!("unexpected error: {err}"),
+        }
     }
 
     #[test]
     fn compact_phase2_finished_result_result_preserves_err() {
-        let err = compact_phase2_finished_result_result(Err("boom".to_string()), false)
-            .expect_err("error should be preserved");
-        assert_eq!(err, "boom");
+        match compact_phase2_finished_result_result(Err("boom".to_string()), false) {
+            FinishedPublicationPointResult::Err(err) => assert_eq!(err, "boom"),
+            FinishedPublicationPointResult::Ok { .. } => panic!("error should be preserved"),
+        }
     }
 }

src/validation/tree_runner.rs

@@ -27,8 +27,8 @@ use crate::report::{RfcRef, Warning};
 use crate::storage::{
     PackFile, PackTime, RawByHashEntry, RocksStore, ValidatedCaInstanceResult, VcirArtifactKind,
     VcirArtifactRole, VcirArtifactValidationStatus, VcirAuditSummary, VcirCcrManifestProjection,
-    VcirChildEntry, VcirInstanceGate, VcirLocalOutput, VcirOutputType, VcirRelatedArtifact,
-    VcirSummary,
+    VcirChildEntry, VcirInstanceGate, VcirLocalOutput, VcirLocalOutputPayload, VcirOutputType,
+    VcirRelatedArtifact, VcirReplaceTimingBreakdown, VcirSourceObjectType, VcirSummary,
 };
 use crate::sync::repo::{
     sync_publication_point, sync_publication_point_replay, sync_publication_point_replay_delta,
@@ -58,13 +58,17 @@ use std::collections::{HashMap, HashSet};
 use std::sync::{Arc, Mutex};
 
 use base64::Engine as _;
-use serde::Deserialize;
-use serde_json::json;
 use x509_parser::prelude::FromDer;
 use x509_parser::x509::SubjectPublicKeyInfo;
 
 use vcir_der::encode_access_description_der_for_vcir_ccr_projection;
 
+fn sha256_hex_to_32(hex_value: &str) -> [u8; 32] {
+    let mut out = [0u8; 32];
+    hex::decode_to_slice(hex_value, &mut out).expect("internal sha256 hex should decode");
+    out
+}
+
 #[derive(Clone, Debug, Default)]
 pub(crate) struct BuildVcirTimingBreakdown {
     pub(crate) select_crl_ms: u64,
@@ -80,9 +84,9 @@ pub(crate) struct PersistVcirTimingBreakdown {
     pub(crate) embedded_collect_ms: u64,
     pub(crate) embedded_store_ms: u64,
     pub(crate) build_vcir_ms: u64,
-    pub(crate) previous_load_ms: u64,
     pub(crate) replace_vcir_ms: u64,
     pub(crate) build_vcir: BuildVcirTimingBreakdown,
+    pub(crate) replace_vcir: VcirReplaceTimingBreakdown,
 }
 
 #[derive(Clone, Debug)]
@@ -311,7 +315,7 @@ impl<'a> Rpkiv1PublicationPointRunner<'a> {
                 self.store,
                 ca,
                 &pack,
-                &objects,
+                &mut objects,
                 &warnings,
                 &child_audits,
                 &discovered_children,
@@ -666,7 +670,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
                             self.validation_time,
                             self.timing.as_ref(),
                             phase2_pool,
-                            self.persist_vcir,
+                            false,
                         )
                     } else if let Some(phase2_config) = self.parallel_phase2_config.as_ref() {
                         process_publication_point_for_issuer_parallel_roa_with_options(
@@ -679,7 +683,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
                             self.validation_time,
                             self.timing.as_ref(),
                             phase2_config,
-                            self.persist_vcir,
+                            false,
                         )
                     } else {
                         crate::validation::objects::process_publication_point_for_issuer_with_options(
@@ -691,7 +695,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
                             ca.effective_as_resources.as_ref(),
                             self.validation_time,
                             self.timing.as_ref(),
-                            self.persist_vcir,
+                            false,
                         )
                     }
                 };
@@ -747,7 +751,6 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
                         "persist_embedded_collect_ms": persist_vcir_timing.embedded_collect_ms,
                         "persist_embedded_store_ms": persist_vcir_timing.embedded_store_ms,
                         "persist_build_vcir_ms": persist_vcir_timing.build_vcir_ms,
-                        "persist_previous_load_ms": persist_vcir_timing.previous_load_ms,
                         "persist_replace_vcir_ms": persist_vcir_timing.replace_vcir_ms,
                         "persist_select_crl_ms": persist_vcir_timing.build_vcir.select_crl_ms,
                         "persist_current_ca_decode_ms": persist_vcir_timing.build_vcir.current_ca_decode_ms,
@@ -755,6 +758,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
                         "persist_child_entries_ms": persist_vcir_timing.build_vcir.child_entries_ms,
                         "persist_related_artifacts_ms": persist_vcir_timing.build_vcir.related_artifacts_ms,
                         "persist_vcir_struct_ms": persist_vcir_timing.build_vcir.struct_build_ms,
+                        "persist_replace_breakdown": &persist_vcir_timing.replace_vcir,
                         "audit_build_ms": audit_build_ms,
                         "warning_count": result.warnings.len(),
                         "vrp_count": result.objects.vrps.len(),
@@ -790,7 +794,6 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
                             "persist_embedded_collect_ms": persist_vcir_timing.embedded_collect_ms,
                             "persist_embedded_store_ms": persist_vcir_timing.embedded_store_ms,
                             "persist_build_vcir_ms": persist_vcir_timing.build_vcir_ms,
-                            "persist_previous_load_ms": persist_vcir_timing.previous_load_ms,
                             "persist_replace_vcir_ms": persist_vcir_timing.replace_vcir_ms,
                             "persist_select_crl_ms": persist_vcir_timing.build_vcir.select_crl_ms,
                             "persist_current_ca_decode_ms": persist_vcir_timing.build_vcir.current_ca_decode_ms,
@@ -798,6 +801,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
                             "persist_child_entries_ms": persist_vcir_timing.build_vcir.child_entries_ms,
                             "persist_related_artifacts_ms": persist_vcir_timing.build_vcir.related_artifacts_ms,
                             "persist_vcir_struct_ms": persist_vcir_timing.build_vcir.struct_build_ms,
+                            "persist_replace_breakdown": &persist_vcir_timing.replace_vcir,
                             "audit_build_ms": audit_build_ms,
                         }),
                     );
@@ -1012,26 +1016,6 @@ struct VcirReuseProjection {
     warnings: Vec<Warning>,
 }
 
-#[derive(Debug, Deserialize)]
-struct VcirVrpPayload {
-    asn: u32,
-    prefix: String,
-    max_length: u16,
-}
-
-#[derive(Debug, Deserialize)]
-struct VcirAspaPayload {
-    customer_as_id: u32,
-    provider_as_ids: Vec<u32>,
-}
-
-#[derive(Debug, Deserialize)]
-struct VcirRouterKeyPayload {
-    as_id: u32,
-    ski_hex: String,
-    spki_der_base64: String,
-}
-
 fn discover_children_from_fresh_snapshot_with_audit<P: PublicationPointData>(
     issuer: &CaInstanceHandle,
     publication_point: &P,
@@ -2299,7 +2283,7 @@ fn build_objects_output_from_vcir(
                     local.source_object_uri.clone(),
                     ObjectAuditEntry {
                         rsync_uri: local.source_object_uri.clone(),
-                        sha256_hex: local.source_object_hash.clone(),
+                        sha256_hex: local.source_object_hash_hex(),
                         kind: audit_kind_for_vcir_output_type(local.output_type),
                         result: AuditObjectResult::Error,
                         detail: Some(
@@ -2315,7 +2299,7 @@ fn build_objects_output_from_vcir(
                 .entry(local.source_object_uri.clone())
                 .or_insert_with(|| ObjectAuditEntry {
                     rsync_uri: local.source_object_uri.clone(),
-                    sha256_hex: local.source_object_hash.clone(),
+                    sha256_hex: local.source_object_hash_hex(),
                     kind: audit_kind_for_vcir_output_type(local.output_type),
                     result: AuditObjectResult::Skipped,
                     detail: Some("skipped: cached local output expired".to_string()),
@@ -2332,7 +2316,7 @@ fn build_objects_output_from_vcir(
                         local.source_object_uri.clone(),
                         ObjectAuditEntry {
                             rsync_uri: local.source_object_uri.clone(),
-                            sha256_hex: local.source_object_hash.clone(),
+                            sha256_hex: local.source_object_hash_hex(),
                             kind: AuditObjectKind::Roa,
                             result: AuditObjectResult::Ok,
                             detail: None,
@@ -2348,7 +2332,7 @@ fn build_objects_output_from_vcir(
                         local.source_object_uri.clone(),
                         ObjectAuditEntry {
                             rsync_uri: local.source_object_uri.clone(),
-                            sha256_hex: local.source_object_hash.clone(),
+                            sha256_hex: local.source_object_hash_hex(),
                             kind: AuditObjectKind::Roa,
                             result: AuditObjectResult::Error,
                             detail: Some(format!("cached ROA local output parse failed: {e}")),
@@ -2364,7 +2348,7 @@ fn build_objects_output_from_vcir(
                         local.source_object_uri.clone(),
                         ObjectAuditEntry {
                             rsync_uri: local.source_object_uri.clone(),
-                            sha256_hex: local.source_object_hash.clone(),
+                            sha256_hex: local.source_object_hash_hex(),
                             kind: AuditObjectKind::Aspa,
                             result: AuditObjectResult::Ok,
                             detail: None,
@@ -2380,7 +2364,7 @@ fn build_objects_output_from_vcir(
                         local.source_object_uri.clone(),
                         ObjectAuditEntry {
                             rsync_uri: local.source_object_uri.clone(),
-                            sha256_hex: local.source_object_hash.clone(),
+                            sha256_hex: local.source_object_hash_hex(),
                             kind: AuditObjectKind::Aspa,
                             result: AuditObjectResult::Error,
                             detail: Some(format!("cached ASPA local output parse failed: {e}")),
@@ -2395,7 +2379,7 @@ fn build_objects_output_from_vcir(
                         local.source_object_uri.clone(),
                         ObjectAuditEntry {
                             rsync_uri: local.source_object_uri.clone(),
-                            sha256_hex: local.source_object_hash.clone(),
+                            sha256_hex: local.source_object_hash_hex(),
                             kind: AuditObjectKind::RouterCertificate,
                             result: AuditObjectResult::Ok,
                             detail: Some("cached Router Key local output restored".to_string()),
@@ -2411,7 +2395,7 @@ fn build_objects_output_from_vcir(
                         local.source_object_uri.clone(),
                         ObjectAuditEntry {
                             rsync_uri: local.source_object_uri.clone(),
-                            sha256_hex: local.source_object_hash.clone(),
+                            sha256_hex: local.source_object_hash_hex(),
                             kind: AuditObjectKind::RouterCertificate,
                             result: AuditObjectResult::Error,
                             detail: Some(format!(
@@ -2435,68 +2419,55 @@ fn build_objects_output_from_vcir(
 }
 
 fn parse_vcir_vrp_output(local: &VcirLocalOutput) -> Result<Vrp, String> {
-    let payload: VcirVrpPayload = serde_json::from_str(&local.payload_json)
-        .map_err(|e| format!("invalid VRP payload JSON: {e}"))?;
-    Ok(Vrp {
-        asn: payload.asn,
-        prefix: parse_vcir_prefix(&payload.prefix)?,
-        max_length: payload.max_length,
-    })
+    match &local.payload {
+        VcirLocalOutputPayload::Vrp {
+            asn,
+            afi,
+            prefix_len,
+            addr,
+            max_length,
+        } => Ok(Vrp {
+            asn: *asn,
+            prefix: crate::data_model::roa::IpPrefix {
+                afi: *afi,
+                prefix_len: *prefix_len,
+                addr: *addr,
+            },
+            max_length: *max_length,
+        }),
+        _ => Err("VCIR local output payload is not VRP".to_string()),
+    }
 }
 
 fn parse_vcir_aspa_output(local: &VcirLocalOutput) -> Result<AspaAttestation, String> {
-    let payload: VcirAspaPayload = serde_json::from_str(&local.payload_json)
-        .map_err(|e| format!("invalid ASPA payload JSON: {e}"))?;
-    Ok(AspaAttestation {
-        customer_as_id: payload.customer_as_id,
-        provider_as_ids: payload.provider_as_ids,
-    })
+    match &local.payload {
+        VcirLocalOutputPayload::Aspa {
+            customer_as_id,
+            provider_as_ids,
+        } => Ok(AspaAttestation {
+            customer_as_id: *customer_as_id,
+            provider_as_ids: provider_as_ids.clone(),
+        }),
+        _ => Err("VCIR local output payload is not ASPA".to_string()),
+    }
 }
 
 fn parse_vcir_router_key_output(local: &VcirLocalOutput) -> Result<RouterKeyPayload, String> {
-    let payload: VcirRouterKeyPayload = serde_json::from_str(&local.payload_json)
-        .map_err(|e| format!("invalid Router Key payload JSON: {e}"))?;
-    let ski =
-        hex::decode(&payload.ski_hex).map_err(|e| format!("invalid Router Key SKI hex: {e}"))?;
-    let spki_der = base64::engine::general_purpose::STANDARD
-        .decode(&payload.spki_der_base64)
-        .map_err(|e| format!("invalid Router Key SPKI base64: {e}"))?;
-    Ok(RouterKeyPayload {
-        as_id: payload.as_id,
+    match &local.payload {
+        VcirLocalOutputPayload::RouterKey {
+            as_id,
             ski,
             spki_der,
+        } => Ok(RouterKeyPayload {
+            as_id: *as_id,
+            ski: ski.clone(),
+            spki_der: spki_der.clone(),
             source_object_uri: local.source_object_uri.clone(),
-        source_object_hash: local.source_object_hash.clone(),
-        source_ee_cert_hash: local.source_ee_cert_hash.clone(),
+            source_object_hash: local.source_object_hash_hex(),
+            source_ee_cert_hash: local.source_ee_cert_hash_hex(),
             item_effective_until: local.item_effective_until.clone(),
-    })
-}
-
-fn parse_vcir_prefix(prefix: &str) -> Result<crate::data_model::roa::IpPrefix, String> {
-    let (addr, len) = prefix
-        .split_once('/')
-        .ok_or_else(|| format!("prefix missing '/': {prefix}"))?;
-    let prefix_len = len
-        .parse::<u16>()
-        .map_err(|e| format!("invalid prefix length '{len}': {e}"))?;
-    let ip = addr
-        .parse::<std::net::IpAddr>()
-        .map_err(|e| format!("invalid IP address '{addr}': {e}"))?;
-    match ip {
-        std::net::IpAddr::V4(v4) => Ok(crate::data_model::roa::IpPrefix {
-            afi: RoaAfi::Ipv4,
-            prefix_len,
-            addr: {
-                let mut addr = [0u8; 16];
-                addr[..4].copy_from_slice(&v4.octets());
-                addr
-            },
-        }),
-        std::net::IpAddr::V6(v6) => Ok(crate::data_model::roa::IpPrefix {
-            afi: RoaAfi::Ipv6,
-            prefix_len,
-            addr: v6.octets(),
         }),
+        _ => Err("VCIR local output payload is not Router Key".to_string()),
     }
 }
 
@@ -2582,7 +2553,7 @@ fn persist_vcir_for_fresh_result(
     store: &RocksStore,
     ca: &CaInstanceHandle,
     pack: &PublicationPointSnapshot,
-    objects: &crate::validation::objects::ObjectsOutput,
+    objects: &mut crate::validation::objects::ObjectsOutput,
     warnings: &[Warning],
     child_audits: &[ObjectAuditEntry],
     discovered_children: &[DiscoveredChildCaInstance],
@@ -2605,7 +2576,7 @@ fn persist_vcir_for_fresh_result_with_timing(
     store: &RocksStore,
     ca: &CaInstanceHandle,
     pack: &PublicationPointSnapshot,
-    objects: &crate::validation::objects::ObjectsOutput,
+    objects: &mut crate::validation::objects::ObjectsOutput,
     warnings: &[Warning],
     child_audits: &[ObjectAuditEntry],
     discovered_children: &[DiscoveredChildCaInstance],
@@ -2635,17 +2606,12 @@ fn persist_vcir_for_fresh_result_with_timing(
     timing.build_vcir_ms = build_vcir_started.elapsed().as_millis() as u64;
     timing.build_vcir = build_vcir_timing;
 
-    let previous_load_started = std::time::Instant::now();
-    let previous = store
-        .get_vcir(&pack.manifest_rsync_uri)
-        .map_err(|e| format!("load existing VCIR failed: {e}"))?;
-    timing.previous_load_ms = previous_load_started.elapsed().as_millis() as u64;
-
     let replace_vcir_started = std::time::Instant::now();
-    store
-        .replace_vcir_and_audit_rule_indexes(previous.as_ref(), &vcir)
-        .map_err(|e| format!("store VCIR and audit rule index failed: {e}"))?;
+    let replace_timing = store
+        .replace_vcir_and_manifest_replay_meta(&vcir)
+        .map_err(|e| format!("store VCIR and manifest replay meta failed: {e}"))?;
     timing.replace_vcir_ms = replace_vcir_started.elapsed().as_millis() as u64;
+    timing.replace_vcir = replace_timing;
 
     Ok(timing)
 }
@@ -2653,7 +2619,7 @@ fn persist_vcir_for_fresh_result_with_timing(
 fn build_vcir_from_fresh_result(
     ca: &CaInstanceHandle,
     pack: &PublicationPointSnapshot,
-    objects: &crate::validation::objects::ObjectsOutput,
+    objects: &mut crate::validation::objects::ObjectsOutput,
     warnings: &[Warning],
     child_audits: &[ObjectAuditEntry],
     discovered_children: &[DiscoveredChildCaInstance],
@@ -2674,7 +2640,7 @@ fn build_vcir_from_fresh_result(
 fn build_vcir_from_fresh_result_with_timing(
     ca: &CaInstanceHandle,
     pack: &PublicationPointSnapshot,
-    objects: &crate::validation::objects::ObjectsOutput,
+    objects: &mut crate::validation::objects::ObjectsOutput,
     warnings: &[Warning],
     child_audits: &[ObjectAuditEntry],
     discovered_children: &[DiscoveredChildCaInstance],
@@ -2692,7 +2658,7 @@ fn build_vcir_from_fresh_result_with_timing(
     timing.current_ca_decode_ms = current_ca_decode_started.elapsed().as_millis() as u64;
 
     let local_outputs_started = std::time::Instant::now();
-    let local_outputs = build_vcir_local_outputs(ca, pack, objects)?;
+    let local_outputs = take_or_build_vcir_local_outputs(ca, pack, objects)?;
     timing.local_outputs_ms = local_outputs_started.elapsed().as_millis() as u64;
 
     let child_entries_started = std::time::Instant::now();
@@ -2710,6 +2676,18 @@ fn build_vcir_from_fresh_result_with_timing(
     timing.related_artifacts_ms = related_artifacts_started.elapsed().as_millis() as u64;
     let ccr_manifest_projection =
         build_vcir_ccr_manifest_projection_from_fresh(ca, pack, &child_entries)?;
+    let local_vrp_count = local_outputs
+        .iter()
+        .filter(|output| output.output_type == VcirOutputType::Vrp)
+        .count() as u32;
+    let local_aspa_count = local_outputs
+        .iter()
+        .filter(|output| output.output_type == VcirOutputType::Aspa)
+        .count() as u32;
+    let local_router_key_count = local_outputs
+        .iter()
+        .filter(|output| output.output_type == VcirOutputType::RouterKey)
+        .count() as u32;
     let accepted_object_count = related_artifacts
         .iter()
         .filter(|artifact| artifact.validation_status == VcirArtifactValidationStatus::Accepted)
@@ -2768,21 +2746,12 @@ fn build_vcir_from_fresh_result_with_timing(
             ),
         },
         child_entries,
-        local_outputs: local_outputs.clone(),
-        related_artifacts: related_artifacts.clone(),
+        local_outputs,
+        related_artifacts,
         summary: VcirSummary {
-            local_vrp_count: local_outputs
-                .iter()
-                .filter(|output| output.output_type == VcirOutputType::Vrp)
-                .count() as u32,
-            local_aspa_count: local_outputs
-                .iter()
-                .filter(|output| output.output_type == VcirOutputType::Aspa)
-                .count() as u32,
-            local_router_key_count: local_outputs
-                .iter()
-                .filter(|output| output.output_type == VcirOutputType::RouterKey)
-                .count() as u32,
+            local_vrp_count,
+            local_aspa_count,
+            local_router_key_count,
             child_count: discovered_children.len() as u32,
             accepted_object_count,
             rejected_object_count,
@@ -2799,6 +2768,17 @@ fn build_vcir_from_fresh_result_with_timing(
     Ok((vcir, timing))
 }
 
+fn take_or_build_vcir_local_outputs(
+    ca: &CaInstanceHandle,
+    pack: &PublicationPointSnapshot,
+    objects: &mut crate::validation::objects::ObjectsOutput,
+) -> Result<Vec<VcirLocalOutput>, String> {
+    if !objects.local_outputs_cache.is_empty() {
+        return Ok(std::mem::take(&mut objects.local_outputs_cache));
+    }
+    build_vcir_local_outputs(ca, pack, objects)
+}
+
 fn build_vcir_ccr_manifest_projection_from_fresh(
     ca: &CaInstanceHandle,
     pack: &PublicationPointSnapshot,
@@ -2892,7 +2872,7 @@ fn select_manifest_current_crl_from_snapshot(
 }
 
 fn build_vcir_local_outputs(
-    ca: &CaInstanceHandle,
+    _ca: &CaInstanceHandle,
     pack: &PublicationPointSnapshot,
     objects: &crate::validation::objects::ObjectsOutput,
 ) -> Result<Vec<VcirLocalOutput>, String> {
@@ -2930,12 +2910,6 @@ fn build_vcir_local_outputs(
                 PackTime::from_utc_offset_datetime(ee.resource_cert.tbs.validity_not_after);
             for vrp in roa_to_vrps_for_vcir(&roa) {
                 let prefix = vrp_prefix_to_string(&vrp);
-                let payload_json = json!({
-                    "asn": vrp.asn,
-                    "prefix": prefix,
-                    "max_length": vrp.max_length,
-                })
-                .to_string();
                 let rule_hash = sha256_hex(
                     format!(
                         "roa-rule:{}:{}:{}:{}",
@@ -2944,20 +2918,20 @@ fn build_vcir_local_outputs(
                     .as_bytes(),
                 );
                 out.push(VcirLocalOutput {
-                    output_id: rule_hash.clone(),
                     output_type: VcirOutputType::Vrp,
                     item_effective_until: item_effective_until.clone(),
                     source_object_uri: file.rsync_uri.clone(),
-                    source_object_type: "roa".to_string(),
-                    source_object_hash: source_object_hash.clone(),
-                    source_ee_cert_hash: source_ee_cert_hash.clone(),
-                    payload_json,
-                    rule_hash,
-                    validation_path_hint: vec![
-                        ca.manifest_rsync_uri.clone(),
-                        file.rsync_uri.clone(),
-                        source_object_hash.clone(),
-                    ],
+                    source_object_type: VcirSourceObjectType::Roa,
+                    source_object_hash: file.sha256,
+                    source_ee_cert_hash: sha256_hex_to_32(&source_ee_cert_hash),
+                    payload: VcirLocalOutputPayload::Vrp {
+                        asn: vrp.asn,
+                        afi: vrp.prefix.afi,
+                        prefix_len: vrp.prefix.prefix_len,
+                        addr: vrp.prefix.addr,
+                        max_length: vrp.max_length,
+                    },
+                    rule_hash: sha256_hex_to_32(&rule_hash),
                 });
             }
         } else if accepted_aspa_uris.contains(file.rsync_uri.as_str()) {
@@ -2970,11 +2944,6 @@ fn build_vcir_local_outputs(
             let source_ee_cert_hash = sha256_hex(ee.raw_der.as_slice());
             let item_effective_until =
                 PackTime::from_utc_offset_datetime(ee.resource_cert.tbs.validity_not_after);
-            let payload_json = json!({
-                "customer_as_id": aspa.aspa.customer_as_id,
-                "provider_as_ids": aspa.aspa.provider_as_ids,
-            })
-            .to_string();
             let providers = aspa
                 .aspa
                 .provider_as_ids
@@ -2990,20 +2959,17 @@ fn build_vcir_local_outputs(
                 .as_bytes(),
             );
             out.push(VcirLocalOutput {
-                output_id: rule_hash.clone(),
                 output_type: VcirOutputType::Aspa,
                 item_effective_until,
                 source_object_uri: file.rsync_uri.clone(),
-                source_object_type: "aspa".to_string(),
-                source_object_hash: source_object_hash.clone(),
-                source_ee_cert_hash,
-                payload_json,
-                rule_hash,
-                validation_path_hint: vec![
-                    ca.manifest_rsync_uri.clone(),
-                    file.rsync_uri.clone(),
-                    source_object_hash,
-                ],
+                source_object_type: VcirSourceObjectType::Aspa,
+                source_object_hash: file.sha256,
+                source_ee_cert_hash: sha256_hex_to_32(&source_ee_cert_hash),
+                payload: VcirLocalOutputPayload::Aspa {
+                    customer_as_id: aspa.aspa.customer_as_id,
+                    provider_as_ids: aspa.aspa.provider_as_ids.clone(),
+                },
+                rule_hash: sha256_hex_to_32(&rule_hash),
             });
         }
     }
@@ -3011,7 +2977,7 @@ fn build_vcir_local_outputs(
 }
 
 pub(crate) fn build_router_key_local_outputs(
-    ca: &CaInstanceHandle,
+    _ca: &CaInstanceHandle,
     router_keys: &[RouterKeyPayload],
 ) -> Vec<VcirLocalOutput> {
     router_keys
@@ -3028,25 +2994,18 @@ pub(crate) fn build_router_key_local_outputs(
                 .as_bytes(),
             );
             VcirLocalOutput {
-                output_id: rule_hash.clone(),
                 output_type: VcirOutputType::RouterKey,
                 item_effective_until: router_key.item_effective_until.clone(),
                 source_object_uri: router_key.source_object_uri.clone(),
-                source_object_type: "router_key".to_string(),
-                source_object_hash: router_key.source_object_hash.clone(),
-                source_ee_cert_hash: router_key.source_ee_cert_hash.clone(),
-                payload_json: json!({
-                    "as_id": router_key.as_id,
-                    "ski_hex": ski_hex,
-                    "spki_der_base64": spki_der_base64,
-                })
-                .to_string(),
-                rule_hash,
-                validation_path_hint: vec![
-                    ca.manifest_rsync_uri.clone(),
-                    router_key.source_object_uri.clone(),
-                    router_key.source_object_hash.clone(),
-                ],
+                source_object_type: VcirSourceObjectType::RouterKey,
+                source_object_hash: sha256_hex_to_32(&router_key.source_object_hash),
+                source_ee_cert_hash: sha256_hex_to_32(&router_key.source_ee_cert_hash),
+                payload: VcirLocalOutputPayload::RouterKey {
+                    as_id: router_key.as_id,
+                    ski: router_key.ski.clone(),
+                    spki_der: router_key.spki_der.clone(),
+                },
+                rule_hash: sha256_hex_to_32(&rule_hash),
             }
         })
         .collect()

src/validation/tree_runner/tests.rs

@@ -1,12 +1,13 @@
 use super::*;
 use crate::data_model::rc::ResourceCertificate;
+use crate::data_model::roa::RoaAfi;
 use crate::fetch::rsync::LocalDirRsyncFetcher;
 use crate::fetch::rsync::{RsyncFetchError, RsyncFetcher};
 use crate::storage::{
     PackFile, PackTime, RawByHashEntry, RocksStore, ValidatedCaInstanceResult,
     ValidatedManifestMeta, VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus,
-    VcirAuditSummary, VcirChildEntry, VcirInstanceGate, VcirLocalOutput, VcirOutputType,
-    VcirRelatedArtifact, VcirSummary,
+    VcirAuditSummary, VcirChildEntry, VcirInstanceGate, VcirLocalOutput, VcirLocalOutputPayload,
+    VcirOutputType, VcirRelatedArtifact, VcirSourceObjectType, VcirSummary,
 };
 use crate::sync::rrdp::Fetcher;
 use crate::validation::publication_point::PublicationPointSnapshot;
@@ -16,6 +17,16 @@ use std::process::Command;
 use std::sync::Arc;
 use std::sync::atomic::{AtomicUsize, Ordering};
 
+fn sha256_32(input: &[u8]) -> [u8; 32] {
+    sha256_hex_to_32(&sha256_hex(input))
+}
+
+fn ipv4_addr(octets: [u8; 4]) -> [u8; 16] {
+    let mut addr = [0u8; 16];
+    addr[..4].copy_from_slice(&octets);
+    addr
+}
+
 struct NeverHttpFetcher;
 impl Fetcher for NeverHttpFetcher {
     fn fetch(&self, _uri: &str) -> Result<Vec<u8>, String> {
@@ -569,44 +580,53 @@ fn sample_vcir_for_projection(
         }],
         local_outputs: vec![
             VcirLocalOutput {
-                    output_id: sha256_hex(b"vrp-out"),
                 output_type: VcirOutputType::Vrp,
-                    item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(30)),
+                item_effective_until: PackTime::from_utc_offset_datetime(
+                    now + time::Duration::minutes(30),
+                ),
                 source_object_uri: roa_uri.clone(),
-                    source_object_type: "roa".to_string(),
-                    source_object_hash: roa_hash.clone(),
-                    source_ee_cert_hash: ee_hash.clone(),
-                    payload_json: serde_json::json!({"asn": 64496, "prefix": "203.0.113.0/24", "max_length": 24}).to_string(),
-                    rule_hash: sha256_hex(b"roa-rule"),
-                    validation_path_hint: vec![manifest_uri.clone(), roa_uri.clone()],
+                source_object_type: VcirSourceObjectType::Roa,
+                source_object_hash: sha256_hex_to_32(&roa_hash),
+                source_ee_cert_hash: sha256_hex_to_32(&ee_hash),
+                payload: VcirLocalOutputPayload::Vrp {
+                    asn: 64496,
+                    afi: RoaAfi::Ipv4,
+                    prefix_len: 24,
+                    addr: ipv4_addr([203, 0, 113, 0]),
+                    max_length: 24,
+                },
+                rule_hash: sha256_32(b"roa-rule"),
             },
             VcirLocalOutput {
-                    output_id: sha256_hex(b"aspa-out"),
                 output_type: VcirOutputType::Aspa,
-                    item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(30)),
+                item_effective_until: PackTime::from_utc_offset_datetime(
+                    now + time::Duration::minutes(30),
+                ),
                 source_object_uri: aspa_uri.clone(),
-                    source_object_type: "aspa".to_string(),
-                    source_object_hash: aspa_hash.clone(),
-                    source_ee_cert_hash: ee_hash,
-                    payload_json: serde_json::json!({"customer_as_id": 64496, "provider_as_ids": [64497, 64498]}).to_string(),
-                    rule_hash: sha256_hex(b"aspa-rule"),
-                    validation_path_hint: vec![manifest_uri.clone(), aspa_uri.clone()],
+                source_object_type: VcirSourceObjectType::Aspa,
+                source_object_hash: sha256_hex_to_32(&aspa_hash),
+                source_ee_cert_hash: sha256_hex_to_32(&ee_hash),
+                payload: VcirLocalOutputPayload::Aspa {
+                    customer_as_id: 64496,
+                    provider_as_ids: vec![64497, 64498],
+                },
+                rule_hash: sha256_32(b"aspa-rule"),
             },
             VcirLocalOutput {
-                    output_id: sha256_hex(b"router-key-out"),
                 output_type: VcirOutputType::RouterKey,
-                    item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(30)),
+                item_effective_until: PackTime::from_utc_offset_datetime(
+                    now + time::Duration::minutes(30),
+                ),
                 source_object_uri: router_uri.clone(),
-                    source_object_type: "router_key".to_string(),
-                    source_object_hash: router_hash.clone(),
-                    source_ee_cert_hash: router_hash.clone(),
-                    payload_json: serde_json::json!({
-                        "as_id": 64496,
-                        "ski_hex": "11".repeat(20),
-                        "spki_der_base64": base64::engine::general_purpose::STANDARD.encode([0x30u8, 0x00]),
-                    }).to_string(),
-                    rule_hash: sha256_hex(b"router-key-rule"),
-                    validation_path_hint: vec![manifest_uri.clone(), router_uri.clone()],
+                source_object_type: VcirSourceObjectType::RouterKey,
+                source_object_hash: sha256_hex_to_32(&router_hash),
+                source_ee_cert_hash: sha256_hex_to_32(&router_hash),
+                payload: VcirLocalOutputPayload::RouterKey {
+                    as_id: 64496,
+                    ski: vec![0x11; 20],
+                    spki_der: vec![0x30, 0x00],
+                },
+                rule_hash: sha256_32(b"router-key-rule"),
             },
         ],
         related_artifacts: vec![
@@ -716,16 +736,20 @@ fn build_vcir_local_outputs_prefers_cached_outputs() {
         rrdp_notification_uri: None,
     };
     let cached = vec![VcirLocalOutput {
-        output_id: "cached-output".to_string(),
         output_type: VcirOutputType::Vrp,
         item_effective_until: pack.next_update.clone(),
         source_object_uri: "rsync://example.test/repo/issuer/a.roa".to_string(),
-        source_object_type: "roa".to_string(),
-        source_object_hash: sha256_hex(b"cached-roa"),
-        source_ee_cert_hash: sha256_hex(b"cached-ee"),
-        payload_json: "{\"asn\":64500}".to_string(),
-        rule_hash: sha256_hex(b"cached-rule"),
-        validation_path_hint: vec![pack.manifest_rsync_uri.clone()],
+        source_object_type: VcirSourceObjectType::Roa,
+        source_object_hash: sha256_32(b"cached-roa"),
+        source_ee_cert_hash: sha256_32(b"cached-ee"),
+        payload: VcirLocalOutputPayload::Vrp {
+            asn: 64500,
+            afi: RoaAfi::Ipv4,
+            prefix_len: 24,
+            addr: ipv4_addr([203, 0, 113, 0]),
+            max_length: 24,
+        },
+        rule_hash: sha256_32(b"cached-rule"),
     }];
     let outputs = build_vcir_local_outputs(
         &ca,
@@ -801,7 +825,7 @@ fn persist_vcir_non_repository_evidence_stores_current_ca_cert_only() {
         .expect("first local output");
     assert!(
         store
-            .get_raw_by_hash_entry(&first_output.source_ee_cert_hash)
+            .get_raw_by_hash_entry(&first_output.source_ee_cert_hash_hex())
             .expect("load source ee raw")
             .is_none()
     );
@@ -838,8 +862,11 @@ fn build_router_key_local_outputs_encodes_router_key_payloads() {
     );
     assert_eq!(outputs.len(), 1);
     assert_eq!(outputs[0].output_type, VcirOutputType::RouterKey);
-    assert_eq!(outputs[0].source_object_type, "router_key");
-    assert!(outputs[0].payload_json.contains("spki_der_base64"));
+    assert_eq!(
+        outputs[0].source_object_type,
+        VcirSourceObjectType::RouterKey
+    );
+    assert!(outputs[0].payload_json().contains("spki_der_base64"));
 }
 
 #[test]
@@ -996,7 +1023,7 @@ fn finalize_fresh_publication_point_releases_local_outputs_cache_after_persist()
 }
 
 #[test]
-fn persist_vcir_for_fresh_result_stores_vcir_and_audit_indexes_for_real_snapshot() {
+fn persist_vcir_for_fresh_result_stores_vcir_and_replay_meta_for_real_snapshot() {
     let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests();
     let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca");
     let objects = crate::validation::objects::process_publication_point_snapshot_for_issuer(
@@ -1029,7 +1056,17 @@ fn persist_vcir_for_fresh_result_stores_vcir_and_audit_indexes_for_real_snapshot
             rrdp_notification_uri: None,
         };
 
-    persist_vcir_for_fresh_result(&store, &ca, &pack, &objects, &[], &[], &[], validation_time)
+    let mut objects = objects;
+    persist_vcir_for_fresh_result(
+        &store,
+        &ca,
+        &pack,
+        &mut objects,
+        &[],
+        &[],
+        &[],
+        validation_time,
+    )
     .expect("persist vcir for fresh result");
 
     let vcir = store
@@ -1054,12 +1091,15 @@ fn persist_vcir_for_fresh_result_stores_vcir_and_audit_indexes_for_real_snapshot
         vcir.ccr_manifest_projection.manifest_size,
         pack.manifest_bytes.len() as u64
     );
-    let first_output = vcir.local_outputs.first().expect("local outputs stored");
-    assert!(
-        store
-            .get_audit_rule_index_entry(crate::storage::AuditRuleKind::Roa, &first_output.rule_hash)
-            .expect("get audit rule index entry")
-            .is_some()
+    assert!(vcir.local_outputs.first().is_some(), "local outputs stored");
+    let replay_meta = store
+        .get_manifest_replay_meta(&pack.manifest_rsync_uri)
+        .expect("get replay meta")
+        .expect("replay meta exists");
+    assert_eq!(replay_meta.manifest_rsync_uri, pack.manifest_rsync_uri);
+    assert_eq!(
+        replay_meta.manifest_sha256,
+        sha2::Sha256::digest(&pack.manifest_bytes).to_vec()
     );
 }
 
@@ -1531,26 +1571,16 @@ fn runner_offline_rsync_fixture_produces_pack_and_warnings() {
         .iter()
         .find(|output| output.output_type == crate::storage::VcirOutputType::Vrp)
         .expect("first VCIR VRP output");
-    let audit_rule = store
-        .get_audit_rule_index_entry(crate::storage::AuditRuleKind::Roa, &first_vrp.rule_hash)
-        .expect("get audit rule index")
-        .expect("audit rule index exists");
-    assert_eq!(audit_rule.manifest_rsync_uri, manifest_rsync_uri);
-    assert_eq!(audit_rule.output_id, first_vrp.output_id);
-    let trace = crate::audit_trace::trace_rule_to_root(
-        &store,
-        crate::storage::AuditRuleKind::Roa,
-        &first_vrp.rule_hash,
-    )
-    .expect("trace rule")
-    .expect("trace exists");
-    assert_eq!(trace.chain_leaf_to_root.len(), 1);
+    assert!(!first_vrp.rule_hash_hex().is_empty());
+    assert!(!first_vrp.output_id().is_empty());
+    let replay_meta = store
+        .get_manifest_replay_meta(&manifest_rsync_uri)
+        .expect("get replay meta")
+        .expect("replay meta exists");
     assert_eq!(
-        trace.chain_leaf_to_root[0].manifest_rsync_uri,
+        replay_meta.manifest_rsync_uri,
         manifest_rsync_uri
     );
-    assert!(trace.source_object_raw.raw_present);
-    assert!(trace.source_ee_cert_raw.raw_present);
 }
 
 #[test]
@@ -2791,10 +2821,11 @@ fn fresh_and_reuse_paths_produce_equivalent_ccr_manifest_projection() {
     let child_discovery =
         discover_children_from_fresh_snapshot_with_audit(&ca, &pack, validation_time, None)
             .expect("discover children");
+    let mut objects = empty_objects_output();
     let fresh_vcir = build_vcir_from_fresh_result(
         &ca,
         &pack,
-        &empty_objects_output(),
+        &mut objects,
         &[],
         &child_discovery.audits,
         &child_discovery.children,
@@ -2913,70 +2944,77 @@ fn build_objects_output_from_vcir_tracks_expired_and_invalid_cached_outputs() {
     }
 
     vcir.local_outputs.push(VcirLocalOutput {
-        output_id: sha256_hex(b"bad-time"),
         output_type: VcirOutputType::Vrp,
         item_effective_until: PackTime {
             rfc3339_utc: "bad-time-value".to_string(),
         },
         source_object_uri: bad_time_uri.clone(),
-        source_object_type: "roa".to_string(),
-        source_object_hash: sha256_hex(b"bad-time-src"),
-        source_ee_cert_hash: sha256_hex(b"bad-time-ee"),
-        payload_json:
-            serde_json::json!({"asn": 64496, "prefix": "203.0.113.0/24", "max_length": 24})
-                .to_string(),
-        rule_hash: sha256_hex(b"bad-time-rule"),
-        validation_path_hint: vec![vcir.manifest_rsync_uri.clone(), bad_time_uri.clone()],
+        source_object_type: VcirSourceObjectType::Roa,
+        source_object_hash: sha256_32(b"bad-time-src"),
+        source_ee_cert_hash: sha256_32(b"bad-time-ee"),
+        payload: VcirLocalOutputPayload::Vrp {
+            asn: 64496,
+            afi: RoaAfi::Ipv4,
+            prefix_len: 24,
+            addr: ipv4_addr([203, 0, 113, 0]),
+            max_length: 24,
+        },
+        rule_hash: sha256_32(b"bad-time-rule"),
     });
     vcir.local_outputs.push(VcirLocalOutput {
-        output_id: sha256_hex(b"expired"),
         output_type: VcirOutputType::Aspa,
         item_effective_until: PackTime::from_utc_offset_datetime(now - time::Duration::minutes(1)),
         source_object_uri: expired_uri.clone(),
-        source_object_type: "aspa".to_string(),
-        source_object_hash: sha256_hex(b"expired-src"),
-        source_ee_cert_hash: sha256_hex(b"expired-ee"),
-        payload_json: serde_json::json!({"customer_as_id": 64500, "provider_as_ids": [64501]})
-            .to_string(),
-        rule_hash: sha256_hex(b"expired-rule"),
-        validation_path_hint: vec![vcir.manifest_rsync_uri.clone(), expired_uri.clone()],
+        source_object_type: VcirSourceObjectType::Aspa,
+        source_object_hash: sha256_32(b"expired-src"),
+        source_ee_cert_hash: sha256_32(b"expired-ee"),
+        payload: VcirLocalOutputPayload::Aspa {
+            customer_as_id: 64500,
+            provider_as_ids: vec![64501],
+        },
+        rule_hash: sha256_32(b"expired-rule"),
     });
     vcir.local_outputs.push(VcirLocalOutput {
-        output_id: sha256_hex(b"bad-json"),
         output_type: VcirOutputType::Vrp,
         item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(5)),
         source_object_uri: bad_json_uri.clone(),
-        source_object_type: "roa".to_string(),
-        source_object_hash: sha256_hex(b"bad-json-src"),
-        source_ee_cert_hash: sha256_hex(b"bad-json-ee"),
-        payload_json: "{not-json".to_string(),
-        rule_hash: sha256_hex(b"bad-json-rule"),
-        validation_path_hint: vec![vcir.manifest_rsync_uri.clone(), bad_json_uri.clone()],
+        source_object_type: VcirSourceObjectType::Roa,
+        source_object_hash: sha256_32(b"bad-json-src"),
+        source_ee_cert_hash: sha256_32(b"bad-json-ee"),
+        payload: VcirLocalOutputPayload::Aspa {
+            customer_as_id: 64510,
+            provider_as_ids: vec![64511],
+        },
+        rule_hash: sha256_32(b"bad-json-rule"),
     });
     vcir.local_outputs.push(VcirLocalOutput {
-        output_id: sha256_hex(b"bad-prefix"),
         output_type: VcirOutputType::Vrp,
         item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(5)),
         source_object_uri: bad_prefix_uri.clone(),
-        source_object_type: "roa".to_string(),
-        source_object_hash: sha256_hex(b"bad-prefix-src"),
-        source_ee_cert_hash: sha256_hex(b"bad-prefix-ee"),
-        payload_json: serde_json::json!({"asn": 64510, "prefix": "203.0.113.0", "max_length": 24})
-            .to_string(),
-        rule_hash: sha256_hex(b"bad-prefix-rule"),
-        validation_path_hint: vec![vcir.manifest_rsync_uri.clone(), bad_prefix_uri.clone()],
+        source_object_type: VcirSourceObjectType::Roa,
+        source_object_hash: sha256_32(b"bad-prefix-src"),
+        source_ee_cert_hash: sha256_32(b"bad-prefix-ee"),
+        payload: VcirLocalOutputPayload::Aspa {
+            customer_as_id: 64512,
+            provider_as_ids: vec![64513],
+        },
+        rule_hash: sha256_32(b"bad-prefix-rule"),
     });
     vcir.local_outputs.push(VcirLocalOutput {
-        output_id: sha256_hex(b"bad-aspa"),
         output_type: VcirOutputType::Aspa,
         item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(5)),
         source_object_uri: bad_aspa_uri.clone(),
-        source_object_type: "aspa".to_string(),
-        source_object_hash: sha256_hex(b"bad-aspa-src"),
-        source_ee_cert_hash: sha256_hex(b"bad-aspa-ee"),
-        payload_json: serde_json::json!({"customer_as_id": 64520}).to_string(),
-        rule_hash: sha256_hex(b"bad-aspa-rule"),
-        validation_path_hint: vec![vcir.manifest_rsync_uri.clone(), bad_aspa_uri.clone()],
+        source_object_type: VcirSourceObjectType::Aspa,
+        source_object_hash: sha256_32(b"bad-aspa-src"),
+        source_ee_cert_hash: sha256_32(b"bad-aspa-ee"),
+        payload: VcirLocalOutputPayload::Vrp {
+            asn: 64520,
+            afi: RoaAfi::Ipv4,
+            prefix_len: 24,
+            addr: ipv4_addr([198, 51, 100, 0]),
+            max_length: 24,
+        },
+        rule_hash: sha256_32(b"bad-aspa-rule"),
     });
 
     let mut warnings = Vec::new();

tests/test_objects_process_publication_point_snapshot.rs

@@ -596,7 +596,7 @@ fn process_snapshot_for_issuer_populates_local_outputs_cache_from_real_cernet_fi
     assert!(
         out.local_outputs_cache
             .iter()
-            .all(|entry| entry.source_object_type == "roa")
+            .all(|entry| entry.source_object_type == rpki::storage::VcirSourceObjectType::Roa)
     );
 }