diff --git a/scripts/coverage.sh b/scripts/coverage.sh index fca3131..25f9c00 100755 --- a/scripts/coverage.sh +++ b/scripts/coverage.sh @@ -27,7 +27,7 @@ cleanup() { } trap cleanup EXIT -IGNORE_REGEX='src/bin/repository_view_stats\.rs|src/bin/trace_arin_missing_vrps\.rs|src/bin/db_stats\.rs|src/bin/rrdp_state_dump\.rs|src/bin/ccr_dump\.rs|src/bin/ccr_verify\.rs|src/bin/ccr_to_routinator_csv\.rs|src/bin/ccr_to_compare_views\.rs|src/bin/cir_materialize\.rs|src/bin/cir_extract_inputs\.rs|src/bin/cir_drop_report\.rs|src/bin/cir_ta_only_fixture\.rs|src/bin/cir_dump_reject_list\.rs|src/bin/rpki_object_parse\.rs|src/bin/triage_ccr_cir_pair\.rs|src/bin/rpki_artifact_metrics\.rs|src/bin/rpki_daemon\.rs|src/bin/sequence_triage_ccr_cir\.rs|src/tools/rpki_artifact_metrics\.rs|src/ccr/compare_view\.rs|src/progress_log\.rs|src/cli\.rs|src/validation/run_tree_from_tal\.rs|src/validation/tree_parallel\.rs|src/validation/tree_runner\.rs|src/validation/from_tal\.rs|src/sync/store_projection\.rs|src/sync/repo\.rs|src/sync/rrdp\.rs|src/storage\.rs|src/cir/materialize\.rs' +IGNORE_REGEX='src/bin/repository_view_stats\.rs|src/bin/db_stats\.rs|src/bin/rrdp_state_dump\.rs|src/bin/ccr_dump\.rs|src/bin/ccr_verify\.rs|src/bin/ccr_to_routinator_csv\.rs|src/bin/ccr_to_compare_views\.rs|src/bin/cir_materialize\.rs|src/bin/cir_extract_inputs\.rs|src/bin/cir_drop_report\.rs|src/bin/cir_ta_only_fixture\.rs|src/bin/cir_dump_reject_list\.rs|src/bin/rpki_object_parse\.rs|src/bin/triage_ccr_cir_pair\.rs|src/bin/rpki_artifact_metrics\.rs|src/bin/rpki_daemon\.rs|src/bin/sequence_triage_ccr_cir\.rs|src/tools/rpki_artifact_metrics\.rs|src/ccr/compare_view\.rs|src/progress_log\.rs|src/cli\.rs|src/validation/run_tree_from_tal\.rs|src/validation/tree_parallel\.rs|src/validation/tree_runner\.rs|src/validation/from_tal\.rs|src/sync/store_projection\.rs|src/sync/repo\.rs|src/sync/rrdp\.rs|src/storage\.rs|src/cir/materialize\.rs' # Preserve colored output even though we post-process output by running under a pseudo-TTY. # We run tests only once, then generate both CLI text + HTML reports without rerunning tests. diff --git a/scripts/experiments/feature043/run_sequence_triage_experiment.py b/scripts/experiments/feature043/run_sequence_triage_experiment.py index 7a1db15..d0fdb12 100755 --- a/scripts/experiments/feature043/run_sequence_triage_experiment.py +++ b/scripts/experiments/feature043/run_sequence_triage_experiment.py @@ -75,6 +75,30 @@ def rsync_run_artifacts_from_remote(target: str, source: str | Path, destination run_local([*rsync_base, f"{target}:{source}/{name}", f"{destination}/"]) +def rsync_remote_analysis_from_remote(target: str, remote_exp_root: str | Path, local_exp_root: Path) -> None: + local_exp_root.mkdir(parents=True, exist_ok=True) + rsync_base = ["rsync", "-az", "--ignore-missing-args", "--partial", "--partial-dir=.rsync-partial"] + for name in [ + "left-sequence.jsonl", + "right-sequence.jsonl", + "run-progress.json", + "sequence-triage-time.txt", + "sequence-triage/sequence-triage.json", + ]: + destination = local_exp_root / Path(name).parent + destination.mkdir(parents=True, exist_ok=True) + run_local([*rsync_base, f"{target}:{remote_exp_root}/{name}", f"{destination}/"]) + + +def same_remote_location( + left_target: str, + left_root: str | Path, + right_target: str, + right_root: str | Path, +) -> bool: + return left_target == right_target and str(left_root) == str(right_root) + + def load_json(path: Path) -> Any: with path.open("r", encoding="utf-8") as handle: return json.load(handle) @@ -258,9 +282,31 @@ def prepare_remote(ssh_target: str, remote_root: Path, needs_rpki_client: bool) rsync_to_remote(ssh_target, detect_libtls_path(rpki_client_bin), remote_root / "lib" / "libtls.so.28") +def prepare_remote_once( + prepared: dict[tuple[str, str], bool], + ssh_target: str, + remote_root: Path, + needs_rpki_client: bool, +) -> None: + key = (ssh_target, str(remote_root)) + already_has_rpki_client = prepared.get(key) + if already_has_rpki_client is not None and (already_has_rpki_client or not needs_rpki_client): + return + prepare_remote(ssh_target, remote_root, needs_rpki_client) + prepared[key] = bool(already_has_rpki_client or needs_rpki_client) + + def side_config(name: str) -> dict[str, Any]: if name == "ours-standard": return {"rpKind": "ours", "mode": "standard", "protocol": "rrdp+rsync", "rsyncScope": "module-root"} + if name == "ours-strict-all": + return { + "rpKind": "ours", + "mode": "strict-all", + "protocol": "rrdp+rsync", + "rsyncScope": "module-root", + "strictPolicies": "name,cms-der,signed-attrs", + } if name == "rpki-client-standard": return {"rpKind": "rpki-client", "mode": "standard", "protocol": "rrdp+rsync"} raise SystemExit(f"unknown side config: {name}") @@ -305,6 +351,8 @@ def build_remote_command(remote_root: Path, side_name: str, side: dict[str, Any] "--repo-bytes-db", str(state_dir / "repo-bytes.db"), "--rsync-scope", side.get("rsyncScope", "module-root"), ] + if side.get("strictPolicies"): + argv.extend(["--strict", str(side["strictPolicies"])]) for rir in rirs: argv.extend(["--tal-path", str(remote_root / "fixtures" / "tal" / fixture_name(rir, "tal"))]) argv.extend(["--ta-path", str(remote_root / "fixtures" / "ta" / fixture_name(rir, "ta"))]) @@ -357,7 +405,22 @@ def build_remote_command(remote_root: Path, side_name: str, side: dict[str, Any] "import json, pathlib, sys\n" "run_dir=pathlib.Path(sys.argv[1]); side_name=sys.argv[2]; side_label=sys.argv[3]; seq=int(sys.argv[4]); sync_mode=sys.argv[5]\n" "def read(p):\n return p.read_text().strip() if p.exists() else None\n" - "meta={'sideName':side_name,'sideLabel':side_label,'seq':seq,'syncMode':sync_mode,'startedAt':read(run_dir/'started-at.txt'),'finishedAt':read(run_dir/'finished-at.txt'),'exitCode':int(read(run_dir/'exit-code.txt') or '1')}\n" + "def counts_from_report():\n" + " p=run_dir/'report.json'\n" + " if not p.exists():\n" + " return {}\n" + " try:\n" + " report=json.load(open(p))\n" + " except Exception:\n" + " return {}\n" + " meta=report.get('metadata') if isinstance(report, dict) else None\n" + " if isinstance(meta, dict):\n" + " return {'vrps': int(meta.get('vrps') or 0), 'vaps': int(meta.get('vaps') or meta.get('aspas') or 0), 'publicationPoints': int(meta.get('repositories') or 0), 'warnings': 0}\n" + " pps=report.get('publication_points', []) if isinstance(report, dict) else []\n" + " tree=report.get('tree', {}) if isinstance(report, dict) else {}\n" + " pp_warnings=sum(len(pp.get('warnings', [])) for pp in pps if isinstance(pp, dict)) if isinstance(pps, list) else 0\n" + " return {'vrps': len(report.get('vrps', [])), 'vaps': len(report.get('aspas', [])), 'publicationPoints': len(pps) if isinstance(pps, list) else 0, 'warnings': len(tree.get('warnings', [])) + pp_warnings if isinstance(tree, dict) else pp_warnings}\n" + "meta={'sideName':side_name,'sideLabel':side_label,'seq':seq,'syncMode':sync_mode,'startedAt':read(run_dir/'started-at.txt'),'finishedAt':read(run_dir/'finished-at.txt'),'exitCode':int(read(run_dir/'exit-code.txt') or '1'),'counts':counts_from_report()}\n" "json.dump(meta, open(run_dir/'remote-run-meta.json','w'), indent=2, sort_keys=True); print()\n" "REMOTE_META" ) @@ -370,6 +433,165 @@ def run_remote_sample(ssh_target: str, remote_root: Path, side_name: str, side: return run_dir +def append_remote_sequence_item( + ssh_target: str, + remote_root: Path, + side_name: str, + side_label: str, + seq: int, + run_dir: Path, + schedule_mode: str, +) -> dict[str, Any]: + remote_exp_root = remote_root / "experiments" / "sequence" + seq_path = remote_exp_root / ("left-sequence.jsonl" if side_label == "A" else "right-sequence.jsonl") + side_value = "left" if side_label == "A" else "right" + script = f""" +set -euo pipefail +python3 - <<'REMOTE_SEQUENCE_ITEM' {shlex.quote(str(remote_exp_root))} {shlex.quote(str(seq_path))} {shlex.quote(str(run_dir))} {shlex.quote(str(remote_root / 'bin' / 'cir_dump_reject_list'))} {shlex.quote(side_name)} {shlex.quote(side_label)} {seq} {shlex.quote(side_value)} {shlex.quote(schedule_mode)} +import hashlib, json, os, pathlib, subprocess, sys +exp_root=pathlib.Path(sys.argv[1]) +seq_path=pathlib.Path(sys.argv[2]) +run_dir=pathlib.Path(sys.argv[3]) +cir_dump=pathlib.Path(sys.argv[4]) +side_name=sys.argv[5] +side_label=sys.argv[6] +seq=int(sys.argv[7]) +side_value=sys.argv[8] +schedule_mode=sys.argv[9] +def read(p): + return p.read_text().strip() if p.exists() else None +def sha256_file(p): + h=hashlib.sha256() + with open(p, 'rb') as f: + for chunk in iter(lambda: f.read(1024 * 1024), b''): + h.update(chunk) + return h.hexdigest() +def parse_elapsed_to_ms(raw): + raw=raw.strip() + if not raw: + return 0 + if '-' in raw: + days, raw=raw.split('-', 1) + else: + days='0' + parts=raw.split(':') + if len(parts) == 3: + hours, minutes, seconds=parts + elif len(parts) == 2: + hours='0'; minutes, seconds=parts + else: + hours='0'; minutes='0'; seconds=parts[0] + return int(round((int(days)*86400 + int(hours)*3600 + int(minutes)*60 + float(seconds))*1000)) +def parse_time_file(p): + data={{}} + if not p.exists(): + return data + for line in p.read_text(errors='replace').splitlines(): + if 'Elapsed (wall clock) time' in line: + elapsed=line.rsplit('):', 1)[1] if '):' in line else line.rsplit(':', 1)[1] + data['wallMs']=parse_elapsed_to_ms(elapsed) + elif 'Maximum resident set size' in line: + try: + data['maxRssKb']=int(line.rsplit(':', 1)[1].strip()) + except ValueError: + pass + return data +def cir_counts(cir): + values={{}} + if not cir.exists(): + return {{}} + result=subprocess.run([str(cir_dump), '--cir', str(cir), '--limit', '0'], text=True, capture_output=True, check=True) + for line in result.stdout.splitlines(): + if '=' not in line: + continue + key, value=line.split('=', 1) + if key in ('object_count', 'trust_anchor_count', 'reject_count'): + values[key]=int(value) + return {{'cirObjectCount': values.get('object_count', 0), 'cirTrustAnchorCount': values.get('trust_anchor_count', 0), 'cirRejectCount': values.get('reject_count', 0)}} +meta=json.load(open(run_dir/'remote-run-meta.json')) +time_info=parse_time_file(run_dir/'process-time.txt') +counts=dict(meta.get('counts') or {{}}) +ccr=run_dir/'result.ccr' +cir=run_dir/'result.cir' +if meta.get('exitCode') != 0: + raise SystemExit(f"remote run failed: exitCode={{meta.get('exitCode')}} runDir={{run_dir}}") +missing=[str(path) for path in (ccr, cir) if not path.exists()] +if missing: + raise SystemExit(f"remote run missing required artifact(s): {{missing}}") +counts.update(cir_counts(cir)) +item={{ + 'schemaVersion': 1, + 'rpId': side_name, + 'side': side_value, + 'seq': seq, + 'runId': f'{{side_label}}-{{seq:04d}}', + 'syncMode': 'snapshot' if seq == 1 else 'delta', + 'status': 'success' if meta.get('exitCode') == 0 else 'failed', + 'startTime': meta.get('startedAt'), + 'finishTime': meta.get('finishedAt'), + 'validationTime': None, + 'ccrPath': os.path.relpath(ccr, exp_root), + 'cirPath': os.path.relpath(cir, exp_root), + 'ccrSha256': sha256_file(ccr) if ccr.exists() else None, + 'cirSha256': sha256_file(cir) if cir.exists() else None, + 'wallMs': time_info.get('wallMs'), + 'maxRssKb': time_info.get('maxRssKb'), + 'vrps': counts.get('vrps'), + 'vaps': counts.get('vaps'), + 'publicationPoints': counts.get('publicationPoints'), + 'cirObjectCount': counts.get('cirObjectCount'), + 'cirRejectCount': counts.get('cirRejectCount'), + 'cirTrustAnchorCount': counts.get('cirTrustAnchorCount'), + 'scheduleMode': schedule_mode, +}} +seq_path.parent.mkdir(parents=True, exist_ok=True) +with open(seq_path, 'a', encoding='utf-8') as handle: + handle.write(json.dumps(item, sort_keys=True, ensure_ascii=False) + '\\n') +print(json.dumps(item, sort_keys=True, ensure_ascii=False)) +REMOTE_SEQUENCE_ITEM +""" + result = ssh_script(ssh_target, script, capture=True) + lines = [line for line in result.stdout.splitlines() if line.strip()] + if not lines: + raise SystemExit("remote sequence item append produced no output") + return json.loads(lines[-1]) + + +def cleanup_remote_run_nonessential(ssh_target: str, run_dir: Path) -> None: + keep = { + "result.ccr", + "result.cir", + "process-time.txt", + "remote-run-meta.json", + "exit-code.txt", + "started-at.txt", + "finished-at.txt", + "stdout.log", + "stderr.log", + } + keep_json = json.dumps(sorted(keep), ensure_ascii=False) + script = f""" +set -euo pipefail +python3 - <<'REMOTE_CLEAN' {shlex.quote(str(run_dir))} {shlex.quote(keep_json)} +import json, pathlib, sys +run_dir=pathlib.Path(sys.argv[1]) +keep=set(json.loads(sys.argv[2])) +removed=0 +if run_dir.exists(): + for path in run_dir.iterdir(): + if path.name in keep or path.is_dir(): + continue + try: + removed += path.stat().st_size + path.unlink() + except FileNotFoundError: + pass +print(f'cleaned_nonessential_bytes={{removed}}') +REMOTE_CLEAN +""" + ssh_script(ssh_target, script) + + def cir_counts(cir_path: Path) -> dict[str, int]: result = run_local([str(REPO_ROOT / "target" / "release" / "cir_dump_reject_list"), "--cir", str(cir_path), "--limit", "0"], capture=True) values: dict[str, int] = {} @@ -413,7 +635,9 @@ def build_sequence_item(local_root: Path, side_name: str, side_label: str, side: cir = run_dir / "result.cir" meta = load_json(run_dir / "remote-run-meta.json") time_info = parse_time_file(run_dir / "process-time.txt") - counts = report_counts(run_dir / "report.json", side["rpKind"]) + counts = dict(meta.get("counts") or {}) + if not counts: + counts = report_counts(run_dir / "report.json", side["rpKind"]) counts.update(cir_counts(cir)) return { "schemaVersion": 1, @@ -455,8 +679,58 @@ def run_sequence_triage(local_exp_root: Path, args: argparse.Namespace) -> None: ]) +def run_sequence_triage_remote(ssh_target: str, remote_root: Path, args: argparse.Namespace) -> None: + remote_exp_root = remote_root / "experiments" / "sequence" + compare_dir = remote_exp_root / "sequence-triage" + time_path = remote_exp_root / "sequence-triage-time.txt" + command = " ".join( + shlex.quote(item) + for item in [ + str(remote_root / "bin" / "sequence_triage_ccr_cir"), + "--left-sequence", str(remote_exp_root / "left-sequence.jsonl"), + "--right-sequence", str(remote_exp_root / "right-sequence.jsonl"), + "--out-dir", str(compare_dir), + "--align-window-runs", str(args.align_window_runs), + "--align-window-secs", str(args.align_window_secs), + "--sample-limit", str(args.sample_limit), + "--timeline-sample-limit", str(args.timeline_sample_limit), + ] + ) + ssh_script( + ssh_target, + "set -euo pipefail; " + f"rm -rf {shlex.quote(str(compare_dir))} {shlex.quote(str(time_path))}; " + f"/usr/bin/time -v -o {shlex.quote(str(time_path))} -- {command}", + ) + + +def sync_side_to_analysis_remote( + source_ssh_target: str, + source_remote_root: Path, + analysis_ssh_target: str, + analysis_remote_root: Path, + side_label: str, +) -> None: + source_exp_root = source_remote_root / "experiments" / "sequence" + analysis_exp_root = analysis_remote_root / "experiments" / "sequence" + sequence_name = "left-sequence.jsonl" if side_label == "A" else "right-sequence.jsonl" + if same_remote_location(source_ssh_target, source_exp_root, analysis_ssh_target, analysis_exp_root): + return + side_dir = source_exp_root / side_label + script = ( + "set -euo pipefail; " + f"ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new {shlex.quote(analysis_ssh_target)} " + f"{shlex.quote('mkdir -p ' + shlex.quote(str(analysis_exp_root / side_label)))}; " + f"rsync -az --delete {shlex.quote(str(side_dir))}/ {shlex.quote(analysis_ssh_target)}:{shlex.quote(str(analysis_exp_root / side_label))}/; " + f"rsync -az {shlex.quote(str(source_exp_root / sequence_name))} " + f"{shlex.quote(analysis_ssh_target)}:{shlex.quote(str(analysis_exp_root / sequence_name))}" + ) + ssh_script(source_ssh_target, script) + + def run_side_sequence( args: argparse.Namespace, + ssh_target: str, remote_root: Path, local_exp_root: Path, side_label: str, @@ -468,13 +742,14 @@ def run_side_sequence( side_progress: list[dict[str, Any]] = [] for seq in range(1, args.samples_per_side + 1): side_progress.append( - run_one_side_sample(args, remote_root, local_exp_root, side_label, side_name, side, seq_path, seq, rirs) + run_one_side_sample(args, ssh_target, remote_root, local_exp_root, side_label, side_name, side, seq_path, seq, rirs) ) return side_progress def run_one_side_sample( args: argparse.Namespace, + ssh_target: str, remote_root: Path, local_exp_root: Path, side_label: str, @@ -489,11 +764,24 @@ def run_one_side_sample( f"[run] {side_label} {side_name} seq={seq} rirs={rir_label} schedule={args.schedule_mode}", flush=True, ) - remote_run_dir = run_remote_sample(args.ssh_target, remote_root, side_name, side, side_label, seq, rirs) - local_run_dir = local_exp_root / side_label / f"run_{seq:04d}" - rsync_run_artifacts_from_remote(args.ssh_target, remote_run_dir, local_run_dir) - item = build_sequence_item(local_exp_root, side_name, side_label, side, seq, local_run_dir) - item["scheduleMode"] = args.schedule_mode + remote_run_dir = run_remote_sample(ssh_target, remote_root, side_name, side, side_label, seq, rirs) + if args.remote_triage: + item = append_remote_sequence_item( + ssh_target, + remote_root, + side_name, + side_label, + seq, + remote_run_dir, + args.schedule_mode, + ) + if args.cleanup_run_nonessential: + cleanup_remote_run_nonessential(ssh_target, remote_run_dir) + else: + local_run_dir = local_exp_root / side_label / f"run_{seq:04d}" + rsync_run_artifacts_from_remote(ssh_target, remote_run_dir, local_run_dir) + item = build_sequence_item(local_exp_root, side_name, side_label, side, seq, local_run_dir) + item["scheduleMode"] = args.schedule_mode append_jsonl(seq_path, item) print( f"[done] {side_label} seq={seq} wallMs={item.get('wallMs')} vrps={item.get('vrps')} vaps={item.get('vaps')} objects={item.get('cirObjectCount')} rejects={item.get('cirRejectCount')}", @@ -510,6 +798,12 @@ def run_experiment(args: argparse.Namespace) -> None: right = side_config(args.right) run_root = Path(args.run_root).resolve() remote_root = Path(args.remote_root) + left_ssh_target = args.left_ssh_target or args.ssh_target + right_ssh_target = args.right_ssh_target or args.ssh_target + analysis_ssh_target = args.analysis_ssh_target or left_ssh_target + left_remote_root = Path(args.left_remote_root or args.remote_root) + right_remote_root = Path(args.right_remote_root or args.remote_root) + analysis_remote_root = Path(args.analysis_remote_root or args.remote_root) run_root.mkdir(parents=True, exist_ok=True) write_json(run_root / "experiment-config.json", { "schemaVersion": 1, @@ -520,6 +814,12 @@ def run_experiment(args: argparse.Namespace) -> None: "rirs": rirs, "scheduleMode": args.schedule_mode, "remoteRoot": str(remote_root), + "leftSshTarget": left_ssh_target, + "rightSshTarget": right_ssh_target, + "analysisSshTarget": analysis_ssh_target, + "leftRemoteRoot": str(left_remote_root), + "rightRemoteRoot": str(right_remote_root), + "analysisRemoteRoot": str(analysis_remote_root), "sshTarget": args.ssh_target, }) if args.dry_run: @@ -532,7 +832,10 @@ def run_experiment(args: argparse.Namespace) -> None: "triage": str(run_root / "experiments" / "sequence" / "sequence-triage" / "sequence-triage.json"), }, indent=2)) return - prepare_remote(args.ssh_target, remote_root, needs_rpki_client=(left["rpKind"] == "rpki-client" or right["rpKind"] == "rpki-client")) + prepared_remotes: dict[tuple[str, str], bool] = {} + prepare_remote_once(prepared_remotes, left_ssh_target, left_remote_root, needs_rpki_client=(left["rpKind"] == "rpki-client")) + prepare_remote_once(prepared_remotes, right_ssh_target, right_remote_root, needs_rpki_client=(right["rpKind"] == "rpki-client")) + prepare_remote_once(prepared_remotes, analysis_ssh_target, analysis_remote_root, needs_rpki_client=False) local_exp_root = run_root / "experiments" / "sequence" left_seq_path = local_exp_root / "left-sequence.jsonl" right_seq_path = local_exp_root / "right-sequence.jsonl" @@ -541,27 +844,63 @@ def run_experiment(args: argparse.Namespace) -> None: progress: list[dict[str, Any]] = [] if args.schedule_mode == "interleaved": for seq in range(1, args.samples_per_side + 1): - for side_label, side_name, side, seq_path in [ - ("A", args.left, left, left_seq_path), - ("B", args.right, right, right_seq_path), + for side_label, ssh_target, side_remote_root, side_name, side, seq_path in [ + ("A", left_ssh_target, left_remote_root, args.left, left, left_seq_path), + ("B", right_ssh_target, right_remote_root, args.right, right, right_seq_path), ]: progress.append( - run_one_side_sample(args, remote_root, local_exp_root, side_label, side_name, side, seq_path, seq, rirs) + run_one_side_sample( + args, + ssh_target, + side_remote_root, + local_exp_root, + side_label, + side_name, + side, + seq_path, + seq, + rirs, + ) ) else: with ThreadPoolExecutor(max_workers=2) as executor: futures = [ - executor.submit(run_side_sequence, args, remote_root, local_exp_root, "A", args.left, left, left_seq_path, rirs), - executor.submit(run_side_sequence, args, remote_root, local_exp_root, "B", args.right, right, right_seq_path, rirs), + executor.submit(run_side_sequence, args, left_ssh_target, left_remote_root, local_exp_root, "A", args.left, left, left_seq_path, rirs), + executor.submit(run_side_sequence, args, right_ssh_target, right_remote_root, local_exp_root, "B", args.right, right, right_seq_path, rirs), ] for future in as_completed(futures): progress.extend(future.result()) progress.sort(key=lambda item: (str(item.get("side")), int(item.get("seq") or 0))) write_json(local_exp_root / "run-progress.json", progress) - run_sequence_triage(local_exp_root, args) - ssh_script(args.ssh_target, f"df -h /data / > {shlex.quote(str(remote_root / 'df-after.txt'))} 2>&1 || true; free -h > {shlex.quote(str(remote_root / 'free-after.txt'))} 2>&1 || true") + if args.remote_triage: + sync_side_to_analysis_remote(left_ssh_target, left_remote_root, analysis_ssh_target, analysis_remote_root, "A") + sync_side_to_analysis_remote(right_ssh_target, right_remote_root, analysis_ssh_target, analysis_remote_root, "B") + remote_exp_root = analysis_remote_root / "experiments" / "sequence" + remote_progress = json.dumps(progress, sort_keys=True, ensure_ascii=False) + ssh_script( + analysis_ssh_target, + "set -euo pipefail; " + f"cat > {shlex.quote(str(remote_exp_root / 'run-progress.json'))} <<'REMOTE_PROGRESS_JSON'\n" + f"{remote_progress}\n" + "REMOTE_PROGRESS_JSON\n", + ) + run_sequence_triage_remote(analysis_ssh_target, analysis_remote_root, args) + if args.fetch_remote_analysis: + rsync_remote_analysis_from_remote(analysis_ssh_target, remote_exp_root, local_exp_root) + else: + run_sequence_triage(local_exp_root, args) + ssh_script(analysis_ssh_target, f"df -h /data / > {shlex.quote(str(analysis_remote_root / 'df-after.txt'))} 2>&1 || true; free -h > {shlex.quote(str(analysis_remote_root / 'free-after.txt'))} 2>&1 || true") compare_dir = local_exp_root / "sequence-triage" - print(json.dumps({"runRoot": str(run_root), "remoteRoot": str(remote_root), "triage": str(compare_dir / "sequence-triage.json")}, indent=2)) + remote_compare_dir = analysis_remote_root / "experiments" / "sequence" / "sequence-triage" + print(json.dumps({ + "runRoot": str(run_root), + "remoteRoot": str(remote_root), + "leftRemoteRoot": str(left_remote_root), + "rightRemoteRoot": str(right_remote_root), + "analysisRemoteRoot": str(analysis_remote_root), + "triage": str(compare_dir / "sequence-triage.json") if not args.remote_triage or args.fetch_remote_analysis else None, + "remoteTriage": str(remote_compare_dir / "sequence-triage.json") if args.remote_triage else None, + }, indent=2)) def main() -> None: @@ -569,6 +908,12 @@ def main() -> None: parser.add_argument("--run-root", required=True) parser.add_argument("--remote-root", required=True) parser.add_argument("--ssh-target", default=os.environ.get("SSH_TARGET", "root@47.251.56.108")) + parser.add_argument("--left-ssh-target") + parser.add_argument("--right-ssh-target") + parser.add_argument("--analysis-ssh-target") + parser.add_argument("--left-remote-root") + parser.add_argument("--right-remote-root") + parser.add_argument("--analysis-remote-root") parser.add_argument("--left", default="ours-standard") parser.add_argument("--right", default="rpki-client-standard") parser.add_argument("--samples-per-side", type=int, default=3) @@ -581,6 +926,9 @@ def main() -> None: parser.add_argument("--dry-run", action="store_true") parser.add_argument("--skip-build", action="store_true", help="reuse existing release binaries") parser.add_argument("--triage-only", action="store_true", help="only rerun local sequence triage for an existing run root") + parser.add_argument("--remote-triage", action="store_true", help="keep CIR/CCR on remote, write sequence JSONL remotely, and run triage on remote") + parser.add_argument("--fetch-remote-analysis", action="store_true", help="when --remote-triage is set, fetch only small sequence/triage JSON outputs; never fetch CIR/CCR") + parser.add_argument("--cleanup-run-nonessential", action="store_true", help="after each successful remote sequence item, remove report/log/CSV files and keep only CIR/CCR/timing/meta") args = parser.parse_args() if args.samples_per_side < 2: raise SystemExit("--samples-per-side must be >= 2") diff --git a/scripts/manual_sync/delta_sync.sh b/scripts/manual_sync/delta_sync.sh index 75b2ef6..a7e342e 100755 --- a/scripts/manual_sync/delta_sync.sh +++ b/scripts/manual_sync/delta_sync.sh @@ -389,7 +389,6 @@ def fmt_db_stats(db: dict) -> str: "repository_view", "raw_by_hash", "vcir", - "audit_rule_index", "rrdp_source", "rrdp_source_member", "rrdp_uri_owner", diff --git a/src/audit_trace.rs b/src/audit_trace.rs deleted file mode 100644 index d0e5540..0000000 --- a/src/audit_trace.rs +++ /dev/null @@ -1,893 +0,0 @@ -use crate::data_model::aspa::AspaObject; -use crate::data_model::manifest::ManifestObject; -use crate::data_model::roa::RoaObject; -use crate::storage::{ - AuditRuleIndexEntry, AuditRuleKind, RawByHashEntry, RocksStore, ValidatedCaInstanceResult, - VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus, VcirLocalOutput, - VcirOutputType, -}; -use serde::Serialize; -use std::collections::HashSet; - -#[derive(Debug, thiserror::Error)] -pub enum AuditTraceError { - #[error("storage error: {0}")] - Storage(#[from] crate::storage::StorageError), - - #[error("audit rule index points to missing VCIR: {manifest_rsync_uri}")] - MissingVcir { manifest_rsync_uri: String }, - - #[error( - "audit rule index points to missing local output: rule_hash={rule_hash}, output_id={output_id}, manifest={manifest_rsync_uri}" - )] - MissingLocalOutput { - rule_hash: String, - output_id: String, - manifest_rsync_uri: String, - }, - - #[error("detected VCIR parent cycle at {manifest_rsync_uri}")] - ParentCycle { manifest_rsync_uri: String }, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize)] -pub struct AuditTraceRawRef { - pub sha256_hex: String, - pub raw_present: bool, - #[serde(skip_serializing_if = "Vec::is_empty", default)] - pub origin_uris: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub object_type: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub byte_len: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize)] -pub struct AuditTraceArtifact { - pub artifact_role: VcirArtifactRole, - pub artifact_kind: VcirArtifactKind, - #[serde(skip_serializing_if = "Option::is_none")] - pub uri: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub object_type: Option, - pub validation_status: VcirArtifactValidationStatus, - pub raw: AuditTraceRawRef, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize)] -pub struct AuditTraceChainNode { - pub manifest_rsync_uri: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub parent_manifest_rsync_uri: Option, - pub tal_id: String, - pub ca_subject_name: String, - pub ca_ski: String, - pub issuer_ski: String, - pub current_manifest_rsync_uri: String, - pub current_crl_rsync_uri: String, - pub last_successful_validation_time_rfc3339_utc: String, - pub local_output_count: usize, - pub child_count: usize, - pub related_artifacts: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize)] -pub struct AuditTraceResolvedOutput { - pub output_id: String, - pub output_type: VcirOutputType, - pub rule_hash: String, - pub source_object_uri: String, - pub source_object_type: String, - pub source_object_hash: String, - pub source_ee_cert_hash: String, - pub item_effective_until_rfc3339_utc: String, - pub payload_json: String, - pub validation_path_hint: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize)] -pub struct AuditRuleTrace { - pub rule: AuditRuleIndexEntry, - pub resolved_output: AuditTraceResolvedOutput, - pub source_object_raw: AuditTraceRawRef, - pub source_ee_cert_raw: AuditTraceRawRef, - pub chain_leaf_to_root: Vec, -} - -pub fn trace_rule_to_root( - store: &RocksStore, - kind: AuditRuleKind, - rule_hash: &str, -) -> Result, AuditTraceError> { - let Some(rule) = store.get_audit_rule_index_entry(kind, rule_hash)? else { - return Ok(None); - }; - - let Some(leaf_vcir) = store.get_vcir(&rule.manifest_rsync_uri)? else { - return Err(AuditTraceError::MissingVcir { - manifest_rsync_uri: rule.manifest_rsync_uri.clone(), - }); - }; - - let Some(local_output) = leaf_vcir - .local_outputs - .iter() - .find(|output| output.output_id == rule.output_id && output.rule_hash == rule.rule_hash) - .or_else(|| { - leaf_vcir - .local_outputs - .iter() - .find(|output| output.rule_hash == rule.rule_hash) - }) - .cloned() - else { - return Err(AuditTraceError::MissingLocalOutput { - rule_hash: rule.rule_hash.clone(), - output_id: rule.output_id.clone(), - manifest_rsync_uri: rule.manifest_rsync_uri.clone(), - }); - }; - - let chain = trace_vcir_chain_to_root(store, &leaf_vcir.manifest_rsync_uri)? - .expect("leaf VCIR already loaded must exist"); - - Ok(Some(AuditRuleTrace { - rule, - resolved_output: resolved_output_from_local(&local_output), - source_object_raw: resolve_raw_ref( - store, - &local_output.source_object_hash, - Some(&local_output.source_object_uri), - Some(local_output.source_object_type.as_str()), - )?, - source_ee_cert_raw: resolve_source_ee_cert_raw_ref(store, &local_output)?, - chain_leaf_to_root: chain, - })) -} - -pub fn trace_vcir_chain_to_root( - store: &RocksStore, - manifest_rsync_uri: &str, -) -> Result>, AuditTraceError> { - let Some(mut current) = store.get_vcir(manifest_rsync_uri)? else { - return Ok(None); - }; - - let mut seen = HashSet::new(); - let mut chain = Vec::new(); - loop { - if !seen.insert(current.manifest_rsync_uri.clone()) { - return Err(AuditTraceError::ParentCycle { - manifest_rsync_uri: current.manifest_rsync_uri, - }); - } - let parent = current.parent_manifest_rsync_uri.clone(); - chain.push(trace_chain_node(store, ¤t)?); - let Some(parent_manifest_rsync_uri) = parent else { - break; - }; - let Some(parent_vcir) = store.get_vcir(&parent_manifest_rsync_uri)? else { - return Err(AuditTraceError::MissingVcir { - manifest_rsync_uri: parent_manifest_rsync_uri, - }); - }; - current = parent_vcir; - } - - Ok(Some(chain)) -} - -fn trace_chain_node( - store: &RocksStore, - vcir: &ValidatedCaInstanceResult, -) -> Result { - let mut related_artifacts = Vec::with_capacity(vcir.related_artifacts.len()); - for artifact in &vcir.related_artifacts { - related_artifacts.push(AuditTraceArtifact { - artifact_role: artifact.artifact_role, - artifact_kind: artifact.artifact_kind, - uri: artifact.uri.clone(), - object_type: artifact.object_type.clone(), - validation_status: artifact.validation_status, - raw: resolve_raw_ref( - store, - &artifact.sha256, - artifact.uri.as_deref(), - artifact.object_type.as_deref(), - )?, - }); - } - - Ok(AuditTraceChainNode { - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - parent_manifest_rsync_uri: vcir.parent_manifest_rsync_uri.clone(), - tal_id: vcir.tal_id.clone(), - ca_subject_name: vcir.ca_subject_name.clone(), - ca_ski: vcir.ca_ski.clone(), - issuer_ski: vcir.issuer_ski.clone(), - current_manifest_rsync_uri: vcir.current_manifest_rsync_uri.clone(), - current_crl_rsync_uri: vcir.current_crl_rsync_uri.clone(), - last_successful_validation_time_rfc3339_utc: vcir - .last_successful_validation_time - .rfc3339_utc - .clone(), - local_output_count: vcir.local_outputs.len(), - child_count: vcir.child_entries.len(), - related_artifacts, - }) -} - -fn resolved_output_from_local(local: &VcirLocalOutput) -> AuditTraceResolvedOutput { - AuditTraceResolvedOutput { - output_id: local.output_id.clone(), - output_type: local.output_type, - rule_hash: local.rule_hash.clone(), - source_object_uri: local.source_object_uri.clone(), - source_object_type: local.source_object_type.clone(), - source_object_hash: local.source_object_hash.clone(), - source_ee_cert_hash: local.source_ee_cert_hash.clone(), - item_effective_until_rfc3339_utc: local.item_effective_until.rfc3339_utc.clone(), - payload_json: local.payload_json.clone(), - validation_path_hint: local.validation_path_hint.clone(), - } -} - -fn resolve_raw_ref( - store: &RocksStore, - sha256_hex: &str, - fallback_uri: Option<&str>, - fallback_object_type: Option<&str>, -) -> Result { - let raw = store.get_raw_by_hash_entry(sha256_hex)?; - if raw.is_some() { - return Ok(raw_ref_from_entry(sha256_hex, raw.as_ref())); - } - - let blob = store.get_blob_bytes(sha256_hex)?; - match blob { - Some(bytes) => Ok(AuditTraceRawRef { - sha256_hex: sha256_hex.to_string(), - raw_present: true, - origin_uris: fallback_uri - .map(|uri| vec![uri.to_string()]) - .unwrap_or_default(), - object_type: fallback_object_type.map(str::to_string), - byte_len: Some(bytes.len()), - }), - None => Ok(raw_ref_from_entry(sha256_hex, None)), - } -} - -fn resolve_source_ee_cert_raw_ref( - store: &RocksStore, - local: &VcirLocalOutput, -) -> Result { - let raw = store.get_raw_by_hash_entry(&local.source_ee_cert_hash)?; - if raw.is_some() { - return Ok(raw_ref_from_entry(&local.source_ee_cert_hash, raw.as_ref())); - } - - let source_bytes = store.get_blob_bytes(&local.source_object_hash)?; - let Some(source_bytes) = source_bytes else { - return Ok(raw_ref_from_entry(&local.source_ee_cert_hash, None)); - }; - - let derived = match local.source_object_type.as_str() { - "roa" => RoaObject::decode_der(&source_bytes).ok().and_then(|roa| { - roa.signed_object - .signed_data - .certificates - .first() - .map(|cert| cert.raw_der.to_vec()) - }), - "aspa" => AspaObject::decode_der(&source_bytes).ok().and_then(|aspa| { - aspa.signed_object - .signed_data - .certificates - .first() - .map(|cert| cert.raw_der.to_vec()) - }), - "mft" => ManifestObject::decode_der(&source_bytes) - .ok() - .and_then(|manifest| { - manifest - .signed_object - .signed_data - .certificates - .first() - .map(|cert| cert.raw_der.to_vec()) - }), - "router_key" => Some(source_bytes), - _ => None, - }; - - let Some(ee_der) = derived else { - return Ok(raw_ref_from_entry(&local.source_ee_cert_hash, None)); - }; - if crate::audit::sha256_hex(ee_der.as_slice()) != local.source_ee_cert_hash { - return Ok(raw_ref_from_entry(&local.source_ee_cert_hash, None)); - } - - Ok(AuditTraceRawRef { - sha256_hex: local.source_ee_cert_hash.clone(), - raw_present: true, - origin_uris: Vec::new(), - object_type: Some("cer".to_string()), - byte_len: Some(ee_der.len()), - }) -} - -fn raw_ref_from_entry(sha256_hex: &str, entry: Option<&RawByHashEntry>) -> AuditTraceRawRef { - match entry { - Some(entry) => AuditTraceRawRef { - sha256_hex: sha256_hex.to_string(), - raw_present: true, - origin_uris: entry.origin_uris.clone(), - object_type: entry.object_type.clone(), - byte_len: Some(entry.bytes.len()), - }, - None => AuditTraceRawRef { - sha256_hex: sha256_hex.to_string(), - raw_present: false, - origin_uris: Vec::new(), - object_type: None, - byte_len: None, - }, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::audit::sha256_hex; - use crate::data_model::roa::RoaObject; - use crate::storage::{ - PackTime, ValidatedManifestMeta, VcirAuditSummary, VcirCcrManifestProjection, - VcirChildEntry, VcirInstanceGate, VcirRelatedArtifact, VcirSummary, - }; - use base64::Engine as _; - - fn sample_vcir( - manifest_rsync_uri: &str, - parent_manifest_rsync_uri: Option<&str>, - tal_id: &str, - local_output: Option, - related_artifacts: Vec, - ) -> ValidatedCaInstanceResult { - let now = time::OffsetDateTime::now_utc(); - let next = PackTime::from_utc_offset_datetime(now + time::Duration::hours(1)); - let local_outputs: Vec = local_output.into_iter().collect(); - let ccr_manifest_projection = VcirCcrManifestProjection { - manifest_rsync_uri: manifest_rsync_uri.to_string(), - manifest_sha256: vec![0x44; 32], - manifest_size: 2048, - manifest_ee_aki: vec![0x55; 20], - manifest_number_be: vec![1], - manifest_this_update: PackTime::from_utc_offset_datetime(now), - manifest_sia_locations_der: vec![vec![ - 0x30, 0x11, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x05, 0x86, 0x05, - b'r', b's', b'y', b'n', b'c', - ]], - subordinate_skis: vec![vec![0x33; 20]], - }; - ValidatedCaInstanceResult { - manifest_rsync_uri: manifest_rsync_uri.to_string(), - parent_manifest_rsync_uri: parent_manifest_rsync_uri.map(str::to_string), - tal_id: tal_id.to_string(), - ca_subject_name: format!("CN={manifest_rsync_uri}"), - ca_ski: "11".repeat(20), - issuer_ski: "22".repeat(20), - last_successful_validation_time: PackTime::from_utc_offset_datetime(now), - current_manifest_rsync_uri: manifest_rsync_uri.to_string(), - current_crl_rsync_uri: manifest_rsync_uri.replace(".mft", ".crl"), - validated_manifest_meta: ValidatedManifestMeta { - validated_manifest_number: vec![1], - validated_manifest_this_update: PackTime::from_utc_offset_datetime(now), - validated_manifest_next_update: next.clone(), - }, - ccr_manifest_projection, - instance_gate: VcirInstanceGate { - manifest_next_update: next.clone(), - current_crl_next_update: next.clone(), - self_ca_not_after: PackTime::from_utc_offset_datetime( - now + time::Duration::hours(2), - ), - instance_effective_until: next, - }, - child_entries: vec![VcirChildEntry { - child_manifest_rsync_uri: "rsync://example.test/child/child.mft".to_string(), - child_cert_rsync_uri: "rsync://example.test/parent/child.cer".to_string(), - child_cert_hash: sha256_hex(b"child-cert"), - child_ski: "33".repeat(20), - child_rsync_base_uri: "rsync://example.test/child/".to_string(), - child_publication_point_rsync_uri: "rsync://example.test/child/".to_string(), - child_rrdp_notification_uri: Some( - "https://example.test/child/notify.xml".to_string(), - ), - child_effective_ip_resources: None, - child_effective_as_resources: None, - accepted_at_validation_time: PackTime::from_utc_offset_datetime(now), - }], - summary: VcirSummary { - local_vrp_count: local_outputs - .iter() - .filter(|output| output.output_type == VcirOutputType::Vrp) - .count() as u32, - local_aspa_count: local_outputs - .iter() - .filter(|output| output.output_type == VcirOutputType::Aspa) - .count() as u32, - local_router_key_count: local_outputs - .iter() - .filter(|output| output.output_type == VcirOutputType::RouterKey) - .count() as u32, - child_count: 1, - accepted_object_count: related_artifacts.len() as u32, - rejected_object_count: 0, - }, - local_outputs, - related_artifacts, - audit_summary: VcirAuditSummary { - failed_fetch_eligible: true, - last_failed_fetch_reason: None, - warning_count: 0, - audit_flags: Vec::new(), - }, - } - } - - fn sample_local_output(manifest_rsync_uri: &str) -> VcirLocalOutput { - let now = time::OffsetDateTime::now_utc(); - VcirLocalOutput { - output_id: sha256_hex(b"vrp-output"), - output_type: VcirOutputType::Vrp, - item_effective_until: PackTime::from_utc_offset_datetime( - now + time::Duration::minutes(30), - ), - source_object_uri: "rsync://example.test/leaf/a.roa".to_string(), - source_object_type: "roa".to_string(), - source_object_hash: sha256_hex(b"roa-raw"), - source_ee_cert_hash: sha256_hex(b"roa-ee"), - payload_json: - serde_json::json!({"asn": 64496, "prefix": "203.0.113.0/24", "max_length": 24}) - .to_string(), - rule_hash: sha256_hex(b"roa-rule"), - validation_path_hint: vec![ - manifest_rsync_uri.to_string(), - "rsync://example.test/leaf/a.roa".to_string(), - sha256_hex(b"roa-raw"), - ], - } - } - - fn sample_artifacts(manifest_rsync_uri: &str, roa_hash: &str) -> Vec { - vec![ - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::Manifest, - artifact_kind: VcirArtifactKind::Mft, - uri: Some(manifest_rsync_uri.to_string()), - sha256: sha256_hex(manifest_rsync_uri.as_bytes()), - object_type: Some("mft".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - }, - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::CurrentCrl, - artifact_kind: VcirArtifactKind::Crl, - uri: Some(manifest_rsync_uri.replace(".mft", ".crl")), - sha256: sha256_hex(format!("{}-crl", manifest_rsync_uri).as_bytes()), - object_type: Some("crl".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - }, - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::SignedObject, - artifact_kind: VcirArtifactKind::Roa, - uri: Some("rsync://example.test/leaf/a.roa".to_string()), - sha256: roa_hash.to_string(), - object_type: Some("roa".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - }, - ] - } - - fn put_raw_evidence(store: &RocksStore, bytes: &[u8], uri: &str, object_type: &str) { - let mut entry = RawByHashEntry::from_bytes(sha256_hex(bytes), bytes.to_vec()); - entry.origin_uris.push(uri.to_string()); - entry.object_type = Some(object_type.to_string()); - entry.encoding = Some("der".to_string()); - store - .put_raw_by_hash_entry(&entry) - .expect("put raw evidence"); - } - - fn put_blob_only(store: &RocksStore, bytes: &[u8]) { - store - .put_blob_bytes_batch(&[(sha256_hex(bytes), bytes.to_vec())]) - .expect("put blob bytes"); - } - - #[test] - fn trace_rule_to_root_returns_leaf_to_root_chain_and_evidence_refs() { - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - - let root_manifest = "rsync://example.test/root/root.mft"; - let leaf_manifest = "rsync://example.test/leaf/leaf.mft"; - let local = sample_local_output(leaf_manifest); - let leaf_vcir = sample_vcir( - leaf_manifest, - Some(root_manifest), - "test-tal", - Some(local.clone()), - sample_artifacts(leaf_manifest, &local.source_object_hash), - ); - let root_vcir = sample_vcir( - root_manifest, - None, - "test-tal", - None, - sample_artifacts(root_manifest, &sha256_hex(b"root-object")), - ); - store.put_vcir(&leaf_vcir).expect("put leaf vcir"); - store.put_vcir(&root_vcir).expect("put root vcir"); - - let rule_entry = AuditRuleIndexEntry { - kind: AuditRuleKind::Roa, - rule_hash: local.rule_hash.clone(), - manifest_rsync_uri: leaf_manifest.to_string(), - source_object_uri: local.source_object_uri.clone(), - source_object_hash: local.source_object_hash.clone(), - output_id: local.output_id.clone(), - item_effective_until: local.item_effective_until.clone(), - }; - store - .put_audit_rule_index_entry(&rule_entry) - .expect("put rule index"); - - put_raw_evidence(&store, leaf_manifest.as_bytes(), leaf_manifest, "mft"); - put_raw_evidence( - &store, - format!("{}-crl", leaf_manifest).as_bytes(), - &leaf_manifest.replace(".mft", ".crl"), - "crl", - ); - put_raw_evidence(&store, b"roa-raw", &local.source_object_uri, "roa"); - put_raw_evidence(&store, b"roa-ee", "rsync://example.test/leaf/a.ee", "cer"); - put_raw_evidence(&store, root_manifest.as_bytes(), root_manifest, "mft"); - put_raw_evidence( - &store, - format!("{}-crl", root_manifest).as_bytes(), - &root_manifest.replace(".mft", ".crl"), - "crl", - ); - - let trace = trace_rule_to_root(&store, AuditRuleKind::Roa, &local.rule_hash) - .expect("trace rule") - .expect("trace exists"); - - assert_eq!(trace.rule, rule_entry); - assert_eq!(trace.resolved_output.output_id, local.output_id); - assert_eq!(trace.chain_leaf_to_root.len(), 2); - assert_eq!( - trace.chain_leaf_to_root[0].manifest_rsync_uri, - leaf_manifest - ); - assert_eq!( - trace.chain_leaf_to_root[1].manifest_rsync_uri, - root_manifest - ); - assert_eq!( - trace.chain_leaf_to_root[0] - .parent_manifest_rsync_uri - .as_deref(), - Some(root_manifest) - ); - assert!(trace.source_object_raw.raw_present); - assert!(trace.source_ee_cert_raw.raw_present); - assert!( - trace.chain_leaf_to_root[0] - .related_artifacts - .iter() - .any(|artifact| { - artifact.uri.as_deref() == Some(leaf_manifest) && artifact.raw.raw_present - }) - ); - } - - #[test] - fn trace_rule_to_root_supports_router_key_rules() { - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let manifest = "rsync://example.test/router/leaf.mft"; - let mut local = sample_local_output(manifest); - local.output_type = VcirOutputType::RouterKey; - local.source_object_uri = "rsync://example.test/router/router.cer".to_string(); - local.source_object_type = "router_key".to_string(); - local.payload_json = serde_json::json!({ - "as_id": 64496, - "ski_hex": "11".repeat(20), - "spki_der_base64": base64::engine::general_purpose::STANDARD.encode([0x30u8, 0x00]), - }) - .to_string(); - let mut vcir = sample_vcir( - manifest, - None, - "test-tal", - Some(local), - sample_artifacts(manifest, &sha256_hex(b"router-object")), - ); - vcir.local_outputs[0].output_type = VcirOutputType::RouterKey; - vcir.local_outputs[0].source_object_uri = - "rsync://example.test/router/router.cer".to_string(); - vcir.local_outputs[0].source_object_type = "router_key".to_string(); - vcir.local_outputs[0].payload_json = serde_json::json!({ - "as_id": 64496, - "ski_hex": "11".repeat(20), - "spki_der_base64": base64::engine::general_purpose::STANDARD.encode([0x30u8, 0x00]), - }) - .to_string(); - vcir.summary.local_vrp_count = 0; - vcir.summary.local_router_key_count = 1; - store.put_vcir(&vcir).expect("put vcir"); - let rule_entry = AuditRuleIndexEntry { - kind: AuditRuleKind::RouterKey, - rule_hash: vcir.local_outputs[0].rule_hash.clone(), - manifest_rsync_uri: manifest.to_string(), - source_object_uri: vcir.local_outputs[0].source_object_uri.clone(), - source_object_hash: vcir.local_outputs[0].source_object_hash.clone(), - output_id: vcir.local_outputs[0].output_id.clone(), - item_effective_until: vcir.local_outputs[0].item_effective_until.clone(), - }; - store - .put_audit_rule_index_entry(&rule_entry) - .expect("put rule"); - let trace = trace_rule_to_root(&store, AuditRuleKind::RouterKey, &rule_entry.rule_hash) - .expect("trace rule") - .expect("trace exists"); - assert_eq!(trace.rule.kind, AuditRuleKind::RouterKey); - assert_eq!(trace.resolved_output.output_type, VcirOutputType::RouterKey); - } - - #[test] - fn trace_rule_to_root_lazily_derives_source_ee_cert_when_raw_is_missing() { - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let manifest = "rsync://example.test/leaf/leaf.mft"; - let roa_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142071.roa"); - let roa_bytes = std::fs::read(&roa_path).expect("read ROA fixture"); - let roa = RoaObject::decode_der(&roa_bytes).expect("decode ROA fixture"); - let local = VcirLocalOutput { - output_id: sha256_hex(b"lazy-vrp-output"), - output_type: VcirOutputType::Vrp, - item_effective_until: PackTime::from_utc_offset_datetime( - time::OffsetDateTime::now_utc() + time::Duration::minutes(30), - ), - source_object_uri: "rsync://example.test/leaf/a.roa".to_string(), - source_object_type: "roa".to_string(), - source_object_hash: sha256_hex(&roa_bytes), - source_ee_cert_hash: sha256_hex( - roa.signed_object.signed_data.certificates[0] - .raw_der - .as_slice(), - ), - payload_json: - serde_json::json!({"asn": 64496, "prefix": "203.0.113.0/24", "max_length": 24}) - .to_string(), - rule_hash: sha256_hex(b"lazy-roa-rule"), - validation_path_hint: vec![manifest.to_string()], - }; - let vcir = sample_vcir( - manifest, - None, - "test-tal", - Some(local.clone()), - sample_artifacts(manifest, &local.source_object_hash), - ); - store.put_vcir(&vcir).expect("put vcir"); - let rule_entry = AuditRuleIndexEntry { - kind: AuditRuleKind::Roa, - rule_hash: local.rule_hash.clone(), - manifest_rsync_uri: manifest.to_string(), - source_object_uri: local.source_object_uri.clone(), - source_object_hash: local.source_object_hash.clone(), - output_id: local.output_id.clone(), - item_effective_until: local.item_effective_until.clone(), - }; - store - .put_audit_rule_index_entry(&rule_entry) - .expect("put rule index"); - - put_raw_evidence(&store, manifest.as_bytes(), manifest, "mft"); - put_raw_evidence( - &store, - format!("{}-crl", manifest).as_bytes(), - &manifest.replace(".mft", ".crl"), - "crl", - ); - put_raw_evidence(&store, &roa_bytes, &local.source_object_uri, "roa"); - - let trace = trace_rule_to_root(&store, AuditRuleKind::Roa, &local.rule_hash) - .expect("trace rule") - .expect("trace exists"); - assert!(trace.source_object_raw.raw_present); - assert!(trace.source_ee_cert_raw.raw_present); - assert_eq!(trace.source_ee_cert_raw.object_type.as_deref(), Some("cer")); - } - - #[test] - fn trace_rule_to_root_uses_blob_only_fallback_for_source_object_raw() { - let store_dir = tempfile::tempdir().expect("store dir"); - let main_db = store_dir.path().join("main-db"); - let raw_db = store_dir.path().join("raw-store.db"); - let store = - RocksStore::open_with_external_raw_store(&main_db, &raw_db).expect("open rocksdb"); - let manifest = "rsync://example.test/leaf/leaf.mft"; - let roa_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142071.roa"); - let roa_bytes = std::fs::read(&roa_path).expect("read ROA fixture"); - let roa = RoaObject::decode_der(&roa_bytes).expect("decode ROA fixture"); - let local = VcirLocalOutput { - output_id: sha256_hex(b"blob-only-vrp-output"), - output_type: VcirOutputType::Vrp, - item_effective_until: PackTime::from_utc_offset_datetime( - time::OffsetDateTime::now_utc() + time::Duration::minutes(30), - ), - source_object_uri: "rsync://example.test/leaf/blob-only.roa".to_string(), - source_object_type: "roa".to_string(), - source_object_hash: sha256_hex(&roa_bytes), - source_ee_cert_hash: sha256_hex( - roa.signed_object.signed_data.certificates[0] - .raw_der - .as_slice(), - ), - payload_json: - serde_json::json!({"asn": 64496, "prefix": "203.0.113.0/24", "max_length": 24}) - .to_string(), - rule_hash: sha256_hex(b"blob-only-roa-rule"), - validation_path_hint: vec![manifest.to_string()], - }; - let vcir = sample_vcir( - manifest, - None, - "test-tal", - Some(local.clone()), - sample_artifacts(manifest, &local.source_object_hash), - ); - store.put_vcir(&vcir).expect("put vcir"); - let rule_entry = AuditRuleIndexEntry { - kind: AuditRuleKind::Roa, - rule_hash: local.rule_hash.clone(), - manifest_rsync_uri: manifest.to_string(), - source_object_uri: local.source_object_uri.clone(), - source_object_hash: local.source_object_hash.clone(), - output_id: local.output_id.clone(), - item_effective_until: local.item_effective_until.clone(), - }; - store - .put_audit_rule_index_entry(&rule_entry) - .expect("put rule index"); - - put_raw_evidence(&store, manifest.as_bytes(), manifest, "mft"); - put_raw_evidence( - &store, - format!("{}-crl", manifest).as_bytes(), - &manifest.replace(".mft", ".crl"), - "crl", - ); - put_blob_only(&store, &roa_bytes); - - let trace = trace_rule_to_root(&store, AuditRuleKind::Roa, &local.rule_hash) - .expect("trace rule") - .expect("trace exists"); - assert!(trace.source_object_raw.raw_present); - assert_eq!( - trace.source_object_raw.origin_uris, - vec![local.source_object_uri.clone()] - ); - assert_eq!(trace.source_object_raw.object_type.as_deref(), Some("roa")); - assert_eq!(trace.source_object_raw.byte_len, Some(roa_bytes.len())); - assert!(trace.source_ee_cert_raw.raw_present); - } - - #[test] - fn trace_rule_to_root_returns_none_for_missing_rule_index() { - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - - assert!( - trace_rule_to_root(&store, AuditRuleKind::Roa, &sha256_hex(b"missing")) - .expect("missing trace ok") - .is_none() - ); - } - - #[test] - fn trace_rule_to_root_errors_when_index_points_to_missing_vcir() { - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let rule_hash = sha256_hex(b"missing-vcir-rule"); - store - .put_audit_rule_index_entry(&AuditRuleIndexEntry { - kind: AuditRuleKind::Roa, - rule_hash: rule_hash.clone(), - manifest_rsync_uri: "rsync://example.test/missing.mft".to_string(), - source_object_uri: "rsync://example.test/missing.roa".to_string(), - source_object_hash: sha256_hex(b"missing-source"), - output_id: sha256_hex(b"missing-output"), - item_effective_until: PackTime::from_utc_offset_datetime( - time::OffsetDateTime::now_utc() + time::Duration::minutes(1), - ), - }) - .expect("put rule index"); - - let err = trace_rule_to_root(&store, AuditRuleKind::Roa, &rule_hash).unwrap_err(); - assert!(matches!( - err, - AuditTraceError::MissingVcir { manifest_rsync_uri } - if manifest_rsync_uri == "rsync://example.test/missing.mft" - )); - } - - #[test] - fn trace_rule_to_root_errors_when_vcir_local_output_is_missing() { - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let manifest = "rsync://example.test/leaf/leaf.mft"; - let vcir = sample_vcir( - manifest, - None, - "test-tal", - None, - sample_artifacts(manifest, &sha256_hex(b"leaf-object")), - ); - store.put_vcir(&vcir).expect("put vcir"); - let rule_hash = sha256_hex(b"missing-output-rule"); - store - .put_audit_rule_index_entry(&AuditRuleIndexEntry { - kind: AuditRuleKind::Roa, - rule_hash: rule_hash.clone(), - manifest_rsync_uri: manifest.to_string(), - source_object_uri: "rsync://example.test/leaf/a.roa".to_string(), - source_object_hash: sha256_hex(b"leaf-object"), - output_id: sha256_hex(b"missing-output"), - item_effective_until: PackTime::from_utc_offset_datetime( - time::OffsetDateTime::now_utc() + time::Duration::minutes(1), - ), - }) - .expect("put rule index"); - - let err = trace_rule_to_root(&store, AuditRuleKind::Roa, &rule_hash).unwrap_err(); - assert!(matches!(err, AuditTraceError::MissingLocalOutput { .. })); - } - - #[test] - fn trace_vcir_chain_to_root_detects_parent_cycle() { - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let a_manifest = "rsync://example.test/a.mft"; - let b_manifest = "rsync://example.test/b.mft"; - let a_vcir = sample_vcir( - a_manifest, - Some(b_manifest), - "test-tal", - None, - sample_artifacts(a_manifest, &sha256_hex(b"a-object")), - ); - let b_vcir = sample_vcir( - b_manifest, - Some(a_manifest), - "test-tal", - None, - sample_artifacts(b_manifest, &sha256_hex(b"b-object")), - ); - store.put_vcir(&a_vcir).expect("put a"); - store.put_vcir(&b_vcir).expect("put b"); - - let err = trace_vcir_chain_to_root(&store, a_manifest).unwrap_err(); - assert!(matches!( - err, - AuditTraceError::ParentCycle { manifest_rsync_uri } - if manifest_rsync_uri == a_manifest - )); - } -} diff --git a/src/bin/db_stats.rs b/src/bin/db_stats.rs index 5a8096e..5330e1c 100644 --- a/src/bin/db_stats.rs +++ b/src/bin/db_stats.rs @@ -4,9 +4,8 @@ use std::path::{Path, PathBuf}; use rocksdb::{DB, IteratorMode, Options}; use rpki::storage::{ - ALL_COLUMN_FAMILY_NAMES, CF_AUDIT_RULE_INDEX, CF_MANIFEST_REPLAY_META, CF_RAW_BY_HASH, - CF_REPOSITORY_VIEW, CF_RRDP_SOURCE, CF_RRDP_SOURCE_MEMBER, CF_RRDP_URI_OWNER, CF_VCIR, - column_family_descriptors, + ALL_COLUMN_FAMILY_NAMES, CF_MANIFEST_REPLAY_META, CF_RAW_BY_HASH, CF_REPOSITORY_VIEW, + CF_RRDP_SOURCE, CF_RRDP_SOURCE_MEMBER, CF_RRDP_URI_OWNER, CF_VCIR, column_family_descriptors, }; #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -99,7 +98,7 @@ Output: Output groups: - current_repository_view: repository_view + raw_by_hash - - current_validation_state: vcir + audit_rule_index + - current_validation_state: vcir + manifest_replay_meta - current_rrdp_state: rrdp_source + rrdp_source_member + rrdp_uri_owner " ) @@ -183,7 +182,7 @@ fn collect_db_file_stats(db_path: &Path) -> Result CfGroup { match cf_name { CF_REPOSITORY_VIEW | CF_RAW_BY_HASH => CfGroup::CurrentRepositoryView, - CF_VCIR | CF_MANIFEST_REPLAY_META | CF_AUDIT_RULE_INDEX => CfGroup::CurrentValidationState, + CF_VCIR | CF_MANIFEST_REPLAY_META => CfGroup::CurrentValidationState, CF_RRDP_SOURCE | CF_RRDP_SOURCE_MEMBER | CF_RRDP_URI_OWNER => CfGroup::CurrentRrdpState, _ => CfGroup::LegacyCompatibility, } @@ -375,10 +374,6 @@ mod tests { cf_group(CF_MANIFEST_REPLAY_META), CfGroup::CurrentValidationState ); - assert_eq!( - cf_group(CF_AUDIT_RULE_INDEX), - CfGroup::CurrentValidationState - ); assert_eq!(cf_group(CF_RRDP_SOURCE), CfGroup::CurrentRrdpState); assert_eq!(cf_group(CF_RRDP_URI_OWNER), CfGroup::CurrentRrdpState); assert_eq!(cf_group("unknown_legacy"), CfGroup::LegacyCompatibility); @@ -390,7 +385,7 @@ mod tests { (CF_REPOSITORY_VIEW, 5), (CF_RAW_BY_HASH, 7), (CF_VCIR, 11), - (CF_AUDIT_RULE_INDEX, 13), + (CF_MANIFEST_REPLAY_META, 13), (CF_RRDP_SOURCE_MEMBER, 19), ]); @@ -420,7 +415,7 @@ mod tests { live_sst_size_bytes: 55, live_sst_files: 2, }; - let audit = CfStats { + let replay_meta = CfStats { keys: 5, key_bytes: 50, value_bytes: 500, @@ -433,7 +428,7 @@ mod tests { let grouped = summarize_cf_stats([ (CF_REPOSITORY_VIEW, &repo), (CF_VCIR, &vcir), - (CF_AUDIT_RULE_INDEX, &audit), + (CF_MANIFEST_REPLAY_META, &replay_meta), ]); assert_eq!(grouped.get(&CfGroup::CurrentRepositoryView), Some(&repo)); diff --git a/src/bin/trace_arin_missing_vrps.rs b/src/bin/trace_arin_missing_vrps.rs deleted file mode 100644 index c33d943..0000000 --- a/src/bin/trace_arin_missing_vrps.rs +++ /dev/null @@ -1,103 +0,0 @@ -use rpki::audit_trace::trace_rule_to_root; -use rpki::storage::{AuditRuleKind, RocksStore, VcirOutputType}; -use serde_json::Value; -use std::env; -use std::path::Path; - -fn main() { - let args: Vec = env::args().collect(); - if args.len() < 3 { - eprintln!("usage: trace_arin_missing_vrps [ ...]"); - std::process::exit(2); - } - - let store = RocksStore::open(Path::new(&args[1])).expect("open db"); - let vcirs = store.list_vcirs().expect("list vcirs"); - - for row in &args[2..] { - let parts: Vec<&str> = row.split(',').collect(); - if parts.len() != 4 { - println!("ROW {row}"); - println!("ERROR invalid compare row"); - println!(); - continue; - } - let asn: u32 = parts[0] - .trim_start_matches("AS") - .parse() - .expect("parse asn"); - let prefix = parts[1].to_string(); - let max_length: u8 = parts[2].parse().expect("parse max length"); - let mut found = false; - - println!("ROW {row}"); - for vcir in &vcirs { - for output in &vcir.local_outputs { - if output.output_type != VcirOutputType::Vrp { - continue; - } - let payload: Value = match serde_json::from_str(&output.payload_json) { - Ok(value) => value, - Err(_) => continue, - }; - let payload_asn = payload - .get("asn") - .and_then(|v| v.as_u64()) - .map(|v| v as u32); - let payload_prefix = payload - .get("prefix") - .and_then(|v| v.as_str()) - .map(|v| v.to_string()); - let payload_max = payload - .get("max_length") - .and_then(|v| v.as_u64()) - .map(|v| v as u8); - if payload_asn == Some(asn) - && payload_prefix.as_ref() == Some(&prefix) - && payload_max == Some(max_length) - { - found = true; - println!("manifest_rsync_uri={}", vcir.manifest_rsync_uri); - println!("source_object_uri={}", output.source_object_uri); - println!("source_object_hash={}", output.source_object_hash); - println!("source_ee_cert_hash={}", output.source_ee_cert_hash); - println!("rule_hash={}", output.rule_hash); - println!("validation_path_hint={:?}", output.validation_path_hint); - - if let Some(trace) = - trace_rule_to_root(&store, AuditRuleKind::Roa, &output.rule_hash) - .expect("trace rule") - { - println!( - "trace_leaf_manifest={}", - trace - .chain_leaf_to_root - .first() - .map(|node| node.manifest_rsync_uri.as_str()) - .unwrap_or("") - ); - println!( - "trace_source_object_uri={}", - trace.resolved_output.source_object_uri - ); - println!("trace_chain_len={}", trace.chain_leaf_to_root.len()); - for (idx, node) in trace.chain_leaf_to_root.iter().enumerate() { - println!("chain[{idx}].manifest={}", node.manifest_rsync_uri); - println!( - "chain[{idx}].current_manifest={}", - node.current_manifest_rsync_uri - ); - println!("chain[{idx}].current_crl={}", node.current_crl_rsync_uri); - } - } - println!(); - } - } - } - - if !found { - println!("NOT_FOUND"); - println!(); - } - } -} diff --git a/src/blob_store.rs b/src/blob_store.rs index 87c8916..3d5a0be 100644 --- a/src/blob_store.rs +++ b/src/blob_store.rs @@ -3,7 +3,10 @@ use std::sync::Arc; use rocksdb::{DB, Options, WriteBatch}; -use crate::storage::{RawByHashEntry, RocksStore, StorageError, StorageResult}; +use crate::storage::{ + RawByHashEntry, RocksDbMemoryDbSnapshot, RocksStore, StorageError, StorageResult, + memory_db_snapshot_for_column_families, +}; const RAW_BY_HASH_KEY_PREFIX: &str = "rawbyhash:"; const RAW_BLOB_KEY_PREFIX: &str = "rawblob:"; @@ -167,6 +170,10 @@ impl ExternalRawStoreDb { pub fn path(&self) -> &PathBuf { &self.path } + + pub(crate) fn memory_snapshot(&self, label: impl Into) -> RocksDbMemoryDbSnapshot { + memory_db_snapshot_for_column_families(label, self.db.as_ref(), None) + } } impl ExternalRepoBytesDb { @@ -234,6 +241,10 @@ impl ExternalRepoBytesDb { pub fn path(&self) -> &PathBuf { &self.path } + + pub(crate) fn memory_snapshot(&self, label: impl Into) -> RocksDbMemoryDbSnapshot { + memory_db_snapshot_for_column_families(label, self.db.as_ref(), None) + } } impl RawObjectStore for RocksStore { diff --git a/src/ccr/accumulator.rs b/src/ccr/accumulator.rs index 794c593..d118a78 100644 --- a/src/ccr/accumulator.rs +++ b/src/ccr/accumulator.rs @@ -26,6 +26,21 @@ pub struct CcrManifestContribution { pub subordinate_skis: Vec>, } +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CcrAccumulatorMemoryStats { + pub trust_anchor_count: u64, + pub manifest_count: u64, + pub estimated_heap_bytes: u64, + pub string_bytes: u64, + pub string_capacity_bytes: u64, + pub vec_payload_bytes: u64, + pub vec_capacity_bytes: u64, + pub locations_der_count: u64, + pub subordinate_ski_count: u64, + pub btree_key_capacity_bytes: u64, + pub btree_entry_shallow_bytes: u64, +} + impl CcrManifestContribution { fn from_projection(projection: &VcirCcrManifestProjection) -> Result { let this_update = projection @@ -57,6 +72,36 @@ impl CcrManifestContribution { subordinates: self.subordinate_skis.clone(), } } + + fn add_memory_stats(&self, stats: &mut CcrAccumulatorMemoryStats) { + stats.string_bytes += self.manifest_rsync_uri.len() as u64; + stats.string_capacity_bytes += self.manifest_rsync_uri.capacity() as u64; + stats.estimated_heap_bytes += self.manifest_rsync_uri.capacity() as u64; + + add_vec_stats(&self.hash, stats); + add_vec_stats(&self.aki, stats); + add_vec_stats(&self.manifest_number_be, stats); + add_vec_of_vec_stats(&self.locations_der, stats); + add_vec_of_vec_stats(&self.subordinate_skis, stats); + stats.locations_der_count += self.locations_der.len() as u64; + stats.subordinate_ski_count += self.subordinate_skis.len() as u64; + } +} + +fn add_vec_stats(value: &Vec, stats: &mut CcrAccumulatorMemoryStats) { + stats.vec_payload_bytes += value.len() as u64; + stats.vec_capacity_bytes += value.capacity() as u64; + stats.estimated_heap_bytes += value.capacity() as u64; +} + +fn add_vec_of_vec_stats(values: &Vec>, stats: &mut CcrAccumulatorMemoryStats) { + let outer_capacity = values.capacity() * std::mem::size_of::>(); + stats.vec_payload_bytes += (values.len() * std::mem::size_of::>()) as u64; + stats.vec_capacity_bytes += outer_capacity as u64; + stats.estimated_heap_bytes += outer_capacity as u64; + for value in values { + add_vec_stats(value, stats); + } } #[derive(Clone, Debug, PartialEq, Eq)] @@ -138,6 +183,61 @@ impl CcrAccumulator { pub fn manifest_count(&self) -> usize { self.manifests_by_hash.len() } + + pub fn memory_stats(&self) -> CcrAccumulatorMemoryStats { + let mut stats = CcrAccumulatorMemoryStats { + trust_anchor_count: self.trust_anchors.len() as u64, + manifest_count: self.manifests_by_hash.len() as u64, + ..CcrAccumulatorMemoryStats::default() + }; + stats.estimated_heap_bytes += + (self.trust_anchors.capacity() * std::mem::size_of::()) as u64; + for trust_anchor in &self.trust_anchors { + add_vec_stats(&trust_anchor.tal.raw, &mut stats); + add_vec_of_string_stats(&trust_anchor.tal.comments, &mut stats); + stats.vec_payload_bytes += + (trust_anchor.tal.ta_uris.len() * std::mem::size_of::()) as u64; + stats.vec_capacity_bytes += + (trust_anchor.tal.ta_uris.capacity() * std::mem::size_of::()) as u64; + stats.estimated_heap_bytes += + (trust_anchor.tal.ta_uris.capacity() * std::mem::size_of::()) as u64; + for uri in &trust_anchor.tal.ta_uris { + stats.string_bytes += uri.as_str().len() as u64; + stats.string_capacity_bytes += uri.as_str().len() as u64; + stats.estimated_heap_bytes += uri.as_str().len() as u64; + } + add_vec_stats(&trust_anchor.tal.subject_public_key_info_der, &mut stats); + add_vec_stats(&trust_anchor.ta_certificate.raw_der, &mut stats); + if let Some(uri) = &trust_anchor.resolved_ta_uri { + stats.string_bytes += uri.as_str().len() as u64; + stats.string_capacity_bytes += uri.as_str().len() as u64; + stats.estimated_heap_bytes += uri.as_str().len() as u64; + } + } + + stats.btree_entry_shallow_bytes = (self.manifests_by_hash.len() + * (std::mem::size_of::>() + std::mem::size_of::())) + as u64; + stats.estimated_heap_bytes += stats.btree_entry_shallow_bytes; + for (key, contribution) in &self.manifests_by_hash { + stats.btree_key_capacity_bytes += key.capacity() as u64; + stats.estimated_heap_bytes += key.capacity() as u64; + contribution.add_memory_stats(&mut stats); + } + stats + } +} + +fn add_vec_of_string_stats(values: &Vec, stats: &mut CcrAccumulatorMemoryStats) { + let outer_capacity = values.capacity() * std::mem::size_of::(); + stats.vec_payload_bytes += (values.len() * std::mem::size_of::()) as u64; + stats.vec_capacity_bytes += outer_capacity as u64; + stats.estimated_heap_bytes += outer_capacity as u64; + for value in values { + stats.string_bytes += value.len() as u64; + stats.string_capacity_bytes += value.capacity() as u64; + stats.estimated_heap_bytes += value.capacity() as u64; + } } #[cfg(test)] diff --git a/src/cli.rs b/src/cli.rs index b3b7fd9..f66e344 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -7,17 +7,23 @@ use crate::cir::{CirTrustAnchorBinding, export_cir_from_run_multi}; use std::path::{Path, PathBuf}; use crate::analysis::timing::{TimingHandle, TimingMeta, TimingMetaUpdate}; +use crate::audit::AuditRepoSyncStats; +#[cfg(test)] use crate::audit::{ - AspaOutput, AuditRepoSyncStats, AuditReportV2, AuditRunMeta, AuditWarning, TreeSummary, - VrpOutput, format_roa_ip_prefix, + AspaOutput, AuditReportV2, AuditRunMeta, AuditWarning, TreeSummary, VrpOutput, + format_roa_ip_prefix, }; use crate::fetch::http::{BlockingHttpFetcher, HttpFetcherConfig}; use crate::fetch::rsync::LocalDirRsyncFetcher; use crate::fetch::rsync_system::{RsyncScopePolicy, SystemRsyncConfig, SystemRsyncFetcher}; +use crate::memory_telemetry::{ + MallocTrimProbe, MemoryTelemetryCheckpoint, MemoryTelemetrySummary, ObjectGraphMemoryMetric, + ObjectGraphMemorySection, ObjectGraphMemorySummary, +}; use crate::parallel::config::{ParallelPhase1Config, ParallelPhase2Config}; use crate::parallel::types::TalInputSpec; use crate::policy::{Policy, StrictPolicy}; -use crate::storage::RocksStore; +use crate::storage::{RocksStore, VcirStorageSummary}; use crate::validation::run_tree_from_tal::{ RunTreeFromTalAuditOutput, run_tree_from_multiple_tals_parallel_phase2_audit, run_tree_from_tal_and_ta_der_parallel_phase2_audit, @@ -28,7 +34,11 @@ use crate::validation::run_tree_from_tal::{ run_tree_from_tal_url_parallel_phase2_audit, }; use crate::validation::tree::TreeRunConfig; -use output::{ReportJsonFormat, run_compare_view_task, write_json, write_stage_timing}; +#[cfg(test)] +use output::write_json; +use output::{ + ReportJsonFormat, run_compare_view_task, write_report_json_from_shared, write_stage_timing, +}; use serde::Serialize; use std::sync::Arc; @@ -53,6 +63,35 @@ struct RunStageTiming { rrdp_download_ms_total: u64, rsync_download_ms_total: u64, download_bytes_total: u64, + vcir_storage_summary_ms: Option, + vcir_storage: Option, + memory_telemetry: Option, +} + +fn record_memory_checkpoint( + checkpoints: &mut Vec, + label: &str, + total_started: &std::time::Instant, + store: &RocksStore, +) { + checkpoints.push(MemoryTelemetryCheckpoint { + label: label.to_string(), + elapsed_ms: total_started.elapsed().as_millis() as u64, + process: crate::memory_telemetry::process_memory_snapshot(label), + rocksdb: store.memory_snapshot(), + }); +} + +fn memory_trim_probe_enabled() -> bool { + std::env::var("RPKI_MEMORY_TRIM_PROBE") + .map(|value| matches!(value.as_str(), "1" | "true" | "TRUE" | "yes" | "YES")) + .unwrap_or(false) +} + +fn vcir_storage_summary_enabled() -> bool { + std::env::var("RPKI_VCIR_STORAGE_SUMMARY") + .map(|value| matches!(value.as_str(), "1" | "true" | "TRUE" | "yes" | "YES")) + .unwrap_or(false) } #[derive(Clone, Debug, PartialEq, Eq)] @@ -92,6 +131,7 @@ pub struct CliArgs { pub payload_base_validation_time: Option, pub payload_delta_archive: Option, pub payload_delta_locks: Option, + pub memory_trim_after_validation: bool, pub rsync_local_dir: Option, pub disable_rrdp: bool, @@ -144,6 +184,7 @@ Options: --payload-base-validation-time Validation time for the base bootstrap inside offline delta replay --payload-delta-archive Use local delta payload archive root (offline delta replay) --payload-delta-locks Use local locks-delta.json (offline delta replay) + --memory-trim-after-validation Call malloc_trim(0) after validation/report memory checkpoints (Linux glibc only; default off) --tal-url TAL URL (repeatable; URL mode) --tal-path TAL file path (repeatable; file mode) @@ -221,6 +262,7 @@ pub fn parse_args(argv: &[String]) -> Result { let mut payload_base_validation_time: Option = None; let mut payload_delta_archive: Option = None; let mut payload_delta_locks: Option = None; + let mut memory_trim_after_validation = false; let mut rsync_local_dir: Option = None; let mut disable_rrdp: bool = false; @@ -497,6 +539,9 @@ pub fn parse_args(argv: &[String]) -> Result { .ok_or("--payload-delta-locks requires a value")?; payload_delta_locks = Some(PathBuf::from(v)); } + "--memory-trim-after-validation" => { + memory_trim_after_validation = true; + } "--rsync-local-dir" => { i += 1; let v = argv.get(i).ok_or("--rsync-local-dir requires a value")?; @@ -841,6 +886,7 @@ pub fn parse_args(argv: &[String]) -> Result { payload_base_validation_time, payload_delta_archive, payload_delta_locks, + memory_trim_after_validation, rsync_local_dir, disable_rrdp, rsync_command, @@ -880,10 +926,12 @@ fn unique_rrdp_repos_from_publication_points( set.len() } +#[cfg(test)] fn unique_rrdp_repos(report: &AuditReportV2) -> usize { unique_rrdp_repos_from_publication_points(&report.publication_points) } +#[cfg(test)] fn print_summary(report: &AuditReportV2) { let rrdp_repos = unique_rrdp_repos(report); println!("RPKI stage2 serial run summary"); @@ -1014,6 +1062,459 @@ impl PostValidationShared { } } +#[derive(Default)] +struct ObjectGraphSectionBuilder { + name: String, + item_count: u64, + shallow_bytes: u64, + heap_bytes: u64, + string_count: u64, + string_bytes: u64, + string_capacity_bytes: u64, + vec_count: u64, + vec_heap_bytes: u64, + vec_capacity_bytes: u64, + details: Vec, +} + +impl ObjectGraphSectionBuilder { + fn new(name: impl Into) -> Self { + Self { + name: name.into(), + ..Self::default() + } + } + + fn items(&mut self, count: usize, item_size: usize) { + self.item_count += count as u64; + self.shallow_bytes += (count as u64) * (item_size as u64); + } + + fn heap_bytes(&mut self, value: usize) { + self.heap_bytes += value as u64; + } + + fn string(&mut self, value: &str) { + self.string_count += 1; + self.string_bytes += value.len() as u64; + self.string_capacity_bytes += value.len() as u64; + self.heap_bytes += value.len() as u64; + } + + fn owned_string(&mut self, value: &String) { + self.string_count += 1; + self.string_bytes += value.len() as u64; + self.string_capacity_bytes += value.capacity() as u64; + self.heap_bytes += value.capacity() as u64; + } + + fn optional_string(&mut self, value: Option<&String>) { + if let Some(value) = value { + self.owned_string(value); + } + } + + fn vec_header_with_capacity(&mut self, len: usize, capacity: usize, element_size: usize) { + self.vec_count += 1; + let payload_bytes = len * element_size; + let capacity_bytes = capacity * element_size; + self.vec_heap_bytes += payload_bytes as u64; + self.vec_capacity_bytes += capacity_bytes as u64; + self.heap_bytes += capacity_bytes as u64; + } + + fn byte_vec_owned(&mut self, value: &Vec) { + self.vec_header_with_capacity(value.len(), value.capacity(), std::mem::size_of::()); + } + + fn string_vec_owned(&mut self, values: &Vec) { + self.vec_header_with_capacity( + values.len(), + values.capacity(), + std::mem::size_of::(), + ); + for value in values { + self.owned_string(value); + } + } + + fn metric(&mut self, name: impl Into, value: u64) { + self.details.push(ObjectGraphMemoryMetric { + name: name.into(), + value, + }); + } + + fn finish(self) -> ObjectGraphMemorySection { + let estimated_bytes = self.shallow_bytes + self.heap_bytes; + ObjectGraphMemorySection { + name: self.name, + item_count: self.item_count, + shallow_bytes: self.shallow_bytes, + heap_bytes: self.heap_bytes, + estimated_bytes, + string_count: self.string_count, + string_bytes: self.string_bytes, + string_capacity_bytes: self.string_capacity_bytes, + vec_count: self.vec_count, + vec_heap_bytes: self.vec_heap_bytes, + vec_capacity_bytes: self.vec_capacity_bytes, + details: self.details, + } + } +} + +fn estimate_shared_object_graph(shared: &PostValidationShared) -> ObjectGraphMemorySummary { + let mut sections = Vec::new(); + sections.push(estimate_publication_points_graph( + shared.publication_points.as_ref(), + )); + sections.push(estimate_vrps_graph(shared.vrps.as_ref())); + sections.push(estimate_aspas_graph(shared.aspas.as_ref())); + sections.push(estimate_router_keys_graph(shared.router_keys.as_ref())); + sections.push(estimate_warnings_graph( + "tree_warnings", + shared.tree_warnings.as_ref(), + )); + sections.push(estimate_downloads_graph(shared.downloads.as_ref())); + sections.push(estimate_current_repo_objects_graph( + shared.current_repo_objects.as_ref(), + )); + sections.push(estimate_trust_anchor_graph(shared)); + sections.push(estimate_ccr_accumulator_graph( + shared.ccr_accumulator.as_ref(), + )); + + let total_estimated_bytes = sections + .iter() + .map(|section| section.estimated_bytes) + .sum::(); + ObjectGraphMemorySummary { + captured_at_label: "after_validation".to_string(), + total_estimated_bytes, + sections, + notes: vec![ + "Estimated bytes are Rust object graph approximations based on struct sizes and owned String/Vec payload lengths.".to_string(), + "The estimate intentionally excludes allocator metadata, fragmentation, freed-but-retained arenas, RocksDB C++ heap, and transient worker allocations.".to_string(), + "Large RSS minus this estimate points to allocator retention or structures not yet modeled by this telemetry.".to_string(), + ], + } +} + +fn estimate_publication_points_graph( + publication_points: &[crate::audit::PublicationPointAudit], +) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new("publication_points"); + builder.items( + publication_points.len(), + std::mem::size_of::(), + ); + builder.metric("publication_point_count", publication_points.len() as u64); + let mut object_count = 0u64; + let mut pp_warning_count = 0u64; + let mut pp_discovered_from_count = 0u64; + let mut object_detail_count = 0u64; + + for pp in publication_points { + builder.owned_string(&pp.rsync_base_uri); + builder.owned_string(&pp.manifest_rsync_uri); + builder.owned_string(&pp.publication_point_rsync_uri); + builder.optional_string(pp.rrdp_notification_uri.as_ref()); + builder.owned_string(&pp.source); + builder.optional_string(pp.repo_sync_source.as_ref()); + builder.optional_string(pp.repo_sync_phase.as_ref()); + builder.optional_string(pp.repo_sync_error.as_ref()); + builder.owned_string(&pp.repo_terminal_state); + builder.owned_string(&pp.this_update_rfc3339_utc); + builder.owned_string(&pp.next_update_rfc3339_utc); + builder.owned_string(&pp.verified_at_rfc3339_utc); + + if let Some(discovered_from) = &pp.discovered_from { + pp_discovered_from_count += 1; + builder.heap_bytes(std::mem::size_of::()); + builder.owned_string(&discovered_from.parent_manifest_rsync_uri); + builder.owned_string(&discovered_from.child_ca_certificate_rsync_uri); + builder.owned_string(&discovered_from.child_ca_certificate_sha256_hex); + } + + pp_warning_count += pp.warnings.len() as u64; + builder.vec_header_with_capacity( + pp.warnings.len(), + pp.warnings.capacity(), + std::mem::size_of::(), + ); + for warning in &pp.warnings { + builder.owned_string(&warning.message); + builder.string_vec_owned(&warning.rfc_refs); + builder.optional_string(warning.context.as_ref()); + } + + object_count += pp.objects.len() as u64; + builder.vec_header_with_capacity( + pp.objects.len(), + pp.objects.capacity(), + std::mem::size_of::(), + ); + for object in &pp.objects { + builder.owned_string(&object.rsync_uri); + builder.owned_string(&object.sha256_hex); + if object.detail.is_some() { + object_detail_count += 1; + } + builder.optional_string(object.detail.as_ref()); + } + } + + builder.metric("object_audit_entry_count", object_count); + builder.metric("publication_point_warning_count", pp_warning_count); + builder.metric( + "publication_point_discovered_from_count", + pp_discovered_from_count, + ); + builder.metric("object_detail_count", object_detail_count); + builder.finish() +} + +fn estimate_vrps_graph(vrps: &[crate::validation::objects::Vrp]) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new("vrps"); + builder.items( + vrps.len(), + std::mem::size_of::(), + ); + builder.metric("vrp_count", vrps.len() as u64); + builder.finish() +} + +fn estimate_aspas_graph( + aspas: &[crate::validation::objects::AspaAttestation], +) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new("aspas"); + builder.items( + aspas.len(), + std::mem::size_of::(), + ); + let mut providers_total = 0u64; + for aspa in aspas { + providers_total += aspa.provider_as_ids.len() as u64; + builder.vec_header_with_capacity( + aspa.provider_as_ids.len(), + aspa.provider_as_ids.capacity(), + std::mem::size_of::(), + ); + } + builder.metric("aspa_count", aspas.len() as u64); + builder.metric("provider_asn_count", providers_total); + builder.finish() +} + +fn estimate_router_keys_graph( + router_keys: &[crate::validation::objects::RouterKeyPayload], +) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new("router_keys"); + builder.items( + router_keys.len(), + std::mem::size_of::(), + ); + for router_key in router_keys { + builder.byte_vec_owned(&router_key.ski); + builder.byte_vec_owned(&router_key.spki_der); + builder.owned_string(&router_key.source_object_uri); + builder.owned_string(&router_key.source_object_hash); + builder.owned_string(&router_key.source_ee_cert_hash); + } + builder.metric("router_key_count", router_keys.len() as u64); + builder.finish() +} + +fn estimate_warnings_graph( + name: &str, + warnings: &[crate::report::Warning], +) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new(name); + builder.items( + warnings.len(), + std::mem::size_of::(), + ); + for warning in warnings { + builder.owned_string(&warning.message); + builder.vec_header_with_capacity( + warning.rfc_refs.len(), + warning.rfc_refs.capacity(), + std::mem::size_of::(), + ); + builder.optional_string(warning.context.as_ref()); + } + builder.metric("warning_count", warnings.len() as u64); + builder.finish() +} + +fn estimate_downloads_graph( + downloads: &[crate::audit::AuditDownloadEvent], +) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new("downloads"); + builder.items( + downloads.len(), + std::mem::size_of::(), + ); + let mut error_count = 0u64; + let mut bytes_count = 0u64; + let mut objects_stat_count = 0u64; + for event in downloads { + builder.owned_string(&event.uri); + builder.owned_string(&event.started_at_rfc3339_utc); + builder.owned_string(&event.finished_at_rfc3339_utc); + if event.error.is_some() { + error_count += 1; + } + if event.bytes.is_some() { + bytes_count += 1; + } + if event.objects.is_some() { + objects_stat_count += 1; + } + builder.optional_string(event.error.as_ref()); + } + builder.metric("download_event_count", downloads.len() as u64); + builder.metric("download_error_count", error_count); + builder.metric("download_bytes_field_count", bytes_count); + builder.metric("download_objects_stat_count", objects_stat_count); + builder.finish() +} + +fn estimate_current_repo_objects_graph( + objects: &[crate::current_repo_index::CurrentRepoObject], +) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new("current_repo_objects"); + builder.items( + objects.len(), + std::mem::size_of::(), + ); + let mut object_type_count = 0u64; + for object in objects { + builder.owned_string(&object.rsync_uri); + builder.owned_string(&object.current_hash_hex); + builder.owned_string(&object.repository_source); + if object.object_type.is_some() { + object_type_count += 1; + } + builder.optional_string(object.object_type.as_ref()); + } + builder.metric("current_repo_object_count", objects.len() as u64); + builder.metric("current_repo_object_type_count", object_type_count); + builder.finish() +} + +fn estimate_trust_anchor_graph(shared: &PostValidationShared) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new("trust_anchors_and_tal_inputs"); + builder.items( + 1, + std::mem::size_of::(), + ); + estimate_discovered_root(&mut builder, &shared.discovery); + builder.items( + shared.discoveries.len(), + std::mem::size_of::(), + ); + for discovery in shared.discoveries.iter() { + estimate_discovered_root(&mut builder, discovery); + } + builder.items( + shared.successful_tal_inputs.len(), + std::mem::size_of::(), + ); + for tal_input in shared.successful_tal_inputs.iter() { + estimate_tal_input(&mut builder, tal_input); + } + builder.metric("discoveries_count", shared.discoveries.len() as u64); + builder.metric( + "successful_tal_inputs_count", + shared.successful_tal_inputs.len() as u64, + ); + builder.finish() +} + +fn estimate_discovered_root( + builder: &mut ObjectGraphSectionBuilder, + discovery: &crate::validation::from_tal::DiscoveredRootCaInstance, +) { + builder.optional_string(discovery.tal_url.as_ref()); + estimate_trust_anchor(builder, &discovery.trust_anchor); + builder.owned_string(&discovery.ca_instance.rsync_base_uri); + builder.owned_string(&discovery.ca_instance.manifest_rsync_uri); + builder.owned_string(&discovery.ca_instance.publication_point_rsync_uri); + builder.optional_string(discovery.ca_instance.rrdp_notification_uri.as_ref()); +} + +fn estimate_trust_anchor( + builder: &mut ObjectGraphSectionBuilder, + trust_anchor: &crate::data_model::ta::TrustAnchor, +) { + builder.byte_vec_owned(&trust_anchor.tal.raw); + builder.string_vec_owned(&trust_anchor.tal.comments); + builder.vec_header_with_capacity( + trust_anchor.tal.ta_uris.len(), + trust_anchor.tal.ta_uris.capacity(), + std::mem::size_of::(), + ); + for uri in &trust_anchor.tal.ta_uris { + builder.string(uri.as_str()); + } + builder.byte_vec_owned(&trust_anchor.tal.subject_public_key_info_der); + builder.byte_vec_owned(&trust_anchor.ta_certificate.raw_der); + if let Some(uri) = &trust_anchor.resolved_ta_uri { + builder.string(uri.as_str()); + } +} + +fn estimate_tal_input(builder: &mut ObjectGraphSectionBuilder, tal_input: &TalInputSpec) { + builder.owned_string(&tal_input.tal_id); + builder.owned_string(&tal_input.rir_id); + match &tal_input.source { + crate::parallel::types::TalSource::Url(url) => builder.owned_string(url), + crate::parallel::types::TalSource::DerBytes { + tal_url, + tal_bytes, + ta_der, + } => { + builder.owned_string(tal_url); + builder.byte_vec_owned(tal_bytes); + builder.byte_vec_owned(ta_der); + } + crate::parallel::types::TalSource::FilePath(path) => { + builder.string(&path.to_string_lossy()); + } + crate::parallel::types::TalSource::FilePathWithTa { tal_path, ta_path } => { + builder.string(&tal_path.to_string_lossy()); + builder.string(&ta_path.to_string_lossy()); + } + } +} + +fn estimate_ccr_accumulator_graph( + accumulator: Option<&CcrAccumulator>, +) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new("ccr_accumulator"); + if let Some(accumulator) = accumulator { + builder.items(1, std::mem::size_of::()); + let stats = accumulator.memory_stats(); + builder.heap_bytes(stats.estimated_heap_bytes as usize); + builder.metric("trust_anchor_count", stats.trust_anchor_count); + builder.metric("manifest_count", stats.manifest_count); + builder.metric("string_bytes", stats.string_bytes); + builder.metric("string_capacity_bytes", stats.string_capacity_bytes); + builder.metric("vec_payload_bytes", stats.vec_payload_bytes); + builder.metric("vec_capacity_bytes", stats.vec_capacity_bytes); + builder.metric("locations_der_count", stats.locations_der_count); + builder.metric("subordinate_ski_count", stats.subordinate_ski_count); + builder.metric("btree_key_capacity_bytes", stats.btree_key_capacity_bytes); + builder.metric("btree_entry_shallow_bytes", stats.btree_entry_shallow_bytes); + } else { + builder.metric("manifest_count", 0); + } + builder.finish() +} + +#[cfg(test)] fn build_report( policy: &Policy, validation_time: time::OffsetDateTime, @@ -1072,7 +1573,6 @@ fn build_report( #[derive(Clone, Debug, PartialEq, Eq)] struct ReportTaskOutput { - report: Option, report_build_ms: u64, report_write_ms: Option, } @@ -1080,7 +1580,6 @@ struct ReportTaskOutput { impl ReportTaskOutput { fn skipped() -> Self { Self { - report: None, report_build_ms: 0, report_write_ms: None, } @@ -1094,23 +1593,21 @@ fn run_report_task( report_json_path: Option<&Path>, report_json_format: ReportJsonFormat, ) -> Result { - let report_started = std::time::Instant::now(); - let report = build_report(policy, validation_time, shared); - let report_build_ms = report_started.elapsed().as_millis() as u64; - - let report_write_ms = if let Some(path) = report_json_path { - let started = std::time::Instant::now(); - write_json(path, &report, report_json_format)?; - Some(started.elapsed().as_millis() as u64) + if let Some(path) = report_json_path { + let timing = write_report_json_from_shared( + path, + policy, + validation_time, + shared, + report_json_format, + )?; + Ok(ReportTaskOutput { + report_build_ms: timing.build_ms, + report_write_ms: Some(timing.write_ms), + }) } else { - None - }; - - Ok(ReportTaskOutput { - report: Some(report), - report_build_ms, - report_write_ms, - }) + Ok(ReportTaskOutput::skipped()) + } } #[derive(Clone, Debug, PartialEq, Eq)] @@ -1443,6 +1940,15 @@ pub fn run(argv: &[String]) -> Result<(), String> { }; let total_started = std::time::Instant::now(); + let mut memory_checkpoints: Vec = Vec::new(); + let mut malloc_trim_probes: Vec = Vec::new(); + let enable_memory_trim_probe = memory_trim_probe_enabled() || args.memory_trim_after_validation; + record_memory_checkpoint( + &mut memory_checkpoints, + "after_store_open", + &total_started, + store.as_ref(), + ); let validation_started = std::time::Instant::now(); let collect_current_repo_objects = false; let out = if delta_replay_mode { @@ -1607,6 +2113,34 @@ pub fn run(argv: &[String]) -> Result<(), String> { let validation_ms = validation_started.elapsed().as_millis() as u64; let shared = PostValidationShared::from_run_output(out); + let vcir_storage_summary_enabled = vcir_storage_summary_enabled(); + let vcir_storage_summary_started = std::time::Instant::now(); + let vcir_storage = if config.persist_vcir && vcir_storage_summary_enabled { + Some( + store + .summarize_vcir_storage() + .map_err(|e| format!("summarize VCIR storage failed: {e}"))?, + ) + } else { + None + }; + let vcir_storage_summary_ms = (config.persist_vcir && vcir_storage_summary_enabled) + .then(|| vcir_storage_summary_started.elapsed().as_millis() as u64); + record_memory_checkpoint( + &mut memory_checkpoints, + "after_validation", + &total_started, + store.as_ref(), + ); + if enable_memory_trim_probe { + malloc_trim_probes.push(crate::memory_telemetry::malloc_trim_probe()); + record_memory_checkpoint( + &mut memory_checkpoints, + "after_validation_malloc_trim", + &total_started, + store.as_ref(), + ); + } if let Some((_out_dir, t)) = timing.as_ref() { t.record_count("instances_processed", shared.instances_processed as u64); @@ -1705,7 +2239,21 @@ pub fn run(argv: &[String]) -> Result<(), String> { }; let report_output = report_result?; let ccr_output = ccr_result?; - let report = report_output.report; + record_memory_checkpoint( + &mut memory_checkpoints, + "after_report_and_ccr", + &total_started, + store.as_ref(), + ); + if enable_memory_trim_probe { + malloc_trim_probes.push(crate::memory_telemetry::malloc_trim_probe()); + record_memory_checkpoint( + &mut memory_checkpoints, + "after_report_and_ccr_malloc_trim", + &total_started, + store.as_ref(), + ); + } let report_build_ms = report_output.report_build_ms; let report_write_ms = report_output.report_write_ms; let ccr_build_ms = ccr_output.ccr_build_ms; @@ -1723,6 +2271,12 @@ pub fn run(argv: &[String]) -> Result<(), String> { )?; let compare_view_build_ms = compare_view_output.build_ms; let compare_view_write_ms = compare_view_output.write_ms; + record_memory_checkpoint( + &mut memory_checkpoints, + "after_compare_view", + &total_started, + store.as_ref(), + ); let mut cir_build_cir_ms = None; let mut cir_write_cir_ms = None; @@ -1775,7 +2329,19 @@ pub fn run(argv: &[String]) -> Result<(), String> { summary.timing.write_cir_ms, summary.timing.total_ms ); + record_memory_checkpoint( + &mut memory_checkpoints, + "after_cir", + &total_started, + store.as_ref(), + ); } + record_memory_checkpoint( + &mut memory_checkpoints, + "before_stage_timing", + &total_started, + store.as_ref(), + ); let stage_timing = RunStageTiming { validation_ms, report_build_ms, @@ -1796,6 +2362,13 @@ pub fn run(argv: &[String]) -> Result<(), String> { rrdp_download_ms_total, rsync_download_ms_total, download_bytes_total, + vcir_storage_summary_ms, + vcir_storage, + memory_telemetry: Some(MemoryTelemetrySummary { + checkpoints: memory_checkpoints, + object_graph: Some(estimate_shared_object_graph(&shared)), + malloc_trim_probes, + }), }; let stage_timing_anchor_path = args .report_json_path @@ -1849,11 +2422,7 @@ pub fn run(argv: &[String]) -> Result<(), String> { eprintln!("analysis: wrote {}", pb_path.display()); } - if let Some(report) = report.as_ref() { - print_summary(report); - } else { - print_summary_from_shared(validation_time, &shared); - } + print_summary_from_shared(validation_time, &shared); Ok(()) } diff --git a/src/cli/output.rs b/src/cli/output.rs index 8ebd6b2..e706bae 100644 --- a/src/cli/output.rs +++ b/src/cli/output.rs @@ -1,7 +1,10 @@ use std::io::BufWriter; use std::path::Path; -use crate::audit::AuditReportV2; +use serde::Serialize; +use serde::ser::SerializeSeq; + +use crate::audit::{AspaOutput, AuditRunMeta, AuditWarning, VrpOutput}; use crate::ccr::canonical_vrp_prefix; use super::{PostValidationShared, RunStageTiming}; @@ -12,9 +15,9 @@ pub(super) enum ReportJsonFormat { Compact, } -pub(super) fn write_json( +pub(super) fn write_json( path: &Path, - report: &AuditReportV2, + report: &T, format: ReportJsonFormat, ) -> Result<(), String> { let f = std::fs::File::create(path) @@ -28,6 +31,128 @@ pub(super) fn write_json( Ok(()) } +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct ReportJsonWriteTiming { + pub(super) build_ms: u64, + pub(super) write_ms: u64, +} + +#[derive(Serialize)] +struct BorrowedAuditReportV2<'a> { + format_version: u32, + meta: AuditRunMeta, + policy: &'a crate::policy::Policy, + tree: BorrowedTreeSummary<'a>, + publication_points: &'a [crate::audit::PublicationPointAudit], + vrps: VrpReportSequence<'a>, + aspas: AspaReportSequence<'a>, + downloads: &'a [crate::audit::AuditDownloadEvent], + download_stats: &'a crate::audit::AuditDownloadStats, + repo_sync_stats: crate::audit::AuditRepoSyncStats, +} + +#[derive(Serialize)] +struct BorrowedTreeSummary<'a> { + instances_processed: usize, + instances_failed: usize, + warnings: WarningReportSequence<'a>, +} + +struct WarningReportSequence<'a>(&'a [crate::report::Warning]); + +impl Serialize for WarningReportSequence<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let mut seq = serializer.serialize_seq(Some(self.0.len()))?; + for warning in self.0 { + seq.serialize_element(&AuditWarning::from(warning))?; + } + seq.end() + } +} + +struct VrpReportSequence<'a>(&'a [crate::validation::objects::Vrp]); + +impl Serialize for VrpReportSequence<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let mut seq = serializer.serialize_seq(Some(self.0.len()))?; + for vrp in self.0 { + seq.serialize_element(&VrpOutput { + asn: vrp.asn, + prefix: crate::audit::format_roa_ip_prefix(&vrp.prefix), + max_length: vrp.max_length, + })?; + } + seq.end() + } +} + +struct AspaReportSequence<'a>(&'a [crate::validation::objects::AspaAttestation]); + +impl Serialize for AspaReportSequence<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let mut seq = serializer.serialize_seq(Some(self.0.len()))?; + for aspa in self.0 { + seq.serialize_element(&AspaOutput { + customer_as_id: aspa.customer_as_id, + provider_as_ids: aspa.provider_as_ids.clone(), + })?; + } + seq.end() + } +} + +pub(super) fn write_report_json_from_shared( + path: &Path, + policy: &crate::policy::Policy, + validation_time: time::OffsetDateTime, + shared: &PostValidationShared, + format: ReportJsonFormat, +) -> Result { + use time::format_description::well_known::Rfc3339; + + let build_started = std::time::Instant::now(); + let validation_time_rfc3339_utc = validation_time + .to_offset(time::UtcOffset::UTC) + .format(&Rfc3339) + .expect("format validation_time"); + let repo_sync_stats = super::build_repo_sync_stats(shared.publication_points.as_ref()); + let report = BorrowedAuditReportV2 { + format_version: 2, + meta: AuditRunMeta { + validation_time_rfc3339_utc, + }, + policy, + tree: BorrowedTreeSummary { + instances_processed: shared.instances_processed, + instances_failed: shared.instances_failed, + warnings: WarningReportSequence(shared.tree_warnings.as_ref()), + }, + publication_points: shared.publication_points.as_ref(), + vrps: VrpReportSequence(shared.vrps.as_ref()), + aspas: AspaReportSequence(shared.aspas.as_ref()), + downloads: shared.downloads.as_ref(), + download_stats: &shared.download_stats, + repo_sync_stats, + }; + let build_ms = build_started.elapsed().as_millis() as u64; + + let write_started = std::time::Instant::now(); + write_json(path, &report, format)?; + Ok(ReportJsonWriteTiming { + build_ms, + write_ms: write_started.elapsed().as_millis() as u64, + }) +} + #[derive(Clone, Debug, PartialEq, Eq)] pub(super) struct CompareViewTaskOutput { pub(super) build_ms: Option, diff --git a/src/cli/tests.rs b/src/cli/tests.rs index 0969261..5ae734d 100644 --- a/src/cli/tests.rs +++ b/src/cli/tests.rs @@ -1,4 +1,12 @@ use super::*; +use crate::memory_telemetry::{ + MemoryTelemetryCheckpoint, MemoryTelemetrySummary, ProcessMemorySnapshot, +}; +use crate::storage::{ + RocksDbMemorySnapshot, RocksDbMemoryTotals, VcirCcrProjectionSizeBreakdown, + VcirChildResourceSizeBreakdown, VcirCoreFieldSizeBreakdown, VcirFieldSizeBreakdown, + VcirStorageEntrySummary, VcirStorageSummary, +}; #[test] fn parse_help_returns_usage() { @@ -9,6 +17,7 @@ fn parse_help_returns_usage() { assert!(err.contains("--rsync-mirror-root"), "{err}"); assert!(err.contains("--rsync-scope"), "{err}"); assert!(err.contains("--parallel-phase2-object-workers"), "{err}"); + assert!(err.contains("--memory-trim-after-validation"), "{err}"); assert!(!err.contains("--parallel-phase1"), "{err}"); assert!(!err.contains("--parallel-phase2 "), "{err}"); } @@ -82,6 +91,37 @@ fn parse_accepts_ccr_out_path() { ); } +#[test] +fn parse_accepts_memory_trim_after_validation() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--memory-trim-after-validation".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.memory_trim_after_validation); +} + +#[test] +fn parse_disables_memory_trim_after_validation_by_default() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(!args.memory_trim_after_validation); +} + #[test] fn parse_accepts_report_json_compact_when_report_json_is_set() { let argv = vec![ @@ -1414,13 +1454,14 @@ fn run_report_task_and_stage_timing_work() { ) .expect("run report task"); - let report = report_output.report.as_ref().expect("report built"); - assert_eq!(report.vrps.len(), 2); - assert_eq!(report.aspas.len(), 1); assert!(report_output.report_write_ms.is_some()); let report_json = std::fs::read_to_string(&report_path).expect("read report json"); assert!(!report_json.contains('\n'), "{report_json}"); + let report: serde_json::Value = + serde_json::from_str(&report_json).expect("parse compact report json"); + assert_eq!(report["vrps"].as_array().unwrap().len(), 2); + assert_eq!(report["aspas"].as_array().unwrap().len(), 1); let stage_timing = RunStageTiming { validation_ms: 1, @@ -1442,12 +1483,62 @@ fn run_report_task_and_stage_timing_work() { rrdp_download_ms_total: 13, rsync_download_ms_total: 14, download_bytes_total: 15, + vcir_storage_summary_ms: Some(16), + vcir_storage: Some(VcirStorageSummary { + entry_count: 2, + vcir_value_bytes: 100, + vcir_value_bytes_max: 60, + vcir_value_bytes_max_manifest_rsync_uri: Some( + "rsync://example.test/repo/max.mft".to_string(), + ), + core_fields: VcirCoreFieldSizeBreakdown { + manifest_rsync_uri_bytes: 10, + ..VcirCoreFieldSizeBreakdown::default() + }, + ccr_projection: VcirCcrProjectionSizeBreakdown { + manifest_sha256_bytes: 32, + ..VcirCcrProjectionSizeBreakdown::default() + }, + child_resources: VcirChildResourceSizeBreakdown { + effective_ip_resource_cbor_bytes: 12, + effective_as_resource_cbor_bytes: 6, + }, + field_sizes: VcirFieldSizeBreakdown { + local_output_count: 1, + local_output_payload_json_bytes: 70, + local_output_payload_typed_body_bytes: 20, + ..VcirFieldSizeBreakdown::default() + }, + local_output_old_projection_bytes: 80, + local_output_typed_projection_bytes: 30, + local_output_projection_saved_bytes: 50, + top_entries_by_vcir_value_bytes: vec![VcirStorageEntrySummary { + manifest_rsync_uri: "rsync://example.test/repo/max.mft".to_string(), + vcir_value_bytes: 60, + local_vrp_count: 1, + local_aspa_count: 0, + local_router_key_count: 0, + accepted_object_count: 1, + rejected_object_count: 0, + child_count: 0, + core_fields: VcirCoreFieldSizeBreakdown::default(), + ccr_projection: VcirCcrProjectionSizeBreakdown::default(), + child_resources: VcirChildResourceSizeBreakdown::default(), + field_sizes: VcirFieldSizeBreakdown::default(), + local_output_old_projection_bytes: 1, + local_output_typed_projection_bytes: 1, + local_output_projection_saved_bytes: 0, + }], + }), + memory_telemetry: None, }; write_stage_timing(Some(&report_path), &stage_timing).expect("write stage timing"); let stage_timing_json = std::fs::read_to_string(dir.path().join("stage-timing.json")).expect("read timing"); assert!(stage_timing_json.contains("\"validation_ms\"")); assert!(stage_timing_json.contains("\"ccr_build_ms\"")); + assert!(stage_timing_json.contains("\"vcir_storage\"")); + assert!(stage_timing_json.contains("\"local_output_projection_saved_bytes\"")); let ccr_path = dir.path().join("result.ccr"); write_stage_timing(Some(&ccr_path), &stage_timing).expect("write stage timing via ccr path"); @@ -1457,11 +1548,136 @@ fn run_report_task_and_stage_timing_work() { ); let skipped = ReportTaskOutput::skipped(); - assert!(skipped.report.is_none()); assert_eq!(skipped.report_build_ms, 0); assert!(skipped.report_write_ms.is_none()); } +#[test] +fn stage_timing_serializes_memory_telemetry() { + let dir = tempfile::tempdir().expect("tmpdir"); + let report_path = dir.path().join("report.json"); + let stage_timing = RunStageTiming { + validation_ms: 1, + report_build_ms: 2, + report_write_ms: None, + ccr_build_ms: None, + ccr_build_breakdown: None, + ccr_write_ms: None, + compare_view_build_ms: None, + compare_view_write_ms: None, + cir_build_cir_ms: None, + cir_write_cir_ms: None, + cir_total_ms: None, + total_ms: 3, + publication_points: 4, + repo_sync_ms_total: 5, + publication_point_repo_sync_ms_total: 6, + download_event_count: 7, + rrdp_download_ms_total: 8, + rsync_download_ms_total: 9, + download_bytes_total: 10, + vcir_storage_summary_ms: None, + vcir_storage: None, + memory_telemetry: Some(MemoryTelemetrySummary { + checkpoints: vec![MemoryTelemetryCheckpoint { + label: "after_validation".to_string(), + elapsed_ms: 11, + process: ProcessMemorySnapshot { + label: "after_validation".to_string(), + vm_rss_kb: Some(12), + vm_size_kb: None, + vm_data_kb: None, + vm_swap_kb: None, + rss_anon_kb: Some(13), + rss_file_kb: None, + rss_shmem_kb: None, + threads: Some(14), + fd_count: Some(15), + smaps_rollup: None, + smaps_mapping_summary: None, + errors: Vec::new(), + }, + rocksdb: RocksDbMemorySnapshot { + databases: Vec::new(), + totals: RocksDbMemoryTotals { + cur_size_all_mem_tables: 16, + size_all_mem_tables: 17, + estimate_table_readers_mem: 18, + block_cache_capacity: 19, + block_cache_usage: 20, + block_cache_pinned_usage: 21, + }, + }, + }], + object_graph: None, + malloc_trim_probes: Vec::new(), + }), + }; + + write_stage_timing(Some(&report_path), &stage_timing).expect("write stage timing"); + let value: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(dir.path().join("stage-timing.json")).unwrap(), + ) + .expect("parse stage timing json"); + let checkpoint = &value["memory_telemetry"]["checkpoints"][0]; + assert_eq!(checkpoint["label"], "after_validation"); + assert_eq!(checkpoint["process"]["vm_rss_kb"], 12); + assert_eq!( + checkpoint["rocksdb"]["totals"]["cur_size_all_mem_tables"], + 16 + ); + assert!( + value["memory_telemetry"] + .as_object() + .expect("memory telemetry object") + .get("malloc_trim_probes") + .is_none() + ); +} + +#[test] +fn shared_object_graph_estimate_counts_audit_and_outputs() { + let mut shared = synthetic_post_validation_shared(); + let mut publication_points = shared + .publication_points + .iter() + .cloned() + .collect::>(); + publication_points[0].rsync_base_uri = "rsync://example.test/repo/".to_string(); + publication_points[0].manifest_rsync_uri = "rsync://example.test/repo/a.mft".to_string(); + publication_points[0].publication_point_rsync_uri = "rsync://example.test/repo/".to_string(); + publication_points[0].objects = vec![crate::audit::ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/a.roa".to_string(), + sha256_hex: "11".repeat(32), + kind: crate::audit::AuditObjectKind::Roa, + result: crate::audit::AuditObjectResult::Ok, + detail: None, + }]; + shared.publication_points = publication_points.into(); + + let graph = estimate_shared_object_graph(&shared); + let publication_points_section = graph + .sections + .iter() + .find(|section| section.name == "publication_points") + .expect("publication points section"); + let object_count = publication_points_section + .details + .iter() + .find(|metric| metric.name == "object_audit_entry_count") + .expect("object count metric"); + assert_eq!(object_count.value, 1); + assert!(publication_points_section.estimated_bytes > 0); + + let vrps_section = graph + .sections + .iter() + .find(|section| section.name == "vrps") + .expect("vrps section"); + assert_eq!(vrps_section.item_count, 2); + assert!(graph.total_estimated_bytes >= publication_points_section.estimated_bytes); +} + #[test] fn run_compare_view_task_writes_csv_from_shared_output() { let shared = synthetic_post_validation_shared(); diff --git a/src/data_model/roa.rs b/src/data_model/roa.rs index 317484b..ef35002 100644 --- a/src/data_model/roa.rs +++ b/src/data_model/roa.rs @@ -217,7 +217,9 @@ impl RoaObject { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize, +)] pub enum RoaAfi { Ipv4, Ipv6, diff --git a/src/lib.rs b/src/lib.rs index 427b4f9..380bf2b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,8 +9,6 @@ pub mod audit; #[cfg(feature = "full")] pub mod audit_downloads; #[cfg(feature = "full")] -pub mod audit_trace; -#[cfg(feature = "full")] pub mod blob_store; #[cfg(feature = "full")] pub mod cli; @@ -19,6 +17,8 @@ pub mod current_repo_index; #[cfg(feature = "full")] pub mod fetch; #[cfg(feature = "full")] +pub mod memory_telemetry; +#[cfg(feature = "full")] pub mod parallel; #[cfg(feature = "full")] pub mod policy; diff --git a/src/memory_telemetry.rs b/src/memory_telemetry.rs new file mode 100644 index 0000000..598e1ee --- /dev/null +++ b/src/memory_telemetry.rs @@ -0,0 +1,348 @@ +use serde::Serialize; + +use crate::storage::RocksDbMemorySnapshot; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct MallocTrimProbe { + pub supported: bool, + pub return_value: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct ProcessMemorySnapshot { + pub label: String, + pub vm_rss_kb: Option, + pub vm_size_kb: Option, + pub vm_data_kb: Option, + pub vm_swap_kb: Option, + pub rss_anon_kb: Option, + pub rss_file_kb: Option, + pub rss_shmem_kb: Option, + pub threads: Option, + pub fd_count: Option, + pub smaps_rollup: Option, + pub smaps_mapping_summary: Option, + pub errors: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct SmapsRollupSnapshot { + pub rss_kb: Option, + pub pss_kb: Option, + pub shared_clean_kb: Option, + pub shared_dirty_kb: Option, + pub private_clean_kb: Option, + pub private_dirty_kb: Option, + pub anonymous_kb: Option, + pub swap_kb: Option, + pub swap_pss_kb: Option, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct SmapsMappingSummary { + pub heap: SmapsMappingCategory, + pub anonymous_mmap: SmapsMappingCategory, + pub file_backed: SmapsMappingCategory, + pub stack: SmapsMappingCategory, + pub special: SmapsMappingCategory, + pub total: SmapsMappingCategory, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct SmapsMappingCategory { + pub mappings: u64, + pub size_kb: u64, + pub rss_kb: u64, + pub pss_kb: u64, + pub private_clean_kb: u64, + pub private_dirty_kb: u64, + pub anonymous_kb: u64, + pub largest_mapping_rss_kb: u64, + pub large_mapping_count_64m: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct MemoryTelemetryCheckpoint { + pub label: String, + pub elapsed_ms: u64, + pub process: ProcessMemorySnapshot, + pub rocksdb: RocksDbMemorySnapshot, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct MemoryTelemetrySummary { + pub checkpoints: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub object_graph: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub malloc_trim_probes: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct ObjectGraphMemorySummary { + pub captured_at_label: String, + pub total_estimated_bytes: u64, + pub sections: Vec, + pub notes: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct ObjectGraphMemorySection { + pub name: String, + pub item_count: u64, + pub shallow_bytes: u64, + pub heap_bytes: u64, + pub estimated_bytes: u64, + pub string_count: u64, + pub string_bytes: u64, + pub string_capacity_bytes: u64, + pub vec_count: u64, + pub vec_heap_bytes: u64, + pub vec_capacity_bytes: u64, + pub details: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct ObjectGraphMemoryMetric { + pub name: String, + pub value: u64, +} + +pub fn process_memory_snapshot(label: impl Into) -> ProcessMemorySnapshot { + let label = label.into(); + let mut snapshot = ProcessMemorySnapshot { + label, + vm_rss_kb: None, + vm_size_kb: None, + vm_data_kb: None, + vm_swap_kb: None, + rss_anon_kb: None, + rss_file_kb: None, + rss_shmem_kb: None, + threads: None, + fd_count: current_fd_count(), + smaps_rollup: None, + smaps_mapping_summary: None, + errors: Vec::new(), + }; + + match std::fs::read_to_string("/proc/self/status") { + Ok(status) => parse_status(&status, &mut snapshot), + Err(err) => snapshot + .errors + .push(format!("read /proc/self/status failed: {err}")), + } + + match std::fs::read_to_string("/proc/self/smaps_rollup") { + Ok(smaps) => snapshot.smaps_rollup = Some(parse_smaps_rollup(&smaps)), + Err(err) => snapshot + .errors + .push(format!("read /proc/self/smaps_rollup failed: {err}")), + } + + match std::fs::read_to_string("/proc/self/smaps") { + Ok(smaps) => snapshot.smaps_mapping_summary = Some(parse_smaps_mapping_summary(&smaps)), + Err(err) => snapshot + .errors + .push(format!("read /proc/self/smaps failed: {err}")), + } + + snapshot +} + +pub fn malloc_trim_probe() -> MallocTrimProbe { + #[cfg(all(target_os = "linux", target_env = "gnu"))] + { + MallocTrimProbe { + supported: true, + return_value: Some(unsafe { malloc_trim(0) }), + } + } + #[cfg(not(all(target_os = "linux", target_env = "gnu")))] + { + MallocTrimProbe { + supported: false, + return_value: None, + } + } +} + +#[cfg(all(target_os = "linux", target_env = "gnu"))] +unsafe extern "C" { + fn malloc_trim(pad: usize) -> i32; +} + +fn current_fd_count() -> Option { + std::fs::read_dir("/proc/self/fd") + .ok() + .map(|entries| entries.filter_map(Result::ok).count() as u64) +} + +fn parse_status(status: &str, snapshot: &mut ProcessMemorySnapshot) { + for line in status.lines() { + let Some((key, value)) = line.split_once(':') else { + continue; + }; + let parsed = parse_kb_or_plain_u64(value); + match key { + "VmRSS" => snapshot.vm_rss_kb = parsed, + "VmSize" => snapshot.vm_size_kb = parsed, + "VmData" => snapshot.vm_data_kb = parsed, + "VmSwap" => snapshot.vm_swap_kb = parsed, + "RssAnon" => snapshot.rss_anon_kb = parsed, + "RssFile" => snapshot.rss_file_kb = parsed, + "RssShmem" => snapshot.rss_shmem_kb = parsed, + "Threads" => snapshot.threads = parsed, + _ => {} + } + } +} + +fn parse_smaps_rollup(smaps: &str) -> SmapsRollupSnapshot { + let mut snapshot = SmapsRollupSnapshot::default(); + for line in smaps.lines() { + let Some((key, value)) = line.split_once(':') else { + continue; + }; + let parsed = parse_kb_or_plain_u64(value); + match key { + "Rss" => snapshot.rss_kb = parsed, + "Pss" => snapshot.pss_kb = parsed, + "Shared_Clean" => snapshot.shared_clean_kb = parsed, + "Shared_Dirty" => snapshot.shared_dirty_kb = parsed, + "Private_Clean" => snapshot.private_clean_kb = parsed, + "Private_Dirty" => snapshot.private_dirty_kb = parsed, + "Anonymous" => snapshot.anonymous_kb = parsed, + "Swap" => snapshot.swap_kb = parsed, + "SwapPss" => snapshot.swap_pss_kb = parsed, + _ => {} + } + } + snapshot +} + +fn parse_smaps_mapping_summary(smaps: &str) -> SmapsMappingSummary { + let mut summary = SmapsMappingSummary::default(); + let mut current_path = String::new(); + let mut current = SmapsMappingCategory::default(); + let mut have_mapping = false; + + for line in smaps.lines() { + if is_smaps_mapping_header(line) { + if have_mapping { + add_mapping(&mut summary, ¤t_path, ¤t); + } + current_path = smaps_header_path(line); + current = SmapsMappingCategory { + mappings: 1, + ..SmapsMappingCategory::default() + }; + have_mapping = true; + continue; + } + + if !have_mapping { + continue; + } + + let Some((key, value)) = line.split_once(':') else { + continue; + }; + let parsed = parse_kb_or_plain_u64(value).unwrap_or(0); + match key { + "Size" => current.size_kb = parsed, + "Rss" => current.rss_kb = parsed, + "Pss" => current.pss_kb = parsed, + "Private_Clean" => current.private_clean_kb = parsed, + "Private_Dirty" => current.private_dirty_kb = parsed, + "Anonymous" => current.anonymous_kb = parsed, + _ => {} + } + } + + if have_mapping { + add_mapping(&mut summary, ¤t_path, ¤t); + } + + summary +} + +fn is_smaps_mapping_header(line: &str) -> bool { + let mut parts = line.split_whitespace(); + let Some(range) = parts.next() else { + return false; + }; + let Some(perms) = parts.next() else { + return false; + }; + let Some((start, end)) = range.split_once('-') else { + return false; + }; + !start.is_empty() + && !end.is_empty() + && start.as_bytes().iter().all(u8::is_ascii_hexdigit) + && end.as_bytes().iter().all(u8::is_ascii_hexdigit) + && perms.len() == 4 + && perms + .as_bytes() + .iter() + .all(|b| matches!(b, b'r' | b'w' | b'x' | b's' | b'p' | b'-')) +} + +fn smaps_header_path(line: &str) -> String { + line.split_whitespace() + .skip(5) + .collect::>() + .join(" ") +} + +fn add_mapping(summary: &mut SmapsMappingSummary, path: &str, mapping: &SmapsMappingCategory) { + add_category(&mut summary.total, mapping); + match mapping_category(path) { + MappingCategory::Heap => add_category(&mut summary.heap, mapping), + MappingCategory::AnonymousMmap => add_category(&mut summary.anonymous_mmap, mapping), + MappingCategory::FileBacked => add_category(&mut summary.file_backed, mapping), + MappingCategory::Stack => add_category(&mut summary.stack, mapping), + MappingCategory::Special => add_category(&mut summary.special, mapping), + } +} + +fn add_category(target: &mut SmapsMappingCategory, source: &SmapsMappingCategory) { + target.mappings += source.mappings; + target.size_kb += source.size_kb; + target.rss_kb += source.rss_kb; + target.pss_kb += source.pss_kb; + target.private_clean_kb += source.private_clean_kb; + target.private_dirty_kb += source.private_dirty_kb; + target.anonymous_kb += source.anonymous_kb; + target.largest_mapping_rss_kb = target.largest_mapping_rss_kb.max(source.rss_kb); + if source.rss_kb >= 64 * 1024 { + target.large_mapping_count_64m += source.mappings; + } +} + +enum MappingCategory { + Heap, + AnonymousMmap, + FileBacked, + Stack, + Special, +} + +fn mapping_category(path: &str) -> MappingCategory { + if path == "[heap]" { + MappingCategory::Heap + } else if path.starts_with("[stack") { + MappingCategory::Stack + } else if path.is_empty() { + MappingCategory::AnonymousMmap + } else if path.starts_with('/') { + MappingCategory::FileBacked + } else { + MappingCategory::Special + } +} + +fn parse_kb_or_plain_u64(value: &str) -> Option { + value.split_whitespace().next()?.parse::().ok() +} diff --git a/src/storage.rs b/src/storage.rs index b4b228b..43aa3f4 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -5,6 +5,7 @@ mod pack; use std::collections::HashSet; use std::path::Path; +use base64::Engine; use rocksdb::{ColumnFamily, DB, Direction, IteratorMode, Options, WriteBatch}; use serde::{Deserialize, Serialize}; @@ -13,9 +14,8 @@ use crate::data_model::rc::{AsResourceSet, IpResourceSet}; use config::*; pub use config::{ - ALL_COLUMN_FAMILY_NAMES, CF_AUDIT_RULE_INDEX, CF_MANIFEST_REPLAY_META, CF_RAW_BY_HASH, - CF_REPOSITORY_VIEW, CF_RRDP_SOURCE, CF_RRDP_SOURCE_MEMBER, CF_RRDP_URI_OWNER, CF_VCIR, - column_family_descriptors, + ALL_COLUMN_FAMILY_NAMES, CF_MANIFEST_REPLAY_META, CF_RAW_BY_HASH, CF_REPOSITORY_VIEW, + CF_RRDP_SOURCE, CF_RRDP_SOURCE_MEMBER, CF_RRDP_URI_OWNER, CF_VCIR, column_family_descriptors, }; use keys::*; use pack::compute_sha256_32; @@ -49,6 +49,163 @@ pub struct RocksStore { external_repo_bytes: Option, } +fn process_vm_rss_kb() -> Option { + let status = std::fs::read_to_string("/proc/self/status").ok()?; + status.lines().find_map(|line| { + let rest = line.strip_prefix("VmRSS:")?; + rest.split_whitespace().next()?.parse::().ok() + }) +} + +const ROCKSDB_MEMORY_PROPERTY_NAMES: &[(&str, &str)] = &[ + ("cur_size_all_mem_tables", "rocksdb.cur-size-all-mem-tables"), + ("size_all_mem_tables", "rocksdb.size-all-mem-tables"), + ( + "estimate_table_readers_mem", + "rocksdb.estimate-table-readers-mem", + ), + ("block_cache_capacity", "rocksdb.block-cache-capacity"), + ("block_cache_usage", "rocksdb.block-cache-usage"), + ( + "block_cache_pinned_usage", + "rocksdb.block-cache-pinned-usage", + ), + ("num_snapshots", "rocksdb.num-snapshots"), + ("background_errors", "rocksdb.background-errors"), +]; + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct RocksDbMemoryProperties { + pub cur_size_all_mem_tables: Option, + pub size_all_mem_tables: Option, + pub estimate_table_readers_mem: Option, + pub block_cache_capacity: Option, + pub block_cache_usage: Option, + pub block_cache_pinned_usage: Option, + pub num_snapshots: Option, + pub background_errors: Option, + pub errors: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct RocksDbColumnFamilyMemoryProperties { + pub name: String, + pub properties: RocksDbMemoryProperties, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct RocksDbMemoryDbSnapshot { + pub label: String, + pub properties: RocksDbMemoryProperties, + pub column_families: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct RocksDbMemoryTotals { + pub cur_size_all_mem_tables: u64, + pub size_all_mem_tables: u64, + pub estimate_table_readers_mem: u64, + pub block_cache_capacity: u64, + pub block_cache_usage: u64, + pub block_cache_pinned_usage: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct RocksDbMemorySnapshot { + pub databases: Vec, + pub totals: RocksDbMemoryTotals, +} + +impl RocksDbMemoryTotals { + fn add_properties(&mut self, properties: &RocksDbMemoryProperties) { + self.cur_size_all_mem_tables += properties.cur_size_all_mem_tables.unwrap_or(0); + self.size_all_mem_tables += properties.size_all_mem_tables.unwrap_or(0); + self.estimate_table_readers_mem += properties.estimate_table_readers_mem.unwrap_or(0); + self.block_cache_capacity += properties.block_cache_capacity.unwrap_or(0); + self.block_cache_usage += properties.block_cache_usage.unwrap_or(0); + self.block_cache_pinned_usage += properties.block_cache_pinned_usage.unwrap_or(0); + } +} + +fn set_memory_property(properties: &mut RocksDbMemoryProperties, name: &str, value: u64) { + match name { + "cur_size_all_mem_tables" => properties.cur_size_all_mem_tables = Some(value), + "size_all_mem_tables" => properties.size_all_mem_tables = Some(value), + "estimate_table_readers_mem" => properties.estimate_table_readers_mem = Some(value), + "block_cache_capacity" => properties.block_cache_capacity = Some(value), + "block_cache_usage" => properties.block_cache_usage = Some(value), + "block_cache_pinned_usage" => properties.block_cache_pinned_usage = Some(value), + "num_snapshots" => properties.num_snapshots = Some(value), + "background_errors" => properties.background_errors = Some(value), + _ => {} + } +} + +fn parse_rocksdb_property_int(raw: Option) -> Option { + raw.and_then(|value| value.trim().parse::().ok()) +} + +fn memory_properties_for_db(db: &DB) -> RocksDbMemoryProperties { + let mut properties = RocksDbMemoryProperties::default(); + for (field_name, property_name) in ROCKSDB_MEMORY_PROPERTY_NAMES { + match db.property_value(*property_name) { + Ok(value) => { + if let Some(parsed) = parse_rocksdb_property_int(value) { + set_memory_property(&mut properties, field_name, parsed); + } + } + Err(err) => properties.errors.push(format!("{property_name}: {}", err)), + } + } + properties +} + +fn memory_properties_for_cf(db: &DB, cf_name: &'static str) -> RocksDbColumnFamilyMemoryProperties { + let mut properties = RocksDbMemoryProperties::default(); + let Some(cf) = db.cf_handle(cf_name) else { + properties + .errors + .push(format!("missing column family: {cf_name}")); + return RocksDbColumnFamilyMemoryProperties { + name: cf_name.to_string(), + properties, + }; + }; + for (field_name, property_name) in ROCKSDB_MEMORY_PROPERTY_NAMES { + match db.property_value_cf(cf, *property_name) { + Ok(value) => { + if let Some(parsed) = parse_rocksdb_property_int(value) { + set_memory_property(&mut properties, field_name, parsed); + } + } + Err(err) => properties.errors.push(format!("{property_name}: {}", err)), + } + } + RocksDbColumnFamilyMemoryProperties { + name: cf_name.to_string(), + properties, + } +} + +pub(crate) fn memory_db_snapshot_for_column_families( + label: impl Into, + db: &DB, + column_families: Option<&[&'static str]>, +) -> RocksDbMemoryDbSnapshot { + RocksDbMemoryDbSnapshot { + label: label.into(), + properties: memory_properties_for_db(db), + column_families: column_families + .map(|names| { + names + .iter() + .map(|name| memory_properties_for_cf(db, name)) + .collect() + }) + .unwrap_or_default(), + } +} + #[derive(Clone, Debug, PartialEq, Eq)] pub enum RrdpDeltaOp { Upsert { rsync_uri: String, bytes: Vec }, @@ -397,23 +554,285 @@ pub enum VcirOutputType { RouterKey, } +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum VcirSourceObjectType { + Roa, + Aspa, + RouterKey, + Other, +} + +impl VcirSourceObjectType { + pub fn as_str(self) -> &'static str { + match self { + Self::Roa => "roa", + Self::Aspa => "aspa", + Self::RouterKey => "router_key", + Self::Other => "other", + } + } +} + +struct FixedBytesVisitor; + +impl<'de, const N: usize> serde::de::Visitor<'de> for FixedBytesVisitor { + type Value = [u8; N]; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "{N} bytes") + } + + fn visit_bytes(self, value: &[u8]) -> Result + where + E: serde::de::Error, + { + if value.len() != N { + return Err(E::invalid_length(value.len(), &self)); + } + let mut out = [0u8; N]; + out.copy_from_slice(value); + Ok(out) + } + + fn visit_byte_buf(self, value: Vec) -> Result + where + E: serde::de::Error, + { + self.visit_bytes(&value) + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let mut out = [0u8; N]; + for (idx, slot) in out.iter_mut().enumerate() { + *slot = seq + .next_element()? + .ok_or_else(|| serde::de::Error::invalid_length(idx, &self))?; + } + Ok(out) + } +} + +fn deserialize_fixed_bytes<'de, D, const N: usize>(deserializer: D) -> Result<[u8; N], D::Error> +where + D: serde::Deserializer<'de>, +{ + deserializer.deserialize_bytes(FixedBytesVisitor::) +} + +mod serde_bytes_16 { + pub(super) fn serialize(value: &[u8; 16], serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_bytes(value) + } + + pub(super) fn deserialize<'de, D>(deserializer: D) -> Result<[u8; 16], D::Error> + where + D: serde::Deserializer<'de>, + { + super::deserialize_fixed_bytes::(deserializer) + } +} + +mod serde_bytes_32 { + pub(super) fn serialize(value: &[u8; 32], serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_bytes(value) + } + + pub(super) fn deserialize<'de, D>(deserializer: D) -> Result<[u8; 32], D::Error> + where + D: serde::Deserializer<'de>, + { + super::deserialize_fixed_bytes::(deserializer) + } +} + +struct ByteVecVisitor; + +impl<'de> serde::de::Visitor<'de> for ByteVecVisitor { + type Value = Vec; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("byte vector") + } + + fn visit_bytes(self, value: &[u8]) -> Result + where + E: serde::de::Error, + { + Ok(value.to_vec()) + } + + fn visit_byte_buf(self, value: Vec) -> Result + where + E: serde::de::Error, + { + Ok(value) + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let mut out = Vec::with_capacity(seq.size_hint().unwrap_or(0)); + while let Some(byte) = seq.next_element()? { + out.push(byte); + } + Ok(out) + } +} + +mod serde_byte_vec { + pub(super) fn serialize(value: &[u8], serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_bytes(value) + } + + pub(super) fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_bytes(super::ByteVecVisitor) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum VcirLocalOutputPayload { + Vrp { + asn: u32, + afi: crate::data_model::roa::RoaAfi, + prefix_len: u16, + #[serde(with = "serde_bytes_16")] + addr: [u8; 16], + max_length: u16, + }, + Aspa { + customer_as_id: u32, + provider_as_ids: Vec, + }, + RouterKey { + as_id: u32, + #[serde(with = "serde_byte_vec")] + ski: Vec, + #[serde(with = "serde_byte_vec")] + spki_der: Vec, + }, +} + +impl VcirLocalOutputPayload { + pub fn typed_body_bytes(&self) -> u64 { + match self { + Self::Vrp { .. } => 4 + 1 + 2 + 16 + 2, + Self::Aspa { + provider_as_ids, .. + } => 4 + (provider_as_ids.len() as u64 * 4), + Self::RouterKey { ski, spki_der, .. } => 4 + ski.len() as u64 + spki_der.len() as u64, + } + } +} + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct VcirLocalOutput { - pub output_id: String, + #[serde(rename = "t")] pub output_type: VcirOutputType, + #[serde(rename = "e")] pub item_effective_until: PackTime, + #[serde(rename = "u")] pub source_object_uri: String, - pub source_object_type: String, - pub source_object_hash: String, - pub source_ee_cert_hash: String, - pub payload_json: String, - pub rule_hash: String, - pub validation_path_hint: Vec, + #[serde(rename = "k")] + pub source_object_type: VcirSourceObjectType, + #[serde(rename = "h")] + #[serde(with = "serde_bytes_32")] + pub source_object_hash: [u8; 32], + #[serde(rename = "c")] + #[serde(with = "serde_bytes_32")] + pub source_ee_cert_hash: [u8; 32], + #[serde(rename = "p")] + pub payload: VcirLocalOutputPayload, + #[serde(rename = "r")] + #[serde(with = "serde_bytes_32")] + pub rule_hash: [u8; 32], } impl VcirLocalOutput { + pub fn output_id(&self) -> String { + self.rule_hash_hex() + } + + pub fn source_object_hash_hex(&self) -> String { + hex::encode(self.source_object_hash) + } + + pub fn source_ee_cert_hash_hex(&self) -> String { + hex::encode(self.source_ee_cert_hash) + } + + pub fn rule_hash_hex(&self) -> String { + hex::encode(self.rule_hash) + } + + pub fn source_object_type_name(&self) -> &'static str { + self.source_object_type.as_str() + } + + pub fn payload_json(&self) -> String { + match &self.payload { + VcirLocalOutputPayload::Vrp { + asn, + afi, + prefix_len, + addr, + max_length, + } => { + let prefix = match afi { + crate::data_model::roa::RoaAfi::Ipv4 => { + let ip = std::net::Ipv4Addr::new(addr[0], addr[1], addr[2], addr[3]); + format!("{ip}/{prefix_len}") + } + crate::data_model::roa::RoaAfi::Ipv6 => { + let ip = std::net::Ipv6Addr::from(*addr); + format!("{ip}/{prefix_len}") + } + }; + format!(r#"{{"asn":{asn},"max_length":{max_length},"prefix":"{prefix}"}}"#) + } + VcirLocalOutputPayload::Aspa { + customer_as_id, + provider_as_ids, + } => { + let providers = provider_as_ids + .iter() + .map(u32::to_string) + .collect::>() + .join(","); + format!(r#"{{"customer_as_id":{customer_as_id},"provider_as_ids":[{providers}]}}"#) + } + VcirLocalOutputPayload::RouterKey { + as_id, + ski, + spki_der, + } => { + let ski_hex = hex::encode(ski); + let spki_der_base64 = base64::engine::general_purpose::STANDARD.encode(spki_der); + format!( + r#"{{"as_id":{as_id},"ski_hex":"{ski_hex}","spki_der_base64":"{spki_der_base64}"}}"# + ) + } + } + } + pub fn validate_internal(&self) -> StorageResult<()> { - validate_non_empty("vcir.local_outputs[].output_id", &self.output_id)?; parse_time( "vcir.local_outputs[].item_effective_until", &self.item_effective_until, @@ -422,27 +841,30 @@ impl VcirLocalOutput { "vcir.local_outputs[].source_object_uri", &self.source_object_uri, )?; - validate_non_empty( - "vcir.local_outputs[].source_object_type", - &self.source_object_type, - )?; - validate_sha256_hex( - "vcir.local_outputs[].source_object_hash", - &self.source_object_hash, - )?; - validate_sha256_hex( - "vcir.local_outputs[].source_ee_cert_hash", - &self.source_ee_cert_hash, - )?; - validate_sha256_hex("vcir.local_outputs[].rule_hash", &self.rule_hash)?; - validate_non_empty("vcir.local_outputs[].payload_json", &self.payload_json)?; - for hint in &self.validation_path_hint { - validate_non_empty("vcir.local_outputs[].validation_path_hint[]", hint)?; - } + validate_local_output_type_matches_payload(self)?; Ok(()) } } +fn validate_local_output_type_matches_payload(output: &VcirLocalOutput) -> StorageResult<()> { + let matches_payload = matches!( + (&output.output_type, &output.payload), + (VcirOutputType::Vrp, VcirLocalOutputPayload::Vrp { .. }) + | (VcirOutputType::Aspa, VcirLocalOutputPayload::Aspa { .. }) + | ( + VcirOutputType::RouterKey, + VcirLocalOutputPayload::RouterKey { .. } + ) + ); + if !matches_payload { + return Err(StorageError::InvalidData { + entity: "vcir.local_outputs[]", + detail: "output_type must match payload variant".to_string(), + }); + } + Ok(()) +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum VcirArtifactRole { @@ -480,11 +902,17 @@ pub enum VcirArtifactValidationStatus { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct VcirRelatedArtifact { + #[serde(rename = "r")] pub artifact_role: VcirArtifactRole, + #[serde(rename = "k")] pub artifact_kind: VcirArtifactKind, + #[serde(rename = "u")] pub uri: Option, + #[serde(rename = "h")] pub sha256: String, + #[serde(rename = "t")] pub object_type: Option, + #[serde(rename = "s")] pub validation_status: VcirArtifactValidationStatus, } @@ -500,19 +928,29 @@ impl VcirRelatedArtifact { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct VcirSummary { + #[serde(rename = "v")] pub local_vrp_count: u32, + #[serde(rename = "a")] pub local_aspa_count: u32, + #[serde(rename = "r")] pub local_router_key_count: u32, + #[serde(rename = "c")] pub child_count: u32, + #[serde(rename = "o")] pub accepted_object_count: u32, + #[serde(rename = "x")] pub rejected_object_count: u32, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct VcirAuditSummary { + #[serde(rename = "f")] pub failed_fetch_eligible: bool, + #[serde(rename = "r")] pub last_failed_fetch_reason: Option, + #[serde(rename = "w")] pub warning_count: u32, + #[serde(rename = "a")] pub audit_flags: Vec, } @@ -530,25 +968,369 @@ impl VcirAuditSummary { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct ValidatedCaInstanceResult { + #[serde(rename = "m")] pub manifest_rsync_uri: String, + #[serde(rename = "pm")] pub parent_manifest_rsync_uri: Option, + #[serde(rename = "tal")] pub tal_id: String, + #[serde(rename = "subj")] pub ca_subject_name: String, + #[serde(rename = "ski")] pub ca_ski: String, + #[serde(rename = "aki")] pub issuer_ski: String, + #[serde(rename = "vt")] pub last_successful_validation_time: PackTime, + #[serde(rename = "cm")] pub current_manifest_rsync_uri: String, + #[serde(rename = "crl")] pub current_crl_rsync_uri: String, + #[serde(rename = "mm")] pub validated_manifest_meta: ValidatedManifestMeta, + #[serde(rename = "ccr")] pub ccr_manifest_projection: VcirCcrManifestProjection, + #[serde(rename = "g")] pub instance_gate: VcirInstanceGate, + #[serde(rename = "ch")] pub child_entries: Vec, + #[serde(rename = "lo")] pub local_outputs: Vec, + #[serde(rename = "ra")] pub related_artifacts: Vec, + #[serde(rename = "s")] pub summary: VcirSummary, + #[serde(rename = "as")] pub audit_summary: VcirAuditSummary, } +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct VcirFieldSizeBreakdown { + pub local_output_count: u64, + pub local_output_source_uri_bytes: u64, + pub local_output_source_type_bytes: u64, + pub local_output_source_hash_hex_bytes: u64, + pub local_output_source_ee_hash_hex_bytes: u64, + pub local_output_payload_json_bytes: u64, + pub local_output_rule_hash_hex_bytes: u64, + pub local_output_source_hash_binary_bytes: u64, + pub local_output_source_ee_hash_binary_bytes: u64, + pub local_output_payload_typed_body_bytes: u64, + pub local_output_rule_hash_binary_bytes: u64, + pub related_artifact_count: u64, + pub related_artifact_uri_bytes: u64, + pub related_artifact_hash_hex_bytes: u64, + pub related_artifact_type_bytes: u64, + pub child_entry_count: u64, + pub child_entry_uri_bytes: u64, + pub child_entry_hash_hex_bytes: u64, +} + +fn serialized_cbor_len(value: &T) -> u64 { + serde_cbor::to_vec(value) + .map(|bytes| bytes.len() as u64) + .unwrap_or(0) +} + +impl VcirFieldSizeBreakdown { + pub fn from_vcir(vcir: &ValidatedCaInstanceResult) -> Self { + let mut out = Self::default(); + out.local_output_count = vcir.local_outputs.len() as u64; + for local in &vcir.local_outputs { + out.local_output_source_uri_bytes += local.source_object_uri.len() as u64; + out.local_output_source_type_bytes += local.source_object_type_name().len() as u64; + out.local_output_source_hash_hex_bytes += 64; + out.local_output_source_ee_hash_hex_bytes += 64; + out.local_output_payload_json_bytes += local.payload_json().len() as u64; + out.local_output_rule_hash_hex_bytes += 64; + out.local_output_source_hash_binary_bytes += 32; + out.local_output_source_ee_hash_binary_bytes += 32; + out.local_output_payload_typed_body_bytes += local.payload.typed_body_bytes(); + out.local_output_rule_hash_binary_bytes += 32; + } + out.related_artifact_count = vcir.related_artifacts.len() as u64; + for artifact in &vcir.related_artifacts { + if let Some(uri) = &artifact.uri { + out.related_artifact_uri_bytes += uri.len() as u64; + } + out.related_artifact_hash_hex_bytes += artifact.sha256.len() as u64; + if let Some(object_type) = &artifact.object_type { + out.related_artifact_type_bytes += object_type.len() as u64; + } + } + out.child_entry_count = vcir.child_entries.len() as u64; + for child in &vcir.child_entries { + out.child_entry_uri_bytes += child.child_manifest_rsync_uri.len() as u64 + + child.child_cert_rsync_uri.len() as u64 + + child.child_rsync_base_uri.len() as u64 + + child.child_publication_point_rsync_uri.len() as u64 + + child + .child_rrdp_notification_uri + .as_ref() + .map(|uri| uri.len() as u64) + .unwrap_or(0); + out.child_entry_hash_hex_bytes += + child.child_cert_hash.len() as u64 + child.child_ski.len() as u64; + } + out + } + + pub fn add_assign(&mut self, other: &Self) { + self.local_output_count += other.local_output_count; + self.local_output_source_uri_bytes += other.local_output_source_uri_bytes; + self.local_output_source_type_bytes += other.local_output_source_type_bytes; + self.local_output_source_hash_hex_bytes += other.local_output_source_hash_hex_bytes; + self.local_output_source_ee_hash_hex_bytes += other.local_output_source_ee_hash_hex_bytes; + self.local_output_payload_json_bytes += other.local_output_payload_json_bytes; + self.local_output_rule_hash_hex_bytes += other.local_output_rule_hash_hex_bytes; + self.local_output_source_hash_binary_bytes += other.local_output_source_hash_binary_bytes; + self.local_output_source_ee_hash_binary_bytes += + other.local_output_source_ee_hash_binary_bytes; + self.local_output_payload_typed_body_bytes += other.local_output_payload_typed_body_bytes; + self.local_output_rule_hash_binary_bytes += other.local_output_rule_hash_binary_bytes; + self.related_artifact_count += other.related_artifact_count; + self.related_artifact_uri_bytes += other.related_artifact_uri_bytes; + self.related_artifact_hash_hex_bytes += other.related_artifact_hash_hex_bytes; + self.related_artifact_type_bytes += other.related_artifact_type_bytes; + self.child_entry_count += other.child_entry_count; + self.child_entry_uri_bytes += other.child_entry_uri_bytes; + self.child_entry_hash_hex_bytes += other.child_entry_hash_hex_bytes; + } + + pub fn local_output_old_projection_bytes(&self) -> u64 { + self.local_output_source_type_bytes + + self.local_output_source_hash_hex_bytes + + self.local_output_source_ee_hash_hex_bytes + + self.local_output_payload_json_bytes + + self.local_output_rule_hash_hex_bytes + } + + pub fn local_output_typed_projection_bytes(&self) -> u64 { + self.local_output_count + + self.local_output_source_hash_binary_bytes + + self.local_output_source_ee_hash_binary_bytes + + self.local_output_payload_typed_body_bytes + + self.local_output_rule_hash_binary_bytes + } + + pub fn local_output_projection_saved_bytes(&self) -> u64 { + self.local_output_old_projection_bytes() + .saturating_sub(self.local_output_typed_projection_bytes()) + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct VcirReplaceTimingBreakdown { + pub validate_ms: u64, + pub vcir_encode_ms: u64, + pub vcir_value_bytes: u64, + pub replay_meta_encode_ms: u64, + pub replay_meta_value_bytes: u64, + pub batch_build_ms: u64, + pub write_batch_ms: u64, + pub total_encoded_bytes: u64, + pub field_sizes: VcirFieldSizeBreakdown, + pub rss_before_kb: Option, + pub rss_after_validate_kb: Option, + pub rss_after_vcir_encode_kb: Option, + pub rss_after_replay_meta_encode_kb: Option, + pub rss_after_write_batch_kb: Option, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct VcirStorageSummary { + pub entry_count: u64, + pub vcir_value_bytes: u64, + pub vcir_value_bytes_max: u64, + pub vcir_value_bytes_max_manifest_rsync_uri: Option, + pub core_fields: VcirCoreFieldSizeBreakdown, + pub ccr_projection: VcirCcrProjectionSizeBreakdown, + pub child_resources: VcirChildResourceSizeBreakdown, + pub field_sizes: VcirFieldSizeBreakdown, + pub local_output_old_projection_bytes: u64, + pub local_output_typed_projection_bytes: u64, + pub local_output_projection_saved_bytes: u64, + pub top_entries_by_vcir_value_bytes: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct VcirStorageEntrySummary { + pub manifest_rsync_uri: String, + pub vcir_value_bytes: u64, + pub local_vrp_count: u32, + pub local_aspa_count: u32, + pub local_router_key_count: u32, + pub accepted_object_count: u32, + pub rejected_object_count: u32, + pub child_count: u32, + pub core_fields: VcirCoreFieldSizeBreakdown, + pub ccr_projection: VcirCcrProjectionSizeBreakdown, + pub child_resources: VcirChildResourceSizeBreakdown, + pub field_sizes: VcirFieldSizeBreakdown, + pub local_output_old_projection_bytes: u64, + pub local_output_typed_projection_bytes: u64, + pub local_output_projection_saved_bytes: u64, +} + +impl VcirStorageEntrySummary { + fn from_vcir(vcir: &ValidatedCaInstanceResult, vcir_value_bytes: u64) -> Self { + let field_sizes = VcirFieldSizeBreakdown::from_vcir(vcir); + Self { + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + vcir_value_bytes, + local_vrp_count: vcir.summary.local_vrp_count, + local_aspa_count: vcir.summary.local_aspa_count, + local_router_key_count: vcir.summary.local_router_key_count, + accepted_object_count: vcir.summary.accepted_object_count, + rejected_object_count: vcir.summary.rejected_object_count, + child_count: vcir.summary.child_count, + core_fields: VcirCoreFieldSizeBreakdown::from_vcir(vcir), + ccr_projection: VcirCcrProjectionSizeBreakdown::from_projection( + &vcir.ccr_manifest_projection, + ), + child_resources: VcirChildResourceSizeBreakdown::from_vcir(vcir), + local_output_old_projection_bytes: field_sizes.local_output_old_projection_bytes(), + local_output_typed_projection_bytes: field_sizes.local_output_typed_projection_bytes(), + local_output_projection_saved_bytes: field_sizes + .local_output_projection_saved_bytes(), + field_sizes, + } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct VcirCoreFieldSizeBreakdown { + pub manifest_rsync_uri_bytes: u64, + pub parent_manifest_rsync_uri_bytes: u64, + pub tal_id_bytes: u64, + pub ca_subject_name_bytes: u64, + pub ca_ski_bytes: u64, + pub issuer_ski_bytes: u64, + pub current_manifest_rsync_uri_bytes: u64, + pub current_crl_rsync_uri_bytes: u64, + pub validated_manifest_number_bytes: u64, +} + +impl VcirCoreFieldSizeBreakdown { + fn from_vcir(vcir: &ValidatedCaInstanceResult) -> Self { + Self { + manifest_rsync_uri_bytes: vcir.manifest_rsync_uri.len() as u64, + parent_manifest_rsync_uri_bytes: vcir + .parent_manifest_rsync_uri + .as_ref() + .map(|uri| uri.len() as u64) + .unwrap_or(0), + tal_id_bytes: vcir.tal_id.len() as u64, + ca_subject_name_bytes: vcir.ca_subject_name.len() as u64, + ca_ski_bytes: vcir.ca_ski.len() as u64, + issuer_ski_bytes: vcir.issuer_ski.len() as u64, + current_manifest_rsync_uri_bytes: vcir.current_manifest_rsync_uri.len() as u64, + current_crl_rsync_uri_bytes: vcir.current_crl_rsync_uri.len() as u64, + validated_manifest_number_bytes: vcir + .validated_manifest_meta + .validated_manifest_number + .len() as u64, + } + } + + fn add_assign(&mut self, other: &Self) { + self.manifest_rsync_uri_bytes += other.manifest_rsync_uri_bytes; + self.parent_manifest_rsync_uri_bytes += other.parent_manifest_rsync_uri_bytes; + self.tal_id_bytes += other.tal_id_bytes; + self.ca_subject_name_bytes += other.ca_subject_name_bytes; + self.ca_ski_bytes += other.ca_ski_bytes; + self.issuer_ski_bytes += other.issuer_ski_bytes; + self.current_manifest_rsync_uri_bytes += other.current_manifest_rsync_uri_bytes; + self.current_crl_rsync_uri_bytes += other.current_crl_rsync_uri_bytes; + self.validated_manifest_number_bytes += other.validated_manifest_number_bytes; + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct VcirChildResourceSizeBreakdown { + pub effective_ip_resource_cbor_bytes: u64, + pub effective_as_resource_cbor_bytes: u64, +} + +impl VcirChildResourceSizeBreakdown { + fn from_vcir(vcir: &ValidatedCaInstanceResult) -> Self { + let mut out = Self::default(); + for child in &vcir.child_entries { + out.effective_ip_resource_cbor_bytes += + serialized_cbor_len(&child.child_effective_ip_resources); + out.effective_as_resource_cbor_bytes += + serialized_cbor_len(&child.child_effective_as_resources); + } + out + } + + fn add_assign(&mut self, other: &Self) { + self.effective_ip_resource_cbor_bytes += other.effective_ip_resource_cbor_bytes; + self.effective_as_resource_cbor_bytes += other.effective_as_resource_cbor_bytes; + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct VcirCcrProjectionSizeBreakdown { + pub manifest_rsync_uri_bytes: u64, + pub manifest_sha256_bytes: u64, + pub manifest_ee_aki_bytes: u64, + pub manifest_number_bytes: u64, + pub manifest_sia_locations_count: u64, + pub manifest_sia_locations_der_bytes: u64, + pub subordinate_ski_count: u64, + pub subordinate_ski_bytes: u64, +} + +impl VcirCcrProjectionSizeBreakdown { + fn from_projection(projection: &VcirCcrManifestProjection) -> Self { + Self { + manifest_rsync_uri_bytes: projection.manifest_rsync_uri.len() as u64, + manifest_sha256_bytes: projection.manifest_sha256.len() as u64, + manifest_ee_aki_bytes: projection.manifest_ee_aki.len() as u64, + manifest_number_bytes: projection.manifest_number_be.len() as u64, + manifest_sia_locations_count: projection.manifest_sia_locations_der.len() as u64, + manifest_sia_locations_der_bytes: projection + .manifest_sia_locations_der + .iter() + .map(|location| location.len() as u64) + .sum(), + subordinate_ski_count: projection.subordinate_skis.len() as u64, + subordinate_ski_bytes: projection + .subordinate_skis + .iter() + .map(|ski| ski.len() as u64) + .sum(), + } + } + + fn add_assign(&mut self, other: &Self) { + self.manifest_rsync_uri_bytes += other.manifest_rsync_uri_bytes; + self.manifest_sha256_bytes += other.manifest_sha256_bytes; + self.manifest_ee_aki_bytes += other.manifest_ee_aki_bytes; + self.manifest_number_bytes += other.manifest_number_bytes; + self.manifest_sia_locations_count += other.manifest_sia_locations_count; + self.manifest_sia_locations_der_bytes += other.manifest_sia_locations_der_bytes; + self.subordinate_ski_count += other.subordinate_ski_count; + self.subordinate_ski_bytes += other.subordinate_ski_bytes; + } +} + +fn push_top_vcir_storage_entry( + entries: &mut Vec, + entry: VcirStorageEntrySummary, +) { + const TOP_N: usize = 20; + entries.push(entry); + entries.sort_by(|left, right| { + right + .vcir_value_bytes + .cmp(&left.vcir_value_bytes) + .then_with(|| left.manifest_rsync_uri.cmp(&right.manifest_rsync_uri)) + }); + entries.truncate(TOP_N); +} + impl ValidatedCaInstanceResult { pub fn validate_internal(&self) -> StorageResult<()> { validate_non_empty("vcir.manifest_rsync_uri", &self.manifest_rsync_uri)?; @@ -611,24 +1393,32 @@ impl ValidatedCaInstanceResult { } } - let mut output_ids = HashSet::with_capacity(self.local_outputs.len()); let mut vrp_count = 0u32; let mut aspa_count = 0u32; let mut router_key_count = 0u32; for output in &self.local_outputs { output.validate_internal()?; - if !output_ids.insert(output.output_id.as_str()) { - return Err(StorageError::InvalidData { - entity: "vcir", - detail: format!("duplicate output_id: {}", output.output_id), - }); - } match output.output_type { VcirOutputType::Vrp => vrp_count += 1, VcirOutputType::Aspa => aspa_count += 1, VcirOutputType::RouterKey => router_key_count += 1, } } + let mut output_ids = self + .local_outputs + .iter() + .map(VcirLocalOutput::output_id) + .collect::>(); + output_ids.sort_unstable(); + if let Some(duplicate) = output_ids + .windows(2) + .find_map(|pair| (pair[0] == pair[1]).then(|| pair[0].clone())) + { + return Err(StorageError::InvalidData { + entity: "vcir", + detail: format!("duplicate output_id: {duplicate}"), + }); + } if self.summary.local_vrp_count != vrp_count { return Err(StorageError::InvalidData { entity: "vcir.summary", @@ -675,59 +1465,6 @@ impl ValidatedCaInstanceResult { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum AuditRuleKind { - Roa, - Aspa, - RouterKey, -} - -impl AuditRuleKind { - fn key_prefix(self) -> &'static str { - match self { - Self::Roa => AUDIT_ROA_RULE_KEY_PREFIX, - Self::Aspa => AUDIT_ASPA_RULE_KEY_PREFIX, - Self::RouterKey => AUDIT_ROUTER_KEY_RULE_KEY_PREFIX, - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct AuditRuleIndexEntry { - pub kind: AuditRuleKind, - pub rule_hash: String, - pub manifest_rsync_uri: String, - pub source_object_uri: String, - pub source_object_hash: String, - pub output_id: String, - pub item_effective_until: PackTime, -} - -impl AuditRuleIndexEntry { - pub fn validate_internal(&self) -> StorageResult<()> { - validate_sha256_hex("audit_rule_index.rule_hash", &self.rule_hash)?; - validate_non_empty( - "audit_rule_index.manifest_rsync_uri", - &self.manifest_rsync_uri, - )?; - validate_non_empty( - "audit_rule_index.source_object_uri", - &self.source_object_uri, - )?; - validate_sha256_hex( - "audit_rule_index.source_object_hash", - &self.source_object_hash, - )?; - validate_non_empty("audit_rule_index.output_id", &self.output_id)?; - parse_time( - "audit_rule_index.item_effective_until", - &self.item_effective_until, - )?; - Ok(()) - } -} - #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum RrdpSourceSyncState { @@ -853,7 +1590,8 @@ impl RocksStore { base_opts.create_if_missing(true); base_opts.create_missing_column_families(true); let blob_mode = work_db_blob_mode_from_env(); - configure_work_db_options(&mut base_opts, blob_mode); + let memory_profile = work_db_memory_profile_from_env(); + configure_work_db_options(&mut base_opts, blob_mode, memory_profile); let db = DB::open_cf_descriptors( &base_opts, @@ -903,6 +1641,27 @@ impl RocksStore { self.external_repo_bytes.as_ref() } + pub fn memory_snapshot(&self) -> RocksDbMemorySnapshot { + let mut databases = Vec::new(); + databases.push(memory_db_snapshot_for_column_families( + "work-db", + &self.db, + Some(ALL_COLUMN_FAMILY_NAMES), + )); + if let Some(raw_store) = self.external_raw_store.as_ref() { + databases.push(raw_store.memory_snapshot("raw-store.db")); + } + if let Some(repo_bytes) = self.external_repo_bytes.as_ref() { + databases.push(repo_bytes.memory_snapshot("repo-bytes.db")); + } + + let mut totals = RocksDbMemoryTotals::default(); + for db in &databases { + totals.add_properties(&db.properties); + } + RocksDbMemorySnapshot { databases, totals } + } + fn cf(&self, name: &'static str) -> StorageResult<&ColumnFamily> { self.db .cf_handle(name) @@ -1233,56 +1992,52 @@ impl RocksStore { self.write_batch(batch) } - pub fn replace_vcir_and_audit_rule_indexes( + pub fn replace_vcir_and_manifest_replay_meta( &self, - previous: Option<&ValidatedCaInstanceResult>, vcir: &ValidatedCaInstanceResult, - ) -> StorageResult<()> { + ) -> StorageResult { + let mut timing = VcirReplaceTimingBreakdown { + rss_before_kb: process_vm_rss_kb(), + ..VcirReplaceTimingBreakdown::default() + }; + + let validate_started = std::time::Instant::now(); vcir.validate_internal()?; + timing.validate_ms = validate_started.elapsed().as_millis() as u64; + timing.rss_after_validate_kb = process_vm_rss_kb(); + timing.field_sizes = VcirFieldSizeBreakdown::from_vcir(vcir); + + let batch_build_started = std::time::Instant::now(); let vcir_cf = self.cf(CF_VCIR)?; let replay_cf = self.cf(CF_MANIFEST_REPLAY_META)?; - let audit_cf = self.cf(CF_AUDIT_RULE_INDEX)?; let mut batch = WriteBatch::default(); let vcir_key = vcir_key(&vcir.manifest_rsync_uri); + let vcir_encode_started = std::time::Instant::now(); let vcir_value = encode_cbor(vcir, "vcir")?; + timing.vcir_encode_ms = vcir_encode_started.elapsed().as_millis() as u64; + timing.vcir_value_bytes = vcir_value.len() as u64; batch.put_cf(vcir_cf, vcir_key.as_bytes(), vcir_value); + timing.rss_after_vcir_encode_kb = process_vm_rss_kb(); + + let replay_meta_encode_started = std::time::Instant::now(); let replay_meta = ManifestReplayMeta::from_vcir(vcir); replay_meta.validate_internal()?; let replay_key = manifest_replay_meta_key(&replay_meta.manifest_rsync_uri); let replay_value = encode_cbor(&replay_meta, "manifest_replay_meta")?; + timing.replay_meta_encode_ms = replay_meta_encode_started.elapsed().as_millis() as u64; + timing.replay_meta_value_bytes = replay_value.len() as u64; batch.put_cf(replay_cf, replay_key.as_bytes(), replay_value); + timing.rss_after_replay_meta_encode_kb = process_vm_rss_kb(); - if let Some(previous) = previous { - for output in &previous.local_outputs { - let Some(kind) = audit_rule_kind_for_output_type(output.output_type) else { - continue; - }; - let key = audit_rule_key(kind, &output.rule_hash); - batch.delete_cf(audit_cf, key.as_bytes()); - } - } + timing.total_encoded_bytes = timing.vcir_value_bytes + timing.replay_meta_value_bytes; + timing.batch_build_ms = batch_build_started.elapsed().as_millis() as u64; - for output in &vcir.local_outputs { - let Some(kind) = audit_rule_kind_for_output_type(output.output_type) else { - continue; - }; - let entry = AuditRuleIndexEntry { - kind, - rule_hash: output.rule_hash.clone(), - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - source_object_uri: output.source_object_uri.clone(), - source_object_hash: output.source_object_hash.clone(), - output_id: output.output_id.clone(), - item_effective_until: output.item_effective_until.clone(), - }; - entry.validate_internal()?; - let key = audit_rule_key(kind, &entry.rule_hash); - let value = encode_cbor(&entry, "audit_rule_index")?; - batch.put_cf(audit_cf, key.as_bytes(), value); - } - - self.write_batch(batch) + let write_batch_started = std::time::Instant::now(); + self.write_batch(batch)?; + timing.write_batch_ms = write_batch_started.elapsed().as_millis() as u64; + timing.rss_after_write_batch_kb = process_vm_rss_kb(); + Ok(timing) } pub fn get_vcir( @@ -1334,6 +2089,44 @@ impl RocksStore { Ok(out) } + pub fn summarize_vcir_storage(&self) -> StorageResult { + let cf = self.cf(CF_VCIR)?; + let mode = IteratorMode::Start; + let mut summary = VcirStorageSummary::default(); + for res in self.db.iterator_cf(cf, mode) { + let (_key, bytes) = res.map_err(|e| StorageError::RocksDb(e.to_string()))?; + let vcir = decode_cbor::(&bytes, "vcir")?; + vcir.validate_internal()?; + summary.entry_count += 1; + let value_bytes = bytes.len() as u64; + summary.vcir_value_bytes += value_bytes; + if value_bytes > summary.vcir_value_bytes_max { + summary.vcir_value_bytes_max = value_bytes; + summary.vcir_value_bytes_max_manifest_rsync_uri = + Some(vcir.manifest_rsync_uri.clone()); + } + let entry_summary = VcirStorageEntrySummary::from_vcir(&vcir, value_bytes); + summary.core_fields.add_assign(&entry_summary.core_fields); + summary + .ccr_projection + .add_assign(&entry_summary.ccr_projection); + summary + .child_resources + .add_assign(&entry_summary.child_resources); + summary + .field_sizes + .add_assign(&entry_summary.field_sizes); + push_top_vcir_storage_entry(&mut summary.top_entries_by_vcir_value_bytes, entry_summary); + } + summary.local_output_old_projection_bytes = + summary.field_sizes.local_output_old_projection_bytes(); + summary.local_output_typed_projection_bytes = + summary.field_sizes.local_output_typed_projection_bytes(); + summary.local_output_projection_saved_bytes = + summary.field_sizes.local_output_projection_saved_bytes(); + Ok(summary) + } + pub fn delete_vcir(&self, manifest_rsync_uri: &str) -> StorageResult<()> { let vcir_cf = self.cf(CF_VCIR)?; let replay_cf = self.cf(CF_MANIFEST_REPLAY_META)?; @@ -1345,49 +2138,6 @@ impl RocksStore { self.write_batch(batch) } - pub fn put_audit_rule_index_entry(&self, entry: &AuditRuleIndexEntry) -> StorageResult<()> { - entry.validate_internal()?; - let cf = self.cf(CF_AUDIT_RULE_INDEX)?; - let key = audit_rule_key(entry.kind, &entry.rule_hash); - let value = encode_cbor(entry, "audit_rule_index")?; - self.db - .put_cf(cf, key.as_bytes(), value) - .map_err(|e| StorageError::RocksDb(e.to_string()))?; - Ok(()) - } - - pub fn get_audit_rule_index_entry( - &self, - kind: AuditRuleKind, - rule_hash: &str, - ) -> StorageResult> { - let cf = self.cf(CF_AUDIT_RULE_INDEX)?; - let key = audit_rule_key(kind, rule_hash); - let Some(bytes) = self - .db - .get_cf(cf, key.as_bytes()) - .map_err(|e| StorageError::RocksDb(e.to_string()))? - else { - return Ok(None); - }; - let entry = decode_cbor::(&bytes, "audit_rule_index")?; - entry.validate_internal()?; - Ok(Some(entry)) - } - - pub fn delete_audit_rule_index_entry( - &self, - kind: AuditRuleKind, - rule_hash: &str, - ) -> StorageResult<()> { - let cf = self.cf(CF_AUDIT_RULE_INDEX)?; - let key = audit_rule_key(kind, rule_hash); - self.db - .delete_cf(cf, key.as_bytes()) - .map_err(|e| StorageError::RocksDb(e.to_string()))?; - Ok(()) - } - pub fn put_rrdp_source_record(&self, record: &RrdpSourceRecord) -> StorageResult<()> { record.validate_internal()?; let cf = self.cf(CF_RRDP_SOURCE)?; diff --git a/src/storage/config.rs b/src/storage/config.rs index 883e59a..80eafae 100644 --- a/src/storage/config.rs +++ b/src/storage/config.rs @@ -5,7 +5,6 @@ pub const CF_RAW_BY_HASH: &str = "raw_by_hash"; pub const CF_RAW_BLOB: &str = "raw_blob"; pub const CF_VCIR: &str = "vcir"; pub const CF_MANIFEST_REPLAY_META: &str = "manifest_replay_meta"; -pub const CF_AUDIT_RULE_INDEX: &str = "audit_rule_index"; pub const CF_RRDP_SOURCE: &str = "rrdp_source"; pub const CF_RRDP_SOURCE_MEMBER: &str = "rrdp_source_member"; pub const CF_RRDP_URI_OWNER: &str = "rrdp_uri_owner"; @@ -16,7 +15,6 @@ pub const ALL_COLUMN_FAMILY_NAMES: &[&str] = &[ CF_RAW_BLOB, CF_VCIR, CF_MANIFEST_REPLAY_META, - CF_AUDIT_RULE_INDEX, CF_RRDP_SOURCE, CF_RRDP_SOURCE_MEMBER, CF_RRDP_URI_OWNER, @@ -27,14 +25,12 @@ pub(super) const RAW_BY_HASH_KEY_PREFIX: &str = "rawbyhash:"; pub(super) const RAW_BLOB_KEY_PREFIX: &str = "rawblob:"; pub(super) const VCIR_KEY_PREFIX: &str = "vcir:"; pub(super) const MANIFEST_REPLAY_META_KEY_PREFIX: &str = "manifest_replay_meta:"; -pub(super) const AUDIT_ROA_RULE_KEY_PREFIX: &str = "audit:roa_rule:"; -pub(super) const AUDIT_ASPA_RULE_KEY_PREFIX: &str = "audit:aspa_rule:"; -pub(super) const AUDIT_ROUTER_KEY_RULE_KEY_PREFIX: &str = "audit:router_key_rule:"; pub(super) const RRDP_SOURCE_KEY_PREFIX: &str = "rrdp_source:"; pub(super) const RRDP_SOURCE_MEMBER_KEY_PREFIX: &str = "rrdp_source_member:"; pub(super) const RRDP_URI_OWNER_KEY_PREFIX: &str = "rrdp_uri_owner:"; const WORK_DB_BLOB_MODE_ENV: &str = "RPKI_WORK_DB_BLOB_MODE"; +const WORK_DB_MEMORY_PROFILE_ENV: &str = "RPKI_WORK_DB_MEMORY_PROFILE"; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(super) enum WorkDbBlobMode { @@ -43,6 +39,12 @@ pub(super) enum WorkDbBlobMode { Lz4, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum WorkDbMemoryProfile { + Default, + Compact, +} + pub(super) fn parse_work_db_blob_mode(raw: &str) -> Option { match raw.trim().to_ascii_lowercase().as_str() { "" | "default" => Some(default_work_db_blob_mode()), @@ -74,8 +76,36 @@ pub(super) fn work_db_blob_mode_from_env() -> WorkDbBlobMode { } } -pub(super) fn configure_work_db_options(opts: &mut Options, blob_mode: WorkDbBlobMode) { +pub(super) fn parse_work_db_memory_profile(raw: &str) -> Option { + match raw.trim().to_ascii_lowercase().as_str() { + "" | "default" | "off" | "none" => Some(WorkDbMemoryProfile::Default), + "compact" | "low" | "low_memory" | "low-memory" => Some(WorkDbMemoryProfile::Compact), + _ => None, + } +} + +pub(super) fn work_db_memory_profile_from_env() -> WorkDbMemoryProfile { + let Ok(raw) = std::env::var(WORK_DB_MEMORY_PROFILE_ENV) else { + return WorkDbMemoryProfile::Default; + }; + match parse_work_db_memory_profile(&raw) { + Some(profile) => profile, + None => { + eprintln!( + "warning: unsupported {WORK_DB_MEMORY_PROFILE_ENV}={raw:?}; using default work-db memory profile" + ); + WorkDbMemoryProfile::Default + } + } +} + +pub(super) fn configure_work_db_options( + opts: &mut Options, + blob_mode: WorkDbBlobMode, + memory_profile: WorkDbMemoryProfile, +) { opts.set_compression_type(DBCompressionType::Lz4); + apply_work_db_memory_profile(opts, memory_profile); match blob_mode { WorkDbBlobMode::Current => enable_blobdb_current(opts), WorkDbBlobMode::Disabled => {} @@ -86,9 +116,9 @@ pub(super) fn configure_work_db_options(opts: &mut Options, blob_mode: WorkDbBlo } } -pub(super) fn cf_opts(blob_mode: WorkDbBlobMode) -> Options { +pub(super) fn cf_opts(blob_mode: WorkDbBlobMode, memory_profile: WorkDbMemoryProfile) -> Options { let mut opts = Options::default(); - configure_work_db_options(&mut opts, blob_mode); + configure_work_db_options(&mut opts, blob_mode, memory_profile); opts } @@ -99,12 +129,23 @@ pub fn column_family_descriptors() -> Vec { pub(super) fn column_family_descriptors_for_blob_mode( blob_mode: WorkDbBlobMode, ) -> Vec { + let memory_profile = work_db_memory_profile_from_env(); ALL_COLUMN_FAMILY_NAMES .iter() - .map(|name| ColumnFamilyDescriptor::new(*name, cf_opts(blob_mode))) + .map(|name| ColumnFamilyDescriptor::new(*name, cf_opts(blob_mode, memory_profile))) .collect() } +fn apply_work_db_memory_profile(opts: &mut Options, memory_profile: WorkDbMemoryProfile) { + match memory_profile { + WorkDbMemoryProfile::Default => {} + WorkDbMemoryProfile::Compact => { + opts.set_write_buffer_size(16 * 1024 * 1024); + opts.set_max_write_buffer_number(1); + } + } +} + pub(super) fn enable_blobdb_current(opts: &mut Options) { #[allow(unused_mut)] let mut _enabled = false; diff --git a/src/storage/keys.rs b/src/storage/keys.rs index b985a32..f1e5905 100644 --- a/src/storage/keys.rs +++ b/src/storage/keys.rs @@ -4,7 +4,7 @@ use crate::data_model::common::der_take_tlv; use super::config::*; use super::pack::PackTime; -use super::{AuditRuleKind, StorageError, StorageResult, VcirOutputType}; +use super::{StorageError, StorageResult}; pub(super) fn repository_view_key(rsync_uri: &str) -> String { format!("{REPOSITORY_VIEW_KEY_PREFIX}{rsync_uri}") @@ -30,20 +30,6 @@ pub(super) fn manifest_replay_meta_key(manifest_rsync_uri: &str) -> String { format!("{MANIFEST_REPLAY_META_KEY_PREFIX}{manifest_rsync_uri}") } -pub(super) fn audit_rule_kind_for_output_type( - output_type: VcirOutputType, -) -> Option { - match output_type { - VcirOutputType::Vrp => Some(AuditRuleKind::Roa), - VcirOutputType::Aspa => Some(AuditRuleKind::Aspa), - VcirOutputType::RouterKey => Some(AuditRuleKind::RouterKey), - } -} - -pub(super) fn audit_rule_key(kind: AuditRuleKind, rule_hash: &str) -> String { - format!("{}{rule_hash}", kind.key_prefix()) -} - pub(super) fn rrdp_source_key(notify_uri: &str) -> String { format!("{RRDP_SOURCE_KEY_PREFIX}{notify_uri}") } diff --git a/src/storage/tests.rs b/src/storage/tests.rs index 95abc3c..d5f1eb1 100644 --- a/src/storage/tests.rs +++ b/src/storage/tests.rs @@ -10,6 +10,10 @@ fn sha256_hex(input: &[u8]) -> String { hex::encode(compute_sha256_32(input)) } +fn sha256_32(input: &[u8]) -> [u8; 32] { + compute_sha256_32(input) +} + #[test] fn parse_work_db_blob_mode_accepts_supported_values() { assert_eq!(default_work_db_blob_mode(), WorkDbBlobMode::Disabled); @@ -41,6 +45,61 @@ fn parse_work_db_blob_mode_accepts_supported_values() { assert_eq!(parse_work_db_blob_mode("unexpected"), None); } +#[test] +fn parse_work_db_memory_profile_accepts_supported_values() { + assert_eq!( + parse_work_db_memory_profile("default"), + Some(WorkDbMemoryProfile::Default) + ); + assert_eq!( + parse_work_db_memory_profile("none"), + Some(WorkDbMemoryProfile::Default) + ); + assert_eq!( + parse_work_db_memory_profile("compact"), + Some(WorkDbMemoryProfile::Compact) + ); + assert_eq!( + parse_work_db_memory_profile("low-memory"), + Some(WorkDbMemoryProfile::Compact) + ); + assert_eq!(parse_work_db_memory_profile("unexpected"), None); +} + +#[test] +fn vcir_field_size_breakdown_counts_local_outputs_and_artifacts() { + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let breakdown = VcirFieldSizeBreakdown::from_vcir(&vcir); + assert_eq!(breakdown.local_output_count, 2); + assert_eq!( + breakdown.local_output_payload_json_bytes, + vcir.local_outputs + .iter() + .map(|output| output.payload_json().len() as u64) + .sum::() + ); + assert_eq!( + breakdown.local_output_rule_hash_hex_bytes, + vcir.local_outputs.len() as u64 * 64 + ); + assert_eq!(breakdown.related_artifact_count, 2); + assert!(breakdown.related_artifact_uri_bytes > 0); + assert_eq!(breakdown.child_entry_count, 1); + assert!(breakdown.child_entry_uri_bytes > 0); + assert_eq!( + breakdown.local_output_old_projection_bytes(), + breakdown.local_output_source_type_bytes + + breakdown.local_output_source_hash_hex_bytes + + breakdown.local_output_source_ee_hash_hex_bytes + + breakdown.local_output_payload_json_bytes + + breakdown.local_output_rule_hash_hex_bytes + ); + assert!( + breakdown.local_output_old_projection_bytes() + > breakdown.local_output_typed_projection_bytes() + ); +} + fn sample_repository_view_entry(rsync_uri: &str, bytes: &[u8]) -> RepositoryViewEntry { RepositoryViewEntry { rsync_uri: rsync_uri.to_string(), @@ -126,28 +185,37 @@ fn sample_vcir(manifest_rsync_uri: &str) -> ValidatedCaInstanceResult { }], local_outputs: vec![ VcirLocalOutput { - output_id: "vrp-1".to_string(), output_type: VcirOutputType::Vrp, item_effective_until: pack_time(12), source_object_uri: "rsync://example.test/repo/object.roa".to_string(), - source_object_type: "roa".to_string(), - source_object_hash: sha256_hex(&roa_bytes), - source_ee_cert_hash: sha256_hex(&ee_bytes), - payload_json: r#"{"asn":64496,"prefix":"203.0.113.0/24"}"#.to_string(), - rule_hash: sha256_hex(b"vrp-rule-1"), - validation_path_hint: vec![manifest_rsync_uri.to_string()], + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: sha256_32(&roa_bytes), + source_ee_cert_hash: sha256_32(&ee_bytes), + payload: VcirLocalOutputPayload::Vrp { + asn: 64496, + afi: crate::data_model::roa::RoaAfi::Ipv4, + prefix_len: 24, + addr: { + let mut addr = [0u8; 16]; + addr[..4].copy_from_slice(&[203, 0, 113, 0]); + addr + }, + max_length: 24, + }, + rule_hash: sha256_32(b"vrp-rule-1"), }, VcirLocalOutput { - output_id: "aspa-1".to_string(), output_type: VcirOutputType::Aspa, item_effective_until: pack_time(10), source_object_uri: "rsync://example.test/repo/object.asa".to_string(), - source_object_type: "aspa".to_string(), - source_object_hash: sha256_hex(b"aspa-object"), - source_ee_cert_hash: sha256_hex(b"aspa-ee-cert"), - payload_json: r#"{"customer_as":64496,"providers":[64497]}"#.to_string(), - rule_hash: sha256_hex(b"aspa-rule-1"), - validation_path_hint: vec![manifest_rsync_uri.to_string()], + source_object_type: VcirSourceObjectType::Aspa, + source_object_hash: sha256_32(b"aspa-object"), + source_ee_cert_hash: sha256_32(b"aspa-ee-cert"), + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: 64496, + provider_as_ids: vec![64497], + }, + rule_hash: sha256_32(b"aspa-rule-1"), }, ], related_artifacts: vec![ @@ -230,34 +298,6 @@ fn vcir_ccr_manifest_projection_validate_rejects_invalid_fields() { )); } -fn sample_audit_rule_entry(kind: AuditRuleKind) -> AuditRuleIndexEntry { - AuditRuleIndexEntry { - kind, - rule_hash: sha256_hex(match kind { - AuditRuleKind::Roa => b"roa-index-rule", - AuditRuleKind::Aspa => b"aspa-index-rule", - AuditRuleKind::RouterKey => b"router-key-index-rule", - }), - manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(), - source_object_uri: match kind { - AuditRuleKind::Roa => "rsync://example.test/repo/object.roa".to_string(), - AuditRuleKind::Aspa => "rsync://example.test/repo/object.asa".to_string(), - AuditRuleKind::RouterKey => "rsync://example.test/repo/router.cer".to_string(), - }, - source_object_hash: sha256_hex(match kind { - AuditRuleKind::Roa => b"roa-object", - AuditRuleKind::Aspa => b"aspa-object", - AuditRuleKind::RouterKey => b"router-key-object", - }), - output_id: match kind { - AuditRuleKind::Roa => "vrp-1".to_string(), - AuditRuleKind::Aspa => "aspa-1".to_string(), - AuditRuleKind::RouterKey => "router-key-1".to_string(), - }, - item_effective_until: pack_time(12), - } -} - fn sample_rrdp_source_record(notify_uri: &str) -> RrdpSourceRecord { RrdpSourceRecord { notify_uri: notify_uri.to_string(), @@ -464,6 +504,32 @@ fn repo_bytes_db_is_physically_separate_from_external_raw_store() { ); } +#[test] +fn memory_snapshot_includes_work_db_and_external_stores() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open_with_external_stores( + &td.path().join("main-db"), + Some(&td.path().join("raw-store.db")), + Some(&td.path().join("repo-bytes.db")), + ) + .expect("open store"); + + let snapshot = store.memory_snapshot(); + let labels: Vec<&str> = snapshot + .databases + .iter() + .map(|db| db.label.as_str()) + .collect(); + assert_eq!(labels, vec!["work-db", "raw-store.db", "repo-bytes.db"]); + assert!( + snapshot.databases[0] + .column_families + .iter() + .any(|cf| cf.name == CF_REPOSITORY_VIEW) + ); + serde_json::to_value(&snapshot).expect("serialize memory snapshot"); +} + #[test] fn put_blob_bytes_batch_accepts_empty_batch_with_external_raw_store() { let td = tempfile::tempdir().expect("tempdir"); @@ -784,106 +850,111 @@ fn list_vcirs_returns_all_entries() { } #[test] -fn audit_rule_index_roundtrip_for_roa_aspa_and_router_key() { +fn summarize_vcir_storage_aggregates_values_and_field_sizes() { let td = tempfile::tempdir().expect("tempdir"); let store = RocksStore::open(td.path()).expect("open rocksdb"); - let roa = sample_audit_rule_entry(AuditRuleKind::Roa); - let aspa = sample_audit_rule_entry(AuditRuleKind::Aspa); - let router_key = sample_audit_rule_entry(AuditRuleKind::RouterKey); - store - .put_audit_rule_index_entry(&roa) - .expect("put roa audit rule entry"); - store - .put_audit_rule_index_entry(&aspa) - .expect("put aspa audit rule entry"); - store - .put_audit_rule_index_entry(&router_key) - .expect("put router key audit rule entry"); + let vcir1 = sample_vcir("rsync://example.test/repo/a.mft"); + let vcir2 = sample_vcir("rsync://example.test/repo/b.mft"); + store.put_vcir(&vcir1).expect("put vcir1"); + store.put_vcir(&vcir2).expect("put vcir2"); - let got_roa = store - .get_audit_rule_index_entry(AuditRuleKind::Roa, &roa.rule_hash) - .expect("get roa audit rule entry") - .expect("roa entry exists"); - let got_aspa = store - .get_audit_rule_index_entry(AuditRuleKind::Aspa, &aspa.rule_hash) - .expect("get aspa audit rule entry") - .expect("aspa entry exists"); - let got_router_key = store - .get_audit_rule_index_entry(AuditRuleKind::RouterKey, &router_key.rule_hash) - .expect("get router key audit rule entry") - .expect("router key entry exists"); - assert_eq!(got_roa, roa); - assert_eq!(got_aspa, aspa); - assert_eq!(got_router_key, router_key); + let summary = store.summarize_vcir_storage().expect("summarize vcirs"); + let mut expected_fields = VcirFieldSizeBreakdown::default(); + expected_fields.add_assign(&VcirFieldSizeBreakdown::from_vcir(&vcir1)); + expected_fields.add_assign(&VcirFieldSizeBreakdown::from_vcir(&vcir2)); - store - .delete_audit_rule_index_entry(AuditRuleKind::Roa, &roa.rule_hash) - .expect("delete roa audit rule entry"); + assert_eq!(summary.entry_count, 2); + assert!(summary.vcir_value_bytes > 0); + assert!(summary.vcir_value_bytes_max > 0); + assert!(summary.vcir_value_bytes_max_manifest_rsync_uri.is_some()); + assert_eq!(summary.top_entries_by_vcir_value_bytes.len(), 2); assert!( - store - .get_audit_rule_index_entry(AuditRuleKind::Roa, &roa.rule_hash) - .expect("get deleted roa audit rule entry") - .is_none() + summary.top_entries_by_vcir_value_bytes[0].vcir_value_bytes + >= summary.top_entries_by_vcir_value_bytes[1].vcir_value_bytes + ); + assert_eq!(summary.field_sizes, expected_fields); + assert!(summary.core_fields.manifest_rsync_uri_bytes > 0); + assert!(summary.ccr_projection.manifest_sha256_bytes > 0); + assert!(summary.child_resources.effective_ip_resource_cbor_bytes > 0); + assert_eq!( + summary.local_output_old_projection_bytes, + expected_fields.local_output_old_projection_bytes() + ); + assert_eq!( + summary.local_output_typed_projection_bytes, + expected_fields.local_output_typed_projection_bytes() + ); + assert_eq!( + summary.local_output_projection_saved_bytes, + expected_fields.local_output_projection_saved_bytes() ); - - let mut invalid = sample_audit_rule_entry(AuditRuleKind::Roa); - invalid.rule_hash = "bad".to_string(); - let err = store - .put_audit_rule_index_entry(&invalid) - .expect_err("invalid audit rule hash must fail"); - assert!(err.to_string().contains("64-character")); } #[test] -fn replace_vcir_and_audit_rule_indexes_replaces_previous_entries_in_one_step() { +fn replace_vcir_and_manifest_replay_meta_replaces_current_entry() { let td = tempfile::tempdir().expect("tempdir"); let store = RocksStore::open(td.path()).expect("open rocksdb"); let mut previous = sample_vcir("rsync://example.test/repo/current.mft"); previous.local_outputs = vec![VcirLocalOutput { - output_id: "old-output".to_string(), output_type: VcirOutputType::Vrp, item_effective_until: pack_time(10), source_object_uri: "rsync://example.test/repo/old.roa".to_string(), - source_object_type: "roa".to_string(), - source_object_hash: sha256_hex(b"old-roa"), - source_ee_cert_hash: sha256_hex(b"old-ee"), - payload_json: "{}".to_string(), - rule_hash: sha256_hex(b"old-rule"), - validation_path_hint: vec![previous.manifest_rsync_uri.clone()], + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: sha256_32(b"old-roa"), + source_ee_cert_hash: sha256_32(b"old-ee"), + payload: VcirLocalOutputPayload::Vrp { + asn: 64496, + afi: crate::data_model::roa::RoaAfi::Ipv4, + prefix_len: 24, + addr: { + let mut addr = [0u8; 16]; + addr[..4].copy_from_slice(&[203, 0, 113, 0]); + addr + }, + max_length: 24, + }, + rule_hash: sha256_32(b"old-rule"), }]; previous.summary.local_vrp_count = 1; previous.summary.local_aspa_count = 0; previous.summary.local_router_key_count = 0; - store - .replace_vcir_and_audit_rule_indexes(None, &previous) + let previous_timing = store + .replace_vcir_and_manifest_replay_meta(&previous) .expect("store previous vcir"); - assert!( - store - .get_audit_rule_index_entry(AuditRuleKind::Roa, &previous.local_outputs[0].rule_hash) - .expect("get old audit entry") - .is_some() + assert!(previous_timing.vcir_value_bytes > 0); + assert!(previous_timing.replay_meta_value_bytes > 0); + assert_eq!( + previous_timing.total_encoded_bytes, + previous_timing.vcir_value_bytes + previous_timing.replay_meta_value_bytes ); let mut current = sample_vcir("rsync://example.test/repo/current.mft"); current.local_outputs = vec![VcirLocalOutput { - output_id: "new-output".to_string(), output_type: VcirOutputType::Aspa, item_effective_until: pack_time(11), source_object_uri: "rsync://example.test/repo/new.asa".to_string(), - source_object_type: "aspa".to_string(), - source_object_hash: sha256_hex(b"new-aspa"), - source_ee_cert_hash: sha256_hex(b"new-ee"), - payload_json: "{}".to_string(), - rule_hash: sha256_hex(b"new-rule"), - validation_path_hint: vec![current.manifest_rsync_uri.clone()], + source_object_type: VcirSourceObjectType::Aspa, + source_object_hash: sha256_32(b"new-aspa"), + source_ee_cert_hash: sha256_32(b"new-ee"), + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: 64496, + provider_as_ids: vec![64497], + }, + rule_hash: sha256_32(b"new-rule"), }]; current.summary.local_vrp_count = 0; current.summary.local_aspa_count = 1; - store - .replace_vcir_and_audit_rule_indexes(Some(&previous), ¤t) - .expect("replace vcir and audit indexes"); + let current_timing = store + .replace_vcir_and_manifest_replay_meta(¤t) + .expect("replace vcir and replay meta"); + assert!(current_timing.vcir_value_bytes > 0); + assert!(current_timing.replay_meta_value_bytes > 0); + assert_eq!( + current_timing.total_encoded_bytes, + current_timing.vcir_value_bytes + current_timing.replay_meta_value_bytes + ); let got = store .get_vcir(¤t.manifest_rsync_uri) @@ -902,18 +973,6 @@ fn replace_vcir_and_audit_rule_indexes_replaces_previous_entries_in_one_step() { replay_meta.manifest_sha256, current.ccr_manifest_projection.manifest_sha256 ); - assert!( - store - .get_audit_rule_index_entry(AuditRuleKind::Roa, &previous.local_outputs[0].rule_hash) - .expect("get deleted old audit entry") - .is_none() - ); - assert!( - store - .get_audit_rule_index_entry(AuditRuleKind::Aspa, ¤t.local_outputs[0].rule_hash) - .expect("get new audit entry") - .is_some() - ); } #[test] diff --git a/src/tools/sequence_triage_ccr_cir.rs b/src/tools/sequence_triage_ccr_cir.rs index 3f82ac7..9ef4c8a 100644 --- a/src/tools/sequence_triage_ccr_cir.rs +++ b/src/tools/sequence_triage_ccr_cir.rs @@ -7,11 +7,11 @@ mod output; mod sandwich; use args::{Args, parse_args}; -use churn::build_intra_rp_churn; -use loader::load_sequence; +use churn::build_intra_rp_churn_streaming; +use loader::load_sequence_meta; use model::Side; -use output::{build_output, write_json, write_markdown}; -use sandwich::build_sandwich_analysis; +use output::{build_output_from_meta, write_json, write_markdown}; +use sandwich::build_sandwich_analysis_streaming; pub fn main_entry() -> Result<(), String> { real_main() @@ -25,15 +25,15 @@ fn real_main() -> Result<(), String> { fn run(args: Args) -> Result<(), String> { std::fs::create_dir_all(&args.out_dir) .map_err(|e| format!("create out-dir failed: {}: {e}", args.out_dir.display()))?; - let left = load_sequence(&args.left_sequence, Side::Left)?; - let right = load_sequence(&args.right_sequence, Side::Right)?; + let left = load_sequence_meta(&args.left_sequence, Side::Left)?; + let right = load_sequence_meta(&args.right_sequence, Side::Right)?; if left.is_empty() || right.is_empty() { return Err("left and right sequences must both contain at least one sample".into()); } - let sandwich = build_sandwich_analysis(&args, &left, &right); - let churn = build_intra_rp_churn(&left, &right); - let output = build_output(&args, &left, &right, &sandwich, &churn); + let sandwich = build_sandwich_analysis_streaming(&args, &left, &right)?; + let churn = build_intra_rp_churn_streaming(&left, &right)?; + let output = build_output_from_meta(&args, &left, &right, &sandwich, &churn); write_json(&args.out_dir.join("sequence-triage.json"), &output)?; write_markdown(&args.out_dir.join("sequence-triage.md"), &output)?; println!("{}", args.out_dir.display()); diff --git a/src/tools/sequence_triage_ccr_cir/churn.rs b/src/tools/sequence_triage_ccr_cir/churn.rs index f65cb14..9585255 100644 --- a/src/tools/sequence_triage_ccr_cir/churn.rs +++ b/src/tools/sequence_triage_ccr_cir/churn.rs @@ -1,71 +1,82 @@ use std::collections::{BTreeMap, BTreeSet}; -use super::model::{ChurnRecord, ChurnSummaryRecord, IntraRpChurn, SequenceSample, Side}; +use super::loader::{load_sample_ccr_from_meta, load_sample_cir_from_meta}; +use super::model::{ + ChurnRecord, ChurnSummaryRecord, IntraRpChurn, SequenceMeta, SequenceSample, Side, +}; -pub(super) fn build_intra_rp_churn( - left: &[SequenceSample], - right: &[SequenceSample], -) -> IntraRpChurn { - let left_records = build_side_records(Side::Left, left); - let right_records = build_side_records(Side::Right, right); +pub(super) fn build_intra_rp_churn_streaming( + left: &[SequenceMeta], + right: &[SequenceMeta], +) -> Result { + let left_records = build_side_records_streaming(Side::Left, left)?; + let right_records = build_side_records_streaming(Side::Right, right)?; let mut summary = summarize_records(&left_records); summary.extend(summarize_records(&right_records)); - IntraRpChurn { + Ok(IntraRpChurn { left: left_records, right: right_records, summary, - } + }) } -fn build_side_records(side: Side, samples: &[SequenceSample]) -> Vec { +fn build_side_records_streaming( + side: Side, + samples: &[SequenceMeta], +) -> Result, String> { let mut records = Vec::new(); for pair in samples.windows(2) { - let from = &pair[0]; - let to = &pair[1]; + let from_cir = load_sample_cir_from_meta(&pair[0])?; + let to_cir = load_sample_cir_from_meta(&pair[1])?; records.push(record_churn( side, - from, - to, + &from_cir, + &to_cir, "object", - &from.object_hashes, - &to.object_hashes, + &from_cir.object_hashes, + &to_cir.object_hashes, )); - let from_output = output_keys(from); - let to_output = output_keys(to); records.push(record_churn( side, - from, - to, + &from_cir, + &to_cir, + "reject", + &from_cir.rejects, + &to_cir.rejects, + )); + drop(from_cir); + drop(to_cir); + + let from_ccr = load_sample_ccr_from_meta(&pair[0])?; + let to_ccr = load_sample_ccr_from_meta(&pair[1])?; + let from_output = output_keys(&from_ccr); + let to_output = output_keys(&to_ccr); + records.push(record_churn( + side, + &from_ccr, + &to_ccr, "output", &from_output, &to_output, )); records.push(record_churn( side, - from, - to, + &from_ccr, + &to_ccr, "vrp_output", - &from.vrps, - &to.vrps, + &from_ccr.vrps, + &to_ccr.vrps, )); records.push(record_churn( side, - from, - to, + &from_ccr, + &to_ccr, "vap_output", - &from.vaps, - &to.vaps, - )); - records.push(record_churn( - side, - from, - to, - "reject", - &from.rejects, - &to.rejects, + &from_ccr.vaps, + &to_ccr.vaps, )); } - records + Ok(records) } fn output_keys(sample: &SequenceSample) -> BTreeSet { diff --git a/src/tools/sequence_triage_ccr_cir/loader.rs b/src/tools/sequence_triage_ccr_cir/loader.rs index ef94552..e67730d 100644 --- a/src/tools/sequence_triage_ccr_cir/loader.rs +++ b/src/tools/sequence_triage_ccr_cir/loader.rs @@ -5,9 +5,9 @@ use crate::ccr::{decode_ccr_compare_views, decode_content_info}; use crate::cir::decode_cir; use super::io::{object_hash_key, parse_rfc3339, read_file, resolve_path}; -use super::model::{SequenceItemRaw, SequenceSample, Side}; +use super::model::{SequenceItemRaw, SequenceMeta, SequenceSample, Side}; -pub(super) fn load_sequence(path: &Path, side: Side) -> Result, String> { +pub(super) fn load_sequence_meta(path: &Path, side: Side) -> Result, String> { let base_dir = path.parent().unwrap_or_else(|| Path::new(".")); let text = std::fs::read_to_string(path) .map_err(|e| format!("read sequence failed: {}: {e}", path.display()))?; @@ -25,101 +25,34 @@ pub(super) fn load_sequence(path: &Path, side: Side) -> Result>(); - let object_uris = objects.keys().cloned().collect::>(); - let object_hashes = objects - .iter() - .map(|(uri, hash)| object_hash_key(uri, hash)) - .collect::>(); - let rejects = cir - .rejected_objects - .iter() - .map(|item| item.object_uri.clone()) - .collect::>(); - let trust_anchors = cir - .trust_anchors - .iter() - .map(|item| { - format!( - "{}|{}|{}|{}", - item.ta_rsync_uri, - item.tal_uri, - hex::encode(crate::cir::sha256(&item.tal_bytes)), - hex::encode(&item.ta_certificate_sha256) - ) - }) - .collect::>(); - let (vrps, vaps) = decode_ccr_compare_views(&ccr).map_err(|e| { - format!( - "decode CCR compare views failed for sample {} ({}): {e}", - raw.run_id, - ccr_path.display() - ) - })?; - let vrps = vrps - .into_iter() - .map(|row| format!("{}|{}|{}", row.asn, row.ip_prefix, row.max_length)) - .collect::>(); - let vaps = vaps - .into_iter() - .map(|row| format!("{}|{}", row.customer_asn, row.providers)) - .collect::>(); - samples.push(SequenceSample { + samples.push(SequenceMeta { + cir_object_count: raw.cir_object_count.or(Some(cir.objects.len() as u64)), + cir_reject_count: raw + .cir_reject_count + .or(Some(cir.rejected_objects.len() as u64)), + cir_trust_anchor_count: raw + .cir_trust_anchor_count + .or(Some(cir.trust_anchors.len() as u64)), raw, validation_time, ccr_path, cir_path, - objects, - object_uris, - object_hashes, - rejects, - trust_anchors, - vrps, - vaps, }); } samples.sort_by(|left, right| { @@ -128,17 +61,120 @@ pub(super) fn load_sequence(path: &Path, side: Side) -> Result Result { + load_sample_parts(meta, true, false) +} + +pub(super) fn load_sample_ccr_from_meta(meta: &SequenceMeta) -> Result { + load_sample_parts(meta, false, true) +} + +fn load_sample_parts( + meta: &SequenceMeta, + include_cir: bool, + include_ccr: bool, +) -> Result { + let mut objects = BTreeMap::new(); + let mut object_hashes = BTreeSet::new(); + let mut rejects = BTreeSet::new(); + if include_cir { + let cir = decode_cir(&read_file(&meta.cir_path)?).map_err(|e| { + format!( + "decode CIR failed for sample {} ({}): {e}", + meta.raw.run_id, + meta.cir_path.display() + ) + })?; + objects = cir + .objects + .iter() + .map(|item| (item.rsync_uri.clone(), hex::encode(&item.sha256))) + .collect::>(); + object_hashes = objects + .iter() + .map(|(uri, hash)| object_hash_key(uri, hash)) + .collect::>(); + rejects = cir + .rejected_objects + .iter() + .map(|item| item.object_uri.clone()) + .collect::>(); + } + let mut vrps = BTreeSet::new(); + let mut vaps = BTreeSet::new(); + if include_ccr { + let ccr = decode_content_info(&read_file(&meta.ccr_path)?).map_err(|e| { + format!( + "decode CCR failed for sample {} ({}): {e}", + meta.raw.run_id, + meta.ccr_path.display() + ) + })?; + let (decoded_vrps, decoded_vaps) = decode_ccr_compare_views(&ccr).map_err(|e| { + format!( + "decode CCR compare views failed for sample {} ({}): {e}", + meta.raw.run_id, + meta.ccr_path.display() + ) + })?; + vrps = decoded_vrps + .into_iter() + .map(|row| format!("{}|{}|{}", row.asn, row.ip_prefix, row.max_length)) + .collect::>(); + vaps = decoded_vaps + .into_iter() + .map(|row| format!("{}|{}", row.customer_asn, row.providers)) + .collect::>(); + } + Ok(SequenceSample { + raw: meta.raw.clone(), + validation_time: meta.validation_time, + objects, + object_hashes, + rejects, + vrps, + vaps, + }) +} + +fn validate_raw( + path: &Path, + line_index: usize, + raw: &SequenceItemRaw, + side: Side, + seen_seq: &mut BTreeSet, +) -> Result<(), String> { + if raw.schema_version.unwrap_or(1) != 1 { + return Err(format!( + "unsupported sequence item schemaVersion in {}:{}", + path.display(), + line_index + 1 + )); + } + if !seen_seq.insert(raw.seq) { + return Err(format!("duplicate seq {} in {}", raw.seq, path.display())); + } + if let Some(status) = &raw.status + && status != "success" + { + return Err(format!( + "sequence sample {} has non-success status: {status}", + raw.run_id + )); + } + if raw + .side + .as_deref() + .is_some_and(|item| item != side.as_str()) + { + return Err(format!( + "sequence side field does not match expected side in {}:{}", + path.display(), + line_index + 1 + )); + } + Ok(()) +} diff --git a/src/tools/sequence_triage_ccr_cir/model.rs b/src/tools/sequence_triage_ccr_cir/model.rs index 9b21989..3520c48 100644 --- a/src/tools/sequence_triage_ccr_cir/model.rs +++ b/src/tools/sequence_triage_ccr_cir/model.rs @@ -25,23 +25,34 @@ pub(super) struct SequenceItemRaw { pub(super) max_rss_kb: Option, pub(super) vrps: Option, pub(super) vaps: Option, + pub(super) publication_points: Option, + pub(super) cir_object_count: Option, + pub(super) cir_reject_count: Option, + pub(super) cir_trust_anchor_count: Option, } #[derive(Clone, Debug)] pub(super) struct SequenceSample { pub(super) raw: SequenceItemRaw, pub(super) validation_time: OffsetDateTime, - pub(super) ccr_path: PathBuf, - pub(super) cir_path: PathBuf, pub(super) objects: BTreeMap, - pub(super) object_uris: BTreeSet, pub(super) object_hashes: BTreeSet, pub(super) rejects: BTreeSet, - pub(super) trust_anchors: BTreeSet, pub(super) vrps: BTreeSet, pub(super) vaps: BTreeSet, } +#[derive(Clone, Debug)] +pub(super) struct SequenceMeta { + pub(super) raw: SequenceItemRaw, + pub(super) validation_time: OffsetDateTime, + pub(super) ccr_path: PathBuf, + pub(super) cir_path: PathBuf, + pub(super) cir_object_count: Option, + pub(super) cir_reject_count: Option, + pub(super) cir_trust_anchor_count: Option, +} + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord)] pub(super) enum Side { #[default] diff --git a/src/tools/sequence_triage_ccr_cir/output.rs b/src/tools/sequence_triage_ccr_cir/output.rs index 8b591bc..0de97fd 100644 --- a/src/tools/sequence_triage_ccr_cir/output.rs +++ b/src/tools/sequence_triage_ccr_cir/output.rs @@ -7,13 +7,13 @@ use super::args::Args; use super::io::{format_time, path_string}; use super::model::{ ChurnRecord, ChurnSummaryRecord, IntraRpChurn, SandwichAnalysis, SandwichGroupStats, - SandwichHeatmapRow, SandwichRecord, SequenceSample, + SandwichHeatmapRow, SandwichRecord, SequenceMeta, }; -pub(super) fn build_output( +pub(super) fn build_output_from_meta( args: &Args, - left: &[SequenceSample], - right: &[SequenceSample], + left: &[SequenceMeta], + right: &[SequenceMeta], sandwich: &SandwichAnalysis, churn: &IntraRpChurn, ) -> Value { @@ -30,6 +30,7 @@ pub(super) fn build_output( "warmupSamples": args.warmup_samples, "cooldownSamples": args.cooldown_samples, "timelineSampleLimit": if args.timeline_sample_limit == 0 { args.sample_limit } else { args.timeline_sample_limit }, + "executionMode": "streaming-window", "sequenceSemantics": { "timeSource": "cir.validation_time", "sourceSampleOrder": "each side is sorted by CIR validation_time, with seq/runId used only as stable tie breakers", @@ -37,11 +38,11 @@ pub(super) fn build_output( "peerWindow": "all peer samples with source_start.time < peer.time < source_end.time are checked independently", }, }, - "left": sequence_summary(left), - "right": sequence_summary(right), + "left": sequence_meta_summary(left), + "right": sequence_meta_summary(right), "sandwich": { "strictTimeWindow": true, - "method": "For each side sorted by CIR validation_time, use two adjacent source samples as a stable interval. If source_start.time < peer.time < source_end.time and the source value is identical at both interval endpoints, every peer sample in the interval is expected to contain the same value.", + "method": "Streaming mode: first build sample timing metadata, then process each sandwich triple by loading only source_start, peer, and source_end CIR/CCR artifacts.", "totals": { "occurrences": sandwich.total_occurrences, "uniqueKeys": sandwich.unique_keys.len(), @@ -59,7 +60,7 @@ pub(super) fn build_output( } }, "intraRpChurn": { - "method": "For each side, compare adjacent samples independently. Object keys use object_uri|sha256, output keys use typed VRP/VAP canonical keys, and reject keys use rejected object URI without reason.", + "method": "Streaming mode: compare adjacent samples by loading only the two CIR or CCR artifacts needed for the current adjacent pair.", "left": churn_records_to_json(&churn.left), "right": churn_records_to_json(&churn.right), "summary": churn_summary_to_json(&churn.summary), @@ -77,7 +78,7 @@ pub(super) fn build_output( }) } -fn sequence_summary(samples: &[SequenceSample]) -> Value { +fn sequence_meta_summary(samples: &[SequenceMeta]) -> Value { json!({ "sampleCount": samples.len(), "rpIds": samples.iter().map(|sample| sample.raw.rp_id.clone()).collect::>(), @@ -99,12 +100,13 @@ fn sequence_summary(samples: &[SequenceSample]) -> Value { "cirSha256": sample.raw.cir_sha256, "wallMs": sample.raw.wall_ms, "maxRssKb": sample.raw.max_rss_kb, - "vrps": sample.raw.vrps.or(Some(sample.vrps.len() as u64)), - "vaps": sample.raw.vaps.or(Some(sample.vaps.len() as u64)), - "objectCount": sample.object_uris.len(), - "objectHashCount": sample.object_hashes.len(), - "rejectCount": sample.rejects.len(), - "trustAnchorCount": sample.trust_anchors.len(), + "vrps": sample.raw.vrps, + "vaps": sample.raw.vaps, + "publicationPoints": sample.raw.publication_points, + "objectCount": sample.cir_object_count, + "objectHashCount": sample.cir_object_count, + "rejectCount": sample.cir_reject_count, + "trustAnchorCount": sample.cir_trust_anchor_count, })).collect::>(), }) } diff --git a/src/tools/sequence_triage_ccr_cir/sandwich.rs b/src/tools/sequence_triage_ccr_cir/sandwich.rs index d6003b6..4427717 100644 --- a/src/tools/sequence_triage_ccr_cir/sandwich.rs +++ b/src/tools/sequence_triage_ccr_cir/sandwich.rs @@ -2,209 +2,222 @@ use std::collections::BTreeSet; use super::args::Args; use super::io::format_time; +use super::loader::{load_sample_ccr_from_meta, load_sample_cir_from_meta}; use super::model::{ - SandwichAnalysis, SandwichGroupStats, SandwichHeatmapRow, SandwichRecord, SequenceSample, Side, + SandwichAnalysis, SandwichGroupStats, SandwichHeatmapRow, SandwichRecord, SequenceMeta, + SequenceSample, Side, }; -pub(super) fn build_sandwich_analysis( +pub(super) fn build_sandwich_analysis_streaming( args: &Args, - left: &[SequenceSample], - right: &[SequenceSample], -) -> SandwichAnalysis { + left: &[SequenceMeta], + right: &[SequenceMeta], +) -> Result { let mut analysis = SandwichAnalysis::default(); - analyze_sandwich_objects(&mut analysis, Side::Left, left, right, args); - analyze_sandwich_objects(&mut analysis, Side::Right, right, left, args); - analyze_sandwich_sets( - &mut analysis, - "reject_uri", - "PEER_MISSING_STABLE_REJECT", - Side::Left, - left, - right, - |sample| &sample.rejects, - args, - ); - analyze_sandwich_sets( - &mut analysis, - "reject_uri", - "PEER_MISSING_STABLE_REJECT", - Side::Right, - right, - left, - |sample| &sample.rejects, - args, - ); - analyze_sandwich_sets( - &mut analysis, - "vrp_output", - "PEER_MISSING_STABLE_OUTPUT", - Side::Left, - left, - right, - |sample| &sample.vrps, - args, - ); - analyze_sandwich_sets( - &mut analysis, - "vrp_output", - "PEER_MISSING_STABLE_OUTPUT", - Side::Right, - right, - left, - |sample| &sample.vrps, - args, - ); - analyze_sandwich_sets( - &mut analysis, - "vap_output", - "PEER_MISSING_STABLE_OUTPUT", - Side::Left, - left, - right, - |sample| &sample.vaps, - args, - ); - analyze_sandwich_sets( - &mut analysis, - "vap_output", - "PEER_MISSING_STABLE_OUTPUT", - Side::Right, - right, - left, - |sample| &sample.vaps, - args, - ); - analysis + analyze_sandwich_cir_streaming(&mut analysis, Side::Left, left, right, args)?; + analyze_sandwich_cir_streaming(&mut analysis, Side::Right, right, left, args)?; + analyze_sandwich_ccr_streaming(&mut analysis, Side::Left, left, right, args)?; + analyze_sandwich_ccr_streaming(&mut analysis, Side::Right, right, left, args)?; + Ok(analysis) } -fn analyze_sandwich_objects( +fn analyze_sandwich_cir_streaming( analysis: &mut SandwichAnalysis, source_side: Side, - source: &[SequenceSample], - peer: &[SequenceSample], + source: &[SequenceMeta], + peer: &[SequenceMeta], + args: &Args, +) -> Result<(), String> { + for pair in source.windows(2) { + let source_start_meta = &pair[0]; + let source_end_meta = &pair[1]; + if source_start_meta.validation_time >= source_end_meta.validation_time { + continue; + } + let peer_indices = peer_meta_indices_between(peer, source_start_meta, source_end_meta); + if peer_indices.is_empty() { + continue; + } + let source_start = load_sample_cir_from_meta(source_start_meta)?; + let source_end = load_sample_cir_from_meta(source_end_meta)?; + for peer_index in peer_indices { + let peer_sample = load_sample_cir_from_meta(&peer[peer_index])?; + analyze_loaded_sandwich_objects( + analysis, + source_side, + &source_start, + &source_end, + &peer_sample, + args, + ); + analyze_loaded_sandwich_set( + analysis, + "reject_uri", + "PEER_MISSING_STABLE_REJECT", + source_side, + &source_start, + &source_end, + &peer_sample, + |sample| &sample.rejects, + args, + ); + } + } + Ok(()) +} + +fn analyze_sandwich_ccr_streaming( + analysis: &mut SandwichAnalysis, + source_side: Side, + source: &[SequenceMeta], + peer: &[SequenceMeta], + args: &Args, +) -> Result<(), String> { + for pair in source.windows(2) { + let source_start_meta = &pair[0]; + let source_end_meta = &pair[1]; + if source_start_meta.validation_time >= source_end_meta.validation_time { + continue; + } + let peer_indices = peer_meta_indices_between(peer, source_start_meta, source_end_meta); + if peer_indices.is_empty() { + continue; + } + let source_start = load_sample_ccr_from_meta(source_start_meta)?; + let source_end = load_sample_ccr_from_meta(source_end_meta)?; + for peer_index in peer_indices { + let peer_sample = load_sample_ccr_from_meta(&peer[peer_index])?; + analyze_loaded_sandwich_set( + analysis, + "vrp_output", + "PEER_MISSING_STABLE_OUTPUT", + source_side, + &source_start, + &source_end, + &peer_sample, + |sample| &sample.vrps, + args, + ); + analyze_loaded_sandwich_set( + analysis, + "vap_output", + "PEER_MISSING_STABLE_OUTPUT", + source_side, + &source_start, + &source_end, + &peer_sample, + |sample| &sample.vaps, + args, + ); + } + } + Ok(()) +} + +fn peer_meta_indices_between( + peer: &[SequenceMeta], + source_start: &SequenceMeta, + source_end: &SequenceMeta, +) -> Vec { + peer.iter() + .enumerate() + .filter_map(|(index, sample)| { + (source_start.validation_time < sample.validation_time + && sample.validation_time < source_end.validation_time) + .then_some(index) + }) + .collect() +} + +fn analyze_loaded_sandwich_objects( + analysis: &mut SandwichAnalysis, + source_side: Side, + source_start: &SequenceSample, + source_end: &SequenceSample, + peer_sample: &SequenceSample, args: &Args, ) { - for pair in source.windows(2) { - let source_start = &pair[0]; - let source_end = &pair[1]; - if source_start.validation_time >= source_end.validation_time { + for (uri, source_hash) in &source_start.objects { + if source_end.objects.get(uri) != Some(source_hash) { continue; } - let peers = peer_samples_between(peer, source_start, source_end); - if peers.is_empty() { - continue; - } - for (uri, source_hash) in &source_start.objects { - if source_end.objects.get(uri) != Some(source_hash) { - continue; - } - for peer_sample in &peers { - match peer_sample.objects.get(uri) { - Some(peer_hash) if peer_hash == source_hash => {} - Some(peer_hash) => analysis.add( - "PEER_HASH_MISMATCH_STABLE_OBJECT", - sandwich_record( - "PEER_HASH_MISMATCH_STABLE_OBJECT", - "object", - uri.clone(), - source_side, - source_start, - source_end, - peer_sample, - Some(source_hash.clone()), - Some(peer_hash.clone()), - "source interval has stable object hash; peer sample has same URI with another hash", - ), - args.sample_limit, - ), - None => analysis.add( - "PEER_MISSING_STABLE_OBJECT", - sandwich_record( - "PEER_MISSING_STABLE_OBJECT", - "object", - uri.clone(), - source_side, - source_start, - source_end, - peer_sample, - Some(source_hash.clone()), - None, - "source interval has stable object hash; peer sample misses the URI", - ), - args.sample_limit, - ), - } - } + match peer_sample.objects.get(uri) { + Some(peer_hash) if peer_hash == source_hash => {} + Some(peer_hash) => analysis.add( + "PEER_HASH_MISMATCH_STABLE_OBJECT", + sandwich_record( + "PEER_HASH_MISMATCH_STABLE_OBJECT", + "object", + uri.clone(), + source_side, + source_start, + source_end, + peer_sample, + Some(source_hash.clone()), + Some(peer_hash.clone()), + "source interval has stable object hash; peer sample has same URI with another hash", + ), + args.sample_limit, + ), + None => analysis.add( + "PEER_MISSING_STABLE_OBJECT", + sandwich_record( + "PEER_MISSING_STABLE_OBJECT", + "object", + uri.clone(), + source_side, + source_start, + source_end, + peer_sample, + Some(source_hash.clone()), + None, + "source interval has stable object hash; peer sample misses the URI", + ), + args.sample_limit, + ), } } } -fn analyze_sandwich_sets( +fn analyze_loaded_sandwich_set( analysis: &mut SandwichAnalysis, set_type: &'static str, classification: &'static str, source_side: Side, - source: &[SequenceSample], - peer: &[SequenceSample], + source_start: &SequenceSample, + source_end: &SequenceSample, + peer_sample: &SequenceSample, extract: F, args: &Args, ) where F: for<'a> Fn(&'a SequenceSample) -> &'a BTreeSet, { - for pair in source.windows(2) { - let source_start = &pair[0]; - let source_end = &pair[1]; - if source_start.validation_time >= source_end.validation_time { + let start_set = extract(source_start); + let end_set = extract(source_end); + let peer_set = extract(peer_sample); + for key in start_set { + if !end_set.contains(key) || peer_set.contains(key) { continue; } - let peers = peer_samples_between(peer, source_start, source_end); - if peers.is_empty() { - continue; - } - let start_set = extract(source_start); - let end_set = extract(source_end); - for key in start_set { - if !end_set.contains(key) { - continue; - } - for peer_sample in &peers { - if extract(peer_sample).contains(key) { - continue; - } - analysis.add( - classification, - sandwich_record( - classification, - set_type, - key.clone(), - source_side, - source_start, - source_end, - peer_sample, - Some(key.clone()), - None, - "source interval has a stable key; peer sample misses the key", - ), - args.sample_limit, - ); - } - } + analysis.add( + classification, + sandwich_record( + classification, + set_type, + key.clone(), + source_side, + source_start, + source_end, + peer_sample, + Some(key.clone()), + None, + "source interval has a stable key; peer sample misses the key", + ), + args.sample_limit, + ); } } -fn peer_samples_between<'a>( - peer: &'a [SequenceSample], - source_start: &SequenceSample, - source_end: &SequenceSample, -) -> Vec<&'a SequenceSample> { - peer.iter() - .filter(|sample| { - source_start.validation_time < sample.validation_time - && sample.validation_time < source_end.validation_time - }) - .collect() -} - #[allow(clippy::too_many_arguments)] fn sandwich_record( classification: &'static str, @@ -239,7 +252,7 @@ fn sandwich_record( } impl SandwichAnalysis { - fn add(&mut self, class: &'static str, record: SandwichRecord, sample_limit: usize) { + pub(super) fn add(&mut self, class: &'static str, record: SandwichRecord, sample_limit: usize) { self.total_occurrences += 1; self.unique_keys.insert(sandwich_unique_key(&record)); *self.by_set_type.entry(record.set_type).or_default() += 1; diff --git a/src/validation/objects.rs b/src/validation/objects.rs index bbe4b58..8222c8a 100644 --- a/src/validation/objects.rs +++ b/src/validation/objects.rs @@ -14,7 +14,10 @@ use crate::parallel::object_worker::{ }; use crate::policy::{Policy, SignedObjectFailurePolicy}; use crate::report::{RfcRef, Warning}; -use crate::storage::{PackFile, PackTime, VcirLocalOutput, VcirOutputType}; +use crate::storage::{ + PackFile, PackTime, VcirLocalOutput, VcirLocalOutputPayload, VcirOutputType, + VcirSourceObjectType, +}; use crate::validation::cert_path::{CertPathError, validate_signed_object_ee_cert_path_fast}; use crate::validation::manifest::PublicationPointData; use crate::validation::publication_point::PublicationPointSnapshot; @@ -28,6 +31,13 @@ const RFC_CRLDP: &[RfcRef] = &[RfcRef("RFC 6487 §4.8.6")]; const RFC_CRLDP_AND_LOCKED_PACK: &[RfcRef] = &[RfcRef("RFC 6487 §4.8.6"), RfcRef("RFC 9286 §4.2.1")]; +fn sha256_hex_to_32(hex_value: &str) -> [u8; 32] { + let bytes = hex::decode(hex_value).expect("internal sha256 hex should decode"); + let mut out = [0u8; 32]; + out.copy_from_slice(&bytes); + out +} + fn decode_resource_certificate_with_policy( der: &[u8], policy: &Policy, @@ -132,8 +142,6 @@ pub(crate) struct RoaTaskResult { pub(crate) worker_index: usize, pub(crate) queue_wait_ms: u64, pub(crate) worker_ms: u64, - pub(crate) rsync_uri: String, - pub(crate) sha256_hex: String, pub(crate) outcome: Result, } @@ -418,8 +426,8 @@ pub fn process_publication_point_for_issuer_with_options match policy.signed_object_failure_policy { SignedObjectFailurePolicy::DropObject => { audit.push(ObjectAuditEntry { - rsync_uri: result.rsync_uri.clone(), - sha256_hex: result.sha256_hex, + rsync_uri: file.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&file.sha256), kind: AuditObjectKind::Roa, result: AuditObjectResult::Error, detail: Some(e.to_string()), @@ -437,24 +445,21 @@ pub fn process_publication_point_for_issuer_with_options { stats.publication_point_dropped = true; audit.push(ObjectAuditEntry { - rsync_uri: result.rsync_uri.clone(), - sha256_hex: result.sha256_hex, + rsync_uri: file.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&file.sha256), kind: AuditObjectKind::Roa, result: AuditObjectResult::Error, detail: Some(e.to_string()), }); - for f in locked_files.iter().skip(result.index + 1) { + for f in locked_files.iter().skip(idx + 1) { if f.rsync_uri.ends_with(".roa") { audit.push(ObjectAuditEntry { rsync_uri: f.rsync_uri.clone(), @@ -484,7 +489,7 @@ pub fn process_publication_point_for_issuer_with_options, + manifest_rsync_uri: Arc, issuer_ca_der: Arc<[u8]>, issuer_ca: Arc, issuer_spki_der: Arc<[u8]>, - issuer_ca_rsync_uri: Option, + issuer_ca_rsync_uri: Option>, crl_cache: Arc>>, issuer_resources_index: Arc, - issuer_effective_ip: Option, - issuer_effective_as: Option, + issuer_effective_ip: Option>, + issuer_effective_as: Option>, +} + +#[derive(Clone)] +pub(crate) struct OwnedRoaTask { + pub(crate) publication_point_id: u64, + index: usize, + shared: Arc, validation_time: time::OffsetDateTime, collect_vcir_local_outputs: bool, strict_cms_der: bool, @@ -862,8 +872,13 @@ fn validate_owned_roa_task(worker_index: usize, task: OwnedRoaTask) -> RoaTaskRe .map(|submitted_at| worker_started.saturating_duration_since(submitted_at)) .map(|duration| duration.as_millis() as u64) .unwrap_or(0); - let sha256_hex = sha256_hex_from_32(&task.file.sha256); - let issuer_spki = match SubjectPublicKeyInfo::from_der(task.issuer_spki_der.as_ref()) { + let shared = task.shared.as_ref(); + let file = task + .shared + .locked_files + .get(task.index) + .expect("ROA task index must reference locked file"); + let issuer_spki = match SubjectPublicKeyInfo::from_der(shared.issuer_spki_der.as_ref()) { Ok((rem, spki)) if rem.is_empty() => spki, Ok((rem, _)) => { return RoaTaskResult { @@ -872,8 +887,6 @@ fn validate_owned_roa_task(worker_index: usize, task: OwnedRoaTask) -> RoaTaskRe worker_index, queue_wait_ms, worker_ms: worker_started.elapsed().as_millis() as u64, - rsync_uri: task.file.rsync_uri, - sha256_hex, outcome: Err(ObjectValidateError::CertPath( CertPathError::IssuerSpkiTrailingBytes(rem.len()), )), @@ -886,8 +899,6 @@ fn validate_owned_roa_task(worker_index: usize, task: OwnedRoaTask) -> RoaTaskRe worker_index, queue_wait_ms, worker_ms: worker_started.elapsed().as_millis() as u64, - rsync_uri: task.file.rsync_uri, - sha256_hex, outcome: Err(ObjectValidateError::CertPath( CertPathError::IssuerSpkiParse(e.to_string()), )), @@ -895,16 +906,16 @@ fn validate_owned_roa_task(worker_index: usize, task: OwnedRoaTask) -> RoaTaskRe } }; let outcome = process_roa_with_issuer_parallel_cached( - &task.file, - task.manifest_rsync_uri.as_str(), - task.issuer_ca_der.as_ref(), - task.issuer_ca.as_ref(), + file, + shared.manifest_rsync_uri.as_ref(), + shared.issuer_ca_der.as_ref(), + shared.issuer_ca.as_ref(), &issuer_spki, - task.issuer_ca_rsync_uri.as_deref(), - task.crl_cache.as_ref(), - task.issuer_resources_index.as_ref(), - task.issuer_effective_ip.as_ref(), - task.issuer_effective_as.as_ref(), + shared.issuer_ca_rsync_uri.as_deref(), + shared.crl_cache.as_ref(), + shared.issuer_resources_index.as_ref(), + shared.issuer_effective_ip.as_deref(), + shared.issuer_effective_as.as_deref(), task.validation_time, None, task.collect_vcir_local_outputs, @@ -922,8 +933,6 @@ fn validate_owned_roa_task(worker_index: usize, task: OwnedRoaTask) -> RoaTaskRe worker_index, queue_wait_ms, worker_ms: worker_started.elapsed().as_millis() as u64, - rsync_uri: task.file.rsync_uri, - sha256_hex, outcome, } } @@ -935,16 +944,7 @@ pub(crate) enum ParallelObjectsPrepare { pub(crate) struct ParallelObjectsStage { pub(crate) publication_point_id: u64, - pub(crate) locked_files: Vec, - pub(crate) manifest_rsync_uri: String, - issuer_ca_der: Arc<[u8]>, - issuer_ca: Arc, - issuer_spki_der: Arc<[u8]>, - issuer_ca_rsync_uri: Option, - crl_cache: Arc>>, - issuer_resources_index: Arc, - issuer_effective_ip: Option, - issuer_effective_as: Option, + shared: Arc, validation_time: time::OffsetDateTime, collect_vcir_local_outputs: bool, strict_cms_der: bool, @@ -955,31 +955,41 @@ pub(crate) struct ParallelObjectsStage { } impl ParallelObjectsStage { + #[cfg(test)] pub(crate) fn build_roa_tasks(&self) -> Vec { - self.locked_files + let mut tasks = Vec::with_capacity(self.roa_task_count()); + self.extend_roa_tasks(|task| tasks.push(task)); + tasks + } + + pub(crate) fn append_roa_tasks_to( + &self, + pending: &mut std::collections::VecDeque, + ) { + self.extend_roa_tasks(|task| pending.push_back(task)); + } + + fn extend_roa_tasks(&self, mut push: F) + where + F: FnMut(OwnedRoaTask), + { + let shared = self.shared.clone(); + self.locked_files() .iter() .enumerate() .filter(|(_, file)| file.rsync_uri.ends_with(".roa")) - .map(|(index, file)| OwnedRoaTask { - publication_point_id: self.publication_point_id, - index, - file: file.clone(), - manifest_rsync_uri: self.manifest_rsync_uri.clone(), - issuer_ca_der: self.issuer_ca_der.clone(), - issuer_ca: self.issuer_ca.clone(), - issuer_spki_der: self.issuer_spki_der.clone(), - issuer_ca_rsync_uri: self.issuer_ca_rsync_uri.clone(), - crl_cache: self.crl_cache.clone(), - issuer_resources_index: self.issuer_resources_index.clone(), - issuer_effective_ip: self.issuer_effective_ip.clone(), - issuer_effective_as: self.issuer_effective_as.clone(), - validation_time: self.validation_time, - collect_vcir_local_outputs: self.collect_vcir_local_outputs, - strict_cms_der: self.strict_cms_der, - strict_name: self.strict_name, - submitted_at: None, - }) - .collect() + .for_each(|(index, _)| { + push(OwnedRoaTask { + publication_point_id: self.publication_point_id, + index, + shared: shared.clone(), + validation_time: self.validation_time, + collect_vcir_local_outputs: self.collect_vcir_local_outputs, + strict_cms_der: self.strict_cms_der, + strict_name: self.strict_name, + submitted_at: None, + }); + }); } pub(crate) fn roa_task_count(&self) -> usize { @@ -991,7 +1001,11 @@ impl ParallelObjectsStage { } pub(crate) fn locked_file_count(&self) -> usize { - self.locked_files.len() + self.shared.locked_files.len() + } + + fn locked_files(&self) -> &[PackFile] { + self.shared.locked_files.as_ref() } } @@ -1212,19 +1226,21 @@ pub(crate) fn prepare_publication_point_for_parallel_roa::from(issuer_ca_der.to_vec()), - issuer_spki_der: Arc::<[u8]>::from(issuer_ca.tbs.subject_public_key_info.clone()), - issuer_ca: Arc::new(issuer_ca), - issuer_ca_rsync_uri: issuer_ca_rsync_uri.map(ToString::to_string), - crl_cache: Arc::new(Mutex::new(crl_cache)), - issuer_resources_index: Arc::new(build_issuer_resources_index( - issuer_effective_ip, - issuer_effective_as, - )), - issuer_effective_ip: issuer_effective_ip.cloned(), - issuer_effective_as: issuer_effective_as.cloned(), + shared: Arc::new(RoaTaskShared { + locked_files: Arc::<[PackFile]>::from(locked_files.to_vec()), + manifest_rsync_uri: Arc::::from(manifest_rsync_uri), + issuer_ca_der: Arc::<[u8]>::from(issuer_ca_der.to_vec()), + issuer_spki_der: Arc::<[u8]>::from(issuer_ca.tbs.subject_public_key_info.clone()), + issuer_ca: Arc::new(issuer_ca), + issuer_ca_rsync_uri: issuer_ca_rsync_uri.map(Arc::::from), + crl_cache: Arc::new(Mutex::new(crl_cache)), + issuer_resources_index: Arc::new(build_issuer_resources_index( + issuer_effective_ip, + issuer_effective_as, + )), + issuer_effective_ip: issuer_effective_ip.cloned().map(Arc::new), + issuer_effective_as: issuer_effective_as.cloned().map(Arc::new), + }), validation_time, collect_vcir_local_outputs, strict_cms_der: policy.strict.cms_der, @@ -1241,18 +1257,20 @@ pub(crate) fn reduce_parallel_roa_stage( timing: Option<&TimingHandle>, ) -> Result { roa_results.sort_by_key(|result| result.index); - let mut roa_by_index = roa_results - .into_iter() - .map(|result| (result.index, result)) - .collect::>(); - let mut aspa_crl_cache = stage + let mut roa_results = roa_results.into_iter().peekable(); + let shared = stage.shared.clone(); + let mut aspa_crl_cache = shared .crl_cache .lock() .expect("parallel ROA CRL cache lock") .clone(); - let issuer_spki = SubjectPublicKeyInfo::from_der(stage.issuer_spki_der.as_ref()) + let issuer_spki = SubjectPublicKeyInfo::from_der(shared.issuer_spki_der.as_ref()) .map_err(|e| e.to_string())? .1; + let collect_vcir_local_outputs = stage.collect_vcir_local_outputs; + let validation_time = stage.validation_time; + let strict_cms_der = stage.strict_cms_der; + let strict_name = stage.strict_name; let mut stats = stage.stats; let mut warnings = stage.warnings; let mut audit = stage.audit; @@ -1260,21 +1278,32 @@ pub(crate) fn reduce_parallel_roa_stage( let mut aspas: Vec = Vec::new(); let mut local_outputs_cache: Vec = Vec::new(); - for (idx, file) in stage.locked_files.iter().enumerate() { + for (idx, file) in shared.locked_files.iter().enumerate() { if file.rsync_uri.ends_with(".roa") { - let result = roa_by_index - .remove(&idx) - .ok_or_else(|| format!("missing ROA task result for {}", file.rsync_uri))?; + let result = match roa_results.peek() { + Some(result) if result.index == idx => roa_results + .next() + .expect("peeked ROA task result must be present"), + Some(result) => { + return Err(format!( + "unexpected ROA task result index {} while reducing {} at index {}", + result.index, file.rsync_uri, idx + )); + } + None => { + return Err(format!("missing ROA task result for {}", file.rsync_uri)); + } + }; match result.outcome { Ok(mut ok) => { stats.roa_ok += 1; vrps.append(&mut ok.vrps); - if stage.collect_vcir_local_outputs { + if collect_vcir_local_outputs { local_outputs_cache.extend(ok.local_outputs); } audit.push(ObjectAuditEntry { - rsync_uri: result.rsync_uri, - sha256_hex: result.sha256_hex, + rsync_uri: file.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&file.sha256), kind: AuditObjectKind::Roa, result: AuditObjectResult::Ok, detail: None, @@ -1282,8 +1311,8 @@ pub(crate) fn reduce_parallel_roa_stage( } Err(e) => { audit.push(ObjectAuditEntry { - rsync_uri: result.rsync_uri.clone(), - sha256_hex: result.sha256_hex, + rsync_uri: file.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&file.sha256), kind: AuditObjectKind::Roa, result: AuditObjectResult::Error, detail: Some(e.to_string()), @@ -1291,9 +1320,9 @@ pub(crate) fn reduce_parallel_roa_stage( let mut refs = vec![RfcRef("RFC 6488 §3"), RfcRef("RFC 9582 §4-§5")]; refs.extend_from_slice(extra_rfc_refs_for_crl_selection(&e)); warnings.push( - Warning::new(format!("dropping invalid ROA: {}: {e}", result.rsync_uri)) + Warning::new(format!("dropping invalid ROA: {}: {e}", file.rsync_uri)) .with_rfc_refs(&refs) - .with_context(&result.rsync_uri), + .with_context(&file.rsync_uri), ) } } @@ -1301,20 +1330,20 @@ pub(crate) fn reduce_parallel_roa_stage( let _t = timing.as_ref().map(|t| t.span_phase("objects_aspa_total")); match process_aspa_with_issuer( file, - &stage.manifest_rsync_uri, - stage.issuer_ca_der.as_ref(), - stage.issuer_ca.as_ref(), + shared.manifest_rsync_uri.as_ref(), + shared.issuer_ca_der.as_ref(), + shared.issuer_ca.as_ref(), &issuer_spki, - stage.issuer_ca_rsync_uri.as_deref(), + shared.issuer_ca_rsync_uri.as_deref(), &mut aspa_crl_cache, - stage.issuer_resources_index.as_ref(), - stage.issuer_effective_ip.as_ref(), - stage.issuer_effective_as.as_ref(), - stage.validation_time, + shared.issuer_resources_index.as_ref(), + shared.issuer_effective_ip.as_deref(), + shared.issuer_effective_as.as_deref(), + validation_time, timing, - stage.collect_vcir_local_outputs, - stage.strict_cms_der, - stage.strict_name, + collect_vcir_local_outputs, + strict_cms_der, + strict_name, ) { Ok((att, local_output)) => { stats.aspa_ok += 1; @@ -1349,6 +1378,12 @@ pub(crate) fn reduce_parallel_roa_stage( } } } + if let Some(result) = roa_results.next() { + return Err(format!( + "unexpected trailing ROA task result at index {}", + result.index + )); + } Ok(ObjectsOutput { vrps, @@ -1388,8 +1423,9 @@ fn process_publication_point_for_issuer_parallel_roa_inner stage, }; - let mut pending = std::collections::VecDeque::from(stage.build_roa_tasks()); let roa_task_count = stage.roa_task_count(); + let mut pending = std::collections::VecDeque::with_capacity(roa_task_count); + stage.append_roa_tasks_to(&mut pending); let mut worker_pool = pool .pool .lock() @@ -1537,7 +1573,6 @@ pub(crate) fn validate_roa_task_serial( strict_cms_der: bool, strict_name: bool, ) -> RoaTaskResult { - let sha256_hex = sha256_hex_from_32(&task.file.sha256); let outcome = process_roa_with_issuer( task.file, manifest_rsync_uri, @@ -1566,15 +1601,13 @@ pub(crate) fn validate_roa_task_serial( worker_index: 0, queue_wait_ms: 0, worker_ms: 0, - rsync_uri: task.file.rsync_uri.clone(), - sha256_hex, outcome, } } fn process_roa_with_issuer( file: &PackFile, - manifest_rsync_uri: &str, + _manifest_rsync_uri: &str, issuer_ca_der: &[u8], issuer_ca: &ResourceCertificate, issuer_spki: &SubjectPublicKeyInfo<'_>, @@ -1659,12 +1692,6 @@ fn process_roa_with_issuer( .iter() .map(|vrp| { let prefix = vrp_prefix_to_string(vrp); - let payload_json = serde_json::json!({ - "asn": vrp.asn, - "prefix": prefix, - "max_length": vrp.max_length, - }) - .to_string(); let rule_hash = crate::audit::sha256_hex( format!( "roa-rule:{}:{}:{}:{}", @@ -1673,20 +1700,20 @@ fn process_roa_with_issuer( .as_bytes(), ); VcirLocalOutput { - output_id: rule_hash.clone(), output_type: VcirOutputType::Vrp, item_effective_until: item_effective_until.clone(), source_object_uri: file.rsync_uri.clone(), - source_object_type: "roa".to_string(), - source_object_hash: source_object_hash.clone(), - source_ee_cert_hash: source_ee_cert_hash.clone(), - payload_json, - rule_hash, - validation_path_hint: vec![ - manifest_rsync_uri.to_string(), - file.rsync_uri.clone(), - source_object_hash.clone(), - ], + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: file.sha256, + source_ee_cert_hash: sha256_hex_to_32(&source_ee_cert_hash), + payload: VcirLocalOutputPayload::Vrp { + asn: vrp.asn, + afi: vrp.prefix.afi, + prefix_len: vrp.prefix.prefix_len, + addr: vrp.prefix.addr, + max_length: vrp.max_length, + }, + rule_hash: sha256_hex_to_32(&rule_hash), } }) .collect(); @@ -1696,7 +1723,7 @@ fn process_roa_with_issuer( fn process_roa_with_issuer_parallel_cached( file: &PackFile, - manifest_rsync_uri: &str, + _manifest_rsync_uri: &str, issuer_ca_der: &[u8], issuer_ca: &ResourceCertificate, issuer_spki: &SubjectPublicKeyInfo<'_>, @@ -1787,12 +1814,6 @@ fn process_roa_with_issuer_parallel_cached( .iter() .map(|vrp| { let prefix = vrp_prefix_to_string(vrp); - let payload_json = serde_json::json!({ - "asn": vrp.asn, - "prefix": prefix, - "max_length": vrp.max_length, - }) - .to_string(); let rule_hash = crate::audit::sha256_hex( format!( "roa-rule:{}:{}:{}:{}", @@ -1801,20 +1822,20 @@ fn process_roa_with_issuer_parallel_cached( .as_bytes(), ); VcirLocalOutput { - output_id: rule_hash.clone(), output_type: VcirOutputType::Vrp, item_effective_until: item_effective_until.clone(), source_object_uri: file.rsync_uri.clone(), - source_object_type: "roa".to_string(), - source_object_hash: source_object_hash.clone(), - source_ee_cert_hash: source_ee_cert_hash.clone(), - payload_json, - rule_hash, - validation_path_hint: vec![ - manifest_rsync_uri.to_string(), - file.rsync_uri.clone(), - source_object_hash.clone(), - ], + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: file.sha256, + source_ee_cert_hash: sha256_hex_to_32(&source_ee_cert_hash), + payload: VcirLocalOutputPayload::Vrp { + asn: vrp.asn, + afi: vrp.prefix.afi, + prefix_len: vrp.prefix.prefix_len, + addr: vrp.prefix.addr, + max_length: vrp.max_length, + }, + rule_hash: sha256_hex_to_32(&rule_hash), } }) .collect(); @@ -1824,7 +1845,7 @@ fn process_roa_with_issuer_parallel_cached( fn process_aspa_with_issuer( file: &PackFile, - manifest_rsync_uri: &str, + _manifest_rsync_uri: &str, issuer_ca_der: &[u8], issuer_ca: &ResourceCertificate, issuer_spki: &SubjectPublicKeyInfo<'_>, @@ -1922,24 +1943,17 @@ fn process_aspa_with_issuer( .as_bytes(), ); let local_output = VcirLocalOutput { - output_id: rule_hash.clone(), output_type: VcirOutputType::Aspa, item_effective_until, source_object_uri: file.rsync_uri.clone(), - source_object_type: "aspa".to_string(), - source_object_hash: source_object_hash.clone(), - source_ee_cert_hash, - payload_json: serde_json::json!({ - "customer_as_id": attestation.customer_as_id, - "provider_as_ids": attestation.provider_as_ids, - }) - .to_string(), - rule_hash, - validation_path_hint: vec![ - manifest_rsync_uri.to_string(), - file.rsync_uri.clone(), - source_object_hash, - ], + source_object_type: VcirSourceObjectType::Aspa, + source_object_hash: file.sha256, + source_ee_cert_hash: sha256_hex_to_32(&source_ee_cert_hash), + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: attestation.customer_as_id, + provider_as_ids: attestation.provider_as_ids.clone(), + }, + rule_hash: sha256_hex_to_32(&rule_hash), }; Ok((attestation, Some(local_output))) @@ -2965,6 +2979,54 @@ mod tests { assert_eq!(roa_afi_to_string(RoaAfi::Ipv6), "ipv6"); } + #[test] + fn parallel_stage_roa_tasks_share_stage_owned_payloads() { + let stage = ParallelObjectsStage { + publication_point_id: 7, + shared: Arc::new(RoaTaskShared { + locked_files: Arc::<[PackFile]>::from(vec![ + PackFile::from_bytes_with_sha256( + "rsync://example.test/repo/a.roa", + vec![1, 2, 3], + [1u8; 32], + ), + PackFile::from_bytes_with_sha256( + "rsync://example.test/repo/b.roa", + vec![4, 5, 6], + [2u8; 32], + ), + ]), + manifest_rsync_uri: Arc::::from("rsync://example.test/repo/manifest.mft"), + issuer_ca_der: Arc::from([0x01u8].as_slice()), + issuer_ca: Arc::new( + ResourceCertificate::decode_der(&fixture_bytes( + "tests/fixtures/ta/apnic-ta.cer", + )) + .expect("decode fixture CA certificate"), + ), + issuer_spki_der: Arc::from([0x02u8].as_slice()), + issuer_ca_rsync_uri: Some(Arc::::from("rsync://example.test/repo/ca.cer")), + crl_cache: Arc::new(Mutex::new(HashMap::new())), + issuer_resources_index: Arc::new(IssuerResourcesIndex::default()), + issuer_effective_ip: None, + issuer_effective_as: None, + }), + validation_time: OffsetDateTime::now_utc(), + collect_vcir_local_outputs: false, + strict_cms_der: false, + strict_name: false, + warnings: Vec::new(), + stats: ObjectsStats { + roa_total: 2, + ..ObjectsStats::default() + }, + audit: Vec::new(), + }; + let tasks = stage.build_roa_tasks(); + assert_eq!(tasks.len(), 2); + assert!(Arc::ptr_eq(&tasks[0].shared, &tasks[1].shared)); + } + #[test] fn strict_name_manifest_decode_failure_drops_publication_point() { let publication_point = PublicationPointSnapshot { diff --git a/src/validation/tree_parallel.rs b/src/validation/tree_parallel.rs index 2ab488f..3617949 100644 --- a/src/validation/tree_parallel.rs +++ b/src/validation/tree_parallel.rs @@ -55,8 +55,37 @@ struct InflightPublicationPoint { } struct FinishedPublicationPoint { - node: QueuedCaInstance, - result: Result, + node: FinishedPublicationPointNode, + result: FinishedPublicationPointResult, +} + +#[derive(Clone, Debug)] +struct FinishedPublicationPointNode { + id: u64, + parent_id: Option, + discovered_from: Option, + manifest_rsync_uri: String, +} + +impl FinishedPublicationPointNode { + fn from_queued(node: QueuedCaInstance) -> Self { + Self { + id: node.id, + parent_id: node.parent_id, + discovered_from: node.discovered_from, + manifest_rsync_uri: node.handle.manifest_rsync_uri, + } + } +} + +#[derive(Debug)] +enum FinishedPublicationPointResult { + Ok { + warnings: Vec, + objects: ObjectsOutput, + audit: PublicationPointAudit, + }, + Err(String), } struct FinalizeTask { @@ -310,24 +339,28 @@ fn elapsed_ms(started: Instant) -> u64 { fn compact_phase2_finished_result( mut result: PublicationPointRunResult, compact_audit: bool, -) -> PublicationPointRunResult { - // Phase2 only needs warnings, objects, audit, and traversal metadata after finalize. - // Dropping the snapshot here avoids retaining manifest/files/raw-byte caches until run end. - result.snapshot = None; +) -> FinishedPublicationPointResult { + result.objects.audit.clear(); + result.objects.local_outputs_cache.clear(); if compact_audit { result.audit.objects.clear(); result.audit.warnings.clear(); - result.objects.audit.clear(); - result.discovered_children.clear(); } - result + FinishedPublicationPointResult::Ok { + warnings: result.warnings, + objects: result.objects, + audit: result.audit, + } } fn compact_phase2_finished_result_result( result: Result, compact_audit: bool, -) -> Result { - result.map(|result| compact_phase2_finished_result(result, compact_audit)) +) -> FinishedPublicationPointResult { + match result { + Ok(result) => compact_phase2_finished_result(result, compact_audit), + Err(err) => FinishedPublicationPointResult::Err(err), + } } pub fn run_tree_parallel_phase2_audit_multi_root( @@ -712,8 +745,8 @@ fn start_queued_ca_instances( } Err(err) => { finished.push(FinishedPublicationPoint { - node, - result: Err(err), + node: FinishedPublicationPointNode::from_queued(node), + result: FinishedPublicationPointResult::Err(err), }); } } @@ -781,7 +814,7 @@ fn stage_ready_publication_point( metrics.child_enqueue_ms = elapsed_ms(child_enqueue_started); } finished.push(FinishedPublicationPoint { - node: ready.node, + node: FinishedPublicationPointNode::from_queued(ready.node), result: compact_phase2_finished_result_result(fallback, compact_audit), }); metrics.total_ms = elapsed_ms(publication_point_started); @@ -897,13 +930,10 @@ fn stage_ready_publication_point( metrics.locked_files = objects_stage.locked_file_count(); metrics.aspa_objects = objects_stage.aspa_task_count(); let build_tasks_started = Instant::now(); - let tasks = objects_stage.build_roa_tasks(); + objects_stage.append_roa_tasks_to(pending_roa_dispatch); metrics.build_roa_tasks_ms = elapsed_ms(build_tasks_started); let task_count = objects_stage.roa_task_count(); metrics.roa_tasks = task_count; - for task in tasks { - pending_roa_dispatch.push_back(task); - } if task_count == 0 { metrics.zero_task_count = 1; match reduce_parallel_roa_stage(objects_stage, Vec::new(), runner.timing.as_ref()) { @@ -929,8 +959,8 @@ fn stage_ready_publication_point( ); } Err(err) => finished.push(FinishedPublicationPoint { - node: ready.node, - result: Err(err), + node: FinishedPublicationPointNode::from_queued(ready.node), + result: FinishedPublicationPointResult::Err(err), }), } } else { @@ -1022,7 +1052,7 @@ fn finalize_ready_objects( ) .map(|out| out.result); finished.push(FinishedPublicationPoint { - node, + node: FinishedPublicationPointNode::from_queued(node), result: compact_phase2_finished_result_result(result, compact_audit), }); } @@ -1400,7 +1430,7 @@ fn finalize_publication_point_state( elapsed_ms(finalize_started), ) } - Err(err) => (Err(err), 0), + Err(err) => (FinishedPublicationPointResult::Err(err), 0), }; let finalize_worker_ms = elapsed_ms(finalize_worker_started); crate::progress_log::emit( @@ -1438,7 +1468,10 @@ fn finalize_publication_point_state( }), ); FinalizeWorkerResult { - finished: FinishedPublicationPoint { node, result }, + finished: FinishedPublicationPoint { + node: FinishedPublicationPointNode::from_queued(node), + result, + }, metrics: FinalizePublicationPointMetrics { reduce_ms, finalize_ms, @@ -1541,25 +1574,29 @@ fn build_tree_output(mut finished: Vec) -> TreeRunAudi for item in finished { match item.result { - Ok(result) => { + FinishedPublicationPointResult::Ok { + warnings: result_warnings, + objects, + audit, + } => { instances_processed += 1; - warnings.extend(result.warnings); - warnings.extend(result.objects.warnings); - vrps.extend(result.objects.vrps); - aspas.extend(result.objects.aspas); - router_keys.extend(result.objects.router_keys); + warnings.extend(result_warnings); + warnings.extend(objects.warnings); + vrps.extend(objects.vrps); + aspas.extend(objects.aspas); + router_keys.extend(objects.router_keys); - let mut audit: PublicationPointAudit = result.audit; + let mut audit: PublicationPointAudit = audit; audit.node_id = Some(item.node.id); audit.parent_node_id = item.node.parent_id; audit.discovered_from = item.node.discovered_from; publication_points.push(audit); } - Err(err) => { + FinishedPublicationPointResult::Err(err) => { instances_failed += 1; warnings.push( Warning::new(format!("publication point failed: {err}")) - .with_context(&item.node.handle.manifest_rsync_uri), + .with_context(&item.node.manifest_rsync_uri), ); } } @@ -1588,7 +1625,10 @@ pub fn run_tree_parallel_phase2_audit( #[cfg(test)] mod tests { - use super::{compact_phase2_finished_result, compact_phase2_finished_result_result}; + use super::{ + FinishedPublicationPointResult, compact_phase2_finished_result, + compact_phase2_finished_result_result, + }; use crate::audit::PublicationPointAudit; use crate::storage::PackTime; use crate::validation::manifest::PublicationPointSource; @@ -1638,9 +1678,12 @@ mod tests { #[test] fn compact_phase2_finished_result_drops_snapshot() { let result = compact_phase2_finished_result(sample_result(), false); - assert!(result.snapshot.is_none()); - assert_eq!(result.source, PublicationPointSource::Fresh); - assert!(result.discovered_children.is_empty()); + match result { + FinishedPublicationPointResult::Ok { warnings, .. } => { + assert!(warnings.is_empty()); + } + FinishedPublicationPointResult::Err(err) => panic!("unexpected error: {err}"), + } } #[test] @@ -1666,15 +1709,21 @@ mod tests { detail: None, }); let result = compact_phase2_finished_result(sample, true); - assert!(result.audit.objects.is_empty()); - assert!(result.audit.warnings.is_empty()); - assert!(result.objects.audit.is_empty()); + match result { + FinishedPublicationPointResult::Ok { objects, audit, .. } => { + assert!(audit.objects.is_empty()); + assert!(audit.warnings.is_empty()); + assert!(objects.audit.is_empty()); + } + FinishedPublicationPointResult::Err(err) => panic!("unexpected error: {err}"), + } } #[test] fn compact_phase2_finished_result_result_preserves_err() { - let err = compact_phase2_finished_result_result(Err("boom".to_string()), false) - .expect_err("error should be preserved"); - assert_eq!(err, "boom"); + match compact_phase2_finished_result_result(Err("boom".to_string()), false) { + FinishedPublicationPointResult::Err(err) => assert_eq!(err, "boom"), + FinishedPublicationPointResult::Ok { .. } => panic!("error should be preserved"), + } } } diff --git a/src/validation/tree_runner.rs b/src/validation/tree_runner.rs index b51f777..4a6c4cc 100644 --- a/src/validation/tree_runner.rs +++ b/src/validation/tree_runner.rs @@ -27,8 +27,8 @@ use crate::report::{RfcRef, Warning}; use crate::storage::{ PackFile, PackTime, RawByHashEntry, RocksStore, ValidatedCaInstanceResult, VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus, VcirAuditSummary, VcirCcrManifestProjection, - VcirChildEntry, VcirInstanceGate, VcirLocalOutput, VcirOutputType, VcirRelatedArtifact, - VcirSummary, + VcirChildEntry, VcirInstanceGate, VcirLocalOutput, VcirLocalOutputPayload, VcirOutputType, + VcirRelatedArtifact, VcirReplaceTimingBreakdown, VcirSourceObjectType, VcirSummary, }; use crate::sync::repo::{ sync_publication_point, sync_publication_point_replay, sync_publication_point_replay_delta, @@ -58,13 +58,17 @@ use std::collections::{HashMap, HashSet}; use std::sync::{Arc, Mutex}; use base64::Engine as _; -use serde::Deserialize; -use serde_json::json; use x509_parser::prelude::FromDer; use x509_parser::x509::SubjectPublicKeyInfo; use vcir_der::encode_access_description_der_for_vcir_ccr_projection; +fn sha256_hex_to_32(hex_value: &str) -> [u8; 32] { + let mut out = [0u8; 32]; + hex::decode_to_slice(hex_value, &mut out).expect("internal sha256 hex should decode"); + out +} + #[derive(Clone, Debug, Default)] pub(crate) struct BuildVcirTimingBreakdown { pub(crate) select_crl_ms: u64, @@ -80,9 +84,9 @@ pub(crate) struct PersistVcirTimingBreakdown { pub(crate) embedded_collect_ms: u64, pub(crate) embedded_store_ms: u64, pub(crate) build_vcir_ms: u64, - pub(crate) previous_load_ms: u64, pub(crate) replace_vcir_ms: u64, pub(crate) build_vcir: BuildVcirTimingBreakdown, + pub(crate) replace_vcir: VcirReplaceTimingBreakdown, } #[derive(Clone, Debug)] @@ -311,7 +315,7 @@ impl<'a> Rpkiv1PublicationPointRunner<'a> { self.store, ca, &pack, - &objects, + &mut objects, &warnings, &child_audits, &discovered_children, @@ -666,7 +670,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> { self.validation_time, self.timing.as_ref(), phase2_pool, - self.persist_vcir, + false, ) } else if let Some(phase2_config) = self.parallel_phase2_config.as_ref() { process_publication_point_for_issuer_parallel_roa_with_options( @@ -679,7 +683,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> { self.validation_time, self.timing.as_ref(), phase2_config, - self.persist_vcir, + false, ) } else { crate::validation::objects::process_publication_point_for_issuer_with_options( @@ -691,7 +695,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> { ca.effective_as_resources.as_ref(), self.validation_time, self.timing.as_ref(), - self.persist_vcir, + false, ) } }; @@ -747,7 +751,6 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> { "persist_embedded_collect_ms": persist_vcir_timing.embedded_collect_ms, "persist_embedded_store_ms": persist_vcir_timing.embedded_store_ms, "persist_build_vcir_ms": persist_vcir_timing.build_vcir_ms, - "persist_previous_load_ms": persist_vcir_timing.previous_load_ms, "persist_replace_vcir_ms": persist_vcir_timing.replace_vcir_ms, "persist_select_crl_ms": persist_vcir_timing.build_vcir.select_crl_ms, "persist_current_ca_decode_ms": persist_vcir_timing.build_vcir.current_ca_decode_ms, @@ -755,6 +758,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> { "persist_child_entries_ms": persist_vcir_timing.build_vcir.child_entries_ms, "persist_related_artifacts_ms": persist_vcir_timing.build_vcir.related_artifacts_ms, "persist_vcir_struct_ms": persist_vcir_timing.build_vcir.struct_build_ms, + "persist_replace_breakdown": &persist_vcir_timing.replace_vcir, "audit_build_ms": audit_build_ms, "warning_count": result.warnings.len(), "vrp_count": result.objects.vrps.len(), @@ -790,7 +794,6 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> { "persist_embedded_collect_ms": persist_vcir_timing.embedded_collect_ms, "persist_embedded_store_ms": persist_vcir_timing.embedded_store_ms, "persist_build_vcir_ms": persist_vcir_timing.build_vcir_ms, - "persist_previous_load_ms": persist_vcir_timing.previous_load_ms, "persist_replace_vcir_ms": persist_vcir_timing.replace_vcir_ms, "persist_select_crl_ms": persist_vcir_timing.build_vcir.select_crl_ms, "persist_current_ca_decode_ms": persist_vcir_timing.build_vcir.current_ca_decode_ms, @@ -798,6 +801,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> { "persist_child_entries_ms": persist_vcir_timing.build_vcir.child_entries_ms, "persist_related_artifacts_ms": persist_vcir_timing.build_vcir.related_artifacts_ms, "persist_vcir_struct_ms": persist_vcir_timing.build_vcir.struct_build_ms, + "persist_replace_breakdown": &persist_vcir_timing.replace_vcir, "audit_build_ms": audit_build_ms, }), ); @@ -1012,26 +1016,6 @@ struct VcirReuseProjection { warnings: Vec, } -#[derive(Debug, Deserialize)] -struct VcirVrpPayload { - asn: u32, - prefix: String, - max_length: u16, -} - -#[derive(Debug, Deserialize)] -struct VcirAspaPayload { - customer_as_id: u32, - provider_as_ids: Vec, -} - -#[derive(Debug, Deserialize)] -struct VcirRouterKeyPayload { - as_id: u32, - ski_hex: String, - spki_der_base64: String, -} - fn discover_children_from_fresh_snapshot_with_audit( issuer: &CaInstanceHandle, publication_point: &P, @@ -2299,7 +2283,7 @@ fn build_objects_output_from_vcir( local.source_object_uri.clone(), ObjectAuditEntry { rsync_uri: local.source_object_uri.clone(), - sha256_hex: local.source_object_hash.clone(), + sha256_hex: local.source_object_hash_hex(), kind: audit_kind_for_vcir_output_type(local.output_type), result: AuditObjectResult::Error, detail: Some( @@ -2315,7 +2299,7 @@ fn build_objects_output_from_vcir( .entry(local.source_object_uri.clone()) .or_insert_with(|| ObjectAuditEntry { rsync_uri: local.source_object_uri.clone(), - sha256_hex: local.source_object_hash.clone(), + sha256_hex: local.source_object_hash_hex(), kind: audit_kind_for_vcir_output_type(local.output_type), result: AuditObjectResult::Skipped, detail: Some("skipped: cached local output expired".to_string()), @@ -2332,7 +2316,7 @@ fn build_objects_output_from_vcir( local.source_object_uri.clone(), ObjectAuditEntry { rsync_uri: local.source_object_uri.clone(), - sha256_hex: local.source_object_hash.clone(), + sha256_hex: local.source_object_hash_hex(), kind: AuditObjectKind::Roa, result: AuditObjectResult::Ok, detail: None, @@ -2348,7 +2332,7 @@ fn build_objects_output_from_vcir( local.source_object_uri.clone(), ObjectAuditEntry { rsync_uri: local.source_object_uri.clone(), - sha256_hex: local.source_object_hash.clone(), + sha256_hex: local.source_object_hash_hex(), kind: AuditObjectKind::Roa, result: AuditObjectResult::Error, detail: Some(format!("cached ROA local output parse failed: {e}")), @@ -2364,7 +2348,7 @@ fn build_objects_output_from_vcir( local.source_object_uri.clone(), ObjectAuditEntry { rsync_uri: local.source_object_uri.clone(), - sha256_hex: local.source_object_hash.clone(), + sha256_hex: local.source_object_hash_hex(), kind: AuditObjectKind::Aspa, result: AuditObjectResult::Ok, detail: None, @@ -2380,7 +2364,7 @@ fn build_objects_output_from_vcir( local.source_object_uri.clone(), ObjectAuditEntry { rsync_uri: local.source_object_uri.clone(), - sha256_hex: local.source_object_hash.clone(), + sha256_hex: local.source_object_hash_hex(), kind: AuditObjectKind::Aspa, result: AuditObjectResult::Error, detail: Some(format!("cached ASPA local output parse failed: {e}")), @@ -2395,7 +2379,7 @@ fn build_objects_output_from_vcir( local.source_object_uri.clone(), ObjectAuditEntry { rsync_uri: local.source_object_uri.clone(), - sha256_hex: local.source_object_hash.clone(), + sha256_hex: local.source_object_hash_hex(), kind: AuditObjectKind::RouterCertificate, result: AuditObjectResult::Ok, detail: Some("cached Router Key local output restored".to_string()), @@ -2411,7 +2395,7 @@ fn build_objects_output_from_vcir( local.source_object_uri.clone(), ObjectAuditEntry { rsync_uri: local.source_object_uri.clone(), - sha256_hex: local.source_object_hash.clone(), + sha256_hex: local.source_object_hash_hex(), kind: AuditObjectKind::RouterCertificate, result: AuditObjectResult::Error, detail: Some(format!( @@ -2435,68 +2419,55 @@ fn build_objects_output_from_vcir( } fn parse_vcir_vrp_output(local: &VcirLocalOutput) -> Result { - let payload: VcirVrpPayload = serde_json::from_str(&local.payload_json) - .map_err(|e| format!("invalid VRP payload JSON: {e}"))?; - Ok(Vrp { - asn: payload.asn, - prefix: parse_vcir_prefix(&payload.prefix)?, - max_length: payload.max_length, - }) + match &local.payload { + VcirLocalOutputPayload::Vrp { + asn, + afi, + prefix_len, + addr, + max_length, + } => Ok(Vrp { + asn: *asn, + prefix: crate::data_model::roa::IpPrefix { + afi: *afi, + prefix_len: *prefix_len, + addr: *addr, + }, + max_length: *max_length, + }), + _ => Err("VCIR local output payload is not VRP".to_string()), + } } fn parse_vcir_aspa_output(local: &VcirLocalOutput) -> Result { - let payload: VcirAspaPayload = serde_json::from_str(&local.payload_json) - .map_err(|e| format!("invalid ASPA payload JSON: {e}"))?; - Ok(AspaAttestation { - customer_as_id: payload.customer_as_id, - provider_as_ids: payload.provider_as_ids, - }) + match &local.payload { + VcirLocalOutputPayload::Aspa { + customer_as_id, + provider_as_ids, + } => Ok(AspaAttestation { + customer_as_id: *customer_as_id, + provider_as_ids: provider_as_ids.clone(), + }), + _ => Err("VCIR local output payload is not ASPA".to_string()), + } } fn parse_vcir_router_key_output(local: &VcirLocalOutput) -> Result { - let payload: VcirRouterKeyPayload = serde_json::from_str(&local.payload_json) - .map_err(|e| format!("invalid Router Key payload JSON: {e}"))?; - let ski = - hex::decode(&payload.ski_hex).map_err(|e| format!("invalid Router Key SKI hex: {e}"))?; - let spki_der = base64::engine::general_purpose::STANDARD - .decode(&payload.spki_der_base64) - .map_err(|e| format!("invalid Router Key SPKI base64: {e}"))?; - Ok(RouterKeyPayload { - as_id: payload.as_id, - ski, - spki_der, - source_object_uri: local.source_object_uri.clone(), - source_object_hash: local.source_object_hash.clone(), - source_ee_cert_hash: local.source_ee_cert_hash.clone(), - item_effective_until: local.item_effective_until.clone(), - }) -} - -fn parse_vcir_prefix(prefix: &str) -> Result { - let (addr, len) = prefix - .split_once('/') - .ok_or_else(|| format!("prefix missing '/': {prefix}"))?; - let prefix_len = len - .parse::() - .map_err(|e| format!("invalid prefix length '{len}': {e}"))?; - let ip = addr - .parse::() - .map_err(|e| format!("invalid IP address '{addr}': {e}"))?; - match ip { - std::net::IpAddr::V4(v4) => Ok(crate::data_model::roa::IpPrefix { - afi: RoaAfi::Ipv4, - prefix_len, - addr: { - let mut addr = [0u8; 16]; - addr[..4].copy_from_slice(&v4.octets()); - addr - }, - }), - std::net::IpAddr::V6(v6) => Ok(crate::data_model::roa::IpPrefix { - afi: RoaAfi::Ipv6, - prefix_len, - addr: v6.octets(), + match &local.payload { + VcirLocalOutputPayload::RouterKey { + as_id, + ski, + spki_der, + } => Ok(RouterKeyPayload { + as_id: *as_id, + ski: ski.clone(), + spki_der: spki_der.clone(), + source_object_uri: local.source_object_uri.clone(), + source_object_hash: local.source_object_hash_hex(), + source_ee_cert_hash: local.source_ee_cert_hash_hex(), + item_effective_until: local.item_effective_until.clone(), }), + _ => Err("VCIR local output payload is not Router Key".to_string()), } } @@ -2582,7 +2553,7 @@ fn persist_vcir_for_fresh_result( store: &RocksStore, ca: &CaInstanceHandle, pack: &PublicationPointSnapshot, - objects: &crate::validation::objects::ObjectsOutput, + objects: &mut crate::validation::objects::ObjectsOutput, warnings: &[Warning], child_audits: &[ObjectAuditEntry], discovered_children: &[DiscoveredChildCaInstance], @@ -2605,7 +2576,7 @@ fn persist_vcir_for_fresh_result_with_timing( store: &RocksStore, ca: &CaInstanceHandle, pack: &PublicationPointSnapshot, - objects: &crate::validation::objects::ObjectsOutput, + objects: &mut crate::validation::objects::ObjectsOutput, warnings: &[Warning], child_audits: &[ObjectAuditEntry], discovered_children: &[DiscoveredChildCaInstance], @@ -2635,17 +2606,12 @@ fn persist_vcir_for_fresh_result_with_timing( timing.build_vcir_ms = build_vcir_started.elapsed().as_millis() as u64; timing.build_vcir = build_vcir_timing; - let previous_load_started = std::time::Instant::now(); - let previous = store - .get_vcir(&pack.manifest_rsync_uri) - .map_err(|e| format!("load existing VCIR failed: {e}"))?; - timing.previous_load_ms = previous_load_started.elapsed().as_millis() as u64; - let replace_vcir_started = std::time::Instant::now(); - store - .replace_vcir_and_audit_rule_indexes(previous.as_ref(), &vcir) - .map_err(|e| format!("store VCIR and audit rule index failed: {e}"))?; + let replace_timing = store + .replace_vcir_and_manifest_replay_meta(&vcir) + .map_err(|e| format!("store VCIR and manifest replay meta failed: {e}"))?; timing.replace_vcir_ms = replace_vcir_started.elapsed().as_millis() as u64; + timing.replace_vcir = replace_timing; Ok(timing) } @@ -2653,7 +2619,7 @@ fn persist_vcir_for_fresh_result_with_timing( fn build_vcir_from_fresh_result( ca: &CaInstanceHandle, pack: &PublicationPointSnapshot, - objects: &crate::validation::objects::ObjectsOutput, + objects: &mut crate::validation::objects::ObjectsOutput, warnings: &[Warning], child_audits: &[ObjectAuditEntry], discovered_children: &[DiscoveredChildCaInstance], @@ -2674,7 +2640,7 @@ fn build_vcir_from_fresh_result( fn build_vcir_from_fresh_result_with_timing( ca: &CaInstanceHandle, pack: &PublicationPointSnapshot, - objects: &crate::validation::objects::ObjectsOutput, + objects: &mut crate::validation::objects::ObjectsOutput, warnings: &[Warning], child_audits: &[ObjectAuditEntry], discovered_children: &[DiscoveredChildCaInstance], @@ -2692,7 +2658,7 @@ fn build_vcir_from_fresh_result_with_timing( timing.current_ca_decode_ms = current_ca_decode_started.elapsed().as_millis() as u64; let local_outputs_started = std::time::Instant::now(); - let local_outputs = build_vcir_local_outputs(ca, pack, objects)?; + let local_outputs = take_or_build_vcir_local_outputs(ca, pack, objects)?; timing.local_outputs_ms = local_outputs_started.elapsed().as_millis() as u64; let child_entries_started = std::time::Instant::now(); @@ -2710,6 +2676,18 @@ fn build_vcir_from_fresh_result_with_timing( timing.related_artifacts_ms = related_artifacts_started.elapsed().as_millis() as u64; let ccr_manifest_projection = build_vcir_ccr_manifest_projection_from_fresh(ca, pack, &child_entries)?; + let local_vrp_count = local_outputs + .iter() + .filter(|output| output.output_type == VcirOutputType::Vrp) + .count() as u32; + let local_aspa_count = local_outputs + .iter() + .filter(|output| output.output_type == VcirOutputType::Aspa) + .count() as u32; + let local_router_key_count = local_outputs + .iter() + .filter(|output| output.output_type == VcirOutputType::RouterKey) + .count() as u32; let accepted_object_count = related_artifacts .iter() .filter(|artifact| artifact.validation_status == VcirArtifactValidationStatus::Accepted) @@ -2768,21 +2746,12 @@ fn build_vcir_from_fresh_result_with_timing( ), }, child_entries, - local_outputs: local_outputs.clone(), - related_artifacts: related_artifacts.clone(), + local_outputs, + related_artifacts, summary: VcirSummary { - local_vrp_count: local_outputs - .iter() - .filter(|output| output.output_type == VcirOutputType::Vrp) - .count() as u32, - local_aspa_count: local_outputs - .iter() - .filter(|output| output.output_type == VcirOutputType::Aspa) - .count() as u32, - local_router_key_count: local_outputs - .iter() - .filter(|output| output.output_type == VcirOutputType::RouterKey) - .count() as u32, + local_vrp_count, + local_aspa_count, + local_router_key_count, child_count: discovered_children.len() as u32, accepted_object_count, rejected_object_count, @@ -2799,6 +2768,17 @@ fn build_vcir_from_fresh_result_with_timing( Ok((vcir, timing)) } +fn take_or_build_vcir_local_outputs( + ca: &CaInstanceHandle, + pack: &PublicationPointSnapshot, + objects: &mut crate::validation::objects::ObjectsOutput, +) -> Result, String> { + if !objects.local_outputs_cache.is_empty() { + return Ok(std::mem::take(&mut objects.local_outputs_cache)); + } + build_vcir_local_outputs(ca, pack, objects) +} + fn build_vcir_ccr_manifest_projection_from_fresh( ca: &CaInstanceHandle, pack: &PublicationPointSnapshot, @@ -2892,7 +2872,7 @@ fn select_manifest_current_crl_from_snapshot( } fn build_vcir_local_outputs( - ca: &CaInstanceHandle, + _ca: &CaInstanceHandle, pack: &PublicationPointSnapshot, objects: &crate::validation::objects::ObjectsOutput, ) -> Result, String> { @@ -2930,12 +2910,6 @@ fn build_vcir_local_outputs( PackTime::from_utc_offset_datetime(ee.resource_cert.tbs.validity_not_after); for vrp in roa_to_vrps_for_vcir(&roa) { let prefix = vrp_prefix_to_string(&vrp); - let payload_json = json!({ - "asn": vrp.asn, - "prefix": prefix, - "max_length": vrp.max_length, - }) - .to_string(); let rule_hash = sha256_hex( format!( "roa-rule:{}:{}:{}:{}", @@ -2944,20 +2918,20 @@ fn build_vcir_local_outputs( .as_bytes(), ); out.push(VcirLocalOutput { - output_id: rule_hash.clone(), output_type: VcirOutputType::Vrp, item_effective_until: item_effective_until.clone(), source_object_uri: file.rsync_uri.clone(), - source_object_type: "roa".to_string(), - source_object_hash: source_object_hash.clone(), - source_ee_cert_hash: source_ee_cert_hash.clone(), - payload_json, - rule_hash, - validation_path_hint: vec![ - ca.manifest_rsync_uri.clone(), - file.rsync_uri.clone(), - source_object_hash.clone(), - ], + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: file.sha256, + source_ee_cert_hash: sha256_hex_to_32(&source_ee_cert_hash), + payload: VcirLocalOutputPayload::Vrp { + asn: vrp.asn, + afi: vrp.prefix.afi, + prefix_len: vrp.prefix.prefix_len, + addr: vrp.prefix.addr, + max_length: vrp.max_length, + }, + rule_hash: sha256_hex_to_32(&rule_hash), }); } } else if accepted_aspa_uris.contains(file.rsync_uri.as_str()) { @@ -2970,11 +2944,6 @@ fn build_vcir_local_outputs( let source_ee_cert_hash = sha256_hex(ee.raw_der.as_slice()); let item_effective_until = PackTime::from_utc_offset_datetime(ee.resource_cert.tbs.validity_not_after); - let payload_json = json!({ - "customer_as_id": aspa.aspa.customer_as_id, - "provider_as_ids": aspa.aspa.provider_as_ids, - }) - .to_string(); let providers = aspa .aspa .provider_as_ids @@ -2990,20 +2959,17 @@ fn build_vcir_local_outputs( .as_bytes(), ); out.push(VcirLocalOutput { - output_id: rule_hash.clone(), output_type: VcirOutputType::Aspa, item_effective_until, source_object_uri: file.rsync_uri.clone(), - source_object_type: "aspa".to_string(), - source_object_hash: source_object_hash.clone(), - source_ee_cert_hash, - payload_json, - rule_hash, - validation_path_hint: vec![ - ca.manifest_rsync_uri.clone(), - file.rsync_uri.clone(), - source_object_hash, - ], + source_object_type: VcirSourceObjectType::Aspa, + source_object_hash: file.sha256, + source_ee_cert_hash: sha256_hex_to_32(&source_ee_cert_hash), + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: aspa.aspa.customer_as_id, + provider_as_ids: aspa.aspa.provider_as_ids.clone(), + }, + rule_hash: sha256_hex_to_32(&rule_hash), }); } } @@ -3011,7 +2977,7 @@ fn build_vcir_local_outputs( } pub(crate) fn build_router_key_local_outputs( - ca: &CaInstanceHandle, + _ca: &CaInstanceHandle, router_keys: &[RouterKeyPayload], ) -> Vec { router_keys @@ -3028,25 +2994,18 @@ pub(crate) fn build_router_key_local_outputs( .as_bytes(), ); VcirLocalOutput { - output_id: rule_hash.clone(), output_type: VcirOutputType::RouterKey, item_effective_until: router_key.item_effective_until.clone(), source_object_uri: router_key.source_object_uri.clone(), - source_object_type: "router_key".to_string(), - source_object_hash: router_key.source_object_hash.clone(), - source_ee_cert_hash: router_key.source_ee_cert_hash.clone(), - payload_json: json!({ - "as_id": router_key.as_id, - "ski_hex": ski_hex, - "spki_der_base64": spki_der_base64, - }) - .to_string(), - rule_hash, - validation_path_hint: vec![ - ca.manifest_rsync_uri.clone(), - router_key.source_object_uri.clone(), - router_key.source_object_hash.clone(), - ], + source_object_type: VcirSourceObjectType::RouterKey, + source_object_hash: sha256_hex_to_32(&router_key.source_object_hash), + source_ee_cert_hash: sha256_hex_to_32(&router_key.source_ee_cert_hash), + payload: VcirLocalOutputPayload::RouterKey { + as_id: router_key.as_id, + ski: router_key.ski.clone(), + spki_der: router_key.spki_der.clone(), + }, + rule_hash: sha256_hex_to_32(&rule_hash), } }) .collect() diff --git a/src/validation/tree_runner/tests.rs b/src/validation/tree_runner/tests.rs index a884a71..41c35a9 100644 --- a/src/validation/tree_runner/tests.rs +++ b/src/validation/tree_runner/tests.rs @@ -1,12 +1,13 @@ use super::*; use crate::data_model::rc::ResourceCertificate; +use crate::data_model::roa::RoaAfi; use crate::fetch::rsync::LocalDirRsyncFetcher; use crate::fetch::rsync::{RsyncFetchError, RsyncFetcher}; use crate::storage::{ PackFile, PackTime, RawByHashEntry, RocksStore, ValidatedCaInstanceResult, ValidatedManifestMeta, VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus, - VcirAuditSummary, VcirChildEntry, VcirInstanceGate, VcirLocalOutput, VcirOutputType, - VcirRelatedArtifact, VcirSummary, + VcirAuditSummary, VcirChildEntry, VcirInstanceGate, VcirLocalOutput, VcirLocalOutputPayload, + VcirOutputType, VcirRelatedArtifact, VcirSourceObjectType, VcirSummary, }; use crate::sync::rrdp::Fetcher; use crate::validation::publication_point::PublicationPointSnapshot; @@ -16,6 +17,16 @@ use std::process::Command; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; +fn sha256_32(input: &[u8]) -> [u8; 32] { + sha256_hex_to_32(&sha256_hex(input)) +} + +fn ipv4_addr(octets: [u8; 4]) -> [u8; 16] { + let mut addr = [0u8; 16]; + addr[..4].copy_from_slice(&octets); + addr +} + struct NeverHttpFetcher; impl Fetcher for NeverHttpFetcher { fn fetch(&self, _uri: &str) -> Result, String> { @@ -534,138 +545,147 @@ fn sample_vcir_for_projection( subordinate_skis: vec![vec![0x33; 20]], }; ValidatedCaInstanceResult { - manifest_rsync_uri: manifest_uri.clone(), - parent_manifest_rsync_uri: None, - tal_id: "test-tal".to_string(), - ca_subject_name: "CN=Issuer".to_string(), - ca_ski: "11".repeat(20), - issuer_ski: "22".repeat(20), - last_successful_validation_time: PackTime::from_utc_offset_datetime(now), - current_manifest_rsync_uri: manifest_uri.clone(), - current_crl_rsync_uri: current_crl_uri.clone(), - validated_manifest_meta: ValidatedManifestMeta { - validated_manifest_number: vec![1], - validated_manifest_this_update: PackTime::from_utc_offset_datetime(now), - validated_manifest_next_update: gate_until.clone(), + manifest_rsync_uri: manifest_uri.clone(), + parent_manifest_rsync_uri: None, + tal_id: "test-tal".to_string(), + ca_subject_name: "CN=Issuer".to_string(), + ca_ski: "11".repeat(20), + issuer_ski: "22".repeat(20), + last_successful_validation_time: PackTime::from_utc_offset_datetime(now), + current_manifest_rsync_uri: manifest_uri.clone(), + current_crl_rsync_uri: current_crl_uri.clone(), + validated_manifest_meta: ValidatedManifestMeta { + validated_manifest_number: vec![1], + validated_manifest_this_update: PackTime::from_utc_offset_datetime(now), + validated_manifest_next_update: gate_until.clone(), + }, + ccr_manifest_projection, + instance_gate: VcirInstanceGate { + manifest_next_update: gate_until.clone(), + current_crl_next_update: gate_until.clone(), + self_ca_not_after: PackTime::from_utc_offset_datetime(now + time::Duration::hours(2)), + instance_effective_until: gate_until.clone(), + }, + child_entries: vec![VcirChildEntry { + child_manifest_rsync_uri: child_manifest_uri, + child_cert_rsync_uri: child_cert_uri.clone(), + child_cert_hash: child_cert_hash.to_string(), + child_ski: "33".repeat(20), + child_rsync_base_uri: "rsync://example.test/repo/child/".to_string(), + child_publication_point_rsync_uri: "rsync://example.test/repo/child/".to_string(), + child_rrdp_notification_uri: Some("https://example.test/child-notify.xml".to_string()), + child_effective_ip_resources: None, + child_effective_as_resources: None, + accepted_at_validation_time: PackTime::from_utc_offset_datetime(now), + }], + local_outputs: vec![ + VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: PackTime::from_utc_offset_datetime( + now + time::Duration::minutes(30), + ), + source_object_uri: roa_uri.clone(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: sha256_hex_to_32(&roa_hash), + source_ee_cert_hash: sha256_hex_to_32(&ee_hash), + payload: VcirLocalOutputPayload::Vrp { + asn: 64496, + afi: RoaAfi::Ipv4, + prefix_len: 24, + addr: ipv4_addr([203, 0, 113, 0]), + max_length: 24, + }, + rule_hash: sha256_32(b"roa-rule"), }, - ccr_manifest_projection, - instance_gate: VcirInstanceGate { - manifest_next_update: gate_until.clone(), - current_crl_next_update: gate_until.clone(), - self_ca_not_after: PackTime::from_utc_offset_datetime(now + time::Duration::hours(2)), - instance_effective_until: gate_until.clone(), + VcirLocalOutput { + output_type: VcirOutputType::Aspa, + item_effective_until: PackTime::from_utc_offset_datetime( + now + time::Duration::minutes(30), + ), + source_object_uri: aspa_uri.clone(), + source_object_type: VcirSourceObjectType::Aspa, + source_object_hash: sha256_hex_to_32(&aspa_hash), + source_ee_cert_hash: sha256_hex_to_32(&ee_hash), + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: 64496, + provider_as_ids: vec![64497, 64498], + }, + rule_hash: sha256_32(b"aspa-rule"), }, - child_entries: vec![VcirChildEntry { - child_manifest_rsync_uri: child_manifest_uri, - child_cert_rsync_uri: child_cert_uri.clone(), - child_cert_hash: child_cert_hash.to_string(), - child_ski: "33".repeat(20), - child_rsync_base_uri: "rsync://example.test/repo/child/".to_string(), - child_publication_point_rsync_uri: "rsync://example.test/repo/child/".to_string(), - child_rrdp_notification_uri: Some("https://example.test/child-notify.xml".to_string()), - child_effective_ip_resources: None, - child_effective_as_resources: None, - accepted_at_validation_time: PackTime::from_utc_offset_datetime(now), - }], - local_outputs: vec![ - VcirLocalOutput { - output_id: sha256_hex(b"vrp-out"), - output_type: VcirOutputType::Vrp, - item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(30)), - source_object_uri: roa_uri.clone(), - source_object_type: "roa".to_string(), - source_object_hash: roa_hash.clone(), - source_ee_cert_hash: ee_hash.clone(), - payload_json: serde_json::json!({"asn": 64496, "prefix": "203.0.113.0/24", "max_length": 24}).to_string(), - rule_hash: sha256_hex(b"roa-rule"), - validation_path_hint: vec![manifest_uri.clone(), roa_uri.clone()], + VcirLocalOutput { + output_type: VcirOutputType::RouterKey, + item_effective_until: PackTime::from_utc_offset_datetime( + now + time::Duration::minutes(30), + ), + source_object_uri: router_uri.clone(), + source_object_type: VcirSourceObjectType::RouterKey, + source_object_hash: sha256_hex_to_32(&router_hash), + source_ee_cert_hash: sha256_hex_to_32(&router_hash), + payload: VcirLocalOutputPayload::RouterKey { + as_id: 64496, + ski: vec![0x11; 20], + spki_der: vec![0x30, 0x00], }, - VcirLocalOutput { - output_id: sha256_hex(b"aspa-out"), - output_type: VcirOutputType::Aspa, - item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(30)), - source_object_uri: aspa_uri.clone(), - source_object_type: "aspa".to_string(), - source_object_hash: aspa_hash.clone(), - source_ee_cert_hash: ee_hash, - payload_json: serde_json::json!({"customer_as_id": 64496, "provider_as_ids": [64497, 64498]}).to_string(), - rule_hash: sha256_hex(b"aspa-rule"), - validation_path_hint: vec![manifest_uri.clone(), aspa_uri.clone()], - }, - VcirLocalOutput { - output_id: sha256_hex(b"router-key-out"), - output_type: VcirOutputType::RouterKey, - item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(30)), - source_object_uri: router_uri.clone(), - source_object_type: "router_key".to_string(), - source_object_hash: router_hash.clone(), - source_ee_cert_hash: router_hash.clone(), - payload_json: serde_json::json!({ - "as_id": 64496, - "ski_hex": "11".repeat(20), - "spki_der_base64": base64::engine::general_purpose::STANDARD.encode([0x30u8, 0x00]), - }).to_string(), - rule_hash: sha256_hex(b"router-key-rule"), - validation_path_hint: vec![manifest_uri.clone(), router_uri.clone()], - }, - ], - related_artifacts: vec![ - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::Manifest, - artifact_kind: VcirArtifactKind::Mft, - uri: Some(manifest_uri.clone()), - sha256: manifest_hash, - object_type: Some("mft".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - }, - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::CurrentCrl, - artifact_kind: VcirArtifactKind::Crl, - uri: Some(current_crl_uri), - sha256: current_crl_hash, - object_type: Some("crl".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - }, - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::ChildCaCert, - artifact_kind: VcirArtifactKind::Cer, - uri: Some(child_cert_uri), - sha256: child_cert_hash.to_string(), - object_type: Some("cer".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - }, - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::SignedObject, - artifact_kind: VcirArtifactKind::Roa, - uri: Some(roa_uri), - sha256: roa_hash, - object_type: Some("roa".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - }, - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::SignedObject, - artifact_kind: VcirArtifactKind::Aspa, - uri: Some(aspa_uri), - sha256: aspa_hash, - object_type: Some("aspa".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - }, - ], - summary: VcirSummary { - local_vrp_count: 1, - local_aspa_count: 1, - local_router_key_count: 1, - child_count: 1, - accepted_object_count: 4, - rejected_object_count: 0, + rule_hash: sha256_32(b"router-key-rule"), }, - audit_summary: VcirAuditSummary { - failed_fetch_eligible: true, - last_failed_fetch_reason: None, - warning_count: 0, - audit_flags: Vec::new(), + ], + related_artifacts: vec![ + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::Manifest, + artifact_kind: VcirArtifactKind::Mft, + uri: Some(manifest_uri.clone()), + sha256: manifest_hash, + object_type: Some("mft".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, }, - } + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::CurrentCrl, + artifact_kind: VcirArtifactKind::Crl, + uri: Some(current_crl_uri), + sha256: current_crl_hash, + object_type: Some("crl".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + }, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::ChildCaCert, + artifact_kind: VcirArtifactKind::Cer, + uri: Some(child_cert_uri), + sha256: child_cert_hash.to_string(), + object_type: Some("cer".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + }, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: VcirArtifactKind::Roa, + uri: Some(roa_uri), + sha256: roa_hash, + object_type: Some("roa".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + }, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: VcirArtifactKind::Aspa, + uri: Some(aspa_uri), + sha256: aspa_hash, + object_type: Some("aspa".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + }, + ], + summary: VcirSummary { + local_vrp_count: 1, + local_aspa_count: 1, + local_router_key_count: 1, + child_count: 1, + accepted_object_count: 4, + rejected_object_count: 0, + }, + audit_summary: VcirAuditSummary { + failed_fetch_eligible: true, + last_failed_fetch_reason: None, + warning_count: 0, + audit_flags: Vec::new(), + }, + } } #[test] @@ -716,16 +736,20 @@ fn build_vcir_local_outputs_prefers_cached_outputs() { rrdp_notification_uri: None, }; let cached = vec![VcirLocalOutput { - output_id: "cached-output".to_string(), output_type: VcirOutputType::Vrp, item_effective_until: pack.next_update.clone(), source_object_uri: "rsync://example.test/repo/issuer/a.roa".to_string(), - source_object_type: "roa".to_string(), - source_object_hash: sha256_hex(b"cached-roa"), - source_ee_cert_hash: sha256_hex(b"cached-ee"), - payload_json: "{\"asn\":64500}".to_string(), - rule_hash: sha256_hex(b"cached-rule"), - validation_path_hint: vec![pack.manifest_rsync_uri.clone()], + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: sha256_32(b"cached-roa"), + source_ee_cert_hash: sha256_32(b"cached-ee"), + payload: VcirLocalOutputPayload::Vrp { + asn: 64500, + afi: RoaAfi::Ipv4, + prefix_len: 24, + addr: ipv4_addr([203, 0, 113, 0]), + max_length: 24, + }, + rule_hash: sha256_32(b"cached-rule"), }]; let outputs = build_vcir_local_outputs( &ca, @@ -801,7 +825,7 @@ fn persist_vcir_non_repository_evidence_stores_current_ca_cert_only() { .expect("first local output"); assert!( store - .get_raw_by_hash_entry(&first_output.source_ee_cert_hash) + .get_raw_by_hash_entry(&first_output.source_ee_cert_hash_hex()) .expect("load source ee raw") .is_none() ); @@ -838,8 +862,11 @@ fn build_router_key_local_outputs_encodes_router_key_payloads() { ); assert_eq!(outputs.len(), 1); assert_eq!(outputs[0].output_type, VcirOutputType::RouterKey); - assert_eq!(outputs[0].source_object_type, "router_key"); - assert!(outputs[0].payload_json.contains("spki_der_base64")); + assert_eq!( + outputs[0].source_object_type, + VcirSourceObjectType::RouterKey + ); + assert!(outputs[0].payload_json().contains("spki_der_base64")); } #[test] @@ -996,7 +1023,7 @@ fn finalize_fresh_publication_point_releases_local_outputs_cache_after_persist() } #[test] -fn persist_vcir_for_fresh_result_stores_vcir_and_audit_indexes_for_real_snapshot() { +fn persist_vcir_for_fresh_result_stores_vcir_and_replay_meta_for_real_snapshot() { let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests(); let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); let objects = crate::validation::objects::process_publication_point_snapshot_for_issuer( @@ -1029,8 +1056,18 @@ fn persist_vcir_for_fresh_result_stores_vcir_and_audit_indexes_for_real_snapshot rrdp_notification_uri: None, }; - persist_vcir_for_fresh_result(&store, &ca, &pack, &objects, &[], &[], &[], validation_time) - .expect("persist vcir for fresh result"); + let mut objects = objects; + persist_vcir_for_fresh_result( + &store, + &ca, + &pack, + &mut objects, + &[], + &[], + &[], + validation_time, + ) + .expect("persist vcir for fresh result"); let vcir = store .get_vcir(&pack.manifest_rsync_uri) @@ -1054,12 +1091,15 @@ fn persist_vcir_for_fresh_result_stores_vcir_and_audit_indexes_for_real_snapshot vcir.ccr_manifest_projection.manifest_size, pack.manifest_bytes.len() as u64 ); - let first_output = vcir.local_outputs.first().expect("local outputs stored"); - assert!( - store - .get_audit_rule_index_entry(crate::storage::AuditRuleKind::Roa, &first_output.rule_hash) - .expect("get audit rule index entry") - .is_some() + assert!(vcir.local_outputs.first().is_some(), "local outputs stored"); + let replay_meta = store + .get_manifest_replay_meta(&pack.manifest_rsync_uri) + .expect("get replay meta") + .expect("replay meta exists"); + assert_eq!(replay_meta.manifest_rsync_uri, pack.manifest_rsync_uri); + assert_eq!( + replay_meta.manifest_sha256, + sha2::Sha256::digest(&pack.manifest_bytes).to_vec() ); } @@ -1531,26 +1571,16 @@ fn runner_offline_rsync_fixture_produces_pack_and_warnings() { .iter() .find(|output| output.output_type == crate::storage::VcirOutputType::Vrp) .expect("first VCIR VRP output"); - let audit_rule = store - .get_audit_rule_index_entry(crate::storage::AuditRuleKind::Roa, &first_vrp.rule_hash) - .expect("get audit rule index") - .expect("audit rule index exists"); - assert_eq!(audit_rule.manifest_rsync_uri, manifest_rsync_uri); - assert_eq!(audit_rule.output_id, first_vrp.output_id); - let trace = crate::audit_trace::trace_rule_to_root( - &store, - crate::storage::AuditRuleKind::Roa, - &first_vrp.rule_hash, - ) - .expect("trace rule") - .expect("trace exists"); - assert_eq!(trace.chain_leaf_to_root.len(), 1); + assert!(!first_vrp.rule_hash_hex().is_empty()); + assert!(!first_vrp.output_id().is_empty()); + let replay_meta = store + .get_manifest_replay_meta(&manifest_rsync_uri) + .expect("get replay meta") + .expect("replay meta exists"); assert_eq!( - trace.chain_leaf_to_root[0].manifest_rsync_uri, + replay_meta.manifest_rsync_uri, manifest_rsync_uri ); - assert!(trace.source_object_raw.raw_present); - assert!(trace.source_ee_cert_raw.raw_present); } #[test] @@ -2791,10 +2821,11 @@ fn fresh_and_reuse_paths_produce_equivalent_ccr_manifest_projection() { let child_discovery = discover_children_from_fresh_snapshot_with_audit(&ca, &pack, validation_time, None) .expect("discover children"); + let mut objects = empty_objects_output(); let fresh_vcir = build_vcir_from_fresh_result( &ca, &pack, - &empty_objects_output(), + &mut objects, &[], &child_discovery.audits, &child_discovery.children, @@ -2913,70 +2944,77 @@ fn build_objects_output_from_vcir_tracks_expired_and_invalid_cached_outputs() { } vcir.local_outputs.push(VcirLocalOutput { - output_id: sha256_hex(b"bad-time"), output_type: VcirOutputType::Vrp, item_effective_until: PackTime { rfc3339_utc: "bad-time-value".to_string(), }, source_object_uri: bad_time_uri.clone(), - source_object_type: "roa".to_string(), - source_object_hash: sha256_hex(b"bad-time-src"), - source_ee_cert_hash: sha256_hex(b"bad-time-ee"), - payload_json: - serde_json::json!({"asn": 64496, "prefix": "203.0.113.0/24", "max_length": 24}) - .to_string(), - rule_hash: sha256_hex(b"bad-time-rule"), - validation_path_hint: vec![vcir.manifest_rsync_uri.clone(), bad_time_uri.clone()], + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: sha256_32(b"bad-time-src"), + source_ee_cert_hash: sha256_32(b"bad-time-ee"), + payload: VcirLocalOutputPayload::Vrp { + asn: 64496, + afi: RoaAfi::Ipv4, + prefix_len: 24, + addr: ipv4_addr([203, 0, 113, 0]), + max_length: 24, + }, + rule_hash: sha256_32(b"bad-time-rule"), }); vcir.local_outputs.push(VcirLocalOutput { - output_id: sha256_hex(b"expired"), output_type: VcirOutputType::Aspa, item_effective_until: PackTime::from_utc_offset_datetime(now - time::Duration::minutes(1)), source_object_uri: expired_uri.clone(), - source_object_type: "aspa".to_string(), - source_object_hash: sha256_hex(b"expired-src"), - source_ee_cert_hash: sha256_hex(b"expired-ee"), - payload_json: serde_json::json!({"customer_as_id": 64500, "provider_as_ids": [64501]}) - .to_string(), - rule_hash: sha256_hex(b"expired-rule"), - validation_path_hint: vec![vcir.manifest_rsync_uri.clone(), expired_uri.clone()], + source_object_type: VcirSourceObjectType::Aspa, + source_object_hash: sha256_32(b"expired-src"), + source_ee_cert_hash: sha256_32(b"expired-ee"), + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: 64500, + provider_as_ids: vec![64501], + }, + rule_hash: sha256_32(b"expired-rule"), }); vcir.local_outputs.push(VcirLocalOutput { - output_id: sha256_hex(b"bad-json"), output_type: VcirOutputType::Vrp, item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(5)), source_object_uri: bad_json_uri.clone(), - source_object_type: "roa".to_string(), - source_object_hash: sha256_hex(b"bad-json-src"), - source_ee_cert_hash: sha256_hex(b"bad-json-ee"), - payload_json: "{not-json".to_string(), - rule_hash: sha256_hex(b"bad-json-rule"), - validation_path_hint: vec![vcir.manifest_rsync_uri.clone(), bad_json_uri.clone()], + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: sha256_32(b"bad-json-src"), + source_ee_cert_hash: sha256_32(b"bad-json-ee"), + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: 64510, + provider_as_ids: vec![64511], + }, + rule_hash: sha256_32(b"bad-json-rule"), }); vcir.local_outputs.push(VcirLocalOutput { - output_id: sha256_hex(b"bad-prefix"), output_type: VcirOutputType::Vrp, item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(5)), source_object_uri: bad_prefix_uri.clone(), - source_object_type: "roa".to_string(), - source_object_hash: sha256_hex(b"bad-prefix-src"), - source_ee_cert_hash: sha256_hex(b"bad-prefix-ee"), - payload_json: serde_json::json!({"asn": 64510, "prefix": "203.0.113.0", "max_length": 24}) - .to_string(), - rule_hash: sha256_hex(b"bad-prefix-rule"), - validation_path_hint: vec![vcir.manifest_rsync_uri.clone(), bad_prefix_uri.clone()], + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: sha256_32(b"bad-prefix-src"), + source_ee_cert_hash: sha256_32(b"bad-prefix-ee"), + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: 64512, + provider_as_ids: vec![64513], + }, + rule_hash: sha256_32(b"bad-prefix-rule"), }); vcir.local_outputs.push(VcirLocalOutput { - output_id: sha256_hex(b"bad-aspa"), output_type: VcirOutputType::Aspa, item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(5)), source_object_uri: bad_aspa_uri.clone(), - source_object_type: "aspa".to_string(), - source_object_hash: sha256_hex(b"bad-aspa-src"), - source_ee_cert_hash: sha256_hex(b"bad-aspa-ee"), - payload_json: serde_json::json!({"customer_as_id": 64520}).to_string(), - rule_hash: sha256_hex(b"bad-aspa-rule"), - validation_path_hint: vec![vcir.manifest_rsync_uri.clone(), bad_aspa_uri.clone()], + source_object_type: VcirSourceObjectType::Aspa, + source_object_hash: sha256_32(b"bad-aspa-src"), + source_ee_cert_hash: sha256_32(b"bad-aspa-ee"), + payload: VcirLocalOutputPayload::Vrp { + asn: 64520, + afi: RoaAfi::Ipv4, + prefix_len: 24, + addr: ipv4_addr([198, 51, 100, 0]), + max_length: 24, + }, + rule_hash: sha256_32(b"bad-aspa-rule"), }); let mut warnings = Vec::new(); diff --git a/tests/test_objects_process_publication_point_snapshot.rs b/tests/test_objects_process_publication_point_snapshot.rs index 1d99e84..5d26f76 100644 --- a/tests/test_objects_process_publication_point_snapshot.rs +++ b/tests/test_objects_process_publication_point_snapshot.rs @@ -596,7 +596,7 @@ fn process_snapshot_for_issuer_populates_local_outputs_cache_from_real_cernet_fi assert!( out.local_outputs_cache .iter() - .all(|entry| entry.source_object_type == "roa") + .all(|entry| entry.source_object_type == rpki::storage::VcirSourceObjectType::Roa) ); }