#!/usr/bin/env python3 from __future__ import annotations import argparse import json import os import shlex import shutil import subprocess import sys import time from dataclasses import dataclass from pathlib import Path from typing import Any SCRIPT_DIR = Path(__file__).resolve().parent REPO_ROOT = SCRIPT_DIR.parents[2] DEV_ROOT = REPO_ROOT.parents[1] PORTABLE_ROOT = DEV_ROOT / "rpki-client-portable" FEATURE_BUNDLE = SCRIPT_DIR / "feature035_bundle.py" EXPERIMENTS_PATH = SCRIPT_DIR / "experiments.json" FIXTURE_MANIFEST_PATH = SCRIPT_DIR / "fixture-manifest.json" def load_json(path: Path) -> Any: with path.open("r", encoding="utf-8") as handle: return json.load(handle) def write_json(path: Path, value: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", encoding="utf-8") as handle: json.dump(value, handle, indent=2, ensure_ascii=False) handle.write("\n") def ensure_dir(path: Path) -> None: path.mkdir(parents=True, exist_ok=True) def git_commit(repo_root: Path) -> str: result = run_local( ["git", "-C", str(repo_root), "rev-parse", "--short", "HEAD"], capture=True, check=False, ) return result.stdout.strip() if result.returncode == 0 else "" def run_local( argv: list[str], *, cwd: Path | None = None, check: bool = True, capture: bool = False, env: dict[str, str] | None = None, ) -> subprocess.CompletedProcess[str]: result = subprocess.run( argv, cwd=str(cwd) if cwd else None, text=True, check=False, capture_output=capture, env=env, ) if check and result.returncode != 0: raise SystemExit( f"command failed ({result.returncode}): {' '.join(shlex.quote(item) for item in argv)}\n" f"stdout:\n{result.stdout if result.stdout else ''}\n" f"stderr:\n{result.stderr if result.stderr else ''}" ) return result def ssh_script(target: str, script: str, *, check: bool = True) -> subprocess.CompletedProcess[str]: result = subprocess.run( ["ssh", target, "bash", "-s"], input=script, text=True, check=False, ) if check and result.returncode != 0: raise SystemExit(f"remote script failed ({result.returncode}) on {target}") return result def rsync_to_remote(target: str, source: Path, destination: str) -> None: run_local(["rsync", "-a", str(source), f"{target}:{destination}"]) def rsync_dir_to_remote(target: str, source: Path, destination: str) -> None: run_local(["rsync", "-a", f"{source}/", f"{target}:{destination}/"]) def rsync_from_remote(target: str, source: str, destination: Path) -> None: ensure_dir(destination) run_local(["rsync", "-a", f"{target}:{source}/", f"{destination}/"]) def rel_cmd(parts: list[str]) -> str: return shlex.join(str(part) for part in parts) def utc_stamp() -> str: return time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) def detect_libtls_path(rpki_client_bin: Path) -> Path: ldd = run_local(["ldd", str(rpki_client_bin)], capture=True) for line in ldd.stdout.splitlines(): if "libtls.so.28" not in line: continue if "=>" not in line: continue candidate = line.split("=>", 1)[1].strip().split(" ", 1)[0] path = Path(candidate) if path.is_file(): return path fallback = DEV_ROOT / ".cache" / "rpki-client-9.8-cir" / "libtls.so.28" if fallback.is_file(): return fallback raise SystemExit("unable to locate libtls.so.28 for rpki-client") def parse_elapsed_to_ms(raw: str) -> int: raw = raw.strip() if not raw: return 0 if "-" in raw: days, raw = raw.split("-", 1) else: days = "0" parts = raw.split(":") try: if len(parts) == 4: days = str(int(days) + int(parts[0])) hours, minutes, seconds = parts[1:] elif len(parts) == 3: hours, minutes, seconds = parts elif len(parts) == 2: hours = "0" minutes, seconds = parts else: hours = "0" minutes = "0" seconds = parts[0] total_seconds = ( int(days) * 86400 + int(hours) * 3600 + int(minutes) * 60 + float(seconds) ) except ValueError: time_part = raw.rsplit(":", 1)[-1] total_seconds = float(time_part) return int(round(total_seconds * 1000)) def parse_time_file(path: Path) -> dict[str, Any]: data: dict[str, Any] = {} if not path.is_file(): return data for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): if "Elapsed (wall clock) time" in line and ":" in line: if "):" in line: elapsed = line.rsplit("):", 1)[1] else: elapsed = line.rsplit(":", 1)[1] data["wallMs"] = parse_elapsed_to_ms(elapsed) elif "Maximum resident set size" in line and ":" in line: try: data["maxRssKb"] = int(line.rsplit(":", 1)[1].strip()) except ValueError: pass elif "User time (seconds)" in line and ":" in line: try: data["userSeconds"] = float(line.rsplit(":", 1)[1].strip()) except ValueError: pass elif "System time (seconds)" in line and ":" in line: try: data["systemSeconds"] = float(line.rsplit(":", 1)[1].strip()) except ValueError: pass return data def path_within(base: Path, path: Path) -> str: return path.relative_to(base).as_posix() def load_report_counts(report_path: Path) -> dict[str, Any]: report = load_json(report_path) publication_points = report.get("publication_points", []) tree = report.get("tree", {}) warnings = tree.get("warnings", []) return { "vrps": len(report.get("vrps", [])), "aspas": len(report.get("aspas", [])), "publicationPoints": len(publication_points), "warnings": len(warnings) + sum(len(pp.get("warnings", [])) for pp in publication_points if isinstance(pp, dict)), "treeInstancesProcessed": tree.get("instances_processed"), "treeInstancesFailed": tree.get("instances_failed"), "reportJson": report, } def load_rpki_client_counts(report_path: Path) -> dict[str, Any]: report = load_json(report_path) metadata = report.get("metadata", {}) roas = report.get("roas", []) aspas = report.get("aspas", []) return { "vrps": int(metadata.get("vrps", len(roas))), "uniqueVrps": int(metadata.get("uniquevrps", len(roas))), "aspas": int(metadata.get("aspas", len(aspas))), "uniqueVaps": int(metadata.get("uniquevaps", len(aspas))), "repositories": int(metadata.get("repositories", 0)), "warnings": 0, "reportJson": report, } def load_cir_counts(cir_path: Path) -> dict[str, int]: if not cir_path.is_file(): return { "cirObjectCount": 0, "cirRejectCount": 0, "cirTrustAnchorCount": 0, } result = run_local( [ str(REPO_ROOT / "target" / "release" / "cir_dump_reject_list"), "--cir", str(cir_path), "--limit", "0", ], capture=True, check=False, ) if result.returncode != 0: raise SystemExit(f"decode CIR failed: {cir_path}\n{result.stderr}") values: dict[str, int] = {} for line in result.stdout.splitlines(): if "=" not in line: continue key, value = line.split("=", 1) if key in {"object_count", "trust_anchor_count", "reject_count"}: values[key] = int(value) return { "cirObjectCount": values.get("object_count", 0), "cirRejectCount": values.get("reject_count", 0), "cirTrustAnchorCount": values.get("trust_anchor_count", 0), } def copy_rpki_client_outputs(run_dir: Path) -> None: if (run_dir / "json").is_file(): shutil.copy2(run_dir / "json", run_dir / "report.json") if (run_dir / "rpki.ccr").is_file(): shutil.copy2(run_dir / "rpki.ccr", run_dir / "result.ccr") if (run_dir / "rpki.cir").is_file(): shutil.copy2(run_dir / "rpki.cir", run_dir / "result.cir") def write_rpki_client_stage_timing(run_dir: Path) -> None: report_path = run_dir / "report.json" if not report_path.is_file(): return report = load_json(report_path) metadata = report.get("metadata", {}) stage_timing = { "tool": "rpki-client-portable", "metadata": { "elapsedTimeSeconds": metadata.get("elapsedtime"), "userTimeSeconds": metadata.get("usertime"), "systemTimeSeconds": metadata.get("systemtime"), }, "counts": { "repositories": metadata.get("repositories"), "vrps": metadata.get("vrps"), "uniqueVrps": metadata.get("uniquevrps"), "vaps": metadata.get("vaps"), "uniqueVaps": metadata.get("uniquevaps"), }, } write_json(run_dir / "stage-timing.json", stage_timing) def build_tool_binaries() -> None: run_local( [ "cargo", "build", "--release", "--bin", "rpki", "--bin", "triage_ccr_cir_pair", "--bin", "cir_dump_reject_list", ], cwd=REPO_ROOT, ) run_local(["make", "-j2"], cwd=PORTABLE_ROOT) def build_fixture_proof(run_root: Path, rirs: list[str]) -> Path: fixture_dir = REPO_ROOT / "tests" / "fixtures" fixture_proof = run_root / "fixture-proof.json" run_local( [ sys.executable, str(FEATURE_BUNDLE), "fixture-proof", "--fixture-dir", str(fixture_dir), "--repo-root", str(run_root), "--rirs", ",".join(rirs), "--out", str(fixture_proof), ] ) return fixture_proof def experiment_steps(exp: dict[str, Any]) -> list[dict[str, str]]: return [ { "step": "snapshot", "left": exp["left"], "right": exp["right"], }, { "step": "delta", "left": exp["left"], "right": exp["right"], }, ] def render_experiment_plan(exp: dict[str, Any], run_root: Path, remote_root: Path, ssh_target: str, rirs: list[str]) -> dict[str, Any]: plan_steps: list[dict[str, Any]] = [] for step in ("snapshot", "delta"): plan_steps.append( { "step": step, "leftRunDir": str(run_root / "experiments" / exp["id"] / "A" / step), "rightRunDir": str(run_root / "experiments" / exp["id"] / "B" / step), "remoteLeftRunDir": str(remote_root / "experiments" / exp["id"] / "A" / step), "remoteRightRunDir": str(remote_root / "experiments" / exp["id"] / "B" / step), "leftCommand": build_side_command(remote_root, exp["id"], exp["left"], "A", step, rirs), "rightCommand": build_side_command(remote_root, exp["id"], exp["right"], "B", step, rirs), } ) return { "id": exp["id"], "left": exp["left"], "right": exp["right"], "sshTarget": ssh_target, "remoteRoot": str(remote_root), "runRoot": str(run_root), "steps": plan_steps, } def build_side_command( remote_root: Path, exp_id: str, side: dict[str, Any], side_label: str, step: str, rirs: list[str], ) -> str: run_dir = remote_root / "experiments" / exp_id / side_label / step state_dir = remote_root / "experiments" / exp_id / side_label / "state" / ("ours" if side["rpKind"] == "ours" else "rpki-client") fixture_root = remote_root / "fixtures" if side["rpKind"] == "ours": argv = [ str(remote_root / "bin" / "rpki"), "--db", str(state_dir / "work-db"), "--raw-store-db", str(state_dir / "raw-store.db"), "--repo-bytes-db", str(state_dir / "repo-bytes.db"), ] for rir in rirs: argv.extend( [ "--tal-path", str(fixture_root / "tal" / fixture_name(rir, "tal")), "--ta-path", str(fixture_root / "ta" / fixture_name(rir, "ta")), ] ) if side["protocol"] == "rsync-only": argv.append("--disable-rrdp") if side["mode"] == "strict-name": argv.extend(["--strict", "name"]) elif side["mode"] == "strict-cms-der": argv.extend(["--strict", "cms-der"]) elif side["mode"] == "strict-signed-attrs": argv.extend(["--strict", "signed-attrs"]) elif side["mode"] == "strict-all": argv.extend(["--strict", "all"]) argv.extend( [ "--report-json", str(run_dir / "report.json"), "--report-json-compact", "--ccr-out", str(run_dir / "result.ccr"), "--cir-enable", "--cir-out", str(run_dir / "result.cir"), ] ) for rir in rirs: argv.extend(["--cir-tal-uri", cir_tal_uri_for_rir(rir)]) argv.extend( [ "--vrps-csv-out", str(run_dir / "vrps.csv"), "--vaps-csv-out", str(run_dir / "vaps.csv"), "--compare-view-trust-anchor", compare_view_trust_anchor(rirs), ] ) return "cd {run_dir} && {prefix} /usr/bin/time -v -o process-time.txt -- {cmd} > stdout.log 2> stderr.log".format( run_dir=shlex.quote(str(run_dir)), prefix=_setup_prefix(step, state_dir, "ours"), cmd=rel_cmd(argv), ) argv = [ str(remote_root / "bin" / "rpki-client"), "-vv", "-S", str(state_dir / "rpki-client-skiplist"), ] if side["protocol"] == "rsync-only": argv.append("-R") for rir in rirs: argv.extend( [ "-t", str(fixture_root / "tal" / fixture_name(rir, "tal")), "-T", f"{fixture_desc(rir)}:{state_dir / 'cache' / 'fixtures' / fixture_name(rir, 'ta')}", ] ) argv.extend(["-d", str(state_dir / "cache"), str(run_dir)]) return "cd {run_dir} && {prefix} LD_LIBRARY_PATH={lib} /usr/bin/time -v -o process-time.txt -- {cmd} > stdout.log 2> stderr.log".format( run_dir=shlex.quote(str(run_dir)), prefix=_setup_prefix(step, state_dir, "rpki-client"), lib=shlex.quote(str(remote_root / "lib")), cmd=rel_cmd(argv), ) def _setup_prefix(step: str, state_dir: Path, kind: str) -> str: if step == "snapshot": if kind == "ours": return "rm -rf {state} && mkdir -p {state}/work-db {state}/rsync-mirror && chmod -R 0777 {state} && ".format( state=shlex.quote(str(state_dir)) ) return "rm -rf {state} && mkdir -p {state}/cache {state}/cache/fixtures && touch {state}/rpki-client-skiplist && chmod -R 0777 {state} && ".format( state=shlex.quote(str(state_dir)) ) if kind == "ours": return "mkdir -p {state}/work-db {state}/rsync-mirror && chmod -R 0777 {state} && ".format( state=shlex.quote(str(state_dir)) ) return "mkdir -p {state}/cache {state}/cache/fixtures && touch {state}/rpki-client-skiplist && chmod -R 0777 {state} && ".format( state=shlex.quote(str(state_dir)) ) def fixture_name(rir: str, kind: str) -> str: fixture_manifest = load_json(FIXTURE_MANIFEST_PATH) return Path(fixture_manifest["rirs"][rir][kind]).name def fixture_desc(rir: str) -> str: return { "afrinic": "afrinic", "apnic": "apnic-rfc7730-https", "arin": "arin", "lacnic": "lacnic", "ripe": "ripe-ncc", }[rir] def cir_tal_uri_for_rir(rir: str) -> str: return { "afrinic": "https://rpki.afrinic.net/tal/afrinic.tal", "apnic": "https://rpki.apnic.net/tal/apnic-rfc7730-https.tal", "arin": "https://www.arin.net/resources/manage/rpki/arin.tal", "lacnic": "https://www.lacnic.net/innovaportal/file/4983/1/lacnic.tal", "ripe": "https://tal.rpki.ripe.net/ripe-ncc.tal", }[rir] def compare_view_trust_anchor(rirs: list[str]) -> str: return "all5" if len(rirs) > 1 else rirs[0] def sanitize_run_meta( run_root: Path, exp_id: str, side_label: str, step: str, rp_kind: str, rp_mode: str, protocol: str, strict_policies: str, rirs: list[str], run_dir: Path, fixture_proof: Path, result_ccr: Path, result_cir: Path, report_json: Path, stage_timing_json: Path, process_time: Path, stdout_log: Path, stderr_log: Path, exit_code: int, counts: dict[str, Any], time_info: dict[str, Any], fixture_pinned: bool, ) -> Path: repo_root = run_root run_meta_path = run_dir / "run-meta.json" run_meta_args = [ sys.executable, str(FEATURE_BUNDLE), "run-meta", "--out", str(run_meta_path), "--repo-root", str(repo_root), "--experiment-id", exp_id, "--side", "left" if side_label == "A" else "right", "--side-label", side_label, "--step", step, "--run-id", f"{side_label}-{step}", "--rp-kind", rp_kind, "--rp-binary", "bin/rpki" if rp_kind == "ours" else "bin/rpki-client", "--rp-version", "portable" if rp_kind == "rpki-client" else "ours", "--rp-mode", rp_mode, "--protocol-mode", protocol, "--strict-policies", strict_policies, "--rirs", ",".join(rirs), "--argv-json", json.dumps([]), "--env-json", json.dumps({}), "--cwd", str(run_root), "--reset-before-run", "--state-root", str(run_dir.parent / "state"), "--db", str(run_dir.parent / "state" / "work-db"), "--repo-bytes-db", str(run_dir.parent / "state" / "repo-bytes.db"), "--raw-store-db", str(run_dir.parent / "state" / "raw-store.db"), "--rsync-mirror-root", str(run_dir.parent / "state" / "rsync-mirror"), "--cache-root", str(run_dir.parent / "state" / ("cache" if rp_kind == "rpki-client" else "work-db")), "--ccr", str(result_ccr), "--cir", str(result_cir), "--fixture-proof", str(fixture_proof), "--fixture-proof-summary-json", json.dumps( { "taFixturePinned": fixture_pinned, "taOnlineFetchObserved": False, "trustAnchorCount": len(rirs), } ), "--report-json", str(report_json), "--stage-timing-json", str(stage_timing_json), "--stdout-log", str(stdout_log), "--stderr-log", str(stderr_log), "--process-time", str(process_time), "--exit-code", str(exit_code), "--wall-ms", str(time_info.get("wallMs", 0)), "--max-rss-kb", str(time_info.get("maxRssKb", 0)), "--vrps", str(counts.get("vrps", 0)), "--vaps", str(counts.get("aspas", 0)), "--publication-points", str(counts.get("publicationPoints", counts.get("repositories", 0))), "--warnings", str(counts.get("warnings", 0)), "--cir-object-count", str(counts.get("cirObjectCount", 0)), "--cir-reject-count", str(counts.get("cirRejectCount", 0)), "--cir-trust-anchor-count", str(counts.get("cirTrustAnchorCount", len(rirs))), "--host", os.uname().nodename, "--platform", sys.platform, ] run_local(run_meta_args) return run_meta_path def run_remote_step( ssh_target: str, remote_root: Path, exp_id: str, side_label: str, step: str, side: dict[str, Any], rirs: list[str], ) -> None: exp_root = remote_root / "experiments" / exp_id run_dir = exp_root / side_label / step state_dir = exp_root / side_label / "state" / ("ours" if side["rpKind"] == "ours" else "rpki-client") ensure = [ f"mkdir -p {shlex.quote(str(run_dir))}", f"mkdir -p {shlex.quote(str(run_dir.parent))}", f"chmod 0777 {shlex.quote(str(run_dir))}", ] if side["rpKind"] == "ours": ensure.extend( [ f"mkdir -p {shlex.quote(str(state_dir / 'work-db'))}", f"mkdir -p {shlex.quote(str(state_dir / 'rsync-mirror'))}", f"chmod -R 0777 {shlex.quote(str(exp_root / side_label / 'state'))}", ] ) if step == "snapshot": ensure.insert(0, f"rm -rf {shlex.quote(str(exp_root / side_label / 'state' / 'ours'))}") argv = [ str(remote_root / "bin" / "rpki"), "--db", str(state_dir / "work-db"), "--raw-store-db", str(state_dir / "raw-store.db"), "--repo-bytes-db", str(state_dir / "repo-bytes.db"), ] for rir in rirs: argv.extend( [ "--tal-path", str(remote_root / "fixtures" / "tal" / fixture_name(rir, "tal")), "--ta-path", str(remote_root / "fixtures" / "ta" / fixture_name(rir, "ta")), ] ) if side["protocol"] == "rsync-only": argv.append("--disable-rrdp") if side["mode"] == "strict-name": argv.extend(["--strict", "name"]) elif side["mode"] == "strict-cms-der": argv.extend(["--strict", "cms-der"]) elif side["mode"] == "strict-signed-attrs": argv.extend(["--strict", "signed-attrs"]) elif side["mode"] == "strict-all": argv.extend(["--strict", "all"]) argv.extend( [ "--report-json", str(run_dir / "report.json"), "--report-json-compact", "--ccr-out", str(run_dir / "result.ccr"), "--cir-enable", "--cir-out", str(run_dir / "result.cir"), ] ) for rir in rirs: argv.extend(["--cir-tal-uri", cir_tal_uri_for_rir(rir)]) argv.extend( [ "--vrps-csv-out", str(run_dir / "vrps.csv"), "--vaps-csv-out", str(run_dir / "vaps.csv"), "--compare-view-trust-anchor", compare_view_trust_anchor(rirs), ] ) else: ensure.extend( [ f"mkdir -p {shlex.quote(str(state_dir / 'cache'))}", f"mkdir -p {shlex.quote(str(state_dir / 'cache' / 'fixtures'))}", f"touch {shlex.quote(str(state_dir / 'rpki-client-skiplist'))}", f"chmod -R 0777 {shlex.quote(str(exp_root / side_label / 'state'))}", ] ) if step == "snapshot": ensure.insert(0, f"rm -rf {shlex.quote(str(exp_root / side_label / 'state' / 'rpki-client'))}") for rir in rirs: ensure.append( f"cp -f {shlex.quote(str(remote_root / 'fixtures' / 'ta' / fixture_name(rir, 'ta')))} {shlex.quote(str(state_dir / 'cache' / 'fixtures' / fixture_name(rir, 'ta')))}" ) argv = [ str(remote_root / "bin" / "rpki-client"), "-vv", "-S", str(state_dir / "rpki-client-skiplist"), ] if side["protocol"] == "rsync-only": argv.append("-R") for rir in rirs: argv.extend( [ "-t", str(remote_root / "fixtures" / "tal" / fixture_name(rir, "tal")), "-T", f"{fixture_desc(rir)}:{state_dir / 'cache' / 'fixtures' / fixture_name(rir, 'ta')}", ] ) argv.extend(["-d", str(state_dir / "cache"), str(run_dir)]) time_prefix = "/usr/bin/time" if side["rpKind"] == "ours": time_prefix = "env RPKI_PROGRESS_LOG=1 RPKI_PROGRESS_SLOW_SECS=10 /usr/bin/time" elif side["rpKind"] == "rpki-client": time_prefix = f"env LD_LIBRARY_PATH={shlex.quote(str(remote_root / 'lib'))} /usr/bin/time" command = ( "set -euo pipefail; " + "; ".join(ensure) + "; " + "set +e; " + time_prefix + " -v -o " + shlex.quote(str(run_dir / "process-time.txt")) + " -- " + rel_cmd(argv) + " > " + shlex.quote(str(run_dir / "stdout.log")) + " 2> " + shlex.quote(str(run_dir / "stderr.log")) + "; ec=$?; set -e; printf '%s\n' \"$ec\" > " + shlex.quote(str(run_dir / "exit-code.txt")) + "; true" ) ssh_script(ssh_target, command) if side["rpKind"] == "rpki-client": copy_cmd = ( f"[ -f {shlex.quote(str(run_dir / 'json'))} ] && cp -f {shlex.quote(str(run_dir / 'json'))} {shlex.quote(str(run_dir / 'report.json'))} || true; " f"[ -f {shlex.quote(str(run_dir / 'rpki.ccr'))} ] && cp -f {shlex.quote(str(run_dir / 'rpki.ccr'))} {shlex.quote(str(run_dir / 'result.ccr'))} || true; " f"[ -f {shlex.quote(str(run_dir / 'rpki.cir'))} ] && cp -f {shlex.quote(str(run_dir / 'rpki.cir'))} {shlex.quote(str(run_dir / 'result.cir'))} || true; " f"[ -f {shlex.quote(str(run_dir / 'report.json'))} ] && python3 - <<'PY' {shlex.quote(str(run_dir / 'report.json'))} {shlex.quote(str(run_dir / 'stage-timing.json'))} || true\n" "import json, sys\n" "report = json.load(open(sys.argv[1]))\n" "meta = report.get('metadata', {})\n" "stage = {\n" " 'tool': 'rpki-client-portable',\n" " 'metadata': {\n" " 'elapsedTimeSeconds': meta.get('elapsedtime'),\n" " 'userTimeSeconds': meta.get('usertime'),\n" " 'systemTimeSeconds': meta.get('systemtime'),\n" " },\n" " 'counts': {\n" " 'repositories': meta.get('repositories'),\n" " 'vrps': meta.get('vrps'),\n" " 'uniqueVrps': meta.get('uniquevrps'),\n" " 'vaps': meta.get('vaps'),\n" " 'uniqueVaps': meta.get('uniquevaps'),\n" " }\n" "}\n" "json.dump(stage, open(sys.argv[2], 'w'), indent=2)\n" "print()\n" "PY" ) ssh_script(ssh_target, copy_cmd) def parse_exit_code(path: Path) -> int: if not path.is_file(): return 1 return int(path.read_text(encoding="utf-8").strip() or "1") def generate_run_meta( local_exp_root: Path, exp_id: str, side_label: str, step: str, side: dict[str, Any], rirs: list[str], fixture_proof: Path, run_dir: Path, ) -> Path: result_ccr = run_dir / "result.ccr" result_cir = run_dir / "result.cir" report_json = run_dir / "report.json" stage_timing_json = run_dir / "stage-timing.json" process_time = run_dir / "process-time.txt" stdout_log = run_dir / "stdout.log" stderr_log = run_dir / "stderr.log" exit_code = parse_exit_code(run_dir / "exit-code.txt") time_info = parse_time_file(process_time) if side["rpKind"] == "ours": counts = load_report_counts(report_json) counts.update(load_cir_counts(result_cir)) else: counts = load_rpki_client_counts(report_json) counts.update(load_cir_counts(result_cir)) strict_policies = "" if side["mode"] == "strict-name": strict_policies = "name" elif side["mode"] == "strict-cms-der": strict_policies = "cms-der" elif side["mode"] == "strict-signed-attrs": strict_policies = "signed-attrs" elif side["mode"] == "strict-all": strict_policies = "all" meta_path = local_exp_root / side_label / step / "run-meta.json" generate_meta_args = [ sys.executable, str(FEATURE_BUNDLE), "run-meta", "--out", str(meta_path), "--repo-root", str(local_exp_root), "--experiment-id", exp_id, "--side", "left" if side_label == "A" else "right", "--side-label", side_label, "--step", step, "--run-id", f"{side_label}-{step}", "--rp-kind", side["rpKind"], "--rp-binary", "bin/rpki" if side["rpKind"] == "ours" else "bin/rpki-client", "--rp-version", "portable-9.8" if side["rpKind"] == "rpki-client" else "ours-dev", "--rp-git-commit", git_commit(REPO_ROOT) or "", "--rp-mode", side["mode"], "--protocol-mode", side["protocol"], "--strict-policies", strict_policies, "--rirs", ",".join(rirs), "--argv-json", json.dumps([]), "--env-json", json.dumps({}), "--cwd", str(local_exp_root), "--state-root", str(local_exp_root / side_label / "state"), "--db", str(local_exp_root / side_label / "state" / "ours" / "work-db"), "--repo-bytes-db", str(local_exp_root / side_label / "state" / "ours" / "repo-bytes.db"), "--raw-store-db", str(local_exp_root / side_label / "state" / "ours" / "raw-store.db"), "--rsync-mirror-root", str(local_exp_root / side_label / "state" / "ours" / "rsync-mirror"), "--cache-root", str(local_exp_root / side_label / "state" / "rpki-client" / "cache"), "--ccr", str(result_ccr), "--cir", str(result_cir), "--fixture-proof", str(fixture_proof), "--fixture-proof-summary-json", json.dumps( { "taFixturePinned": True, "taOnlineFetchObserved": False, "trustAnchorCount": len(rirs), } ), "--report-json", str(report_json), "--stage-timing-json", str(stage_timing_json), "--stdout-log", str(stdout_log), "--stderr-log", str(stderr_log), "--process-time", str(process_time), "--exit-code", str(exit_code), "--wall-ms", str(int(time_info.get("wallMs", 0))), "--max-rss-kb", str(int(time_info.get("maxRssKb", 0))), "--vrps", str(int(counts.get("vrps", 0))), "--vaps", str(int(counts.get("aspas", 0))), "--publication-points", str(int(counts.get("publicationPoints", counts.get("repositories", 0)))), "--warnings", str(int(counts.get("warnings", 0))), "--cir-object-count", str(int(counts.get("cirObjectCount", 0))), "--cir-reject-count", str(int(counts.get("cirRejectCount", 0))), "--cir-trust-anchor-count", str(int(counts.get("cirTrustAnchorCount", len(rirs)))), "--host", os.uname().nodename, "--platform", sys.platform, ] if step == "snapshot": insert_at = generate_meta_args.index("--state-root") generate_meta_args.insert(insert_at, "--reset-before-run") run_local(generate_meta_args) return meta_path def run_experiment( ssh_target: str, local_run_root: Path, remote_root: Path, exp: dict[str, Any], rirs: list[str], dry_run: bool = False, ) -> dict[str, Any]: exp_id = exp["id"] local_exp_root = local_run_root / "experiments" / exp_id remote_exp_root = remote_root / "experiments" / exp_id ensure_dir(local_exp_root) if dry_run: return render_experiment_plan(exp, local_run_root, remote_root, ssh_target, rirs) fixture_proof = build_fixture_proof(local_exp_root, rirs) preflight = ( "set -euo pipefail; " f"df -h /data / || true; " "systemctl disable --now rpki-client.timer >/dev/null 2>&1 || true; " "systemctl stop rpki-client.service >/dev/null 2>&1 || true; " "pkill -f '[/]rpki-client([[:space:]]|$)' >/dev/null 2>&1 || true; " "pkill -f '[/]routinator([[:space:]]|$)' >/dev/null 2>&1 || true; " "id -u _rpki-client >/dev/null 2>&1 || useradd -r -M -s /usr/sbin/nologin _rpki-client || true; " f"mkdir -p {shlex.quote(str(remote_root / 'bin'))} {shlex.quote(str(remote_root / 'lib'))} {shlex.quote(str(remote_root / 'fixtures' / 'tal'))} {shlex.quote(str(remote_root / 'fixtures' / 'ta'))} {shlex.quote(str(remote_exp_root))}" ) ssh_script(ssh_target, preflight) rsync_dir_to_remote(ssh_target, REPO_ROOT / "tests" / "fixtures" / "tal", remote_root / "fixtures" / "tal") rsync_dir_to_remote(ssh_target, REPO_ROOT / "tests" / "fixtures" / "ta", remote_root / "fixtures" / "ta") if exp["left"]["rpKind"] == "rpki-client" or exp["right"]["rpKind"] == "rpki-client": rsync_to_remote(ssh_target, detect_libtls_path(PORTABLE_ROOT / "src" / "rpki-client"), str(remote_root / "lib" / "libtls.so.28")) rsync_to_remote(ssh_target, PORTABLE_ROOT / "src" / "rpki-client", str(remote_root / "bin" / "rpki-client")) rsync_to_remote(ssh_target, REPO_ROOT / "target" / "release" / "rpki", str(remote_root / "bin" / "rpki")) rsync_to_remote(ssh_target, REPO_ROOT / "target" / "release" / "triage_ccr_cir_pair", str(remote_root / "bin" / "triage_ccr_cir_pair")) rsync_to_remote(ssh_target, REPO_ROOT / "target" / "release" / "cir_dump_reject_list", str(remote_root / "bin" / "cir_dump_reject_list")) step_results: list[dict[str, Any]] = [] for step in ("snapshot", "delta"): for side_label, side in (("A", exp["left"]), ("B", exp["right"])): run_remote_step(ssh_target, remote_root, exp_id, side_label, step, side, rirs) local_step_root = local_exp_root / side_label / step remote_step_root = remote_exp_root / side_label / step rsync_from_remote(ssh_target, str(remote_step_root), local_step_root) if side["rpKind"] == "rpki-client": copy_rpki_client_outputs(local_step_root) write_rpki_client_stage_timing(local_step_root) # ensure ours has a stable stage timing artifact too if not (local_step_root / "stage-timing.json").is_file() and (local_step_root / "report.json").is_file(): report = load_json(local_step_root / "report.json") stage_timing = { "tool": side["rpKind"], "counts": { "publicationPoints": len(report.get("publication_points", [])), "vrps": len(report.get("vrps", [])), "aspas": len(report.get("aspas", [])), }, } write_json(local_step_root / "stage-timing.json", stage_timing) meta_path = generate_run_meta( local_exp_root, exp_id, side_label, step, side, rirs, fixture_proof, local_step_root, ) step_results.append( { "side": side_label, "step": step, "runDir": str(local_step_root), "meta": str(meta_path), } ) compare_dir = local_exp_root / "compare" / step ensure_dir(compare_dir) triage_cmd = [ str(REPO_ROOT / "target" / "release" / "triage_ccr_cir_pair"), "--left-ccr", str(local_exp_root / "A" / step / "result.ccr"), "--left-cir", str(local_exp_root / "A" / step / "result.cir"), "--left-meta", str(local_exp_root / "A" / step / "run-meta.json"), "--right-ccr", str(local_exp_root / "B" / step / "result.ccr"), "--right-cir", str(local_exp_root / "B" / step / "result.cir"), "--right-meta", str(local_exp_root / "B" / step / "run-meta.json"), "--out-dir", str(compare_dir), "--sample-limit", "200", "--compare-view-trust-anchor", compare_view_trust_anchor(rirs), ] run_local(triage_cmd, cwd=local_exp_root) step_summary = { "step": step, "compareDir": str(compare_dir), "triage": load_json(compare_dir / "triage.json"), } write_json(local_exp_root / "compare" / f"{step}.summary.json", step_summary) step_results.append(step_summary) experiment_summary = { "schemaVersion": 1, "experimentId": exp_id, "left": exp["left"], "right": exp["right"], "rirs": rirs, "steps": step_results, } write_json(local_exp_root / "experiment-summary.json", experiment_summary) with (local_exp_root / "experiment-summary.md").open("w", encoding="utf-8") as handle: handle.write(f"# {exp_id}\n\n") handle.write(json.dumps(experiment_summary, indent=2, ensure_ascii=False)) handle.write("\n") rsync_dir_to_remote(ssh_target, local_exp_root, str(remote_exp_root)) return experiment_summary def is_rpki_client_experiment(exp: dict[str, Any]) -> bool: return exp["left"]["rpKind"] == "rpki-client" or exp["right"]["rpKind"] == "rpki-client" def main() -> None: parser = argparse.ArgumentParser(description="Feature #035 experiment driver") parser.add_argument("--run-root", required=True) parser.add_argument("--remote-root", required=True) parser.add_argument("--ssh-target", default=os.environ.get("SSH_TARGET", "root@47.251.56.108")) parser.add_argument("--experiment", action="append", help="Experiment id to run; repeatable") parser.add_argument("--all", action="store_true", help="Run all experiments from experiments.json") parser.add_argument("--dry-run", action="store_true") parser.add_argument("--rirs", default="afrinic,apnic,arin,lacnic,ripe") args = parser.parse_args() experiments_doc = load_json(EXPERIMENTS_PATH) experiments = experiments_doc["experiments"] selected_ids = set(args.experiment or []) if args.all or not selected_ids: selected = experiments else: selected = [exp for exp in experiments if exp["id"] in selected_ids] if not selected: raise SystemExit("no experiments selected") rirs = [item.strip() for item in args.rirs.split(",") if item.strip()] if not rirs: raise SystemExit("--rirs must not be empty") run_root = Path(args.run_root).resolve() remote_root = Path(args.remote_root) ensure_dir(run_root) if not args.dry_run: build_tool_binaries() if args.dry_run: plans = [ run_experiment(args.ssh_target, run_root, remote_root, exp, rirs, dry_run=True) for exp in selected ] print(json.dumps({"schemaVersion": 1, "dryRun": True, "experiments": plans}, indent=2, ensure_ascii=False)) return summary = { "schemaVersion": 1, "generatedAtUtc": utc_stamp(), "runRoot": str(run_root), "remoteRoot": str(remote_root), "sshTarget": args.ssh_target, "experiments": [], } for exp in selected: summary["experiments"].append( run_experiment(args.ssh_target, run_root, remote_root, exp, rirs, dry_run=False) ) write_json(run_root / "feature035-summary.json", summary) print(json.dumps(summary, indent=2, ensure_ascii=False)) if __name__ == "__main__": main()