#!/usr/bin/env python3 from __future__ import annotations import argparse import json import os import shlex import shutil import subprocess import sys import time from dataclasses import dataclass from pathlib import Path from typing import Any SCRIPT_DIR = Path(__file__).resolve().parent REPO_ROOT = SCRIPT_DIR.parents[2] DEV_ROOT = REPO_ROOT.parents[1] PORTABLE_ROOT = DEV_ROOT / "rpki-client-portable" FEATURE_BUNDLE = SCRIPT_DIR / "feature035_bundle.py" EXPERIMENTS_PATH = SCRIPT_DIR / "experiments.json" FIXTURE_MANIFEST_PATH = SCRIPT_DIR / "fixture-manifest.json" def load_json(path: Path) -> Any: with path.open("r", encoding="utf-8") as handle: return json.load(handle) def write_json(path: Path, value: Any) -> None: path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", encoding="utf-8") as handle: json.dump(value, handle, indent=2, ensure_ascii=False) handle.write("\n") def ensure_dir(path: Path) -> None: path.mkdir(parents=True, exist_ok=True) def git_commit(repo_root: Path) -> str: result = run_local( ["git", "-C", str(repo_root), "rev-parse", "--short", "HEAD"], capture=True, check=False, ) return result.stdout.strip() if result.returncode == 0 else "" def run_local( argv: list[str], *, cwd: Path | None = None, check: bool = True, capture: bool = False, env: dict[str, str] | None = None, ) -> subprocess.CompletedProcess[str]: result = subprocess.run( argv, cwd=str(cwd) if cwd else None, text=True, check=False, capture_output=capture, env=env, ) if check and result.returncode != 0: raise SystemExit( f"command failed ({result.returncode}): {' '.join(shlex.quote(item) for item in argv)}\n" f"stdout:\n{result.stdout if result.stdout else ''}\n" f"stderr:\n{result.stderr if result.stderr else ''}" ) return result def ssh_script(target: str, script: str, *, check: bool = True) -> subprocess.CompletedProcess[str]: result = subprocess.run( ["ssh", target, "bash", "-s"], input=script, text=True, check=False, ) if check and result.returncode != 0: raise SystemExit(f"remote script failed ({result.returncode}) on {target}") return result def rsync_to_remote(target: str, source: Path, destination: str) -> None: run_local(["rsync", "-a", str(source), f"{target}:{destination}"]) def rsync_dir_to_remote(target: str, source: Path, destination: str) -> None: run_local(["rsync", "-a", f"{source}/", f"{target}:{destination}/"]) def rsync_from_remote(target: str, source: str, destination: Path) -> None: ensure_dir(destination) run_local(["rsync", "-a", f"{target}:{source}/", f"{destination}/"]) def rel_cmd(parts: list[str]) -> str: return shlex.join(str(part) for part in parts) def utc_stamp() -> str: return time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()) def detect_libtls_path(rpki_client_bin: Path) -> Path: ldd = run_local(["ldd", str(rpki_client_bin)], capture=True) for line in ldd.stdout.splitlines(): if "libtls.so.28" not in line: continue if "=>" not in line: continue candidate = line.split("=>", 1)[1].strip().split(" ", 1)[0] path = Path(candidate) if path.is_file(): return path fallback = DEV_ROOT / ".cache" / "rpki-client-9.8-cir" / "libtls.so.28" if fallback.is_file(): return fallback raise SystemExit("unable to locate libtls.so.28 for rpki-client") def parse_elapsed_to_ms(raw: str) -> int: raw = raw.strip() if not raw: return 0 if "-" in raw: days, raw = raw.split("-", 1) else: days = "0" parts = raw.split(":") try: if len(parts) == 4: days = str(int(days) + int(parts[0])) hours, minutes, seconds = parts[1:] elif len(parts) == 3: hours, minutes, seconds = parts elif len(parts) == 2: hours = "0" minutes, seconds = parts else: hours = "0" minutes = "0" seconds = parts[0] total_seconds = ( int(days) * 86400 + int(hours) * 3600 + int(minutes) * 60 + float(seconds) ) except ValueError: time_part = raw.rsplit(":", 1)[-1] total_seconds = float(time_part) return int(round(total_seconds * 1000)) def parse_time_file(path: Path) -> dict[str, Any]: data: dict[str, Any] = {} if not path.is_file(): return data for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): if "Elapsed (wall clock) time" in line and ":" in line: if "):" in line: elapsed = line.rsplit("):", 1)[1] else: elapsed = line.rsplit(":", 1)[1] data["wallMs"] = parse_elapsed_to_ms(elapsed) elif "Maximum resident set size" in line and ":" in line: try: data["maxRssKb"] = int(line.rsplit(":", 1)[1].strip()) except ValueError: pass elif "User time (seconds)" in line and ":" in line: try: data["userSeconds"] = float(line.rsplit(":", 1)[1].strip()) except ValueError: pass elif "System time (seconds)" in line and ":" in line: try: data["systemSeconds"] = float(line.rsplit(":", 1)[1].strip()) except ValueError: pass return data def path_within(base: Path, path: Path) -> str: return path.relative_to(base).as_posix() def load_report_counts(report_path: Path) -> dict[str, Any]: report = load_json(report_path) publication_points = report.get("publication_points", []) tree = report.get("tree", {}) warnings = tree.get("warnings", []) return { "vrps": len(report.get("vrps", [])), "aspas": len(report.get("aspas", [])), "publicationPoints": len(publication_points), "warnings": len(warnings) + sum(len(pp.get("warnings", [])) for pp in publication_points if isinstance(pp, dict)), "treeInstancesProcessed": tree.get("instances_processed"), "treeInstancesFailed": tree.get("instances_failed"), "reportJson": report, } def load_rpki_client_counts(report_path: Path) -> dict[str, Any]: report = load_json(report_path) metadata = report.get("metadata", {}) roas = report.get("roas", []) aspas = report.get("aspas", []) return { "vrps": int(metadata.get("vrps", len(roas))), "uniqueVrps": int(metadata.get("uniquevrps", len(roas))), "aspas": int(metadata.get("aspas", len(aspas))), "uniqueVaps": int(metadata.get("uniquevaps", len(aspas))), "repositories": int(metadata.get("repositories", 0)), "warnings": 0, "reportJson": report, } def load_cir_counts(cir_path: Path) -> dict[str, int]: if not cir_path.is_file(): return { "cirObjectCount": 0, "cirRejectCount": 0, "cirTrustAnchorCount": 0, } result = run_local( [ str(REPO_ROOT / "target" / "release" / "cir_dump_reject_list"), "--cir", str(cir_path), "--limit", "0", ], capture=True, check=False, ) if result.returncode != 0: raise SystemExit(f"decode CIR failed: {cir_path}\n{result.stderr}") values: dict[str, int] = {} for line in result.stdout.splitlines(): if "=" not in line: continue key, value = line.split("=", 1) if key in {"object_count", "trust_anchor_count", "reject_count"}: values[key] = int(value) return { "cirObjectCount": values.get("object_count", 0), "cirRejectCount": values.get("reject_count", 0), "cirTrustAnchorCount": values.get("trust_anchor_count", 0), } def copy_rpki_client_outputs(run_dir: Path) -> None: if (run_dir / "json").is_file(): shutil.copy2(run_dir / "json", run_dir / "report.json") if (run_dir / "rpki.ccr").is_file(): shutil.copy2(run_dir / "rpki.ccr", run_dir / "result.ccr") if (run_dir / "rpki.cir").is_file(): shutil.copy2(run_dir / "rpki.cir", run_dir / "result.cir") def write_rpki_client_stage_timing(run_dir: Path) -> None: report_path = run_dir / "report.json" if not report_path.is_file(): return report = load_json(report_path) metadata = report.get("metadata", {}) stage_timing = { "tool": "rpki-client-portable", "metadata": { "elapsedTimeSeconds": metadata.get("elapsedtime"), "userTimeSeconds": metadata.get("usertime"), "systemTimeSeconds": metadata.get("systemtime"), }, "counts": { "repositories": metadata.get("repositories"), "vrps": metadata.get("vrps"), "uniqueVrps": metadata.get("uniquevrps"), "vaps": metadata.get("vaps"), "uniqueVaps": metadata.get("uniquevaps"), }, } write_json(run_dir / "stage-timing.json", stage_timing) def build_tool_binaries() -> None: run_local( [ "cargo", "build", "--release", "--bin", "rpki", "--bin", "triage_ccr_cir_pair", "--bin", "cir_dump_reject_list", ], cwd=REPO_ROOT, ) rpki_client_bin = PORTABLE_ROOT / "src" / "rpki-client" if not rpki_client_bin.is_file(): run_local(["make", "-j2"], cwd=PORTABLE_ROOT) smoke = run_local( [str(rpki_client_bin), "-T", "invalid"], capture=True, check=False, ) if "--ta-fixture requires :" not in (smoke.stderr + smoke.stdout): cache_bin = DEV_ROOT / ".cache" / "rpki-client-9.8-cir" / "rpki-client" cache_smoke = run_local( [str(cache_bin), "-T", "invalid"], capture=True, check=False, ) if cache_bin.is_file() else None if cache_smoke and "--ta-fixture requires :" in (cache_smoke.stderr + cache_smoke.stdout): shutil.copy2(cache_bin, rpki_client_bin) else: raise SystemExit( "rpki-client binary lacks local TA fixture support (-T); " "switch to feature/cir-output-for-rp-compare and rebuild or restore the cached CIR+TA-fixture binary" ) def build_fixture_proof(run_root: Path, rirs: list[str]) -> Path: fixture_dir = REPO_ROOT / "tests" / "fixtures" fixture_proof = run_root / "fixture-proof.json" run_local( [ sys.executable, str(FEATURE_BUNDLE), "fixture-proof", "--fixture-dir", str(fixture_dir), "--repo-root", str(run_root), "--rirs", ",".join(rirs), "--out", str(fixture_proof), ] ) return fixture_proof def experiment_steps(exp: dict[str, Any]) -> list[dict[str, str]]: return [ { "step": "snapshot", "left": exp["left"], "right": exp["right"], }, { "step": "delta", "left": exp["left"], "right": exp["right"], }, ] def render_experiment_plan(exp: dict[str, Any], run_root: Path, remote_root: Path, ssh_target: str, rirs: list[str]) -> dict[str, Any]: plan_steps: list[dict[str, Any]] = [] for step in ("snapshot", "delta"): plan_steps.append( { "step": step, "leftRunDir": str(run_root / "experiments" / exp["id"] / "A" / step), "rightRunDir": str(run_root / "experiments" / exp["id"] / "B" / step), "remoteLeftRunDir": str(remote_root / "experiments" / exp["id"] / "A" / step), "remoteRightRunDir": str(remote_root / "experiments" / exp["id"] / "B" / step), "leftCommand": build_side_command(remote_root, exp["id"], exp["left"], "A", step, rirs), "rightCommand": build_side_command(remote_root, exp["id"], exp["right"], "B", step, rirs), } ) return { "id": exp["id"], "left": exp["left"], "right": exp["right"], "sshTarget": ssh_target, "remoteRoot": str(remote_root), "runRoot": str(run_root), "steps": plan_steps, } def build_side_command( remote_root: Path, exp_id: str, side: dict[str, Any], side_label: str, step: str, rirs: list[str], ) -> str: run_dir = remote_root / "experiments" / exp_id / side_label / step state_dir = remote_root / "experiments" / exp_id / side_label / "state" / ("ours" if side["rpKind"] == "ours" else "rpki-client") fixture_root = remote_root / "fixtures" if side["rpKind"] == "ours": argv = [ str(remote_root / "bin" / "rpki"), "--db", str(state_dir / "work-db"), "--raw-store-db", str(state_dir / "raw-store.db"), "--repo-bytes-db", str(state_dir / "repo-bytes.db"), ] for rir in rirs: argv.extend( [ "--tal-path", str(fixture_root / "tal" / fixture_name(rir, "tal")), "--ta-path", str(fixture_root / "ta" / fixture_name(rir, "ta")), ] ) if side["protocol"] == "rsync-only": argv.append("--disable-rrdp") rsync_scope = side.get("rsyncScope") if rsync_scope: argv.extend(["--rsync-scope", str(rsync_scope)]) if side["mode"] == "strict-name": argv.extend(["--strict", "name"]) elif side["mode"] == "strict-cms-der": argv.extend(["--strict", "cms-der"]) elif side["mode"] == "strict-signed-attrs": argv.extend(["--strict", "signed-attrs"]) elif side["mode"] == "strict-all": argv.extend(["--strict", "all"]) argv.extend( [ "--report-json", str(run_dir / "report.json"), "--report-json-compact", "--ccr-out", str(run_dir / "result.ccr"), "--cir-enable", "--cir-out", str(run_dir / "result.cir"), ] ) for rir in rirs: argv.extend(["--cir-tal-uri", cir_tal_uri_for_rir(rir)]) argv.extend( [ "--vrps-csv-out", str(run_dir / "vrps.csv"), "--vaps-csv-out", str(run_dir / "vaps.csv"), ] ) return "cd {run_dir} && {prefix} /usr/bin/time -v -o process-time.txt -- {cmd} > stdout.log 2> stderr.log".format( run_dir=shlex.quote(str(run_dir)), prefix=_setup_prefix(step, state_dir, "ours"), cmd=rel_cmd(argv), ) argv = [ str(remote_root / "bin" / "rpki-client"), "-vv", "-S", str(state_dir / "rpki-client-skiplist"), ] if side["protocol"] == "rsync-only": argv.append("-R") for rir in rirs: argv.extend( [ "-t", str(fixture_root / "tal" / fixture_name(rir, "tal")), "-T", f"{fixture_desc(rir)}:{state_dir / 'cache' / 'fixtures' / fixture_name(rir, 'ta')}", ] ) argv.extend(["-d", str(state_dir / "cache"), str(run_dir)]) return "cd {run_dir} && {prefix} LD_LIBRARY_PATH={lib} /usr/bin/time -v -o process-time.txt -- {cmd} > stdout.log 2> stderr.log".format( run_dir=shlex.quote(str(run_dir)), prefix=_setup_prefix(step, state_dir, "rpki-client"), lib=shlex.quote(str(remote_root / "lib")), cmd=rel_cmd(argv), ) def _setup_prefix(step: str, state_dir: Path, kind: str) -> str: if step == "snapshot": if kind == "ours": return "rm -rf {state} && mkdir -p {state}/work-db {state}/rsync-mirror && chmod -R 0777 {state} && ".format( state=shlex.quote(str(state_dir)) ) return "rm -rf {state} && mkdir -p {state}/cache {state}/cache/fixtures && touch {state}/rpki-client-skiplist && chmod -R 0777 {state} && ".format( state=shlex.quote(str(state_dir)) ) if kind == "ours": return "mkdir -p {state}/work-db {state}/rsync-mirror && chmod -R 0777 {state} && ".format( state=shlex.quote(str(state_dir)) ) return "mkdir -p {state}/cache {state}/cache/fixtures && touch {state}/rpki-client-skiplist && chmod -R 0777 {state} && ".format( state=shlex.quote(str(state_dir)) ) def fixture_name(rir: str, kind: str) -> str: fixture_manifest = load_json(FIXTURE_MANIFEST_PATH) return Path(fixture_manifest["rirs"][rir][kind]).name def fixture_desc(rir: str) -> str: return { "afrinic": "afrinic", "apnic": "apnic-rfc7730-https", "arin": "arin", "lacnic": "lacnic", "ripe": "ripe-ncc", }[rir] def cir_tal_uri_for_rir(rir: str) -> str: return { "afrinic": "https://rpki.afrinic.net/tal/afrinic.tal", "apnic": "https://rpki.apnic.net/tal/apnic-rfc7730-https.tal", "arin": "https://www.arin.net/resources/manage/rpki/arin.tal", "lacnic": "https://www.lacnic.net/innovaportal/file/4983/1/lacnic.tal", "ripe": "https://tal.rpki.ripe.net/ripe-ncc.tal", }[rir] def run_remote_step( ssh_target: str, remote_root: Path, exp_id: str, side_label: str, step: str, side: dict[str, Any], rirs: list[str], ) -> None: exp_root = remote_root / "experiments" / exp_id run_dir = exp_root / side_label / step state_dir = exp_root / side_label / "state" / ("ours" if side["rpKind"] == "ours" else "rpki-client") ensure = [ f"mkdir -p {shlex.quote(str(run_dir))}", f"mkdir -p {shlex.quote(str(run_dir.parent))}", f"chmod 0777 {shlex.quote(str(run_dir))}", ] if side["rpKind"] == "ours": ensure.extend( [ f"mkdir -p {shlex.quote(str(state_dir / 'work-db'))}", f"mkdir -p {shlex.quote(str(state_dir / 'rsync-mirror'))}", f"chmod -R 0777 {shlex.quote(str(exp_root / side_label / 'state'))}", ] ) if step == "snapshot": ensure.insert(0, f"rm -rf {shlex.quote(str(exp_root / side_label / 'state' / 'ours'))}") argv = [ str(remote_root / "bin" / "rpki"), "--db", str(state_dir / "work-db"), "--raw-store-db", str(state_dir / "raw-store.db"), "--repo-bytes-db", str(state_dir / "repo-bytes.db"), ] for rir in rirs: argv.extend( [ "--tal-path", str(remote_root / "fixtures" / "tal" / fixture_name(rir, "tal")), "--ta-path", str(remote_root / "fixtures" / "ta" / fixture_name(rir, "ta")), ] ) if side["protocol"] == "rsync-only": argv.append("--disable-rrdp") rsync_scope = side.get("rsyncScope") if rsync_scope: argv.extend(["--rsync-scope", str(rsync_scope)]) if side["mode"] == "strict-name": argv.extend(["--strict", "name"]) elif side["mode"] == "strict-cms-der": argv.extend(["--strict", "cms-der"]) elif side["mode"] == "strict-signed-attrs": argv.extend(["--strict", "signed-attrs"]) elif side["mode"] == "strict-all": argv.extend(["--strict", "all"]) argv.extend( [ "--report-json", str(run_dir / "report.json"), "--report-json-compact", "--ccr-out", str(run_dir / "result.ccr"), "--cir-enable", "--cir-out", str(run_dir / "result.cir"), ] ) for rir in rirs: argv.extend(["--cir-tal-uri", cir_tal_uri_for_rir(rir)]) argv.extend( [ "--vrps-csv-out", str(run_dir / "vrps.csv"), "--vaps-csv-out", str(run_dir / "vaps.csv"), ] ) else: ensure.extend( [ f"mkdir -p {shlex.quote(str(state_dir / 'cache'))}", f"mkdir -p {shlex.quote(str(state_dir / 'cache' / 'fixtures'))}", f"touch {shlex.quote(str(state_dir / 'rpki-client-skiplist'))}", f"chmod -R 0777 {shlex.quote(str(exp_root / side_label / 'state'))}", ] ) if step == "snapshot": ensure.insert(0, f"rm -rf {shlex.quote(str(exp_root / side_label / 'state' / 'rpki-client'))}") for rir in rirs: ensure.append( f"cp -f {shlex.quote(str(remote_root / 'fixtures' / 'ta' / fixture_name(rir, 'ta')))} {shlex.quote(str(state_dir / 'cache' / 'fixtures' / fixture_name(rir, 'ta')))}" ) argv = [ str(remote_root / "bin" / "rpki-client"), "-vv", "-S", str(state_dir / "rpki-client-skiplist"), ] if side["protocol"] == "rsync-only": argv.append("-R") for rir in rirs: argv.extend( [ "-t", str(remote_root / "fixtures" / "tal" / fixture_name(rir, "tal")), "-T", f"{fixture_desc(rir)}:{state_dir / 'cache' / 'fixtures' / fixture_name(rir, 'ta')}", ] ) argv.extend(["-d", str(state_dir / "cache"), str(run_dir)]) time_prefix = "/usr/bin/time" if side["rpKind"] == "ours": time_prefix = "env RPKI_PROGRESS_LOG=1 RPKI_PROGRESS_SLOW_SECS=10 /usr/bin/time" elif side["rpKind"] == "rpki-client": time_prefix = f"env LD_LIBRARY_PATH={shlex.quote(str(remote_root / 'lib'))} /usr/bin/time" command = ( "set -euo pipefail; " + "; ".join(ensure) + "; " + "set +e; " + time_prefix + " -v -o " + shlex.quote(str(run_dir / "process-time.txt")) + " -- " + rel_cmd(argv) + " > " + shlex.quote(str(run_dir / "stdout.log")) + " 2> " + shlex.quote(str(run_dir / "stderr.log")) + "; ec=$?; set -e; printf '%s\n' \"$ec\" > " + shlex.quote(str(run_dir / "exit-code.txt")) + "; true" ) ssh_script(ssh_target, command) if side["rpKind"] == "rpki-client": copy_cmd = ( f"[ -f {shlex.quote(str(run_dir / 'json'))} ] && cp -f {shlex.quote(str(run_dir / 'json'))} {shlex.quote(str(run_dir / 'report.json'))} || true; " f"[ -f {shlex.quote(str(run_dir / 'rpki.ccr'))} ] && cp -f {shlex.quote(str(run_dir / 'rpki.ccr'))} {shlex.quote(str(run_dir / 'result.ccr'))} || true; " f"[ -f {shlex.quote(str(run_dir / 'rpki.cir'))} ] && cp -f {shlex.quote(str(run_dir / 'rpki.cir'))} {shlex.quote(str(run_dir / 'result.cir'))} || true; " f"[ -f {shlex.quote(str(run_dir / 'report.json'))} ] && python3 - <<'PY' {shlex.quote(str(run_dir / 'report.json'))} {shlex.quote(str(run_dir / 'stage-timing.json'))} || true\n" "import json, sys\n" "report = json.load(open(sys.argv[1]))\n" "meta = report.get('metadata', {})\n" "stage = {\n" " 'tool': 'rpki-client-portable',\n" " 'metadata': {\n" " 'elapsedTimeSeconds': meta.get('elapsedtime'),\n" " 'userTimeSeconds': meta.get('usertime'),\n" " 'systemTimeSeconds': meta.get('systemtime'),\n" " },\n" " 'counts': {\n" " 'repositories': meta.get('repositories'),\n" " 'vrps': meta.get('vrps'),\n" " 'uniqueVrps': meta.get('uniquevrps'),\n" " 'vaps': meta.get('vaps'),\n" " 'uniqueVaps': meta.get('uniquevaps'),\n" " }\n" "}\n" "json.dump(stage, open(sys.argv[2], 'w'), indent=2)\n" "print()\n" "PY" ) ssh_script(ssh_target, copy_cmd) def parse_exit_code(path: Path) -> int: if not path.is_file(): return 1 return int(path.read_text(encoding="utf-8").strip() or "1") def generate_run_meta( local_exp_root: Path, exp_id: str, side_label: str, step: str, side: dict[str, Any], rirs: list[str], fixture_proof: Path, run_dir: Path, ) -> Path: result_ccr = run_dir / "result.ccr" result_cir = run_dir / "result.cir" report_json = run_dir / "report.json" stage_timing_json = run_dir / "stage-timing.json" process_time = run_dir / "process-time.txt" stdout_log = run_dir / "stdout.log" stderr_log = run_dir / "stderr.log" exit_code = parse_exit_code(run_dir / "exit-code.txt") time_info = parse_time_file(process_time) if side["rpKind"] == "ours": counts = load_report_counts(report_json) counts.update(load_cir_counts(result_cir)) else: counts = load_rpki_client_counts(report_json) counts.update(load_cir_counts(result_cir)) strict_policies = "" if side["mode"] == "strict-name": strict_policies = "name" elif side["mode"] == "strict-cms-der": strict_policies = "cms-der" elif side["mode"] == "strict-signed-attrs": strict_policies = "signed-attrs" elif side["mode"] == "strict-all": strict_policies = "all" meta_path = local_exp_root / side_label / step / "run-meta.json" generate_meta_args = [ sys.executable, str(FEATURE_BUNDLE), "run-meta", "--out", str(meta_path), "--repo-root", str(local_exp_root), "--experiment-id", exp_id, "--side", "left" if side_label == "A" else "right", "--side-label", side_label, "--step", step, "--run-id", f"{side_label}-{step}", "--rp-kind", side["rpKind"], "--rp-binary", "bin/rpki" if side["rpKind"] == "ours" else "bin/rpki-client", "--rp-version", "portable-9.8" if side["rpKind"] == "rpki-client" else "ours-dev", "--rp-git-commit", git_commit(REPO_ROOT) or "", "--rp-mode", side["mode"], "--protocol-mode", side["protocol"] + (f"/rsync-scope:{side['rsyncScope']}" if side.get("rsyncScope") else ""), "--strict-policies", strict_policies, "--rirs", ",".join(rirs), "--argv-json", json.dumps([]), "--env-json", json.dumps({}), "--cwd", str(local_exp_root), "--state-root", str(local_exp_root / side_label / "state"), "--db", str(local_exp_root / side_label / "state" / "ours" / "work-db"), "--repo-bytes-db", str(local_exp_root / side_label / "state" / "ours" / "repo-bytes.db"), "--raw-store-db", str(local_exp_root / side_label / "state" / "ours" / "raw-store.db"), "--rsync-mirror-root", str(local_exp_root / side_label / "state" / "ours" / "rsync-mirror"), "--cache-root", str(local_exp_root / side_label / "state" / "rpki-client" / "cache"), "--ccr", str(result_ccr), "--cir", str(result_cir), "--fixture-proof", str(fixture_proof), "--fixture-proof-summary-json", json.dumps( { "taFixturePinned": True, "taOnlineFetchObserved": False, "trustAnchorCount": len(rirs), } ), "--report-json", str(report_json), "--stage-timing-json", str(stage_timing_json), "--stdout-log", str(stdout_log), "--stderr-log", str(stderr_log), "--process-time", str(process_time), "--exit-code", str(exit_code), "--wall-ms", str(int(time_info.get("wallMs", 0))), "--max-rss-kb", str(int(time_info.get("maxRssKb", 0))), "--vrps", str(int(counts.get("vrps", 0))), "--vaps", str(int(counts.get("aspas", 0))), "--publication-points", str(int(counts.get("publicationPoints", counts.get("repositories", 0)))), "--warnings", str(int(counts.get("warnings", 0))), "--cir-object-count", str(int(counts.get("cirObjectCount", 0))), "--cir-reject-count", str(int(counts.get("cirRejectCount", 0))), "--cir-trust-anchor-count", str(int(counts.get("cirTrustAnchorCount", len(rirs)))), "--host", os.uname().nodename, "--platform", sys.platform, ] if step == "snapshot": insert_at = generate_meta_args.index("--state-root") generate_meta_args.insert(insert_at, "--reset-before-run") run_local(generate_meta_args) return meta_path def run_experiment( ssh_target: str, local_run_root: Path, remote_root: Path, exp: dict[str, Any], rirs: list[str], dry_run: bool = False, ) -> dict[str, Any]: exp_id = exp["id"] local_exp_root = local_run_root / "experiments" / exp_id remote_exp_root = remote_root / "experiments" / exp_id ensure_dir(local_exp_root) if dry_run: return render_experiment_plan(exp, local_run_root, remote_root, ssh_target, rirs) fixture_proof = build_fixture_proof(local_exp_root, rirs) preflight = ( "set -euo pipefail; " f"df -h /data / || true; " "systemctl disable --now rpki-client.timer >/dev/null 2>&1 || true; " "systemctl stop rpki-client.service >/dev/null 2>&1 || true; " "pkill -f '[/]rpki-client([[:space:]]|$)' >/dev/null 2>&1 || true; " "pkill -f '[/]routinator([[:space:]]|$)' >/dev/null 2>&1 || true; " "id -u _rpki-client >/dev/null 2>&1 || useradd -r -M -s /usr/sbin/nologin _rpki-client || true; " f"mkdir -p {shlex.quote(str(remote_root / 'bin'))} {shlex.quote(str(remote_root / 'lib'))} {shlex.quote(str(remote_root / 'fixtures' / 'tal'))} {shlex.quote(str(remote_root / 'fixtures' / 'ta'))} {shlex.quote(str(remote_exp_root))}" ) ssh_script(ssh_target, preflight) rsync_dir_to_remote(ssh_target, REPO_ROOT / "tests" / "fixtures" / "tal", remote_root / "fixtures" / "tal") rsync_dir_to_remote(ssh_target, REPO_ROOT / "tests" / "fixtures" / "ta", remote_root / "fixtures" / "ta") if exp["left"]["rpKind"] == "rpki-client" or exp["right"]["rpKind"] == "rpki-client": rsync_to_remote(ssh_target, detect_libtls_path(PORTABLE_ROOT / "src" / "rpki-client"), str(remote_root / "lib" / "libtls.so.28")) rsync_to_remote(ssh_target, PORTABLE_ROOT / "src" / "rpki-client", str(remote_root / "bin" / "rpki-client")) rsync_to_remote(ssh_target, REPO_ROOT / "target" / "release" / "rpki", str(remote_root / "bin" / "rpki")) rsync_to_remote(ssh_target, REPO_ROOT / "target" / "release" / "triage_ccr_cir_pair", str(remote_root / "bin" / "triage_ccr_cir_pair")) rsync_to_remote(ssh_target, REPO_ROOT / "target" / "release" / "cir_dump_reject_list", str(remote_root / "bin" / "cir_dump_reject_list")) step_results: list[dict[str, Any]] = [] for step in ("snapshot", "delta"): for side_label, side in (("A", exp["left"]), ("B", exp["right"])): run_remote_step(ssh_target, remote_root, exp_id, side_label, step, side, rirs) local_step_root = local_exp_root / side_label / step remote_step_root = remote_exp_root / side_label / step rsync_from_remote(ssh_target, str(remote_step_root), local_step_root) if side["rpKind"] == "rpki-client": copy_rpki_client_outputs(local_step_root) write_rpki_client_stage_timing(local_step_root) # ensure ours has a stable stage timing artifact too if not (local_step_root / "stage-timing.json").is_file() and (local_step_root / "report.json").is_file(): report = load_json(local_step_root / "report.json") stage_timing = { "tool": side["rpKind"], "counts": { "publicationPoints": len(report.get("publication_points", [])), "vrps": len(report.get("vrps", [])), "aspas": len(report.get("aspas", [])), }, } write_json(local_step_root / "stage-timing.json", stage_timing) meta_path = generate_run_meta( local_exp_root, exp_id, side_label, step, side, rirs, fixture_proof, local_step_root, ) step_results.append( { "side": side_label, "step": step, "runDir": str(local_step_root), "meta": str(meta_path), } ) compare_dir = local_exp_root / "compare" / step ensure_dir(compare_dir) triage_cmd = [ str(REPO_ROOT / "target" / "release" / "triage_ccr_cir_pair"), "--left-ccr", str(local_exp_root / "A" / step / "result.ccr"), "--left-cir", str(local_exp_root / "A" / step / "result.cir"), "--right-ccr", str(local_exp_root / "B" / step / "result.ccr"), "--right-cir", str(local_exp_root / "B" / step / "result.cir"), "--out-dir", str(compare_dir), "--sample-limit", "200", ] run_local(triage_cmd, cwd=local_exp_root) step_summary = { "step": step, "compareDir": str(compare_dir), "triage": load_json(compare_dir / "triage.json"), } write_json(local_exp_root / "compare" / f"{step}.summary.json", step_summary) step_results.append(step_summary) experiment_summary = { "schemaVersion": 1, "experimentId": exp_id, "left": exp["left"], "right": exp["right"], "rirs": rirs, "steps": step_results, } write_json(local_exp_root / "experiment-summary.json", experiment_summary) with (local_exp_root / "experiment-summary.md").open("w", encoding="utf-8") as handle: handle.write(f"# {exp_id}\n\n") handle.write(json.dumps(experiment_summary, indent=2, ensure_ascii=False)) handle.write("\n") rsync_dir_to_remote(ssh_target, local_exp_root, str(remote_exp_root)) return experiment_summary def is_rpki_client_experiment(exp: dict[str, Any]) -> bool: return exp["left"]["rpKind"] == "rpki-client" or exp["right"]["rpKind"] == "rpki-client" def main() -> None: parser = argparse.ArgumentParser(description="Feature #035 experiment driver") parser.add_argument("--run-root", required=True) parser.add_argument("--remote-root", required=True) parser.add_argument("--ssh-target", default=os.environ.get("SSH_TARGET", "root@47.251.56.108")) parser.add_argument("--experiment", action="append", help="Experiment id to run; repeatable") parser.add_argument("--all", action="store_true", help="Run all experiments from experiments.json") parser.add_argument("--dry-run", action="store_true") parser.add_argument("--rirs", default="afrinic,apnic,arin,lacnic,ripe") args = parser.parse_args() experiments_doc = load_json(EXPERIMENTS_PATH) experiments = experiments_doc["experiments"] selected_ids = set(args.experiment or []) if args.all or not selected_ids: selected = experiments else: selected = [exp for exp in experiments if exp["id"] in selected_ids] if not selected: raise SystemExit("no experiments selected") rirs = [item.strip() for item in args.rirs.split(",") if item.strip()] if not rirs: raise SystemExit("--rirs must not be empty") run_root = Path(args.run_root).resolve() remote_root = Path(args.remote_root) ensure_dir(run_root) if not args.dry_run: build_tool_binaries() if args.dry_run: plans = [ run_experiment(args.ssh_target, run_root, remote_root, exp, rirs, dry_run=True) for exp in selected ] print(json.dumps({"schemaVersion": 1, "dryRun": True, "experiments": plans}, indent=2, ensure_ascii=False)) return summary = { "schemaVersion": 1, "generatedAtUtc": utc_stamp(), "runRoot": str(run_root), "remoteRoot": str(remote_root), "sshTarget": args.ssh_target, "experiments": [], } for exp in selected: summary["experiments"].append( run_experiment(args.ssh_target, run_root, remote_root, exp, rirs, dry_run=False) ) write_json(run_root / "feature035-summary.json", summary) print(json.dumps(summary, indent=2, ensure_ascii=False)) if __name__ == "__main__": main()