1037 lines
37 KiB
Python
Executable File
1037 lines
37 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import shlex
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
|
REPO_ROOT = SCRIPT_DIR.parents[2]
|
|
DEV_ROOT = REPO_ROOT.parents[1]
|
|
PORTABLE_ROOT = DEV_ROOT / "rpki-client-portable"
|
|
FEATURE_BUNDLE = SCRIPT_DIR / "feature035_bundle.py"
|
|
EXPERIMENTS_PATH = SCRIPT_DIR / "experiments.json"
|
|
FIXTURE_MANIFEST_PATH = SCRIPT_DIR / "fixture-manifest.json"
|
|
|
|
|
|
def load_json(path: Path) -> Any:
|
|
with path.open("r", encoding="utf-8") as handle:
|
|
return json.load(handle)
|
|
|
|
|
|
def write_json(path: Path, value: Any) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
with path.open("w", encoding="utf-8") as handle:
|
|
json.dump(value, handle, indent=2, ensure_ascii=False)
|
|
handle.write("\n")
|
|
|
|
|
|
def ensure_dir(path: Path) -> None:
|
|
path.mkdir(parents=True, exist_ok=True)
|
|
|
|
|
|
def git_commit(repo_root: Path) -> str:
|
|
result = run_local(
|
|
["git", "-C", str(repo_root), "rev-parse", "--short", "HEAD"],
|
|
capture=True,
|
|
check=False,
|
|
)
|
|
return result.stdout.strip() if result.returncode == 0 else ""
|
|
|
|
|
|
def run_local(
|
|
argv: list[str],
|
|
*,
|
|
cwd: Path | None = None,
|
|
check: bool = True,
|
|
capture: bool = False,
|
|
env: dict[str, str] | None = None,
|
|
) -> subprocess.CompletedProcess[str]:
|
|
result = subprocess.run(
|
|
argv,
|
|
cwd=str(cwd) if cwd else None,
|
|
text=True,
|
|
check=False,
|
|
capture_output=capture,
|
|
env=env,
|
|
)
|
|
if check and result.returncode != 0:
|
|
raise SystemExit(
|
|
f"command failed ({result.returncode}): {' '.join(shlex.quote(item) for item in argv)}\n"
|
|
f"stdout:\n{result.stdout if result.stdout else ''}\n"
|
|
f"stderr:\n{result.stderr if result.stderr else ''}"
|
|
)
|
|
return result
|
|
|
|
|
|
def ssh_script(target: str, script: str, *, check: bool = True) -> subprocess.CompletedProcess[str]:
|
|
result = subprocess.run(
|
|
["ssh", target, "bash", "-s"],
|
|
input=script,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
if check and result.returncode != 0:
|
|
raise SystemExit(f"remote script failed ({result.returncode}) on {target}")
|
|
return result
|
|
|
|
|
|
def rsync_to_remote(target: str, source: Path, destination: str) -> None:
|
|
run_local(["rsync", "-a", str(source), f"{target}:{destination}"])
|
|
|
|
|
|
def rsync_dir_to_remote(target: str, source: Path, destination: str) -> None:
|
|
run_local(["rsync", "-a", f"{source}/", f"{target}:{destination}/"])
|
|
|
|
|
|
def rsync_from_remote(target: str, source: str, destination: Path) -> None:
|
|
ensure_dir(destination)
|
|
run_local(["rsync", "-a", f"{target}:{source}/", f"{destination}/"])
|
|
|
|
|
|
def rel_cmd(parts: list[str]) -> str:
|
|
return shlex.join(str(part) for part in parts)
|
|
|
|
|
|
def utc_stamp() -> str:
|
|
return time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())
|
|
|
|
|
|
def detect_libtls_path(rpki_client_bin: Path) -> Path:
|
|
ldd = run_local(["ldd", str(rpki_client_bin)], capture=True)
|
|
for line in ldd.stdout.splitlines():
|
|
if "libtls.so.28" not in line:
|
|
continue
|
|
if "=>" not in line:
|
|
continue
|
|
candidate = line.split("=>", 1)[1].strip().split(" ", 1)[0]
|
|
path = Path(candidate)
|
|
if path.is_file():
|
|
return path
|
|
fallback = DEV_ROOT / ".cache" / "rpki-client-9.8-cir" / "libtls.so.28"
|
|
if fallback.is_file():
|
|
return fallback
|
|
raise SystemExit("unable to locate libtls.so.28 for rpki-client")
|
|
|
|
|
|
def parse_elapsed_to_ms(raw: str) -> int:
|
|
raw = raw.strip()
|
|
if not raw:
|
|
return 0
|
|
if "-" in raw:
|
|
days, raw = raw.split("-", 1)
|
|
else:
|
|
days = "0"
|
|
parts = raw.split(":")
|
|
try:
|
|
if len(parts) == 4:
|
|
days = str(int(days) + int(parts[0]))
|
|
hours, minutes, seconds = parts[1:]
|
|
elif len(parts) == 3:
|
|
hours, minutes, seconds = parts
|
|
elif len(parts) == 2:
|
|
hours = "0"
|
|
minutes, seconds = parts
|
|
else:
|
|
hours = "0"
|
|
minutes = "0"
|
|
seconds = parts[0]
|
|
total_seconds = (
|
|
int(days) * 86400 + int(hours) * 3600 + int(minutes) * 60 + float(seconds)
|
|
)
|
|
except ValueError:
|
|
time_part = raw.rsplit(":", 1)[-1]
|
|
total_seconds = float(time_part)
|
|
return int(round(total_seconds * 1000))
|
|
|
|
|
|
def parse_time_file(path: Path) -> dict[str, Any]:
|
|
data: dict[str, Any] = {}
|
|
if not path.is_file():
|
|
return data
|
|
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
|
|
if "Elapsed (wall clock) time" in line and ":" in line:
|
|
if "):" in line:
|
|
elapsed = line.rsplit("):", 1)[1]
|
|
else:
|
|
elapsed = line.rsplit(":", 1)[1]
|
|
data["wallMs"] = parse_elapsed_to_ms(elapsed)
|
|
elif "Maximum resident set size" in line and ":" in line:
|
|
try:
|
|
data["maxRssKb"] = int(line.rsplit(":", 1)[1].strip())
|
|
except ValueError:
|
|
pass
|
|
elif "User time (seconds)" in line and ":" in line:
|
|
try:
|
|
data["userSeconds"] = float(line.rsplit(":", 1)[1].strip())
|
|
except ValueError:
|
|
pass
|
|
elif "System time (seconds)" in line and ":" in line:
|
|
try:
|
|
data["systemSeconds"] = float(line.rsplit(":", 1)[1].strip())
|
|
except ValueError:
|
|
pass
|
|
return data
|
|
|
|
|
|
def path_within(base: Path, path: Path) -> str:
|
|
return path.relative_to(base).as_posix()
|
|
|
|
|
|
def load_report_counts(report_path: Path) -> dict[str, Any]:
|
|
report = load_json(report_path)
|
|
publication_points = report.get("publication_points", [])
|
|
tree = report.get("tree", {})
|
|
warnings = tree.get("warnings", [])
|
|
return {
|
|
"vrps": len(report.get("vrps", [])),
|
|
"aspas": len(report.get("aspas", [])),
|
|
"publicationPoints": len(publication_points),
|
|
"warnings": len(warnings)
|
|
+ sum(len(pp.get("warnings", [])) for pp in publication_points if isinstance(pp, dict)),
|
|
"treeInstancesProcessed": tree.get("instances_processed"),
|
|
"treeInstancesFailed": tree.get("instances_failed"),
|
|
"reportJson": report,
|
|
}
|
|
|
|
|
|
def load_rpki_client_counts(report_path: Path) -> dict[str, Any]:
|
|
report = load_json(report_path)
|
|
metadata = report.get("metadata", {})
|
|
roas = report.get("roas", [])
|
|
aspas = report.get("aspas", [])
|
|
return {
|
|
"vrps": int(metadata.get("vrps", len(roas))),
|
|
"uniqueVrps": int(metadata.get("uniquevrps", len(roas))),
|
|
"aspas": int(metadata.get("aspas", len(aspas))),
|
|
"uniqueVaps": int(metadata.get("uniquevaps", len(aspas))),
|
|
"repositories": int(metadata.get("repositories", 0)),
|
|
"warnings": 0,
|
|
"reportJson": report,
|
|
}
|
|
|
|
|
|
def load_cir_counts(cir_path: Path) -> dict[str, int]:
|
|
if not cir_path.is_file():
|
|
return {
|
|
"cirObjectCount": 0,
|
|
"cirRejectCount": 0,
|
|
"cirTrustAnchorCount": 0,
|
|
}
|
|
result = run_local(
|
|
[
|
|
str(REPO_ROOT / "target" / "release" / "cir_dump_reject_list"),
|
|
"--cir",
|
|
str(cir_path),
|
|
"--limit",
|
|
"0",
|
|
],
|
|
capture=True,
|
|
check=False,
|
|
)
|
|
if result.returncode != 0:
|
|
raise SystemExit(f"decode CIR failed: {cir_path}\n{result.stderr}")
|
|
values: dict[str, int] = {}
|
|
for line in result.stdout.splitlines():
|
|
if "=" not in line:
|
|
continue
|
|
key, value = line.split("=", 1)
|
|
if key in {"object_count", "trust_anchor_count", "reject_count"}:
|
|
values[key] = int(value)
|
|
return {
|
|
"cirObjectCount": values.get("object_count", 0),
|
|
"cirRejectCount": values.get("reject_count", 0),
|
|
"cirTrustAnchorCount": values.get("trust_anchor_count", 0),
|
|
}
|
|
|
|
|
|
def copy_rpki_client_outputs(run_dir: Path) -> None:
|
|
if (run_dir / "json").is_file():
|
|
shutil.copy2(run_dir / "json", run_dir / "report.json")
|
|
if (run_dir / "rpki.ccr").is_file():
|
|
shutil.copy2(run_dir / "rpki.ccr", run_dir / "result.ccr")
|
|
if (run_dir / "rpki.cir").is_file():
|
|
shutil.copy2(run_dir / "rpki.cir", run_dir / "result.cir")
|
|
|
|
|
|
def write_rpki_client_stage_timing(run_dir: Path) -> None:
|
|
report_path = run_dir / "report.json"
|
|
if not report_path.is_file():
|
|
return
|
|
report = load_json(report_path)
|
|
metadata = report.get("metadata", {})
|
|
stage_timing = {
|
|
"tool": "rpki-client-portable",
|
|
"metadata": {
|
|
"elapsedTimeSeconds": metadata.get("elapsedtime"),
|
|
"userTimeSeconds": metadata.get("usertime"),
|
|
"systemTimeSeconds": metadata.get("systemtime"),
|
|
},
|
|
"counts": {
|
|
"repositories": metadata.get("repositories"),
|
|
"vrps": metadata.get("vrps"),
|
|
"uniqueVrps": metadata.get("uniquevrps"),
|
|
"vaps": metadata.get("vaps"),
|
|
"uniqueVaps": metadata.get("uniquevaps"),
|
|
},
|
|
}
|
|
write_json(run_dir / "stage-timing.json", stage_timing)
|
|
|
|
|
|
def build_tool_binaries() -> None:
|
|
run_local(
|
|
[
|
|
"cargo",
|
|
"build",
|
|
"--release",
|
|
"--bin",
|
|
"rpki",
|
|
"--bin",
|
|
"triage_ccr_cir_pair",
|
|
"--bin",
|
|
"cir_dump_reject_list",
|
|
],
|
|
cwd=REPO_ROOT,
|
|
)
|
|
rpki_client_bin = PORTABLE_ROOT / "src" / "rpki-client"
|
|
if not rpki_client_bin.is_file():
|
|
run_local(["make", "-j2"], cwd=PORTABLE_ROOT)
|
|
smoke = run_local(
|
|
[str(rpki_client_bin), "-T", "invalid"],
|
|
capture=True,
|
|
check=False,
|
|
)
|
|
if "--ta-fixture requires <tal>:<path>" not in (smoke.stderr + smoke.stdout):
|
|
cache_bin = DEV_ROOT / ".cache" / "rpki-client-9.8-cir" / "rpki-client"
|
|
cache_smoke = run_local(
|
|
[str(cache_bin), "-T", "invalid"],
|
|
capture=True,
|
|
check=False,
|
|
) if cache_bin.is_file() else None
|
|
if cache_smoke and "--ta-fixture requires <tal>:<path>" in (cache_smoke.stderr + cache_smoke.stdout):
|
|
shutil.copy2(cache_bin, rpki_client_bin)
|
|
else:
|
|
raise SystemExit(
|
|
"rpki-client binary lacks local TA fixture support (-T); "
|
|
"switch to feature/cir-output-for-rp-compare and rebuild or restore the cached CIR+TA-fixture binary"
|
|
)
|
|
|
|
|
|
def build_fixture_proof(run_root: Path, rirs: list[str]) -> Path:
|
|
fixture_dir = REPO_ROOT / "tests" / "fixtures"
|
|
fixture_proof = run_root / "fixture-proof.json"
|
|
run_local(
|
|
[
|
|
sys.executable,
|
|
str(FEATURE_BUNDLE),
|
|
"fixture-proof",
|
|
"--fixture-dir",
|
|
str(fixture_dir),
|
|
"--repo-root",
|
|
str(run_root),
|
|
"--rirs",
|
|
",".join(rirs),
|
|
"--out",
|
|
str(fixture_proof),
|
|
]
|
|
)
|
|
return fixture_proof
|
|
|
|
|
|
def experiment_steps(exp: dict[str, Any]) -> list[dict[str, str]]:
|
|
return [
|
|
{
|
|
"step": "snapshot",
|
|
"left": exp["left"],
|
|
"right": exp["right"],
|
|
},
|
|
{
|
|
"step": "delta",
|
|
"left": exp["left"],
|
|
"right": exp["right"],
|
|
},
|
|
]
|
|
|
|
|
|
def render_experiment_plan(exp: dict[str, Any], run_root: Path, remote_root: Path, ssh_target: str, rirs: list[str]) -> dict[str, Any]:
|
|
plan_steps: list[dict[str, Any]] = []
|
|
for step in ("snapshot", "delta"):
|
|
plan_steps.append(
|
|
{
|
|
"step": step,
|
|
"leftRunDir": str(run_root / "experiments" / exp["id"] / "A" / step),
|
|
"rightRunDir": str(run_root / "experiments" / exp["id"] / "B" / step),
|
|
"remoteLeftRunDir": str(remote_root / "experiments" / exp["id"] / "A" / step),
|
|
"remoteRightRunDir": str(remote_root / "experiments" / exp["id"] / "B" / step),
|
|
"leftCommand": build_side_command(remote_root, exp["id"], exp["left"], "A", step, rirs),
|
|
"rightCommand": build_side_command(remote_root, exp["id"], exp["right"], "B", step, rirs),
|
|
}
|
|
)
|
|
return {
|
|
"id": exp["id"],
|
|
"left": exp["left"],
|
|
"right": exp["right"],
|
|
"sshTarget": ssh_target,
|
|
"remoteRoot": str(remote_root),
|
|
"runRoot": str(run_root),
|
|
"steps": plan_steps,
|
|
}
|
|
|
|
|
|
def build_side_command(
|
|
remote_root: Path,
|
|
exp_id: str,
|
|
side: dict[str, Any],
|
|
side_label: str,
|
|
step: str,
|
|
rirs: list[str],
|
|
) -> str:
|
|
run_dir = remote_root / "experiments" / exp_id / side_label / step
|
|
state_dir = remote_root / "experiments" / exp_id / side_label / "state" / ("ours" if side["rpKind"] == "ours" else "rpki-client")
|
|
fixture_root = remote_root / "fixtures"
|
|
if side["rpKind"] == "ours":
|
|
argv = [
|
|
str(remote_root / "bin" / "rpki"),
|
|
"--db",
|
|
str(state_dir / "work-db"),
|
|
"--raw-store-db",
|
|
str(state_dir / "raw-store.db"),
|
|
"--repo-bytes-db",
|
|
str(state_dir / "repo-bytes.db"),
|
|
]
|
|
for rir in rirs:
|
|
argv.extend(
|
|
[
|
|
"--tal-path",
|
|
str(fixture_root / "tal" / fixture_name(rir, "tal")),
|
|
"--ta-path",
|
|
str(fixture_root / "ta" / fixture_name(rir, "ta")),
|
|
]
|
|
)
|
|
if side["protocol"] == "rsync-only":
|
|
argv.append("--disable-rrdp")
|
|
rsync_scope = side.get("rsyncScope")
|
|
if rsync_scope:
|
|
argv.extend(["--rsync-scope", str(rsync_scope)])
|
|
if side["mode"] == "strict-name":
|
|
argv.extend(["--strict", "name"])
|
|
elif side["mode"] == "strict-cms-der":
|
|
argv.extend(["--strict", "cms-der"])
|
|
elif side["mode"] == "strict-signed-attrs":
|
|
argv.extend(["--strict", "signed-attrs"])
|
|
elif side["mode"] == "strict-all":
|
|
argv.extend(["--strict", "all"])
|
|
argv.extend(
|
|
[
|
|
"--report-json",
|
|
str(run_dir / "report.json"),
|
|
"--report-json-compact",
|
|
"--ccr-out",
|
|
str(run_dir / "result.ccr"),
|
|
"--cir-enable",
|
|
"--cir-out",
|
|
str(run_dir / "result.cir"),
|
|
]
|
|
)
|
|
for rir in rirs:
|
|
argv.extend(["--cir-tal-uri", cir_tal_uri_for_rir(rir)])
|
|
argv.extend(
|
|
[
|
|
"--vrps-csv-out",
|
|
str(run_dir / "vrps.csv"),
|
|
"--vaps-csv-out",
|
|
str(run_dir / "vaps.csv"),
|
|
]
|
|
)
|
|
return "cd {run_dir} && {prefix} /usr/bin/time -v -o process-time.txt -- {cmd} > stdout.log 2> stderr.log".format(
|
|
run_dir=shlex.quote(str(run_dir)),
|
|
prefix=_setup_prefix(step, state_dir, "ours"),
|
|
cmd=rel_cmd(argv),
|
|
)
|
|
|
|
argv = [
|
|
str(remote_root / "bin" / "rpki-client"),
|
|
"-vv",
|
|
"-S",
|
|
str(state_dir / "rpki-client-skiplist"),
|
|
]
|
|
if side["protocol"] == "rsync-only":
|
|
argv.append("-R")
|
|
for rir in rirs:
|
|
argv.extend(
|
|
[
|
|
"-t",
|
|
str(fixture_root / "tal" / fixture_name(rir, "tal")),
|
|
"-T",
|
|
f"{fixture_desc(rir)}:{state_dir / 'cache' / 'fixtures' / fixture_name(rir, 'ta')}",
|
|
]
|
|
)
|
|
argv.extend(["-d", str(state_dir / "cache"), str(run_dir)])
|
|
return "cd {run_dir} && {prefix} LD_LIBRARY_PATH={lib} /usr/bin/time -v -o process-time.txt -- {cmd} > stdout.log 2> stderr.log".format(
|
|
run_dir=shlex.quote(str(run_dir)),
|
|
prefix=_setup_prefix(step, state_dir, "rpki-client"),
|
|
lib=shlex.quote(str(remote_root / "lib")),
|
|
cmd=rel_cmd(argv),
|
|
)
|
|
|
|
|
|
def _setup_prefix(step: str, state_dir: Path, kind: str) -> str:
|
|
if step == "snapshot":
|
|
if kind == "ours":
|
|
return "rm -rf {state} && mkdir -p {state}/work-db {state}/rsync-mirror && chmod -R 0777 {state} && ".format(
|
|
state=shlex.quote(str(state_dir))
|
|
)
|
|
return "rm -rf {state} && mkdir -p {state}/cache {state}/cache/fixtures && touch {state}/rpki-client-skiplist && chmod -R 0777 {state} && ".format(
|
|
state=shlex.quote(str(state_dir))
|
|
)
|
|
if kind == "ours":
|
|
return "mkdir -p {state}/work-db {state}/rsync-mirror && chmod -R 0777 {state} && ".format(
|
|
state=shlex.quote(str(state_dir))
|
|
)
|
|
return "mkdir -p {state}/cache {state}/cache/fixtures && touch {state}/rpki-client-skiplist && chmod -R 0777 {state} && ".format(
|
|
state=shlex.quote(str(state_dir))
|
|
)
|
|
|
|
|
|
def fixture_name(rir: str, kind: str) -> str:
|
|
fixture_manifest = load_json(FIXTURE_MANIFEST_PATH)
|
|
return Path(fixture_manifest["rirs"][rir][kind]).name
|
|
|
|
|
|
def fixture_desc(rir: str) -> str:
|
|
return {
|
|
"afrinic": "afrinic",
|
|
"apnic": "apnic-rfc7730-https",
|
|
"arin": "arin",
|
|
"lacnic": "lacnic",
|
|
"ripe": "ripe-ncc",
|
|
}[rir]
|
|
|
|
|
|
def cir_tal_uri_for_rir(rir: str) -> str:
|
|
return {
|
|
"afrinic": "https://rpki.afrinic.net/tal/afrinic.tal",
|
|
"apnic": "https://rpki.apnic.net/tal/apnic-rfc7730-https.tal",
|
|
"arin": "https://www.arin.net/resources/manage/rpki/arin.tal",
|
|
"lacnic": "https://www.lacnic.net/innovaportal/file/4983/1/lacnic.tal",
|
|
"ripe": "https://tal.rpki.ripe.net/ripe-ncc.tal",
|
|
}[rir]
|
|
|
|
|
|
|
|
def run_remote_step(
|
|
ssh_target: str,
|
|
remote_root: Path,
|
|
exp_id: str,
|
|
side_label: str,
|
|
step: str,
|
|
side: dict[str, Any],
|
|
rirs: list[str],
|
|
) -> None:
|
|
exp_root = remote_root / "experiments" / exp_id
|
|
run_dir = exp_root / side_label / step
|
|
state_dir = exp_root / side_label / "state" / ("ours" if side["rpKind"] == "ours" else "rpki-client")
|
|
ensure = [
|
|
f"mkdir -p {shlex.quote(str(run_dir))}",
|
|
f"mkdir -p {shlex.quote(str(run_dir.parent))}",
|
|
f"chmod 0777 {shlex.quote(str(run_dir))}",
|
|
]
|
|
if side["rpKind"] == "ours":
|
|
ensure.extend(
|
|
[
|
|
f"mkdir -p {shlex.quote(str(state_dir / 'work-db'))}",
|
|
f"mkdir -p {shlex.quote(str(state_dir / 'rsync-mirror'))}",
|
|
f"chmod -R 0777 {shlex.quote(str(exp_root / side_label / 'state'))}",
|
|
]
|
|
)
|
|
if step == "snapshot":
|
|
ensure.insert(0, f"rm -rf {shlex.quote(str(exp_root / side_label / 'state' / 'ours'))}")
|
|
argv = [
|
|
str(remote_root / "bin" / "rpki"),
|
|
"--db",
|
|
str(state_dir / "work-db"),
|
|
"--raw-store-db",
|
|
str(state_dir / "raw-store.db"),
|
|
"--repo-bytes-db",
|
|
str(state_dir / "repo-bytes.db"),
|
|
]
|
|
for rir in rirs:
|
|
argv.extend(
|
|
[
|
|
"--tal-path",
|
|
str(remote_root / "fixtures" / "tal" / fixture_name(rir, "tal")),
|
|
"--ta-path",
|
|
str(remote_root / "fixtures" / "ta" / fixture_name(rir, "ta")),
|
|
]
|
|
)
|
|
if side["protocol"] == "rsync-only":
|
|
argv.append("--disable-rrdp")
|
|
rsync_scope = side.get("rsyncScope")
|
|
if rsync_scope:
|
|
argv.extend(["--rsync-scope", str(rsync_scope)])
|
|
if side["mode"] == "strict-name":
|
|
argv.extend(["--strict", "name"])
|
|
elif side["mode"] == "strict-cms-der":
|
|
argv.extend(["--strict", "cms-der"])
|
|
elif side["mode"] == "strict-signed-attrs":
|
|
argv.extend(["--strict", "signed-attrs"])
|
|
elif side["mode"] == "strict-all":
|
|
argv.extend(["--strict", "all"])
|
|
argv.extend(
|
|
[
|
|
"--report-json",
|
|
str(run_dir / "report.json"),
|
|
"--report-json-compact",
|
|
"--ccr-out",
|
|
str(run_dir / "result.ccr"),
|
|
"--cir-enable",
|
|
"--cir-out",
|
|
str(run_dir / "result.cir"),
|
|
]
|
|
)
|
|
for rir in rirs:
|
|
argv.extend(["--cir-tal-uri", cir_tal_uri_for_rir(rir)])
|
|
argv.extend(
|
|
[
|
|
"--vrps-csv-out",
|
|
str(run_dir / "vrps.csv"),
|
|
"--vaps-csv-out",
|
|
str(run_dir / "vaps.csv"),
|
|
]
|
|
)
|
|
else:
|
|
ensure.extend(
|
|
[
|
|
f"mkdir -p {shlex.quote(str(state_dir / 'cache'))}",
|
|
f"mkdir -p {shlex.quote(str(state_dir / 'cache' / 'fixtures'))}",
|
|
f"touch {shlex.quote(str(state_dir / 'rpki-client-skiplist'))}",
|
|
f"chmod -R 0777 {shlex.quote(str(exp_root / side_label / 'state'))}",
|
|
]
|
|
)
|
|
if step == "snapshot":
|
|
ensure.insert(0, f"rm -rf {shlex.quote(str(exp_root / side_label / 'state' / 'rpki-client'))}")
|
|
for rir in rirs:
|
|
ensure.append(
|
|
f"cp -f {shlex.quote(str(remote_root / 'fixtures' / 'ta' / fixture_name(rir, 'ta')))} {shlex.quote(str(state_dir / 'cache' / 'fixtures' / fixture_name(rir, 'ta')))}"
|
|
)
|
|
argv = [
|
|
str(remote_root / "bin" / "rpki-client"),
|
|
"-vv",
|
|
"-S",
|
|
str(state_dir / "rpki-client-skiplist"),
|
|
]
|
|
if side["protocol"] == "rsync-only":
|
|
argv.append("-R")
|
|
for rir in rirs:
|
|
argv.extend(
|
|
[
|
|
"-t",
|
|
str(remote_root / "fixtures" / "tal" / fixture_name(rir, "tal")),
|
|
"-T",
|
|
f"{fixture_desc(rir)}:{state_dir / 'cache' / 'fixtures' / fixture_name(rir, 'ta')}",
|
|
]
|
|
)
|
|
argv.extend(["-d", str(state_dir / "cache"), str(run_dir)])
|
|
|
|
time_prefix = "/usr/bin/time"
|
|
if side["rpKind"] == "ours":
|
|
time_prefix = "env RPKI_PROGRESS_LOG=1 RPKI_PROGRESS_SLOW_SECS=10 /usr/bin/time"
|
|
elif side["rpKind"] == "rpki-client":
|
|
time_prefix = f"env LD_LIBRARY_PATH={shlex.quote(str(remote_root / 'lib'))} /usr/bin/time"
|
|
|
|
command = (
|
|
"set -euo pipefail; "
|
|
+ "; ".join(ensure)
|
|
+ "; "
|
|
+ "set +e; "
|
|
+ time_prefix
|
|
+ " -v -o "
|
|
+ shlex.quote(str(run_dir / "process-time.txt"))
|
|
+ " -- "
|
|
+ rel_cmd(argv)
|
|
+ " > "
|
|
+ shlex.quote(str(run_dir / "stdout.log"))
|
|
+ " 2> "
|
|
+ shlex.quote(str(run_dir / "stderr.log"))
|
|
+ "; ec=$?; set -e; printf '%s\n' \"$ec\" > "
|
|
+ shlex.quote(str(run_dir / "exit-code.txt"))
|
|
+ "; true"
|
|
)
|
|
ssh_script(ssh_target, command)
|
|
|
|
if side["rpKind"] == "rpki-client":
|
|
copy_cmd = (
|
|
f"[ -f {shlex.quote(str(run_dir / 'json'))} ] && cp -f {shlex.quote(str(run_dir / 'json'))} {shlex.quote(str(run_dir / 'report.json'))} || true; "
|
|
f"[ -f {shlex.quote(str(run_dir / 'rpki.ccr'))} ] && cp -f {shlex.quote(str(run_dir / 'rpki.ccr'))} {shlex.quote(str(run_dir / 'result.ccr'))} || true; "
|
|
f"[ -f {shlex.quote(str(run_dir / 'rpki.cir'))} ] && cp -f {shlex.quote(str(run_dir / 'rpki.cir'))} {shlex.quote(str(run_dir / 'result.cir'))} || true; "
|
|
f"[ -f {shlex.quote(str(run_dir / 'report.json'))} ] && python3 - <<'PY' {shlex.quote(str(run_dir / 'report.json'))} {shlex.quote(str(run_dir / 'stage-timing.json'))} || true\n"
|
|
"import json, sys\n"
|
|
"report = json.load(open(sys.argv[1]))\n"
|
|
"meta = report.get('metadata', {})\n"
|
|
"stage = {\n"
|
|
" 'tool': 'rpki-client-portable',\n"
|
|
" 'metadata': {\n"
|
|
" 'elapsedTimeSeconds': meta.get('elapsedtime'),\n"
|
|
" 'userTimeSeconds': meta.get('usertime'),\n"
|
|
" 'systemTimeSeconds': meta.get('systemtime'),\n"
|
|
" },\n"
|
|
" 'counts': {\n"
|
|
" 'repositories': meta.get('repositories'),\n"
|
|
" 'vrps': meta.get('vrps'),\n"
|
|
" 'uniqueVrps': meta.get('uniquevrps'),\n"
|
|
" 'vaps': meta.get('vaps'),\n"
|
|
" 'uniqueVaps': meta.get('uniquevaps'),\n"
|
|
" }\n"
|
|
"}\n"
|
|
"json.dump(stage, open(sys.argv[2], 'w'), indent=2)\n"
|
|
"print()\n"
|
|
"PY"
|
|
)
|
|
ssh_script(ssh_target, copy_cmd)
|
|
|
|
|
|
def parse_exit_code(path: Path) -> int:
|
|
if not path.is_file():
|
|
return 1
|
|
return int(path.read_text(encoding="utf-8").strip() or "1")
|
|
|
|
|
|
def generate_run_meta(
|
|
local_exp_root: Path,
|
|
exp_id: str,
|
|
side_label: str,
|
|
step: str,
|
|
side: dict[str, Any],
|
|
rirs: list[str],
|
|
fixture_proof: Path,
|
|
run_dir: Path,
|
|
) -> Path:
|
|
result_ccr = run_dir / "result.ccr"
|
|
result_cir = run_dir / "result.cir"
|
|
report_json = run_dir / "report.json"
|
|
stage_timing_json = run_dir / "stage-timing.json"
|
|
process_time = run_dir / "process-time.txt"
|
|
stdout_log = run_dir / "stdout.log"
|
|
stderr_log = run_dir / "stderr.log"
|
|
exit_code = parse_exit_code(run_dir / "exit-code.txt")
|
|
time_info = parse_time_file(process_time)
|
|
if side["rpKind"] == "ours":
|
|
counts = load_report_counts(report_json)
|
|
counts.update(load_cir_counts(result_cir))
|
|
else:
|
|
counts = load_rpki_client_counts(report_json)
|
|
counts.update(load_cir_counts(result_cir))
|
|
strict_policies = ""
|
|
if side["mode"] == "strict-name":
|
|
strict_policies = "name"
|
|
elif side["mode"] == "strict-cms-der":
|
|
strict_policies = "cms-der"
|
|
elif side["mode"] == "strict-signed-attrs":
|
|
strict_policies = "signed-attrs"
|
|
elif side["mode"] == "strict-all":
|
|
strict_policies = "all"
|
|
|
|
meta_path = local_exp_root / side_label / step / "run-meta.json"
|
|
generate_meta_args = [
|
|
sys.executable,
|
|
str(FEATURE_BUNDLE),
|
|
"run-meta",
|
|
"--out",
|
|
str(meta_path),
|
|
"--repo-root",
|
|
str(local_exp_root),
|
|
"--experiment-id",
|
|
exp_id,
|
|
"--side",
|
|
"left" if side_label == "A" else "right",
|
|
"--side-label",
|
|
side_label,
|
|
"--step",
|
|
step,
|
|
"--run-id",
|
|
f"{side_label}-{step}",
|
|
"--rp-kind",
|
|
side["rpKind"],
|
|
"--rp-binary",
|
|
"bin/rpki" if side["rpKind"] == "ours" else "bin/rpki-client",
|
|
"--rp-version",
|
|
"portable-9.8" if side["rpKind"] == "rpki-client" else "ours-dev",
|
|
"--rp-git-commit",
|
|
git_commit(REPO_ROOT) or "",
|
|
"--rp-mode",
|
|
side["mode"],
|
|
"--protocol-mode",
|
|
side["protocol"] + (f"/rsync-scope:{side['rsyncScope']}" if side.get("rsyncScope") else ""),
|
|
"--strict-policies",
|
|
strict_policies,
|
|
"--rirs",
|
|
",".join(rirs),
|
|
"--argv-json",
|
|
json.dumps([]),
|
|
"--env-json",
|
|
json.dumps({}),
|
|
"--cwd",
|
|
str(local_exp_root),
|
|
"--state-root",
|
|
str(local_exp_root / side_label / "state"),
|
|
"--db",
|
|
str(local_exp_root / side_label / "state" / "ours" / "work-db"),
|
|
"--repo-bytes-db",
|
|
str(local_exp_root / side_label / "state" / "ours" / "repo-bytes.db"),
|
|
"--raw-store-db",
|
|
str(local_exp_root / side_label / "state" / "ours" / "raw-store.db"),
|
|
"--rsync-mirror-root",
|
|
str(local_exp_root / side_label / "state" / "ours" / "rsync-mirror"),
|
|
"--cache-root",
|
|
str(local_exp_root / side_label / "state" / "rpki-client" / "cache"),
|
|
"--ccr",
|
|
str(result_ccr),
|
|
"--cir",
|
|
str(result_cir),
|
|
"--fixture-proof",
|
|
str(fixture_proof),
|
|
"--fixture-proof-summary-json",
|
|
json.dumps(
|
|
{
|
|
"taFixturePinned": True,
|
|
"taOnlineFetchObserved": False,
|
|
"trustAnchorCount": len(rirs),
|
|
}
|
|
),
|
|
"--report-json",
|
|
str(report_json),
|
|
"--stage-timing-json",
|
|
str(stage_timing_json),
|
|
"--stdout-log",
|
|
str(stdout_log),
|
|
"--stderr-log",
|
|
str(stderr_log),
|
|
"--process-time",
|
|
str(process_time),
|
|
"--exit-code",
|
|
str(exit_code),
|
|
"--wall-ms",
|
|
str(int(time_info.get("wallMs", 0))),
|
|
"--max-rss-kb",
|
|
str(int(time_info.get("maxRssKb", 0))),
|
|
"--vrps",
|
|
str(int(counts.get("vrps", 0))),
|
|
"--vaps",
|
|
str(int(counts.get("aspas", 0))),
|
|
"--publication-points",
|
|
str(int(counts.get("publicationPoints", counts.get("repositories", 0)))),
|
|
"--warnings",
|
|
str(int(counts.get("warnings", 0))),
|
|
"--cir-object-count",
|
|
str(int(counts.get("cirObjectCount", 0))),
|
|
"--cir-reject-count",
|
|
str(int(counts.get("cirRejectCount", 0))),
|
|
"--cir-trust-anchor-count",
|
|
str(int(counts.get("cirTrustAnchorCount", len(rirs)))),
|
|
"--host",
|
|
os.uname().nodename,
|
|
"--platform",
|
|
sys.platform,
|
|
]
|
|
if step == "snapshot":
|
|
insert_at = generate_meta_args.index("--state-root")
|
|
generate_meta_args.insert(insert_at, "--reset-before-run")
|
|
run_local(generate_meta_args)
|
|
return meta_path
|
|
|
|
|
|
def run_experiment(
|
|
ssh_target: str,
|
|
local_run_root: Path,
|
|
remote_root: Path,
|
|
exp: dict[str, Any],
|
|
rirs: list[str],
|
|
dry_run: bool = False,
|
|
) -> dict[str, Any]:
|
|
exp_id = exp["id"]
|
|
local_exp_root = local_run_root / "experiments" / exp_id
|
|
remote_exp_root = remote_root / "experiments" / exp_id
|
|
ensure_dir(local_exp_root)
|
|
if dry_run:
|
|
return render_experiment_plan(exp, local_run_root, remote_root, ssh_target, rirs)
|
|
|
|
fixture_proof = build_fixture_proof(local_exp_root, rirs)
|
|
preflight = (
|
|
"set -euo pipefail; "
|
|
f"df -h /data / || true; "
|
|
"systemctl disable --now rpki-client.timer >/dev/null 2>&1 || true; "
|
|
"systemctl stop rpki-client.service >/dev/null 2>&1 || true; "
|
|
"pkill -f '[/]rpki-client([[:space:]]|$)' >/dev/null 2>&1 || true; "
|
|
"pkill -f '[/]routinator([[:space:]]|$)' >/dev/null 2>&1 || true; "
|
|
"id -u _rpki-client >/dev/null 2>&1 || useradd -r -M -s /usr/sbin/nologin _rpki-client || true; "
|
|
f"mkdir -p {shlex.quote(str(remote_root / 'bin'))} {shlex.quote(str(remote_root / 'lib'))} {shlex.quote(str(remote_root / 'fixtures' / 'tal'))} {shlex.quote(str(remote_root / 'fixtures' / 'ta'))} {shlex.quote(str(remote_exp_root))}"
|
|
)
|
|
ssh_script(ssh_target, preflight)
|
|
rsync_dir_to_remote(ssh_target, REPO_ROOT / "tests" / "fixtures" / "tal", remote_root / "fixtures" / "tal")
|
|
rsync_dir_to_remote(ssh_target, REPO_ROOT / "tests" / "fixtures" / "ta", remote_root / "fixtures" / "ta")
|
|
|
|
if exp["left"]["rpKind"] == "rpki-client" or exp["right"]["rpKind"] == "rpki-client":
|
|
rsync_to_remote(ssh_target, detect_libtls_path(PORTABLE_ROOT / "src" / "rpki-client"), str(remote_root / "lib" / "libtls.so.28"))
|
|
rsync_to_remote(ssh_target, PORTABLE_ROOT / "src" / "rpki-client", str(remote_root / "bin" / "rpki-client"))
|
|
rsync_to_remote(ssh_target, REPO_ROOT / "target" / "release" / "rpki", str(remote_root / "bin" / "rpki"))
|
|
rsync_to_remote(ssh_target, REPO_ROOT / "target" / "release" / "triage_ccr_cir_pair", str(remote_root / "bin" / "triage_ccr_cir_pair"))
|
|
rsync_to_remote(ssh_target, REPO_ROOT / "target" / "release" / "cir_dump_reject_list", str(remote_root / "bin" / "cir_dump_reject_list"))
|
|
|
|
step_results: list[dict[str, Any]] = []
|
|
for step in ("snapshot", "delta"):
|
|
for side_label, side in (("A", exp["left"]), ("B", exp["right"])):
|
|
run_remote_step(ssh_target, remote_root, exp_id, side_label, step, side, rirs)
|
|
local_step_root = local_exp_root / side_label / step
|
|
remote_step_root = remote_exp_root / side_label / step
|
|
rsync_from_remote(ssh_target, str(remote_step_root), local_step_root)
|
|
if side["rpKind"] == "rpki-client":
|
|
copy_rpki_client_outputs(local_step_root)
|
|
write_rpki_client_stage_timing(local_step_root)
|
|
# ensure ours has a stable stage timing artifact too
|
|
if not (local_step_root / "stage-timing.json").is_file() and (local_step_root / "report.json").is_file():
|
|
report = load_json(local_step_root / "report.json")
|
|
stage_timing = {
|
|
"tool": side["rpKind"],
|
|
"counts": {
|
|
"publicationPoints": len(report.get("publication_points", [])),
|
|
"vrps": len(report.get("vrps", [])),
|
|
"aspas": len(report.get("aspas", [])),
|
|
},
|
|
}
|
|
write_json(local_step_root / "stage-timing.json", stage_timing)
|
|
|
|
meta_path = generate_run_meta(
|
|
local_exp_root,
|
|
exp_id,
|
|
side_label,
|
|
step,
|
|
side,
|
|
rirs,
|
|
fixture_proof,
|
|
local_step_root,
|
|
)
|
|
step_results.append(
|
|
{
|
|
"side": side_label,
|
|
"step": step,
|
|
"runDir": str(local_step_root),
|
|
"meta": str(meta_path),
|
|
}
|
|
)
|
|
|
|
compare_dir = local_exp_root / "compare" / step
|
|
ensure_dir(compare_dir)
|
|
triage_cmd = [
|
|
str(REPO_ROOT / "target" / "release" / "triage_ccr_cir_pair"),
|
|
"--left-ccr",
|
|
str(local_exp_root / "A" / step / "result.ccr"),
|
|
"--left-cir",
|
|
str(local_exp_root / "A" / step / "result.cir"),
|
|
"--right-ccr",
|
|
str(local_exp_root / "B" / step / "result.ccr"),
|
|
"--right-cir",
|
|
str(local_exp_root / "B" / step / "result.cir"),
|
|
"--out-dir",
|
|
str(compare_dir),
|
|
"--sample-limit",
|
|
"200",
|
|
]
|
|
run_local(triage_cmd, cwd=local_exp_root)
|
|
|
|
step_summary = {
|
|
"step": step,
|
|
"compareDir": str(compare_dir),
|
|
"triage": load_json(compare_dir / "triage.json"),
|
|
}
|
|
write_json(local_exp_root / "compare" / f"{step}.summary.json", step_summary)
|
|
step_results.append(step_summary)
|
|
|
|
experiment_summary = {
|
|
"schemaVersion": 1,
|
|
"experimentId": exp_id,
|
|
"left": exp["left"],
|
|
"right": exp["right"],
|
|
"rirs": rirs,
|
|
"steps": step_results,
|
|
}
|
|
write_json(local_exp_root / "experiment-summary.json", experiment_summary)
|
|
with (local_exp_root / "experiment-summary.md").open("w", encoding="utf-8") as handle:
|
|
handle.write(f"# {exp_id}\n\n")
|
|
handle.write(json.dumps(experiment_summary, indent=2, ensure_ascii=False))
|
|
handle.write("\n")
|
|
|
|
rsync_dir_to_remote(ssh_target, local_exp_root, str(remote_exp_root))
|
|
return experiment_summary
|
|
|
|
|
|
def is_rpki_client_experiment(exp: dict[str, Any]) -> bool:
|
|
return exp["left"]["rpKind"] == "rpki-client" or exp["right"]["rpKind"] == "rpki-client"
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="Feature #035 experiment driver")
|
|
parser.add_argument("--run-root", required=True)
|
|
parser.add_argument("--remote-root", required=True)
|
|
parser.add_argument("--ssh-target", default=os.environ.get("SSH_TARGET", "root@47.251.56.108"))
|
|
parser.add_argument("--experiment", action="append", help="Experiment id to run; repeatable")
|
|
parser.add_argument("--all", action="store_true", help="Run all experiments from experiments.json")
|
|
parser.add_argument("--dry-run", action="store_true")
|
|
parser.add_argument("--rirs", default="afrinic,apnic,arin,lacnic,ripe")
|
|
args = parser.parse_args()
|
|
|
|
experiments_doc = load_json(EXPERIMENTS_PATH)
|
|
experiments = experiments_doc["experiments"]
|
|
selected_ids = set(args.experiment or [])
|
|
if args.all or not selected_ids:
|
|
selected = experiments
|
|
else:
|
|
selected = [exp for exp in experiments if exp["id"] in selected_ids]
|
|
if not selected:
|
|
raise SystemExit("no experiments selected")
|
|
|
|
rirs = [item.strip() for item in args.rirs.split(",") if item.strip()]
|
|
if not rirs:
|
|
raise SystemExit("--rirs must not be empty")
|
|
|
|
run_root = Path(args.run_root).resolve()
|
|
remote_root = Path(args.remote_root)
|
|
ensure_dir(run_root)
|
|
if not args.dry_run:
|
|
build_tool_binaries()
|
|
|
|
if args.dry_run:
|
|
plans = [
|
|
run_experiment(args.ssh_target, run_root, remote_root, exp, rirs, dry_run=True)
|
|
for exp in selected
|
|
]
|
|
print(json.dumps({"schemaVersion": 1, "dryRun": True, "experiments": plans}, indent=2, ensure_ascii=False))
|
|
return
|
|
|
|
summary = {
|
|
"schemaVersion": 1,
|
|
"generatedAtUtc": utc_stamp(),
|
|
"runRoot": str(run_root),
|
|
"remoteRoot": str(remote_root),
|
|
"sshTarget": args.ssh_target,
|
|
"experiments": [],
|
|
}
|
|
for exp in selected:
|
|
summary["experiments"].append(
|
|
run_experiment(args.ssh_target, run_root, remote_root, exp, rirs, dry_run=False)
|
|
)
|
|
write_json(run_root / "feature035-summary.json", summary)
|
|
print(json.dumps(summary, indent=2, ensure_ascii=False))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|