#!/usr/bin/env python3
from __future__ import annotations

import argparse
import json
import os
import shlex
import shutil
import subprocess
import sys
import time
from dataclasses import dataclass
from pathlib import Path
from typing import Any


SCRIPT_DIR = Path(__file__).resolve().parent
REPO_ROOT = SCRIPT_DIR.parents[2]
DEV_ROOT = REPO_ROOT.parents[1]
PORTABLE_ROOT = DEV_ROOT / "rpki-client-portable"
FEATURE_BUNDLE = SCRIPT_DIR / "feature035_bundle.py"
EXPERIMENTS_PATH = SCRIPT_DIR / "experiments.json"
FIXTURE_MANIFEST_PATH = SCRIPT_DIR / "fixture-manifest.json"


def load_json(path: Path) -> Any:
    with path.open("r", encoding="utf-8") as handle:
        return json.load(handle)


def write_json(path: Path, value: Any) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    with path.open("w", encoding="utf-8") as handle:
        json.dump(value, handle, indent=2, ensure_ascii=False)
        handle.write("\n")


def ensure_dir(path: Path) -> None:
    path.mkdir(parents=True, exist_ok=True)


def git_commit(repo_root: Path) -> str:
    result = run_local(
        ["git", "-C", str(repo_root), "rev-parse", "--short", "HEAD"],
        capture=True,
        check=False,
    )
    return result.stdout.strip() if result.returncode == 0 else ""


def run_local(
    argv: list[str],
    *,
    cwd: Path | None = None,
    check: bool = True,
    capture: bool = False,
    env: dict[str, str] | None = None,
) -> subprocess.CompletedProcess[str]:
    result = subprocess.run(
        argv,
        cwd=str(cwd) if cwd else None,
        text=True,
        check=False,
        capture_output=capture,
        env=env,
    )
    if check and result.returncode != 0:
        raise SystemExit(
            f"command failed ({result.returncode}): {' '.join(shlex.quote(item) for item in argv)}\n"
            f"stdout:\n{result.stdout if result.stdout else ''}\n"
            f"stderr:\n{result.stderr if result.stderr else ''}"
        )
    return result


def ssh_script(target: str, script: str, *, check: bool = True) -> subprocess.CompletedProcess[str]:
    result = subprocess.run(
        ["ssh", target, "bash", "-s"],
        input=script,
        text=True,
        check=False,
    )
    if check and result.returncode != 0:
        raise SystemExit(f"remote script failed ({result.returncode}) on {target}")
    return result


def rsync_to_remote(target: str, source: Path, destination: str) -> None:
    run_local(["rsync", "-a", str(source), f"{target}:{destination}"])


def rsync_dir_to_remote(target: str, source: Path, destination: str) -> None:
    run_local(["rsync", "-a", f"{source}/", f"{target}:{destination}/"])


def rsync_from_remote(target: str, source: str, destination: Path) -> None:
    ensure_dir(destination)
    run_local(["rsync", "-a", f"{target}:{source}/", f"{destination}/"])


def rel_cmd(parts: list[str]) -> str:
    return shlex.join(str(part) for part in parts)


def utc_stamp() -> str:
    return time.strftime("%Y%m%dT%H%M%SZ", time.gmtime())


def detect_libtls_path(rpki_client_bin: Path) -> Path:
    ldd = run_local(["ldd", str(rpki_client_bin)], capture=True)
    for line in ldd.stdout.splitlines():
        if "libtls.so.28" not in line:
            continue
        if "=>" not in line:
            continue
        candidate = line.split("=>", 1)[1].strip().split(" ", 1)[0]
        path = Path(candidate)
        if path.is_file():
            return path
    fallback = DEV_ROOT / ".cache" / "rpki-client-9.8-cir" / "libtls.so.28"
    if fallback.is_file():
        return fallback
    raise SystemExit("unable to locate libtls.so.28 for rpki-client")


def parse_elapsed_to_ms(raw: str) -> int:
    raw = raw.strip()
    if not raw:
        return 0
    if "-" in raw:
        days, raw = raw.split("-", 1)
    else:
        days = "0"
    parts = raw.split(":")
    try:
        if len(parts) == 4:
            days = str(int(days) + int(parts[0]))
            hours, minutes, seconds = parts[1:]
        elif len(parts) == 3:
            hours, minutes, seconds = parts
        elif len(parts) == 2:
            hours = "0"
            minutes, seconds = parts
        else:
            hours = "0"
            minutes = "0"
            seconds = parts[0]
        total_seconds = (
            int(days) * 86400 + int(hours) * 3600 + int(minutes) * 60 + float(seconds)
        )
    except ValueError:
        time_part = raw.rsplit(":", 1)[-1]
        total_seconds = float(time_part)
    return int(round(total_seconds * 1000))


def parse_time_file(path: Path) -> dict[str, Any]:
    data: dict[str, Any] = {}
    if not path.is_file():
        return data
    for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
        if "Elapsed (wall clock) time" in line and ":" in line:
            if "):" in line:
                elapsed = line.rsplit("):", 1)[1]
            else:
                elapsed = line.rsplit(":", 1)[1]
            data["wallMs"] = parse_elapsed_to_ms(elapsed)
        elif "Maximum resident set size" in line and ":" in line:
            try:
                data["maxRssKb"] = int(line.rsplit(":", 1)[1].strip())
            except ValueError:
                pass
        elif "User time (seconds)" in line and ":" in line:
            try:
                data["userSeconds"] = float(line.rsplit(":", 1)[1].strip())
            except ValueError:
                pass
        elif "System time (seconds)" in line and ":" in line:
            try:
                data["systemSeconds"] = float(line.rsplit(":", 1)[1].strip())
            except ValueError:
                pass
    return data


def path_within(base: Path, path: Path) -> str:
    return path.relative_to(base).as_posix()


def load_report_counts(report_path: Path) -> dict[str, Any]:
    report = load_json(report_path)
    publication_points = report.get("publication_points", [])
    tree = report.get("tree", {})
    warnings = tree.get("warnings", [])
    return {
        "vrps": len(report.get("vrps", [])),
        "aspas": len(report.get("aspas", [])),
        "publicationPoints": len(publication_points),
        "warnings": len(warnings)
        + sum(len(pp.get("warnings", [])) for pp in publication_points if isinstance(pp, dict)),
        "treeInstancesProcessed": tree.get("instances_processed"),
        "treeInstancesFailed": tree.get("instances_failed"),
        "reportJson": report,
    }


def load_rpki_client_counts(report_path: Path) -> dict[str, Any]:
    report = load_json(report_path)
    metadata = report.get("metadata", {})
    roas = report.get("roas", [])
    aspas = report.get("aspas", [])
    return {
        "vrps": int(metadata.get("vrps", len(roas))),
        "uniqueVrps": int(metadata.get("uniquevrps", len(roas))),
        "aspas": int(metadata.get("aspas", len(aspas))),
        "uniqueVaps": int(metadata.get("uniquevaps", len(aspas))),
        "repositories": int(metadata.get("repositories", 0)),
        "warnings": 0,
        "reportJson": report,
    }


def load_cir_counts(cir_path: Path) -> dict[str, int]:
    if not cir_path.is_file():
        return {
            "cirObjectCount": 0,
            "cirRejectCount": 0,
            "cirTrustAnchorCount": 0,
        }
    result = run_local(
        [
            str(REPO_ROOT / "target" / "release" / "cir_dump_reject_list"),
            "--cir",
            str(cir_path),
            "--limit",
            "0",
        ],
        capture=True,
        check=False,
    )
    if result.returncode != 0:
        raise SystemExit(f"decode CIR failed: {cir_path}\n{result.stderr}")
    values: dict[str, int] = {}
    for line in result.stdout.splitlines():
        if "=" not in line:
            continue
        key, value = line.split("=", 1)
        if key in {"object_count", "trust_anchor_count", "reject_count"}:
            values[key] = int(value)
    return {
        "cirObjectCount": values.get("object_count", 0),
        "cirRejectCount": values.get("reject_count", 0),
        "cirTrustAnchorCount": values.get("trust_anchor_count", 0),
    }


def copy_rpki_client_outputs(run_dir: Path) -> None:
    if (run_dir / "json").is_file():
        shutil.copy2(run_dir / "json", run_dir / "report.json")
    if (run_dir / "rpki.ccr").is_file():
        shutil.copy2(run_dir / "rpki.ccr", run_dir / "result.ccr")
    if (run_dir / "rpki.cir").is_file():
        shutil.copy2(run_dir / "rpki.cir", run_dir / "result.cir")


def write_rpki_client_stage_timing(run_dir: Path) -> None:
    report_path = run_dir / "report.json"
    if not report_path.is_file():
        return
    report = load_json(report_path)
    metadata = report.get("metadata", {})
    stage_timing = {
        "tool": "rpki-client-portable",
        "metadata": {
            "elapsedTimeSeconds": metadata.get("elapsedtime"),
            "userTimeSeconds": metadata.get("usertime"),
            "systemTimeSeconds": metadata.get("systemtime"),
        },
        "counts": {
            "repositories": metadata.get("repositories"),
            "vrps": metadata.get("vrps"),
            "uniqueVrps": metadata.get("uniquevrps"),
            "vaps": metadata.get("vaps"),
            "uniqueVaps": metadata.get("uniquevaps"),
        },
    }
    write_json(run_dir / "stage-timing.json", stage_timing)


def build_tool_binaries() -> None:
    run_local(
        [
            "cargo",
            "build",
            "--release",
            "--bin",
            "rpki",
            "--bin",
            "triage_ccr_cir_pair",
            "--bin",
            "cir_dump_reject_list",
        ],
        cwd=REPO_ROOT,
    )
    rpki_client_bin = PORTABLE_ROOT / "src" / "rpki-client"
    if not rpki_client_bin.is_file():
        run_local(["make", "-j2"], cwd=PORTABLE_ROOT)
    smoke = run_local(
        [str(rpki_client_bin), "-T", "invalid"],
        capture=True,
        check=False,
    )
    if "--ta-fixture requires <tal>:<path>" not in (smoke.stderr + smoke.stdout):
        cache_bin = DEV_ROOT / ".cache" / "rpki-client-9.8-cir" / "rpki-client"
        cache_smoke = run_local(
            [str(cache_bin), "-T", "invalid"],
            capture=True,
            check=False,
        ) if cache_bin.is_file() else None
        if cache_smoke and "--ta-fixture requires <tal>:<path>" in (cache_smoke.stderr + cache_smoke.stdout):
            shutil.copy2(cache_bin, rpki_client_bin)
        else:
            raise SystemExit(
                "rpki-client binary lacks local TA fixture support (-T); "
                "switch to feature/cir-output-for-rp-compare and rebuild or restore the cached CIR+TA-fixture binary"
            )


def build_fixture_proof(run_root: Path, rirs: list[str]) -> Path:
    fixture_dir = REPO_ROOT / "tests" / "fixtures"
    fixture_proof = run_root / "fixture-proof.json"
    run_local(
        [
            sys.executable,
            str(FEATURE_BUNDLE),
            "fixture-proof",
            "--fixture-dir",
            str(fixture_dir),
            "--repo-root",
            str(run_root),
            "--rirs",
            ",".join(rirs),
            "--out",
            str(fixture_proof),
        ]
    )
    return fixture_proof


def experiment_steps(exp: dict[str, Any]) -> list[dict[str, str]]:
    return [
        {
            "step": "snapshot",
            "left": exp["left"],
            "right": exp["right"],
        },
        {
            "step": "delta",
            "left": exp["left"],
            "right": exp["right"],
        },
    ]


def render_experiment_plan(exp: dict[str, Any], run_root: Path, remote_root: Path, ssh_target: str, rirs: list[str]) -> dict[str, Any]:
    plan_steps: list[dict[str, Any]] = []
    for step in ("snapshot", "delta"):
        plan_steps.append(
            {
                "step": step,
                "leftRunDir": str(run_root / "experiments" / exp["id"] / "A" / step),
                "rightRunDir": str(run_root / "experiments" / exp["id"] / "B" / step),
                "remoteLeftRunDir": str(remote_root / "experiments" / exp["id"] / "A" / step),
                "remoteRightRunDir": str(remote_root / "experiments" / exp["id"] / "B" / step),
                "leftCommand": build_side_command(remote_root, exp["id"], exp["left"], "A", step, rirs),
                "rightCommand": build_side_command(remote_root, exp["id"], exp["right"], "B", step, rirs),
            }
        )
    return {
        "id": exp["id"],
        "left": exp["left"],
        "right": exp["right"],
        "sshTarget": ssh_target,
        "remoteRoot": str(remote_root),
        "runRoot": str(run_root),
        "steps": plan_steps,
    }


def build_side_command(
    remote_root: Path,
    exp_id: str,
    side: dict[str, Any],
    side_label: str,
    step: str,
    rirs: list[str],
) -> str:
    run_dir = remote_root / "experiments" / exp_id / side_label / step
    state_dir = remote_root / "experiments" / exp_id / side_label / "state" / ("ours" if side["rpKind"] == "ours" else "rpki-client")
    fixture_root = remote_root / "fixtures"
    if side["rpKind"] == "ours":
        argv = [
            str(remote_root / "bin" / "rpki"),
            "--db",
            str(state_dir / "work-db"),
            "--raw-store-db",
            str(state_dir / "raw-store.db"),
            "--repo-bytes-db",
            str(state_dir / "repo-bytes.db"),
        ]
        for rir in rirs:
            argv.extend(
                [
                    "--tal-path",
                    str(fixture_root / "tal" / fixture_name(rir, "tal")),
                    "--ta-path",
                    str(fixture_root / "ta" / fixture_name(rir, "ta")),
                ]
            )
        if side["protocol"] == "rsync-only":
            argv.append("--disable-rrdp")
        rsync_scope = side.get("rsyncScope")
        if rsync_scope:
            argv.extend(["--rsync-scope", str(rsync_scope)])
        if side["mode"] == "strict-name":
            argv.extend(["--strict", "name"])
        elif side["mode"] == "strict-cms-der":
            argv.extend(["--strict", "cms-der"])
        elif side["mode"] == "strict-signed-attrs":
            argv.extend(["--strict", "signed-attrs"])
        elif side["mode"] == "strict-all":
            argv.extend(["--strict", "all"])
        argv.extend(
            [
                "--report-json",
                str(run_dir / "report.json"),
                "--report-json-compact",
                "--ccr-out",
                str(run_dir / "result.ccr"),
                "--cir-enable",
                "--cir-out",
                str(run_dir / "result.cir"),
            ]
        )
        for rir in rirs:
            argv.extend(["--cir-tal-uri", cir_tal_uri_for_rir(rir)])
        argv.extend(
            [
                "--vrps-csv-out",
                str(run_dir / "vrps.csv"),
                "--vaps-csv-out",
                str(run_dir / "vaps.csv"),
            ]
        )
        return "cd {run_dir} && {prefix} /usr/bin/time -v -o process-time.txt -- {cmd} > stdout.log 2> stderr.log".format(
            run_dir=shlex.quote(str(run_dir)),
            prefix=_setup_prefix(step, state_dir, "ours"),
            cmd=rel_cmd(argv),
        )

    argv = [
        str(remote_root / "bin" / "rpki-client"),
        "-vv",
        "-S",
        str(state_dir / "rpki-client-skiplist"),
    ]
    if side["protocol"] == "rsync-only":
        argv.append("-R")
    for rir in rirs:
        argv.extend(
            [
                "-t",
                str(fixture_root / "tal" / fixture_name(rir, "tal")),
                "-T",
                f"{fixture_desc(rir)}:{state_dir / 'cache' / 'fixtures' / fixture_name(rir, 'ta')}",
            ]
        )
    argv.extend(["-d", str(state_dir / "cache"), str(run_dir)])
    return "cd {run_dir} && {prefix} LD_LIBRARY_PATH={lib} /usr/bin/time -v -o process-time.txt -- {cmd} > stdout.log 2> stderr.log".format(
        run_dir=shlex.quote(str(run_dir)),
        prefix=_setup_prefix(step, state_dir, "rpki-client"),
        lib=shlex.quote(str(remote_root / "lib")),
        cmd=rel_cmd(argv),
    )


def _setup_prefix(step: str, state_dir: Path, kind: str) -> str:
    if step == "snapshot":
        if kind == "ours":
            return "rm -rf {state} && mkdir -p {state}/work-db {state}/rsync-mirror && chmod -R 0777 {state} && ".format(
                state=shlex.quote(str(state_dir))
            )
        return "rm -rf {state} && mkdir -p {state}/cache {state}/cache/fixtures && touch {state}/rpki-client-skiplist && chmod -R 0777 {state} && ".format(
            state=shlex.quote(str(state_dir))
        )
    if kind == "ours":
        return "mkdir -p {state}/work-db {state}/rsync-mirror && chmod -R 0777 {state} && ".format(
            state=shlex.quote(str(state_dir))
        )
    return "mkdir -p {state}/cache {state}/cache/fixtures && touch {state}/rpki-client-skiplist && chmod -R 0777 {state} && ".format(
        state=shlex.quote(str(state_dir))
    )


def fixture_name(rir: str, kind: str) -> str:
    fixture_manifest = load_json(FIXTURE_MANIFEST_PATH)
    return Path(fixture_manifest["rirs"][rir][kind]).name


def fixture_desc(rir: str) -> str:
    return {
        "afrinic": "afrinic",
        "apnic": "apnic-rfc7730-https",
        "arin": "arin",
        "lacnic": "lacnic",
        "ripe": "ripe-ncc",
    }[rir]


def cir_tal_uri_for_rir(rir: str) -> str:
    return {
        "afrinic": "https://rpki.afrinic.net/tal/afrinic.tal",
        "apnic": "https://rpki.apnic.net/tal/apnic-rfc7730-https.tal",
        "arin": "https://www.arin.net/resources/manage/rpki/arin.tal",
        "lacnic": "https://www.lacnic.net/innovaportal/file/4983/1/lacnic.tal",
        "ripe": "https://tal.rpki.ripe.net/ripe-ncc.tal",
    }[rir]



def run_remote_step(
    ssh_target: str,
    remote_root: Path,
    exp_id: str,
    side_label: str,
    step: str,
    side: dict[str, Any],
    rirs: list[str],
) -> None:
    exp_root = remote_root / "experiments" / exp_id
    run_dir = exp_root / side_label / step
    state_dir = exp_root / side_label / "state" / ("ours" if side["rpKind"] == "ours" else "rpki-client")
    ensure = [
        f"mkdir -p {shlex.quote(str(run_dir))}",
        f"mkdir -p {shlex.quote(str(run_dir.parent))}",
        f"chmod 0777 {shlex.quote(str(run_dir))}",
    ]
    if side["rpKind"] == "ours":
        ensure.extend(
            [
                f"mkdir -p {shlex.quote(str(state_dir / 'work-db'))}",
                f"mkdir -p {shlex.quote(str(state_dir / 'rsync-mirror'))}",
                f"chmod -R 0777 {shlex.quote(str(exp_root / side_label / 'state'))}",
            ]
        )
        if step == "snapshot":
            ensure.insert(0, f"rm -rf {shlex.quote(str(exp_root / side_label / 'state' / 'ours'))}")
        argv = [
            str(remote_root / "bin" / "rpki"),
            "--db",
            str(state_dir / "work-db"),
            "--raw-store-db",
            str(state_dir / "raw-store.db"),
            "--repo-bytes-db",
            str(state_dir / "repo-bytes.db"),
        ]
        for rir in rirs:
            argv.extend(
                [
                    "--tal-path",
                    str(remote_root / "fixtures" / "tal" / fixture_name(rir, "tal")),
                    "--ta-path",
                    str(remote_root / "fixtures" / "ta" / fixture_name(rir, "ta")),
                ]
            )
        if side["protocol"] == "rsync-only":
            argv.append("--disable-rrdp")
        rsync_scope = side.get("rsyncScope")
        if rsync_scope:
            argv.extend(["--rsync-scope", str(rsync_scope)])
        if side["mode"] == "strict-name":
            argv.extend(["--strict", "name"])
        elif side["mode"] == "strict-cms-der":
            argv.extend(["--strict", "cms-der"])
        elif side["mode"] == "strict-signed-attrs":
            argv.extend(["--strict", "signed-attrs"])
        elif side["mode"] == "strict-all":
            argv.extend(["--strict", "all"])
        argv.extend(
            [
                "--report-json",
                str(run_dir / "report.json"),
                "--report-json-compact",
                "--ccr-out",
                str(run_dir / "result.ccr"),
                "--cir-enable",
                "--cir-out",
                str(run_dir / "result.cir"),
            ]
        )
        for rir in rirs:
            argv.extend(["--cir-tal-uri", cir_tal_uri_for_rir(rir)])
        argv.extend(
            [
                "--vrps-csv-out",
                str(run_dir / "vrps.csv"),
                "--vaps-csv-out",
                str(run_dir / "vaps.csv"),
            ]
        )
    else:
        ensure.extend(
            [
                f"mkdir -p {shlex.quote(str(state_dir / 'cache'))}",
                f"mkdir -p {shlex.quote(str(state_dir / 'cache' / 'fixtures'))}",
                f"touch {shlex.quote(str(state_dir / 'rpki-client-skiplist'))}",
                f"chmod -R 0777 {shlex.quote(str(exp_root / side_label / 'state'))}",
            ]
        )
        if step == "snapshot":
            ensure.insert(0, f"rm -rf {shlex.quote(str(exp_root / side_label / 'state' / 'rpki-client'))}")
        for rir in rirs:
            ensure.append(
                f"cp -f {shlex.quote(str(remote_root / 'fixtures' / 'ta' / fixture_name(rir, 'ta')))} {shlex.quote(str(state_dir / 'cache' / 'fixtures' / fixture_name(rir, 'ta')))}"
            )
        argv = [
            str(remote_root / "bin" / "rpki-client"),
            "-vv",
            "-S",
            str(state_dir / "rpki-client-skiplist"),
        ]
        if side["protocol"] == "rsync-only":
            argv.append("-R")
        for rir in rirs:
            argv.extend(
                [
                    "-t",
                    str(remote_root / "fixtures" / "tal" / fixture_name(rir, "tal")),
                    "-T",
                    f"{fixture_desc(rir)}:{state_dir / 'cache' / 'fixtures' / fixture_name(rir, 'ta')}",
                ]
            )
        argv.extend(["-d", str(state_dir / "cache"), str(run_dir)])

    time_prefix = "/usr/bin/time"
    if side["rpKind"] == "ours":
        time_prefix = "env RPKI_PROGRESS_LOG=1 RPKI_PROGRESS_SLOW_SECS=10 /usr/bin/time"
    elif side["rpKind"] == "rpki-client":
        time_prefix = f"env LD_LIBRARY_PATH={shlex.quote(str(remote_root / 'lib'))} /usr/bin/time"

    command = (
        "set -euo pipefail; "
        + "; ".join(ensure)
        + "; "
        + "set +e; "
        + time_prefix
        + " -v -o "
        + shlex.quote(str(run_dir / "process-time.txt"))
        + " -- "
        + rel_cmd(argv)
        + " > "
        + shlex.quote(str(run_dir / "stdout.log"))
        + " 2> "
        + shlex.quote(str(run_dir / "stderr.log"))
        + "; ec=$?; set -e; printf '%s\n' \"$ec\" > "
        + shlex.quote(str(run_dir / "exit-code.txt"))
        + "; true"
    )
    ssh_script(ssh_target, command)

    if side["rpKind"] == "rpki-client":
        copy_cmd = (
            f"[ -f {shlex.quote(str(run_dir / 'json'))} ] && cp -f {shlex.quote(str(run_dir / 'json'))} {shlex.quote(str(run_dir / 'report.json'))} || true; "
            f"[ -f {shlex.quote(str(run_dir / 'rpki.ccr'))} ] && cp -f {shlex.quote(str(run_dir / 'rpki.ccr'))} {shlex.quote(str(run_dir / 'result.ccr'))} || true; "
            f"[ -f {shlex.quote(str(run_dir / 'rpki.cir'))} ] && cp -f {shlex.quote(str(run_dir / 'rpki.cir'))} {shlex.quote(str(run_dir / 'result.cir'))} || true; "
            f"[ -f {shlex.quote(str(run_dir / 'report.json'))} ] && python3 - <<'PY' {shlex.quote(str(run_dir / 'report.json'))} {shlex.quote(str(run_dir / 'stage-timing.json'))} || true\n"
            "import json, sys\n"
            "report = json.load(open(sys.argv[1]))\n"
            "meta = report.get('metadata', {})\n"
            "stage = {\n"
            "  'tool': 'rpki-client-portable',\n"
            "  'metadata': {\n"
            "    'elapsedTimeSeconds': meta.get('elapsedtime'),\n"
            "    'userTimeSeconds': meta.get('usertime'),\n"
            "    'systemTimeSeconds': meta.get('systemtime'),\n"
            "  },\n"
            "  'counts': {\n"
            "    'repositories': meta.get('repositories'),\n"
            "    'vrps': meta.get('vrps'),\n"
            "    'uniqueVrps': meta.get('uniquevrps'),\n"
            "    'vaps': meta.get('vaps'),\n"
            "    'uniqueVaps': meta.get('uniquevaps'),\n"
            "  }\n"
            "}\n"
            "json.dump(stage, open(sys.argv[2], 'w'), indent=2)\n"
            "print()\n"
            "PY"
        )
        ssh_script(ssh_target, copy_cmd)


def parse_exit_code(path: Path) -> int:
    if not path.is_file():
        return 1
    return int(path.read_text(encoding="utf-8").strip() or "1")


def generate_run_meta(
    local_exp_root: Path,
    exp_id: str,
    side_label: str,
    step: str,
    side: dict[str, Any],
    rirs: list[str],
    fixture_proof: Path,
    run_dir: Path,
) -> Path:
    result_ccr = run_dir / "result.ccr"
    result_cir = run_dir / "result.cir"
    report_json = run_dir / "report.json"
    stage_timing_json = run_dir / "stage-timing.json"
    process_time = run_dir / "process-time.txt"
    stdout_log = run_dir / "stdout.log"
    stderr_log = run_dir / "stderr.log"
    exit_code = parse_exit_code(run_dir / "exit-code.txt")
    time_info = parse_time_file(process_time)
    if side["rpKind"] == "ours":
        counts = load_report_counts(report_json)
        counts.update(load_cir_counts(result_cir))
    else:
        counts = load_rpki_client_counts(report_json)
        counts.update(load_cir_counts(result_cir))
    strict_policies = ""
    if side["mode"] == "strict-name":
        strict_policies = "name"
    elif side["mode"] == "strict-cms-der":
        strict_policies = "cms-der"
    elif side["mode"] == "strict-signed-attrs":
        strict_policies = "signed-attrs"
    elif side["mode"] == "strict-all":
        strict_policies = "all"

    meta_path = local_exp_root / side_label / step / "run-meta.json"
    generate_meta_args = [
        sys.executable,
        str(FEATURE_BUNDLE),
        "run-meta",
        "--out",
        str(meta_path),
        "--repo-root",
        str(local_exp_root),
        "--experiment-id",
        exp_id,
        "--side",
        "left" if side_label == "A" else "right",
        "--side-label",
        side_label,
        "--step",
        step,
        "--run-id",
        f"{side_label}-{step}",
        "--rp-kind",
        side["rpKind"],
        "--rp-binary",
        "bin/rpki" if side["rpKind"] == "ours" else "bin/rpki-client",
        "--rp-version",
        "portable-9.8" if side["rpKind"] == "rpki-client" else "ours-dev",
        "--rp-git-commit",
        git_commit(REPO_ROOT) or "",
        "--rp-mode",
        side["mode"],
        "--protocol-mode",
        side["protocol"] + (f"/rsync-scope:{side['rsyncScope']}" if side.get("rsyncScope") else ""),
        "--strict-policies",
        strict_policies,
        "--rirs",
        ",".join(rirs),
        "--argv-json",
        json.dumps([]),
        "--env-json",
        json.dumps({}),
        "--cwd",
        str(local_exp_root),
        "--state-root",
        str(local_exp_root / side_label / "state"),
        "--db",
        str(local_exp_root / side_label / "state" / "ours" / "work-db"),
        "--repo-bytes-db",
        str(local_exp_root / side_label / "state" / "ours" / "repo-bytes.db"),
        "--raw-store-db",
        str(local_exp_root / side_label / "state" / "ours" / "raw-store.db"),
        "--rsync-mirror-root",
        str(local_exp_root / side_label / "state" / "ours" / "rsync-mirror"),
        "--cache-root",
        str(local_exp_root / side_label / "state" / "rpki-client" / "cache"),
        "--ccr",
        str(result_ccr),
        "--cir",
        str(result_cir),
        "--fixture-proof",
        str(fixture_proof),
        "--fixture-proof-summary-json",
        json.dumps(
            {
                "taFixturePinned": True,
                "taOnlineFetchObserved": False,
                "trustAnchorCount": len(rirs),
            }
        ),
        "--report-json",
        str(report_json),
        "--stage-timing-json",
        str(stage_timing_json),
        "--stdout-log",
        str(stdout_log),
        "--stderr-log",
        str(stderr_log),
        "--process-time",
        str(process_time),
        "--exit-code",
        str(exit_code),
        "--wall-ms",
        str(int(time_info.get("wallMs", 0))),
        "--max-rss-kb",
        str(int(time_info.get("maxRssKb", 0))),
        "--vrps",
        str(int(counts.get("vrps", 0))),
        "--vaps",
        str(int(counts.get("aspas", 0))),
        "--publication-points",
        str(int(counts.get("publicationPoints", counts.get("repositories", 0)))),
        "--warnings",
        str(int(counts.get("warnings", 0))),
        "--cir-object-count",
        str(int(counts.get("cirObjectCount", 0))),
        "--cir-reject-count",
        str(int(counts.get("cirRejectCount", 0))),
        "--cir-trust-anchor-count",
        str(int(counts.get("cirTrustAnchorCount", len(rirs)))),
        "--host",
        os.uname().nodename,
        "--platform",
        sys.platform,
    ]
    if step == "snapshot":
        insert_at = generate_meta_args.index("--state-root")
        generate_meta_args.insert(insert_at, "--reset-before-run")
    run_local(generate_meta_args)
    return meta_path


def run_experiment(
    ssh_target: str,
    local_run_root: Path,
    remote_root: Path,
    exp: dict[str, Any],
    rirs: list[str],
    dry_run: bool = False,
) -> dict[str, Any]:
    exp_id = exp["id"]
    local_exp_root = local_run_root / "experiments" / exp_id
    remote_exp_root = remote_root / "experiments" / exp_id
    ensure_dir(local_exp_root)
    if dry_run:
        return render_experiment_plan(exp, local_run_root, remote_root, ssh_target, rirs)

    fixture_proof = build_fixture_proof(local_exp_root, rirs)
    preflight = (
        "set -euo pipefail; "
        f"df -h /data / || true; "
        "systemctl disable --now rpki-client.timer >/dev/null 2>&1 || true; "
        "systemctl stop rpki-client.service >/dev/null 2>&1 || true; "
        "pkill -f '[/]rpki-client([[:space:]]|$)' >/dev/null 2>&1 || true; "
        "pkill -f '[/]routinator([[:space:]]|$)' >/dev/null 2>&1 || true; "
        "id -u _rpki-client >/dev/null 2>&1 || useradd -r -M -s /usr/sbin/nologin _rpki-client || true; "
        f"mkdir -p {shlex.quote(str(remote_root / 'bin'))} {shlex.quote(str(remote_root / 'lib'))} {shlex.quote(str(remote_root / 'fixtures' / 'tal'))} {shlex.quote(str(remote_root / 'fixtures' / 'ta'))} {shlex.quote(str(remote_exp_root))}"
    )
    ssh_script(ssh_target, preflight)
    rsync_dir_to_remote(ssh_target, REPO_ROOT / "tests" / "fixtures" / "tal", remote_root / "fixtures" / "tal")
    rsync_dir_to_remote(ssh_target, REPO_ROOT / "tests" / "fixtures" / "ta", remote_root / "fixtures" / "ta")

    if exp["left"]["rpKind"] == "rpki-client" or exp["right"]["rpKind"] == "rpki-client":
        rsync_to_remote(ssh_target, detect_libtls_path(PORTABLE_ROOT / "src" / "rpki-client"), str(remote_root / "lib" / "libtls.so.28"))
        rsync_to_remote(ssh_target, PORTABLE_ROOT / "src" / "rpki-client", str(remote_root / "bin" / "rpki-client"))
    rsync_to_remote(ssh_target, REPO_ROOT / "target" / "release" / "rpki", str(remote_root / "bin" / "rpki"))
    rsync_to_remote(ssh_target, REPO_ROOT / "target" / "release" / "triage_ccr_cir_pair", str(remote_root / "bin" / "triage_ccr_cir_pair"))
    rsync_to_remote(ssh_target, REPO_ROOT / "target" / "release" / "cir_dump_reject_list", str(remote_root / "bin" / "cir_dump_reject_list"))

    step_results: list[dict[str, Any]] = []
    for step in ("snapshot", "delta"):
        for side_label, side in (("A", exp["left"]), ("B", exp["right"])):
            run_remote_step(ssh_target, remote_root, exp_id, side_label, step, side, rirs)
            local_step_root = local_exp_root / side_label / step
            remote_step_root = remote_exp_root / side_label / step
            rsync_from_remote(ssh_target, str(remote_step_root), local_step_root)
            if side["rpKind"] == "rpki-client":
                copy_rpki_client_outputs(local_step_root)
                write_rpki_client_stage_timing(local_step_root)
            # ensure ours has a stable stage timing artifact too
            if not (local_step_root / "stage-timing.json").is_file() and (local_step_root / "report.json").is_file():
                report = load_json(local_step_root / "report.json")
                stage_timing = {
                    "tool": side["rpKind"],
                    "counts": {
                        "publicationPoints": len(report.get("publication_points", [])),
                        "vrps": len(report.get("vrps", [])),
                        "aspas": len(report.get("aspas", [])),
                    },
                }
                write_json(local_step_root / "stage-timing.json", stage_timing)

            meta_path = generate_run_meta(
                local_exp_root,
                exp_id,
                side_label,
                step,
                side,
                rirs,
                fixture_proof,
                local_step_root,
            )
            step_results.append(
                {
                    "side": side_label,
                    "step": step,
                    "runDir": str(local_step_root),
                    "meta": str(meta_path),
                }
            )

        compare_dir = local_exp_root / "compare" / step
        ensure_dir(compare_dir)
        triage_cmd = [
            str(REPO_ROOT / "target" / "release" / "triage_ccr_cir_pair"),
            "--left-ccr",
            str(local_exp_root / "A" / step / "result.ccr"),
            "--left-cir",
            str(local_exp_root / "A" / step / "result.cir"),
            "--right-ccr",
            str(local_exp_root / "B" / step / "result.ccr"),
            "--right-cir",
            str(local_exp_root / "B" / step / "result.cir"),
            "--out-dir",
            str(compare_dir),
            "--sample-limit",
            "200",
        ]
        run_local(triage_cmd, cwd=local_exp_root)

        step_summary = {
            "step": step,
            "compareDir": str(compare_dir),
            "triage": load_json(compare_dir / "triage.json"),
        }
        write_json(local_exp_root / "compare" / f"{step}.summary.json", step_summary)
        step_results.append(step_summary)

    experiment_summary = {
        "schemaVersion": 1,
        "experimentId": exp_id,
        "left": exp["left"],
        "right": exp["right"],
        "rirs": rirs,
        "steps": step_results,
    }
    write_json(local_exp_root / "experiment-summary.json", experiment_summary)
    with (local_exp_root / "experiment-summary.md").open("w", encoding="utf-8") as handle:
        handle.write(f"# {exp_id}\n\n")
        handle.write(json.dumps(experiment_summary, indent=2, ensure_ascii=False))
        handle.write("\n")

    rsync_dir_to_remote(ssh_target, local_exp_root, str(remote_exp_root))
    return experiment_summary


def is_rpki_client_experiment(exp: dict[str, Any]) -> bool:
    return exp["left"]["rpKind"] == "rpki-client" or exp["right"]["rpKind"] == "rpki-client"


def main() -> None:
    parser = argparse.ArgumentParser(description="Feature #035 experiment driver")
    parser.add_argument("--run-root", required=True)
    parser.add_argument("--remote-root", required=True)
    parser.add_argument("--ssh-target", default=os.environ.get("SSH_TARGET", "root@47.251.56.108"))
    parser.add_argument("--experiment", action="append", help="Experiment id to run; repeatable")
    parser.add_argument("--all", action="store_true", help="Run all experiments from experiments.json")
    parser.add_argument("--dry-run", action="store_true")
    parser.add_argument("--rirs", default="afrinic,apnic,arin,lacnic,ripe")
    args = parser.parse_args()

    experiments_doc = load_json(EXPERIMENTS_PATH)
    experiments = experiments_doc["experiments"]
    selected_ids = set(args.experiment or [])
    if args.all or not selected_ids:
        selected = experiments
    else:
        selected = [exp for exp in experiments if exp["id"] in selected_ids]
        if not selected:
            raise SystemExit("no experiments selected")

    rirs = [item.strip() for item in args.rirs.split(",") if item.strip()]
    if not rirs:
        raise SystemExit("--rirs must not be empty")

    run_root = Path(args.run_root).resolve()
    remote_root = Path(args.remote_root)
    ensure_dir(run_root)
    if not args.dry_run:
        build_tool_binaries()

    if args.dry_run:
        plans = [
            run_experiment(args.ssh_target, run_root, remote_root, exp, rirs, dry_run=True)
            for exp in selected
        ]
        print(json.dumps({"schemaVersion": 1, "dryRun": True, "experiments": plans}, indent=2, ensure_ascii=False))
        return

    summary = {
        "schemaVersion": 1,
        "generatedAtUtc": utc_stamp(),
        "runRoot": str(run_root),
        "remoteRoot": str(remote_root),
        "sshTarget": args.ssh_target,
        "experiments": [],
    }
    for exp in selected:
        summary["experiments"].append(
            run_experiment(args.ssh_target, run_root, remote_root, exp, rirs, dry_run=False)
        )
    write_json(run_root / "feature035-summary.json", summary)
    print(json.dumps(summary, indent=2, ensure_ascii=False))


if __name__ == "__main__":
    main()
