20260413_2 并行化架构优化第一阶段，apnic 串行98s->并行74s，传输层任务并行，同repo内发布点串行

2026-04-15 09:53:11 +08:00
parent af1c2c7f88
commit 585c41b83b
21 changed files with 5098 additions and 73 deletions
--- a/scripts/periodic/run_apnic_ours_parallel_round_remote.sh
+++ b/scripts/periodic/run_apnic_ours_parallel_round_remote.sh
@@ -0,0 +1,156 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/periodic/run_apnic_ours_parallel_round_remote.sh \
+    --run-root <path> \
+    --round-id <round-XXX> \
+    --kind <snapshot|delta> \
+    --ssh-target <user@host> \
+    --remote-root <path> \
+    [--scheduled-at <RFC3339>] \
+    [--skip-sync]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+RUN_ROOT=""
+ROUND_ID=""
+KIND=""
+SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
+REMOTE_ROOT=""
+SCHEDULED_AT=""
+SKIP_SYNC=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --run-root) RUN_ROOT="$2"; shift 2 ;;
+    --round-id) ROUND_ID="$2"; shift 2 ;;
+    --kind) KIND="$2"; shift 2 ;;
+    --ssh-target) SSH_TARGET="$2"; shift 2 ;;
+    --remote-root) REMOTE_ROOT="$2"; shift 2 ;;
+    --scheduled-at) SCHEDULED_AT="$2"; shift 2 ;;
+    --skip-sync) SKIP_SYNC=1; shift 1 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$RUN_ROOT" && -n "$ROUND_ID" && -n "$KIND" && -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
+[[ "$KIND" == "snapshot" || "$KIND" == "delta" ]] || { echo "--kind must be snapshot or delta" >&2; exit 2; }
+
+LOCAL_OUT="$RUN_ROOT/rounds/$ROUND_ID/ours"
+REMOTE_REPO="$REMOTE_ROOT/repo"
+REMOTE_OUT="$REMOTE_ROOT/rounds/$ROUND_ID/ours"
+REMOTE_WORK_DB="$REMOTE_ROOT/state/ours/work-db"
+REMOTE_RAW_STORE="$REMOTE_ROOT/state/ours/raw-store.db"
+
+mkdir -p "$LOCAL_OUT"
+
+if [[ "$SKIP_SYNC" -eq 0 ]]; then
+  ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT'"
+  rsync -a --delete \
+    --exclude target \
+    --exclude .git \
+    "$ROOT_DIR/" "$SSH_TARGET:$REMOTE_REPO/"
+  ssh "$SSH_TARGET" "mkdir -p '$REMOTE_REPO/target/release' '$REMOTE_OUT' '$REMOTE_ROOT/state/ours'"
+  rsync -a "$ROOT_DIR/target/release/rpki" "$SSH_TARGET:$REMOTE_REPO/target/release/"
+else
+  ssh "$SSH_TARGET" "mkdir -p '$REMOTE_OUT' '$REMOTE_ROOT/state/ours'"
+fi
+
+ssh "$SSH_TARGET" \
+  REMOTE_REPO="$REMOTE_REPO" \
+  REMOTE_OUT="$REMOTE_OUT" \
+  REMOTE_WORK_DB="$REMOTE_WORK_DB" \
+  REMOTE_RAW_STORE="$REMOTE_RAW_STORE" \
+  KIND="$KIND" \
+  ROUND_ID="$ROUND_ID" \
+  SCHEDULED_AT="$SCHEDULED_AT" \
+  'bash -s' <<'EOS'
+set -euo pipefail
+
+cd "$REMOTE_REPO"
+mkdir -p "$REMOTE_OUT"
+
+if [[ "$KIND" == "snapshot" ]]; then
+  rm -rf "$REMOTE_WORK_DB" "$REMOTE_RAW_STORE"
+fi
+mkdir -p "$(dirname "$REMOTE_WORK_DB")"
+
+started_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+started_at_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+
+ccr_out="$REMOTE_OUT/result.ccr"
+cir_out="$REMOTE_OUT/result.cir"
+report_out="$REMOTE_OUT/report.json"
+run_log="$REMOTE_OUT/run.log"
+meta_out="$REMOTE_OUT/round-result.json"
+
+set +e
+env RPKI_PROGRESS_LOG=1 RPKI_PROGRESS_SLOW_SECS=0 target/release/rpki \
+  --db "$REMOTE_WORK_DB" \
+  --raw-store-db "$REMOTE_RAW_STORE" \
+  --tal-path tests/fixtures/tal/apnic-rfc7730-https.tal \
+  --ta-path tests/fixtures/ta/apnic-ta.cer \
+  --parallel-phase1 \
+  --ccr-out "$ccr_out" \
+  --report-json "$report_out" \
+  --cir-enable \
+  --cir-out "$cir_out" \
+  --cir-tal-uri "https://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer" \
+  >"$run_log" 2>&1
+exit_code=$?
+set -e
+
+finished_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+finished_at_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+
+python3 - <<'PY' "$meta_out" "$ROUND_ID" "$KIND" "$SCHEDULED_AT" "$started_at_iso" "$finished_at_iso" "$REMOTE_WORK_DB" "$REMOTE_RAW_STORE" "$exit_code" "$started_at_ms" "$finished_at_ms"
+import json, sys
+(
+    path,
+    round_id,
+    kind,
+    scheduled_at,
+    started_at,
+    finished_at,
+    work_db,
+    raw_store,
+    exit_code,
+    start_ms,
+    end_ms,
+) = sys.argv[1:]
+with open(path, "w", encoding="utf-8") as fh:
+    json.dump(
+        {
+            "roundId": round_id,
+            "kind": kind,
+            "scheduledAt": scheduled_at or None,
+            "startedAt": started_at,
+            "finishedAt": finished_at,
+            "durationMs": int(end_ms) - int(start_ms),
+            "remoteWorkDbPath": work_db,
+            "remoteRawStoreDbPath": raw_store,
+            "exitCode": int(exit_code),
+        },
+        fh,
+        indent=2,
+    )
+PY
+
+exit "$exit_code"
+EOS
+
+rsync -a "$SSH_TARGET:$REMOTE_OUT/" "$LOCAL_OUT/"
+echo "$LOCAL_OUT"
--- a/scripts/periodic/run_apnic_parallel_dual_rp_periodic_ccr_compare.sh
+++ b/scripts/periodic/run_apnic_parallel_dual_rp_periodic_ccr_compare.sh
@@ -0,0 +1,280 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/periodic/run_apnic_parallel_dual_rp_periodic_ccr_compare.sh \
+    --run-root <path> \
+    [--ssh-target <user@host>] \
+    [--remote-root <path>] \
+    [--rpki-client-bin <path>] \
+    [--round-count <n>] \
+    [--interval-secs <n>] \
+    [--start-at <RFC3339>] \
+    [--dry-run]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+RUN_ROOT=""
+SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
+REMOTE_ROOT=""
+RPKI_CLIENT_BIN="${RPKI_CLIENT_BIN:-/home/yuyr/dev/rpki-client-9.7/build-m5/src/rpki-client}"
+ROUND_COUNT=10
+INTERVAL_SECS=600
+START_AT=""
+DRY_RUN=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --run-root) RUN_ROOT="$2"; shift 2 ;;
+    --ssh-target) SSH_TARGET="$2"; shift 2 ;;
+    --remote-root) REMOTE_ROOT="$2"; shift 2 ;;
+    --rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
+    --round-count) ROUND_COUNT="$2"; shift 2 ;;
+    --interval-secs) INTERVAL_SECS="$2"; shift 2 ;;
+    --start-at) START_AT="$2"; shift 2 ;;
+    --dry-run) DRY_RUN=1; shift 1 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$RUN_ROOT" ]] || { usage >&2; exit 2; }
+[[ "$ROUND_COUNT" =~ ^[0-9]+$ ]] || { echo "--round-count must be an integer" >&2; exit 2; }
+[[ "$INTERVAL_SECS" =~ ^[0-9]+$ ]] || { echo "--interval-secs must be an integer" >&2; exit 2; }
+if [[ "$DRY_RUN" -ne 1 ]]; then
+  [[ -n "$REMOTE_ROOT" ]] || { echo "--remote-root is required unless --dry-run" >&2; exit 2; }
+  [[ -x "$RPKI_CLIENT_BIN" ]] || { echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2; exit 2; }
+fi
+
+mkdir -p "$RUN_ROOT"
+
+python3 - <<'PY' "$RUN_ROOT" "$SSH_TARGET" "$REMOTE_ROOT" "$ROUND_COUNT" "$INTERVAL_SECS" "$START_AT" "$DRY_RUN"
+import json
+import sys
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+
+run_root = Path(sys.argv[1]).resolve()
+ssh_target = sys.argv[2]
+remote_root = sys.argv[3]
+round_count = int(sys.argv[4])
+interval_secs = int(sys.argv[5])
+start_at_arg = sys.argv[6]
+dry_run = bool(int(sys.argv[7]))
+
+def parse_rfc3339_utc(value: str) -> datetime:
+    return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc)
+
+def fmt(dt: datetime) -> str:
+    return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+base_time = parse_rfc3339_utc(start_at_arg) if start_at_arg else datetime.now(timezone.utc)
+
+(run_root / "rounds").mkdir(parents=True, exist_ok=True)
+(run_root / "state" / "ours" / "work-db").mkdir(parents=True, exist_ok=True)
+(run_root / "state" / "ours" / "raw-store.db").mkdir(parents=True, exist_ok=True)
+(run_root / "state" / "rpki-client" / "cache").mkdir(parents=True, exist_ok=True)
+(run_root / "state" / "rpki-client" / "out").mkdir(parents=True, exist_ok=True)
+
+meta = {
+    "version": 1,
+    "rir": "apnic",
+    "roundCount": round_count,
+    "intervalSecs": interval_secs,
+    "baseScheduledAt": fmt(base_time),
+    "mode": "dry_run" if dry_run else "remote_periodic",
+    "execution": {
+        "mode": "remote",
+        "sshTarget": ssh_target,
+        "remoteRoot": remote_root or None,
+    },
+}
+(run_root / "meta.json").write_text(json.dumps(meta, indent=2), encoding="utf-8")
+
+rounds = []
+for idx in range(round_count):
+    round_id = f"round-{idx+1:03d}"
+    kind = "snapshot" if idx == 0 else "delta"
+    scheduled_at = base_time + timedelta(seconds=interval_secs * idx)
+    round_dir = run_root / "rounds" / round_id
+    for name in ("ours", "rpki-client", "compare"):
+        (round_dir / name).mkdir(parents=True, exist_ok=True)
+    round_meta = {
+        "roundId": round_id,
+        "kind": kind,
+        "scheduledAt": fmt(scheduled_at),
+        "status": "dry_run" if dry_run else "pending",
+        "paths": {
+            "ours": f"rounds/{round_id}/ours",
+            "rpkiClient": f"rounds/{round_id}/rpki-client",
+            "compare": f"rounds/{round_id}/compare",
+        },
+    }
+    (round_dir / "round-meta.json").write_text(json.dumps(round_meta, indent=2), encoding="utf-8")
+    rounds.append(round_meta)
+
+final_summary = {
+    "version": 1,
+    "status": "dry_run" if dry_run else "pending",
+    "roundCount": round_count,
+    "allMatch": None,
+    "rounds": rounds,
+}
+(run_root / "final-summary.json").write_text(json.dumps(final_summary, indent=2), encoding="utf-8")
+PY
+
+if [[ "$DRY_RUN" -eq 1 ]]; then
+  echo "$RUN_ROOT"
+  exit 0
+fi
+
+if [[ ! -x "$ROOT_DIR/target/release/rpki" || ! -x "$ROOT_DIR/target/release/ccr_to_compare_views" ]]; then
+  (
+    cd "$ROOT_DIR"
+    cargo build --release --bin rpki --bin ccr_to_compare_views
+  )
+fi
+
+ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT'"
+rsync -a --delete \
+  --exclude target \
+  --exclude .git \
+  "$ROOT_DIR/" "$SSH_TARGET:$REMOTE_ROOT/repo/"
+ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT/repo/target/release' '$REMOTE_ROOT/bin' '$REMOTE_ROOT/rounds' '$REMOTE_ROOT/state/ours' '$REMOTE_ROOT/state/rpki-client'"
+rsync -a "$ROOT_DIR/target/release/rpki" "$SSH_TARGET:$REMOTE_ROOT/repo/target/release/"
+rsync -a "$RPKI_CLIENT_BIN" "$SSH_TARGET:$REMOTE_ROOT/bin/rpki-client"
+
+for idx in $(seq 1 "$ROUND_COUNT"); do
+  ROUND_ID="$(printf 'round-%03d' "$idx")"
+  ROUND_DIR="$RUN_ROOT/rounds/$ROUND_ID"
+  SCHEDULED_AT="$(python3 - <<'PY' "$ROUND_DIR/round-meta.json"
+import json, sys
+print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['scheduledAt'])
+PY
+)"
+  python3 - <<'PY' "$SCHEDULED_AT"
+from datetime import datetime, timezone
+import sys, time
+scheduled = datetime.fromisoformat(sys.argv[1].replace("Z", "+00:00")).astimezone(timezone.utc)
+delay = (scheduled - datetime.now(timezone.utc)).total_seconds()
+if delay > 0:
+    time.sleep(delay)
+PY
+
+  "$ROOT_DIR/scripts/periodic/run_apnic_ours_parallel_round_remote.sh" \
+    --run-root "$RUN_ROOT" \
+    --round-id "$ROUND_ID" \
+    --kind "$(python3 - <<'PY' "$ROUND_DIR/round-meta.json"
+import json, sys
+print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['kind'])
+PY
+)" \
+    --ssh-target "$SSH_TARGET" \
+    --remote-root "$REMOTE_ROOT" \
+    --scheduled-at "$SCHEDULED_AT" \
+    --skip-sync &
+  OURS_PID=$!
+
+  "$ROOT_DIR/scripts/periodic/run_apnic_rpki_client_round_remote.sh" \
+    --run-root "$RUN_ROOT" \
+    --round-id "$ROUND_ID" \
+    --kind "$(python3 - <<'PY' "$ROUND_DIR/round-meta.json"
+import json, sys
+print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['kind'])
+PY
+)" \
+    --ssh-target "$SSH_TARGET" \
+    --remote-root "$REMOTE_ROOT" \
+    --scheduled-at "$SCHEDULED_AT" \
+    --rpki-client-bin "$RPKI_CLIENT_BIN" \
+    --skip-sync &
+  CLIENT_PID=$!
+
+  set +e
+  wait "$OURS_PID"; OURS_STATUS=$?
+  wait "$CLIENT_PID"; CLIENT_STATUS=$?
+  set -e
+
+  rsync -az "$SSH_TARGET:$REMOTE_ROOT/rounds/$ROUND_ID/ours/" "$ROUND_DIR/ours/"
+  rsync -az "$SSH_TARGET:$REMOTE_ROOT/rounds/$ROUND_ID/rpki-client/" "$ROUND_DIR/rpki-client/"
+
+  if [[ "$OURS_STATUS" -eq 0 && "$CLIENT_STATUS" -eq 0 \
+        && -f "$ROUND_DIR/ours/result.ccr" && -f "$ROUND_DIR/rpki-client/result.ccr" ]]; then
+    "$ROOT_DIR/scripts/periodic/compare_ccr_round.sh" \
+      --ours-ccr "$ROUND_DIR/ours/result.ccr" \
+      --rpki-client-ccr "$ROUND_DIR/rpki-client/result.ccr" \
+      --out-dir "$ROUND_DIR/compare" \
+      --trust-anchor apnic >/dev/null
+  fi
+
+  python3 - <<'PY' "$ROUND_DIR/round-meta.json" "$ROUND_DIR/ours/round-result.json" "$ROUND_DIR/rpki-client/round-result.json" "$ROUND_DIR/compare/compare-summary.json"
+import json, sys
+from datetime import datetime, timezone
+from pathlib import Path
+meta_path, ours_path, client_path, compare_path = sys.argv[1:]
+meta = json.load(open(meta_path, 'r', encoding='utf-8'))
+ours = json.load(open(ours_path, 'r', encoding='utf-8'))
+client = json.load(open(client_path, 'r', encoding='utf-8'))
+scheduled = datetime.fromisoformat(meta['scheduledAt'].replace('Z', '+00:00')).astimezone(timezone.utc)
+started = [
+    datetime.fromisoformat(v.replace('Z', '+00:00')).astimezone(timezone.utc)
+    for v in [ours.get('startedAt'), client.get('startedAt')] if v
+]
+finished = [
+    datetime.fromisoformat(v.replace('Z', '+00:00')).astimezone(timezone.utc)
+    for v in [ours.get('finishedAt'), client.get('finishedAt')] if v
+]
+if started:
+    start_at = min(started)
+    meta['startedAt'] = start_at.strftime('%Y-%m-%dT%H:%M:%SZ')
+    meta['startLagMs'] = max(int((start_at - scheduled).total_seconds() * 1000), 0)
+if finished:
+    finish_at = max(finished)
+    meta['finishedAt'] = finish_at.strftime('%Y-%m-%dT%H:%M:%SZ')
+meta['status'] = 'completed' if ours.get('exitCode') == 0 and client.get('exitCode') == 0 else 'failed'
+meta['ours'] = {'exitCode': ours.get('exitCode'), 'durationMs': ours.get('durationMs')}
+meta['rpkiClient'] = {'exitCode': client.get('exitCode'), 'durationMs': client.get('durationMs')}
+if Path(compare_path).exists():
+    compare = json.load(open(compare_path, 'r', encoding='utf-8'))
+    meta['compare'] = {
+        'allMatch': compare.get('allMatch'),
+        'vrpMatch': compare.get('vrps', {}).get('match'),
+        'vapMatch': compare.get('vaps', {}).get('match'),
+        'oursVrps': compare.get('vrps', {}).get('ours'),
+        'rpkiClientVrps': compare.get('vrps', {}).get('rpkiClient'),
+        'oursVaps': compare.get('vaps', {}).get('ours'),
+        'rpkiClientVaps': compare.get('vaps', {}).get('rpkiClient'),
+    }
+json.dump(meta, open(meta_path, 'w', encoding='utf-8'), indent=2)
+PY
+
+  ssh "$SSH_TARGET" "rm -rf '$REMOTE_ROOT/rounds/$ROUND_ID'"
+done
+
+python3 - <<'PY' "$RUN_ROOT/final-summary.json" "$RUN_ROOT/rounds"
+import json, sys
+from pathlib import Path
+summary_path = Path(sys.argv[1])
+rounds_root = Path(sys.argv[2])
+rounds = []
+all_match = True
+for round_dir in sorted(rounds_root.glob('round-*')):
+    meta = json.load(open(round_dir / 'round-meta.json', 'r', encoding='utf-8'))
+    rounds.append(meta)
+    compare = meta.get('compare')
+    if compare is None or compare.get('allMatch') is not True:
+        all_match = False
+summary = {
+    'version': 1,
+    'status': 'completed',
+    'roundCount': len(rounds),
+    'allMatch': all_match,
+    'rounds': rounds,
+}
+json.dump(summary, open(summary_path, 'w', encoding='utf-8'), indent=2)
+PY
+
+echo "$RUN_ROOT"
--- a/scripts/periodic/run_apnic_rpki_client_round_remote.sh
+++ b/scripts/periodic/run_apnic_rpki_client_round_remote.sh
@@ -0,0 +1,163 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/periodic/run_apnic_rpki_client_round_remote.sh \
+    --run-root <path> \
+    --round-id <round-XXX> \
+    --kind <snapshot|delta> \
+    --ssh-target <user@host> \
+    --remote-root <path> \
+    [--scheduled-at <RFC3339>] \
+    [--rpki-client-bin <local path>] \
+    [--skip-sync]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+RUN_ROOT=""
+ROUND_ID=""
+KIND=""
+SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
+REMOTE_ROOT=""
+SCHEDULED_AT=""
+RPKI_CLIENT_BIN="${RPKI_CLIENT_BIN:-/home/yuyr/dev/rpki-client-9.7/build-m5/src/rpki-client}"
+SKIP_SYNC=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --run-root) RUN_ROOT="$2"; shift 2 ;;
+    --round-id) ROUND_ID="$2"; shift 2 ;;
+    --kind) KIND="$2"; shift 2 ;;
+    --ssh-target) SSH_TARGET="$2"; shift 2 ;;
+    --remote-root) REMOTE_ROOT="$2"; shift 2 ;;
+    --scheduled-at) SCHEDULED_AT="$2"; shift 2 ;;
+    --rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
+    --skip-sync) SKIP_SYNC=1; shift 1 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$RUN_ROOT" && -n "$ROUND_ID" && -n "$KIND" && -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
+[[ "$KIND" == "snapshot" || "$KIND" == "delta" ]] || { echo "--kind must be snapshot or delta" >&2; exit 2; }
+[[ -x "$RPKI_CLIENT_BIN" ]] || { echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2; exit 2; }
+
+LOCAL_OUT="$RUN_ROOT/rounds/$ROUND_ID/rpki-client"
+REMOTE_REPO="$REMOTE_ROOT/repo"
+REMOTE_BIN_DIR="$REMOTE_ROOT/bin"
+REMOTE_BIN="$REMOTE_BIN_DIR/rpki-client"
+REMOTE_OUT="$REMOTE_ROOT/rounds/$ROUND_ID/rpki-client"
+REMOTE_CACHE="$REMOTE_ROOT/state/rpki-client/cache"
+REMOTE_STATE_OUT="$REMOTE_ROOT/state/rpki-client/out"
+REMOTE_STATE_ROOT="$REMOTE_ROOT/state/rpki-client"
+
+mkdir -p "$LOCAL_OUT"
+
+if [[ "$SKIP_SYNC" -eq 0 ]]; then
+  ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT'"
+  rsync -a --delete \
+    --exclude target \
+    --exclude .git \
+    "$ROOT_DIR/" "$SSH_TARGET:$REMOTE_REPO/"
+  ssh "$SSH_TARGET" "mkdir -p '$REMOTE_BIN_DIR' '$REMOTE_OUT' '$REMOTE_STATE_ROOT'"
+  rsync -a "$RPKI_CLIENT_BIN" "$SSH_TARGET:$REMOTE_BIN"
+else
+  ssh "$SSH_TARGET" "mkdir -p '$REMOTE_BIN_DIR' '$REMOTE_OUT' '$REMOTE_STATE_ROOT'"
+fi
+
+ssh "$SSH_TARGET" \
+  REMOTE_ROOT="$REMOTE_ROOT" \
+  REMOTE_BIN="$REMOTE_BIN" \
+  REMOTE_OUT="$REMOTE_OUT" \
+  REMOTE_CACHE="$REMOTE_CACHE" \
+  REMOTE_STATE_OUT="$REMOTE_STATE_OUT" \
+  REMOTE_STATE_ROOT="$REMOTE_STATE_ROOT" \
+  KIND="$KIND" \
+  ROUND_ID="$ROUND_ID" \
+  SCHEDULED_AT="$SCHEDULED_AT" \
+  'bash -s' <<'EOS'
+set -euo pipefail
+
+cd "$REMOTE_ROOT"
+mkdir -p "$REMOTE_OUT"
+
+if [[ "$KIND" == "snapshot" ]]; then
+  rm -rf "$REMOTE_CACHE" "$REMOTE_STATE_OUT" "$REMOTE_STATE_ROOT/ta" "$REMOTE_STATE_ROOT/.ta"
+fi
+mkdir -p "$REMOTE_CACHE" "$REMOTE_STATE_OUT"
+chmod 0777 "$REMOTE_STATE_ROOT" "$REMOTE_CACHE" "$REMOTE_STATE_OUT"
+
+started_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+started_at_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+
+ccr_out="$REMOTE_OUT/result.ccr"
+run_log="$REMOTE_OUT/run.log"
+meta_out="$REMOTE_OUT/round-result.json"
+
+set +e
+(
+  cd "$REMOTE_STATE_ROOT"
+  "$REMOTE_BIN" -vv -t "../../repo/tests/fixtures/tal/apnic-rfc7730-https.tal" -d "cache" "out"
+) >"$run_log" 2>&1
+exit_code=$?
+set -e
+
+if [[ -f "$REMOTE_STATE_OUT/rpki.ccr" ]]; then
+  cp "$REMOTE_STATE_OUT/rpki.ccr" "$ccr_out"
+fi
+if [[ -f "$REMOTE_STATE_OUT/openbgpd" ]]; then
+  cp "$REMOTE_STATE_OUT/openbgpd" "$REMOTE_OUT/openbgpd"
+fi
+
+finished_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+finished_at_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+
+python3 - <<'PY' "$meta_out" "$ROUND_ID" "$KIND" "$SCHEDULED_AT" "$started_at_iso" "$finished_at_iso" "$REMOTE_CACHE" "$REMOTE_STATE_OUT" "$exit_code" "$started_at_ms" "$finished_at_ms"
+import json, sys
+(
+    path,
+    round_id,
+    kind,
+    scheduled_at,
+    started_at,
+    finished_at,
+    cache_path,
+    out_path,
+    exit_code,
+    start_ms,
+    end_ms,
+) = sys.argv[1:]
+with open(path, "w", encoding="utf-8") as fh:
+    json.dump(
+        {
+            "roundId": round_id,
+            "kind": kind,
+            "scheduledAt": scheduled_at or None,
+            "startedAt": started_at,
+            "finishedAt": finished_at,
+            "durationMs": int(end_ms) - int(start_ms),
+            "remoteCachePath": cache_path,
+            "remoteOutPath": out_path,
+            "exitCode": int(exit_code),
+        },
+        fh,
+        indent=2,
+    )
+PY
+
+exit "$exit_code"
+EOS
+
+rsync -a "$SSH_TARGET:$REMOTE_OUT/" "$LOCAL_OUT/"
+echo "$LOCAL_OUT"