NASP/rpki
84
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260413 增加定时周期任务与rpki-client对比，发现rpki-client rsync降权问题，改到tmp目录执行，执行两轮step发现输入基本对齐

Browse Source
This commit is contained in:
yuyr
2026-04-13 16:30:36 +08:00
parent e45830d79f
commit af1c2c7f88
5 changed files with 763 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

123
scripts/periodic/compare_ccr_round.sh Executable file
View File

@@ -0,0 +1,123 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'EOF'
Usage:
./scripts/periodic/compare_ccr_round.sh \
--ours-ccr <path> \
--rpki-client-ccr <path> \
--out-dir <path> \
[--trust-anchor <name>]
EOF
}
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
OURS_CCR=""
CLIENT_CCR=""
OUT_DIR=""
TRUST_ANCHOR="unknown"
CCR_TO_COMPARE_VIEWS_BIN="$ROOT_DIR/target/release/ccr_to_compare_views"
while [[ $# -gt 0 ]]; do
case "$1" in
--ours-ccr) OURS_CCR="$2"; shift 2 ;;
--rpki-client-ccr) CLIENT_CCR="$2"; shift 2 ;;
--out-dir) OUT_DIR="$2"; shift 2 ;;
--trust-anchor) TRUST_ANCHOR="$2"; shift 2 ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
esac
done
[[ -n "$OURS_CCR" && -n "$CLIENT_CCR" && -n "$OUT_DIR" ]] || { usage >&2; exit 2; }
mkdir -p "$OUT_DIR"
if [[ ! -x "$CCR_TO_COMPARE_VIEWS_BIN" ]]; then
(
cd "$ROOT_DIR"
cargo build --release --bin ccr_to_compare_views
)
fi
OURS_VRPS="$OUT_DIR/ours-vrps.csv"
OURS_VAPS="$OUT_DIR/ours-vaps.csv"
CLIENT_VRPS="$OUT_DIR/rpki-client-vrps.csv"
CLIENT_VAPS="$OUT_DIR/rpki-client-vaps.csv"
SUMMARY_JSON="$OUT_DIR/compare-summary.json"
SUMMARY_MD="$OUT_DIR/compare-summary.md"
"$CCR_TO_COMPARE_VIEWS_BIN" \
--ccr "$OURS_CCR" \
--vrps-out "$OURS_VRPS" \
--vaps-out "$OURS_VAPS" \
--trust-anchor "$TRUST_ANCHOR"
"$CCR_TO_COMPARE_VIEWS_BIN" \
--ccr "$CLIENT_CCR" \
--vrps-out "$CLIENT_VRPS" \
--vaps-out "$CLIENT_VAPS" \
--trust-anchor "$TRUST_ANCHOR"
python3 - <<'PY' "$OURS_VRPS" "$CLIENT_VRPS" "$OURS_VAPS" "$CLIENT_VAPS" "$SUMMARY_JSON" "$SUMMARY_MD"
import csv
import json
import sys
from pathlib import Path
ours_vrps_path, client_vrps_path, ours_vaps_path, client_vaps_path, json_out, md_out = sys.argv[1:]
def rows(path):
with open(path, newline="") as f:
return list(csv.reader(f))[1:]
ours_vrps = {tuple(r) for r in rows(ours_vrps_path)}
client_vrps = {tuple(r) for r in rows(client_vrps_path)}
ours_vaps = {tuple(r) for r in rows(ours_vaps_path)}
client_vaps = {tuple(r) for r in rows(client_vaps_path)}
summary = {
"vrps": {
"ours": len(ours_vrps),
"rpkiClient": len(client_vrps),
"match": ours_vrps == client_vrps,
"onlyInOurs": sorted(ours_vrps - client_vrps)[:20],
"onlyInRpkiClient": sorted(client_vrps - ours_vrps)[:20],
},
"vaps": {
"ours": len(ours_vaps),
"rpkiClient": len(client_vaps),
"match": ours_vaps == client_vaps,
"onlyInOurs": sorted(ours_vaps - client_vaps)[:20],
"onlyInRpkiClient": sorted(client_vaps - ours_vaps)[:20],
},
}
summary["allMatch"] = summary["vrps"]["match"] and summary["vaps"]["match"]
Path(json_out).write_text(json.dumps(summary, indent=2), encoding="utf-8")
lines = [
"# Round Compare Summary",
"",
f"- `allMatch`: `{summary['allMatch']}`",
f"- `vrpMatch`: `{summary['vrps']['match']}`",
f"- `vapMatch`: `{summary['vaps']['match']}`",
f"- `ours_vrps`: `{summary['vrps']['ours']}`",
f"- `rpki_client_vrps`: `{summary['vrps']['rpkiClient']}`",
f"- `ours_vaps`: `{summary['vaps']['ours']}`",
f"- `rpki_client_vaps`: `{summary['vaps']['rpkiClient']}`",
"",
"## Sample Differences",
"",
f"- `vrps.onlyInOurs`: `{len(summary['vrps']['onlyInOurs'])}`",
f"- `vrps.onlyInRpkiClient`: `{len(summary['vrps']['onlyInRpkiClient'])}`",
f"- `vaps.onlyInOurs`: `{len(summary['vaps']['onlyInOurs'])}`",
f"- `vaps.onlyInRpkiClient`: `{len(summary['vaps']['onlyInRpkiClient'])}`",
]
Path(md_out).write_text("\n".join(lines) + "\n", encoding="utf-8")
PY
echo "$OUT_DIR"

317
scripts/periodic/run_arin_dual_rp_periodic_ccr_compare.sh Executable file
View File

@@ -0,0 +1,317 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'EOF'
Usage:
./scripts/periodic/run_arin_dual_rp_periodic_ccr_compare.sh \
--run-root <path> \
[--ssh-target <user@host>] \
[--remote-root <path>] \
[--rpki-client-bin <path>] \
[--round-count <n>] \
[--interval-secs <n>] \
[--start-at <RFC3339>] \
[--dry-run]
M1 behavior:
- creates the periodic run skeleton
- writes per-round scheduling metadata
- does not execute RP binaries yet
EOF
}
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
RUN_ROOT=""
SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
REMOTE_ROOT=""
RPKI_CLIENT_BIN="${RPKI_CLIENT_BIN:-/home/yuyr/dev/rpki-client-9.7/build-m5/src/rpki-client}"
ROUND_COUNT=10
INTERVAL_SECS=600
START_AT=""
DRY_RUN=0
while [[ $# -gt 0 ]]; do
case "$1" in
--run-root) RUN_ROOT="$2"; shift 2 ;;
--ssh-target) SSH_TARGET="$2"; shift 2 ;;
--remote-root) REMOTE_ROOT="$2"; shift 2 ;;
--rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
--round-count) ROUND_COUNT="$2"; shift 2 ;;
--interval-secs) INTERVAL_SECS="$2"; shift 2 ;;
--start-at) START_AT="$2"; shift 2 ;;
--dry-run) DRY_RUN=1; shift 1 ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
esac
done
[[ -n "$RUN_ROOT" ]] || { usage >&2; exit 2; }
[[ "$ROUND_COUNT" =~ ^[0-9]+$ ]] || { echo "--round-count must be an integer" >&2; exit 2; }
[[ "$INTERVAL_SECS" =~ ^[0-9]+$ ]] || { echo "--interval-secs must be an integer" >&2; exit 2; }
if [[ "$DRY_RUN" -ne 1 ]]; then
[[ -n "$REMOTE_ROOT" ]] || { echo "--remote-root is required unless --dry-run" >&2; exit 2; }
[[ -x "$RPKI_CLIENT_BIN" ]] || { echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2; exit 2; }
fi
mkdir -p "$RUN_ROOT"
python3 - <<'PY' "$RUN_ROOT" "$SSH_TARGET" "$REMOTE_ROOT" "$ROUND_COUNT" "$INTERVAL_SECS" "$START_AT" "$DRY_RUN"
import json
import sys
from datetime import datetime, timedelta, timezone
from pathlib import Path
run_root = Path(sys.argv[1]).resolve()
ssh_target = sys.argv[2]
remote_root = sys.argv[3]
round_count = int(sys.argv[4])
interval_secs = int(sys.argv[5])
start_at_arg = sys.argv[6]
dry_run = bool(int(sys.argv[7]))
def parse_rfc3339_utc(value: str) -> datetime:
return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc)
def fmt(dt: datetime) -> str:
return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
base_time = (
parse_rfc3339_utc(start_at_arg)
if start_at_arg
else datetime.now(timezone.utc)
)
(run_root / "rounds").mkdir(parents=True, exist_ok=True)
(run_root / "state" / "ours" / "work-db").mkdir(parents=True, exist_ok=True)
(run_root / "state" / "ours" / "raw-store.db").mkdir(parents=True, exist_ok=True)
(run_root / "state" / "rpki-client" / "cache").mkdir(parents=True, exist_ok=True)
(run_root / "state" / "rpki-client" / "out").mkdir(parents=True, exist_ok=True)
meta = {
"version": 1,
"rir": "arin",
"roundCount": round_count,
"intervalSecs": interval_secs,
"baseScheduledAt": fmt(base_time),
"mode": "dry_run" if dry_run else "skeleton_only",
"execution": {
"mode": "remote",
"sshTarget": ssh_target,
"remoteRoot": remote_root or None,
},
"state": {
"ours": {
"workDbPath": "state/ours/work-db",
"rawStoreDbPath": "state/ours/raw-store.db",
"remoteWorkDbPath": "state/ours/work-db",
"remoteRawStoreDbPath": "state/ours/raw-store.db",
},
"rpkiClient": {
"cachePath": "state/rpki-client/cache",
"outPath": "state/rpki-client/out",
"remoteCachePath": "state/rpki-client/cache",
"remoteOutPath": "state/rpki-client/out",
},
},
}
(run_root / "meta.json").write_text(json.dumps(meta, indent=2), encoding="utf-8")
rounds = []
for idx in range(round_count):
round_id = f"round-{idx+1:03d}"
kind = "snapshot" if idx == 0 else "delta"
scheduled_at = base_time + timedelta(seconds=interval_secs * idx)
round_dir = run_root / "rounds" / round_id
for name in ("ours", "rpki-client", "compare"):
(round_dir / name).mkdir(parents=True, exist_ok=True)
# M1 only builds the schedule skeleton, so lag is defined relative to the schedule model.
started_at = scheduled_at if dry_run else None
finished_at = scheduled_at if dry_run else None
round_meta = {
"roundId": round_id,
"kind": kind,
"scheduledAt": fmt(scheduled_at),
"startedAt": fmt(started_at) if started_at else None,
"finishedAt": fmt(finished_at) if finished_at else None,
"startLagMs": 0 if dry_run else None,
"status": "dry_run" if dry_run else "pending",
"paths": {
"ours": f"rounds/{round_id}/ours",
"rpkiClient": f"rounds/{round_id}/rpki-client",
"compare": f"rounds/{round_id}/compare",
},
}
(round_dir / "round-meta.json").write_text(
json.dumps(round_meta, indent=2), encoding="utf-8"
)
rounds.append(round_meta)
final_summary = {
"version": 1,
"status": "dry_run" if dry_run else "pending",
"roundCount": round_count,
"allMatch": None,
"rounds": rounds,
}
(run_root / "final-summary.json").write_text(
json.dumps(final_summary, indent=2), encoding="utf-8"
)
PY
if [[ "$DRY_RUN" -eq 1 ]]; then
echo "$RUN_ROOT"
exit 0
fi
if [[ ! -x "$ROOT_DIR/target/release/rpki" || ! -x "$ROOT_DIR/target/release/ccr_to_compare_views" ]]; then
(
cd "$ROOT_DIR"
cargo build --release --bin rpki --bin ccr_to_compare_views
)
fi
ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT'"
rsync -a --delete \
--exclude target \
--exclude .git \
"$ROOT_DIR/" "$SSH_TARGET:$REMOTE_ROOT/repo/"
ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT/repo/target/release' '$REMOTE_ROOT/bin' '$REMOTE_ROOT/rounds' '$REMOTE_ROOT/state/ours' '$REMOTE_ROOT/state/rpki-client'"
rsync -a "$ROOT_DIR/target/release/rpki" "$SSH_TARGET:$REMOTE_ROOT/repo/target/release/"
rsync -a "$RPKI_CLIENT_BIN" "$SSH_TARGET:$REMOTE_ROOT/bin/rpki-client"
for idx in $(seq 1 "$ROUND_COUNT"); do
ROUND_ID="$(printf 'round-%03d' "$idx")"
ROUND_DIR="$RUN_ROOT/rounds/$ROUND_ID"
SCHEDULED_AT="$(python3 - <<'PY' "$ROUND_DIR/round-meta.json"
import json, sys
print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['scheduledAt'])
PY
)"
python3 - <<'PY' "$SCHEDULED_AT"
from datetime import datetime, timezone
import sys, time
scheduled = datetime.fromisoformat(sys.argv[1].replace("Z", "+00:00")).astimezone(timezone.utc)
now = datetime.now(timezone.utc)
delay = (scheduled - now).total_seconds()
if delay > 0:
time.sleep(delay)
PY
"$ROOT_DIR/scripts/periodic/run_arin_ours_round_remote.sh" \
--run-root "$RUN_ROOT" \
--round-id "$ROUND_ID" \
--kind "$(python3 - <<'PY' "$ROUND_DIR/round-meta.json"
import json, sys
print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['kind'])
PY
)" \
--ssh-target "$SSH_TARGET" \
--remote-root "$REMOTE_ROOT" \
--scheduled-at "$SCHEDULED_AT" \
--skip-sync &
OURS_PID=$!
"$ROOT_DIR/scripts/periodic/run_arin_rpki_client_round_remote.sh" \
--run-root "$RUN_ROOT" \
--round-id "$ROUND_ID" \
--kind "$(python3 - <<'PY' "$ROUND_DIR/round-meta.json"
import json, sys
print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['kind'])
PY
)" \
--ssh-target "$SSH_TARGET" \
--remote-root "$REMOTE_ROOT" \
--scheduled-at "$SCHEDULED_AT" \
--rpki-client-bin "$RPKI_CLIENT_BIN" \
--skip-sync &
CLIENT_PID=$!
set +e
wait "$OURS_PID"
OURS_STATUS=$?
wait "$CLIENT_PID"
CLIENT_STATUS=$?
set -e
if [[ "$OURS_STATUS" -eq 0 && "$CLIENT_STATUS" -eq 0 \
&& -f "$ROUND_DIR/ours/result.ccr" && -f "$ROUND_DIR/rpki-client/result.ccr" ]]; then
"$ROOT_DIR/scripts/periodic/compare_ccr_round.sh" \
--ours-ccr "$ROUND_DIR/ours/result.ccr" \
--rpki-client-ccr "$ROUND_DIR/rpki-client/result.ccr" \
--out-dir "$ROUND_DIR/compare"
fi
python3 - <<'PY' "$ROUND_DIR/round-meta.json" "$ROUND_DIR/ours/round-result.json" "$ROUND_DIR/rpki-client/round-result.json" "$ROUND_DIR/compare/compare-summary.json"
import json, sys
from datetime import datetime, timezone
round_meta_path, ours_result_path, client_result_path, compare_path = sys.argv[1:]
meta = json.load(open(round_meta_path, 'r', encoding='utf-8'))
ours = json.load(open(ours_result_path, 'r', encoding='utf-8'))
client = json.load(open(client_result_path, 'r', encoding='utf-8'))
scheduled = datetime.fromisoformat(meta['scheduledAt'].replace('Z', '+00:00')).astimezone(timezone.utc)
started_candidates = []
for item in (ours, client):
if item.get('startedAt'):
started_candidates.append(datetime.fromisoformat(item['startedAt'].replace('Z', '+00:00')).astimezone(timezone.utc))
finished_candidates = []
for item in (ours, client):
if item.get('finishedAt'):
finished_candidates.append(datetime.fromisoformat(item['finishedAt'].replace('Z', '+00:00')).astimezone(timezone.utc))
if started_candidates:
started_at = min(started_candidates)
meta['startedAt'] = started_at.strftime('%Y-%m-%dT%H:%M:%SZ')
lag_ms = int((started_at - scheduled).total_seconds() * 1000)
meta['startLagMs'] = max(lag_ms, 0)
if finished_candidates:
finished_at = max(finished_candidates)
meta['finishedAt'] = finished_at.strftime('%Y-%m-%dT%H:%M:%SZ')
meta['status'] = 'completed' if ours.get('exitCode') == 0 and client.get('exitCode') == 0 else 'failed'
meta['ours'] = {
'exitCode': ours.get('exitCode'),
'durationMs': json.load(open(ours_result_path.replace('round-result.json', 'timing.json'), 'r', encoding='utf-8')).get('durationMs'),
}
meta['rpkiClient'] = {
'exitCode': client.get('exitCode'),
'durationMs': json.load(open(client_result_path.replace('round-result.json', 'timing.json'), 'r', encoding='utf-8')).get('durationMs'),
}
if compare_path and __import__('pathlib').Path(compare_path).exists():
compare = json.load(open(compare_path, 'r', encoding='utf-8'))
meta['compare'] = {
'allMatch': compare.get('allMatch'),
'vrpMatch': compare.get('vrps', {}).get('match'),
'vapMatch': compare.get('vaps', {}).get('match'),
}
json.dump(meta, open(round_meta_path, 'w', encoding='utf-8'), indent=2)
PY
done
python3 - <<'PY' "$RUN_ROOT/final-summary.json" "$RUN_ROOT/rounds"
import json, sys
from pathlib import Path
summary_path = Path(sys.argv[1])
rounds_root = Path(sys.argv[2])
rounds = []
all_match = True
for round_dir in sorted(rounds_root.glob('round-*')):
meta = json.load(open(round_dir / 'round-meta.json', 'r', encoding='utf-8'))
rounds.append(meta)
compare = meta.get('compare')
if compare is None or compare.get('allMatch') is not True:
all_match = False
summary = {
'version': 1,
'status': 'completed',
'roundCount': len(rounds),
'allMatch': all_match,
'rounds': rounds,
}
json.dump(summary, open(summary_path, 'w', encoding='utf-8'), indent=2)
PY
echo "$RUN_ROOT"

155
scripts/periodic/run_arin_ours_round_remote.sh Executable file
View File

@@ -0,0 +1,155 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'EOF'
Usage:
./scripts/periodic/run_arin_ours_round_remote.sh \
--run-root <path> \
--round-id <round-XXX> \
--kind <snapshot|delta> \
--ssh-target <user@host> \
--remote-root <path> \
[--scheduled-at <RFC3339>] \
[--skip-sync]
EOF
}
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
RUN_ROOT=""
ROUND_ID=""
KIND=""
SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
REMOTE_ROOT=""
SCHEDULED_AT=""
SKIP_SYNC=0
while [[ $# -gt 0 ]]; do
case "$1" in
--run-root) RUN_ROOT="$2"; shift 2 ;;
--round-id) ROUND_ID="$2"; shift 2 ;;
--kind) KIND="$2"; shift 2 ;;
--ssh-target) SSH_TARGET="$2"; shift 2 ;;
--remote-root) REMOTE_ROOT="$2"; shift 2 ;;
--scheduled-at) SCHEDULED_AT="$2"; shift 2 ;;
--skip-sync) SKIP_SYNC=1; shift 1 ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
esac
done
[[ -n "$RUN_ROOT" && -n "$ROUND_ID" && -n "$KIND" && -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
[[ "$KIND" == "snapshot" || "$KIND" == "delta" ]] || { echo "--kind must be snapshot or delta" >&2; exit 2; }
LOCAL_OUT="$RUN_ROOT/rounds/$ROUND_ID/ours"
REMOTE_REPO="$REMOTE_ROOT/repo"
REMOTE_OUT="$REMOTE_ROOT/rounds/$ROUND_ID/ours"
REMOTE_WORK_DB="$REMOTE_ROOT/state/ours/work-db"
REMOTE_RAW_STORE="$REMOTE_ROOT/state/ours/raw-store.db"
mkdir -p "$LOCAL_OUT"
if [[ "$SKIP_SYNC" -eq 0 ]]; then
ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT'"
rsync -a --delete \
--exclude target \
--exclude .git \
"$ROOT_DIR/" "$SSH_TARGET:$REMOTE_REPO/"
ssh "$SSH_TARGET" "mkdir -p '$REMOTE_REPO/target/release' '$REMOTE_OUT' '$REMOTE_ROOT/state/ours'"
rsync -a "$ROOT_DIR/target/release/rpki" "$SSH_TARGET:$REMOTE_REPO/target/release/"
else
ssh "$SSH_TARGET" "mkdir -p '$REMOTE_OUT' '$REMOTE_ROOT/state/ours'"
fi
ssh "$SSH_TARGET" \
REMOTE_REPO="$REMOTE_REPO" \
REMOTE_OUT="$REMOTE_OUT" \
REMOTE_WORK_DB="$REMOTE_WORK_DB" \
REMOTE_RAW_STORE="$REMOTE_RAW_STORE" \
KIND="$KIND" \
ROUND_ID="$ROUND_ID" \
SCHEDULED_AT="$SCHEDULED_AT" \
'bash -s' <<'EOS'
set -euo pipefail
cd "$REMOTE_REPO"
mkdir -p "$REMOTE_OUT"
if [[ "$KIND" == "snapshot" ]]; then
rm -rf "$REMOTE_WORK_DB" "$REMOTE_RAW_STORE"
fi
mkdir -p "$(dirname "$REMOTE_WORK_DB")"
started_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
started_at_ms="$(python3 - <<'PY'
import time
print(int(time.time() * 1000))
PY
)"
ccr_out="$REMOTE_OUT/result.ccr"
report_out="$REMOTE_OUT/report.json"
run_log="$REMOTE_OUT/run.log"
timing_out="$REMOTE_OUT/timing.json"
meta_out="$REMOTE_OUT/round-result.json"
set +e
env RPKI_PROGRESS_LOG=1 target/release/rpki \
--db "$REMOTE_WORK_DB" \
--raw-store-db "$REMOTE_RAW_STORE" \
--tal-path tests/fixtures/tal/arin.tal \
--ta-path tests/fixtures/ta/arin-ta.cer \
--ccr-out "$ccr_out" \
--report-json "$report_out" \
>"$run_log" 2>&1
exit_code=$?
set -e
finished_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
finished_at_ms="$(python3 - <<'PY'
import time
print(int(time.time() * 1000))
PY
)"
python3 - <<'PY' "$timing_out" "$started_at_ms" "$finished_at_ms" "$started_at_iso" "$finished_at_iso" "$exit_code"
import json, sys
path, start_ms, end_ms, started_at, finished_at, exit_code = sys.argv[1:]
with open(path, "w", encoding="utf-8") as fh:
json.dump(
{
"durationMs": int(end_ms) - int(start_ms),
"startedAt": started_at,
"finishedAt": finished_at,
"exitCode": int(exit_code),
},
fh,
indent=2,
)
PY
python3 - <<'PY' "$meta_out" "$ROUND_ID" "$KIND" "$SCHEDULED_AT" "$started_at_iso" "$finished_at_iso" "$REMOTE_WORK_DB" "$REMOTE_RAW_STORE" "$exit_code"
import json, sys
path, round_id, kind, scheduled_at, started_at, finished_at, work_db, raw_store, exit_code = sys.argv[1:]
with open(path, "w", encoding="utf-8") as fh:
json.dump(
{
"roundId": round_id,
"kind": kind,
"scheduledAt": scheduled_at or None,
"startedAt": started_at,
"finishedAt": finished_at,
"remoteWorkDbPath": work_db,
"remoteRawStoreDbPath": raw_store,
"exitCode": int(exit_code),
},
fh,
indent=2,
)
PY
exit "$exit_code"
EOS
rsync -a "$SSH_TARGET:$REMOTE_OUT/" "$LOCAL_OUT/"
echo "$LOCAL_OUT"

165
scripts/periodic/run_arin_rpki_client_round_remote.sh Executable file
View File

@@ -0,0 +1,165 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'EOF'
Usage:
./scripts/periodic/run_arin_rpki_client_round_remote.sh \
--run-root <path> \
--round-id <round-XXX> \
--kind <snapshot|delta> \
--ssh-target <user@host> \
--remote-root <path> \
[--scheduled-at <RFC3339>] \
[--rpki-client-bin <local path>] \
[--skip-sync]
EOF
}
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
RUN_ROOT=""
ROUND_ID=""
KIND=""
SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
REMOTE_ROOT=""
SCHEDULED_AT=""
RPKI_CLIENT_BIN="${RPKI_CLIENT_BIN:-/home/yuyr/dev/rpki-client-9.7/build-m5/src/rpki-client}"
SKIP_SYNC=0
while [[ $# -gt 0 ]]; do
case "$1" in
--run-root) RUN_ROOT="$2"; shift 2 ;;
--round-id) ROUND_ID="$2"; shift 2 ;;
--kind) KIND="$2"; shift 2 ;;
--ssh-target) SSH_TARGET="$2"; shift 2 ;;
--remote-root) REMOTE_ROOT="$2"; shift 2 ;;
--scheduled-at) SCHEDULED_AT="$2"; shift 2 ;;
--rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
--skip-sync) SKIP_SYNC=1; shift 1 ;;
-h|--help) usage; exit 0 ;;
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
esac
done
[[ -n "$RUN_ROOT" && -n "$ROUND_ID" && -n "$KIND" && -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
[[ "$KIND" == "snapshot" || "$KIND" == "delta" ]] || { echo "--kind must be snapshot or delta" >&2; exit 2; }
[[ -x "$RPKI_CLIENT_BIN" ]] || { echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2; exit 2; }
LOCAL_OUT="$RUN_ROOT/rounds/$ROUND_ID/rpki-client"
REMOTE_REPO="$REMOTE_ROOT/repo"
REMOTE_BIN_DIR="$REMOTE_ROOT/bin"
REMOTE_BIN="$REMOTE_BIN_DIR/rpki-client"
REMOTE_OUT="$REMOTE_ROOT/rounds/$ROUND_ID/rpki-client"
REMOTE_CACHE="$REMOTE_ROOT/state/rpki-client/cache"
REMOTE_STATE_OUT="$REMOTE_ROOT/state/rpki-client/out"
REMOTE_STATE_ROOT="$REMOTE_ROOT/state/rpki-client"
mkdir -p "$LOCAL_OUT"
if [[ "$SKIP_SYNC" -eq 0 ]]; then
ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT'"
rsync -a --delete \
--exclude target \
--exclude .git \
"$ROOT_DIR/" "$SSH_TARGET:$REMOTE_REPO/"
ssh "$SSH_TARGET" "mkdir -p '$REMOTE_BIN_DIR' '$REMOTE_OUT' '$REMOTE_STATE_ROOT'"
rsync -a "$RPKI_CLIENT_BIN" "$SSH_TARGET:$REMOTE_BIN"
else
ssh "$SSH_TARGET" "mkdir -p '$REMOTE_BIN_DIR' '$REMOTE_OUT' '$REMOTE_STATE_ROOT'"
fi
ssh "$SSH_TARGET" \
REMOTE_ROOT="$REMOTE_ROOT" \
REMOTE_REPO="$REMOTE_REPO" \
REMOTE_BIN="$REMOTE_BIN" \
REMOTE_OUT="$REMOTE_OUT" \
REMOTE_CACHE="$REMOTE_CACHE" \
REMOTE_STATE_OUT="$REMOTE_STATE_OUT" \
REMOTE_STATE_ROOT="$REMOTE_STATE_ROOT" \
KIND="$KIND" \
ROUND_ID="$ROUND_ID" \
SCHEDULED_AT="$SCHEDULED_AT" \
'bash -s' <<'EOS'
set -euo pipefail
cd "$REMOTE_ROOT"
mkdir -p "$REMOTE_OUT"
if [[ "$KIND" == "snapshot" ]]; then
rm -rf "$REMOTE_CACHE" "$REMOTE_STATE_OUT" "$REMOTE_STATE_ROOT/ta" "$REMOTE_STATE_ROOT/.ta"
fi
mkdir -p "$REMOTE_CACHE" "$REMOTE_STATE_OUT"
chmod 0777 "$REMOTE_STATE_ROOT" "$REMOTE_CACHE" "$REMOTE_STATE_OUT"
started_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
started_at_ms="$(python3 - <<'PY'
import time
print(int(time.time() * 1000))
PY
)"
ccr_out="$REMOTE_OUT/result.ccr"
run_log="$REMOTE_OUT/run.log"
timing_out="$REMOTE_OUT/timing.json"
meta_out="$REMOTE_OUT/round-result.json"
set +e
(
cd "$REMOTE_STATE_ROOT"
"$REMOTE_BIN" -vv -t "../../repo/tests/fixtures/tal/arin.tal" -d "cache" "out"
) >"$run_log" 2>&1
exit_code=$?
set -e
if [[ -f "$REMOTE_STATE_OUT/rpki.ccr" ]]; then
cp "$REMOTE_STATE_OUT/rpki.ccr" "$ccr_out"
fi
finished_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
finished_at_ms="$(python3 - <<'PY'
import time
print(int(time.time() * 1000))
PY
)"
python3 - <<'PY' "$timing_out" "$started_at_ms" "$finished_at_ms" "$started_at_iso" "$finished_at_iso" "$exit_code"
import json, sys
path, start_ms, end_ms, started_at, finished_at, exit_code = sys.argv[1:]
with open(path, "w", encoding="utf-8") as fh:
json.dump(
{
"durationMs": int(end_ms) - int(start_ms),
"startedAt": started_at,
"finishedAt": finished_at,
"exitCode": int(exit_code),
},
fh,
indent=2,
)
PY
python3 - <<'PY' "$meta_out" "$ROUND_ID" "$KIND" "$SCHEDULED_AT" "$started_at_iso" "$finished_at_iso" "$REMOTE_CACHE" "$REMOTE_STATE_OUT" "$exit_code"
import json, sys
path, round_id, kind, scheduled_at, started_at, finished_at, cache_path, out_path, exit_code = sys.argv[1:]
with open(path, "w", encoding="utf-8") as fh:
json.dump(
{
"roundId": round_id,
"kind": kind,
"scheduledAt": scheduled_at or None,
"startedAt": started_at,
"finishedAt": finished_at,
"remoteCachePath": cache_path,
"remoteOutPath": out_path,
"exitCode": int(exit_code),
},
fh,
indent=2,
)
PY
exit "$exit_code"
EOS
rsync -a "$SSH_TARGET:$REMOTE_OUT/" "$LOCAL_OUT/"
echo "$LOCAL_OUT"