#!/usr/bin/env bash set -euo pipefail usage() { cat <<'EOF' Usage: ./scripts/periodic/run_arin_dual_rp_periodic_ccr_compare.sh \ --run-root \ [--ssh-target ] \ [--remote-root ] \ [--rpki-client-bin ] \ [--round-count ] \ [--interval-secs ] \ [--start-at ] \ [--dry-run] M1 behavior: - creates the periodic run skeleton - writes per-round scheduling metadata - does not execute RP binaries yet EOF } ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" RUN_ROOT="" SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}" REMOTE_ROOT="" RPKI_CLIENT_BIN="${RPKI_CLIENT_BIN:-/home/yuyr/dev/rpki-client-9.7/build-m5/src/rpki-client}" ROUND_COUNT=10 INTERVAL_SECS=600 START_AT="" DRY_RUN=0 while [[ $# -gt 0 ]]; do case "$1" in --run-root) RUN_ROOT="$2"; shift 2 ;; --ssh-target) SSH_TARGET="$2"; shift 2 ;; --remote-root) REMOTE_ROOT="$2"; shift 2 ;; --rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;; --round-count) ROUND_COUNT="$2"; shift 2 ;; --interval-secs) INTERVAL_SECS="$2"; shift 2 ;; --start-at) START_AT="$2"; shift 2 ;; --dry-run) DRY_RUN=1; shift 1 ;; -h|--help) usage; exit 0 ;; *) echo "unknown argument: $1" >&2; usage; exit 2 ;; esac done [[ -n "$RUN_ROOT" ]] || { usage >&2; exit 2; } [[ "$ROUND_COUNT" =~ ^[0-9]+$ ]] || { echo "--round-count must be an integer" >&2; exit 2; } [[ "$INTERVAL_SECS" =~ ^[0-9]+$ ]] || { echo "--interval-secs must be an integer" >&2; exit 2; } if [[ "$DRY_RUN" -ne 1 ]]; then [[ -n "$REMOTE_ROOT" ]] || { echo "--remote-root is required unless --dry-run" >&2; exit 2; } [[ -x "$RPKI_CLIENT_BIN" ]] || { echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2; exit 2; } fi mkdir -p "$RUN_ROOT" python3 - <<'PY' "$RUN_ROOT" "$SSH_TARGET" "$REMOTE_ROOT" "$ROUND_COUNT" "$INTERVAL_SECS" "$START_AT" "$DRY_RUN" import json import sys from datetime import datetime, timedelta, timezone from pathlib import Path run_root = Path(sys.argv[1]).resolve() ssh_target = sys.argv[2] remote_root = sys.argv[3] round_count = int(sys.argv[4]) interval_secs = int(sys.argv[5]) start_at_arg = sys.argv[6] dry_run = bool(int(sys.argv[7])) def parse_rfc3339_utc(value: str) -> datetime: return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc) def fmt(dt: datetime) -> str: return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") base_time = ( parse_rfc3339_utc(start_at_arg) if start_at_arg else datetime.now(timezone.utc) ) (run_root / "rounds").mkdir(parents=True, exist_ok=True) (run_root / "state" / "ours" / "work-db").mkdir(parents=True, exist_ok=True) (run_root / "state" / "ours" / "raw-store.db").mkdir(parents=True, exist_ok=True) (run_root / "state" / "rpki-client" / "cache").mkdir(parents=True, exist_ok=True) (run_root / "state" / "rpki-client" / "out").mkdir(parents=True, exist_ok=True) meta = { "version": 1, "rir": "arin", "roundCount": round_count, "intervalSecs": interval_secs, "baseScheduledAt": fmt(base_time), "mode": "dry_run" if dry_run else "skeleton_only", "execution": { "mode": "remote", "sshTarget": ssh_target, "remoteRoot": remote_root or None, }, "state": { "ours": { "workDbPath": "state/ours/work-db", "rawStoreDbPath": "state/ours/raw-store.db", "remoteWorkDbPath": "state/ours/work-db", "remoteRawStoreDbPath": "state/ours/raw-store.db", }, "rpkiClient": { "cachePath": "state/rpki-client/cache", "outPath": "state/rpki-client/out", "remoteCachePath": "state/rpki-client/cache", "remoteOutPath": "state/rpki-client/out", }, }, } (run_root / "meta.json").write_text(json.dumps(meta, indent=2), encoding="utf-8") rounds = [] for idx in range(round_count): round_id = f"round-{idx+1:03d}" kind = "snapshot" if idx == 0 else "delta" scheduled_at = base_time + timedelta(seconds=interval_secs * idx) round_dir = run_root / "rounds" / round_id for name in ("ours", "rpki-client", "compare"): (round_dir / name).mkdir(parents=True, exist_ok=True) # M1 only builds the schedule skeleton, so lag is defined relative to the schedule model. started_at = scheduled_at if dry_run else None finished_at = scheduled_at if dry_run else None round_meta = { "roundId": round_id, "kind": kind, "scheduledAt": fmt(scheduled_at), "startedAt": fmt(started_at) if started_at else None, "finishedAt": fmt(finished_at) if finished_at else None, "startLagMs": 0 if dry_run else None, "status": "dry_run" if dry_run else "pending", "paths": { "ours": f"rounds/{round_id}/ours", "rpkiClient": f"rounds/{round_id}/rpki-client", "compare": f"rounds/{round_id}/compare", }, } (round_dir / "round-meta.json").write_text( json.dumps(round_meta, indent=2), encoding="utf-8" ) rounds.append(round_meta) final_summary = { "version": 1, "status": "dry_run" if dry_run else "pending", "roundCount": round_count, "allMatch": None, "rounds": rounds, } (run_root / "final-summary.json").write_text( json.dumps(final_summary, indent=2), encoding="utf-8" ) PY if [[ "$DRY_RUN" -eq 1 ]]; then echo "$RUN_ROOT" exit 0 fi if [[ ! -x "$ROOT_DIR/target/release/rpki" || ! -x "$ROOT_DIR/target/release/ccr_to_compare_views" ]]; then ( cd "$ROOT_DIR" cargo build --release --bin rpki --bin ccr_to_compare_views ) fi ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT'" rsync -a --delete \ --exclude target \ --exclude .git \ "$ROOT_DIR/" "$SSH_TARGET:$REMOTE_ROOT/repo/" ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT/repo/target/release' '$REMOTE_ROOT/bin' '$REMOTE_ROOT/rounds' '$REMOTE_ROOT/state/ours' '$REMOTE_ROOT/state/rpki-client'" rsync -a "$ROOT_DIR/target/release/rpki" "$SSH_TARGET:$REMOTE_ROOT/repo/target/release/" rsync -a "$RPKI_CLIENT_BIN" "$SSH_TARGET:$REMOTE_ROOT/bin/rpki-client" for idx in $(seq 1 "$ROUND_COUNT"); do ROUND_ID="$(printf 'round-%03d' "$idx")" ROUND_DIR="$RUN_ROOT/rounds/$ROUND_ID" SCHEDULED_AT="$(python3 - <<'PY' "$ROUND_DIR/round-meta.json" import json, sys print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['scheduledAt']) PY )" python3 - <<'PY' "$SCHEDULED_AT" from datetime import datetime, timezone import sys, time scheduled = datetime.fromisoformat(sys.argv[1].replace("Z", "+00:00")).astimezone(timezone.utc) now = datetime.now(timezone.utc) delay = (scheduled - now).total_seconds() if delay > 0: time.sleep(delay) PY "$ROOT_DIR/scripts/periodic/run_arin_ours_round_remote.sh" \ --run-root "$RUN_ROOT" \ --round-id "$ROUND_ID" \ --kind "$(python3 - <<'PY' "$ROUND_DIR/round-meta.json" import json, sys print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['kind']) PY )" \ --ssh-target "$SSH_TARGET" \ --remote-root "$REMOTE_ROOT" \ --scheduled-at "$SCHEDULED_AT" \ --skip-sync & OURS_PID=$! "$ROOT_DIR/scripts/periodic/run_arin_rpki_client_round_remote.sh" \ --run-root "$RUN_ROOT" \ --round-id "$ROUND_ID" \ --kind "$(python3 - <<'PY' "$ROUND_DIR/round-meta.json" import json, sys print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['kind']) PY )" \ --ssh-target "$SSH_TARGET" \ --remote-root "$REMOTE_ROOT" \ --scheduled-at "$SCHEDULED_AT" \ --rpki-client-bin "$RPKI_CLIENT_BIN" \ --skip-sync & CLIENT_PID=$! set +e wait "$OURS_PID" OURS_STATUS=$? wait "$CLIENT_PID" CLIENT_STATUS=$? set -e if [[ "$OURS_STATUS" -eq 0 && "$CLIENT_STATUS" -eq 0 \ && -f "$ROUND_DIR/ours/result.ccr" && -f "$ROUND_DIR/rpki-client/result.ccr" ]]; then "$ROOT_DIR/scripts/periodic/compare_ccr_round.sh" \ --ours-ccr "$ROUND_DIR/ours/result.ccr" \ --rpki-client-ccr "$ROUND_DIR/rpki-client/result.ccr" \ --out-dir "$ROUND_DIR/compare" fi python3 - <<'PY' "$ROUND_DIR/round-meta.json" "$ROUND_DIR/ours/round-result.json" "$ROUND_DIR/rpki-client/round-result.json" "$ROUND_DIR/compare/compare-summary.json" import json, sys from datetime import datetime, timezone round_meta_path, ours_result_path, client_result_path, compare_path = sys.argv[1:] meta = json.load(open(round_meta_path, 'r', encoding='utf-8')) ours = json.load(open(ours_result_path, 'r', encoding='utf-8')) client = json.load(open(client_result_path, 'r', encoding='utf-8')) scheduled = datetime.fromisoformat(meta['scheduledAt'].replace('Z', '+00:00')).astimezone(timezone.utc) started_candidates = [] for item in (ours, client): if item.get('startedAt'): started_candidates.append(datetime.fromisoformat(item['startedAt'].replace('Z', '+00:00')).astimezone(timezone.utc)) finished_candidates = [] for item in (ours, client): if item.get('finishedAt'): finished_candidates.append(datetime.fromisoformat(item['finishedAt'].replace('Z', '+00:00')).astimezone(timezone.utc)) if started_candidates: started_at = min(started_candidates) meta['startedAt'] = started_at.strftime('%Y-%m-%dT%H:%M:%SZ') lag_ms = int((started_at - scheduled).total_seconds() * 1000) meta['startLagMs'] = max(lag_ms, 0) if finished_candidates: finished_at = max(finished_candidates) meta['finishedAt'] = finished_at.strftime('%Y-%m-%dT%H:%M:%SZ') meta['status'] = 'completed' if ours.get('exitCode') == 0 and client.get('exitCode') == 0 else 'failed' meta['ours'] = { 'exitCode': ours.get('exitCode'), 'durationMs': json.load(open(ours_result_path.replace('round-result.json', 'timing.json'), 'r', encoding='utf-8')).get('durationMs'), } meta['rpkiClient'] = { 'exitCode': client.get('exitCode'), 'durationMs': json.load(open(client_result_path.replace('round-result.json', 'timing.json'), 'r', encoding='utf-8')).get('durationMs'), } if compare_path and __import__('pathlib').Path(compare_path).exists(): compare = json.load(open(compare_path, 'r', encoding='utf-8')) meta['compare'] = { 'allMatch': compare.get('allMatch'), 'vrpMatch': compare.get('vrps', {}).get('match'), 'vapMatch': compare.get('vaps', {}).get('match'), } json.dump(meta, open(round_meta_path, 'w', encoding='utf-8'), indent=2) PY done python3 - <<'PY' "$RUN_ROOT/final-summary.json" "$RUN_ROOT/rounds" import json, sys from pathlib import Path summary_path = Path(sys.argv[1]) rounds_root = Path(sys.argv[2]) rounds = [] all_match = True for round_dir in sorted(rounds_root.glob('round-*')): meta = json.load(open(round_dir / 'round-meta.json', 'r', encoding='utf-8')) rounds.append(meta) compare = meta.get('compare') if compare is None or compare.get('allMatch') is not True: all_match = False summary = { 'version': 1, 'status': 'completed', 'roundCount': len(rounds), 'allMatch': all_match, 'rounds': rounds, } json.dump(summary, open(summary_path, 'w', encoding='utf-8'), indent=2) PY echo "$RUN_ROOT"