318 lines
11 KiB
Bash
Executable File
318 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
usage() {
|
|
cat <<'EOF'
|
|
Usage:
|
|
./scripts/periodic/run_arin_dual_rp_periodic_ccr_compare.sh \
|
|
--run-root <path> \
|
|
[--ssh-target <user@host>] \
|
|
[--remote-root <path>] \
|
|
[--rpki-client-bin <path>] \
|
|
[--round-count <n>] \
|
|
[--interval-secs <n>] \
|
|
[--start-at <RFC3339>] \
|
|
[--dry-run]
|
|
|
|
M1 behavior:
|
|
- creates the periodic run skeleton
|
|
- writes per-round scheduling metadata
|
|
- does not execute RP binaries yet
|
|
EOF
|
|
}
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
RUN_ROOT=""
|
|
SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
|
|
REMOTE_ROOT=""
|
|
RPKI_CLIENT_BIN="${RPKI_CLIENT_BIN:-/home/yuyr/dev/rpki-client-9.7/build-m5/src/rpki-client}"
|
|
ROUND_COUNT=10
|
|
INTERVAL_SECS=600
|
|
START_AT=""
|
|
DRY_RUN=0
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--run-root) RUN_ROOT="$2"; shift 2 ;;
|
|
--ssh-target) SSH_TARGET="$2"; shift 2 ;;
|
|
--remote-root) REMOTE_ROOT="$2"; shift 2 ;;
|
|
--rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
|
|
--round-count) ROUND_COUNT="$2"; shift 2 ;;
|
|
--interval-secs) INTERVAL_SECS="$2"; shift 2 ;;
|
|
--start-at) START_AT="$2"; shift 2 ;;
|
|
--dry-run) DRY_RUN=1; shift 1 ;;
|
|
-h|--help) usage; exit 0 ;;
|
|
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
[[ -n "$RUN_ROOT" ]] || { usage >&2; exit 2; }
|
|
[[ "$ROUND_COUNT" =~ ^[0-9]+$ ]] || { echo "--round-count must be an integer" >&2; exit 2; }
|
|
[[ "$INTERVAL_SECS" =~ ^[0-9]+$ ]] || { echo "--interval-secs must be an integer" >&2; exit 2; }
|
|
if [[ "$DRY_RUN" -ne 1 ]]; then
|
|
[[ -n "$REMOTE_ROOT" ]] || { echo "--remote-root is required unless --dry-run" >&2; exit 2; }
|
|
[[ -x "$RPKI_CLIENT_BIN" ]] || { echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2; exit 2; }
|
|
fi
|
|
|
|
mkdir -p "$RUN_ROOT"
|
|
|
|
python3 - <<'PY' "$RUN_ROOT" "$SSH_TARGET" "$REMOTE_ROOT" "$ROUND_COUNT" "$INTERVAL_SECS" "$START_AT" "$DRY_RUN"
|
|
import json
|
|
import sys
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
|
|
run_root = Path(sys.argv[1]).resolve()
|
|
ssh_target = sys.argv[2]
|
|
remote_root = sys.argv[3]
|
|
round_count = int(sys.argv[4])
|
|
interval_secs = int(sys.argv[5])
|
|
start_at_arg = sys.argv[6]
|
|
dry_run = bool(int(sys.argv[7]))
|
|
|
|
|
|
def parse_rfc3339_utc(value: str) -> datetime:
|
|
return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc)
|
|
|
|
|
|
def fmt(dt: datetime) -> str:
|
|
return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
|
|
|
|
base_time = (
|
|
parse_rfc3339_utc(start_at_arg)
|
|
if start_at_arg
|
|
else datetime.now(timezone.utc)
|
|
)
|
|
|
|
(run_root / "rounds").mkdir(parents=True, exist_ok=True)
|
|
(run_root / "state" / "ours" / "work-db").mkdir(parents=True, exist_ok=True)
|
|
(run_root / "state" / "ours" / "raw-store.db").mkdir(parents=True, exist_ok=True)
|
|
(run_root / "state" / "rpki-client" / "cache").mkdir(parents=True, exist_ok=True)
|
|
(run_root / "state" / "rpki-client" / "out").mkdir(parents=True, exist_ok=True)
|
|
|
|
meta = {
|
|
"version": 1,
|
|
"rir": "arin",
|
|
"roundCount": round_count,
|
|
"intervalSecs": interval_secs,
|
|
"baseScheduledAt": fmt(base_time),
|
|
"mode": "dry_run" if dry_run else "skeleton_only",
|
|
"execution": {
|
|
"mode": "remote",
|
|
"sshTarget": ssh_target,
|
|
"remoteRoot": remote_root or None,
|
|
},
|
|
"state": {
|
|
"ours": {
|
|
"workDbPath": "state/ours/work-db",
|
|
"rawStoreDbPath": "state/ours/raw-store.db",
|
|
"remoteWorkDbPath": "state/ours/work-db",
|
|
"remoteRawStoreDbPath": "state/ours/raw-store.db",
|
|
},
|
|
"rpkiClient": {
|
|
"cachePath": "state/rpki-client/cache",
|
|
"outPath": "state/rpki-client/out",
|
|
"remoteCachePath": "state/rpki-client/cache",
|
|
"remoteOutPath": "state/rpki-client/out",
|
|
},
|
|
},
|
|
}
|
|
(run_root / "meta.json").write_text(json.dumps(meta, indent=2), encoding="utf-8")
|
|
|
|
rounds = []
|
|
for idx in range(round_count):
|
|
round_id = f"round-{idx+1:03d}"
|
|
kind = "snapshot" if idx == 0 else "delta"
|
|
scheduled_at = base_time + timedelta(seconds=interval_secs * idx)
|
|
round_dir = run_root / "rounds" / round_id
|
|
for name in ("ours", "rpki-client", "compare"):
|
|
(round_dir / name).mkdir(parents=True, exist_ok=True)
|
|
|
|
# M1 only builds the schedule skeleton, so lag is defined relative to the schedule model.
|
|
started_at = scheduled_at if dry_run else None
|
|
finished_at = scheduled_at if dry_run else None
|
|
round_meta = {
|
|
"roundId": round_id,
|
|
"kind": kind,
|
|
"scheduledAt": fmt(scheduled_at),
|
|
"startedAt": fmt(started_at) if started_at else None,
|
|
"finishedAt": fmt(finished_at) if finished_at else None,
|
|
"startLagMs": 0 if dry_run else None,
|
|
"status": "dry_run" if dry_run else "pending",
|
|
"paths": {
|
|
"ours": f"rounds/{round_id}/ours",
|
|
"rpkiClient": f"rounds/{round_id}/rpki-client",
|
|
"compare": f"rounds/{round_id}/compare",
|
|
},
|
|
}
|
|
(round_dir / "round-meta.json").write_text(
|
|
json.dumps(round_meta, indent=2), encoding="utf-8"
|
|
)
|
|
rounds.append(round_meta)
|
|
|
|
final_summary = {
|
|
"version": 1,
|
|
"status": "dry_run" if dry_run else "pending",
|
|
"roundCount": round_count,
|
|
"allMatch": None,
|
|
"rounds": rounds,
|
|
}
|
|
(run_root / "final-summary.json").write_text(
|
|
json.dumps(final_summary, indent=2), encoding="utf-8"
|
|
)
|
|
PY
|
|
|
|
if [[ "$DRY_RUN" -eq 1 ]]; then
|
|
echo "$RUN_ROOT"
|
|
exit 0
|
|
fi
|
|
|
|
if [[ ! -x "$ROOT_DIR/target/release/rpki" || ! -x "$ROOT_DIR/target/release/ccr_to_compare_views" ]]; then
|
|
(
|
|
cd "$ROOT_DIR"
|
|
cargo build --release --bin rpki --bin ccr_to_compare_views
|
|
)
|
|
fi
|
|
|
|
ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT'"
|
|
rsync -a --delete \
|
|
--exclude target \
|
|
--exclude .git \
|
|
"$ROOT_DIR/" "$SSH_TARGET:$REMOTE_ROOT/repo/"
|
|
ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT/repo/target/release' '$REMOTE_ROOT/bin' '$REMOTE_ROOT/rounds' '$REMOTE_ROOT/state/ours' '$REMOTE_ROOT/state/rpki-client'"
|
|
rsync -a "$ROOT_DIR/target/release/rpki" "$SSH_TARGET:$REMOTE_ROOT/repo/target/release/"
|
|
rsync -a "$RPKI_CLIENT_BIN" "$SSH_TARGET:$REMOTE_ROOT/bin/rpki-client"
|
|
|
|
for idx in $(seq 1 "$ROUND_COUNT"); do
|
|
ROUND_ID="$(printf 'round-%03d' "$idx")"
|
|
ROUND_DIR="$RUN_ROOT/rounds/$ROUND_ID"
|
|
SCHEDULED_AT="$(python3 - <<'PY' "$ROUND_DIR/round-meta.json"
|
|
import json, sys
|
|
print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['scheduledAt'])
|
|
PY
|
|
)"
|
|
|
|
python3 - <<'PY' "$SCHEDULED_AT"
|
|
from datetime import datetime, timezone
|
|
import sys, time
|
|
scheduled = datetime.fromisoformat(sys.argv[1].replace("Z", "+00:00")).astimezone(timezone.utc)
|
|
now = datetime.now(timezone.utc)
|
|
delay = (scheduled - now).total_seconds()
|
|
if delay > 0:
|
|
time.sleep(delay)
|
|
PY
|
|
|
|
"$ROOT_DIR/scripts/periodic/run_arin_ours_round_remote.sh" \
|
|
--run-root "$RUN_ROOT" \
|
|
--round-id "$ROUND_ID" \
|
|
--kind "$(python3 - <<'PY' "$ROUND_DIR/round-meta.json"
|
|
import json, sys
|
|
print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['kind'])
|
|
PY
|
|
)" \
|
|
--ssh-target "$SSH_TARGET" \
|
|
--remote-root "$REMOTE_ROOT" \
|
|
--scheduled-at "$SCHEDULED_AT" \
|
|
--skip-sync &
|
|
OURS_PID=$!
|
|
|
|
"$ROOT_DIR/scripts/periodic/run_arin_rpki_client_round_remote.sh" \
|
|
--run-root "$RUN_ROOT" \
|
|
--round-id "$ROUND_ID" \
|
|
--kind "$(python3 - <<'PY' "$ROUND_DIR/round-meta.json"
|
|
import json, sys
|
|
print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['kind'])
|
|
PY
|
|
)" \
|
|
--ssh-target "$SSH_TARGET" \
|
|
--remote-root "$REMOTE_ROOT" \
|
|
--scheduled-at "$SCHEDULED_AT" \
|
|
--rpki-client-bin "$RPKI_CLIENT_BIN" \
|
|
--skip-sync &
|
|
CLIENT_PID=$!
|
|
|
|
set +e
|
|
wait "$OURS_PID"
|
|
OURS_STATUS=$?
|
|
wait "$CLIENT_PID"
|
|
CLIENT_STATUS=$?
|
|
set -e
|
|
|
|
if [[ "$OURS_STATUS" -eq 0 && "$CLIENT_STATUS" -eq 0 \
|
|
&& -f "$ROUND_DIR/ours/result.ccr" && -f "$ROUND_DIR/rpki-client/result.ccr" ]]; then
|
|
"$ROOT_DIR/scripts/periodic/compare_ccr_round.sh" \
|
|
--ours-ccr "$ROUND_DIR/ours/result.ccr" \
|
|
--rpki-client-ccr "$ROUND_DIR/rpki-client/result.ccr" \
|
|
--out-dir "$ROUND_DIR/compare"
|
|
fi
|
|
|
|
python3 - <<'PY' "$ROUND_DIR/round-meta.json" "$ROUND_DIR/ours/round-result.json" "$ROUND_DIR/rpki-client/round-result.json" "$ROUND_DIR/compare/compare-summary.json"
|
|
import json, sys
|
|
from datetime import datetime, timezone
|
|
round_meta_path, ours_result_path, client_result_path, compare_path = sys.argv[1:]
|
|
meta = json.load(open(round_meta_path, 'r', encoding='utf-8'))
|
|
ours = json.load(open(ours_result_path, 'r', encoding='utf-8'))
|
|
client = json.load(open(client_result_path, 'r', encoding='utf-8'))
|
|
scheduled = datetime.fromisoformat(meta['scheduledAt'].replace('Z', '+00:00')).astimezone(timezone.utc)
|
|
started_candidates = []
|
|
for item in (ours, client):
|
|
if item.get('startedAt'):
|
|
started_candidates.append(datetime.fromisoformat(item['startedAt'].replace('Z', '+00:00')).astimezone(timezone.utc))
|
|
finished_candidates = []
|
|
for item in (ours, client):
|
|
if item.get('finishedAt'):
|
|
finished_candidates.append(datetime.fromisoformat(item['finishedAt'].replace('Z', '+00:00')).astimezone(timezone.utc))
|
|
if started_candidates:
|
|
started_at = min(started_candidates)
|
|
meta['startedAt'] = started_at.strftime('%Y-%m-%dT%H:%M:%SZ')
|
|
lag_ms = int((started_at - scheduled).total_seconds() * 1000)
|
|
meta['startLagMs'] = max(lag_ms, 0)
|
|
if finished_candidates:
|
|
finished_at = max(finished_candidates)
|
|
meta['finishedAt'] = finished_at.strftime('%Y-%m-%dT%H:%M:%SZ')
|
|
meta['status'] = 'completed' if ours.get('exitCode') == 0 and client.get('exitCode') == 0 else 'failed'
|
|
meta['ours'] = {
|
|
'exitCode': ours.get('exitCode'),
|
|
'durationMs': json.load(open(ours_result_path.replace('round-result.json', 'timing.json'), 'r', encoding='utf-8')).get('durationMs'),
|
|
}
|
|
meta['rpkiClient'] = {
|
|
'exitCode': client.get('exitCode'),
|
|
'durationMs': json.load(open(client_result_path.replace('round-result.json', 'timing.json'), 'r', encoding='utf-8')).get('durationMs'),
|
|
}
|
|
if compare_path and __import__('pathlib').Path(compare_path).exists():
|
|
compare = json.load(open(compare_path, 'r', encoding='utf-8'))
|
|
meta['compare'] = {
|
|
'allMatch': compare.get('allMatch'),
|
|
'vrpMatch': compare.get('vrps', {}).get('match'),
|
|
'vapMatch': compare.get('vaps', {}).get('match'),
|
|
}
|
|
json.dump(meta, open(round_meta_path, 'w', encoding='utf-8'), indent=2)
|
|
PY
|
|
done
|
|
|
|
python3 - <<'PY' "$RUN_ROOT/final-summary.json" "$RUN_ROOT/rounds"
|
|
import json, sys
|
|
from pathlib import Path
|
|
summary_path = Path(sys.argv[1])
|
|
rounds_root = Path(sys.argv[2])
|
|
rounds = []
|
|
all_match = True
|
|
for round_dir in sorted(rounds_root.glob('round-*')):
|
|
meta = json.load(open(round_dir / 'round-meta.json', 'r', encoding='utf-8'))
|
|
rounds.append(meta)
|
|
compare = meta.get('compare')
|
|
if compare is None or compare.get('allMatch') is not True:
|
|
all_match = False
|
|
summary = {
|
|
'version': 1,
|
|
'status': 'completed',
|
|
'roundCount': len(rounds),
|
|
'allMatch': all_match,
|
|
'rounds': rounds,
|
|
}
|
|
json.dump(summary, open(summary_path, 'w', encoding='utf-8'), indent=2)
|
|
PY
|
|
|
|
echo "$RUN_ROOT"
|