20260413 增加定时周期任务与rpki-client对比，发现rpki-client rsync降权问题，改到tmp目录执行，执行两轮step发现输入基本对齐
This commit is contained in:
parent
e45830d79f
commit
af1c2c7f88
123
scripts/periodic/compare_ccr_round.sh
Executable file
123
scripts/periodic/compare_ccr_round.sh
Executable file
@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
./scripts/periodic/compare_ccr_round.sh \
|
||||
--ours-ccr <path> \
|
||||
--rpki-client-ccr <path> \
|
||||
--out-dir <path> \
|
||||
[--trust-anchor <name>]
|
||||
EOF
|
||||
}
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
OURS_CCR=""
|
||||
CLIENT_CCR=""
|
||||
OUT_DIR=""
|
||||
TRUST_ANCHOR="unknown"
|
||||
CCR_TO_COMPARE_VIEWS_BIN="$ROOT_DIR/target/release/ccr_to_compare_views"
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--ours-ccr) OURS_CCR="$2"; shift 2 ;;
|
||||
--rpki-client-ccr) CLIENT_CCR="$2"; shift 2 ;;
|
||||
--out-dir) OUT_DIR="$2"; shift 2 ;;
|
||||
--trust-anchor) TRUST_ANCHOR="$2"; shift 2 ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -n "$OURS_CCR" && -n "$CLIENT_CCR" && -n "$OUT_DIR" ]] || { usage >&2; exit 2; }
|
||||
|
||||
mkdir -p "$OUT_DIR"
|
||||
|
||||
if [[ ! -x "$CCR_TO_COMPARE_VIEWS_BIN" ]]; then
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
cargo build --release --bin ccr_to_compare_views
|
||||
)
|
||||
fi
|
||||
|
||||
OURS_VRPS="$OUT_DIR/ours-vrps.csv"
|
||||
OURS_VAPS="$OUT_DIR/ours-vaps.csv"
|
||||
CLIENT_VRPS="$OUT_DIR/rpki-client-vrps.csv"
|
||||
CLIENT_VAPS="$OUT_DIR/rpki-client-vaps.csv"
|
||||
SUMMARY_JSON="$OUT_DIR/compare-summary.json"
|
||||
SUMMARY_MD="$OUT_DIR/compare-summary.md"
|
||||
|
||||
"$CCR_TO_COMPARE_VIEWS_BIN" \
|
||||
--ccr "$OURS_CCR" \
|
||||
--vrps-out "$OURS_VRPS" \
|
||||
--vaps-out "$OURS_VAPS" \
|
||||
--trust-anchor "$TRUST_ANCHOR"
|
||||
|
||||
"$CCR_TO_COMPARE_VIEWS_BIN" \
|
||||
--ccr "$CLIENT_CCR" \
|
||||
--vrps-out "$CLIENT_VRPS" \
|
||||
--vaps-out "$CLIENT_VAPS" \
|
||||
--trust-anchor "$TRUST_ANCHOR"
|
||||
|
||||
python3 - <<'PY' "$OURS_VRPS" "$CLIENT_VRPS" "$OURS_VAPS" "$CLIENT_VAPS" "$SUMMARY_JSON" "$SUMMARY_MD"
|
||||
import csv
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ours_vrps_path, client_vrps_path, ours_vaps_path, client_vaps_path, json_out, md_out = sys.argv[1:]
|
||||
|
||||
|
||||
def rows(path):
|
||||
with open(path, newline="") as f:
|
||||
return list(csv.reader(f))[1:]
|
||||
|
||||
|
||||
ours_vrps = {tuple(r) for r in rows(ours_vrps_path)}
|
||||
client_vrps = {tuple(r) for r in rows(client_vrps_path)}
|
||||
ours_vaps = {tuple(r) for r in rows(ours_vaps_path)}
|
||||
client_vaps = {tuple(r) for r in rows(client_vaps_path)}
|
||||
|
||||
summary = {
|
||||
"vrps": {
|
||||
"ours": len(ours_vrps),
|
||||
"rpkiClient": len(client_vrps),
|
||||
"match": ours_vrps == client_vrps,
|
||||
"onlyInOurs": sorted(ours_vrps - client_vrps)[:20],
|
||||
"onlyInRpkiClient": sorted(client_vrps - ours_vrps)[:20],
|
||||
},
|
||||
"vaps": {
|
||||
"ours": len(ours_vaps),
|
||||
"rpkiClient": len(client_vaps),
|
||||
"match": ours_vaps == client_vaps,
|
||||
"onlyInOurs": sorted(ours_vaps - client_vaps)[:20],
|
||||
"onlyInRpkiClient": sorted(client_vaps - ours_vaps)[:20],
|
||||
},
|
||||
}
|
||||
summary["allMatch"] = summary["vrps"]["match"] and summary["vaps"]["match"]
|
||||
|
||||
Path(json_out).write_text(json.dumps(summary, indent=2), encoding="utf-8")
|
||||
|
||||
lines = [
|
||||
"# Round Compare Summary",
|
||||
"",
|
||||
f"- `allMatch`: `{summary['allMatch']}`",
|
||||
f"- `vrpMatch`: `{summary['vrps']['match']}`",
|
||||
f"- `vapMatch`: `{summary['vaps']['match']}`",
|
||||
f"- `ours_vrps`: `{summary['vrps']['ours']}`",
|
||||
f"- `rpki_client_vrps`: `{summary['vrps']['rpkiClient']}`",
|
||||
f"- `ours_vaps`: `{summary['vaps']['ours']}`",
|
||||
f"- `rpki_client_vaps`: `{summary['vaps']['rpkiClient']}`",
|
||||
"",
|
||||
"## Sample Differences",
|
||||
"",
|
||||
f"- `vrps.onlyInOurs`: `{len(summary['vrps']['onlyInOurs'])}`",
|
||||
f"- `vrps.onlyInRpkiClient`: `{len(summary['vrps']['onlyInRpkiClient'])}`",
|
||||
f"- `vaps.onlyInOurs`: `{len(summary['vaps']['onlyInOurs'])}`",
|
||||
f"- `vaps.onlyInRpkiClient`: `{len(summary['vaps']['onlyInRpkiClient'])}`",
|
||||
]
|
||||
Path(md_out).write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
PY
|
||||
|
||||
echo "$OUT_DIR"
|
||||
317
scripts/periodic/run_arin_dual_rp_periodic_ccr_compare.sh
Executable file
317
scripts/periodic/run_arin_dual_rp_periodic_ccr_compare.sh
Executable file
@ -0,0 +1,317 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
./scripts/periodic/run_arin_dual_rp_periodic_ccr_compare.sh \
|
||||
--run-root <path> \
|
||||
[--ssh-target <user@host>] \
|
||||
[--remote-root <path>] \
|
||||
[--rpki-client-bin <path>] \
|
||||
[--round-count <n>] \
|
||||
[--interval-secs <n>] \
|
||||
[--start-at <RFC3339>] \
|
||||
[--dry-run]
|
||||
|
||||
M1 behavior:
|
||||
- creates the periodic run skeleton
|
||||
- writes per-round scheduling metadata
|
||||
- does not execute RP binaries yet
|
||||
EOF
|
||||
}
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
RUN_ROOT=""
|
||||
SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
|
||||
REMOTE_ROOT=""
|
||||
RPKI_CLIENT_BIN="${RPKI_CLIENT_BIN:-/home/yuyr/dev/rpki-client-9.7/build-m5/src/rpki-client}"
|
||||
ROUND_COUNT=10
|
||||
INTERVAL_SECS=600
|
||||
START_AT=""
|
||||
DRY_RUN=0
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--run-root) RUN_ROOT="$2"; shift 2 ;;
|
||||
--ssh-target) SSH_TARGET="$2"; shift 2 ;;
|
||||
--remote-root) REMOTE_ROOT="$2"; shift 2 ;;
|
||||
--rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
|
||||
--round-count) ROUND_COUNT="$2"; shift 2 ;;
|
||||
--interval-secs) INTERVAL_SECS="$2"; shift 2 ;;
|
||||
--start-at) START_AT="$2"; shift 2 ;;
|
||||
--dry-run) DRY_RUN=1; shift 1 ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -n "$RUN_ROOT" ]] || { usage >&2; exit 2; }
|
||||
[[ "$ROUND_COUNT" =~ ^[0-9]+$ ]] || { echo "--round-count must be an integer" >&2; exit 2; }
|
||||
[[ "$INTERVAL_SECS" =~ ^[0-9]+$ ]] || { echo "--interval-secs must be an integer" >&2; exit 2; }
|
||||
if [[ "$DRY_RUN" -ne 1 ]]; then
|
||||
[[ -n "$REMOTE_ROOT" ]] || { echo "--remote-root is required unless --dry-run" >&2; exit 2; }
|
||||
[[ -x "$RPKI_CLIENT_BIN" ]] || { echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2; exit 2; }
|
||||
fi
|
||||
|
||||
mkdir -p "$RUN_ROOT"
|
||||
|
||||
python3 - <<'PY' "$RUN_ROOT" "$SSH_TARGET" "$REMOTE_ROOT" "$ROUND_COUNT" "$INTERVAL_SECS" "$START_AT" "$DRY_RUN"
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
run_root = Path(sys.argv[1]).resolve()
|
||||
ssh_target = sys.argv[2]
|
||||
remote_root = sys.argv[3]
|
||||
round_count = int(sys.argv[4])
|
||||
interval_secs = int(sys.argv[5])
|
||||
start_at_arg = sys.argv[6]
|
||||
dry_run = bool(int(sys.argv[7]))
|
||||
|
||||
|
||||
def parse_rfc3339_utc(value: str) -> datetime:
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc)
|
||||
|
||||
|
||||
def fmt(dt: datetime) -> str:
|
||||
return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
base_time = (
|
||||
parse_rfc3339_utc(start_at_arg)
|
||||
if start_at_arg
|
||||
else datetime.now(timezone.utc)
|
||||
)
|
||||
|
||||
(run_root / "rounds").mkdir(parents=True, exist_ok=True)
|
||||
(run_root / "state" / "ours" / "work-db").mkdir(parents=True, exist_ok=True)
|
||||
(run_root / "state" / "ours" / "raw-store.db").mkdir(parents=True, exist_ok=True)
|
||||
(run_root / "state" / "rpki-client" / "cache").mkdir(parents=True, exist_ok=True)
|
||||
(run_root / "state" / "rpki-client" / "out").mkdir(parents=True, exist_ok=True)
|
||||
|
||||
meta = {
|
||||
"version": 1,
|
||||
"rir": "arin",
|
||||
"roundCount": round_count,
|
||||
"intervalSecs": interval_secs,
|
||||
"baseScheduledAt": fmt(base_time),
|
||||
"mode": "dry_run" if dry_run else "skeleton_only",
|
||||
"execution": {
|
||||
"mode": "remote",
|
||||
"sshTarget": ssh_target,
|
||||
"remoteRoot": remote_root or None,
|
||||
},
|
||||
"state": {
|
||||
"ours": {
|
||||
"workDbPath": "state/ours/work-db",
|
||||
"rawStoreDbPath": "state/ours/raw-store.db",
|
||||
"remoteWorkDbPath": "state/ours/work-db",
|
||||
"remoteRawStoreDbPath": "state/ours/raw-store.db",
|
||||
},
|
||||
"rpkiClient": {
|
||||
"cachePath": "state/rpki-client/cache",
|
||||
"outPath": "state/rpki-client/out",
|
||||
"remoteCachePath": "state/rpki-client/cache",
|
||||
"remoteOutPath": "state/rpki-client/out",
|
||||
},
|
||||
},
|
||||
}
|
||||
(run_root / "meta.json").write_text(json.dumps(meta, indent=2), encoding="utf-8")
|
||||
|
||||
rounds = []
|
||||
for idx in range(round_count):
|
||||
round_id = f"round-{idx+1:03d}"
|
||||
kind = "snapshot" if idx == 0 else "delta"
|
||||
scheduled_at = base_time + timedelta(seconds=interval_secs * idx)
|
||||
round_dir = run_root / "rounds" / round_id
|
||||
for name in ("ours", "rpki-client", "compare"):
|
||||
(round_dir / name).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# M1 only builds the schedule skeleton, so lag is defined relative to the schedule model.
|
||||
started_at = scheduled_at if dry_run else None
|
||||
finished_at = scheduled_at if dry_run else None
|
||||
round_meta = {
|
||||
"roundId": round_id,
|
||||
"kind": kind,
|
||||
"scheduledAt": fmt(scheduled_at),
|
||||
"startedAt": fmt(started_at) if started_at else None,
|
||||
"finishedAt": fmt(finished_at) if finished_at else None,
|
||||
"startLagMs": 0 if dry_run else None,
|
||||
"status": "dry_run" if dry_run else "pending",
|
||||
"paths": {
|
||||
"ours": f"rounds/{round_id}/ours",
|
||||
"rpkiClient": f"rounds/{round_id}/rpki-client",
|
||||
"compare": f"rounds/{round_id}/compare",
|
||||
},
|
||||
}
|
||||
(round_dir / "round-meta.json").write_text(
|
||||
json.dumps(round_meta, indent=2), encoding="utf-8"
|
||||
)
|
||||
rounds.append(round_meta)
|
||||
|
||||
final_summary = {
|
||||
"version": 1,
|
||||
"status": "dry_run" if dry_run else "pending",
|
||||
"roundCount": round_count,
|
||||
"allMatch": None,
|
||||
"rounds": rounds,
|
||||
}
|
||||
(run_root / "final-summary.json").write_text(
|
||||
json.dumps(final_summary, indent=2), encoding="utf-8"
|
||||
)
|
||||
PY
|
||||
|
||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||
echo "$RUN_ROOT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ ! -x "$ROOT_DIR/target/release/rpki" || ! -x "$ROOT_DIR/target/release/ccr_to_compare_views" ]]; then
|
||||
(
|
||||
cd "$ROOT_DIR"
|
||||
cargo build --release --bin rpki --bin ccr_to_compare_views
|
||||
)
|
||||
fi
|
||||
|
||||
ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT'"
|
||||
rsync -a --delete \
|
||||
--exclude target \
|
||||
--exclude .git \
|
||||
"$ROOT_DIR/" "$SSH_TARGET:$REMOTE_ROOT/repo/"
|
||||
ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT/repo/target/release' '$REMOTE_ROOT/bin' '$REMOTE_ROOT/rounds' '$REMOTE_ROOT/state/ours' '$REMOTE_ROOT/state/rpki-client'"
|
||||
rsync -a "$ROOT_DIR/target/release/rpki" "$SSH_TARGET:$REMOTE_ROOT/repo/target/release/"
|
||||
rsync -a "$RPKI_CLIENT_BIN" "$SSH_TARGET:$REMOTE_ROOT/bin/rpki-client"
|
||||
|
||||
for idx in $(seq 1 "$ROUND_COUNT"); do
|
||||
ROUND_ID="$(printf 'round-%03d' "$idx")"
|
||||
ROUND_DIR="$RUN_ROOT/rounds/$ROUND_ID"
|
||||
SCHEDULED_AT="$(python3 - <<'PY' "$ROUND_DIR/round-meta.json"
|
||||
import json, sys
|
||||
print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['scheduledAt'])
|
||||
PY
|
||||
)"
|
||||
|
||||
python3 - <<'PY' "$SCHEDULED_AT"
|
||||
from datetime import datetime, timezone
|
||||
import sys, time
|
||||
scheduled = datetime.fromisoformat(sys.argv[1].replace("Z", "+00:00")).astimezone(timezone.utc)
|
||||
now = datetime.now(timezone.utc)
|
||||
delay = (scheduled - now).total_seconds()
|
||||
if delay > 0:
|
||||
time.sleep(delay)
|
||||
PY
|
||||
|
||||
"$ROOT_DIR/scripts/periodic/run_arin_ours_round_remote.sh" \
|
||||
--run-root "$RUN_ROOT" \
|
||||
--round-id "$ROUND_ID" \
|
||||
--kind "$(python3 - <<'PY' "$ROUND_DIR/round-meta.json"
|
||||
import json, sys
|
||||
print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['kind'])
|
||||
PY
|
||||
)" \
|
||||
--ssh-target "$SSH_TARGET" \
|
||||
--remote-root "$REMOTE_ROOT" \
|
||||
--scheduled-at "$SCHEDULED_AT" \
|
||||
--skip-sync &
|
||||
OURS_PID=$!
|
||||
|
||||
"$ROOT_DIR/scripts/periodic/run_arin_rpki_client_round_remote.sh" \
|
||||
--run-root "$RUN_ROOT" \
|
||||
--round-id "$ROUND_ID" \
|
||||
--kind "$(python3 - <<'PY' "$ROUND_DIR/round-meta.json"
|
||||
import json, sys
|
||||
print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['kind'])
|
||||
PY
|
||||
)" \
|
||||
--ssh-target "$SSH_TARGET" \
|
||||
--remote-root "$REMOTE_ROOT" \
|
||||
--scheduled-at "$SCHEDULED_AT" \
|
||||
--rpki-client-bin "$RPKI_CLIENT_BIN" \
|
||||
--skip-sync &
|
||||
CLIENT_PID=$!
|
||||
|
||||
set +e
|
||||
wait "$OURS_PID"
|
||||
OURS_STATUS=$?
|
||||
wait "$CLIENT_PID"
|
||||
CLIENT_STATUS=$?
|
||||
set -e
|
||||
|
||||
if [[ "$OURS_STATUS" -eq 0 && "$CLIENT_STATUS" -eq 0 \
|
||||
&& -f "$ROUND_DIR/ours/result.ccr" && -f "$ROUND_DIR/rpki-client/result.ccr" ]]; then
|
||||
"$ROOT_DIR/scripts/periodic/compare_ccr_round.sh" \
|
||||
--ours-ccr "$ROUND_DIR/ours/result.ccr" \
|
||||
--rpki-client-ccr "$ROUND_DIR/rpki-client/result.ccr" \
|
||||
--out-dir "$ROUND_DIR/compare"
|
||||
fi
|
||||
|
||||
python3 - <<'PY' "$ROUND_DIR/round-meta.json" "$ROUND_DIR/ours/round-result.json" "$ROUND_DIR/rpki-client/round-result.json" "$ROUND_DIR/compare/compare-summary.json"
|
||||
import json, sys
|
||||
from datetime import datetime, timezone
|
||||
round_meta_path, ours_result_path, client_result_path, compare_path = sys.argv[1:]
|
||||
meta = json.load(open(round_meta_path, 'r', encoding='utf-8'))
|
||||
ours = json.load(open(ours_result_path, 'r', encoding='utf-8'))
|
||||
client = json.load(open(client_result_path, 'r', encoding='utf-8'))
|
||||
scheduled = datetime.fromisoformat(meta['scheduledAt'].replace('Z', '+00:00')).astimezone(timezone.utc)
|
||||
started_candidates = []
|
||||
for item in (ours, client):
|
||||
if item.get('startedAt'):
|
||||
started_candidates.append(datetime.fromisoformat(item['startedAt'].replace('Z', '+00:00')).astimezone(timezone.utc))
|
||||
finished_candidates = []
|
||||
for item in (ours, client):
|
||||
if item.get('finishedAt'):
|
||||
finished_candidates.append(datetime.fromisoformat(item['finishedAt'].replace('Z', '+00:00')).astimezone(timezone.utc))
|
||||
if started_candidates:
|
||||
started_at = min(started_candidates)
|
||||
meta['startedAt'] = started_at.strftime('%Y-%m-%dT%H:%M:%SZ')
|
||||
lag_ms = int((started_at - scheduled).total_seconds() * 1000)
|
||||
meta['startLagMs'] = max(lag_ms, 0)
|
||||
if finished_candidates:
|
||||
finished_at = max(finished_candidates)
|
||||
meta['finishedAt'] = finished_at.strftime('%Y-%m-%dT%H:%M:%SZ')
|
||||
meta['status'] = 'completed' if ours.get('exitCode') == 0 and client.get('exitCode') == 0 else 'failed'
|
||||
meta['ours'] = {
|
||||
'exitCode': ours.get('exitCode'),
|
||||
'durationMs': json.load(open(ours_result_path.replace('round-result.json', 'timing.json'), 'r', encoding='utf-8')).get('durationMs'),
|
||||
}
|
||||
meta['rpkiClient'] = {
|
||||
'exitCode': client.get('exitCode'),
|
||||
'durationMs': json.load(open(client_result_path.replace('round-result.json', 'timing.json'), 'r', encoding='utf-8')).get('durationMs'),
|
||||
}
|
||||
if compare_path and __import__('pathlib').Path(compare_path).exists():
|
||||
compare = json.load(open(compare_path, 'r', encoding='utf-8'))
|
||||
meta['compare'] = {
|
||||
'allMatch': compare.get('allMatch'),
|
||||
'vrpMatch': compare.get('vrps', {}).get('match'),
|
||||
'vapMatch': compare.get('vaps', {}).get('match'),
|
||||
}
|
||||
json.dump(meta, open(round_meta_path, 'w', encoding='utf-8'), indent=2)
|
||||
PY
|
||||
done
|
||||
|
||||
python3 - <<'PY' "$RUN_ROOT/final-summary.json" "$RUN_ROOT/rounds"
|
||||
import json, sys
|
||||
from pathlib import Path
|
||||
summary_path = Path(sys.argv[1])
|
||||
rounds_root = Path(sys.argv[2])
|
||||
rounds = []
|
||||
all_match = True
|
||||
for round_dir in sorted(rounds_root.glob('round-*')):
|
||||
meta = json.load(open(round_dir / 'round-meta.json', 'r', encoding='utf-8'))
|
||||
rounds.append(meta)
|
||||
compare = meta.get('compare')
|
||||
if compare is None or compare.get('allMatch') is not True:
|
||||
all_match = False
|
||||
summary = {
|
||||
'version': 1,
|
||||
'status': 'completed',
|
||||
'roundCount': len(rounds),
|
||||
'allMatch': all_match,
|
||||
'rounds': rounds,
|
||||
}
|
||||
json.dump(summary, open(summary_path, 'w', encoding='utf-8'), indent=2)
|
||||
PY
|
||||
|
||||
echo "$RUN_ROOT"
|
||||
155
scripts/periodic/run_arin_ours_round_remote.sh
Executable file
155
scripts/periodic/run_arin_ours_round_remote.sh
Executable file
@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
./scripts/periodic/run_arin_ours_round_remote.sh \
|
||||
--run-root <path> \
|
||||
--round-id <round-XXX> \
|
||||
--kind <snapshot|delta> \
|
||||
--ssh-target <user@host> \
|
||||
--remote-root <path> \
|
||||
[--scheduled-at <RFC3339>] \
|
||||
[--skip-sync]
|
||||
EOF
|
||||
}
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
RUN_ROOT=""
|
||||
ROUND_ID=""
|
||||
KIND=""
|
||||
SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
|
||||
REMOTE_ROOT=""
|
||||
SCHEDULED_AT=""
|
||||
SKIP_SYNC=0
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--run-root) RUN_ROOT="$2"; shift 2 ;;
|
||||
--round-id) ROUND_ID="$2"; shift 2 ;;
|
||||
--kind) KIND="$2"; shift 2 ;;
|
||||
--ssh-target) SSH_TARGET="$2"; shift 2 ;;
|
||||
--remote-root) REMOTE_ROOT="$2"; shift 2 ;;
|
||||
--scheduled-at) SCHEDULED_AT="$2"; shift 2 ;;
|
||||
--skip-sync) SKIP_SYNC=1; shift 1 ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -n "$RUN_ROOT" && -n "$ROUND_ID" && -n "$KIND" && -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
|
||||
[[ "$KIND" == "snapshot" || "$KIND" == "delta" ]] || { echo "--kind must be snapshot or delta" >&2; exit 2; }
|
||||
|
||||
LOCAL_OUT="$RUN_ROOT/rounds/$ROUND_ID/ours"
|
||||
REMOTE_REPO="$REMOTE_ROOT/repo"
|
||||
REMOTE_OUT="$REMOTE_ROOT/rounds/$ROUND_ID/ours"
|
||||
REMOTE_WORK_DB="$REMOTE_ROOT/state/ours/work-db"
|
||||
REMOTE_RAW_STORE="$REMOTE_ROOT/state/ours/raw-store.db"
|
||||
|
||||
mkdir -p "$LOCAL_OUT"
|
||||
|
||||
if [[ "$SKIP_SYNC" -eq 0 ]]; then
|
||||
ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT'"
|
||||
rsync -a --delete \
|
||||
--exclude target \
|
||||
--exclude .git \
|
||||
"$ROOT_DIR/" "$SSH_TARGET:$REMOTE_REPO/"
|
||||
ssh "$SSH_TARGET" "mkdir -p '$REMOTE_REPO/target/release' '$REMOTE_OUT' '$REMOTE_ROOT/state/ours'"
|
||||
rsync -a "$ROOT_DIR/target/release/rpki" "$SSH_TARGET:$REMOTE_REPO/target/release/"
|
||||
else
|
||||
ssh "$SSH_TARGET" "mkdir -p '$REMOTE_OUT' '$REMOTE_ROOT/state/ours'"
|
||||
fi
|
||||
|
||||
ssh "$SSH_TARGET" \
|
||||
REMOTE_REPO="$REMOTE_REPO" \
|
||||
REMOTE_OUT="$REMOTE_OUT" \
|
||||
REMOTE_WORK_DB="$REMOTE_WORK_DB" \
|
||||
REMOTE_RAW_STORE="$REMOTE_RAW_STORE" \
|
||||
KIND="$KIND" \
|
||||
ROUND_ID="$ROUND_ID" \
|
||||
SCHEDULED_AT="$SCHEDULED_AT" \
|
||||
'bash -s' <<'EOS'
|
||||
set -euo pipefail
|
||||
|
||||
cd "$REMOTE_REPO"
|
||||
mkdir -p "$REMOTE_OUT"
|
||||
|
||||
if [[ "$KIND" == "snapshot" ]]; then
|
||||
rm -rf "$REMOTE_WORK_DB" "$REMOTE_RAW_STORE"
|
||||
fi
|
||||
mkdir -p "$(dirname "$REMOTE_WORK_DB")"
|
||||
|
||||
started_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
started_at_ms="$(python3 - <<'PY'
|
||||
import time
|
||||
print(int(time.time() * 1000))
|
||||
PY
|
||||
)"
|
||||
|
||||
ccr_out="$REMOTE_OUT/result.ccr"
|
||||
report_out="$REMOTE_OUT/report.json"
|
||||
run_log="$REMOTE_OUT/run.log"
|
||||
timing_out="$REMOTE_OUT/timing.json"
|
||||
meta_out="$REMOTE_OUT/round-result.json"
|
||||
|
||||
set +e
|
||||
env RPKI_PROGRESS_LOG=1 target/release/rpki \
|
||||
--db "$REMOTE_WORK_DB" \
|
||||
--raw-store-db "$REMOTE_RAW_STORE" \
|
||||
--tal-path tests/fixtures/tal/arin.tal \
|
||||
--ta-path tests/fixtures/ta/arin-ta.cer \
|
||||
--ccr-out "$ccr_out" \
|
||||
--report-json "$report_out" \
|
||||
>"$run_log" 2>&1
|
||||
exit_code=$?
|
||||
set -e
|
||||
|
||||
finished_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
finished_at_ms="$(python3 - <<'PY'
|
||||
import time
|
||||
print(int(time.time() * 1000))
|
||||
PY
|
||||
)"
|
||||
|
||||
python3 - <<'PY' "$timing_out" "$started_at_ms" "$finished_at_ms" "$started_at_iso" "$finished_at_iso" "$exit_code"
|
||||
import json, sys
|
||||
path, start_ms, end_ms, started_at, finished_at, exit_code = sys.argv[1:]
|
||||
with open(path, "w", encoding="utf-8") as fh:
|
||||
json.dump(
|
||||
{
|
||||
"durationMs": int(end_ms) - int(start_ms),
|
||||
"startedAt": started_at,
|
||||
"finishedAt": finished_at,
|
||||
"exitCode": int(exit_code),
|
||||
},
|
||||
fh,
|
||||
indent=2,
|
||||
)
|
||||
PY
|
||||
|
||||
python3 - <<'PY' "$meta_out" "$ROUND_ID" "$KIND" "$SCHEDULED_AT" "$started_at_iso" "$finished_at_iso" "$REMOTE_WORK_DB" "$REMOTE_RAW_STORE" "$exit_code"
|
||||
import json, sys
|
||||
path, round_id, kind, scheduled_at, started_at, finished_at, work_db, raw_store, exit_code = sys.argv[1:]
|
||||
with open(path, "w", encoding="utf-8") as fh:
|
||||
json.dump(
|
||||
{
|
||||
"roundId": round_id,
|
||||
"kind": kind,
|
||||
"scheduledAt": scheduled_at or None,
|
||||
"startedAt": started_at,
|
||||
"finishedAt": finished_at,
|
||||
"remoteWorkDbPath": work_db,
|
||||
"remoteRawStoreDbPath": raw_store,
|
||||
"exitCode": int(exit_code),
|
||||
},
|
||||
fh,
|
||||
indent=2,
|
||||
)
|
||||
PY
|
||||
|
||||
exit "$exit_code"
|
||||
EOS
|
||||
|
||||
rsync -a "$SSH_TARGET:$REMOTE_OUT/" "$LOCAL_OUT/"
|
||||
echo "$LOCAL_OUT"
|
||||
165
scripts/periodic/run_arin_rpki_client_round_remote.sh
Executable file
165
scripts/periodic/run_arin_rpki_client_round_remote.sh
Executable file
@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
cat <<'EOF'
|
||||
Usage:
|
||||
./scripts/periodic/run_arin_rpki_client_round_remote.sh \
|
||||
--run-root <path> \
|
||||
--round-id <round-XXX> \
|
||||
--kind <snapshot|delta> \
|
||||
--ssh-target <user@host> \
|
||||
--remote-root <path> \
|
||||
[--scheduled-at <RFC3339>] \
|
||||
[--rpki-client-bin <local path>] \
|
||||
[--skip-sync]
|
||||
EOF
|
||||
}
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
RUN_ROOT=""
|
||||
ROUND_ID=""
|
||||
KIND=""
|
||||
SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
|
||||
REMOTE_ROOT=""
|
||||
SCHEDULED_AT=""
|
||||
RPKI_CLIENT_BIN="${RPKI_CLIENT_BIN:-/home/yuyr/dev/rpki-client-9.7/build-m5/src/rpki-client}"
|
||||
SKIP_SYNC=0
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--run-root) RUN_ROOT="$2"; shift 2 ;;
|
||||
--round-id) ROUND_ID="$2"; shift 2 ;;
|
||||
--kind) KIND="$2"; shift 2 ;;
|
||||
--ssh-target) SSH_TARGET="$2"; shift 2 ;;
|
||||
--remote-root) REMOTE_ROOT="$2"; shift 2 ;;
|
||||
--scheduled-at) SCHEDULED_AT="$2"; shift 2 ;;
|
||||
--rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
|
||||
--skip-sync) SKIP_SYNC=1; shift 1 ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) echo "unknown argument: $1" >&2; usage; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -n "$RUN_ROOT" && -n "$ROUND_ID" && -n "$KIND" && -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
|
||||
[[ "$KIND" == "snapshot" || "$KIND" == "delta" ]] || { echo "--kind must be snapshot or delta" >&2; exit 2; }
|
||||
[[ -x "$RPKI_CLIENT_BIN" ]] || { echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2; exit 2; }
|
||||
|
||||
LOCAL_OUT="$RUN_ROOT/rounds/$ROUND_ID/rpki-client"
|
||||
REMOTE_REPO="$REMOTE_ROOT/repo"
|
||||
REMOTE_BIN_DIR="$REMOTE_ROOT/bin"
|
||||
REMOTE_BIN="$REMOTE_BIN_DIR/rpki-client"
|
||||
REMOTE_OUT="$REMOTE_ROOT/rounds/$ROUND_ID/rpki-client"
|
||||
REMOTE_CACHE="$REMOTE_ROOT/state/rpki-client/cache"
|
||||
REMOTE_STATE_OUT="$REMOTE_ROOT/state/rpki-client/out"
|
||||
REMOTE_STATE_ROOT="$REMOTE_ROOT/state/rpki-client"
|
||||
|
||||
mkdir -p "$LOCAL_OUT"
|
||||
|
||||
if [[ "$SKIP_SYNC" -eq 0 ]]; then
|
||||
ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT'"
|
||||
rsync -a --delete \
|
||||
--exclude target \
|
||||
--exclude .git \
|
||||
"$ROOT_DIR/" "$SSH_TARGET:$REMOTE_REPO/"
|
||||
ssh "$SSH_TARGET" "mkdir -p '$REMOTE_BIN_DIR' '$REMOTE_OUT' '$REMOTE_STATE_ROOT'"
|
||||
rsync -a "$RPKI_CLIENT_BIN" "$SSH_TARGET:$REMOTE_BIN"
|
||||
else
|
||||
ssh "$SSH_TARGET" "mkdir -p '$REMOTE_BIN_DIR' '$REMOTE_OUT' '$REMOTE_STATE_ROOT'"
|
||||
fi
|
||||
|
||||
ssh "$SSH_TARGET" \
|
||||
REMOTE_ROOT="$REMOTE_ROOT" \
|
||||
REMOTE_REPO="$REMOTE_REPO" \
|
||||
REMOTE_BIN="$REMOTE_BIN" \
|
||||
REMOTE_OUT="$REMOTE_OUT" \
|
||||
REMOTE_CACHE="$REMOTE_CACHE" \
|
||||
REMOTE_STATE_OUT="$REMOTE_STATE_OUT" \
|
||||
REMOTE_STATE_ROOT="$REMOTE_STATE_ROOT" \
|
||||
KIND="$KIND" \
|
||||
ROUND_ID="$ROUND_ID" \
|
||||
SCHEDULED_AT="$SCHEDULED_AT" \
|
||||
'bash -s' <<'EOS'
|
||||
set -euo pipefail
|
||||
|
||||
cd "$REMOTE_ROOT"
|
||||
mkdir -p "$REMOTE_OUT"
|
||||
|
||||
if [[ "$KIND" == "snapshot" ]]; then
|
||||
rm -rf "$REMOTE_CACHE" "$REMOTE_STATE_OUT" "$REMOTE_STATE_ROOT/ta" "$REMOTE_STATE_ROOT/.ta"
|
||||
fi
|
||||
mkdir -p "$REMOTE_CACHE" "$REMOTE_STATE_OUT"
|
||||
chmod 0777 "$REMOTE_STATE_ROOT" "$REMOTE_CACHE" "$REMOTE_STATE_OUT"
|
||||
|
||||
started_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
started_at_ms="$(python3 - <<'PY'
|
||||
import time
|
||||
print(int(time.time() * 1000))
|
||||
PY
|
||||
)"
|
||||
|
||||
ccr_out="$REMOTE_OUT/result.ccr"
|
||||
run_log="$REMOTE_OUT/run.log"
|
||||
timing_out="$REMOTE_OUT/timing.json"
|
||||
meta_out="$REMOTE_OUT/round-result.json"
|
||||
|
||||
set +e
|
||||
(
|
||||
cd "$REMOTE_STATE_ROOT"
|
||||
"$REMOTE_BIN" -vv -t "../../repo/tests/fixtures/tal/arin.tal" -d "cache" "out"
|
||||
) >"$run_log" 2>&1
|
||||
exit_code=$?
|
||||
set -e
|
||||
|
||||
if [[ -f "$REMOTE_STATE_OUT/rpki.ccr" ]]; then
|
||||
cp "$REMOTE_STATE_OUT/rpki.ccr" "$ccr_out"
|
||||
fi
|
||||
|
||||
finished_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
finished_at_ms="$(python3 - <<'PY'
|
||||
import time
|
||||
print(int(time.time() * 1000))
|
||||
PY
|
||||
)"
|
||||
|
||||
python3 - <<'PY' "$timing_out" "$started_at_ms" "$finished_at_ms" "$started_at_iso" "$finished_at_iso" "$exit_code"
|
||||
import json, sys
|
||||
path, start_ms, end_ms, started_at, finished_at, exit_code = sys.argv[1:]
|
||||
with open(path, "w", encoding="utf-8") as fh:
|
||||
json.dump(
|
||||
{
|
||||
"durationMs": int(end_ms) - int(start_ms),
|
||||
"startedAt": started_at,
|
||||
"finishedAt": finished_at,
|
||||
"exitCode": int(exit_code),
|
||||
},
|
||||
fh,
|
||||
indent=2,
|
||||
)
|
||||
PY
|
||||
|
||||
python3 - <<'PY' "$meta_out" "$ROUND_ID" "$KIND" "$SCHEDULED_AT" "$started_at_iso" "$finished_at_iso" "$REMOTE_CACHE" "$REMOTE_STATE_OUT" "$exit_code"
|
||||
import json, sys
|
||||
path, round_id, kind, scheduled_at, started_at, finished_at, cache_path, out_path, exit_code = sys.argv[1:]
|
||||
with open(path, "w", encoding="utf-8") as fh:
|
||||
json.dump(
|
||||
{
|
||||
"roundId": round_id,
|
||||
"kind": kind,
|
||||
"scheduledAt": scheduled_at or None,
|
||||
"startedAt": started_at,
|
||||
"finishedAt": finished_at,
|
||||
"remoteCachePath": cache_path,
|
||||
"remoteOutPath": out_path,
|
||||
"exitCode": int(exit_code),
|
||||
},
|
||||
fh,
|
||||
indent=2,
|
||||
)
|
||||
PY
|
||||
|
||||
exit "$exit_code"
|
||||
EOS
|
||||
|
||||
rsync -a "$SSH_TARGET:$REMOTE_OUT/" "$LOCAL_OUT/"
|
||||
echo "$LOCAL_OUT"
|
||||
@ -6,7 +6,7 @@
|
||||
**Expires: [Date, e.g., October 2026]** April 2026
|
||||
|
||||
# A Profile for Resource Public Key Infrastructure (RPKI) Canonical Input Representation (CIR)
|
||||
## draft-yirong-sidrops-rpki-cir-00
|
||||
## draft-yu-sidrops-rpki-cir-00
|
||||
|
||||
### Abstract
|
||||
|
||||
@ -14,11 +14,7 @@ This document specifies a Canonical Input Representation (CIR) content type for
|
||||
|
||||
### Status of This Memo
|
||||
|
||||
This Internet-Draft is submitted in full conformance with the provisions of BCP 78 and BCP 79.
|
||||
|
||||
Internet-Drafts are working documents of the Internet Engineering Task Force (IETF). Note that other groups may also distribute working documents as Internet-Drafts. The list of current Internet-Drafts is at [https://datatracker.ietf.org/drafts/current/](https://datatracker.ietf.org/drafts/current/).
|
||||
|
||||
Internet-Drafts are draft documents valid for a maximum of six months and may be updated, replaced, or obsoleted by other documents at any time. It is inappropriate to use Internet-Drafts as reference material or to cite them other than as "work in progress."
|
||||
TBD
|
||||
|
||||
### Table of Contents
|
||||
|
||||
@ -46,7 +42,7 @@ This document specifies a Canonical Input Representation (CIR) content type for
|
||||
|
||||
A Relying Party (RP) fetches RPKI objects from publication points using protocols such as rsync [RFC5781] or RRDP [RFC8182] prior to executing cryptographic validation. While the Canonical Cache Representation (CCR) [draft-ietf-sidrops-rpki-ccr] accurately describes the subset of objects that successfully passed validation, it inherently omits objects that were rejected due to format errors, invalid signatures, or expired timestamps (survivorship bias).
|
||||
|
||||
CIR records the precise mapping of object URIs to their cryptographic hashes *before* validation occurs. By decoupling the network transport layer from the validation layer, CIR allows researchers and operators to reconstruct the exact physical file tree (the "dirty inputs") perceived by an observation point.
|
||||
CIR records the precise mapping of object URIs to their cryptographic hashes *before* validation occurs. By decoupling the network transport layer from the validation layer, CIR allows researchers and operators to reconstruct the exact physical file tree (the "dirty inputs") perceived by a vantage point.
|
||||
|
||||
#### 1.1. Requirements Language
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user