#!/usr/bin/env bash set -euo pipefail # Full sync + validation from a TAL URL (default: APNIC). # # Produces artifacts under OUT_DIR (default: target/live/manual_sync): # - *_db_* RocksDB directory # - *_report_*.json audit report # - *_run_*.log stdout/stderr log (includes summary) # - *_db_stats_*.txt db_stats --exact output # - *_rrdp_state_*.tsv rrdp_state_dump --view legacy-state output ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" cd "$ROOT_DIR" TAL_URL="${TAL_URL:-https://tal.apnic.net/tal-archive/apnic-rfc7730-https.tal}" HTTP_TIMEOUT_SECS="${HTTP_TIMEOUT_SECS:-1800}" RSYNC_TIMEOUT_SECS="${RSYNC_TIMEOUT_SECS:-1800}" RSYNC_MIRROR_ROOT="${RSYNC_MIRROR_ROOT:-}" VALIDATION_TIME="${VALIDATION_TIME:-}" OUT_DIR="${OUT_DIR:-$ROOT_DIR/target/live/manual_sync}" mkdir -p "$OUT_DIR" TS="$(date -u +%Y%m%dT%H%M%SZ)" RUN_NAME="${RUN_NAME:-apnic_full_${TS}}" DB_DIR="${DB_DIR:-$OUT_DIR/${RUN_NAME}_db}" REPORT_JSON="${REPORT_JSON:-$OUT_DIR/${RUN_NAME}_report.json}" RUN_LOG="${RUN_LOG:-$OUT_DIR/${RUN_NAME}_run.log}" DB_STATS_TXT="${DB_STATS_TXT:-$OUT_DIR/${RUN_NAME}_db_stats.txt}" RRDP_STATE_TSV="${RRDP_STATE_TSV:-$OUT_DIR/${RUN_NAME}_rrdp_state.tsv}" RUN_META_JSON="${RUN_META_JSON:-$OUT_DIR/${RUN_NAME}_meta.json}" SUMMARY_MD="${SUMMARY_MD:-$OUT_DIR/${RUN_NAME}_summary.md}" echo "== rpki manual full sync ==" >&2 echo "tal_url=$TAL_URL" >&2 echo "db=$DB_DIR" >&2 echo "report_json=$REPORT_JSON" >&2 echo "out_dir=$OUT_DIR" >&2 cmd=(cargo run --release --bin rpki -- \ --db "$DB_DIR" \ --tal-url "$TAL_URL" \ --http-timeout-secs "$HTTP_TIMEOUT_SECS" \ --rsync-timeout-secs "$RSYNC_TIMEOUT_SECS" \ --report-json "$REPORT_JSON") if [[ -n "${RSYNC_MIRROR_ROOT}" ]]; then cmd+=(--rsync-mirror-root "$RSYNC_MIRROR_ROOT") fi if [[ -n "${VALIDATION_TIME}" ]]; then cmd+=(--validation-time "$VALIDATION_TIME") fi script_start_s="$(date +%s)" run_start_s="$(date +%s)" ( echo "# command:" printf '%q ' "${cmd[@]}" echo echo "${cmd[@]}" ) 2>&1 | tee "$RUN_LOG" >/dev/null run_end_s="$(date +%s)" run_duration_s="$((run_end_s - run_start_s))" echo "== db_stats (exact) ==" >&2 db_stats_start_s="$(date +%s)" cargo run --release --bin db_stats -- --db "$DB_DIR" --exact 2>&1 | tee "$DB_STATS_TXT" >/dev/null db_stats_end_s="$(date +%s)" db_stats_duration_s="$((db_stats_end_s - db_stats_start_s))" echo "== rrdp_state_dump (legacy-state) ==" >&2 state_start_s="$(date +%s)" cargo run --release --bin rrdp_state_dump -- --db "$DB_DIR" --view legacy-state >"$RRDP_STATE_TSV" state_end_s="$(date +%s)" state_duration_s="$((state_end_s - state_start_s))" script_end_s="$(date +%s)" total_duration_s="$((script_end_s - script_start_s))" echo "== write run meta + summary ==" >&2 TAL_URL="$TAL_URL" \ DB_DIR="$DB_DIR" \ REPORT_JSON="$REPORT_JSON" \ RUN_LOG="$RUN_LOG" \ HTTP_TIMEOUT_SECS="$HTTP_TIMEOUT_SECS" \ RSYNC_TIMEOUT_SECS="$RSYNC_TIMEOUT_SECS" \ VALIDATION_TIME_ARG="$VALIDATION_TIME" \ RUN_DURATION_S="$run_duration_s" \ DB_STATS_DURATION_S="$db_stats_duration_s" \ STATE_DURATION_S="$state_duration_s" \ TOTAL_DURATION_S="$total_duration_s" \ python3 - "$REPORT_JSON" "$RUN_META_JSON" "$SUMMARY_MD" <<'PY' import json import os import sys from datetime import datetime, timezone from pathlib import Path report_path = Path(sys.argv[1]) meta_path = Path(sys.argv[2]) summary_path = Path(sys.argv[3]) rep = json.loads(report_path.read_text(encoding="utf-8")) now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") download_stats = rep.get("download_stats") or {} meta = { "recorded_at_utc": now, "tal_url": os.environ["TAL_URL"], "db_dir": os.environ["DB_DIR"], "report_json": os.environ["REPORT_JSON"], "run_log": os.environ["RUN_LOG"], "validation_time_rfc3339_utc": rep["meta"]["validation_time_rfc3339_utc"], "http_timeout_secs": int(os.environ["HTTP_TIMEOUT_SECS"]), "rsync_timeout_secs": int(os.environ["RSYNC_TIMEOUT_SECS"]), "validation_time_arg": os.environ.get("VALIDATION_TIME_ARG",""), "durations_secs": { "rpki_run": int(os.environ["RUN_DURATION_S"]), "db_stats_exact": int(os.environ["DB_STATS_DURATION_S"]), "rrdp_state_dump": int(os.environ["STATE_DURATION_S"]), "total_script": int(os.environ["TOTAL_DURATION_S"]), }, "counts": { "publication_points_processed": rep["tree"]["instances_processed"], "publication_points_failed": rep["tree"]["instances_failed"], "vrps": len(rep["vrps"]), "aspas": len(rep["aspas"]), "audit_publication_points": len(rep["publication_points"]), }, "download_stats": download_stats, } meta_path.write_text(json.dumps(meta, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") lines = [] lines.append("# Manual full sync summary\\n\\n") lines.append(f"- recorded_at_utc: `{now}`\\n") lines.append(f"- tal_url: `{meta['tal_url']}`\\n") lines.append(f"- db: `{meta['db_dir']}`\\n") lines.append(f"- report_json: `{meta['report_json']}`\\n") lines.append(f"- validation_time: `{meta['validation_time_rfc3339_utc']}`\\n\\n") lines.append("## Results\\n\\n") lines.append("| metric | value |\\n") lines.append("|---|---:|\\n") for k in ["publication_points_processed","publication_points_failed","vrps","aspas","audit_publication_points"]: lines.append(f"| {k} | {meta['counts'][k]} |\\n") lines.append("\\n") lines.append("## Durations (seconds)\\n\\n") lines.append("| step | seconds |\\n") lines.append("|---|---:|\\n") for k,v in meta["durations_secs"].items(): lines.append(f"| {k} | {v} |\\n") lines.append("\\n") lines.append("## Download Stats\\n\\n") lines.append("- raw events: `report_json.downloads`\\n") lines.append("- aggregated: `report_json.download_stats` (copied into meta.json)\\n\\n") def fmt_u(v): if v is None: return "" return str(v) by_kind = download_stats.get("by_kind") or {} lines.append("| kind | ok | fail | duration_ms_total | bytes_total | objects_count_total | objects_bytes_total |\\n") lines.append("|---|---:|---:|---:|---:|---:|---:|\\n") for kind in sorted(by_kind.keys()): st = by_kind[kind] or {} lines.append( f"| {kind} | {st.get('ok_total',0)} | {st.get('fail_total',0)} | {st.get('duration_ms_total',0)} | {fmt_u(st.get('bytes_total'))} | {fmt_u(st.get('objects_count_total'))} | {fmt_u(st.get('objects_bytes_total'))} |\\n" ) lines.append("\\n") summary_path.write_text("".join(lines), encoding="utf-8") print(summary_path) PY echo "== done ==" >&2 echo "artifacts:" >&2 echo "- db: $DB_DIR" >&2 echo "- report: $REPORT_JSON" >&2 echo "- run log: $RUN_LOG" >&2 echo "- db stats: $DB_STATS_TXT" >&2 echo "- rrdp state: $RRDP_STATE_TSV" >&2 echo "- meta json: $RUN_META_JSON" >&2 echo "- summary md: $SUMMARY_MD" >&2