diff --git a/deploy/arm64-compose/.env.example b/deploy/arm64-compose/.env.example index ff4d61c..b294ffc 100644 --- a/deploy/arm64-compose/.env.example +++ b/deploy/arm64-compose/.env.example @@ -8,6 +8,11 @@ RPKI_IMAGE=ours-rp-runtime-arm64:dev # 核心 RP 镜像强制以 ARM64 运行;远端233为 x86_64,需要 binfmt/qemu 支持。 RPKI_PLATFORM=linux/arm64 +# metrics sidecar 独立镜像。 +METRICS_IMAGE=ours-rp-metrics-arm64:dev +METRICS_PLATFORM=linux/arm64 +MONITOR_PLATFORM=linux/arm64 + # 验收默认只跑 APNIC;all5 可改成 afrinic,apnic,arin,lacnic,ripe。 RIRS=apnic @@ -80,6 +85,8 @@ METRICS_PORT=9556 METRICS_POLL_SECS=10 # Prometheus / Grafana 配置。 +PROMETHEUS_IMAGE=prom/prometheus:v2.55.1 +GRAFANA_IMAGE=grafana/grafana:11.3.1 PROMETHEUS_PORT=9090 PROMETHEUS_RETENTION=7d GRAFANA_PORT=3000 diff --git a/deploy/arm64-compose/docker-compose.yml b/deploy/arm64-compose/docker-compose.yml index 5016931..c1da79d 100644 --- a/deploy/arm64-compose/docker-compose.yml +++ b/deploy/arm64-compose/docker-compose.yml @@ -22,8 +22,8 @@ services: - core artifact-metrics: - image: ${RPKI_IMAGE:-ours-rp-runtime-arm64:dev} - platform: ${RPKI_PLATFORM:-linux/arm64} + image: ${METRICS_IMAGE:-ours-rp-metrics-arm64:dev} + platform: ${METRICS_PLATFORM:-linux/arm64} container_name: ours-rp-arm64-artifact-metrics env_file: - ./.env @@ -49,6 +49,7 @@ services: prometheus: image: ${PROMETHEUS_IMAGE:-prom/prometheus:v2.55.1} + platform: ${MONITOR_PLATFORM:-linux/arm64} container_name: ours-rp-arm64-prometheus command: - --config.file=/etc/prometheus/prometheus.yml @@ -68,6 +69,7 @@ services: grafana: image: ${GRAFANA_IMAGE:-grafana/grafana:11.3.1} + platform: ${MONITOR_PLATFORM:-linux/arm64} container_name: ours-rp-arm64-grafana depends_on: - prometheus diff --git a/deploy/arm64-installer/.env.example b/deploy/arm64-installer/.env.example index 96dc20e..5b534f2 100644 --- a/deploy/arm64-installer/.env.example +++ b/deploy/arm64-installer/.env.example @@ -14,6 +14,10 @@ COMPOSE_PROJECT_NAME=ours-rp-arm64-installer RPKI_IMAGE=ours-rp-runtime-arm64:dev RPKI_PLATFORM=linux/arm64 +# Metrics image loaded from images/*.tar.gz by install.sh. +METRICS_IMAGE=ours-rp-metrics-arm64:dev +METRICS_PLATFORM=linux/arm64 + # Restart policy for the soak container. Production default keeps the daemon alive. # For finite acceptance tests such as MAX_RUNS=3, set SOAK_RESTART_POLICY=no to avoid an extra restarted run. SOAK_RESTART_POLICY=unless-stopped diff --git a/deploy/arm64-installer/compose/docker-compose.yml b/deploy/arm64-installer/compose/docker-compose.yml index 6c72e5e..ad67a4a 100644 --- a/deploy/arm64-installer/compose/docker-compose.yml +++ b/deploy/arm64-installer/compose/docker-compose.yml @@ -22,8 +22,8 @@ services: - core artifact-metrics: - image: ${RPKI_IMAGE:-ours-rp-runtime-arm64:dev} - platform: ${RPKI_PLATFORM:-linux/arm64} + image: ${METRICS_IMAGE:-ours-rp-metrics-arm64:dev} + platform: ${METRICS_PLATFORM:-linux/arm64} container_name: ${COMPOSE_PROJECT_NAME:-ours-rp-package-installer}-artifact-metrics env_file: - ../.env diff --git a/deploy/arm64-installer/docs/README.en.md b/deploy/arm64-installer/docs/README.en.md index 0f9b6ef..a0d3ebb 100644 --- a/deploy/arm64-installer/docs/README.en.md +++ b/deploy/arm64-installer/docs/README.en.md @@ -4,7 +4,7 @@ This package deploys ours RP on Linux servers whose host architecture matches the packaged installer metadata, using Docker Compose and continuous all-five RIR validation. -The package includes architecture-matched ours RP runtime, Prometheus and Grafana images, so deployment does not need to pull application images on the target host. Runtime state, run artifacts, logs, Prometheus data and Grafana data are persisted through host bind mounts. +The package includes four architecture-matched images: ours RP runtime, artifact metrics, Prometheus, and Grafana. Deployment does not need to pull application images on the target host. Runtime state, run artifacts, logs, Prometheus data and Grafana data are persisted through host bind mounts. By default the host architecture must match `PACKAGE_ARCH` from `PACKAGE-MANIFEST.env`. If you intentionally want to run this package on a different host architecture through QEMU/binfmt, set: @@ -38,6 +38,9 @@ Defaults: - `PERIODIC_SNAPSHOT_MAX_DELTAS=100` - `HOST_DATA_DIR=__HOST_DATA_DIR__` - `SOAK_RESTART_POLICY=unless-stopped` +- `RPKI_IMAGE=ours-rp-runtime-__PACKAGE_ARCH__:dev` +- `METRICS_IMAGE=ours-rp-metrics-__PACKAGE_ARCH__:dev` +- `METRICS_PLATFORM=__PACKAGE_PLATFORM__` - `MONITOR_PLATFORM=__PACKAGE_PLATFORM__` - `ALLOW_CROSS_ARCH=0` @@ -45,6 +48,12 @@ Defaults: If there is no successful run under `HOST_DATA_DIR/runs`, `start.sh` starts the core `ours-rp-soak` service first and waits for the first snapshot to succeed before starting metrics, Prometheus and Grafana. +Container/image mapping: + +- `ours-rp-soak` uses `RPKI_IMAGE` +- `artifact-metrics` uses `METRICS_IMAGE` +- `prometheus` / `grafana` use their monitor images + The first snapshot refreshes live TA certificates before starting the RP process. ## Architecture Guardrails diff --git a/deploy/arm64-installer/docs/README.zh-CN.md b/deploy/arm64-installer/docs/README.zh-CN.md index 023fd16..41feb75 100644 --- a/deploy/arm64-installer/docs/README.zh-CN.md +++ b/deploy/arm64-installer/docs/README.zh-CN.md @@ -4,7 +4,7 @@ 本安装包用于在目标架构与包元数据匹配的 Linux 服务器上,通过 Docker Compose 部署 ours RP,并持续运行 all5 RIR 同步验证任务。 -安装包内置与包架构一致的 ours RP runtime、Prometheus、Grafana 镜像,部署时不需要现场拉取应用镜像。运行产物、状态数据库、日志、Prometheus 和 Grafana 数据均通过宿主机目录挂载保存。 +安装包内置与包架构一致的四类镜像:ours RP runtime、artifact metrics、Prometheus、Grafana。部署时不需要现场拉取应用镜像。运行产物、状态数据库、日志、Prometheus 和 Grafana 数据均通过宿主机目录挂载保存。 默认情况下,宿主架构必须与 `PACKAGE-MANIFEST.env` 中的 `PACKAGE_ARCH` 匹配;若确需在异构主机上通过 QEMU/binfmt 运行,必须显式设置: @@ -38,6 +38,9 @@ vim .env - `PERIODIC_SNAPSHOT_MAX_DELTAS=100` - `HOST_DATA_DIR=__HOST_DATA_DIR__` - `SOAK_RESTART_POLICY=unless-stopped` +- `RPKI_IMAGE=ours-rp-runtime-__PACKAGE_ARCH__:dev` +- `METRICS_IMAGE=ours-rp-metrics-__PACKAGE_ARCH__:dev` +- `METRICS_PLATFORM=__PACKAGE_PLATFORM__` - `MONITOR_PLATFORM=__PACKAGE_PLATFORM__` - `ALLOW_CROSS_ARCH=0` @@ -45,6 +48,12 @@ vim .env 如果 `HOST_DATA_DIR/runs` 下没有成功 run,`start.sh` 会先启动核心 `ours-rp-soak`,等待第一轮 snapshot 成功后再启动 metrics、Prometheus 和 Grafana。 +容器镜像分工: + +- `ours-rp-soak` 使用 `RPKI_IMAGE` +- `artifact-metrics` 使用 `METRICS_IMAGE` +- `prometheus` / `grafana` 使用各自 monitor image + 第一轮 snapshot 会先拉取 live TA,避免 clean state 使用旧 fixture TA。 ## 架构检查 diff --git a/deploy/arm64-installer/docs/operations.en.md b/deploy/arm64-installer/docs/operations.en.md index 1695e0e..39cf800 100644 --- a/deploy/arm64-installer/docs/operations.en.md +++ b/deploy/arm64-installer/docs/operations.en.md @@ -10,7 +10,7 @@ The installer is idempotent: - existing `.env` is kept; - existing Docker/Compose installation is reused; -- repeated loading of packaged ours RP, Prometheus and Grafana images for the package architecture is safe; +- repeated loading of packaged runtime, metrics, Prometheus and Grafana images for the package architecture is safe; - existing data directory is reused; - matching host/package architecture is required by default. @@ -51,7 +51,7 @@ Then rerun `./install.sh` or `./self-check.sh`, and only then will the scripts a Important checks: - Docker/Compose availability; -- runtime, Prometheus and Grafana images exist; +- runtime, metrics, Prometheus and Grafana images exist; - `HOST_DATA_DIR` is writable; - Compose config is valid; - latest run status; @@ -62,6 +62,8 @@ Important checks: - `package_arch` - `package_platform` +- `runtime_image` +- `metrics_image` - `allow_cross_arch` - `periodic_snapshot_reset` - `periodic_snapshot_max_deltas` @@ -76,6 +78,8 @@ Extract the new package into a new directory and explicitly reuse the existing ` If the new package directory already has a `.env`, the upgrade script keeps it. +If an older `.env` does not contain `METRICS_IMAGE` or `METRICS_PLATFORM`, the install/upgrade flow backfills package-matched defaults automatically. + Upgrade does not delete: - `runs/` diff --git a/deploy/arm64-installer/docs/operations.zh-CN.md b/deploy/arm64-installer/docs/operations.zh-CN.md index b2a8dee..f01556d 100644 --- a/deploy/arm64-installer/docs/operations.zh-CN.md +++ b/deploy/arm64-installer/docs/operations.zh-CN.md @@ -10,7 +10,7 @@ - 已有 `.env` 不覆盖; - 已安装 Docker/Compose 则跳过; -- 包内 ours RP、Prometheus、Grafana 对应架构镜像重复加载是安全的; +- 包内 runtime、metrics、Prometheus、Grafana 对应架构镜像重复加载是安全的; - 数据目录已存在则复用; - 默认要求宿主与包架构匹配。 @@ -51,7 +51,7 @@ ALLOW_CROSS_ARCH=1 重点检查项: - Docker/Compose 可用; -- runtime、Prometheus、Grafana 镜像存在; +- runtime、metrics、Prometheus、Grafana 镜像存在; - `HOST_DATA_DIR` 可写; - Compose 配置合法; - 最新 run 状态; @@ -62,6 +62,8 @@ ALLOW_CROSS_ARCH=1 - `package_arch` - `package_platform` +- `runtime_image` +- `metrics_image` - `allow_cross_arch` - `periodic_snapshot_reset` - `periodic_snapshot_max_deltas` @@ -76,6 +78,8 @@ ALLOW_CROSS_ARCH=1 如果新目录已经存在 `.env`,升级脚本会保留它,不覆盖。 +如果旧 `.env` 缺失 `METRICS_IMAGE` 或 `METRICS_PLATFORM`,升级/安装链路会按当前包架构自动补齐默认值。 + 升级不会删除以下数据: - `runs/` diff --git a/deploy/arm64-installer/docs/troubleshooting.en.md b/deploy/arm64-installer/docs/troubleshooting.en.md index d72eab7..46ab470 100644 --- a/deploy/arm64-installer/docs/troubleshooting.en.md +++ b/deploy/arm64-installer/docs/troubleshooting.en.md @@ -44,6 +44,15 @@ docker run --rm --platform __PACKAGE_PLATFORM__ debian:bookworm-slim uname -m For `arm64` packages the expected output is `aarch64`; for `amd64` packages the expected output is `x86_64`. +To confirm the metrics image is also loaded correctly: + +```bash +docker image inspect "$(grep '^METRICS_IMAGE=' .env | cut -d= -f2-)" +docker run --rm --platform __PACKAGE_PLATFORM__ \ + "$(grep '^METRICS_IMAGE=' .env | cut -d= -f2-)" \ + /opt/ours-rp/bin/rpki_artifact_metrics --help +``` + ## First Snapshot Times Out All-five snapshot can be slow, especially under QEMU. Increase timeout: diff --git a/deploy/arm64-installer/docs/troubleshooting.zh-CN.md b/deploy/arm64-installer/docs/troubleshooting.zh-CN.md index 8429a5a..0c73a48 100644 --- a/deploy/arm64-installer/docs/troubleshooting.zh-CN.md +++ b/deploy/arm64-installer/docs/troubleshooting.zh-CN.md @@ -44,6 +44,15 @@ docker run --rm --platform __PACKAGE_PLATFORM__ debian:bookworm-slim uname -m 对于 `arm64` 包,预期输出 `aarch64`;对于 `amd64` 包,预期输出 `x86_64`。 +如需进一步确认 metrics 镜像也已正确加载: + +```bash +docker image inspect "$(grep '^METRICS_IMAGE=' .env | cut -d= -f2-)" +docker run --rm --platform __PACKAGE_PLATFORM__ \ + "$(grep '^METRICS_IMAGE=' .env | cut -d= -f2-)" \ + /opt/ours-rp/bin/rpki_artifact_metrics --help +``` + ## 首轮 snapshot 超时 all5 snapshot 可能很慢,尤其在 QEMU 环境。可以提高超时: diff --git a/deploy/arm64-installer/install.sh b/deploy/arm64-installer/install.sh index 3ae8699..4681b81 100755 --- a/deploy/arm64-installer/install.sh +++ b/deploy/arm64-installer/install.sh @@ -40,6 +40,7 @@ create_data_dirs load_installer_images ensure_binfmt_if_needed verify_runtime_image +verify_metrics_image verify_monitor_images compose_config_path="$(mktemp "${TMPDIR:-/tmp}/ours-rp-compose-config.XXXXXX.yml")" compose_cmd --profile core --profile sidecar --profile monitor config >"$compose_config_path" diff --git a/deploy/arm64-installer/scripts/common.sh b/deploy/arm64-installer/scripts/common.sh index 11008aa..d34e8cb 100644 --- a/deploy/arm64-installer/scripts/common.sh +++ b/deploy/arm64-installer/scripts/common.sh @@ -42,6 +42,26 @@ platform_for_arch() { printf 'linux/%s\n' "$1" } +env_file_has_key() { + local env_path="$1" + local key="$2" + [[ -f "$env_path" ]] && grep -Eq "^${key}=" "$env_path" +} + +persist_env_default() { + local env_path="$1" + local key="$2" + local value="$3" + local tmp_env + if env_file_has_key "$env_path" "$key"; then + return 0 + fi + tmp_env="$(mktemp)" + cat "$env_path" > "$tmp_env" + printf '%s=%s\n' "$key" "$value" >> "$tmp_env" + mv "$tmp_env" "$env_path" +} + env_flag_enabled() { case "${1:-0}" in 1|true|TRUE|yes|YES|on|ON) @@ -85,6 +105,10 @@ effective_package_arch() { normalize_arch "$RPKI_PLATFORM" || die "unsupported RPKI_PLATFORM=${RPKI_PLATFORM}" return 0 fi + if [[ -n "${METRICS_PLATFORM:-}" ]]; then + normalize_arch "$METRICS_PLATFORM" || die "unsupported METRICS_PLATFORM=${METRICS_PLATFORM}" + return 0 + fi if [[ -n "${MONITOR_PLATFORM:-}" ]]; then normalize_arch "$MONITOR_PLATFORM" || die "unsupported MONITOR_PLATFORM=${MONITOR_PLATFORM}" return 0 @@ -170,12 +194,17 @@ load_env() { COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-ours-rp-${PACKAGE_ARCH}-installer}" RPKI_IMAGE="${RPKI_IMAGE:-ours-rp-runtime-${PACKAGE_ARCH}:dev}" RPKI_PLATFORM="${RPKI_PLATFORM:-$PACKAGE_PLATFORM}" + METRICS_IMAGE="${METRICS_IMAGE:-ours-rp-metrics-${PACKAGE_ARCH}:dev}" + METRICS_PLATFORM="${METRICS_PLATFORM:-$PACKAGE_PLATFORM}" MONITOR_PLATFORM="${MONITOR_PLATFORM:-$PACKAGE_PLATFORM}" PROMETHEUS_IMAGE="${PROMETHEUS_IMAGE:-prom/prometheus:v2.55.1}" GRAFANA_IMAGE="${GRAFANA_IMAGE:-grafana/grafana:11.3.1}" METRICS_INSTANCE="${METRICS_INSTANCE:-${PACKAGE_ARCH}-installer}" ALLOW_CROSS_ARCH="${ALLOW_CROSS_ARCH:-0}" FIRST_RUN_WAIT_TIMEOUT_SECS="${FIRST_RUN_WAIT_TIMEOUT_SECS:-7200}" + + persist_env_default "$ENV_FILE" "METRICS_IMAGE" "$METRICS_IMAGE" + persist_env_default "$ENV_FILE" "METRICS_PLATFORM" "$METRICS_PLATFORM" } compose_cmd() { @@ -340,6 +369,7 @@ install_docker_if_missing() { } load_installer_images() { + load_env require_cmd docker shopt -s nullglob local image load_output desired_tag @@ -358,6 +388,9 @@ load_installer_images() { "${image_tar:-}") desired_tag="${image_tag:-$RPKI_IMAGE}" ;; + "${metrics_image_tar:-}") + desired_tag="${metrics_image:-$METRICS_IMAGE}" + ;; "${prometheus_image_tar:-}") desired_tag="${prometheus_image:-$PROMETHEUS_IMAGE}" ;; @@ -377,15 +410,19 @@ ensure_loaded_image_tag() { local desired_tag="$1" local load_output="$2" local image_path="$3" - if docker image inspect "$desired_tag" >/dev/null 2>&1; then - return 0 - fi local loaded_ref="" + local loaded_id="" + local desired_id="" loaded_ref="$(printf '%s\n' "$load_output" | awk -F'Loaded image: ' '/Loaded image: / {print $2; exit}')" if [[ -z "$loaded_ref" ]]; then loaded_ref="$(printf '%s\n' "$load_output" | awk -F'Loaded image ID: ' '/Loaded image ID: / {print $2; exit}')" fi [[ -n "$loaded_ref" ]] || die "unable to determine loaded image reference for $image_path" + loaded_id="$(docker image inspect --format '{{.Id}}' "$loaded_ref" 2>/dev/null || true)" + desired_id="$(docker image inspect --format '{{.Id}}' "$desired_tag" 2>/dev/null || true)" + if [[ -n "$loaded_id" && -n "$desired_id" && "$loaded_id" == "$desired_id" ]]; then + return 0 + fi log "tagging loaded image for package use: source=$loaded_ref target=$desired_tag" docker tag "$loaded_ref" "$desired_tag" } @@ -407,13 +444,17 @@ ensure_binfmt_if_needed() { verify_runtime_image() { load_env require_cmd docker - local help_path - help_path="$(mktemp "${TMPDIR:-/tmp}/ours-rp-rpki-help.XXXXXX")" log "verifying runtime image $RPKI_IMAGE on $RPKI_PLATFORM" - docker image inspect "$RPKI_IMAGE" >/dev/null - docker run --rm --platform "$RPKI_PLATFORM" "$RPKI_IMAGE" /opt/ours-rp/bin/rpki --help >"$help_path" - head -5 "$help_path" || true - rm -f "$help_path" + verify_image_platform "$RPKI_IMAGE" "$RPKI_PLATFORM" "runtime" + verify_image_usage_output "$RPKI_IMAGE" "$RPKI_PLATFORM" "runtime" /opt/ours-rp/bin/rpki --help +} + +verify_metrics_image() { + load_env + require_cmd docker + log "verifying metrics image $METRICS_IMAGE on $METRICS_PLATFORM" + verify_image_platform "$METRICS_IMAGE" "$METRICS_PLATFORM" "metrics" + verify_image_usage_output "$METRICS_IMAGE" "$METRICS_PLATFORM" "metrics" /opt/ours-rp/bin/rpki_artifact_metrics --help } verify_image_platform() { @@ -426,6 +467,32 @@ verify_image_platform() { [[ "$actual_platform" == "$expected_platform" ]] || die "$role image platform mismatch: image=$image expected=$expected_platform actual=$actual_platform" } +verify_image_usage_output() { + local image="$1" + local expected_platform="$2" + local role="$3" + shift 3 + local help_path + local status + help_path="$(mktemp "${TMPDIR:-/tmp}/ours-rp-image-help.XXXXXX")" + set +e + docker run --rm --platform "$expected_platform" "$image" "$@" >"$help_path" 2>&1 + status=$? + set -e + if [[ "$status" != "0" && "$status" != "1" ]]; then + cat "$help_path" >&2 || true + rm -f "$help_path" + die "$role image command failed: image=$image platform=$expected_platform exit=$status" + fi + grep -q '^Usage' "$help_path" || { + cat "$help_path" >&2 || true + rm -f "$help_path" + die "$role image did not emit usage output: image=$image platform=$expected_platform exit=$status" + } + head -5 "$help_path" || true + rm -f "$help_path" +} + verify_monitor_images() { load_env require_cmd docker diff --git a/deploy/arm64-installer/self-check.sh b/deploy/arm64-installer/self-check.sh index fee8468..22ed285 100755 --- a/deploy/arm64-installer/self-check.sh +++ b/deploy/arm64-installer/self-check.sh @@ -32,8 +32,10 @@ create_data_dirs [[ -w "$HOST_DATA_DIR" ]] || die "data dir is not writable: $HOST_DATA_DIR" compose_cmd --profile core --profile sidecar --profile monitor config >/dev/null verify_image_platform "$RPKI_IMAGE" "$RPKI_PLATFORM" "runtime" +verify_image_platform "$METRICS_IMAGE" "$METRICS_PLATFORM" "metrics" verify_monitor_images if [[ "$QUICK" == "0" ]]; then verify_runtime_image + verify_metrics_image fi log "self-check ok" diff --git a/deploy/arm64-installer/status.sh b/deploy/arm64-installer/status.sh index 2602c89..8a676a9 100755 --- a/deploy/arm64-installer/status.sh +++ b/deploy/arm64-installer/status.sh @@ -32,8 +32,11 @@ echo "package_arch=$PACKAGE_ARCH" echo "package_platform=$PACKAGE_PLATFORM" echo "allow_cross_arch=$ALLOW_CROSS_ARCH" echo "arch_mode=$mode" -echo "image=$RPKI_IMAGE" -echo "platform=$RPKI_PLATFORM" +echo "runtime_image=$RPKI_IMAGE" +echo "runtime_platform=$RPKI_PLATFORM" +echo "metrics_image=$METRICS_IMAGE" +echo "metrics_platform=$METRICS_PLATFORM" +echo "monitor_platform=$MONITOR_PLATFORM" echo "rirs=${RIRS:-}" echo "max_runs=${MAX_RUNS:-}" echo "interval_secs=${INTERVAL_SECS:-}" diff --git a/deploy/arm64-installer/upgrade.sh b/deploy/arm64-installer/upgrade.sh index d55af68..f1497b3 100755 --- a/deploy/arm64-installer/upgrade.sh +++ b/deploy/arm64-installer/upgrade.sh @@ -73,7 +73,10 @@ overlay_reused_env() { [[ -n "$key" ]] || continue [[ "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || continue case "$key" in - RPKI_IMAGE|PROMETHEUS_IMAGE|GRAFANA_IMAGE) + PACKAGE_ARCH|PACKAGE_PLATFORM|RPKI_PLATFORM|METRICS_PLATFORM|MONITOR_PLATFORM) + continue + ;; + RPKI_IMAGE|METRICS_IMAGE|PROMETHEUS_IMAGE|GRAFANA_IMAGE) [[ "$UPDATE_PACKAGE_IMAGE" == "0" ]] || continue ;; esac @@ -101,6 +104,7 @@ install_docker_if_missing load_installer_images ensure_binfmt_if_needed verify_runtime_image +verify_metrics_image verify_monitor_images compose_cmd --profile core --profile sidecar --profile monitor up -d --force-recreate "$SCRIPT_DIR/status.sh" --brief || true diff --git a/docker/ours-rp-metrics.Dockerfile b/docker/ours-rp-metrics.Dockerfile new file mode 100644 index 0000000..6954890 --- /dev/null +++ b/docker/ours-rp-metrics.Dockerfile @@ -0,0 +1,110 @@ +ARG BUILDER_IMAGE=rust:1-bookworm +ARG RUNTIME_IMAGE=debian:bookworm-slim + +FROM --platform=$BUILDPLATFORM ${BUILDER_IMAGE} AS builder + +ARG BUILDARCH +ARG TARGETARCH +ARG TARGETPLATFORM + +WORKDIR /src + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + clang \ + cmake \ + git \ + libclang-dev \ + make \ + perl \ + pkg-config \ + python3 \ + && if [ "$TARGETARCH" != "$BUILDARCH" ]; then \ + case "$TARGETARCH" in \ + arm64) \ + apt-get install -y --no-install-recommends \ + g++-aarch64-linux-gnu \ + gcc-aarch64-linux-gnu \ + libc6-dev-arm64-cross \ + ;; \ + amd64) \ + apt-get install -y --no-install-recommends \ + g++-x86-64-linux-gnu \ + gcc-x86-64-linux-gnu \ + libc6-dev-amd64-cross \ + ;; \ + *) \ + echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 2 \ + ;; \ + esac; \ + fi \ + && rm -rf /var/lib/apt/lists/* \ + && case "$TARGETARCH" in \ + amd64|arm64) ;; \ + *) echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 2 ;; \ + esac + +COPY . . + +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + --mount=type=cache,target=/src/target \ + set -eux; \ + case "$TARGETARCH" in \ + amd64) target_triple=x86_64-unknown-linux-gnu ;; \ + arm64) target_triple=aarch64-unknown-linux-gnu ;; \ + *) echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 2 ;; \ + esac; \ + rustup target add "$target_triple"; \ + if [ "$TARGETARCH" != "$BUILDARCH" ]; then \ + case "$TARGETARCH" in \ + arm64) \ + export \ + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc \ + CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc \ + CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ \ + AR_aarch64_unknown_linux_gnu=aarch64-linux-gnu-ar \ + PKG_CONFIG_ALLOW_CROSS=1 \ + ;; \ + amd64) \ + export \ + CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc \ + CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc \ + CXX_x86_64_unknown_linux_gnu=x86_64-linux-gnu-g++ \ + AR_x86_64_unknown_linux_gnu=x86_64-linux-gnu-ar \ + PKG_CONFIG_ALLOW_CROSS=1 \ + ;; \ + *) \ + echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 2 \ + ;; \ + esac; \ + fi; \ + cargo build --release --target "$target_triple" --bin rpki_artifact_metrics; \ + mkdir -p /build-out/bin; \ + cp "target/$target_triple/release/rpki_artifact_metrics" /build-out/bin/ + +FROM --platform=$TARGETPLATFORM ${RUNTIME_IMAGE} AS runtime + +LABEL org.opencontainers.image.title="ours-rp-metrics" \ + org.opencontainers.image.description="Ours RP artifact metrics image for multi-arch Docker Compose deployment" + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + tzdata \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /opt/ours-rp + +COPY --from=builder /build-out/bin/ /opt/ours-rp/bin/ + +RUN chmod +x /opt/ours-rp/bin/* \ + && mkdir -p /var/lib/ours-rp/state /var/lib/ours-rp/runs /var/lib/ours-rp/logs + +ENV RUN_ROOT=/var/lib/ours-rp \ + BIN_DIR=/opt/ours-rp/bin \ + RUST_BACKTRACE=1 + +VOLUME ["/var/lib/ours-rp"] diff --git a/docker/ours-rp-runtime.Dockerfile b/docker/ours-rp-runtime.Dockerfile index dbc0107..fce28ac 100644 --- a/docker/ours-rp-runtime.Dockerfile +++ b/docker/ours-rp-runtime.Dockerfile @@ -84,14 +84,12 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \ cargo build --release --target "$target_triple" \ --bin rpki \ --bin rpki_daemon \ - --bin db_stats \ - --bin rpki_artifact_metrics; \ + --bin db_stats; \ mkdir -p /build-out/bin; \ cp \ "target/$target_triple/release/rpki" \ "target/$target_triple/release/rpki_daemon" \ "target/$target_triple/release/db_stats" \ - "target/$target_triple/release/rpki_artifact_metrics" \ /build-out/bin/ FROM --platform=$TARGETPLATFORM ${RUNTIME_IMAGE} AS runtime diff --git a/scripts/docker/build_arm64_metrics_image.sh b/scripts/docker/build_arm64_metrics_image.sh new file mode 100755 index 0000000..3b239d5 --- /dev/null +++ b/scripts/docker/build_arm64_metrics_image.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec "$SCRIPT_DIR/build_docker_metrics_image.sh" --arch arm64 "$@" diff --git a/scripts/docker/build_docker_installer_package.sh b/scripts/docker/build_docker_installer_package.sh index c06c2bd..5ba2c0a 100755 --- a/scripts/docker/build_docker_installer_package.sh +++ b/scripts/docker/build_docker_installer_package.sh @@ -7,6 +7,8 @@ REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" TARGET_ARCH="${TARGET_ARCH:-}" IMAGE_TAG="${IMAGE_TAG:-}" IMAGE_TAR="${IMAGE_TAR:-}" +METRICS_IMAGE="${METRICS_IMAGE:-}" +METRICS_IMAGE_TAR="${METRICS_IMAGE_TAR:-}" PROMETHEUS_IMAGE="${PROMETHEUS_IMAGE:-prom/prometheus:v2.55.1}" PROMETHEUS_IMAGE_TAR="${PROMETHEUS_IMAGE_TAR:-}" GRAFANA_IMAGE="${GRAFANA_IMAGE:-grafana/grafana:11.3.1}" @@ -24,6 +26,10 @@ Options: --arch Target architecture: amd64|arm64 --image Runtime image tag recorded in package manifest. --image-tar Existing docker save tar/tar.gz to include. + --metrics-image + Metrics image tag recorded in package manifest. + --metrics-image-tar + Existing metrics docker save tar/tar.gz to include. --prometheus-image Prometheus image tag to record and package. --prometheus-image-tar @@ -38,8 +44,8 @@ Options: Package template directory. -h, --help Show help. -If --image-tar is omitted, the script uses the newest -target/-docker/*.tar.gz file. +If --image-tar or --metrics-image-tar is omitted, the script uses the +matching docker-save archive under target/-docker/. USAGE } @@ -69,6 +75,10 @@ default_runtime_tag() { printf 'ours-rp-runtime-%s:dev\n' "$1" } +default_metrics_tag() { + printf 'ours-rp-metrics-%s:dev\n' "$1" +} + default_package_prefix() { printf 'ours-rp-docker-installer-%s\n' "$1" } @@ -153,6 +163,22 @@ save_image_if_needed() { printf '%s\n' "$tar_path" } +default_saved_image_tar() { + local image="$1" + local out_dir="$REPO_ROOT/target/${TARGET_ARCH}-docker" + local safe_name + safe_name="$(safe_tag_name "$image")" + if [[ -f "$out_dir/${safe_name}.tar.gz" ]]; then + printf '%s\n' "$out_dir/${safe_name}.tar.gz" + return 0 + fi + if [[ -f "$out_dir/${safe_name}.tar" ]]; then + printf '%s\n' "$out_dir/${safe_name}.tar" + return 0 + fi + printf '\n' +} + while [[ $# -gt 0 ]]; do case "$1" in --arch) @@ -170,6 +196,14 @@ while [[ $# -gt 0 ]]; do IMAGE_TAR="$2" shift 2 ;; + --metrics-image) + METRICS_IMAGE="$2" + shift 2 + ;; + --metrics-image-tar) + METRICS_IMAGE_TAR="$2" + shift 2 + ;; --prometheus-image) PROMETHEUS_IMAGE="$2" shift 2 @@ -224,6 +258,9 @@ done if [[ -z "$IMAGE_TAG" ]]; then IMAGE_TAG="$(default_runtime_tag "$TARGET_ARCH")" fi +if [[ -z "$METRICS_IMAGE" ]]; then + METRICS_IMAGE="$(default_metrics_tag "$TARGET_ARCH")" +fi if [[ -z "$OUT_DIR" ]]; then OUT_DIR="$REPO_ROOT/target/${TARGET_ARCH}-installer" fi @@ -237,7 +274,10 @@ COMPOSE_PROJECT_DEFAULT="$(default_compose_project "$TARGET_ARCH")" METRICS_INSTANCE_DEFAULT="$(default_metrics_instance "$TARGET_ARCH")" if [[ -z "$IMAGE_TAR" ]]; then - IMAGE_TAR="$(find "$REPO_ROOT/target/${TARGET_ARCH}-docker" -maxdepth 1 -type f \( -name '*.tar.gz' -o -name '*.tar' \) -printf '%T@ %p\n' 2>/dev/null | sort -nr | awk 'NR==1 {print $2}')" + IMAGE_TAR="$(default_saved_image_tar "$IMAGE_TAG")" +fi +if [[ -z "$METRICS_IMAGE_TAR" ]]; then + METRICS_IMAGE_TAR="$(default_saved_image_tar "$METRICS_IMAGE")" fi [[ -n "$IMAGE_TAR" && -f "$IMAGE_TAR" ]] || { @@ -249,6 +289,15 @@ Build one first, for example: EOF exit 2 } +[[ -n "$METRICS_IMAGE_TAR" && -f "$METRICS_IMAGE_TAR" ]] || { + cat >&2 </dev/null || echo unknown)" @@ -261,6 +310,7 @@ rm -rf "$stage" rsync -a --delete "$TEMPLATE_DIR"/ "$stage"/ mkdir -p "$stage/images" cp "$IMAGE_TAR" "$stage/images/" +cp "$METRICS_IMAGE_TAR" "$stage/images/" monitor_image_stage="$OUT_DIR/.monitor-images-$TARGET_ARCH-$timestamp" rm -rf "$monitor_image_stage" @@ -276,6 +326,8 @@ if [[ -f "$stage/.env.example" ]]; then replace_or_append_env "$stage/.env.example" "COMPOSE_PROJECT_NAME" "$COMPOSE_PROJECT_DEFAULT" replace_or_append_env "$stage/.env.example" "RPKI_IMAGE" "$IMAGE_TAG" replace_or_append_env "$stage/.env.example" "RPKI_PLATFORM" "$TARGET_PLATFORM" + replace_or_append_env "$stage/.env.example" "METRICS_IMAGE" "$METRICS_IMAGE" + replace_or_append_env "$stage/.env.example" "METRICS_PLATFORM" "$TARGET_PLATFORM" replace_or_append_env "$stage/.env.example" "HOST_DATA_DIR" "$HOST_DATA_DIR_DEFAULT" replace_or_append_env "$stage/.env.example" "ALLOW_CROSS_ARCH" "0" replace_or_append_env "$stage/.env.example" "METRICS_INSTANCE" "$METRICS_INSTANCE_DEFAULT" @@ -302,6 +354,7 @@ replace_text_in_file "$stage/docs/troubleshooting.zh-CN.md" "__HOST_DATA_DIR__" replace_text_in_file "$stage/docs/troubleshooting.en.md" "__HOST_DATA_DIR__" "$HOST_DATA_DIR_DEFAULT" runtime_tar_sha256="$(sha256sum "$IMAGE_TAR" | awk '{print $1}')" +metrics_tar_sha256="$(sha256sum "$METRICS_IMAGE_TAR" | awk '{print $1}')" prometheus_tar_sha256="$(sha256sum "$prometheus_tar" | awk '{print $1}')" grafana_tar_sha256="$(sha256sum "$grafana_tar" | awk '{print $1}')" @@ -316,20 +369,35 @@ package_arch=$TARGET_ARCH package_platform=$TARGET_PLATFORM cross_arch_default=deny cross_arch_enable_var=ALLOW_CROSS_ARCH +RPKI_IMAGE=$IMAGE_TAG +RPKI_PLATFORM=$TARGET_PLATFORM image_tag=$IMAGE_TAG image_tar=$(basename "$IMAGE_TAR") image_tar_size_bytes=$(wc -c < "$IMAGE_TAR") image_tar_sha256=$runtime_tar_sha256 +rpki_image=$IMAGE_TAG +rpki_platform=$TARGET_PLATFORM +METRICS_IMAGE=$METRICS_IMAGE +METRICS_PLATFORM=$TARGET_PLATFORM +metrics_image=$METRICS_IMAGE +metrics_image_tar=$(basename "$METRICS_IMAGE_TAR") +metrics_image_tar_size_bytes=$(wc -c < "$METRICS_IMAGE_TAR") +metrics_image_tar_sha256=$metrics_tar_sha256 +PROMETHEUS_IMAGE=$PROMETHEUS_IMAGE prometheus_image=$PROMETHEUS_IMAGE prometheus_image_tar=$(basename "$prometheus_tar") prometheus_image_tar_size_bytes=$(wc -c < "$prometheus_tar") prometheus_image_tar_sha256=$prometheus_tar_sha256 +GRAFANA_IMAGE=$GRAFANA_IMAGE grafana_image=$GRAFANA_IMAGE grafana_image_tar=$(basename "$grafana_tar") grafana_image_tar_size_bytes=$(wc -c < "$grafana_tar") grafana_image_tar_sha256=$grafana_tar_sha256 target_platform=$TARGET_PLATFORM +MONITOR_PLATFORM=$TARGET_PLATFORM +rpki_image_tar=$(basename "$IMAGE_TAR") monitor_platform=$TARGET_PLATFORM +metrics_platform=$TARGET_PLATFORM default_host_data_dir=$HOST_DATA_DIR_DEFAULT default_compose_project_name=$COMPOSE_PROJECT_DEFAULT default_metrics_instance=$METRICS_INSTANCE_DEFAULT @@ -340,6 +408,7 @@ package_name: $package_name package_arch: $TARGET_ARCH package_platform: $TARGET_PLATFORM runtime_image: $IMAGE_TAG +metrics_image: $METRICS_IMAGE prometheus_image: $PROMETHEUS_IMAGE grafana_image: $GRAFANA_IMAGE default_host_data_dir: $HOST_DATA_DIR_DEFAULT diff --git a/scripts/docker/build_docker_metrics_image.sh b/scripts/docker/build_docker_metrics_image.sh new file mode 100755 index 0000000..b1e3ed3 --- /dev/null +++ b/scripts/docker/build_docker_metrics_image.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +normalize_arch() { + case "$1" in + amd64|x86_64) + printf 'amd64\n' + ;; + arm64|aarch64) + printf 'arm64\n' + ;; + *) + return 1 + ;; + esac +} + +TARGET_ARCH="${TARGET_ARCH:-}" +IMAGE_TAG="${IMAGE_TAG:-}" +DOCKERFILE="${DOCKERFILE:-$REPO_ROOT/docker/ours-rp-metrics.Dockerfile}" +args=() +image_explicit=0 +dockerfile_explicit=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --arch) + TARGET_ARCH="$(normalize_arch "$2")" || { + echo "unsupported target architecture: $2" >&2 + exit 2 + } + args+=("$1" "$TARGET_ARCH") + shift 2 + ;; + --image) + IMAGE_TAG="$2" + image_explicit=1 + args+=("$1" "$2") + shift 2 + ;; + --dockerfile) + DOCKERFILE="$2" + dockerfile_explicit=1 + args+=("$1" "$2") + shift 2 + ;; + *) + args+=("$1") + shift + ;; + esac +done + +if [[ -z "$TARGET_ARCH" ]]; then + cat >&2 <<'EOF' +missing target architecture. + +Usage: + scripts/docker/build_docker_metrics_image.sh --arch amd64|arm64 [options] +EOF + exit 2 +fi + +if [[ -z "$IMAGE_TAG" ]]; then + IMAGE_TAG="ours-rp-metrics-${TARGET_ARCH}:dev" +fi + +if [[ "$dockerfile_explicit" != "1" ]]; then + args=(--dockerfile "$DOCKERFILE" "${args[@]}") +fi +if [[ "$image_explicit" != "1" ]]; then + args=(--image "$IMAGE_TAG" "${args[@]}") +fi + +exec "$SCRIPT_DIR/build_docker_runtime_image.sh" "${args[@]}" diff --git a/scripts/docker/deploy_remote233_arm64_compose.sh b/scripts/docker/deploy_remote233_arm64_compose.sh index efc33e2..56d0db4 100755 --- a/scripts/docker/deploy_remote233_arm64_compose.sh +++ b/scripts/docker/deploy_remote233_arm64_compose.sh @@ -8,6 +8,8 @@ REMOTE_HOST="${REMOTE_HOST:-root@47.77.204.233}" REMOTE_ROOT="${REMOTE_ROOT:-/root/ours-rp-arm64-compose}" IMAGE_TAG="${IMAGE_TAG:-ours-rp-runtime-arm64:dev}" IMAGE_TAR="${IMAGE_TAR:-$REPO_ROOT/target/arm64-docker/ours-rp-runtime-arm64-dev.tar.gz}" +METRICS_IMAGE="${METRICS_IMAGE:-ours-rp-metrics-arm64:dev}" +METRICS_IMAGE_TAR="${METRICS_IMAGE_TAR:-$REPO_ROOT/target/arm64-docker/ours-rp-metrics-arm64-dev.tar.gz}" EXECUTE="${EXECUTE:-0}" INSTALL_DOCKER="${INSTALL_DOCKER:-1}" INSTALL_BINFMT="${INSTALL_BINFMT:-1}" @@ -27,6 +29,9 @@ Options: --remote-root Remote compose root --image Image tag loaded by docker load --image-tar Local docker save tar.gz + --metrics-image Metrics image tag loaded by docker load + --metrics-image-tar + Local metrics docker save tar.gz --no-install-docker Skip Docker installation --no-binfmt Skip binfmt/qemu installation --start-core Start ours-rp-soak after deploy @@ -57,6 +62,14 @@ while [[ $# -gt 0 ]]; do IMAGE_TAR="$2" shift 2 ;; + --metrics-image) + METRICS_IMAGE="$2" + shift 2 + ;; + --metrics-image-tar) + METRICS_IMAGE_TAR="$2" + shift 2 + ;; --no-install-docker) INSTALL_DOCKER=0 shift @@ -117,6 +130,10 @@ require_command rsync echo "missing image tar: $IMAGE_TAR" >&2 exit 2 } +[[ -f "$METRICS_IMAGE_TAR" ]] || { + echo "missing metrics image tar: $METRICS_IMAGE_TAR" >&2 + exit 2 +} compose_src="$REPO_ROOT/deploy/arm64-compose/" [[ -f "$compose_src/docker-compose.yml" ]] || { @@ -128,6 +145,8 @@ echo "remote=$REMOTE_HOST" echo "remote_root=$REMOTE_ROOT" echo "image=$IMAGE_TAG" echo "image_tar=$IMAGE_TAR" +echo "metrics_image=$METRICS_IMAGE" +echo "metrics_image_tar=$METRICS_IMAGE_TAR" if [[ "$INSTALL_DOCKER" == "1" ]]; then remote_run "if ! command -v docker >/dev/null 2>&1; then apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y docker.io; fi; if ! docker compose version >/dev/null 2>&1; then if apt-cache show docker-compose-plugin >/dev/null 2>&1; then DEBIAN_FRONTEND=noninteractive apt-get install -y docker-compose-plugin; elif apt-cache show docker-compose-v2 >/dev/null 2>&1; then DEBIAN_FRONTEND=noninteractive apt-get install -y docker-compose-v2; elif apt-cache show docker-compose >/dev/null 2>&1; then DEBIAN_FRONTEND=noninteractive apt-get install -y docker-compose; else echo 'no docker compose package found' >&2; exit 2; fi; fi; systemctl enable --now docker || true" @@ -136,10 +155,13 @@ fi remote_run "mkdir -p '$REMOTE_ROOT/images'" run_or_echo rsync -a --delete "$compose_src" "$REMOTE_HOST:$REMOTE_ROOT/" run_or_echo rsync -a "$IMAGE_TAR" "$REMOTE_HOST:$REMOTE_ROOT/images/" +run_or_echo rsync -a "$METRICS_IMAGE_TAR" "$REMOTE_HOST:$REMOTE_ROOT/images/" remote_tar="$REMOTE_ROOT/images/$(basename "$IMAGE_TAR")" +remote_metrics_tar="$REMOTE_ROOT/images/$(basename "$METRICS_IMAGE_TAR")" remote_run "cd '$REMOTE_ROOT' && test -f .env || cp .env.example .env" remote_run "gunzip -c '$remote_tar' | docker load" +remote_run "gunzip -c '$remote_metrics_tar' | docker load" if [[ "$INSTALL_BINFMT" == "1" ]]; then remote_run "docker run --rm --privileged tonistiigi/binfmt --install arm64" @@ -147,6 +169,7 @@ fi remote_run "docker run --rm --platform linux/arm64 '$IMAGE_TAG' uname -m" remote_run "docker run --rm --platform linux/arm64 '$IMAGE_TAG' /opt/ours-rp/bin/rpki --help >/tmp/ours-rp-arm64-help.txt && head -5 /tmp/ours-rp-arm64-help.txt" +remote_run "docker run --rm --platform linux/arm64 '$METRICS_IMAGE' /opt/ours-rp/bin/rpki_artifact_metrics --help >/tmp/ours-rp-arm64-metrics-help.txt && head -5 /tmp/ours-rp-arm64-metrics-help.txt" if [[ "$START_CORE" == "1" ]]; then remote_run "cd '$REMOTE_ROOT' && docker compose --profile core up -d ours-rp-soak"