From 850ff218ce4b70170ba3a15dbb5518e13e21fd95 Mon Sep 17 00:00:00 2001 From: yuyr Date: Thu, 6 Aug 2026 10:48:38 +0800 Subject: [PATCH] add custom TAL installer patch --- deploy/docker-installer/.env.example | 15 + .../compose/docker-compose.yml | 7 + .../custom-fixtures/README.md | 14 + .../custom-tal-patch/README.zh-CN.md | 30 + .../apply_custom_tal_patch.sh | 187 +++++ .../custom-tal-patch/custom-tal.env.example | 15 + .../rollback_custom_tal_patch.sh | 80 +++ .../tools/generate_custom_fixture.py | 636 ++++++++++++++++++ .../custom-tal-patch/tools/serve_https.py | 36 + .../tools/start_fixed_services.sh | 87 +++ .../tools/stop_fixed_services.sh | 23 + scripts/docker/build_custom_tal_patch.sh | 146 ++++ scripts/soak/run_soak.sh | 56 +- 13 files changed, 1329 insertions(+), 3 deletions(-) create mode 100644 deploy/docker-installer/custom-fixtures/README.md create mode 100644 deploy/docker-installer/custom-tal-patch/README.zh-CN.md create mode 100644 deploy/docker-installer/custom-tal-patch/apply_custom_tal_patch.sh create mode 100644 deploy/docker-installer/custom-tal-patch/custom-tal.env.example create mode 100644 deploy/docker-installer/custom-tal-patch/rollback_custom_tal_patch.sh create mode 100644 deploy/docker-installer/custom-tal-patch/tools/generate_custom_fixture.py create mode 100644 deploy/docker-installer/custom-tal-patch/tools/serve_https.py create mode 100644 deploy/docker-installer/custom-tal-patch/tools/start_fixed_services.sh create mode 100644 deploy/docker-installer/custom-tal-patch/tools/stop_fixed_services.sh create mode 100644 scripts/docker/build_custom_tal_patch.sh diff --git a/deploy/docker-installer/.env.example b/deploy/docker-installer/.env.example index 78c7b11..b27ea95 100644 --- a/deploy/docker-installer/.env.example +++ b/deploy/docker-installer/.env.example @@ -37,12 +37,27 @@ RETAIN_RUNS=100 # TAL/TA input mode: # file-with-ta: use packaged fixture TAL + TA only. # file-live-ta: use packaged fixture TAL; snapshot waits for live TA refresh, delta refreshes TA in background. +# custom-file-with-ta: use one custom TAL + TA from the read-only custom fixture mount. # url: pass TAL URL to child process. TAL_INPUT_MODE=file-live-ta LIVE_TA_REFRESH_BEFORE_SNAPSHOT=1 LIVE_TA_REFRESH_CONNECT_TIMEOUT_SECS=15 LIVE_TA_REFRESH_MAX_TIME_SECS=120 +# Custom TAL patch mode. Keep TAL_INPUT_MODE=file-live-ta and the five-RIR RIRS +# list above for the original package behavior. For a private fixture, set: +# RIRS=custom +# TAL_INPUT_MODE=custom-file-with-ta +# The paths below are container paths; CUSTOM_FIXTURE_HOST_DIR is a host path +# resolved relative to the compose directory when left at its default. +CUSTOM_FIXTURE_HOST_DIR=../custom-fixtures +CUSTOM_TAL_PATH=/opt/ours-rp/custom-fixtures/tal/custom.tal +CUSTOM_TA_PATH=/opt/ours-rp/custom-fixtures/ta/custom-ta.cer +CUSTOM_TAL_URI=https://host.docker.internal:18443/tal/custom.tal +# Space-separated container paths; paths must not contain spaces. Set this in +# custom mode when the RRDP server uses a private/self-signed CA. +HTTP_ROOT_CERT_PATHS= + # Sync and runtime behavior. RSYNC_SCOPE=module-root DISABLE_COMPETING_RPS=0 diff --git a/deploy/docker-installer/compose/docker-compose.yml b/deploy/docker-installer/compose/docker-compose.yml index 85e660d..113fc92 100644 --- a/deploy/docker-installer/compose/docker-compose.yml +++ b/deploy/docker-installer/compose/docker-compose.yml @@ -11,8 +11,15 @@ services: RUN_ROOT: /var/lib/ours-rp BIN_DIR: /opt/ours-rp/bin FIXTURE_DIR: /opt/ours-rp/fixtures + CUSTOM_FIXTURE_DIR: /opt/ours-rp/custom-fixtures + extra_hosts: + - "host.docker.internal:host-gateway" volumes: - ../.env:/opt/ours-rp/.env:ro + # Overlay the patched runner while keeping the original runtime image + # and its exact rpki binaries unchanged. + - ../scripts/soak/run_soak.sh:/opt/ours-rp/run_soak.sh:ro + - ${CUSTOM_FIXTURE_HOST_DIR:-../custom-fixtures}:/opt/ours-rp/custom-fixtures:ro - ${HOST_DATA_DIR:-/var/lib/ours-rp-package-installer}/state:/var/lib/ours-rp/state - ${HOST_DATA_DIR:-/var/lib/ours-rp-package-installer}/runs:/var/lib/ours-rp/runs - ${HOST_DATA_DIR:-/var/lib/ours-rp-package-installer}/logs:/var/lib/ours-rp/logs diff --git a/deploy/docker-installer/custom-fixtures/README.md b/deploy/docker-installer/custom-fixtures/README.md new file mode 100644 index 0000000..4632409 --- /dev/null +++ b/deploy/docker-installer/custom-fixtures/README.md @@ -0,0 +1,14 @@ +# Custom TAL fixture mount + +This directory is intentionally shipped as a mount point, not as a repository +of private keys. Run the patch tool `tools/generate_custom_fixture.py` to create +the test-only TAL, TA, RPKI objects, HTTPS CA/server certificates, RRDP files, +and rsync module data below it. + +The generated fixture contains `baseline-v1`, `sync-hash-mismatch` and +`baseline-v2` for snapshot / hash / delta tests, plus +`validation-expired`, `validation-max-length`, +`validation-max-length-invalid`, `validation-roa-prefix-outside`, +`validation-out-of-resource`, `validation-ca-over-resource`, and +`validation-nonstandard` for validation cases. Generated keys are test-only +and must not be used in a production RPKI hierarchy. diff --git a/deploy/docker-installer/custom-tal-patch/README.zh-CN.md b/deploy/docker-installer/custom-tal-patch/README.zh-CN.md new file mode 100644 index 0000000..be1adff --- /dev/null +++ b/deploy/docker-installer/custom-tal-patch/README.zh-CN.md @@ -0,0 +1,30 @@ +# 自建 TAL/TA 补丁 + +本目录描述并生成一个叠加到客户 Arm64 Ours RP 组件包上的测试补丁。补丁基线是源码 commit `9f4f4cf069e9fa8f6f97ed0f1065adac38abe82f`，不会替换客户包中的 runtime image 或 `bin/rpki`、`bin/rpki_daemon` 二进制。 + +补丁做三件事： + +1. 在 runner 中增加 `RIRS=custom`、`TAL_INPUT_MODE=custom-file-with-ta`、自定义 TAL/TA 路径和 `HTTP_ROOT_CERT_PATHS`； +2. 通过 Compose 只读挂载 patched runner、`custom-fixtures/` 和 `host.docker.internal`，连接固定 HTTPS RRDP 与 rsync 服务； +3. 提供固定证书链、RPKI 对象生成器和服务启动脚本，不依赖 Barry、Rapport 或公网 RIR。 + +先停止 Ours RP，再执行： + +```bash +./apply_custom_tal_patch.sh \ + --component-root /opt/ours-rp-rtr-stack/components/ours-rp \ + --enable-custom +``` + +`--enable-custom` 会修改组件 `.env` 的 custom 模式入口；不加该选项只安装 overlay，不改变当前 `.env`。回滚： + +```bash +./rollback_custom_tal_patch.sh \ + --component-root /opt/ours-rp-rtr-stack/components/ours-rp +``` + +补丁应用脚本会校验 `PACKAGE-MANIFEST.env` 的源码 commit 和 `arm64` 架构，并留下 `CUSTOM-TAL-PATCH-MANIFEST.env` 与 `.custom-tal-patch-state/` 备份。升级组件后需要重新检查 patch marker；不要把该 overlay 误当成客户原始安装包。 + +生成器默认把容器访问地址写成 `host.docker.internal`，固定服务端口为 HTTPS RRDP `18443`、rsync `1873`，rsync module 为 `custom`。可用 `--host localhost` 生成给宿主机直接运行 `bin/rpki` 的本地变体。 + +用例包括：`baseline-v1`、`sync-hash-mismatch`、`baseline-v2`、`validation-expired`、`validation-max-length`、`validation-max-length-invalid`、`validation-roa-prefix-outside`、`validation-out-of-resource`、`validation-ca-over-resource`、`validation-nonstandard`。其中 `validation-ca-over-resource` 在 `RESOURCE_VALIDATION_MODE=rfc6487` 下验证严格拒绝；`validation-update-03` 是原包默认的兼容处理模式。 diff --git a/deploy/docker-installer/custom-tal-patch/apply_custom_tal_patch.sh b/deploy/docker-installer/custom-tal-patch/apply_custom_tal_patch.sh new file mode 100644 index 0000000..1b72dea --- /dev/null +++ b/deploy/docker-installer/custom-tal-patch/apply_custom_tal_patch.sh @@ -0,0 +1,187 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PATCH_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +PAYLOAD_ROOT="$SCRIPT_DIR/payload" +STATE_DIR_NAME=".custom-tal-patch-state" +EXPECTED_SOURCE_COMMIT="9f4f4cf069e9fa8f6f97ed0f1065adac38abe82f" + +usage() { + cat <<'USAGE' +Usage: + ./apply_custom_tal_patch.sh --component-root + ./apply_custom_tal_patch.sh --component-root --enable-custom + +The default operation installs the overlay but does not change the existing +.env. --enable-custom writes the custom TAL/TA/root-cert example values into +the component .env so the next run uses the fixed local fixture. +Stop the ours RP service before applying or rolling back the overlay. +USAGE +} + +die() { + echo "error: $*" >&2 + exit 2 +} + +COMPONENT_ROOT="" +ENABLE_CUSTOM=0 +while [[ $# -gt 0 ]]; do + case "$1" in + --component-root) + COMPONENT_ROOT="${2:-}" + shift 2 + ;; + --enable-custom) + ENABLE_CUSTOM=1 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown option: $1" + ;; + esac +done + +[[ -n "$COMPONENT_ROOT" ]] || die "--component-root is required" +COMPONENT_ROOT="$(cd "$COMPONENT_ROOT" 2>/dev/null && pwd)" \ + || die "component root does not exist: $COMPONENT_ROOT" +[[ -f "$COMPONENT_ROOT/PACKAGE-MANIFEST.env" ]] \ + || die "missing component PACKAGE-MANIFEST.env: $COMPONENT_ROOT" +[[ -d "$PAYLOAD_ROOT" ]] || die "missing patch payload: $PAYLOAD_ROOT" + +# shellcheck disable=SC1091 +source "$COMPONENT_ROOT/PACKAGE-MANIFEST.env" +[[ "${source_commit:-}" == "$EXPECTED_SOURCE_COMMIT" ]] \ + || die "base source commit mismatch: ${source_commit:-missing} != $EXPECTED_SOURCE_COMMIT" +[[ "${package_arch:-${PACKAGE_ARCH:-}}" == "arm64" ]] \ + || die "this patch is for the customer Arm64 component, got ${package_arch:-${PACKAGE_ARCH:-missing}}" + +STATE_ROOT="$COMPONENT_ROOT/$STATE_DIR_NAME" +BACKUP_ROOT="$STATE_ROOT/original" +marker="$COMPONENT_ROOT/CUSTOM-TAL-PATCH-MANIFEST.env" + +set_env_value() { + local env_path="$1" + local key="$2" + local value="$3" + local tmp_path="${env_path}.custom-tal-patch.tmp" + [[ -f "$env_path" ]] || die "missing component .env: $env_path" + awk -v key="$key" -v value="$value" ' + BEGIN { found = 0 } + $0 ~ "^" key "=" { print key "=" value; found = 1; next } + { print } + END { if (!found) print key "=" value } + ' "$env_path" > "$tmp_path" + mv "$tmp_path" "$env_path" +} + +copy_payload_file() { + local source_path="$1" + local target_path="$2" + mkdir -p "$(dirname "$target_path")" + cp "$source_path" "$target_path" + chmod --reference="$source_path" "$target_path" 2>/dev/null || true +} + +append_custom_env_example() { + local target_path="$1" + if ! grep -Eq '^CUSTOM_TAL_URI=' "$target_path"; then + { + printf '\n# #142 custom TAL patch settings; original package values above are preserved.\n' + cat "$SCRIPT_DIR/custom-tal.env.example" + } >> "$target_path" + fi +} + +if [[ -e "$STATE_ROOT" ]]; then + die "patch is already applied or an incomplete state exists: $STATE_ROOT" +fi + +mkdir -p "$BACKUP_ROOT" +for relative_path in compose/docker-compose.yml .env.example; do + [[ -f "$COMPONENT_ROOT/$relative_path" ]] \ + || die "missing base file: $COMPONENT_ROOT/$relative_path" + mkdir -p "$BACKUP_ROOT/$(dirname "$relative_path")" + cp -a "$COMPONENT_ROOT/$relative_path" "$BACKUP_ROOT/$relative_path" +done + +runner_path="$COMPONENT_ROOT/scripts/soak/run_soak.sh" +if [[ -e "$runner_path" ]]; then + mkdir -p "$BACKUP_ROOT/scripts/soak" + cp -a "$runner_path" "$BACKUP_ROOT/scripts/soak/run_soak.sh" + printf 'runner_was_present=1\n' > "$STATE_ROOT/state.env" +else + printf 'runner_was_present=0\n' > "$STATE_ROOT/state.env" +fi + +if [[ -d "$COMPONENT_ROOT/custom-fixtures" ]]; then + cp -a "$COMPONENT_ROOT/custom-fixtures" "$BACKUP_ROOT/custom-fixtures" + printf 'custom_fixtures_were_present=1\n' >> "$STATE_ROOT/state.env" +else + printf 'custom_fixtures_were_present=0\n' >> "$STATE_ROOT/state.env" +fi + +if [[ -d "$COMPONENT_ROOT/custom-tal-patch" ]]; then + cp -a "$COMPONENT_ROOT/custom-tal-patch" "$BACKUP_ROOT/custom-tal-patch" + printf 'patch_tools_were_present=1\n' >> "$STATE_ROOT/state.env" +else + printf 'patch_tools_were_present=0\n' >> "$STATE_ROOT/state.env" +fi + +if [[ -f "$COMPONENT_ROOT/custom-tal.env.example" ]]; then + cp -a "$COMPONENT_ROOT/custom-tal.env.example" "$BACKUP_ROOT/custom-tal.env.example" + printf 'custom_env_example_was_present=1\n' >> "$STATE_ROOT/state.env" +else + printf 'custom_env_example_was_present=0\n' >> "$STATE_ROOT/state.env" +fi + +if [[ "$ENABLE_CUSTOM" == "1" ]]; then + [[ -f "$COMPONENT_ROOT/.env" ]] || die "missing component .env: $COMPONENT_ROOT/.env" + cp -a "$COMPONENT_ROOT/.env" "$BACKUP_ROOT/.env" +fi + +copy_payload_file "$PAYLOAD_ROOT/compose/docker-compose.yml" "$COMPONENT_ROOT/compose/docker-compose.yml" +append_custom_env_example "$COMPONENT_ROOT/.env.example" +copy_payload_file "$PAYLOAD_ROOT/scripts/soak/run_soak.sh" "$runner_path" +mkdir -p "$COMPONENT_ROOT/custom-fixtures/tal" \ + "$COMPONENT_ROOT/custom-fixtures/ta" \ + "$COMPONENT_ROOT/custom-fixtures/certs" \ + "$COMPONENT_ROOT/custom-fixtures/services" +if [[ -d "$PAYLOAD_ROOT/custom-fixtures" ]]; then + cp -a "$PAYLOAD_ROOT/custom-fixtures/." "$COMPONENT_ROOT/custom-fixtures/" +fi +mkdir -p "$COMPONENT_ROOT/custom-tal-patch/tools" +cp -a "$PAYLOAD_ROOT/tools/." "$COMPONENT_ROOT/custom-tal-patch/tools/" +copy_payload_file "$SCRIPT_DIR/custom-tal.env.example" \ + "$COMPONENT_ROOT/custom-tal.env.example" + +if [[ "$ENABLE_CUSTOM" == "1" ]]; then + set_env_value "$COMPONENT_ROOT/.env" RIRS custom + set_env_value "$COMPONENT_ROOT/.env" TAL_INPUT_MODE custom-file-with-ta + set_env_value "$COMPONENT_ROOT/.env" CUSTOM_FIXTURE_HOST_DIR ../custom-fixtures + set_env_value "$COMPONENT_ROOT/.env" CUSTOM_TAL_PATH /opt/ours-rp/custom-fixtures/tal/custom.tal + set_env_value "$COMPONENT_ROOT/.env" CUSTOM_TA_PATH /opt/ours-rp/custom-fixtures/ta/custom-ta.cer + set_env_value "$COMPONENT_ROOT/.env" CUSTOM_TAL_URI https://host.docker.internal:18443/tal/custom.tal + set_env_value "$COMPONENT_ROOT/.env" HTTP_ROOT_CERT_PATHS /opt/ours-rp/custom-fixtures/certs/rrdp-ca.pem + printf 'env_was_enabled=1\n' >> "$STATE_ROOT/state.env" +else + printf 'env_was_enabled=0\n' >> "$STATE_ROOT/state.env" +fi + +[[ -f "$SCRIPT_DIR/PATCH-MANIFEST.env" ]] \ + || die "missing PATCH-MANIFEST.env" +cp "$SCRIPT_DIR/PATCH-MANIFEST.env" "$marker" +printf 'patch_root=%s\n' "$PATCH_ROOT" >> "$STATE_ROOT/state.env" +printf 'applied_at_utc=%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$STATE_ROOT/state.env" + +echo "custom TAL patch applied to $COMPONENT_ROOT" +if [[ "$ENABLE_CUSTOM" == "1" ]]; then + echo "custom mode enabled in $COMPONENT_ROOT/.env" +else + echo "custom mode remains disabled; review custom-tal.env.example before enabling it" +fi diff --git a/deploy/docker-installer/custom-tal-patch/custom-tal.env.example b/deploy/docker-installer/custom-tal-patch/custom-tal.env.example new file mode 100644 index 0000000..4215267 --- /dev/null +++ b/deploy/docker-installer/custom-tal-patch/custom-tal.env.example @@ -0,0 +1,15 @@ +# Copy these values into the component .env after applying the patch. +# The patch's --enable-custom option writes the same values automatically. +RIRS=custom +TAL_INPUT_MODE=custom-file-with-ta +CUSTOM_FIXTURE_HOST_DIR=../custom-fixtures +CUSTOM_TAL_PATH=/opt/ours-rp/custom-fixtures/tal/custom.tal +CUSTOM_TA_PATH=/opt/ours-rp/custom-fixtures/ta/custom-ta.cer +CUSTOM_TAL_URI=https://host.docker.internal:18443/tal/custom.tal +HTTP_ROOT_CERT_PATHS=/opt/ours-rp/custom-fixtures/certs/rrdp-ca.pem + +# Recommended for a finite acceptance run; keep the customer default for a +# long-running test only after the fixture service is supervised. +MAX_RUNS=1 +INTERVAL_SECS=0 +SOAK_RESTART_POLICY=no diff --git a/deploy/docker-installer/custom-tal-patch/rollback_custom_tal_patch.sh b/deploy/docker-installer/custom-tal-patch/rollback_custom_tal_patch.sh new file mode 100644 index 0000000..f7f386e --- /dev/null +++ b/deploy/docker-installer/custom-tal-patch/rollback_custom_tal_patch.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'USAGE' +Usage: + ./rollback_custom_tal_patch.sh --component-root + +Stop the ours RP service before rolling back the overlay. +USAGE +} + +die() { + echo "error: $*" >&2 + exit 2 +} + +COMPONENT_ROOT="" +while [[ $# -gt 0 ]]; do + case "$1" in + --component-root) + COMPONENT_ROOT="${2:-}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + die "unknown option: $1" + ;; + esac +done + +[[ -n "$COMPONENT_ROOT" ]] || die "--component-root is required" +COMPONENT_ROOT="$(cd "$COMPONENT_ROOT" 2>/dev/null && pwd)" \ + || die "component root does not exist: $COMPONENT_ROOT" +STATE_ROOT="$COMPONENT_ROOT/.custom-tal-patch-state" +BACKUP_ROOT="$STATE_ROOT/original" +[[ -d "$STATE_ROOT" ]] || die "patch state not found: $STATE_ROOT" +[[ -f "$BACKUP_ROOT/compose/docker-compose.yml" ]] \ + || die "patch backup is incomplete: $BACKUP_ROOT" + +cp -a "$BACKUP_ROOT/compose/docker-compose.yml" "$COMPONENT_ROOT/compose/docker-compose.yml" +cp -a "$BACKUP_ROOT/.env.example" "$COMPONENT_ROOT/.env.example" + +if [[ -f "$BACKUP_ROOT/scripts/soak/run_soak.sh" ]]; then + mkdir -p "$COMPONENT_ROOT/scripts/soak" + cp -a "$BACKUP_ROOT/scripts/soak/run_soak.sh" "$COMPONENT_ROOT/scripts/soak/run_soak.sh" +else + rm -f "$COMPONENT_ROOT/scripts/soak/run_soak.sh" + rmdir "$COMPONENT_ROOT/scripts/soak" 2>/dev/null || true +fi + +if [[ -d "$BACKUP_ROOT/custom-fixtures" ]]; then + rm -rf "$COMPONENT_ROOT/custom-fixtures" + cp -a "$BACKUP_ROOT/custom-fixtures" "$COMPONENT_ROOT/custom-fixtures" +else + rm -rf "$COMPONENT_ROOT/custom-fixtures" +fi + +if [[ -d "$BACKUP_ROOT/custom-tal-patch" ]]; then + rm -rf "$COMPONENT_ROOT/custom-tal-patch" + cp -a "$BACKUP_ROOT/custom-tal-patch" "$COMPONENT_ROOT/custom-tal-patch" +else + rm -rf "$COMPONENT_ROOT/custom-tal-patch" +fi + +if [[ -f "$BACKUP_ROOT/custom-tal.env.example" ]]; then + cp -a "$BACKUP_ROOT/custom-tal.env.example" "$COMPONENT_ROOT/custom-tal.env.example" +else + rm -f "$COMPONENT_ROOT/custom-tal.env.example" +fi + +if [[ -f "$BACKUP_ROOT/.env" ]]; then + cp -a "$BACKUP_ROOT/.env" "$COMPONENT_ROOT/.env" +fi +rm -f "$COMPONENT_ROOT/CUSTOM-TAL-PATCH-MANIFEST.env" +rm -rf "$STATE_ROOT" +echo "custom TAL patch rolled back from $COMPONENT_ROOT" diff --git a/deploy/docker-installer/custom-tal-patch/tools/generate_custom_fixture.py b/deploy/docker-installer/custom-tal-patch/tools/generate_custom_fixture.py new file mode 100644 index 0000000..a0885d3 --- /dev/null +++ b/deploy/docker-installer/custom-tal-patch/tools/generate_custom_fixture.py @@ -0,0 +1,636 @@ +#!/usr/bin/env python3 +"""Generate a small self-contained RPKI repository for installer tests. + +The generator uses Python cryptography and the OpenSSL CLI for CMS signing. +It does not invoke Barry, Rapport, or a public RIR service. +""" +from __future__ import annotations + +import argparse +import base64 +import hashlib +import ipaddress +import shutil +import subprocess +import tempfile +from datetime import datetime, timedelta, timezone +from pathlib import Path + +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.hazmat.primitives.serialization import Encoding, NoEncryption, PrivateFormat, PublicFormat +from cryptography.x509 import AccessDescription, AuthorityInformationAccess, DistributionPoint, UniformResourceIdentifier +from cryptography.x509.oid import AuthorityInformationAccessOID, NameOID, ObjectIdentifier + + +RRDP_HOST = "host.docker.internal" +RRDP_PORT = 18443 +RSYNC_PORT = 1873 +TAL_URI = f"rsync://{RRDP_HOST}:{RSYNC_PORT}/custom/ta.cer" +TAL_HTTPS_URI = f"https://{RRDP_HOST}:{RRDP_PORT}/ta/custom-ta.cer" +RRDP_URI = f"https://{RRDP_HOST}:{RRDP_PORT}/rrdp/notification.xml" +ROOT_REPO_URI = f"rsync://{RRDP_HOST}:{RSYNC_PORT}/custom/root/" +ROOT_MFT_URI = f"{ROOT_REPO_URI}root.mft" +CHILD_REPO_URI = f"rsync://{RRDP_HOST}:{RSYNC_PORT}/custom/child/" +CHILD_MFT_URI = f"{CHILD_REPO_URI}child.mft" +OID_SIA = ObjectIdentifier("1.3.6.1.5.5.7.1.11") +OID_RESOURCES_IP = ObjectIdentifier("1.3.6.1.5.5.7.1.7") +OID_RESOURCES_AS = ObjectIdentifier("1.3.6.1.5.5.7.1.8") +OID_POLICY_IP_AS = ObjectIdentifier("1.3.6.1.5.5.7.14.2") +OID_ROA_ECONTENT = "1.2.840.113549.1.9.16.1.24" +OID_MANIFEST_ECONTENT = "1.2.840.113549.1.9.16.1.26" +OID_SHA256 = "2.16.840.1.101.3.4.2.1" +SIA_CA_REPOSITORY = "1.3.6.1.5.5.7.48.5" +SIA_RPKI_MANIFEST = "1.3.6.1.5.5.7.48.10" +SIA_SIGNED_OBJECT = "1.3.6.1.5.5.7.48.11" +SIA_RPKI_NOTIFY = "1.3.6.1.5.5.7.48.13" +UTC = timezone.utc +VALID_FROM = datetime(2026, 1, 1, tzinfo=UTC) +VALID_TO = datetime(2027, 12, 31, 23, 59, 59, tzinfo=UTC) +CHILD_PREFIX = ipaddress.ip_network("203.0.113.0/24") +SECOND_PREFIX = ipaddress.ip_network("203.0.113.128/25") +OUTSIDE_PREFIX = ipaddress.ip_network("203.0.114.0/24") +TEST_ASN = 64496 +SESSION_ID = "11111111-2222-4333-8444-555555555555" + + +def tlv(tag: int, value: bytes) -> bytes: + length = len(value) + if length < 128: + encoded_length = bytes([length]) + else: + raw = length.to_bytes((length.bit_length() + 7) // 8, "big") + encoded_length = bytes([0x80 | len(raw)]) + raw + return bytes([tag]) + encoded_length + value + + +def seq(*values: bytes) -> bytes: + return tlv(0x30, b"".join(values)) + + +def integer(value: int) -> bytes: + raw = value.to_bytes(max(1, (value.bit_length() + 7) // 8), "big") + if raw[0] & 0x80: + raw = b"\x00" + raw + return tlv(0x02, raw) + + +def oid(oid_value: str) -> bytes: + parts = [int(part) for part in oid_value.split(".")] + encoded = bytearray([40 * parts[0] + parts[1]]) + for part in parts[2:]: + chunks = [part & 0x7F] + part >>= 7 + while part: + chunks.append(0x80 | (part & 0x7F)) + part >>= 7 + encoded.extend(reversed(chunks)) + return tlv(0x06, bytes(encoded)) + + +def octet(value: bytes) -> bytes: + return tlv(0x04, value) + + +def uri(value: str) -> bytes: + return tlv(0x86, value.encode("ascii")) + + +def bit_string(value: bytes, unused: int = 0) -> bytes: + return tlv(0x03, bytes([unused]) + value) + + +def generalized_time(value: datetime) -> bytes: + return tlv(0x18, value.astimezone(UTC).strftime("%Y%m%d%H%M%SZ").encode("ascii")) + + +def context_zero(value: bytes) -> bytes: + return tlv(0xA0, value) + + +def ip_choice(networks: list[ipaddress._BaseNetwork]) -> bytes: + entries = [] + for network in networks: + length = (network.prefixlen + 7) // 8 + raw = network.network_address.packed[:length] + entries.append(bit_string(raw, length * 8 - network.prefixlen)) + return seq(*entries) + + +def ip_resources(networks: list[ipaddress._BaseNetwork]) -> bytes: + families: dict[int, list[ipaddress._BaseNetwork]] = {1: [], 2: []} + for network in networks: + families[1 if network.version == 4 else 2].append(network) + return seq( + *( + seq(octet(afi.to_bytes(2, "big")), ip_choice(items)) + for afi, items in families.items() + if items + ) + ) + + +def ip_inherit() -> bytes: + return seq( + seq(octet(b"\x00\x01"), tlv(0x05, b"")), + seq(octet(b"\x00\x02"), tlv(0x05, b"")), + ) + + +def as_resources(asns: list[int] | None) -> bytes: + if asns is None: + return seq(context_zero(tlv(0x05, b""))) + return seq(context_zero(seq(*(integer(asn) for asn in asns)))) + + +def sia(entries: list[tuple[str, str]]) -> bytes: + return seq(*(seq(oid(method), uri(location)) for method, location in entries)) + + +def write(path: Path, data: bytes, mode: int | None = None) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + if mode is not None: + path.chmod(mode) + + +def pem_cert(cert: x509.Certificate) -> bytes: + return cert.public_bytes(Encoding.PEM) + + +def pem_key(key: rsa.RSAPrivateKey) -> bytes: + return key.private_bytes(Encoding.PEM, PrivateFormat.TraditionalOpenSSL, NoEncryption()) + + +def write_key(path: Path, key: rsa.RSAPrivateKey) -> None: + write(path, pem_key(key), 0o600) + + +def sign_cert( + subject: x509.Name, + issuer: x509.Name, + public_key, + signing_key: rsa.RSAPrivateKey, + serial: int, + is_ca: bool, + ski_key: rsa.RSAPrivateKey, + aki_key: rsa.RSAPrivateKey | None = None, + ip_ext: bytes | None = None, + as_ext: bytes | None = None, + ca_repository: str | None = None, + manifest_uri: str | None = None, + notify_uri: str | None = None, + signed_object_uri: str | None = None, + crl_uri: str | None = None, + ca_issuer_uri: str | None = None, + not_before: datetime = VALID_FROM, + not_after: datetime = VALID_TO, +) -> x509.Certificate: + builder = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(issuer) + .public_key(public_key) + .serial_number(serial) + .not_valid_before(not_before) + .not_valid_after(not_after) + ) + if is_ca: + builder = builder.add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True) + if is_ca: + usage = x509.KeyUsage(False, False, False, False, False, True, True, False, False) + else: + usage = x509.KeyUsage(True, False, False, False, False, False, False, False, False) + builder = builder.add_extension(usage, critical=True) + builder = builder.add_extension(x509.SubjectKeyIdentifier.from_public_key(ski_key.public_key()), critical=False) + if aki_key is not None: + builder = builder.add_extension(x509.AuthorityKeyIdentifier.from_issuer_public_key(aki_key.public_key()), critical=False) + builder = builder.add_extension( + x509.CertificatePolicies([x509.PolicyInformation(OID_POLICY_IP_AS, None)]), + critical=True, + ) + if crl_uri is not None: + builder = builder.add_extension( + x509.CRLDistributionPoints( + [DistributionPoint([UniformResourceIdentifier(crl_uri)], None, None, None)] + ), + critical=False, + ) + if ca_issuer_uri is not None: + builder = builder.add_extension( + AuthorityInformationAccess( + [AccessDescription(AuthorityInformationAccessOID.CA_ISSUERS, UniformResourceIdentifier(ca_issuer_uri))] + ), + critical=False, + ) + sia_entries: list[tuple[str, str]] = [] + if ca_repository is not None: + sia_entries.append((SIA_CA_REPOSITORY, ca_repository)) + if manifest_uri is not None: + sia_entries.append((SIA_RPKI_MANIFEST, manifest_uri)) + if notify_uri is not None: + sia_entries.append((SIA_RPKI_NOTIFY, notify_uri)) + if signed_object_uri is not None: + sia_entries.append((SIA_SIGNED_OBJECT, signed_object_uri)) + if sia_entries: + builder = builder.add_extension(x509.UnrecognizedExtension(OID_SIA, sia(sia_entries)), critical=False) + if ip_ext is not None: + builder = builder.add_extension(x509.UnrecognizedExtension(OID_RESOURCES_IP, ip_ext), critical=True) + if as_ext is not None: + builder = builder.add_extension(x509.UnrecognizedExtension(OID_RESOURCES_AS, as_ext), critical=True) + return builder.sign(signing_key, hashes.SHA256()) + + +def make_crl(subject: x509.Name, key: rsa.RSAPrivateKey, number: int) -> x509.CertificateRevocationList: + return ( + x509.CertificateRevocationListBuilder() + .issuer_name(subject) + .last_update(VALID_FROM + timedelta(days=30)) + .next_update(VALID_TO - timedelta(days=30)) + .add_extension(x509.AuthorityKeyIdentifier.from_issuer_public_key(key.public_key()), critical=False) + .add_extension(x509.CRLNumber(number), critical=False) + .sign(key, hashes.SHA256()) + ) + + +def roa_content(prefix: ipaddress._BaseNetwork, max_length: int) -> bytes: + length = (prefix.prefixlen + 7) // 8 + raw = prefix.network_address.packed[:length] + address = seq(bit_string(raw, length * 8 - prefix.prefixlen), integer(max_length)) + family = seq(octet((1 if prefix.version == 4 else 2).to_bytes(2, "big")), seq(address)) + return seq(integer(TEST_ASN), seq(family)) + + +def manifest_content(number: int, files: list[tuple[str, bytes]]) -> bytes: + entries = [ + seq(tlv(0x16, name.encode("ascii")), bit_string(hashlib.sha256(content).digest())) + for name, content in files + ] + return seq( + integer(number), + generalized_time(VALID_FROM + timedelta(days=30)), + generalized_time(VALID_TO - timedelta(days=30)), + oid(OID_SHA256), + seq(*entries), + ) + + +def write_cms(content: bytes, content_type: str, signer_cert: x509.Certificate, signer_key: rsa.RSAPrivateKey, output: Path, work: Path) -> None: + content_path = work / f"{output.name}.content.der" + signer_path = work / f"{output.name}.signer.pem" + key_path = work / f"{output.name}.key.pem" + write(content_path, content) + write(signer_path, pem_cert(signer_cert)) + write_key(key_path, signer_key) + subprocess.run( + [ + "openssl", "cms", "-sign", "-binary", "-in", str(content_path), + "-signer", str(signer_path), "-inkey", str(key_path), + "-outform", "DER", "-nodetach", "-nosmimecap", "-keyid", "-md", "sha256", + "-econtent_type", content_type, "-out", str(output), + ], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + ) + + +def make_roa( + name: str, + prefix, + max_length: int, + not_before: datetime, + not_after: datetime, + child_name, + child_key, + common: Path, + work: Path, + certificate_prefix=None, +) -> None: + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + cert = sign_cert( + x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, f"#142 EE {name}")]), + child_name, + key.public_key(), + child_key, + x509.random_serial_number(), + False, + key, + child_key, + ip_ext=ip_resources([certificate_prefix or prefix]), + signed_object_uri=f"{CHILD_REPO_URI}{name}", + crl_uri=f"{CHILD_REPO_URI}child.crl", + ca_issuer_uri=f"{CHILD_REPO_URI}child.cer", + not_before=not_before, + not_after=not_after, + ) + write_cms(roa_content(prefix, max_length), OID_ROA_ECONTENT, cert, key, common / name, work) + + +def make_manifest(name: str, number: int, files: list[tuple[str, bytes]], child_name, child_key, common: Path, work: Path) -> bytes: + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + cert = sign_cert( + x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, f"#142 Manifest EE {name}")]), + child_name, + key.public_key(), + child_key, + x509.random_serial_number(), + False, + key, + child_key, + ip_ext=ip_inherit(), + as_ext=as_resources(None), + signed_object_uri=CHILD_MFT_URI if name != "root" else ROOT_MFT_URI, + crl_uri=f"{CHILD_REPO_URI}child.crl" if name != "root" else f"{ROOT_REPO_URI}root.crl", + ca_issuer_uri=f"{CHILD_REPO_URI}child.cer" if name != "root" else f"{ROOT_REPO_URI}ta.cer", + ) + output = common / f"{name}.mft" + write_cms(manifest_content(number, files), OID_MANIFEST_ECONTENT, cert, key, output, work) + return output.read_bytes() + + +def publish(uri: str, path: Path) -> str: + encoded = base64.b64encode(path.read_bytes()).decode("ascii") + return f' {encoded}\n' + + +def snapshot(case_root: Path, names: list[str], serial: int, extra_root_names: list[str] | None = None) -> bytes: + repository = case_root / "repository" + objects = [ + (f"{ROOT_REPO_URI}root.crl", repository / "root/root.crl"), + (f"{ROOT_REPO_URI}child.cer", repository / "root/child.cer"), + *[ + (f"{ROOT_REPO_URI}{name}", repository / f"root/{name}") + for name in extra_root_names or [] + ], + (f"{CHILD_REPO_URI}child.crl", repository / "child/child.crl"), + *[(f"{CHILD_REPO_URI}{name}", repository / f"child/{name}") for name in names], + (ROOT_MFT_URI, repository / "root/root.mft"), + (CHILD_MFT_URI, repository / "child/child.mft"), + ] + body = "".join(publish(uri, path) for uri, path in objects) + return ( + f'\n' + f"{body}\n" + ).encode() + + +def notification(serial: int, snap: bytes, delta: bytes | None) -> bytes: + lines = [ + f'', + f' ', + ] + if delta is not None: + lines.append( + f' ' + ) + return ("\n".join(lines) + "\n\n").encode() + + +def corrupt_first_hash(xml: bytes) -> bytes: + marker = b'hash="' + start = xml.index(marker) + len(marker) + end = xml.index(b'"', start) + value = bytearray(xml[start:end]) + value[0] = ord("0") if value[0] != ord("0") else ord("1") + return xml[:start] + bytes(value) + xml[end:] + + +def delta(add_name: str, add_path: Path, new_mft: Path) -> bytes: + body = publish(CHILD_MFT_URI, new_mft) + body += publish(f"{CHILD_REPO_URI}{add_name}", add_path) + return f'\n{body}\n'.encode() + + +def build_case( + output: Path, + common: Path, + tal: bytes, + ta: bytes, + name: str, + number: int, + names: list[str], + child_mft: bytes, + change: bytes | None = None, + root_mft: bytes | None = None, + extra_root_objects: list[tuple[str, bytes]] | None = None, + notification_hash_mismatch: bool = False, +) -> None: + case_root = output / "cases" / name + repo = case_root / "repository" + (repo / "root").mkdir(parents=True, exist_ok=True) + (repo / "child").mkdir(parents=True, exist_ok=True) + shutil.copy2(common / "ta.cer", repo / "ta.cer") + for item in ("root.crl", "child.cer"): + shutil.copy2(common / item, repo / "root" / item) + (repo / "root/root.mft").write_bytes(root_mft if root_mft is not None else (common / "root.mft").read_bytes()) + for item, content in extra_root_objects or []: + (repo / "root" / item).write_bytes(content) + for item in ( + "child.crl", "valid.roa", "valid2.roa", "expired.roa", "max-length.roa", + "max-length-invalid.roa", "roa-prefix-outside.roa", "out-of-resource.roa", "bad-format.roa", + ): + if (common / item).exists(): + shutil.copy2(common / item, repo / "child" / item) + (repo / "child/child.mft").write_bytes(child_mft) + snap = snapshot(case_root, names, number, [item for item, _ in extra_root_objects or []]) + (case_root / "http/rrdp").mkdir(parents=True, exist_ok=True) + (case_root / "http/tal").mkdir(parents=True, exist_ok=True) + (case_root / "http/ta").mkdir(parents=True, exist_ok=True) + (case_root / "http/rrdp/snapshot.xml").write_bytes(snap) + notification_xml = notification(number, snap, change) + if notification_hash_mismatch: + notification_xml = corrupt_first_hash(notification_xml) + (case_root / "http/rrdp/notification.xml").write_bytes(notification_xml) + if change is not None: + (case_root / "http/rrdp/delta-2.xml").write_bytes(change) + (case_root / "http/tal/custom.tal").write_bytes(tal) + (case_root / "http/ta/custom-ta.cer").write_bytes(ta) + (case_root / "CASE-MANIFEST.txt").write_text( + f"case={name}\nserial={number}\nobjects={','.join(names)}\nsnapshot_sha256={hashlib.sha256(snap).hexdigest()}\n", + encoding="utf-8", + ) + + +def make_https_certificates(output: Path) -> None: + ca_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + ca_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "#142 fixed RRDP test CA")]) + ca_cert = ( + x509.CertificateBuilder() + .subject_name(ca_name).issuer_name(ca_name).public_key(ca_key.public_key()) + .serial_number(x509.random_serial_number()).not_valid_before(VALID_FROM).not_valid_after(VALID_TO) + .add_extension(x509.BasicConstraints(ca=True, path_length=1), critical=True) + .add_extension(x509.KeyUsage(False, False, False, False, False, True, True, False, False), critical=True) + .sign(ca_key, hashes.SHA256()) + ) + server_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + server_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, RRDP_HOST)]) + try: + server_host = x509.IPAddress(ipaddress.ip_address(RRDP_HOST)) + except ValueError: + server_host = x509.DNSName(RRDP_HOST) + san_entries = [server_host] + if RRDP_HOST != "localhost": + san_entries.append(x509.DNSName("localhost")) + server_cert = ( + x509.CertificateBuilder() + .subject_name(server_name).issuer_name(ca_cert.subject).public_key(server_key.public_key()) + .serial_number(x509.random_serial_number()).not_valid_before(VALID_FROM).not_valid_after(VALID_TO) + .add_extension(x509.BasicConstraints(ca=False, path_length=None), critical=True) + .add_extension(x509.KeyUsage(True, False, False, False, False, False, False, False, False), critical=True) + .add_extension(x509.SubjectAlternativeName(san_entries), critical=False) + .sign(ca_key, hashes.SHA256()) + ) + write(output / "certs/rrdp-ca.pem", pem_cert(ca_cert)) + write_key(output / "certs/rrdp-ca.key", ca_key) + write(output / "certs/rrdp-server.pem", pem_cert(server_cert)) + write_key(output / "certs/rrdp-server.key", server_key) + + +def main() -> None: + global RRDP_HOST, TAL_URI, TAL_HTTPS_URI, RRDP_URI + global ROOT_REPO_URI, ROOT_MFT_URI, CHILD_REPO_URI, CHILD_MFT_URI + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path, required=True) + parser.add_argument( + "--host", + default=RRDP_HOST, + help="host embedded in TAL, certificate SIA and RRDP data (default: host.docker.internal)", + ) + args = parser.parse_args() + RRDP_HOST = args.host + TAL_URI = f"rsync://{RRDP_HOST}:{RSYNC_PORT}/custom/ta.cer" + TAL_HTTPS_URI = f"https://{RRDP_HOST}:{RRDP_PORT}/ta/custom-ta.cer" + RRDP_URI = f"https://{RRDP_HOST}:{RRDP_PORT}/rrdp/notification.xml" + ROOT_REPO_URI = f"rsync://{RRDP_HOST}:{RSYNC_PORT}/custom/root/" + ROOT_MFT_URI = f"{ROOT_REPO_URI}root.mft" + CHILD_REPO_URI = f"rsync://{RRDP_HOST}:{RSYNC_PORT}/custom/child/" + CHILD_MFT_URI = f"{CHILD_REPO_URI}child.mft" + output = args.output.resolve() + if output.exists(): + shutil.rmtree(output) + for directory in ("tal", "ta", "certs", "services", "keys"): + (output / directory).mkdir(parents=True) + + root_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + child_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + root_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "#142 Custom RPKI Root")]) + child_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "#142 Custom RPKI Child")]) + root_cert = sign_cert( + root_name, root_name, root_key.public_key(), root_key, 0x14200001, True, root_key, + ip_ext=ip_resources([CHILD_PREFIX]), + as_ext=as_resources([TEST_ASN]), ca_repository=ROOT_REPO_URI, manifest_uri=ROOT_MFT_URI, + notify_uri=RRDP_URI, + ) + child_cert = sign_cert( + child_name, root_name, child_key.public_key(), root_key, 0x14200002, True, child_key, root_key, + ip_ext=ip_resources([CHILD_PREFIX]), as_ext=as_resources([TEST_ASN]), + ca_repository=CHILD_REPO_URI, manifest_uri=CHILD_MFT_URI, notify_uri=RRDP_URI, + crl_uri=f"{ROOT_REPO_URI}root.crl", ca_issuer_uri=f"{ROOT_REPO_URI}child.cer", + ) + bad_child_name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "#142 Custom RPKI Overclaiming Child")]) + bad_child_repo_uri = f"rsync://{RRDP_HOST}:{RSYNC_PORT}/custom/bad-child/" + bad_child_cert = sign_cert( + bad_child_name, root_name, child_key.public_key(), root_key, 0x14200003, True, child_key, root_key, + ip_ext=ip_resources([OUTSIDE_PREFIX]), as_ext=as_resources([TEST_ASN]), + ca_repository=bad_child_repo_uri, manifest_uri=f"{bad_child_repo_uri}bad-child.mft", notify_uri=RRDP_URI, + crl_uri=f"{ROOT_REPO_URI}root.crl", ca_issuer_uri=f"{ROOT_REPO_URI}child.cer", + ) + root_crl = make_crl(root_name, root_key, 1) + child_crl = make_crl(child_name, child_key, 1) + tal = ( + f"{TAL_URI}\n{TAL_HTTPS_URI}\n\n" + + base64.b64encode(root_key.public_key().public_bytes(Encoding.DER, PublicFormat.SubjectPublicKeyInfo)).decode() + + "\n" + ).encode() + write(output / "tal/custom.tal", tal) + write(output / "ta/custom-ta.cer", root_cert.public_bytes(Encoding.DER)) + with tempfile.TemporaryDirectory(prefix="custom-rpki-cms-") as temp: + work = Path(temp) + common = output / ".common" + common.mkdir() + write(common / "ta.cer", root_cert.public_bytes(Encoding.DER)) + write(common / "child.cer", child_cert.public_bytes(Encoding.DER)) + write(common / "bad-child.cer", bad_child_cert.public_bytes(Encoding.DER)) + write(common / "root.crl", root_crl.public_bytes(Encoding.DER)) + write(common / "child.crl", child_crl.public_bytes(Encoding.DER)) + for name, prefix, max_length, before, after, certificate_prefix in ( + ("valid.roa", CHILD_PREFIX, 24, VALID_FROM + timedelta(days=31), VALID_TO - timedelta(days=31), None), + ("valid2.roa", SECOND_PREFIX, 25, VALID_FROM + timedelta(days=31), VALID_TO - timedelta(days=31), None), + ("expired.roa", CHILD_PREFIX, 24, VALID_FROM - timedelta(days=30), VALID_FROM - timedelta(days=1), None), + ("max-length.roa", CHILD_PREFIX, 25, VALID_FROM + timedelta(days=31), VALID_TO - timedelta(days=31), None), + ("max-length-invalid.roa", CHILD_PREFIX, 33, VALID_FROM + timedelta(days=31), VALID_TO - timedelta(days=31), None), + ("roa-prefix-outside.roa", OUTSIDE_PREFIX, 24, VALID_FROM + timedelta(days=31), VALID_TO - timedelta(days=31), CHILD_PREFIX), + ("out-of-resource.roa", OUTSIDE_PREFIX, 24, VALID_FROM + timedelta(days=31), VALID_TO - timedelta(days=31), None), + ): + make_roa( + name, prefix, max_length, before, after, child_name, child_key, common, work, + certificate_prefix=certificate_prefix, + ) + bad = bytearray((common / "valid.roa").read_bytes()) + bad[-1] ^= 1 + write(common / "bad-format.roa", bytes(bad)) + + root_mft = make_manifest( + "root", 1, [ + ("root.crl", (common / "root.crl").read_bytes()), + ("child.cer", (common / "child.cer").read_bytes()), + ], root_name, root_key, common, work, + ) + write(common / "root.mft", root_mft) + bad_root_mft = make_manifest( + "root", 1, [ + ("root.crl", (common / "root.crl").read_bytes()), + ("child.cer", (common / "child.cer").read_bytes()), + ("bad-child.cer", (common / "bad-child.cer").read_bytes()), + ], root_name, root_key, common, work, + ) + write(common / "root.mft", root_mft) + case_specs = ( + ("baseline-v1", 1, ["child.crl", "valid.roa"]), + ("sync-hash-mismatch", 1, ["child.crl", "valid.roa"]), + ("baseline-v2", 2, ["child.crl", "valid.roa", "valid2.roa"]), + ("validation-expired", 1, ["child.crl", "expired.roa"]), + ("validation-max-length", 1, ["child.crl", "max-length.roa"]), + ("validation-max-length-invalid", 1, ["child.crl", "max-length-invalid.roa"]), + ("validation-roa-prefix-outside", 1, ["child.crl", "roa-prefix-outside.roa"]), + ("validation-out-of-resource", 1, ["child.crl", "out-of-resource.roa"]), + ("validation-ca-over-resource", 1, ["child.crl", "valid.roa"]), + ("validation-nonstandard", 1, ["child.crl", "bad-format.roa"]), + ) + child_mfts = {} + for name, number, names in case_specs: + child_mfts[name] = make_manifest( + name, number, [(item, (common / item).read_bytes()) for item in names], + child_name, child_key, common, work, + ) + for name, number, names in case_specs: + change = None + if name == "baseline-v2": + change = delta("valid2.roa", common / "valid2.roa", common / "baseline-v2.mft") + build_case( + output, common, tal, root_cert.public_bytes(Encoding.DER), name, number, names, + child_mfts[name], change, + root_mft=bad_root_mft if name == "validation-ca-over-resource" else None, + extra_root_objects=[("bad-child.cer", (common / "bad-child.cer").read_bytes())] + if name == "validation-ca-over-resource" else None, + notification_hash_mismatch=name == "sync-hash-mismatch", + ) + write_key(output / "keys/root.key", root_key) + write_key(output / "keys/child.key", child_key) + make_https_certificates(output) + shutil.rmtree(output / ".common") + (output / "FIXTURE-MANIFEST.txt").write_text( + "fixture_schema_version=1\nfixture_kind=custom-tal-fixed-local-rpki\n" + f"tal_uri={TAL_URI}\nrrdp_uri={RRDP_URI}\nrsync_module=custom\n" + f"rrdp_port={RRDP_PORT}\nrsync_port={RSYNC_PORT}\n" + "cases=baseline-v1,sync-hash-mismatch,baseline-v2,validation-expired,validation-max-length,validation-max-length-invalid," + "validation-roa-prefix-outside,validation-out-of-resource,validation-ca-over-resource,validation-nonstandard\n", + encoding="utf-8", + ) + print(f"generated custom fixture: {output}") + + +if __name__ == "__main__": + main() diff --git a/deploy/docker-installer/custom-tal-patch/tools/serve_https.py b/deploy/docker-installer/custom-tal-patch/tools/serve_https.py new file mode 100644 index 0000000..d566ddb --- /dev/null +++ b/deploy/docker-installer/custom-tal-patch/tools/serve_https.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Serve a fixture directory over HTTPS with request logging.""" +from __future__ import annotations + +import argparse +import http.server +import ssl +from pathlib import Path + + +class RequestHandler(http.server.SimpleHTTPRequestHandler): + def log_message(self, format: str, *args) -> None: + print("%s %s" % (self.log_date_time_string(), format % args), flush=True) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--directory", type=Path, required=True) + parser.add_argument("--certfile", type=Path, required=True) + parser.add_argument("--keyfile", type=Path, required=True) + parser.add_argument("--host", default="0.0.0.0") + parser.add_argument("--port", type=int, default=18443) + args = parser.parse_args() + handler = lambda *handler_args, directory=str(args.directory): RequestHandler( + *handler_args, directory=directory + ) + server = http.server.ThreadingHTTPServer((args.host, args.port), handler) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(certfile=args.certfile, keyfile=args.keyfile) + server.socket = context.wrap_socket(server.socket, server_side=True) + print(f"https fixture server listening on {args.host}:{args.port} root={args.directory}", flush=True) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/deploy/docker-installer/custom-tal-patch/tools/start_fixed_services.sh b/deploy/docker-installer/custom-tal-patch/tools/start_fixed_services.sh new file mode 100644 index 0000000..a5b58ca --- /dev/null +++ b/deploy/docker-installer/custom-tal-patch/tools/start_fixed_services.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$BASH_SOURCE")" && pwd)" +FIXTURE_ROOT="" +CASE_NAME="baseline-v1" +RRDP_PORT="$(printenv RRDP_PORT 2>/dev/null || true)" +RSYNC_PORT="$(printenv RSYNC_PORT 2>/dev/null || true)" +PID_ROOT="" + +[[ -n "$RRDP_PORT" ]] || RRDP_PORT=18443 +[[ -n "$RSYNC_PORT" ]] || RSYNC_PORT=1873 + +usage() { + cat <<'USAGE' +Usage: + ./start_fixed_services.sh --fixture-root [--case NAME] + +Cases: baseline-v1, sync-hash-mismatch, baseline-v2, validation-expired, + validation-max-length, validation-max-length-invalid, + validation-roa-prefix-outside, validation-out-of-resource, + validation-ca-over-resource, validation-nonstandard +Environment: RRDP_PORT (default 18443), RSYNC_PORT (default 1873) +USAGE +} + +die() { + echo "error: $*" >&2 + exit 2 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --fixture-root) FIXTURE_ROOT="$2"; shift 2 ;; + --case) CASE_NAME="$2"; shift 2 ;; + --pid-root) PID_ROOT="$2"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) die "unknown option: $1" ;; + esac +done + +[[ -n "$FIXTURE_ROOT" ]] || die "--fixture-root is required" +FIXTURE_ROOT="$(cd "$FIXTURE_ROOT" 2>/dev/null && pwd)" || die "fixture root not found" +CASE_ROOT="$FIXTURE_ROOT/cases/$CASE_NAME" +[[ -d "$CASE_ROOT/repository" && -d "$CASE_ROOT/http" ]] || die "fixture case not found: $CASE_NAME" +if [[ -z "$PID_ROOT" ]]; then + PID_ROOT="$FIXTURE_ROOT/services/$CASE_NAME" +fi +mkdir -p "$PID_ROOT" +[[ ! -f "$PID_ROOT/pids.env" ]] || die "services already appear to be running: $PID_ROOT/pids.env" +rm -f "$PID_ROOT/rsyncd.pid" "$PID_ROOT/rsyncd.lock" + +RSYNCD_CONF="$PID_ROOT/rsyncd.conf" +RSYNC_UID="$(id -un)" +RSYNC_GID="$(id -gn)" +cat > "$RSYNCD_CONF" <"$PID_ROOT/rsyncd.stdout.log" 2>&1 & +RSYNC_PID=$! +python3 "$SCRIPT_DIR/serve_https.py" \ + --directory "$CASE_ROOT/http" \ + --certfile "$FIXTURE_ROOT/certs/rrdp-server.pem" \ + --keyfile "$FIXTURE_ROOT/certs/rrdp-server.key" \ + --port "$RRDP_PORT" \ + >"$PID_ROOT/https.log" 2>&1 & +HTTPS_PID=$! + +printf 'case=%s\nrrdp_pid=%s\nrsync_pid=%s\nrrdp_port=%s\nrsync_port=%s\n' \ + "$CASE_NAME" "$HTTPS_PID" "$RSYNC_PID" "$RRDP_PORT" "$RSYNC_PORT" > "$PID_ROOT/pids.env" +sleep 1 +kill -0 "$HTTPS_PID" 2>/dev/null || die "HTTPS fixture server exited; see $PID_ROOT/https.log" +kill -0 "$RSYNC_PID" 2>/dev/null || die "rsync fixture server exited; see $PID_ROOT/rsyncd.stdout.log" +echo "fixed RRDP/rsync services started case=$CASE_NAME rrdp=$RRDP_PORT rsync=$RSYNC_PORT pid_root=$PID_ROOT" diff --git a/deploy/docker-installer/custom-tal-patch/tools/stop_fixed_services.sh b/deploy/docker-installer/custom-tal-patch/tools/stop_fixed_services.sh new file mode 100644 index 0000000..97192d1 --- /dev/null +++ b/deploy/docker-installer/custom-tal-patch/tools/stop_fixed_services.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +PID_ROOT="$1" +[[ -n "$PID_ROOT" ]] || { echo "Usage: $0 " >&2; exit 2; } +[[ -f "$PID_ROOT/pids.env" ]] || { echo "services are not running: $PID_ROOT" >&2; exit 2; } +# shellcheck disable=SC1090 +source "$PID_ROOT/pids.env" +service_pids=("${rrdp_pid:-}" "${rsync_pid:-}") +for pid in "${service_pids[@]}"; do + if [[ "$pid" =~ ^[0-9]+$ ]] && kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + fi +done +for pid in "${service_pids[@]}"; do + [[ "$pid" =~ ^[0-9]+$ ]] || continue + for _ in $(seq 1 50); do + kill -0 "$pid" 2>/dev/null || break + sleep 0.1 + done +done +rm -f "$PID_ROOT/pids.env" +echo "fixed RRDP/rsync services stopped: $PID_ROOT" diff --git a/scripts/docker/build_custom_tal_patch.sh b/scripts/docker/build_custom_tal_patch.sh new file mode 100644 index 0000000..bd322d9 --- /dev/null +++ b/scripts/docker/build_custom_tal_patch.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$BASH_SOURCE")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +PATCH_SOURCE="$REPO_ROOT/deploy/docker-installer/custom-tal-patch" +BASE_COMPONENT_PACKAGE="" +OUT_DIR="$(printenv OUT_DIR 2>/dev/null || true)" +PREFIX="$(printenv PREFIX 2>/dev/null || true)" +ALLOW_DIRTY=0 +[[ -n "$OUT_DIR" ]] || OUT_DIR="$REPO_ROOT/target/custom-tal-patch" +[[ -n "$PREFIX" ]] || PREFIX="ours-rp-custom-tal-patch" + +usage() { + cat <<'USAGE' +Usage: + scripts/docker/build_custom_tal_patch.sh \ + --base-component-package \ + [--out-dir ] [--prefix ] [--allow-dirty] + +The patch targets the Arm64 component package from the customer stack. It +overlays the runner, Compose, fixture mount and test-service tools; it does not +rebuild or replace the runtime image. +USAGE +} + +die() { + echo "error: $*" >&2 + exit 2 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --base-component-package) BASE_COMPONENT_PACKAGE="$2"; shift 2 ;; + --out-dir) OUT_DIR="$2"; shift 2 ;; + --prefix) PREFIX="$2"; shift 2 ;; + --allow-dirty) ALLOW_DIRTY=1; shift ;; + -h|--help) usage; exit 0 ;; + *) die "unknown option: $1" ;; + esac +done + +[[ -f "$BASE_COMPONENT_PACKAGE" ]] || die "missing base component package: $BASE_COMPONENT_PACKAGE" +[[ -d "$PATCH_SOURCE" ]] || die "missing patch source: $PATCH_SOURCE" + +SOURCE_COMMIT="$(git -C "$REPO_ROOT" rev-parse HEAD)" +SOURCE_COMMIT_SHORT="$(git -C "$REPO_ROOT" rev-parse --short=8 HEAD)" +SOURCE_DIRTY=false +if [[ -n "$(git -C "$REPO_ROOT" status --short)" ]]; then + SOURCE_DIRTY=true +fi +if [[ "$SOURCE_DIRTY" == true && "$ALLOW_DIRTY" != 1 ]]; then + die "source worktree is dirty; use --allow-dirty for a development patch" +fi + +package_root="$(tar -tzf "$BASE_COMPONENT_PACKAGE" | awk -F/ '!seen { print $1; seen = 1 }')" +[[ -n "$package_root" ]] || die "cannot determine component package root" +manifest_text="$(tar -xOf "$BASE_COMPONENT_PACKAGE" "$package_root/PACKAGE-MANIFEST.env")" \ + || die "base component package has no PACKAGE-MANIFEST.env" +read_manifest_value() { + printf '%s\n' "$manifest_text" | awk -F= -v key="$1" '$1 == key { print substr($0, index($0, "=") + 1); exit }' +} +base_source_commit="$(read_manifest_value source_commit)" +base_package_arch="$(read_manifest_value package_arch)" +base_runtime_image="$(read_manifest_value runtime_image)" +[[ "$base_source_commit" == 9f4f4cf069e9fa8f6f97ed0f1065adac38abe82f ]] \ + || die "base source commit mismatch: $base_source_commit" +[[ "$base_package_arch" == arm64 ]] || die "base package must be arm64: $base_package_arch" + +timestamp="$(date -u +%Y%m%dT%H%M%SZ)" +patch_name="$PREFIX-$SOURCE_COMMIT_SHORT" +if [[ "$SOURCE_DIRTY" == true ]]; then + patch_name="$patch_name-dirty" +fi +patch_name="$patch_name-$timestamp" +stage="$OUT_DIR/$patch_name" +tar_path="$OUT_DIR/$patch_name.tar.gz" +base_sha256="$(sha256sum "$BASE_COMPONENT_PACKAGE" | awk '{ print $1 }')" +rm -rf "$stage" +mkdir -p "$stage/payload/compose" "$stage/payload/scripts/soak" \ + "$stage/payload/custom-fixtures" "$stage/payload/tools" + +cp "$PATCH_SOURCE/apply_custom_tal_patch.sh" "$stage/" +cp "$PATCH_SOURCE/rollback_custom_tal_patch.sh" "$stage/" +cp "$PATCH_SOURCE/custom-tal.env.example" "$stage/" +cp "$REPO_ROOT/deploy/docker-installer/compose/docker-compose.yml" "$stage/payload/compose/" +cp "$REPO_ROOT/scripts/soak/run_soak.sh" "$stage/payload/scripts/soak/" +cp "$REPO_ROOT/deploy/docker-installer/custom-fixtures/README.md" "$stage/payload/custom-fixtures/" +cp "$PATCH_SOURCE/tools/generate_custom_fixture.py" "$stage/payload/tools/" +cp "$PATCH_SOURCE/tools/serve_https.py" "$stage/payload/tools/" +cp "$PATCH_SOURCE/tools/start_fixed_services.sh" "$stage/payload/tools/" +cp "$PATCH_SOURCE/tools/stop_fixed_services.sh" "$stage/payload/tools/" +chmod +x "$stage/apply_custom_tal_patch.sh" "$stage/rollback_custom_tal_patch.sh" \ + "$stage/payload/scripts/soak/run_soak.sh" "$stage/payload/tools/"*.sh \ + "$stage/payload/tools/"*.py + +{ + echo "patch_schema_version=1" + echo "patch_name=$patch_name" + echo "created_at_utc=$timestamp" + echo "source_commit=$SOURCE_COMMIT" + echo "source_commit_short=$SOURCE_COMMIT_SHORT" + echo "source_dirty=$SOURCE_DIRTY" + echo "base_component_archive=$(basename "$BASE_COMPONENT_PACKAGE")" + echo "base_component_archive_sha256=$base_sha256" + echo "base_component_root=$package_root" + echo "base_source_commit=$base_source_commit" + echo "base_package_arch=$base_package_arch" + echo "base_runtime_image=$base_runtime_image" + echo "overlay_runtime_image_unchanged=true" + echo "custom_mode=RIRS=custom,TAL_INPUT_MODE=custom-file-with-ta" + echo "rrdp_port=18443" + echo "rsync_port=1873" + echo "payload_root=payload" + while IFS= read -r relative_path; do + safe_name="$(printf '%s' "$relative_path" | tr '/.' '__')" + file_sha256="$(sha256sum "$stage/payload/$relative_path" | awk '{ print $1 }')" + echo "payload_"$safe_name"_sha256=$file_sha256" + done < <(cd "$stage/payload" && find . -type f -printf '%P\n' | sort) +} > "$stage/PATCH-MANIFEST.env" + +cat > "$stage/PATCH-SUMMARY.txt" < "$OUT_DIR/$patch_name.summary.env" +echo "patch built: $tar_path" diff --git a/scripts/soak/run_soak.sh b/scripts/soak/run_soak.sh index 15d1b13..d50f4d2 100755 --- a/scripts/soak/run_soak.sh +++ b/scripts/soak/run_soak.sh @@ -15,6 +15,10 @@ INTERVAL_SECS="${INTERVAL_SECS:-0}" STOP_AFTER_SECS="${STOP_AFTER_SECS:-0}" RIRS="${RIRS:-afrinic,apnic,arin,lacnic,ripe}" TAL_INPUT_MODE="${TAL_INPUT_MODE:-file-with-ta}" +CUSTOM_TAL_PATH="${CUSTOM_TAL_PATH:-$PACKAGE_ROOT/custom-fixtures/tal/custom.tal}" +CUSTOM_TA_PATH="${CUSTOM_TA_PATH:-$PACKAGE_ROOT/custom-fixtures/ta/custom-ta.cer}" +CUSTOM_TAL_URI="${CUSTOM_TAL_URI:-https://host.docker.internal:18443/tal/custom.tal}" +HTTP_ROOT_CERT_PATHS="${HTTP_ROOT_CERT_PATHS:-}" RESOURCE_VALIDATION_MODE="${RESOURCE_VALIDATION_MODE:-validation-update-03}" RUN_ROOT="${RUN_ROOT:-$PACKAGE_ROOT}" RETAIN_RUNS="${RETAIN_RUNS:-10}" @@ -118,10 +122,10 @@ validate_rsync_scope() { validate_tal_input_mode() { case "$TAL_INPUT_MODE" in - file-with-ta|file-live-ta|url) + file-with-ta|file-live-ta|custom-file-with-ta|url) ;; *) - die "TAL_INPUT_MODE must be file-with-ta, file-live-ta or url: $TAL_INPUT_MODE" + die "TAL_INPUT_MODE must be file-with-ta, file-live-ta, custom-file-with-ta or url: $TAL_INPUT_MODE" ;; esac } @@ -155,14 +159,46 @@ parse_rirs() { afrinic|apnic|arin|lacnic|ripe) RIR_LIST+=("$normalized") ;; + custom) + [[ "$TAL_INPUT_MODE" == "custom-file-with-ta" ]] \ + || die "RIRS=custom requires TAL_INPUT_MODE=custom-file-with-ta" + RIR_LIST+=("$normalized") + ;; *) - die "invalid RIRS entry: $raw_token; allowed: afrinic,apnic,arin,lacnic,ripe" + die "invalid RIRS entry: $raw_token; allowed: afrinic,apnic,arin,lacnic,ripe,custom (custom mode only)" ;; esac done [[ "${#RIR_LIST[@]}" -gt 0 ]] || die "RIRS must contain at least one RIR" } +validate_custom_inputs() { + local root_cert_path + local -a root_cert_paths=() + + if [[ "$TAL_INPUT_MODE" == "custom-file-with-ta" ]]; then + [[ "${#RIR_LIST[@]}" -eq 1 && "${RIR_LIST[0]}" == "custom" ]] \ + || die "custom-file-with-ta requires exactly RIRS=custom" + [[ -s "$CUSTOM_TAL_PATH" ]] \ + || die "missing custom TAL: $CUSTOM_TAL_PATH" + [[ -s "$CUSTOM_TA_PATH" ]] \ + || die "missing custom TA DER: $CUSTOM_TA_PATH" + [[ -n "$CUSTOM_TAL_URI" ]] \ + || die "CUSTOM_TAL_URI is required in custom-file-with-ta mode" + elif [[ "$RIRS" == *custom* ]]; then + die "RIRS=custom requires TAL_INPUT_MODE=custom-file-with-ta" + fi + + if [[ -n "$HTTP_ROOT_CERT_PATHS" ]]; then + # shellcheck disable=SC2206 + root_cert_paths=( $HTTP_ROOT_CERT_PATHS ) + for root_cert_path in "${root_cert_paths[@]}"; do + [[ -s "$root_cert_path" ]] \ + || die "missing HTTP root certificate: $root_cert_path" + done + fi +} + tal_file_for_rir() { case "$1" in afrinic) printf '%s' "$FIXTURE_DIR/tal/afrinic.tal" ;; @@ -170,6 +206,7 @@ tal_file_for_rir() { arin) printf '%s' "$FIXTURE_DIR/tal/arin.tal" ;; lacnic) printf '%s' "$FIXTURE_DIR/tal/lacnic.tal" ;; ripe) printf '%s' "$FIXTURE_DIR/tal/ripe-ncc.tal" ;; + custom) printf '%s' "$CUSTOM_TAL_PATH" ;; *) die "unknown RIR: $1" ;; esac } @@ -181,6 +218,7 @@ ta_file_for_rir() { arin) printf '%s' "$FIXTURE_DIR/ta/arin-ta.cer" ;; lacnic) printf '%s' "$FIXTURE_DIR/ta/lacnic-ta.cer" ;; ripe) printf '%s' "$FIXTURE_DIR/ta/ripe-ncc-ta.cer" ;; + custom) printf '%s' "$CUSTOM_TA_PATH" ;; *) die "unknown RIR: $1" ;; esac } @@ -192,6 +230,7 @@ tal_url_for_rir() { arin) printf '%s' "https://www.arin.net/resources/manage/rpki/arin.tal" ;; lacnic) printf '%s' "https://www.lacnic.net/innovaportal/file/4983/1/lacnic.tal" ;; ripe) printf '%s' "https://tal.rpki.ripe.net/ripe-ncc.tal" ;; + custom) printf '%s' "$CUSTOM_TAL_URI" ;; *) die "unknown RIR: $1" ;; esac } @@ -1057,6 +1096,16 @@ build_child_args() { CHILD_ARGS+=(--rsync-mirror-root "$TMP_DIR/rsync-mirror-{run_id}") fi + local root_cert_path + local -a root_cert_paths=() + if [[ -n "$HTTP_ROOT_CERT_PATHS" ]]; then + # shellcheck disable=SC2206 + root_cert_paths=( $HTTP_ROOT_CERT_PATHS ) + for root_cert_path in "${root_cert_paths[@]}"; do + CHILD_ARGS+=(--http-root-cert "$root_cert_path") + done + fi + CHILD_ARGS+=( --parallel-phase2-ready-batch-size 256 --parallel-phase2-ready-batch-wall-time-budget-ms 100 @@ -1344,6 +1393,7 @@ main() { validate_positive_int "DB_STATS_EXACT_EVERY" "$DB_STATS_EXACT_EVERY" fi parse_rirs + validate_custom_inputs [[ -x "$RPKI_BIN" ]] || die "missing executable: $RPKI_BIN" [[ -x "$RPKI_DAEMON_BIN" ]] || die "missing executable: $RPKI_DAEMON_BIN"