From 4975128945e56b2b6480488d7a3a766ecf40ae72 Mon Sep 17 00:00:00 2001 From: yuyr Date: Mon, 14 Sep 2026 16:58:08 +0800 Subject: [PATCH] =?UTF-8?q?20260914=20=E5=BC=BA=E5=88=B6=20TOML=20?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E5=B9=B6=E5=AE=8C=E5=96=84=20TA=20=E5=88=B7?= =?UTF-8?q?=E6=96=B0=E4=B8=8E=20daemon?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 12 +- README.md | 43 +- docker/compose.all5-daemon.yaml | 63 +-- docker/compose.all5.yaml | 57 +-- docker/compose.daemon.yaml | 32 +- docker/compose.yaml | 23 +- docker/examples/README.md | 24 + docker/examples/daemon-all5.toml | 39 ++ docker/examples/daemon.toml | 19 + docker/examples/validate-all5.toml | 37 ++ docker/examples/validate.toml | 17 + docs/command-line-reference.md | 217 ++++----- docs/configuration.md | 294 ++++++++++++ docs/conformance-matrix.md | 5 +- docs/docker.md | 312 ++++++------ docs/getting-started.md | 47 +- docs/logging.md | 23 +- docs/releasing.md | 10 + docs/trust-anchors.md | 56 ++- docs/usage.md | 196 ++++---- examples/config/README.md | 32 ++ examples/config/daemon-all5.toml | 45 ++ examples/config/daemon-three.toml | 46 ++ examples/config/validate-fixed.toml | 12 + examples/config/validate-full.toml | 27 ++ examples/config/validate-minimal.toml | 9 + src/cli/anchors.rs | 246 ++++++++++ src/cli/config.rs | 391 +++++++++++++++ src/cli/mod.rs | 276 ++++------- src/cli/publication.rs | 133 ++++++ src/cli/validate.rs | 445 +++++++++--------- src/daemon/mod.rs | 203 +++----- src/daemon/process.rs | 93 +++- src/daemon/tests.rs | 125 ++--- src/main.rs | 18 +- src/repository/sync/rrdp/notification_sync.rs | 47 +- src/repository/sync/rrdp/snapshot_sync.rs | 19 +- src/runtime/post_validation.rs | 54 ++- src/runtime/run.rs | 33 +- src/ta_constraints/implementation.rs | 10 +- src/validation/manifest/models_and_process.rs | 10 +- src/validation/run_tree_from_tal/discovery.rs | 18 +- src/validation/tree_parallel/phase2.rs | 24 +- .../tree_runner/audit_projection.rs | 4 +- tests/cli_errors.rs | 79 +++- tests/logging_contract.rs | 69 +-- tests/support/test_daemon.py | 204 ++++---- tests/synthetic_docker_e2e.sh | 56 ++- 48 files changed, 2795 insertions(+), 1459 deletions(-) create mode 100644 docker/examples/README.md create mode 100644 docker/examples/daemon-all5.toml create mode 100644 docker/examples/daemon.toml create mode 100644 docker/examples/validate-all5.toml create mode 100644 docker/examples/validate.toml create mode 100644 docs/configuration.md create mode 100644 examples/config/README.md create mode 100644 examples/config/daemon-all5.toml create mode 100644 examples/config/daemon-three.toml create mode 100644 examples/config/validate-fixed.toml create mode 100644 examples/config/validate-full.toml create mode 100644 examples/config/validate-minimal.toml create mode 100644 src/cli/anchors.rs create mode 100644 src/cli/config.rs create mode 100644 src/cli/publication.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c846b8..6ec7156 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## 0.1.0 — Unreleased +- Breaking CLI change: mandatory TOML via `validate|daemon --config FILE`; + remove business flags, `--` forwarding and logging environment overrides. +- Read-only configuration checking/printing and top-level version output. +- TAL-driven TA certificate refresh with validated cache fallback; TAL files + remain operator-maintained. +- Explicit success/partial/failed results, strict completion exit code 3, + atomic completion markers and success-only latest publication. +- Complete automatic-TA/text-log Compose TOML examples and stop/edit/reload guide. + + - RPKI validation with multiple TAL/TA inputs and resource constraints. - RRDP snapshot/delta synchronization and same-origin enforcement. - Bounded transport and object worker pools. @@ -9,5 +19,5 @@ - Foreground daemon with intervals, timeouts, restart and retention. - Docker and Compose deployment, including all-five-anchor configuration. -The CLI accepts HTTPS RRDP. Operators supply TALs and matching certificates. +The CLI accepts HTTPS RRDP. Operators supply TALs; matching TA certificates can be fixed or automatically refreshed. No RTR server is included. See the standards matrix and CLI reference for scope. diff --git a/README.md b/README.md index cbed536..6cd9f11 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Panda RPKI +For every field, default, constraint and complete example, see the [TOML configuration reference](docs/configuration.md)。 + Panda RPKI is an RPKI relying party written in Rust. It synchronizes repositories over RRDP, validates signed objects, and exports routing data as CSV and Canonical Cache Representation (CCR). Run a single cycle or use the daemon for continuous operation. @@ -10,41 +12,58 @@ and [security policy](SECURITY.md) before evaluating it for routing use. ## Quick start -Requirements: Linux, Docker, and a TAL with its matching DER trust-anchor -certificate. Follow [input preparation](docs/getting-started.md#prepare-trust-anchors) -to obtain these from your chosen RIR. From the project root, after placing them -in `input/anchor.tal` and `input/anchor.cer`: +Requirements: Linux, Docker, and a TAL. A matching DER trust-anchor certificate +may be supplied in the TOML, or Panda can refresh it from the TAL's URI. Follow +[input preparation](docs/getting-started.md#prepare-trust-anchors) for trusted +RIR inputs. From the project root, after placing `input/anchor.tal` (and, for a +fixed-TA run, `input/anchor.cer`): ```bash docker build -f docker/Dockerfile -t panda-rpki:v0.1.0 . mkdir -p state output +cat > validate.toml <<'TOML' +config-version = 1 +mode = "validate" +out = "/output" +rrdp-state-dir = "/state/repository" +ccr-out = "/output/result.ccr" + +[[anchors]] +id = "anchor" +tal = "/input/anchor.tal" +# ta = "/input/anchor.cer" # uncomment for fixed TA mode +TOML docker run --rm --read-only --tmpfs /tmp \ --user "$(id -u):$(id -g)" \ -v "$PWD/input:/input:ro" \ -v "$PWD/state:/state" \ -v "$PWD/output:/output" \ - panda-rpki:v0.1.0 validate \ - --tal /input/anchor.tal --ta /input/anchor.cer \ - --rrdp-state-dir /state --out /output \ - --ccr-out /output/result.ccr + -v "$PWD/validate.toml:/config/validate.toml:ro" \ + panda-rpki:v0.1.0 validate --config /config/validate.toml ``` +The quick start refreshes the TA certificate automatically; TAL files themselves +remain operator-maintained. For complete single/all-five and daemon TOML files, +see [Compose examples](docker/examples/README.md). + Inspect `output/summary.json` and `output/vrps.csv`. Logs go to stderr. Reuse the state directory to allow delta updates on subsequent runs. For continuous operation see [Usage](docs/usage.md#continuous-operation). ## Features -- Multiple TAL/TA pairs and optional per-anchor resource constraints. +- Multiple named anchors, fixed or automatically refreshed trust anchors, and + optional per-anchor resource constraints. - RRDP snapshot, delta, replace/withdraw and protocol fallback. - HTTPS same-origin checks for RRDP references and redirects. - Independent bounded repository and object worker pools. - CA, CRL, manifest, ROA, ASPA and BGPsec router-certificate processing. -- CSV, CCR, JSON summaries and configurable text or JSON logs. +- CSV, CCR, JSON summaries and configurable text or JSON logs from TOML. - Daemon intervals, persistent state, run retention, timeouts and graceful stop. -Version 0.1.0 accepts HTTPS RRDP through its CLI. Operators supply TALs and -matching TA certificates. No RTR server is included. See the +Version 0.1.0 accepts HTTPS RRDP through its CLI. Operators supply explicit +TALs; automatic TA refresh follows each TAL's HTTPS or rsync URI. No RTR server +is included. See the [standards matrix](docs/conformance-matrix.md) for profiles and limitations. ## Documentation diff --git a/docker/compose.all5-daemon.yaml b/docker/compose.all5-daemon.yaml index d87d7ae..5cea1c2 100644 --- a/docker/compose.all5-daemon.yaml +++ b/docker/compose.all5-daemon.yaml @@ -1,3 +1,6 @@ +# Copy docker/examples/daemon-all5.toml and set CONFIG_FILE to its absolute path. +# Examples use text logs and automatic TA certificate refresh; TALs stay local. +# See docs/docker.md for graceful stop and config reload. services: validator: build: @@ -12,62 +15,10 @@ services: tmpfs: - /tmp volumes: - - ${INPUT_DIR:?set INPUT_DIR to the TAL/TA directory}:/input:ro + - ${CONFIG_FILE:?set CONFIG_FILE to an all-RIR daemon TOML file}:/config/daemon.toml:ro + - ${INPUT_DIR:?set INPUT_DIR to the TAL directory (TA certificates optional)}:/input:ro - ${DAEMON_DIR:?set DAEMON_DIR to a writable dedicated directory}:/data - environment: - PANDA_RPKI_LOG_LEVEL: "${PANDA_RPKI_LOG_LEVEL:-info}" - PANDA_RPKI_LOG_FORMAT: "${PANDA_RPKI_LOG_FORMAT:-text}" - PANDA_RPKI_LOG_PROGRESS_INTERVAL_SECS: "${PANDA_RPKI_LOG_PROGRESS_INTERVAL_SECS:-10}" command: - daemon - - --state-root - - /data - - --interval-secs - - "${PANDA_RPKI_INTERVAL_SECS:-60}" - - --retain-runs - - "${PANDA_RPKI_RETAIN_RUNS:-10}" - - --run-timeout-secs - - "${PANDA_RPKI_RUN_TIMEOUT_SECS:-0}" - - --shutdown-grace-secs - - "${PANDA_RPKI_SHUTDOWN_GRACE_SECS:-30}" - - -- - - --tal - - /input/${AFRINIC_TAL_FILE:?set AFRINIC_TAL_FILE} - - --ta - - /input/${AFRINIC_TA_FILE:?set AFRINIC_TA_FILE} - - --tal-id - - afrinic - - --tal - - /input/${APNIC_TAL_FILE:?set APNIC_TAL_FILE} - - --ta - - /input/${APNIC_TA_FILE:?set APNIC_TA_FILE} - - --tal-id - - apnic - - --tal - - /input/${ARIN_TAL_FILE:?set ARIN_TAL_FILE} - - --ta - - /input/${ARIN_TA_FILE:?set ARIN_TA_FILE} - - --tal-id - - arin - - --tal - - /input/${LACNIC_TAL_FILE:?set LACNIC_TAL_FILE} - - --ta - - /input/${LACNIC_TA_FILE:?set LACNIC_TA_FILE} - - --tal-id - - lacnic - - --tal - - /input/${RIPE_TAL_FILE:?set RIPE_TAL_FILE} - - --ta - - /input/${RIPE_TA_FILE:?set RIPE_TA_FILE} - - --tal-id - - ripe - - --parallel-phase2-object-workers - - "${PANDA_RPKI_WORKERS:-8}" - - --parallel-max-repo-sync-workers-global - - "${PANDA_RPKI_REPO_WORKERS:-8}" - - --parallel-phase2-worker-queue-capacity - - "${PANDA_RPKI_WORKER_QUEUE_CAPACITY:-256}" - - --parallel-repo-worker-queue-capacity - - "${PANDA_RPKI_REPO_QUEUE_CAPACITY:-256}" - - --http-timeout-secs - - "${PANDA_RPKI_HTTP_TIMEOUT_SECS:-600}" + - --config + - /config/daemon.toml diff --git a/docker/compose.all5.yaml b/docker/compose.all5.yaml index 0764f0b..51fdca6 100644 --- a/docker/compose.all5.yaml +++ b/docker/compose.all5.yaml @@ -1,62 +1,23 @@ +# Copy docker/examples/validate-all5.toml and set CONFIG_FILE to its absolute path. +# Examples use text logs and automatic TA certificate refresh; TALs stay local. +# See docs/docker.md for graceful stop and config reload. services: validator: build: context: .. dockerfile: docker/Dockerfile - image: panda-rpki:v0.1.0 + image: "${PANDA_RPKI_IMAGE:-panda-rpki:v0.1.0}" user: "${PUID:-1000}:${PGID:-1000}" read_only: true + init: true tmpfs: - /tmp volumes: - - ${INPUT_DIR:?set INPUT_DIR to the TAL/TA directory}:/input:ro + - ${CONFIG_FILE:?set CONFIG_FILE to an all-RIR validate TOML file}:/config/validate.toml:ro + - ${INPUT_DIR:?set INPUT_DIR to the TAL directory (TA certificates optional)}:/input:ro - ${OUTPUT_DIR:?set OUTPUT_DIR to a writable output directory}:/output - ${STATE_DIR:?set STATE_DIR to a writable state directory}:/state command: - validate - - --tal - - /input/${AFRINIC_TAL_FILE:?set AFRINIC_TAL_FILE} - - --ta - - /input/${AFRINIC_TA_FILE:?set AFRINIC_TA_FILE} - - --tal-id - - afrinic - - --tal - - /input/${APNIC_TAL_FILE:?set APNIC_TAL_FILE} - - --ta - - /input/${APNIC_TA_FILE:?set APNIC_TA_FILE} - - --tal-id - - apnic - - --tal - - /input/${ARIN_TAL_FILE:?set ARIN_TAL_FILE} - - --ta - - /input/${ARIN_TA_FILE:?set ARIN_TA_FILE} - - --tal-id - - arin - - --tal - - /input/${LACNIC_TAL_FILE:?set LACNIC_TAL_FILE} - - --ta - - /input/${LACNIC_TA_FILE:?set LACNIC_TA_FILE} - - --tal-id - - lacnic - - --tal - - /input/${RIPE_TAL_FILE:?set RIPE_TAL_FILE} - - --ta - - /input/${RIPE_TA_FILE:?set RIPE_TA_FILE} - - --tal-id - - ripe - - --parallel-phase2-object-workers - - "${PANDA_RPKI_WORKERS:-8}" - - --parallel-max-repo-sync-workers-global - - "${PANDA_RPKI_REPO_WORKERS:-8}" - - --parallel-phase2-worker-queue-capacity - - "${PANDA_RPKI_WORKER_QUEUE_CAPACITY:-256}" - - --rrdp-state-dir - - /state - - --rrdp-sync-mode - - "${PANDA_RPKI_RRDP_SYNC_MODE:-auto}" - - --http-timeout-secs - - "${PANDA_RPKI_HTTP_TIMEOUT_SECS:-600}" - - --ccr-out - - /output/all5.ccr - - --out - - /output + - --config + - /config/validate.toml diff --git a/docker/compose.daemon.yaml b/docker/compose.daemon.yaml index 344ecc5..aa69c97 100644 --- a/docker/compose.daemon.yaml +++ b/docker/compose.daemon.yaml @@ -1,9 +1,12 @@ +# Copy docker/examples/daemon.toml and set CONFIG_FILE to its absolute path. +# Examples use text logs and automatic TA certificate refresh; TALs stay local. +# See docs/docker.md for graceful stop and config reload. services: validator: build: context: .. dockerfile: docker/Dockerfile - image: panda-rpki:v0.1.0 + image: "${PANDA_RPKI_IMAGE:-panda-rpki:v0.1.0}" user: "${PUID:-1000}:${PGID:-1000}" read_only: true init: true @@ -12,29 +15,10 @@ services: tmpfs: - /tmp volumes: - - ${INPUT_DIR:?set INPUT_DIR to the TAL/TA directory}:/input:ro + - ${CONFIG_FILE:?set CONFIG_FILE to a daemon TOML file}:/config/daemon.toml:ro + - ${INPUT_DIR:?set INPUT_DIR to the TAL directory (TA certificates optional)}:/input:ro - ${DAEMON_DIR:?set DAEMON_DIR to a writable dedicated directory}:/data - environment: - PANDA_RPKI_LOG_LEVEL: "${PANDA_RPKI_LOG_LEVEL:-info}" - PANDA_RPKI_LOG_FORMAT: "${PANDA_RPKI_LOG_FORMAT:-text}" command: - daemon - - --state-root - - /data - - --interval-secs - - "${PANDA_RPKI_INTERVAL_SECS:-60}" - - --retain-runs - - "${PANDA_RPKI_RETAIN_RUNS:-10}" - - --run-timeout-secs - - "${PANDA_RPKI_RUN_TIMEOUT_SECS:-0}" - - --shutdown-grace-secs - - "30" - - -- - - --tal - - /input/${TAL_FILE:?set TAL_FILE} - - --ta - - /input/${TA_FILE:?set TA_FILE} - - --parallel-max-repo-sync-workers-global - - "${PANDA_RPKI_REPO_WORKERS:-8}" - - --parallel-phase2-object-workers - - "${PANDA_RPKI_WORKERS:-8}" + - --config + - /config/daemon.toml diff --git a/docker/compose.yaml b/docker/compose.yaml index 21725cd..80df178 100644 --- a/docker/compose.yaml +++ b/docker/compose.yaml @@ -1,28 +1,23 @@ +# Copy docker/examples/validate.toml and set CONFIG_FILE to its absolute path. +# Examples use text logs and automatic TA certificate refresh; TALs stay local. +# See docs/docker.md for graceful stop and config reload. services: validator: build: context: .. dockerfile: docker/Dockerfile - image: panda-rpki:v0.1.0 + image: "${PANDA_RPKI_IMAGE:-panda-rpki:v0.1.0}" user: "${PUID:-1000}:${PGID:-1000}" read_only: true + init: true tmpfs: - /tmp volumes: - - ${INPUT_DIR:?set INPUT_DIR to the TAL/TA directory}:/input:ro + - ${CONFIG_FILE:?set CONFIG_FILE to a validate TOML file}:/config/validate.toml:ro + - ${INPUT_DIR:?set INPUT_DIR to the TAL directory (TA certificates optional)}:/input:ro - ${OUTPUT_DIR:?set OUTPUT_DIR to a writable output directory}:/output - ${STATE_DIR:?set STATE_DIR to a writable state directory}:/state command: - validate - - --tal - - /input/${TAL_FILE:?set TAL_FILE} - - --ta - - /input/${TA_FILE:?set TA_FILE} - - --rrdp-state-dir - - /state - - --rrdp-sync-mode - - "${PANDA_RPKI_RRDP_SYNC_MODE:-auto}" - - --ccr-out - - /output/result.ccr - - --out - - /output + - --config + - /config/validate.toml diff --git a/docker/examples/README.md b/docker/examples/README.md new file mode 100644 index 0000000..eb5caa3 --- /dev/null +++ b/docker/examples/README.md @@ -0,0 +1,24 @@ +# Container TOML examples + +Copy the appropriate TOML outside this directory and point CONFIG_FILE at the +absolute copy path. See [Docker and Compose](../../docs/docker.md) for mounts, +startup, graceful stop and config reload. + +| File | Mode | Inputs | +| --- | --- | --- | +| validate.toml | Single run | /input/anchor.tal | +| validate-all5.toml | Single run | Five named RIR TALs | +| daemon.toml | Continuous | /input/anchor.tal | +| daemon-all5.toml | Continuous | Five named RIR TALs | + +All examples use text/info diagnostics, require-complete=true, and automatic +TA certificate refresh (ta omitted). TAL files are never automatically updated. +Uncomment an anchor's ta field to use a fixed certificate. Policy +allow-valid-cache permits only a validated cached certificate and marks fallback +partial. First automatic use needs reachable TAL URIs; fixed files are not an +automatic-cache seed. These examples contain no TALs or certificates. + +Container paths differ from native configuration paths. Validate uses +/state/repository and /state/repository.ta-cache under the writable /state mount; +daemon uses /data/state and /data/state.ta-cache. Existing /state databases need +an explicit migration or separate cache mount before switching to these paths. diff --git a/docker/examples/daemon-all5.toml b/docker/examples/daemon-all5.toml new file mode 100644 index 0000000..315ea99 --- /dev/null +++ b/docker/examples/daemon-all5.toml @@ -0,0 +1,39 @@ +# Container paths. Omit ta to refresh the TA certificate each run. +# TAL files themselves are operator-maintained and are never auto-updated. +config-version = 1 +mode = "daemon" +state-root = "/data" +interval-secs = 60 +retain-runs = 20 +shutdown-grace-secs = 30 +# max-runs = 3 # uncomment only for a finite run +run-timeout-secs = 1800 +require-complete = true +ta-refresh-policy = "allow-valid-cache" +log-level = "info" +log-format = "text" + +[[anchors]] +id = "afrinic" +tal = "/input/afrinic.tal" +# ta = "/input/afrinic.cer" # uncomment to pin a fixed TA certificate + +[[anchors]] +id = "apnic" +tal = "/input/apnic.tal" +# ta = "/input/apnic.cer" # uncomment to pin a fixed TA certificate + +[[anchors]] +id = "arin" +tal = "/input/arin.tal" +# ta = "/input/arin.cer" # uncomment to pin a fixed TA certificate + +[[anchors]] +id = "lacnic" +tal = "/input/lacnic.tal" +# ta = "/input/lacnic.cer" # uncomment to pin a fixed TA certificate + +[[anchors]] +id = "ripe" +tal = "/input/ripe.tal" +# ta = "/input/ripe.cer" # uncomment to pin a fixed TA certificate diff --git a/docker/examples/daemon.toml b/docker/examples/daemon.toml new file mode 100644 index 0000000..2970e9d --- /dev/null +++ b/docker/examples/daemon.toml @@ -0,0 +1,19 @@ +# Container paths. Omit ta to refresh the TA certificate each run. +# TAL files themselves are operator-maintained and are never auto-updated. +config-version = 1 +mode = "daemon" +state-root = "/data" +interval-secs = 60 +retain-runs = 20 +shutdown-grace-secs = 30 +# max-runs = 3 # uncomment only for a finite run +run-timeout-secs = 1800 +require-complete = true +ta-refresh-policy = "allow-valid-cache" +log-level = "info" +log-format = "text" + +[[anchors]] +id = "anchor" +tal = "/input/anchor.tal" +# ta = "/input/anchor.cer" # uncomment to pin a fixed TA certificate diff --git a/docker/examples/validate-all5.toml b/docker/examples/validate-all5.toml new file mode 100644 index 0000000..8f69d5f --- /dev/null +++ b/docker/examples/validate-all5.toml @@ -0,0 +1,37 @@ +# Container paths. Omit ta to refresh the TA certificate each run. +# TAL files themselves are operator-maintained and are never auto-updated. +config-version = 1 +mode = "validate" +out = "/output" +rrdp-state-dir = "/state/repository" +ccr-out = "/output/result.ccr" +run-timeout-secs = 1800 +require-complete = true +ta-refresh-policy = "allow-valid-cache" +log-level = "info" +log-format = "text" + +[[anchors]] +id = "afrinic" +tal = "/input/afrinic.tal" +# ta = "/input/afrinic.cer" # uncomment to pin a fixed TA certificate + +[[anchors]] +id = "apnic" +tal = "/input/apnic.tal" +# ta = "/input/apnic.cer" # uncomment to pin a fixed TA certificate + +[[anchors]] +id = "arin" +tal = "/input/arin.tal" +# ta = "/input/arin.cer" # uncomment to pin a fixed TA certificate + +[[anchors]] +id = "lacnic" +tal = "/input/lacnic.tal" +# ta = "/input/lacnic.cer" # uncomment to pin a fixed TA certificate + +[[anchors]] +id = "ripe" +tal = "/input/ripe.tal" +# ta = "/input/ripe.cer" # uncomment to pin a fixed TA certificate diff --git a/docker/examples/validate.toml b/docker/examples/validate.toml new file mode 100644 index 0000000..e6662fa --- /dev/null +++ b/docker/examples/validate.toml @@ -0,0 +1,17 @@ +# Container paths. Omit ta to refresh the TA certificate each run. +# TAL files themselves are operator-maintained and are never auto-updated. +config-version = 1 +mode = "validate" +out = "/output" +rrdp-state-dir = "/state/repository" +ccr-out = "/output/result.ccr" +run-timeout-secs = 1800 +require-complete = true +ta-refresh-policy = "allow-valid-cache" +log-level = "info" +log-format = "text" + +[[anchors]] +id = "anchor" +tal = "/input/anchor.tal" +# ta = "/input/anchor.cer" # uncomment to pin a fixed TA certificate diff --git a/docs/command-line-reference.md b/docs/command-line-reference.md index 6b42c9c..5db2937 100644 --- a/docs/command-line-reference.md +++ b/docs/command-line-reference.md @@ -1,140 +1,127 @@ # Command-line reference -This reference covers v0.1.0. `panda-rpki --help` displays version and usage; -there is no separate `--version` option. +For every field, default, constraint and complete example, see the [TOML configuration reference](configuration.md)。 -## Syntax +Runtime settings come from one explicit TOML file; the binary does not read `PANDA_RPKI_LOG_*` overrides and +does not accept legacy validation flags. + +## Commands ```text -panda-rpki validate --tal FILE --ta FILE --out DIRECTORY [OPTIONS] -panda-rpki daemon --state-root DIRECTORY [DAEMON OPTIONS] -- [VALIDATION OPTIONS] +panda-rpki --help +panda-rpki --version +panda-rpki -V +panda-rpki validate --help +panda-rpki daemon --help +panda-rpki validate --config FILE +panda-rpki validate --config FILE --check-config +panda-rpki validate --config FILE --print-config +panda-rpki daemon --config FILE +panda-rpki daemon --config FILE --check-config +panda-rpki daemon --config FILE --print-config ``` -Use `--name value`, not `--name=value`. Only help has a short alias (`-h`). -Values are case-sensitive; options cannot repeat unless listed as repeatable. -Paths are relative to the process working directory. No configuration-file -option is provided. No arguments or a help request exits successfully. +Only these long options are accepted: `--help`, `--version`, `--config`, +`--check-config` and `--print-config`. `-h` aliases help and `-V` aliases +version. `--config FILE` is required for a normal validate/daemon run and may +also be written as `--config=FILE`. Help does not read a config. Check and print +are mutually exclusive and do not run validation. Unknown, repeated or legacy +options—including a daemon `--` tail—return exit code 2. -## Inputs and outputs +## TOML schema -| Option | Default / requirement | Repeatable | Meaning | -| --- | --- | --- | --- | -| `--tal ` | Required | Yes | Local TAL file. | -| `--ta ` | Required | Yes | Matching DER trust-anchor certificate. | -| `--tal-id ` | TAL filename stem | Yes | Anchor identifier; if supplied, provide one for every TAL. | -| `--ta-constraints =` | Adjacent `.constraints` if present | Yes | Override constraints for a known TAL ID; each ID may occur once. | -| `--out ` | Required | No | Output directory, created if absent. Existing output files may be overwritten. | -| `--ccr-out ` | Disabled | No | CCR DER path, relative to the working directory, not `--out`. | +Every file contains `config-version = 1` and a matching `mode` value: -TAL and TA lists are paired by occurrence order and must have equal lengths. -Use distinct TAL IDs. For `input/anchor.tal`, automatic discovery checks -`input/anchor.constraints`. If absent with no explicit path, no additional -constraints are loaded for that anchor. +```toml +config-version = 1 +mode = "validate" +out = "output" +rrdp-state-dir = "state" +rrdp-sync-mode = "auto" +ccr-out = "output/result.ccr" +publish-dir = "publish" +ta-refresh-policy = "allow-valid-cache" +require-complete = false -## State and network +[[anchors]] +id = "ripe" +tal = "input/ripe.tal" +ta = "input/ripe.cer" # omit for automatic refresh from the TAL URI +constraints = "input/ripe.constraints" +``` -| Option | Default | Meaning | -| --- | --- | --- | -| `--rrdp-state-dir ` | `/.state` | Dedicated persistent root; database is stored in `repository-db/`. | -| `--rrdp-sync-mode ` | `auto` | Starting state requirements described below. | -| `--http-timeout-secs ` | `300` | HTTP timeout seconds; nonnegative with runtime minimum 1. Connection, retry and large-object transport policies also apply. Not a whole-run deadline. | -| `--http-root-cert ` | None | Repeatable extra HTTPS roots; system roots and hostname verification remain enabled. | -| `--max-ca-depth ` | `64` | Nonnegative CA traversal depth limit. | +All paths are relative to the TOML file, not the process working directory. +Absolute paths are accepted. `id` defaults to the TAL filename stem and must be +unique. `constraints` defaults to a same-stem `.constraints` file beside the +TAL. Automatic TA refresh supports HTTPS and rsync, validates the TAL binding, +signature and current validity, and stores a keyed cache at +`.ta-cache/`. `allow-valid-cache` may use a still-valid cached +certificate when refresh fails and marks the run partial; `require-fresh` does +not fall back. -| Mode | Starting state | Behavior | -| --- | --- | --- | -| `auto` | Empty or populated database | Select snapshot, delta or noop per repository. | -| `snapshot` | Missing or empty database | Start fresh; reject a populated database. | -| `delta` | Existing populated database | Resume state, allowing protocol snapshot fallback and new-repository snapshots. | +The remaining common keys and defaults are: -Unrecognized state layouts and unsupported database schemas are rejected. -Use a new dedicated directory for an incompatible format. State includes -repository objects, RRDP sessions and manifest anti-rollback metadata. Do not -share a writable database between processes. Changing `--out` changes default -state location unless `--rrdp-state-dir` is explicit. +| Key | Default | +| --- | ---: | +| `max-ca-depth` | 64 | +| `http-timeout-secs` | 300 (0 is normalized to 1) | +| `http-root-cert` | `[]` | +| `parallel-max-repo-sync-workers-global` | 8 | +| `parallel-phase2-object-workers` | 8 | +| `parallel-phase2-worker-queue-capacity` | 256 | +| `parallel-repo-worker-queue-capacity` | 256 (effective minimum 1024) | +| `log-level` | `info` | +| `log-format` | `text` | +| `log-progress-interval-secs` | 10 | +| `run-timeout-secs` | 0 (disabled) | +| `require-complete` | `false` | -RRDP references and redirects enforce HTTPS and same-origin rules. Extra -trust roots do not relax origin checks. No separate rsync timeout is exposed. +Validate-only keys are `out` (required), `rrdp-state-dir`, `rrdp-sync-mode`, +`ccr-out` and optional `publish-dir`. Daemon-only keys are `state-root` +(required), `interval-secs` (60), `max-runs` (unlimited), `retain-runs` (10) +and `shutdown-grace-secs` (30). Daemon does not accept validate-only keys; it +creates a per-run validate TOML internally. -## Workers and queues +## Inspection and outcomes -| Option | Default | Meaning | -| --- | ---: | --- | -| `--parallel-max-repo-sync-workers-global ` | 8 | Global repository transport workers. | -| `--parallel-phase2-object-workers ` | 8 | Object workers, independent of transport. | -| `--parallel-phase2-worker-queue-capacity ` | 256 | Object-worker queue capacity. | -| `--parallel-repo-worker-queue-capacity ` | 256 | Pending repository results; effective limit is `max(n, 1024)`. | +`--check-config` validates syntax, fields, local TAL/TA/constraints and existing +path types without network access, directory creation, state writes or locks. +`--print-config` emits `configVersion`, `mode`, `effective`, `sources` and +`normalizations` as JSON; defaults are filled and paths are absolute. -All values must be positive. Budgets are shared across TALs. Higher concurrency -can increase memory consumption and does not guarantee proportional speedup. +Validation writes `summary.json`, `vrps.csv`, `vaps.csv`, timing files and an +optional CCR. The final `result-complete.json` is written only after all required +artifacts are ready. `resultStatus` is `success`, `partial` or `failed`: -## Logging +| Status | Default exit | `require-complete = true` | +| --- | ---: | ---: | +| success | 0 | 0 | +| partial | 0 | 3 | +| failed | 2 | 2 | -| Option | Default | Values | -| --- | --- | --- | -| `--log-level ` | `info` | `off`, `error`, `warn`, `info`, `debug`, `trace` | -| `--log-progress-interval-secs ` | `10` | Periodic progress; `0` disables periodic events, not stage events. CLI overrides `PANDA_RPKI_LOG_PROGRESS_INTERVAL_SECS`. | -| `--log-format ` | `text` | `text`, `json` | +`summary.json` adds `trustAnchors`, `taFailedCount`, +`repositoryFailedCount`, `taCacheFallbackCount`, `resultStatus` and +`publicationStatus`. A cache fallback or a failed anchor makes an otherwise +complete artifact set partial; rejected objects alone do not. -Precedence: explicit option, then `PANDA_RPKI_LOG_LEVEL` or -`PANDA_RPKI_LOG_FORMAT`, then default. Each level includes more severe messages. -`off` suppresses logs but does not change exit codes. JSON logs contain -`timestamp`, `level`, `event`, and `fields`, one object per line. Logs use stderr; -help uses stdout; validation results are written to files. +With `publish-dir`, only a complete success is copied to an immutable run: -Other `PANDA_RPKI_*` variables in [Compose](docker.md) are template substitutions, -not configuration variables read directly by the binary. +```text +publish/runs//{vrps.csv,vaps.csv,summary.json,stage-timing.json, + analysis/timing.json,result-complete.json[,result.ccr]} +publish/latest -> runs/ +publish/last-attempt.json +``` -## Daemon +`latest` changes atomically and never moves for partial or failed attempts. +Daemon uses its existing `state-root/runs/run_000001/` layout and atomically +points `state-root/latest` at the newest successful run; it retains completed +runs according to `retain-runs` while preserving the latest pointer. -Daemon runs in the foreground, launching this executable's `validate` command -in a separate process per cycle. Pass validator options after `--`, without -the word `validate`. The controller owns `--out`, `--ccr-out`, -`--rrdp-state-dir` and `--rrdp-sync-mode`; do not pass them after `--`. +## Configuration lifecycle -| Option | Default | Meaning | -| --- | --- | --- | -| `--state-root ` | Required | Dedicated root for state, lifecycle metadata and run outputs. | -| `--interval-secs ` | 60 | Wait after run recording and retention; zero starts the next run immediately. | -| `--max-runs ` | Unlimited | Positive attempt count for this invocation, including failed attempts. | -| `--retain-runs ` | 10 | Positive number of completed run directories retained, including recorded failures. Older completed directories are deleted. | -| `--run-timeout-secs ` | 0 | Whole-run deadline; zero disables. Sends SIGTERM to the child group, then SIGKILL after two seconds if needed. | -| `--shutdown-grace-secs ` | 30 | On SIGINT/SIGTERM, stop starting runs and wait at most n seconds before terminating the active process group. | - -Options cannot repeat. Seconds are nonnegative. Empty state starts with -snapshot, successful continuation requests delta, and recovery after a failed -run with existing state requests auto. Actual operations can include noop and -snapshot fallback. Normal child failures/timeouts are recorded and retried; -corrupt lifecycle metadata or inability to record state stops the controller. - -Restart continues numbering and reuses state. Retention leaves incomplete, -unrelated and symlink run directories untouched. It does not trim history JSONL. -Do not remove a live lock file. After SIGKILL, ensure the old validator child -has stopped before restarting. Stop the entire container for Docker deployments. - -## Outputs and exit status - -| File | Content | -| --- | --- | -| `summary.json` | TAL, worker, validation and actual RRDP operation counts. | -| `vrps.csv` | ROA payloads: `asn,prefix,max_length`. | -| `vaps.csv` | ASPA payloads. | -| `stage-timing.json`, `analysis/timing.json` | Timing and analysis counters. | -| `.vrps-source.csv` | Intermediate CSV used to produce `vrps.csv`. | -| Requested CCR path | Optional CCR DER; automatic per run in daemon mode. | - -Daemon adds `daemon.lock`, `daemon-status.json`, `lifecycle.json`, -`run-summary.jsonl`, `state/repository-db/`, and `runs/run_000001/` etc. -Each run includes validator outputs, `run-meta.json`, `run-summary.json`, -`process-output.txt`, `validator.log` and optional `process-time.txt`. `/usr/bin/time` -provides user/system CPU and peak RSS when installed; Docker includes it. -Controller states: `starting`, `running`, `sleeping`, `exited`, `failed`. - -Help and successful validation return 0; argument and propagated runtime -errors return 2. Completion can include rejected objects or recoverable -repository failures: inspect warnings and outputs too. Failed runs can leave -partial output; use a separate output directory per cycle. - -A bounded daemon returns 2 if any attempt failed, otherwise 0. Stopping during -an interval exits normally. A forcibly terminated current run is recorded as -failed and causes exit 2. +TAL files are never auto-updated. Omitting an anchor's `ta` field refreshes its +TA certificate on each cycle; providing `ta` selects fixed mode. TOML is read +once at process start. Put `log-format = "text"` and `ta-refresh-policy` at the +top level, before any `[[anchors]]` tables. See [Docker reload steps](docker.md#graceful-stop-edit-and-reload) +and [complete container TOML examples](../docker/examples/README.md). diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..83dbf60 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,294 @@ +# Complete TOML configuration reference (schema v1) + +This reference documents all 26 top-level settings and four anchor fields. The product provides validation and a daemon +that schedules validation cycles. SLURM, RTR, ROV, HTTP/API services, offline +validation and additional payload JSON export are outside this configuration. + +## 1. Loading a configuration + +```bash +panda-rpki validate --config /srv/panda/validate.toml --check-config +panda-rpki validate --config /srv/panda/validate.toml --print-config +panda-rpki validate --config /srv/panda/validate.toml +``` + +Use `daemon` and a daemon configuration for continuous operation. Normal runs, +checks and printing all require one explicit file. Check and print are mutually +exclusive. There is no default-file discovery, include, profile merging, or +business-setting override. Old `--tal`, `--ta`, `--out`, `--log-format` options +and daemon `--` forwarding are rejected. `PANDA_RPKI_LOG_*` does not override +TOML. Standard dependency environments such as proxy settings are separate from +this application configuration contract. + +Settings come from TOML, then built-in defaults. Unknown keys, duplicate TOML +keys, incorrect types and a mismatched mode return exit code 2. Quote strings, +leave integers unquoted, use `true`/`false` for booleans, and use `[]` for an empty +array. TOML has no null value: omit optional paths rather than writing `null`, +an empty path, or an empty string. + +Write all top-level settings before the first `[[anchors]]` table. Fields after +that header belong to the current anchor until another table header appears. +For example, putting `log-format` after `tal` makes it an unknown anchor field. + +```toml +config-version = 1 +mode = "validate" +out = "output" +log-format = "text" # Top-level setting: keep this before [[anchors]]. + +[[anchors]] +id = "ripe" +tal = "input/ripe.tal" +``` + +## 2. Common top-level settings + +These settings apply to both commands. Defaults below describe omitted fields. +The examples explicitly choose a 1,800-second timeout and strict completion; +those choices do not change the built-in defaults. + +| Key | Type | Default / required | Meaning and constraints | +| --- | --- | --- | --- | +| `config-version` | Integer | Explicit `1` required | Configuration schema version, not the software version. Missing or unsupported versions are rejected. | +| `mode` | String | Required | `validate` or `daemon`; must match the command. | +| `anchors` | Array of tables | At least one required | Write `[[anchors]]` entries; see section 5. No automatic TAL-directory scan. | +| `ta-refresh-policy` | String | `allow-valid-cache` | On automatic TA download failure, permit a revalidated, usable cached certificate. `require-fresh` prohibits fallback. Does not replace fixed TA files. | +| `max-ca-depth` | Nonnegative integer | 64 | Maximum CA validation-tree depth. Zero is accepted by configuration parsing, but is a depth limit, not unlimited traversal. Usually keep 64. | +| `http-timeout-secs` | Nonnegative integer, seconds | 300 | HTTP fetch timeout. Zero is normalized to one second, not unlimited. Does not set the whole-run or rsync timeout. | +| `http-root-cert` | Array of path strings | `[]` | Additional HTTP TLS root certificates in PEM format. Every file must be readable and parseable. Empty means no custom additions; system TLS trust remains. These are not RPKI TA certificates. | +| `parallel-max-repo-sync-workers-global` | Positive integer | 8 | Global repository-sync worker count. Zero is rejected. | +| `parallel-phase2-object-workers` | Positive integer | 8 | Phase-2 object-processing worker count. Zero is rejected. | +| `parallel-phase2-worker-queue-capacity` | Positive integer | 256 | Phase-2 worker queue capacity. Zero is rejected. | +| `parallel-repo-worker-queue-capacity` | Positive integer | Input default 256; effective 1024 | Repository worker queue capacity. Zero is rejected; values 1–1023 are raised to 1024. Print-config reports normalization. | +| `log-level` | String | `info` | `off`, `error`, `warn`, `info`, `debug`, or `trace`; case-sensitive. | +| `log-format` | String | `text` | Human-readable English diagnostics or `json` events, one object per line. Does not change summary or metadata formats. | +| `log-progress-interval-secs` | Nonnegative integer, seconds | 10 | Periodic progress interval. Zero disables periodic progress, not other stage messages. Levels off/error/warn do not start a progress thread. | +| `require-complete` | Boolean | false | A partial run exits 0 when false, 3 when true. Does not change validation rules, turn partial into success, or permit partial publication. | +| `run-timeout-secs` | Nonnegative integer, seconds | 0 | Whole-run budget; zero disables it. Validate includes runtime TA acquisition and output preparation. Daemon measures from child startup and enforces the budget in the controller. Check/print do not use this budget. | + +Integers are also limited by TOML and the target platform's integer types. +Larger worker counts do not guarantee higher throughput; consider CPU, memory +and repository-service load. + +## 3. Validate-only settings + +These keys are rejected in daemon configurations, even when set to their usual +default values. + +| Key | Type | Default / required | Meaning and constraints | +| --- | --- | --- | --- | +| `out` | Directory path | Required | Current-attempt output: CSV, summary, timing files and completion marker. A partial run can also produce artifacts. | +| `rrdp-state-dir` | Directory path | `/.state` | Persistent protocol state containing repository-db. Reuse for incremental synchronization; runtime access is exclusive. | +| `rrdp-sync-mode` | String | `auto` | `auto` selects synchronization according to available state; `snapshot` requires empty state; `delta` requires an existing database. Individual repositories can still use protocol fallback. | +| `ccr-out` | File path | Omitted: no CCR | For example, `output/result.ccr`. Can be outside out. An existing target must be a file, not a directory. | +| `publish-dir` | Directory path | Omitted: disabled | Immutable successful result copies and latest pointer. Must not equal, contain, or be contained by out or protocol state. Uses additional disk space; Successful copies in publish/runs are not automatically pruned. | + +## 4. Daemon-only settings + +These keys are rejected in validate configurations. + +| Key | Type | Default / required | Meaning and constraints | +| --- | --- | --- | --- | +| `state-root` | Directory path | Required | Controller root; daemon manages state, runs, locks, status files and latest beneath it. | +| `interval-secs` | Nonnegative integer, seconds | 60 | Delay after one cycle finishes before the next starts, not a fixed start-to-start period. Zero runs cycles consecutively. | +| `max-runs` | Positive integer | Omitted: unlimited | Maximum cycles in the current process, not the maximum historical sequence number. Zero is rejected. | +| `retain-runs` | Positive integer | 10 | Retain the newest N completed runs plus an older run referenced by latest. Not a hard limit on every directory. Zero is rejected. | +| `shutdown-grace-secs` | Nonnegative integer, seconds | 30 | Time allowed for the active child to finish after a stop request. Then TERM, followed by KILL after another two seconds. Zero skips waiting for natural completion. Separate from run timeout. | + +Each cycle generates `runs/run_XXXXXX/validator-config.toml` with absolute paths. +The controller supplies output, CCR and synchronization settings. The child +configuration has run-timeout-secs=0 because the parent enforces the budget. +Editing generated child configurations is not a persistent configuration change. + +The controller requests snapshot for empty state, delta after success and auto +after partial/failed. Ordinary partial/failed cycles do not prevent subsequent +cycles. On finite completion or stop, the aggregate exit is 2 if any cycle failed; +otherwise 3 if strict completion is enabled and any cycle was partial; otherwise +0. An exited container with code 3 does not imply Docker failed to stop it. + +## 5. Anchor fields + +| Key | Type | Default / required | Meaning and constraints | +| --- | --- | --- | --- | +| `id` | String | TAL filename stem | Identifier in results, logs and constraints. An explicitly empty string also falls back to the filename stem. Resolved IDs must be nonempty and unique; explicit names are preferred. | +| `tal` | File path | Required | TAL obtained from an independently trusted source; must exist and parse during preflight. Contains certificate URIs and the trusted public key. | +| `ta` | File path | Omitted: automatic mode | If present, selects a fixed DER TA certificate. Preflight checks binding, certificate profile/signature and validity. Network downloads do not silently replace it. | +| `constraints` | File path | Discover same-stem `.constraints` | For ripe.tal, try ripe.constraints. Absence of that file means no additional constraints. An explicit path must exist and parse. Omitting this setting does not disable discovery; an empty string is not a disable switch. | + +Automatic refresh updates the **TA certificate, never the TAL file**. Each +validate run or daemon cycle tries the TAL's HTTPS/rsync URIs in order. There is +no separate background refresh timer, and no invented HTTPS alternative for an +rsync-only TAL. Candidates must pass certificate and TAL-binding validation. + +The cache is `.ta-cache`, keyed by the TAL public-key SHA-256. +Download failure permits fallback only under allow-valid-cache with a usable, +revalidated cached certificate; that makes the result at least partial. +Require-fresh prohibits fallback. Existing fixed `.cer` files do not seed this +cache automatically; switching to automatic mode initially needs reachable URIs. +One failed TA with others completing yields partial; all TAs failing yields +failed. A fixed-certificate error found during preflight exits 2 immediately. + +## 6. Paths, mounts, reload and concurrency + +- `--config FILE` is relative to the process working directory. Paths inside + TOML are relative to the resolved configuration file location; existing + symlink ancestors are resolved. Absolute paths are accepted. There is no `~`, + `$HOME` or `${VAR}` expansion inside TOML strings. +- Moving a configuration can change every relative path. Keep temporary profile + copies in the same directory; concurrent instances additionally need separate + output, state and publication roots. +- Docker TOML uses container paths; Compose CONFIG_FILE/INPUT_DIR/STATE_DIR + variables use host paths. Those variables select mounts/image/UID, not + application settings. +- The TA cache is a sibling of protocol state. With a writable /state mount, + use `rrdp-state-dir="/state/repository"` so repository and repository.ta-cache + are both writable and persistent. Using /state directly places the cache at + /state.ta-cache, which needs a separate mount. Plan stopped migration for + existing databases; changing a path does not migrate the old database. +- Daemon's /data/state and /data/state.ta-cache already share the /data mount. +- TOML is loaded once at process startup. Daemon rereads the configured TAL/TA/ + constraints files each cycle. TOML changes require a process restart. If an + editor or sed -i replaces a single-file bind mount's host inode, recreate the + container so the new file is mounted. +- Runtime locks protect output, state and publication roots. Preflight does not + acquire runtime locks. Passing it does not prove future write permissions, + network access, free space or exclusive access. + +See [Docker operations](docker.md#graceful-stop-edit-and-reload) for complete +stop, check and recreate commands. + +## 7. Results and consistent reads + +| Status | Conditions in brief | Default exit | Strict exit | Update latest | +| --- | --- | --- | --- | --- | +| success | All configured TAs complete, no terminal repository failures or cache degradation, required artifacts complete | 0 | 0 | Yes | +| partial | Some TA/repository terminal failure or valid TA-cache fallback, with usable completed results | 0 | 3 | No | +| failed | Configuration/lock/fatal failure, all TAs fail, timeout, interruption, output or publication failure | 2 | 2 | No | + +Rejecting invalid objects according to the standards does not itself imply +partial. A recoverable RRDP failure followed by valid fallback need not imply +partial either. Read resultStatus and the typed repositoryFailedCount rather +than inferring status from warning counts. + +Out contains the current attempt. Atomic replacement of individual files is +not an atomic multi-file snapshot. The result-complete.json marker is written +last; a partial attempt can have a marker and still be partial. Only success is +copied to publish/runs/ and switches latest. Resolve latest once, then read +all files from that immutable directory. Daemon uses its existing run directory +without a second publication copy. Latest is absent before the first success; +upgrading an existing state root does not create latest from historical runs; +the pointer is established by the first subsequent complete success. + +Text logging affects diagnostics only: summary, completion markers and printed +configuration remain JSON. Daemon lifecycle diagnostics are in container logs; +child validation diagnostics are in that run's validator.log. Historical logs +are not converted. + +## 8. Complete examples and selection + +The [native configuration directory](../examples/config/README.md) contains +copyable files: + +| File | Scenario | Explicit choices | +| --- | --- | --- | +| validate-minimal.toml | Minimal single automatic TA | Required fields only; built-in defaults otherwise | +| validate-full.toml | All common/validate settings and successful publication | Text logs, strict completion, 1,800-second budget | +| validate-fixed.toml | Fixed certificate | Both tal and ta; certificate must match and be valid | +| daemon-all5.toml | Continuous five-TA automatic refresh | All common/daemon settings; max-runs omitted | +| daemon-three.toml | Three five-TA cycles | max-runs=3, interval=0, independent state root | + +For containers, use the [four Docker configurations](../docker/examples/README.md). +Mounting a native relative-path example under /config does not automatically +make its paths correct. + +### 8.1 Minimal complete validate configuration + +Save as /srv/panda/validate.toml and prepare /srv/panda/input/ripe.tal: + +```toml +config-version = 1 +mode = "validate" +out = "output" + +[[anchors]] +id = "ripe" +tal = "input/ripe.tal" +``` + +Run `panda-rpki validate --config /srv/panda/validate.toml`. Default state is +/srv/panda/output/.state; the TA cache is /srv/panda/output/.state.ta-cache. +Defaults are text/info, no CCR, no publication root and no whole-run timeout. +Partial also exits 0 by default. Add require-complete=true at the top level for +strict completion. + +### 8.2 Fixed, automatic and mixed anchors: replacement fragment + +```toml +[[anchors]] +id = "ripe" +tal = "input/ripe.tal" +ta = "input/ripe.cer" # Fixed; remove this key to switch to automatic. + +[[anchors]] +id = "apnic" +tal = "input/apnic.tal" # Automatic. +``` + +This replaces the anchors section of a complete configuration; it is not a +standalone file. Mixed mode needs no extra mode switch. Changing +TA refresh policy does not make an anchor with an explicit ta automatic. + +### 8.3 Logging, TLS and worker tuning: top-level fragment + +```toml +log-format = "json" +log-level = "debug" +log-progress-interval-secs = 0 +http-timeout-secs = 600 +http-root-cert = ["tls/internal-root.pem"] +parallel-phase2-object-workers = 4 +parallel-max-repo-sync-workers-global = 4 +parallel-phase2-worker-queue-capacity = 256 +parallel-repo-worker-queue-capacity = 1024 +``` + +Replace existing top-level settings; do not add duplicate keys. Prepare the +custom TLS PEM file first, or keep [] when no extra roots are needed. This does +not change the TAL public key, relax TA validation or disable system TLS checks. + +## 9. Preflight and troubleshooting + +Check first, print effective configuration, then run normally. Check does not +use the network, take runtime locks or create run directories. It reads TOML, +TALs, fixed TAs, constraints and extra HTTP roots. Print performs the same checks +and emits effective settings, sources and normalizations. + +For example, setting http-timeout-secs to 0 and the repository queue to 256 +produces this excerpt, not the full print output: + +```json +{ + "effective": { + "http-timeout-secs": 1, + "parallel-repo-worker-queue-capacity": 1024 + }, + "normalizations": [ + {"field":"http-timeout-secs","input":0,"effective":1}, + {"field":"parallel-repo-worker-queue-capacity","input":256,"effective":1024} + ] +} +``` + +| Symptom | Explanation and action | +| --- | --- | +| Missing configuration/version error | Explicitly write config-version=1 and pass --config. | +| Mode mismatch | Match the command and mode. | +| Unknown anchor field log-format | Move it before the first [[anchors]] table. | +| Out is not allowed in daemon | Remove validate-only keys; the controller owns output. | +| Zero workers/queues/max-runs/retention | Use positive values; omit max-runs for unlimited cycles. | +| File not found | Resolve paths against the config directory; check container mounts. | +| No automatic TA cache | First use needs a successful download; fixed cer files do not seed the cache. | +| Preflight passes, runtime fails | Check does not test network reachability, runtime locks or actual writes. | +| Logs still JSON | Check top-level settings, recreate the container and inspect new logs. Metadata remains JSON intentionally. | +| Exited with code 3 | Stopped with partial aggregate results; inspect summaries/lifecycle logs rather than repeatedly stopping. | +| Latest absent | No complete success yet; partial does not publish even when it exits 0. | diff --git a/docs/conformance-matrix.md b/docs/conformance-matrix.md index dfd6020..0db3228 100644 --- a/docs/conformance-matrix.md +++ b/docs/conformance-matrix.md @@ -12,7 +12,7 @@ | CMS signed-object profile | RFC 6488 §§2–3; RFC 9589 §4 | `src/model/signed_object.rs` | signed-object profile tests | | RRDP notification, snapshot, delta/replace/withdraw and fallback | RFC 8182 §§3.4–3.5 | `src/repository/sync/rrdp.rs`, `src/repository/sync/repo.rs` | snapshot/delta state-restart unit tests and synthetic Docker tests | | RRDP direct-reference and redirect origin checks | RFC 9674 §3.2 | `src/repository/fetch/http.rs`, `src/repository/sync/rrdp.rs` | cross-origin unit tests | -| Repeated TAL/TA pairing and aggregate output | RFC 8630 §2 for each TAL; aggregation is an implementation choice | `src/cli/mod.rs`, `src/cli/validate.rs`, `src/runtime/` | CLI parser checks and multi-TAL integration evidence | +| Named TOML anchors with TAL/TA binding and aggregate output | RFC 8630 §2 for each TAL; aggregation is an implementation choice | `src/cli/mod.rs`, `src/cli/validate.rs`, `src/runtime/` | CLI parser checks and multi-TAL integration evidence | | Bounded repository and object workers (default 8 each, shared across TALs) | operational support for the standard validation pipeline | `src/scheduler/`, `src/runtime/` | worker-pool unit tests | | Resumable RRDP protocol state | RFC 8182 §3.4 | `src/repository/storage/`, `src/repository/sync/rrdp.rs` | atomic state index and restart delta test | | CCR manifest, ROA, ASPA, trust-anchor and router-key states | `draft-ietf-sidrops-rpki-ccr-11` §§2–4 | `src/ccr/` | encode/decode tests and integration artifacts | @@ -22,7 +22,8 @@ References identify the implemented profiles, not a certification of complete RFC compliance. Draft revisions are intentionally explicit and are not claims of compatibility with later revisions. The CLI uses HTTPS RRDP, requires local -TAL/TA pairs, and does not provide RTR or automatic TA-certificate refresh. +TALs, supports fixed or automatically refreshed TA certificates, and does not provide RTR. +TAL files themselves are not automatically updated. CCR tests establish encoding and payload consistency, not independent revalidation of the entire trust chain by another relying party. diff --git a/docs/docker.md b/docs/docker.md index 95758c4..33a5f79 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -1,199 +1,189 @@ # Docker and Compose -Run commands from the project root. A Linux Docker engine is required for the -documented deployment and integration tests. +For every field, default, constraint and complete example, see the [TOML configuration reference](configuration.md)。 + +The image contains the validator, `rsync`, CA certificates and `/usr/bin/time`. +The root filesystem is read-only; input and configuration are read-only; output and +state mounts are writable. The application receives a subcommand and one configuration option: +`validate|daemon --config FILE`. + +Build the image: ```bash docker build -f docker/Dockerfile -t panda-rpki:v0.1.0 . ``` -The image defaults to an unprivileged user and includes CA certificates, -`rsync` and `/usr/bin/time`. Compose uses a read-only root filesystem and a -temporary `/tmp`. Host input mounts are read-only; state and outputs are writable. +## Single validate run -## Single anchor +Create a TOML whose paths are valid **inside** the container: + +```toml +config-version = 1 +mode = "validate" +out = "/output" +rrdp-state-dir = "/state/repository" +ccr-out = "/output/result.ccr" + +[[anchors]] +id = "anchor" +tal = "/input/anchor.tal" +# ta = "/input/anchor.cer" # uncomment for fixed TA mode +``` + +Run the provided Compose template: ```bash mkdir -p state output +export CONFIG_FILE="$PWD/validate.toml" export INPUT_DIR="$PWD/input" STATE_DIR="$PWD/state" OUTPUT_DIR="$PWD/output" -export TAL_FILE=anchor.tal TA_FILE=anchor.cer export PUID="$(id -u)" PGID="$(id -g)" docker compose -f docker/compose.yaml run --build --rm validator ``` -Supply absolute host paths as above. Relative Compose paths are resolved from -the Compose file's directory, which is `docker/`. Build context is explicitly -the project root. `PANDA_RPKI_RRDP_SYNC_MODE` defaults to `auto`. +The template mounts the TOML read-only at `/config/validate.toml` and invokes +`panda-rpki validate --config /config/validate.toml`. Host variables only select +mounts, image and container UID/GID; they are not read as validator settings. -## All five regional anchors +## All five RIRs -Follow [Prepare all five RIR trust anchors](trust-anchors.md) to download and -check each TAL and matching certificate under `input/`, then run: +Put the five RIR TAL files (TA certificates are optional) under `input/` and list them as named anchors in a +TOML with `/input/...` paths. Then set `CONFIG_FILE` to that file and run: ```bash -mkdir -p state output -export INPUT_DIR="$PWD/input" STATE_DIR="$PWD/state" OUTPUT_DIR="$PWD/output" -export PUID="$(id -u)" PGID="$(id -g)" -export AFRINIC_TAL_FILE=afrinic.tal AFRINIC_TA_FILE=afrinic.cer -export APNIC_TAL_FILE=apnic.tal APNIC_TA_FILE=apnic.cer -export ARIN_TAL_FILE=arin.tal ARIN_TA_FILE=arin.cer -export LACNIC_TAL_FILE=lacnic.tal LACNIC_TA_FILE=lacnic.cer -export RIPE_TAL_FILE=ripe.tal RIPE_TA_FILE=ripe.cer +export CONFIG_FILE="$PWD/all5.toml" +export INPUT_DIR="$PWD/input" STATE_DIR="$PWD/state/all5" OUTPUT_DIR="$PWD/output/all5" docker compose -f docker/compose.all5.yaml run --build --rm validator ``` -| Template variable | Default | Applies to | -| --- | --- | --- | -| `PANDA_RPKI_WORKERS` | 8 | all5 and daemon object workers | -| `PANDA_RPKI_REPO_WORKERS` | 8 | all5 and daemon transport workers | -| `PANDA_RPKI_WORKER_QUEUE_CAPACITY` | 256 | all5 object queue | -| `PANDA_RPKI_REPO_QUEUE_CAPACITY` | 256 | all5 daemon repository queue | -| `PANDA_RPKI_HTTP_TIMEOUT_SECS` | 600 | all5 HTTP timeout (native CLI default is 300) | -| `PANDA_RPKI_RRDP_SYNC_MODE` | auto | single-anchor and all5 | +The application still receives only `validate --config`; workers, timeouts, +logging and sync policy belong in `all5.toml`. -The all5 CCR is `OUTPUT_DIR/all5.ccr`; the single-anchor CCR is -`OUTPUT_DIR/result.ccr`. Reuse `STATE_DIR` and select new `OUTPUT_DIR` values -to retain multiple cycles. These templates take no bundled RIR input files. +## Daemon -## All-five daemon service +Daemon TOML paths are also container paths. `state-root` owns run directories, +protocol state and the `latest` pointer: -The all-five daemon template combines the five regional TAL/TA pairs with the -continuous daemon controller. Prepare the same `input/` directory used by the -all-five one-shot example, then run: +```toml +config-version = 1 +mode = "daemon" +state-root = "/data" +interval-secs = 600 +retain-runs = 10 +run-timeout-secs = 1800 +shutdown-grace-secs = 30 + +[[anchors]] +id = "anchor" +tal = "/input/anchor.tal" +# ta = "/input/anchor.cer" # uncomment for fixed TA mode +``` + +Run the daemon template: ```bash -mkdir -p data/all5-daemon -export INPUT_DIR="$PWD/input" DAEMON_DIR="$PWD/data/all5-daemon" -export AFRINIC_TAL_FILE=afrinic.tal AFRINIC_TA_FILE=afrinic.cer -export APNIC_TAL_FILE=apnic.tal APNIC_TA_FILE=apnic.cer -export ARIN_TAL_FILE=arin.tal ARIN_TA_FILE=arin.cer -export LACNIC_TAL_FILE=lacnic.tal LACNIC_TA_FILE=lacnic.cer -export RIPE_TAL_FILE=ripe.tal RIPE_TA_FILE=ripe.cer +mkdir -p daemon-state +export CONFIG_FILE="$PWD/daemon.toml" +export INPUT_DIR="$PWD/input" DAEMON_DIR="$PWD/daemon-state" export PUID="$(id -u)" PGID="$(id -g)" -docker compose -f docker/compose.all5-daemon.yaml up -d --build -docker compose -f docker/compose.all5-daemon.yaml logs -f validator +docker compose -f docker/compose.daemon.yaml up --build ``` -The template uses a 60-second interval by default, keeps 10 completed runs, -and leaves the daemon running until it is stopped. It writes each run below -`DAEMON_DIR/runs/` and keeps shared protocol state below `DAEMON_DIR/state/`. -The default diagnostic format is `text`, with a source file and line appended -to each diagnostic record; select `json` explicitly for machine consumers. -Set `PANDA_RPKI_INTERVAL_SECS` only when a different interval is intended. Stop -the service without deleting its data with: +For an all-RIR daemon use `docker/compose.all5-daemon.yaml` with an all-RIR +TOML. Stop the whole Compose service for an upgrade so the state-root lock and +validator child are released; do not delete `daemon-state`. + +## Configuration and persistence + +Run `docker compose ... run --rm validator validate --config ... --check-config` +only when the image command is overridden deliberately; the normal templates +already pass the config. Check/print never download TA files or open RocksDB. +Successful validate publication, when `publish-dir` is configured, is an +immutable `publish/runs/` plus `publish/latest`; daemon publication uses its +existing `state-root/runs/run_` directory. Cache files are written beside +the protocol state as `.ta-cache/`. + +If a host path is not writable by the selected UID/GID, pre-create it and fix +ownership. Preserve existing state and run history when changing images. Verify +`summary.json`, `result-complete.json`, `vrps.csv`, `vaps.csv`, timing files and +the `latest` link after a deployment. + +## Complete configuration examples + +Copy a template before editing it; set `CONFIG_FILE` to the copy's absolute path. +These templates use automatic **TA certificate** refresh, text/info logs and +`require-complete = true`. They never download or update TAL files. + +| Compose file | Complete TOML | +| --- | --- | +| `compose.yaml` | [validate.toml](../docker/examples/validate.toml) | +| `compose.all5.yaml` | [validate-all5.toml](../docker/examples/validate-all5.toml) | +| `compose.daemon.yaml` | [daemon.toml](../docker/examples/daemon.toml) | +| `compose.all5-daemon.yaml` | [daemon-all5.toml](../docker/examples/daemon-all5.toml) | + +For example, from the repository root: ```bash -docker compose -f docker/compose.all5-daemon.yaml stop -``` - -## Upgrade a daemon deployment - -An upgrade replaces the image and recreates the container. It does not replace -the host data directory. Keep the same `DAEMON_DIR`, Compose project, TAL/TA -inputs and controller settings so the new image can resume the existing -protocol state. Do not run two writable daemons against the same state root. - -Run the following from the deployment directory that owns the Compose file. Use -a unique image tag for every candidate so the image used by a run is auditable: - -```bash -export COMPOSE_FILE="${COMPOSE_FILE:-docker/compose.all5-daemon.yaml}" -export COMPOSE_PROJECT_NAME=panda-rpki-all5 -export PANDA_RPKI_IMAGE=panda-rpki:2026-09-11-logging -export SOURCE_DIR=/path/to/panda-rpki-oss - -# Keep the existing INPUT_DIR, DAEMON_DIR, TAL/TA variables and settings. -docker compose -p "$COMPOSE_PROJECT_NAME" -f "$COMPOSE_FILE" ps -test "$(jq -r .state "$DAEMON_DIR/daemon-status.json")" = sleeping - -# Build from the source tree before stopping the running service. -docker build -f "$SOURCE_DIR/docker/Dockerfile" -t "$PANDA_RPKI_IMAGE" "$SOURCE_DIR" -docker image inspect "$PANDA_RPKI_IMAGE" --format '{{.Id}}' - -# Save configuration and create a consistent stopped-state backup. -backup="upgrade-backup-$(date -u +%Y%m%dT%H%M%SZ)" -mkdir -p "$backup" -cp "$COMPOSE_FILE" "$backup/compose.yaml" -if test -f .env; then cp .env "$backup/env"; fi -docker compose -p "$COMPOSE_PROJECT_NAME" -f "$COMPOSE_FILE" stop -cp -a "$DAEMON_DIR/state" "$backup/state" -cp -a "$DAEMON_DIR/daemon-status.json" "$DAEMON_DIR/lifecycle.json" "$backup/" - -# Recreate the same service with the new image; bind-mounted data is retained. -docker compose -p "$COMPOSE_PROJECT_NAME" -f "$COMPOSE_FILE" up -d --no-build --force-recreate -docker inspect "$(docker compose -p "$COMPOSE_PROJECT_NAME" -f "$COMPOSE_FILE" ps -q validator)" \ - --format 'image={{.Config.Image}} running={{.State.Running}}' -``` - -After the first upgraded run, verify that the run sequence advanced, the run -completed, and the new log/outputs exist. The exact run directory is printed by -the daemon status and can be selected with `find`: - -```bash -latest_run="$(find "$DAEMON_DIR/runs" -mindepth 1 -maxdepth 1 -type d -name 'run_*' | sort | tail -1)" -test -s "$latest_run/run-summary.json" -test -s "$latest_run/validator.log" -test -s "$latest_run/result.ccr" -grep -Eq '\[src/[^]]+\.rs:[0-9]+\]$' "$latest_run/validator.log" -jq . "$latest_run/run-summary.json" -``` - -The upgrade process preserves completed run directories until normal retention -removes them. Archive important runs separately if they must outlive -`PANDA_RPKI_RETAIN_RUNS`. - -### Roll back - -Stop the upgraded service before changing the image. Restore the previous -Compose file and `.env`, or set `PANDA_RPKI_IMAGE` to the previously verified -tag, then recreate the service with the same project and data directory: - -```bash -docker compose -p "$COMPOSE_PROJECT_NAME" -f "$COMPOSE_FILE" stop -cp "$backup/compose.yaml" "$COMPOSE_FILE" -if test -f "$backup/env"; then cp "$backup/env" .env; fi -unset PANDA_RPKI_IMAGE -docker compose -p "$COMPOSE_PROJECT_NAME" -f "$COMPOSE_FILE" up -d --no-build --force-recreate -``` - -If the new binary changed the state schema or wrote incompatible state, stop the -service and move the current state aside before restoring the matching -stopped-state backup: - -```bash -mv "$DAEMON_DIR/state" "$DAEMON_DIR/state-incompatible-$(date -u +%Y%m%dT%H%M%SZ)" -cp -a "$backup/state" "$DAEMON_DIR/state" -``` - -Never delete `DAEMON_DIR` or use `docker compose down -v` as part of a routine -upgrade. Check [state compatibility](command-line-reference.md#state-and-network) -before deploying a binary with storage changes. - -## Single-anchor daemon service - -```bash -mkdir -p data/daemon -export INPUT_DIR="$PWD/input" DAEMON_DIR="$PWD/data/daemon" -export TAL_FILE=anchor.tal TA_FILE=anchor.cer +cp docker/examples/daemon-all5.toml daemon.toml +export CONFIG_FILE="$PWD/daemon.toml" +export INPUT_DIR="$PWD/input" DAEMON_DIR="$PWD/daemon-state" +mkdir -p "$DAEMON_DIR" export PUID="$(id -u)" PGID="$(id -g)" -export PANDA_RPKI_INTERVAL_SECS=600 PANDA_RPKI_RETAIN_RUNS=10 -docker compose -f docker/compose.daemon.yaml up -d --build -docker compose -f docker/compose.daemon.yaml logs -f validator -# Stop gracefully; persistent host data remains available. -docker compose -f docker/compose.daemon.yaml stop +docker compose -p panda-all5 -f docker/compose.all5-daemon.yaml up -d --build ``` -Defaults: interval 60 seconds, retain 10, `PANDA_RPKI_RUN_TIMEOUT_SECS=0` -(disabled), log level `info`, log format `text`. The daemon template passes -`PANDA_RPKI_LOG_LEVEL` and `PANDA_RPKI_LOG_FORMAT` into the container. -It restarts unless stopped and allows 40 seconds for shutdown, exceeding the -controller's default 30-second grace plus termination wait. +For single validation, `/state` is the writable mount parent. Set +`rrdp-state-dir = "/state/repository"` so both `/state/repository` and its +sibling `/state/repository.ta-cache` are persistent and writable. Existing +fixed-mode deployments with `rrdp-state-dir = "/state"` must plan a stopped +state-layout migration or mount `/state.ta-cache` separately before enabling +automatic mode. Do not silently point an existing database at an empty path. +Daemon needs no such migration: `/data/state` and `/data/state.ta-cache` already +share the writable `/data` mount. -For multiple anchors, append TAL/TA pairs after `--` in the daemon service's -command. For other validator options, edit that same list. A finite run can -be invoked with `docker compose run --rm validator daemon ...` using the -[daemon CLI](command-line-reference.md#daemon); use `--max-runs` before `--`. +## Graceful stop, edit and reload -Compose variables are expanded into arguments; only the two documented log -environment variables are read directly by the binary. For single-anchor/all5 -logs, pass them with `docker compose run -e PANDA_RPKI_LOG_LEVEL=debug ...`. +Use the same project name, Compose file and environment file on every command. +The following assumes the `panda-all5` project created above: + +```bash +dc() { docker compose -p panda-all5 -f docker/compose.all5-daemon.yaml "$@"; } +dc stop --timeout 40 validator +dc ps -a +cp -a "$CONFIG_FILE" "$CONFIG_FILE.bak.$(date -u +%Y%m%dT%H%M%SZ)" +``` + +Edit the existing top-level field to `log-format = "text"`. To enable automatic +TA refresh, remove each desired anchor's `ta` field; keep its `id` and `tal`. +`ta-refresh-policy = "allow-valid-cache"` allows a revalidated cache fallback +and marks that run partial; `"require-fresh"` rejects the fallback. Missing or +invalid cache cannot satisfy the first automatic run if all downloads fail. + +Check the changed file and recreate the service: + +```bash +dc run --rm --no-deps validator daemon --config /config/daemon.toml --check-config +dc run --rm --no-deps validator daemon --config /config/daemon.toml --print-config +# Continue only after Configuration valid. and the expected effective settings. +dc up -d --no-build --force-recreate validator +dc logs --since 2m validator +``` + +Recreation is intentional: editors and `sed -i` can replace the host file's +inode, while an existing single-file bind mount may retain the old file. +`start` or `restart` alone is not the documented configuration reload procedure. +Recreation retains bind-mounted state and run history. + +SIGTERM prevents further cycles and allows the active child up to +`shutdown-grace-secs` (30 here) to finish. If it does not finish, the controller +sends TERM, then KILL after another 2 seconds. Docker's 40-second timeout leaves +room for controller cleanup. It is not an unlimited wait for the current cycle. +To avoid interrupting a long cycle, stop during the daemon's sleeping interval. +`exited` confirms the container stopped; exit 3 means the controller recorded +partial runs under `require-complete`, and does not mean stop failed. Inspect +lifecycle logs to establish whether the last child finished or was interrupted. + +The controller logs are in `docker compose logs`; child diagnostics are in +`data/runs/run_XXXXXX/validator.log` on the host. Existing logs keep their original +format. JSON summary/metadata files remain JSON even with text diagnostics. +`latest` exists only after a complete success; partial runs do not create or +replace it, including the first run after upgrading an old state root. diff --git a/docs/getting-started.md b/docs/getting-started.md index 12b0e15..bd40b44 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -1,5 +1,7 @@ # Getting started +For every field, default, constraint and complete example, see the [TOML configuration reference](configuration.md)。 + Panda RPKI v0.1.0 is a Linux command-line application. You can use Docker without installing Rust, or build a native binary. It requires outbound HTTPS to RPKI repositories, a writable state directory, and enough disk space for repository @@ -7,10 +9,11 @@ objects and outputs. Resource use depends on the selected trust anchors. ## Prepare trust anchors -A trust anchor defines what you trust. Obtain its TAL from the responsible RIR -and obtain the current DER certificate from a certificate URI in that TAL. -Panda RPKI checks that the certificate matches the TAL public key. It takes -both files as explicit inputs; it does not refresh the supplied TA file for you. +A trust anchor defines what you trust. Obtain its TAL from the responsible RIR. +Panda RPKI checks the TAL public key and either uses a configured DER certificate +or refreshes a certificate from the TAL's HTTPS/rsync URI. Automatic refresh +stores a validated last-known certificate beside the protocol state; it never +changes the TAL itself. For example, the [RIPE NCC trust-anchor page](https://www.ripe.net/manage-ips-and-asns/resource-management/rpki/ripe-ncc-rpki-trust-anchor-structure/) publishes the RIPE NCC TAL. From the project root: @@ -20,6 +23,7 @@ mkdir -p input curl --fail --location --proto '=https' --proto-redir '=https' \ --connect-timeout 15 --max-time 120 \ https://tal.rpki.ripe.net/ripe-ncc.tal -o input/anchor.tal +# Optional: download a TA certificate only for fixed mode. curl --fail --location --proto '=https' --proto-redir '=https' \ --connect-timeout 15 --max-time 120 \ https://rpki.ripe.net/ta/ripe-ncc-ta.cer -o input/anchor.cer @@ -36,10 +40,11 @@ commands, input checks, and the filenames used by Compose. ## Docker first run -Follow the [README Quick start](../README.md#quick-start). It creates -`output/summary.json`, `output/vrps.csv` and `output/result.ccr`. State is retained -under `state/repository-db/`. View warnings in the terminal and inspect the -summary to distinguish rejected objects from successful repository updates. +Follow the [README Quick start](../README.md#quick-start). The only application +argument is `--config`; all paths, anchors, workers and logging belong in that +TOML file. It creates `output/summary.json`, `output/vrps.csv` and +`output/result.ccr`. State is retained under `state/repository/repository-db/` and the +automatic TA cache is a sibling `state/repository.ta-cache/` directory. ## Build from source @@ -53,14 +58,30 @@ cargo build --locked --release target/release/panda-rpki --help ``` -Run with the same inputs: +Create a configuration file (paths are relative to the file, or use absolute +container paths): + +```toml +config-version = 1 +mode = "validate" +out = "output/native" +rrdp-state-dir = "state" +ccr-out = "output/native/result.ccr" + +[[anchors]] +id = "anchor" +tal = "input/anchor.tal" +# ta = "input/anchor.cer" # uncomment for fixed mode +``` + +Run with that configuration: ```bash -target/release/panda-rpki validate \ - --tal input/anchor.tal --ta input/anchor.cer \ - --rrdp-state-dir state --out output/native \ - --ccr-out output/native/result.ccr +target/release/panda-rpki validate --config validate.toml ``` +Use `validate --config validate.toml --check-config` before a run for a read-only local check, or +`validate --config validate.toml --print-config` to inspect absolute paths, defaults and field sources. + For tests, install `python3-cryptography` and `openssl`. See [Development](development.md) and [Usage](usage.md) for next steps. diff --git a/docs/logging.md b/docs/logging.md index ecaee80..785d66a 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -1,12 +1,15 @@ # Diagnostic logging +For every field, default, constraint and complete example, see the [TOML configuration reference](configuration.md)。 + The validator writes diagnostics to stderr. Under daemon, that stream is saved as `runs/run_XXXXXX/validator.log`; it contains all enabled levels, not only errors. `process-output.txt` captures ordinary output and is normally empty. The daemon's own lifecycle diagnostics remain in the container log. No duplicate log forwarding is performed. Historical `stderr.log` files are not renamed. -CLI and both daemon Compose files default to `info` and `text`. +TOML defaults and the shipped Compose TOML examples use `info` and `text`. +Compose does not independently select the logging format. Text is written for people, with independently authored English sentences for run lifecycle, validation stages, progress, output files and failures. It does not enumerate event fields or translate event identifiers into prose. Progress reports @@ -23,8 +26,8 @@ not select the format. Text is not a stable machine-parsing interface. 2026-09-11T10:45:36Z INFO [run 230] Validation completed in 82.0 seconds. ``` -Use `--log-format json` or `PANDA_RPKI_LOG_FORMAT=json` for machine consumers; -the JSON event names and fields are unchanged. Both formats redact sensitive data. +Set `log-format = "json"` in the TOML for machine consumers; the JSON event +names and fields are unchanged. Both formats redact sensitive data. Text error reasons are limited to 240 Unicode characters with an explicit notice when shortened. Unknown extension/debug events retain a diagnostic identifier and their message, not an arbitrary field dump. Use JSON (and debug/trace as needed) @@ -79,8 +82,8 @@ unregistered literal call sites. Unknown library-supplied names default to debug ## Progress and volume -`--log-progress-interval-secs` overrides `PANDA_RPKI_LOG_PROGRESS_INTERVAL_SECS`, -default `10`. `0` suppresses periodic events, not stage events. `off/error/warn` +`log-progress-interval-secs` is configured in TOML and defaults to `10`. `0` +suppresses periodic events, not stage events. `off/error/warn` do not create a progress thread. Queue snapshots are refreshed at most every 250ms. `statistics_age_ms` is the age of the queue/PP snapshot, not the repository event counters. `progress_stalled` means no observed completion for 30s, not proof of deadlock. @@ -122,3 +125,13 @@ The `jq` examples require JSON logging to have been explicitly enabled. Retention removes completed run directories including their logs. Archive needed runs before the configured retention window expires. Summary history is append-only. + +## Change an existing daemon to text + +Set `log-format = "text"` at the TOML top level, before `[[anchors]]`. +Keep `log-level = "info"` or the desired existing level. Do not use the removed +`--log-format` flag or `PANDA_RPKI_LOG_FORMAT` environment override. +TOML is read at process start, so stop and recreate the service as described in +[Docker configuration reload](docker.md#graceful-stop-edit-and-reload). +Each new validator child inherits the controller configuration; prior run logs +and JSON summary/metadata files are not converted. diff --git a/docs/releasing.md b/docs/releasing.md index f3482af..e8aec05 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -35,3 +35,13 @@ are not sufficient to reproduce an image. If runtime code or dependencies change, repeat end-to-end snapshot/delta and artifact verification. Documentation-only changes do not establish new runtime performance measurements; retain the revision associated with earlier results. + +## Configuration documentation checks + +Verify all four `docker/examples/*.toml` with the release binary's +`--check-config` and render each matching Compose file with `docker compose config`. +Confirm mandatory TOML, removed business flags/env overrides, text logging, +automatic versus fixed TA certificates, and operator-maintained TALs are +documented consistently. Validate containers need a writable parent for both +protocol state and the sibling TA cache. Configuration reload instructions must +recreate a container after replacing a single-file bind-mounted TOML. diff --git a/docs/trust-anchors.md b/docs/trust-anchors.md index 11767cc..a614253 100644 --- a/docs/trust-anchors.md +++ b/docs/trust-anchors.md @@ -1,9 +1,37 @@ # Prepare all five RIR trust anchors -Panda RPKI requires a TAL and a matching DER-encoded TA certificate for each -anchor. The TAL supplies the trusted public key; the certificate supplies the -current trust-anchor certificate. A certificate alone is not a substitute for -an independently obtained TAL. No RIR inputs are bundled with this project. +For every field, default, constraint and complete example, see the [TOML configuration reference](configuration.md)。 + +Panda RPKI requires a TAL for each configured anchor. The TAL supplies the +trusted public key and its certificate URI. You may pin a matching DER-encoded +TA certificate in TOML, or omit `ta` and let Panda fetch, validate and cache the +current certificate on each run. A certificate alone is not a substitute for an +independently obtained TAL. No RIR inputs are bundled with this project. + +## Automatic and fixed certificate modes + +Automatic mode requires only an independently obtained TAL: + +```toml +# Top-level setting, before any [[anchors]] table: +ta-refresh-policy = "allow-valid-cache" + +[[anchors]] +id = "ripe" +tal = "input/ripe.tal" +# No ta field: refresh the TA certificate at each validate run / daemon cycle. +``` + +Add `ta = "input/ripe.cer"` inside that anchor to select fixed mode. Anchors can +mix modes. Fixed files are never replaced by the automatic cache. The TAL itself +is never automatically updated: operators maintain its URIs and trusted key. +All download candidates and cached fallback certificates must pass TA validation, +TAL key binding and validity checks. A valid-cache fallback yields partial; +`require-fresh` disallows it. Check-config does not test download reachability. + +The download/pair-check sections below are optional for fixed mode. Automatic +mode follows the URIs actually present in the TAL; it does not invent HTTPS +alternatives for an rsync-only TAL. ## Official sources @@ -85,7 +113,7 @@ for rir in afrinic apnic arin lacnic ripe; do done ``` -## Download the TA certificates +## Optional: download certificates for fixed mode After checking that the URI lines match the table, run in the same Bash shell (which defines `fetch_https`): @@ -112,7 +140,7 @@ fetch_https https://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer in Always verify the downloaded certificate against the independently obtained TAL, including when using this alternative endpoint. -## Check the pairs and run +## Optional: check fixed TAL/TA pairs The following checks DER parsing, expiration, and equality of the certificate's SubjectPublicKeyInfo with the TAL key. They are input sanity checks, not a full @@ -136,13 +164,15 @@ for rir in afrinic apnic arin lacnic ripe; do done ``` -You now have ten files in `input/`, named exactly as expected by the -[all-five Compose example](docker.md#all-five-regional-anchors). For the -single-anchor README example only, copy a chosen pair to `input/anchor.tal` -and `input/anchor.cer`, or change the CLI arguments to use its RIR filenames. +You now have five TALs and, if using fixed mode, five matching certificates in +`input/`. Put those paths in the named `[[anchors]]` tables shown in the +[usage example](usage.md#multiple-anchors-and-constraints). To use automatic +refresh, omit the corresponding `ta` line; the first run requires outbound +HTTPS/rsync access and later runs may use the valid keyed cache when policy is +`allow-valid-cache`. -Keep inputs out of version control. Monitor certificate validity and RIR -announcements; Panda RPKI does not refresh supplied TA files automatically. -Stage and check updated pairs before replacing active inputs between runs. +Keep inputs out of version control. Monitor TAL changes and RIR announcements. +Panda RPKI never rewrites supplied TA files; automatic mode refreshes a separate +cache. Stage and check updated pairs before replacing active inputs between runs. Never resolve a key mismatch by replacing the TAL key with the downloaded certificate's key: retrieve the current TAL from the RIR and investigate first. diff --git a/docs/usage.md b/docs/usage.md index 1b98f98..57f2ab1 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -1,110 +1,132 @@ # Usage -Examples assume `panda-rpki` is on PATH; for a local build use -`target/release/panda-rpki`. Prepare TAL/TA files as described in -[Getting started](getting-started.md). +For every field, default, constraint and complete example, see the [TOML configuration reference](configuration.md)。 -## Snapshot and delta +Panda RPKI reads one TOML file at process start. Copy the file to create a +different profile; there is no command-line business-option or environment +override. -Use a fresh state directory for an explicit snapshot: +## Snapshot, delta and automatic refresh -```bash -panda-rpki validate --tal input/anchor.tal --ta input/anchor.cer \ - --rrdp-state-dir state/anchor --rrdp-sync-mode snapshot \ - --out output/snapshot --ccr-out output/snapshot/result.ccr -panda-rpki validate --tal input/anchor.tal --ta input/anchor.cer \ - --rrdp-state-dir state/anchor --rrdp-sync-mode delta \ - --out output/delta-01 --ccr-out output/delta-01/result.ccr +```toml +config-version = 1 +mode = "validate" +out = "output/ripe" +rrdp-state-dir = "state/ripe" +rrdp-sync-mode = "auto" +ccr-out = "output/ripe/result.ccr" +publish-dir = "publish/ripe" + +[[anchors]] +id = "ripe" +tal = "input/ripe.tal" +# ta = "input/ripe.cer" # leave out to refresh from the TAL URI ``` -Subsequent runs reuse state and can use `auto`. Delta mode can include a -protocol-required snapshot or noop. Check actual operation counters in the -summary. Do not discard persistent state between scheduled runs. +```bash +panda-rpki validate --config validate.toml --check-config +panda-rpki validate --config validate.toml +``` + +An empty state selects a snapshot. After a successful run, `auto` can use +delta updates; after an incomplete or failed run it recovers with `auto`. +Explicit `snapshot` requires an empty state and `delta` requires an existing +database. A valid downloaded TA is cached beside the state directory using the +TAL public-key hash. Refresh failures may use that cache under +`allow-valid-cache` and produce `partial`; use `require-fresh` to reject the +fallback. ## Multiple anchors and constraints -```bash -panda-rpki validate \ - --tal input/arin.tal --ta input/arin.cer --tal-id arin \ - --tal input/ripe.tal --ta input/ripe.cer --tal-id ripe \ - --parallel-max-repo-sync-workers-global 8 \ - --parallel-phase2-object-workers 8 \ - --rrdp-state-dir state/multi --out output/multi \ - --ccr-out output/multi/result.ccr +Use one named `[[anchors]]` table per RIR. There is no positional TAL/TA pairing: + +```toml +config-version = 1 +mode = "validate" +out = "output/all5" +rrdp-state-dir = "state/all5" +ccr-out = "output/all5/result.ccr" + +[[anchors]] +id = "afrinic" +tal = "input/afrinic.tal" +# ta = "input/afrinic.cer" # uncomment for fixed TA mode + +[[anchors]] +id = "apnic" +tal = "input/apnic.tal" +# ta = "input/apnic.cer" # uncomment for fixed TA mode + +[[anchors]] +id = "arin" +tal = "input/arin.tal" +# ta = "input/arin.cer" # uncomment for fixed TA mode + +[[anchors]] +id = "lacnic" +tal = "input/lacnic.tal" +# ta = "input/lacnic.cer" # uncomment for fixed TA mode + +[[anchors]] +id = "ripe" +tal = "input/ripe.tal" +# ta = "input/ripe.cer" # uncomment for fixed TA mode +constraints = "input/ripe.constraints" ``` -Add further pairs in the same way. Optional resource constraints are read from -the TAL-adjacent `.constraints` file, or specified with -`--ta-constraints arin=input/arin.constraints`. Rules follow the WG profile -listed in the [standards matrix](conformance-matrix.md). - -A constraint file contains `allow`/`deny` followed by an IPv4/IPv6 prefix, -address range, ASN or ASN range. Blank lines and `#` comments are allowed. -For a synthetic test anchor, an example is: - -```text -# Example resources only: do not apply this policy to a production RIR. -allow 192.0.2.0/24 -deny 192.0.2.128/25 -allow 2001:db8::/32 -allow 64496 - 64511 -``` - -Deny overlap rejects an EE resource; otherwise the resource must be fully -covered by allow rules of its resource family. Unlisted resources are not -implicitly allowed. An empty constraint file therefore differs from having -no constraint file. Overlapping rules are normalized and may produce warnings. +An omitted `constraints` field discovers a same-stem file beside the TAL. An +explicit path must exist and parse. Constraint rules follow the profile in the +[standards matrix](conformance-matrix.md). ## Continuous operation -```bash -panda-rpki daemon --state-root data/daemon \ - --interval-secs 600 --retain-runs 10 -- \ - --tal input/anchor.tal --ta input/anchor.cer \ - --log-level info --log-format json +Daemon configuration keeps the controller and validator settings together: + +```toml +config-version = 1 +mode = "daemon" +state-root = "data/all5-daemon" +interval-secs = 600 +# max-runs = 3 # uncomment for a finite three-cycle run +retain-runs = 10 +run-timeout-secs = 1800 +shutdown-grace-secs = 30 +require-complete = true +ta-refresh-policy = "allow-valid-cache" +log-format = "text" +log-level = "info" + +[[anchors]] +id = "ripe" +tal = "input/ripe.tal" +# ta = "input/ripe.cer" # uncomment for fixed TA mode ``` -The daemon runs in the foreground. It starts with snapshot for empty state and -requests delta after successful runs. The interval starts after each cycle -finishes. Add `--max-runs 3` before `--` for a finite run. SIGTERM requests a -graceful stop. Restart with the same root to continue numbering and state. +```bash +panda-rpki daemon --config daemon.toml --check-config +panda-rpki daemon --config daemon.toml +``` -Each cycle writes to `data/daemon/runs/run_000001/`, then `run_000002/`, etc. -Retention deletes the oldest completed run directories beyond the configured -limit, including their artifacts. Copy anything you need to retain elsewhere. -The append-only summary history is not trimmed by retention. +Each cycle writes `data/all5-daemon/runs/run_000001/`, including the generated +`validator-config.toml`, CSV/CCR/summary/timing artifacts, process metrics and +logs. The daemon points `data/all5-daemon/latest` at the newest successful run; +partial and failed runs remain available for diagnosis. It chooses snapshot for +empty state, delta after success and auto after partial/failed recovery. A +finite daemon exits 2 if any run failed, 3 when only partial runs occurred and +`require-complete` is true, otherwise 0. ## Outputs and diagnostics -`validator.log` is the per-run diagnostic log, not an error-only file. -`process-output.txt` captures ordinary child output and is normally empty. -Old runs keep their historical filenames; collectors support `stderr.log` as a fallback. -Every ten seconds, info logs report the current stage and available progress counters. -Use `--log-progress-interval-secs 0` to disable periodic events; stage events remain. -`statistics_age_ms` identifies stale statistics; `progress_stalled` means no observed -completion for thirty seconds, not a proven deadlock. Totals and percentages are not guessed. -Events include `run_id` and, under daemon, `run_seq`. Controller logs remain separate. +`summary.json` and `result-complete.json` describe the current attempt. Use the +immutable `publish/latest` (validate with `publish-dir`) or daemon `latest` when +consumers need a consistent group of files. `validator.log` contains structured +diagnostics on stderr; `--print-config` is configuration JSON and does not +contain validation payloads. -`vrps.csv` contains `asn,prefix,max_length`. CCR contains canonical state; -duplicate routing payloads can collapse into a single CCR entry. Count raw -rows and normalized unique payloads separately when comparing outputs. -ASPA canonicalization groups providers by customer ASN. +For path, state-layout, RRDP origin or permission errors, inspect the error +first and preserve the existing state directory. Do not share a writable state +directory between concurrent instances. -Use `--log-level debug` for diagnosis, or `trace` for detailed object/worker -events. Logs are written to stderr and can be redirected independently from -outputs. In daemon mode, inspect per-run `validator.log` and the controller's -`daemon-status.json` and `run-summary.jsonl`. - -| Symptom | Action | -| --- | --- | -| TAL/TA key mismatch or expired TA | Refresh the certificate from the TAL's official URI and check the selected pair. | -| State-layout/schema error | Preserve the directory and select a new dedicated root for a compatible run. | -| Snapshot refuses existing state | Use `auto`/`delta`, or choose a fresh directory for a new snapshot. | -| Delta refuses empty state | Establish state with `auto` or `snapshot`. | -| Permission denied in Docker | Pre-create writable host directories and use your UID/GID mapping. | -| RRDP origin rejection | Inspect the publisher's notification/references; extra TLS roots do not bypass origin policy. | -| Daemon root locked | Stop the other instance normally; do not delete its lock file. | -| Unexpected counts or partial output | Inspect warnings and per-run status; use new output paths to avoid stale files. | - -See the [CLI reference](command-line-reference.md) for precise defaults, -timeouts, exit codes and recovery behavior. +TAL files themselves are operator-maintained; automatic refresh changes only +TA certificates in the cache. See [stop/edit/reload](docker.md#graceful-stop-edit-and-reload) +for Docker shutdown semantics, text logging changes and switching anchor modes. diff --git a/examples/config/README.md b/examples/config/README.md new file mode 100644 index 0000000..94c5c9d --- /dev/null +++ b/examples/config/README.md @@ -0,0 +1,32 @@ +# Complete native TOML examples + +See the [complete configuration reference](../../docs/configuration.md) for all +fields, types, defaults and constraints. Copy a file into your deployment +directory and prepare trusted TALs in its input/ directory. The fixed example +also needs a valid, matching ripe.cer. No live inputs are bundled here. + +| File | Purpose | +| --- | --- | +| [validate-minimal.toml](validate-minimal.toml) | Minimal automatic TA; built-in defaults | +| [validate-full.toml](validate-full.toml) | Common/validate settings, strict completion, successful publication | +| [validate-fixed.toml](validate-fixed.toml) | Fixed TA certificate | +| [daemon-all5.toml](daemon-all5.toml) | Five automatic TAs, continuous operation | +| [daemon-three.toml](daemon-three.toml) | Five automatic TAs, three cycles with independent state | + +For example, copy validate-full.toml to /srv/panda/validate.toml and prepare +/srv/panda/input/ripe.tal: + +```bash +panda-rpki validate --config /srv/panda/validate.toml --check-config +panda-rpki validate --config /srv/panda/validate.toml --print-config +panda-rpki validate --config /srv/panda/validate.toml +``` + +Paths are relative to the configuration file. Automatic mode refreshes the TA +certificate, never the TAL. Uncomment ta or constraints only after preparing the +corresponding files. Omitted constraints still discovers a same-stem file when +one exists. + +Full/daemon examples explicitly choose require-complete=true and +run-timeout-secs=1800; built-in defaults are false and 0. Use the +[Docker examples](../../docker/examples/README.md) for container paths. diff --git a/examples/config/daemon-all5.toml b/examples/config/daemon-all5.toml new file mode 100644 index 0000000..b91c1f4 --- /dev/null +++ b/examples/config/daemon-all5.toml @@ -0,0 +1,45 @@ +config-version = 1 +mode = "daemon" +state-root = "data/all5" +interval-secs = 60 +retain-runs = 10 +shutdown-grace-secs = 30 +# Common settings: these choices are explicit, not all built-in defaults. +ta-refresh-policy = "allow-valid-cache" +max-ca-depth = 64 +http-timeout-secs = 300 +http-root-cert = [] # optional: ["tls/internal-root.pem"] +parallel-max-repo-sync-workers-global = 8 +parallel-phase2-object-workers = 8 +parallel-phase2-worker-queue-capacity = 256 +parallel-repo-worker-queue-capacity = 1024 # effective minimum +log-level = "info" +log-format = "text" +log-progress-interval-secs = 10 +require-complete = true # built-in default is false +run-timeout-secs = 1800 # built-in default is 0 (unlimited) + +[[anchors]] +id = "afrinic" +tal = "input/afrinic.tal" +# ta = "input/afrinic.cer" # uncomment for fixed mode + +[[anchors]] +id = "apnic" +tal = "input/apnic.tal" +# ta = "input/apnic.cer" # uncomment for fixed mode + +[[anchors]] +id = "arin" +tal = "input/arin.tal" +# ta = "input/arin.cer" # uncomment for fixed mode + +[[anchors]] +id = "lacnic" +tal = "input/lacnic.tal" +# ta = "input/lacnic.cer" # uncomment for fixed mode + +[[anchors]] +id = "ripe" +tal = "input/ripe.tal" +# ta = "input/ripe.cer" # uncomment for fixed mode diff --git a/examples/config/daemon-three.toml b/examples/config/daemon-three.toml new file mode 100644 index 0000000..58157f7 --- /dev/null +++ b/examples/config/daemon-three.toml @@ -0,0 +1,46 @@ +config-version = 1 +mode = "daemon" +state-root = "data/three" +interval-secs = 0 +max-runs = 3 +retain-runs = 10 +shutdown-grace-secs = 30 +# Common settings: these choices are explicit, not all built-in defaults. +ta-refresh-policy = "allow-valid-cache" +max-ca-depth = 64 +http-timeout-secs = 300 +http-root-cert = [] # optional: ["tls/internal-root.pem"] +parallel-max-repo-sync-workers-global = 8 +parallel-phase2-object-workers = 8 +parallel-phase2-worker-queue-capacity = 256 +parallel-repo-worker-queue-capacity = 1024 # effective minimum +log-level = "info" +log-format = "text" +log-progress-interval-secs = 10 +require-complete = true # built-in default is false +run-timeout-secs = 1800 # built-in default is 0 (unlimited) + +[[anchors]] +id = "afrinic" +tal = "input/afrinic.tal" +# ta = "input/afrinic.cer" # uncomment for fixed mode + +[[anchors]] +id = "apnic" +tal = "input/apnic.tal" +# ta = "input/apnic.cer" # uncomment for fixed mode + +[[anchors]] +id = "arin" +tal = "input/arin.tal" +# ta = "input/arin.cer" # uncomment for fixed mode + +[[anchors]] +id = "lacnic" +tal = "input/lacnic.tal" +# ta = "input/lacnic.cer" # uncomment for fixed mode + +[[anchors]] +id = "ripe" +tal = "input/ripe.tal" +# ta = "input/ripe.cer" # uncomment for fixed mode diff --git a/examples/config/validate-fixed.toml b/examples/config/validate-fixed.toml new file mode 100644 index 0000000..ae552cf --- /dev/null +++ b/examples/config/validate-fixed.toml @@ -0,0 +1,12 @@ +config-version = 1 +mode = "validate" +out = "output/fixed" +rrdp-state-dir = "state/fixed" +log-format = "text" +require-complete = true +run-timeout-secs = 1800 + +[[anchors]] +id = "ripe" +tal = "input/ripe.tal" +ta = "input/ripe.cer" diff --git a/examples/config/validate-full.toml b/examples/config/validate-full.toml new file mode 100644 index 0000000..354e3b0 --- /dev/null +++ b/examples/config/validate-full.toml @@ -0,0 +1,27 @@ +config-version = 1 +mode = "validate" +out = "output/full" +rrdp-state-dir = "state/full" +rrdp-sync-mode = "auto" +ccr-out = "output/full/result.ccr" +publish-dir = "publish/full" +# Common settings: these choices are explicit, not all built-in defaults. +ta-refresh-policy = "allow-valid-cache" +max-ca-depth = 64 +http-timeout-secs = 300 +http-root-cert = [] # optional: ["tls/internal-root.pem"] +parallel-max-repo-sync-workers-global = 8 +parallel-phase2-object-workers = 8 +parallel-phase2-worker-queue-capacity = 256 +parallel-repo-worker-queue-capacity = 1024 # effective minimum +log-level = "info" +log-format = "text" +log-progress-interval-secs = 10 +require-complete = true # built-in default is false +run-timeout-secs = 1800 # built-in default is 0 (unlimited) + +[[anchors]] +id = "ripe" +tal = "input/ripe.tal" +# ta = "input/ripe.cer" # uncomment for fixed mode +# constraints = "input/ripe.constraints" # must exist when explicit diff --git a/examples/config/validate-minimal.toml b/examples/config/validate-minimal.toml new file mode 100644 index 0000000..a536e6b --- /dev/null +++ b/examples/config/validate-minimal.toml @@ -0,0 +1,9 @@ +config-version = 1 +mode = "validate" +out = "output/minimal" + +[[anchors]] +id = "ripe" +tal = "input/ripe.tal" +# ta = "input/ripe.cer" # uncomment for fixed mode +# constraints = "input/ripe.constraints" # must exist when explicit diff --git a/src/cli/anchors.rs b/src/cli/anchors.rs new file mode 100644 index 0000000..3b7eb2d --- /dev/null +++ b/src/cli/anchors.rs @@ -0,0 +1,246 @@ +//! TAL-bound certificate refresh with a validated last-known certificate. +use super::config::Config; +use crate::model::{ta::TrustAnchor, tal::Tal}; +use crate::repository::fetch::{ + http::{BlockingHttpFetcher, HttpFetcherConfig}, + rsync::RsyncFetcher, + rsync_system::{SystemRsyncConfig, SystemRsyncFetcher}, +}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; +use std::{ + fs, + path::{Path, PathBuf}, + time::Duration, +}; +use x509_parser::prelude::{FromDer, X509Certificate}; + +pub(crate) fn validate_certificate(tal: &Tal, bytes: &[u8]) -> Result { + TrustAnchor::bind_der(tal.clone(), bytes, None).map_err(|e| e.to_string())?; + let (_, cert) = X509Certificate::from_der(bytes).map_err(|e| e.to_string())?; + if !cert.validity().is_valid() { + return Err("TA certificate is expired or not yet valid".into()); + } + Ok( + json!({"certificateSha256":hex::encode(Sha256::digest(bytes)),"notBefore":cert.validity().not_before.to_rfc2822().map_err(|e|e.to_string())?,"notAfter":cert.validity().not_after.to_rfc2822().map_err(|e|e.to_string())?}), + ) +} +pub(crate) fn cache_root(state: &Path) -> PathBuf { + let mut name = state.as_os_str().to_os_string(); + name.push(".ta-cache"); + PathBuf::from(name) +} + +pub(crate) fn resolve( + config: &Config, +) -> Result<(Vec, Vec), String> { + let http = BlockingHttpFetcher::new(HttpFetcherConfig { + timeout: Duration::from_secs(config.http_timeout_secs), + extra_root_certificates_pem: config + .http_root_cert + .iter() + .map(fs::read) + .collect::>() + .map_err(|e| e.to_string())?, + ..Default::default() + })?; + let rsync = SystemRsyncFetcher::new(SystemRsyncConfig::default()); + resolve_with(config, |uri| { + match url::Url::parse(uri).map_err(|e| e.to_string())?.scheme() { + "https" => http.fetch_bytes(uri), + "rsync" => rsync.fetch_object(uri).map_err(|e| e.to_string()), + _ => Err("unsupported TA URI scheme".into()), + } + }) +} + +pub(crate) fn resolve_with( + config: &Config, + mut fetch: impl FnMut(&str) -> Result, String>, +) -> Result<(Vec, Vec), String> { + let root = cache_root(config.rrdp_state_dir.as_ref().ok_or("state path missing")?); + let mut inputs = Vec::new(); + let mut records = Vec::new(); + for anchor in &config.anchors { + let mut record = json!({"id":anchor.id,"mode":if anchor.ta.is_some(){"fixed"}else{"auto"},"source":Value::Null,"certificateSha256":Value::Null,"fetchedAt":Value::Null,"notBefore":Value::Null,"notAfter":Value::Null,"refreshFailed":false,"errorCode":Value::Null}); + let attempt = (|| -> Result { + let tal = Tal::decode_bytes(&fs::read(&anchor.tal).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + if let Some(path) = &anchor.ta { + let info = validate_certificate(&tal, &fs::read(path).map_err(|e| e.to_string())?)?; + record + .as_object_mut() + .unwrap() + .extend(info.as_object().unwrap().clone()); + record["source"] = json!("file"); + return Ok(path.clone()); + } + let key = hex::encode(Sha256::digest(&tal.subject_public_key_info_der)); + let path = root.join(format!("{key}.json")); + let der_path = root.join(format!("{key}.cer")); + let mut last_error = "no usable TA URI".to_string(); + let mut error_code = "download_failed"; + for uri in &tal.ta_uris { + match fetch(uri.as_str()) { + Ok(bytes) => match validate_certificate(&tal, &bytes) { + Ok(info) => { + record + .as_object_mut() + .unwrap() + .extend(info.as_object().unwrap().clone()); + record["source"] = json!("download"); + record["fetchedAt"] = json!(super::publication::now()); + // Certificate and metadata are one atomic cache record; the .cer is a runtime projection. + super::publication::write_json( + &path, + &json!({"der":hex::encode(&bytes),"record":record}), + )?; + super::publication::write_atomic(&der_path, &bytes)?; + return Ok(der_path); + } + Err(e) => { + last_error = e; + error_code = "certificate_invalid"; + } + }, + Err(e) => { + last_error = e; + } + } + } + record["refreshFailed"] = json!(true); + record["errorCode"] = json!(error_code); + if config.ta_refresh_policy == "allow-valid-cache" && path.exists() { + let cached: Value = match fs::read(&path) + .map_err(|e| e.to_string()) + .and_then(|bytes| serde_json::from_slice(&bytes).map_err(|e| e.to_string())) + { + Ok(value) => value, + Err(error) => { + record["errorCode"] = json!("cache_unusable"); + return Err(format!("read cached TA metadata failed: {error}")); + } + }; + let bytes = match cached["der"].as_str() { + Some(encoded) => match hex::decode(encoded) { + Ok(bytes) => bytes, + Err(error) => { + record["errorCode"] = json!("cache_unusable"); + return Err(format!("decode cached TA failed: {error}")); + } + }, + None => { + record["errorCode"] = json!("cache_unusable"); + return Err("cached TA metadata has no DER value".into()); + } + }; + let info = validate_certificate(&tal, &bytes).inspect_err(|_| { + record["errorCode"] = json!("cache_unusable"); + })?; + record + .as_object_mut() + .unwrap() + .extend(info.as_object().unwrap().clone()); + record["source"] = json!("cache"); + record["fetchedAt"] = cached["record"]["fetchedAt"].clone(); + super::publication::write_atomic(&der_path, &bytes)?; + crate::logging::emit( + crate::logging::Level::Warn, + "ta_refresh_failed", + || json!({"tal_id":anchor.id,"error":last_error,"fallback":"valid_cache"}), + ); + return Ok(der_path); + } + Err(last_error) + })(); + match attempt { + Ok(path) => inputs.push(anchor.input(&path)), + Err(e) => { + if record["errorCode"].is_null() { + record["errorCode"] = json!("input_invalid"); + } + record["error"] = json!(e); + crate::logging::emit( + crate::logging::Level::Warn, + "trust_anchor_failed", + || json!({"tal_id":anchor.id,"error":e}), + ); + } + } + records.push(record); + } + Ok((inputs, records)) +} + +#[cfg(test)] +mod tests { + use super::{cache_root, resolve_with}; + use crate::cli::config::{Anchor, Config}; + use std::fs; + + fn config(state: &std::path::Path) -> Config { + let fixture = crate::test_support::synthetic_repository(); + Config { + config_version: 1, + mode: "validate".into(), + rrdp_state_dir: Some(state.to_path_buf()), + anchors: vec![Anchor { + id: "custom".into(), + tal: fixture.tal(), + ta: None, + constraints: None, + }], + ..Default::default() + } + } + + #[test] + fn automatic_ta_download_is_bound_and_cached() { + let fixture = crate::test_support::synthetic_repository(); + let directory = tempfile::tempdir().unwrap(); + let config = config(directory.path().join("state").as_path()); + let ta = fs::read(fixture.trust_anchor()).unwrap(); + let (inputs, records) = resolve_with(&config, |_| Ok(ta.clone())).unwrap(); + assert_eq!(inputs.len(), 1); + assert_eq!(records[0]["mode"], "auto"); + assert_eq!(records[0]["source"], "download"); + let mut cached = fs::read_dir(cache_root(config.rrdp_state_dir.as_ref().unwrap())) + .unwrap() + .map(|entry| entry.unwrap().path()) + .collect::>(); + cached.sort(); + assert!( + cached + .iter() + .any(|path| path.extension().is_some_and(|x| x == "json")) + ); + assert!( + cached + .iter() + .any(|path| path.extension().is_some_and(|x| x == "cer")) + ); + } + + #[test] + fn valid_cache_fallback_is_partial_and_bad_cache_is_rejected() { + let fixture = crate::test_support::synthetic_repository(); + let directory = tempfile::tempdir().unwrap(); + let config = config(directory.path().join("state").as_path()); + let ta = fs::read(fixture.trust_anchor()).unwrap(); + resolve_with(&config, |_| Ok(ta.clone())).unwrap(); + let (inputs, records) = resolve_with(&config, |_| Err("offline".into())).unwrap(); + assert_eq!(inputs.len(), 1); + assert_eq!(records[0]["source"], "cache"); + assert_eq!(records[0]["refreshFailed"], true); + + let cache = fs::read_dir(cache_root(config.rrdp_state_dir.as_ref().unwrap())) + .unwrap() + .map(|entry| entry.unwrap().path()) + .find(|path| path.extension().is_some_and(|x| x == "json")) + .unwrap(); + fs::write(cache, br#"{"der":"00"}"#).unwrap(); + let (inputs, records) = resolve_with(&config, |_| Err("offline".into())).unwrap(); + assert!(inputs.is_empty()); + assert_eq!(records[0]["errorCode"], "cache_unusable"); + } +} diff --git a/src/cli/config.rs b/src/cli/config.rs new file mode 100644 index 0000000..bff0375 --- /dev/null +++ b/src/cli/config.rs @@ -0,0 +1,391 @@ +//! One explicit TOML file is the sole source of runtime settings. +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::collections::BTreeSet; +use std::path::{Component, Path, PathBuf}; + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub struct Anchor { + #[serde(default)] + pub id: String, + pub tal: PathBuf, + pub ta: Option, + pub constraints: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(default, deny_unknown_fields, rename_all = "kebab-case")] +pub struct Config { + pub config_version: u32, + pub mode: String, + pub anchors: Vec, + pub ta_refresh_policy: String, + pub max_ca_depth: usize, + pub http_timeout_secs: u64, + pub http_root_cert: Vec, + pub parallel_max_repo_sync_workers_global: usize, + pub parallel_phase2_object_workers: usize, + pub parallel_phase2_worker_queue_capacity: usize, + pub parallel_repo_worker_queue_capacity: usize, + pub log_level: String, + pub log_format: String, + pub log_progress_interval_secs: u64, + pub require_complete: bool, + pub run_timeout_secs: u64, + pub out: Option, + pub rrdp_state_dir: Option, + pub rrdp_sync_mode: String, + pub ccr_out: Option, + pub publish_dir: Option, + pub state_root: Option, + pub interval_secs: u64, + pub max_runs: Option, + pub retain_runs: usize, + pub shutdown_grace_secs: u64, +} +impl Default for Config { + fn default() -> Self { + Self { + config_version: 1, + mode: String::new(), + anchors: Vec::new(), + ta_refresh_policy: "allow-valid-cache".into(), + max_ca_depth: 64, + http_timeout_secs: 300, + http_root_cert: Vec::new(), + parallel_max_repo_sync_workers_global: 8, + parallel_phase2_object_workers: 8, + parallel_phase2_worker_queue_capacity: 256, + parallel_repo_worker_queue_capacity: 256, + log_level: "info".into(), + log_format: "text".into(), + log_progress_interval_secs: 10, + require_complete: false, + run_timeout_secs: 0, + out: None, + rrdp_state_dir: None, + rrdp_sync_mode: "auto".into(), + ccr_out: None, + publish_dir: None, + state_root: None, + interval_secs: 60, + max_runs: None, + retain_runs: 10, + shutdown_grace_secs: 30, + } + } +} + +/// Resolve symlinked existing ancestors without creating a destination. +pub(crate) fn absolute(base: &Path, path: &Path) -> Result { + if path.as_os_str().is_empty() { + return Err("empty path".into()); + } + let joined = if path.is_absolute() { + path.to_path_buf() + } else { + base.join(path) + }; + let mut normalized = PathBuf::new(); + for c in joined.components() { + match c { + Component::ParentDir => { + normalized.pop(); + } + Component::CurDir => {} + x => normalized.push(x), + } + } + let mut ancestor = normalized.as_path(); + let mut missing = Vec::new(); + while !ancestor.exists() { + missing.push(ancestor.file_name().ok_or("invalid path")?.to_os_string()); + ancestor = ancestor.parent().ok_or("invalid path")?; + } + let mut result = ancestor.canonicalize().map_err(|e| e.to_string())?; + for c in missing.into_iter().rev() { + result.push(c); + } + Ok(result) +} + +impl Config { + pub(crate) fn load(path: &Path, mode: &str) -> Result<(Self, Value), String> { + let path = absolute(&std::env::current_dir().map_err(|e| e.to_string())?, path)?; + let raw = std::fs::read_to_string(&path) + .map_err(|e| format!("read config {}: {e}", path.display()))?; + let table: toml::Table = toml::from_str(&raw).map_err(|e| format!("invalid TOML: {e}"))?; + if table + .get("config-version") + .and_then(toml::Value::as_integer) + != Some(1) + { + return Err("config-version must be explicitly set to 1".into()); + } + let mut config: Self = + toml::from_str(&raw).map_err(|e| format!("invalid configuration: {e}"))?; + if config.mode != mode { + return Err(format!("config mode must be {mode}")); + } + let forbidden: &[&str] = if mode == "daemon" { + &[ + "out", + "rrdp-state-dir", + "rrdp-sync-mode", + "ccr-out", + "publish-dir", + ] + } else { + &[ + "state-root", + "interval-secs", + "max-runs", + "retain-runs", + "shutdown-grace-secs", + ] + }; + for key in forbidden { + if table.contains_key(*key) { + return Err(format!("{key} is not allowed in {mode} configuration")); + } + } + let base = path.parent().ok_or("config has no parent")?; + for p in [ + &mut config.out, + &mut config.rrdp_state_dir, + &mut config.ccr_out, + &mut config.publish_dir, + &mut config.state_root, + ] + .into_iter() + .flatten() + { + *p = absolute(base, p)?; + } + if mode == "validate" { + let out = config.out.as_ref().ok_or("out is required")?; + if config.rrdp_state_dir.is_none() { + config.rrdp_state_dir = Some(out.join(".state")); + } + if let Some(publish) = &config.publish_dir { + for other in [out, config.rrdp_state_dir.as_ref().unwrap()] { + if other.starts_with(publish) || publish.starts_with(other) { + return Err("publish-dir must not overlap out or state".into()); + } + } + } + } else if config.state_root.is_none() { + return Err("state-root is required".into()); + } + for p in config.http_root_cert.iter_mut() { + *p = absolute(base, p)?; + } + for anchor in &mut config.anchors { + anchor.tal = absolute(base, &anchor.tal)?; + if anchor.id.is_empty() { + anchor.id = anchor + .tal + .file_stem() + .and_then(|x| x.to_str()) + .ok_or("invalid TAL name")? + .into(); + } + if let Some(p) = &mut anchor.ta { + *p = absolute(base, p)?; + } + if let Some(p) = &mut anchor.constraints { + *p = absolute(base, p)?; + } + } + config.validate_inputs()?; + let mut normalizations = Vec::new(); + if config.http_timeout_secs < 1 { + normalizations.push(json!({"field":"http-timeout-secs","input":0,"effective":1})); + config.http_timeout_secs = 1; + } + if config.parallel_repo_worker_queue_capacity < 1024 { + normalizations.push(json!({"field":"parallel-repo-worker-queue-capacity","input":config.parallel_repo_worker_queue_capacity,"effective":1024})); + config.parallel_repo_worker_queue_capacity = 1024; + } + let mut effective = serde_json::to_value(&config).map_err(|e| e.to_string())?; + for key in forbidden { + effective.as_object_mut().unwrap().remove(*key); + } + let raw_json = serde_json::to_value(&table).map_err(|e| e.to_string())?; + let mut sources = serde_json::Map::new(); + collect_sources( + "", + &effective, + &raw_json, + &format!("file:{}", path.display()), + &mut sources, + ); + let printed = json!({"configVersion":1,"mode":mode,"effective":effective,"sources":sources,"normalizations":normalizations}); + Ok((config, printed)) + } + + pub(crate) fn validate_inputs(&self) -> Result<(), String> { + if !["validate", "daemon"].contains(&self.mode.as_str()) || self.config_version != 1 { + return Err("invalid mode/config-version".into()); + } + if self.anchors.is_empty() { + return Err("anchors must not be empty".into()); + } + if !["allow-valid-cache", "require-fresh"].contains(&self.ta_refresh_policy.as_str()) { + return Err("invalid ta-refresh-policy".into()); + } + if !["auto", "snapshot", "delta"].contains(&self.rrdp_sync_mode.as_str()) { + return Err("invalid rrdp-sync-mode".into()); + } + crate::logging::parse_level(&self.log_level)?; + if !["text", "json"].contains(&self.log_format.as_str()) { + return Err("invalid log-format".into()); + } + if [ + self.parallel_max_repo_sync_workers_global, + self.parallel_phase2_object_workers, + self.parallel_phase2_worker_queue_capacity, + self.parallel_repo_worker_queue_capacity, + self.retain_runs, + ] + .contains(&0) + || self.max_runs == Some(0) + { + return Err("workers, queues, retain-runs and max-runs must be positive".into()); + } + let mut ids = BTreeSet::new(); + for anchor in &self.anchors { + if anchor.id.is_empty() || !ids.insert(&anchor.id) { + return Err("anchor IDs must be nonempty and unique".into()); + } + let tal = crate::model::tal::Tal::decode_bytes( + &std::fs::read(&anchor.tal) + .map_err(|e| format!("read TAL {}: {e}", anchor.tal.display()))?, + ) + .map_err(|e| e.to_string())?; + if let Some(ta) = &anchor.ta { + super::anchors::validate_certificate( + &tal, + &std::fs::read(ta).map_err(|e| format!("read TA: {e}"))?, + )?; + } + let constraints = anchor + .constraints + .clone() + .unwrap_or_else(|| anchor.tal.with_extension("constraints")); + if anchor.constraints.is_some() || constraints.exists() { + crate::ta_constraints::TaConstraintsByTal::load_for_tals( + &[anchor.input(anchor.ta.as_deref().unwrap_or(&anchor.tal))], + &[format!("{}={}", anchor.id, constraints.display())], + )?; + } + } + for path in &self.http_root_cert { + reqwest::Certificate::from_pem( + &std::fs::read(path).map_err(|e| format!("read HTTP root: {e}"))?, + ) + .map_err(|e| e.to_string())?; + } + for p in [ + &self.out, + &self.state_root, + &self.rrdp_state_dir, + &self.publish_dir, + ] + .into_iter() + .flatten() + { + if p.exists() && !p.is_dir() { + return Err(format!("not a directory: {}", p.display())); + } + if let Some(parent) = p.parent() + && parent.exists() + && !parent.is_dir() + { + return Err(format!("parent is not a directory: {}", parent.display())); + } + } + if let Some(p) = &self.ccr_out { + if p.exists() && !p.is_file() { + return Err("ccr-out must be a file".into()); + } + if let Some(parent) = p.parent() + && parent.exists() + && !parent.is_dir() + { + return Err(format!( + "ccr-out parent is not a directory: {}", + parent.display() + )); + } + } + Ok(()) + } + + pub(crate) fn child_config( + &self, + out: &Path, + state: &Path, + mode: &str, + ) -> Result { + let mut child = self.clone(); + child.mode = "validate".into(); + child.out = Some(out.into()); + child.rrdp_state_dir = Some(state.into()); + child.ccr_out = Some(out.join("result.ccr")); + child.publish_dir = None; + child.rrdp_sync_mode = mode.into(); + child.run_timeout_secs = 0; + let mut value = toml::Value::try_from(child).map_err(|e| e.to_string())?; + for key in [ + "state-root", + "interval-secs", + "max-runs", + "retain-runs", + "shutdown-grace-secs", + ] { + value.as_table_mut().unwrap().remove(key); + } + toml::to_string_pretty(&value).map_err(|e| e.to_string()) + } +} +impl Anchor { + pub(crate) fn input(&self, ta: &Path) -> crate::scheduler::types::TalInputSpec { + let mut input = + crate::scheduler::types::TalInputSpec::from_file_path_with_ta(&self.tal, ta); + input.tal_id = self.id.clone(); + input + } +} +fn collect_sources( + prefix: &str, + effective: &Value, + raw: &Value, + file: &str, + sources: &mut serde_json::Map, +) { + match effective { + Value::Object(map) => { + for (key, value) in map { + let name = if prefix.is_empty() { + key.clone() + } else { + format!("{prefix}.{key}") + }; + collect_sources(&name, value, &raw[key], file, sources); + } + } + Value::Array(items) if !items.is_empty() => { + for (i, value) in items.iter().enumerate() { + collect_sources(&format!("{prefix}[{i}]"), value, &raw[i], file, sources); + } + } + _ => { + sources.insert( + prefix.into(), + json!(if raw.is_null() { "default" } else { file }), + ); + } + } +} diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 7306720..37acd11 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -1,17 +1,23 @@ -//! Narrow command-line surface for the v0.1.0 RFC core. - +//! Minimal command surface: one TOML configuration and read-only inspection. +pub(crate) mod anchors; +pub(crate) mod config; +pub(crate) mod publication; mod validate; +use std::path::PathBuf; -/// Reject unrelated state layouts before selecting a synchronization mode. pub(crate) fn check_state_root(root: &std::path::Path) -> Result<(), String> { if !root.exists() { return Ok(()); } for entry in std::fs::read_dir(root).map_err(|e| e.to_string())? { let entry = entry.map_err(|e| e.to_string())?; - if entry.file_name() != "repository-db" - || !entry.file_type().map_err(|e| e.to_string())?.is_dir() - { + let file_type = entry.file_type().map_err(|e| e.to_string())?; + let valid = match entry.file_name().to_string_lossy().as_ref() { + "repository-db" => file_type.is_dir(), + ".panda-rpki.lock" => file_type.is_file(), + _ => false, + }; + if !valid { return Err(format!( "unrecognized state layout at {}; use a fresh state directory", root.display() @@ -20,210 +26,88 @@ pub(crate) fn check_state_root(root: &std::path::Path) -> Result<(), String> { } Ok(()) } - -pub(crate) use validate::{VALIDATE_OPTIONS, VALIDATE_REPEATABLE}; - -/// Bounded default long enough for large public RIR snapshots. -const DEFAULT_HTTP_TIMEOUT_SECS: u64 = 300; -const DEFAULT_WORKER_COUNT: usize = 8; -const DEFAULT_REPO_SYNC_WORKER_COUNT: usize = 8; -const DEFAULT_WORKER_QUEUE_CAPACITY: usize = 256; - pub fn usage() -> String { - let validation = format!( - "panda-rpki {}\n\nUSAGE:\n panda-rpki validate --tal --ta [--tal --ta ...] --out [--rrdp-state-dir ] [--rrdp-sync-mode auto|snapshot|delta] [--ccr-out ] [--parallel-phase2-object-workers ] [--parallel-max-repo-sync-workers-global ] [--parallel-phase2-worker-queue-capacity ] [--parallel-repo-worker-queue-capacity ] [--max-ca-depth ] [--http-timeout-secs ] [--http-root-cert ]... [--log-level off|error|warn|info|debug|trace] [--log-format text|json]\n\nTAL/TA pairs are positional and may be repeated. RRDP state is an explicit protocol-resume directory, not a validation cache. The public v0.1.0 fast path accepts HTTPS RRDP notification/snapshot/delta URLs only.", - crate::VERSION - ); format!( - "{validation}\n --log-progress-interval-secs : periodic info progress (default 10; 0 disables).\n\n{}", - crate::daemon::usage() + "panda-rpki {}\n\nUSAGE:\n panda-rpki validate --config FILE [--check-config | --print-config]\n panda-rpki daemon --config FILE [--check-config | --print-config]\n panda-rpki --help | --version\n\nAll runtime settings belong in TOML. No runtime option or environment overrides.\n\nOPTIONS:\n --config FILE Required TOML configuration (also --config=FILE)\n --check-config Read-only validation, without network or state writes\n --print-config Print effective configuration, defaults and sources as JSON\n -h, --help Show this help\n -V, --version Show version (top level only)\n", + crate::VERSION ) } - -pub fn run(args: I) -> Result<(), String> +pub fn run(args: I) -> Result where I: IntoIterator, S: Into, { let args: Vec = args.into_iter().map(Into::into).collect(); - if args.is_empty() || matches!(args.first().map(String::as_str), Some("--help" | "-h")) { - return Err(usage()); + if args.is_empty() { + print!("{}", usage()); + return Ok(0); } - match args[0].as_str() { - "validate" => validate::run(&args[1..]), - "daemon" => crate::daemon::run(&args[1..]), - _ => Err(format!("unknown command: {}", args[0])), - } -} - -#[derive(Debug)] -pub(crate) struct Options { - values: std::collections::BTreeMap>, -} - -impl Options { - pub(crate) fn parse( - args: &[String], - allowed: &[&str], - repeatable: &[&str], - ) -> Result { - let mut values = std::collections::BTreeMap::>::new(); - let mut index = 0; - while index < args.len() { - let flag = &args[index]; - if matches!(flag.as_str(), "--help" | "-h") { - return Err(usage()); - } - let name = flag - .strip_prefix("--") - .ok_or_else(|| format!("expected an option, got {flag}"))?; - if !allowed.contains(&name) { - return Err(format!("unsupported option: {flag}")); - } - index += 1; - let value = args - .get(index) - .ok_or_else(|| format!("{flag} requires a value"))?; - if value.starts_with("--") { - return Err(format!("{flag} requires a value")); - } - let entry = values.entry(name.to_string()).or_default(); - if !repeatable.contains(&name) && !entry.is_empty() { - return Err(format!("option repeated: {flag}")); - } - entry.push(value.clone()); - index += 1; + if args.len() == 1 && ["--help", "-h", "--version", "-V"].contains(&args[0].as_str()) { + if ["--version", "-V"].contains(&args[0].as_str()) { + println!("panda-rpki {}", crate::VERSION); + } else { + print!("{}", usage()); } - Ok(Self { values }) + return Ok(0); } - - pub(crate) fn required(&self, name: &str) -> Result<&str, String> { - self.values - .get(name) - .and_then(|values| values.first()) - .map(String::as_str) - .ok_or_else(|| format!("--{name} is required")) + let mode = &args[0]; + if !["validate", "daemon"].contains(&mode.as_str()) { + return Err(format!("unknown command: {mode}")); } - - pub(crate) fn values(&self, name: &str) -> &[String] { - self.values.get(name).map(Vec::as_slice).unwrap_or(&[]) + let mut path = None; + let mut help = false; + let mut check = false; + let mut print = false; + let mut i = 1; + while i < args.len() { + let a = &args[i]; + match a.as_str() { + "--help" | "-h" if !help => help = true, + "--check-config" if !check => check = true, + "--print-config" if !print => print = true, + "--config" if path.is_none() => { + i += 1; + let p = args + .get(i) + .filter(|v| !v.starts_with('-')) + .ok_or("--config requires a file")?; + path = Some(PathBuf::from(p)); + } + _ if a.starts_with("--config=") && path.is_none() => { + path = Some(PathBuf::from(&a[9..])) + } + _ => { + return Err(format!( + "unsupported or repeated option: {a}; configure runtime settings in TOML and use --config FILE" + )); + } + } + i += 1; } - - pub(crate) fn parse_or(&self, name: &str, default: T) -> Result - where - T: std::str::FromStr, - T::Err: std::fmt::Display, - { - self.values - .get(name) - .and_then(|values| values.first()) - .map(|value| { - value - .parse() - .map_err(|error| format!("invalid --{name}: {error}")) - }) - .unwrap_or(Ok(default)) + if u8::from(help) + u8::from(check) + u8::from(print) > 1 { + return Err("help, check-config and print-config are mutually exclusive".into()); } - - pub(crate) fn optional_parse(&self, name: &str) -> Result, String> - where - T: std::str::FromStr, - T::Err: std::fmt::Display, - { - self.values - .get(name) - .and_then(|values| values.first()) - .map(|value| { - value - .parse() - .map(Some) - .map_err(|error| format!("invalid --{name}: {error}")) - }) - .unwrap_or(Ok(None)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn state_layout_rejects_unrecognized_entries_without_modifying_them() { - let root = tempfile::tempdir().unwrap(); - check_state_root(root.path()).unwrap(); - std::fs::create_dir(root.path().join("repository-db")).unwrap(); - check_state_root(root.path()).unwrap(); - let unknown = root.path().join("previous-state"); - std::fs::create_dir(&unknown).unwrap(); - std::fs::write(unknown.join("checkpoint"), b"retain").unwrap(); - assert!( - check_state_root(root.path()) - .unwrap_err() - .contains("unrecognized state layout") - ); - assert_eq!( - std::fs::read(unknown.join("checkpoint")).unwrap(), - b"retain" - ); - } - - #[test] - fn parser_rejects_private_or_unknown_options() { - let error = Options::parse(&["--extra".to_string(), "1".to_string()], &["out"], &[]) - .expect_err("unknown option"); - assert!(error.contains("unsupported option")); - } - - #[test] - fn parser_preserves_repeatable_tal_pairs() { - let options = Options::parse( - &[ - "--tal".to_string(), - "one.tal".to_string(), - "--ta".to_string(), - "one.cer".to_string(), - "--tal".to_string(), - "two.tal".to_string(), - "--ta".to_string(), - "two.cer".to_string(), - ], - &["tal", "ta"], - &["tal", "ta"], - ) - .expect("repeatable pairs"); - assert_eq!(options.values("tal"), ["one.tal", "two.tal"]); - assert_eq!(options.values("ta"), ["one.cer", "two.cer"]); - } - - #[test] - fn parser_rejects_non_repeatable_worker_option() { - let error = Options::parse( - &[ - "--workers".to_string(), - "2".to_string(), - "--workers".to_string(), - "3".to_string(), - ], - &["workers"], - &[], - ) - .expect_err("worker option should not repeat"); - assert!(error.contains("option repeated")); - } - - #[test] - fn parser_preserves_repeatable_http_root_certificates() { - let options = Options::parse( - &[ - "--http-root-cert".to_string(), - "one.pem".to_string(), - "--http-root-cert".to_string(), - "two.pem".to_string(), - ], - &["http-root-cert"], - &["http-root-cert"], - ) - .expect("repeatable HTTP roots"); - assert_eq!(options.values("http-root-cert"), ["one.pem", "two.pem"]); + if help { + print!("{}", usage()); + return Ok(0); + } + let path = path.ok_or("--config FILE is required")?; + let (config, effective) = config::Config::load(&path, mode)?; + if check { + println!("Configuration valid."); + return Ok(0); + } + if print { + println!( + "{}", + serde_json::to_string_pretty(&effective).map_err(|e| e.to_string())? + ); + return Ok(0); + } + crate::logging::configure(&config.log_level, &config.log_format)?; + if mode == "daemon" { + crate::daemon::run(config) + } else { + validate::run(config) } } diff --git a/src/cli/publication.rs b/src/cli/publication.rs new file mode 100644 index 0000000..e8999a3 --- /dev/null +++ b/src/cli/publication.rs @@ -0,0 +1,133 @@ +//! Durable single-file writes and immutable successful result publication. +use serde_json::{Value, json}; +use std::{ + fs::{self, File, OpenOptions}, + io::Write, + path::{Path, PathBuf}, +}; + +pub(crate) fn now() -> String { + time::OffsetDateTime::now_utc() + .format(&time::format_description::well_known::Rfc3339) + .expect("UTC") +} +pub(crate) fn write_atomic(path: &Path, bytes: &[u8]) -> Result<(), String> { + let parent = path.parent().ok_or("path has no parent")?; + fs::create_dir_all(parent).map_err(|e| e.to_string())?; + let mut temp = tempfile::NamedTempFile::new_in(parent).map_err(|e| e.to_string())?; + temp.write_all(bytes) + .and_then(|()| temp.as_file().sync_all()) + .map_err(|e| e.to_string())?; + temp.persist(path).map_err(|e| e.to_string())?; + File::open(parent) + .and_then(|f| f.sync_all()) + .map_err(|e| e.to_string()) +} +pub(crate) fn write_json(path: &Path, value: &Value) -> Result<(), String> { + write_atomic( + path, + &serde_json::to_vec_pretty(value).map_err(|e| e.to_string())?, + ) +} + +pub(crate) struct Locks(Vec); +impl Locks { + pub(crate) fn acquire(paths: &[PathBuf]) -> Result { + let mut paths = paths.to_vec(); + paths.sort(); + paths.dedup(); + let mut locks = Self(Vec::new()); + for path in paths { + // The lock belongs inside the protected directory. This keeps a + // container with a read-only root filesystem from trying to create + // `/output.lock` or `/state.lock` beside a bind mount. + fs::create_dir_all(&path).map_err(|e| e.to_string())?; + let lock_path = path.join(".panda-rpki.lock"); + let f = OpenOptions::new() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(&lock_path) + .map_err(|e| e.to_string())?; + f.try_lock() + .map_err(|e| format!("path is locked: {}: {e}", path.display()))?; + locks.0.push(f); + } + Ok(locks) + } +} + +pub(crate) fn switch_latest(root: &Path, target: &Path) -> Result<(), String> { + let tmp = root.join(format!(".latest-{}", uuid::Uuid::new_v4())); + std::os::unix::fs::symlink(target, &tmp).map_err(|e| e.to_string())?; + if let Err(e) = fs::rename(&tmp, root.join("latest")) { + let _ = fs::remove_file(tmp); + return Err(e.to_string()); + } + File::open(root) + .and_then(|f| f.sync_all()) + .map_err(|e| e.to_string()) +} +pub(crate) fn attempt(root: &Path, id: &str, status: &str) -> Result<(), String> { + let published = fs::read_link(root.join("latest")) + .ok() + .and_then(|p| p.file_name().map(|s| s.to_string_lossy().into_owned())); + write_json( + &root.join("last-attempt.json"), + &json!({"runId":id,"resultStatus":status,"finishedAt":now(),"publishedRunId":published}), + ) +} +pub(crate) fn publish( + root: &Path, + id: &str, + files: &[(PathBuf, PathBuf)], + summary: &Value, +) -> Result<(), String> { + fs::create_dir_all(root.join("runs")).map_err(|e| e.to_string())?; + let staging = tempfile::Builder::new() + .prefix(".staging-") + .tempdir_in(root) + .map_err(|e| e.to_string())?; + for (source, relative) in files { + write_atomic( + &staging.path().join(relative), + &fs::read(source).map_err(|e| e.to_string())?, + )?; + } + let mut published_summary = summary.clone(); + published_summary["publicationStatus"] = json!("published"); + write_json(&staging.path().join("summary.json"), &published_summary)?; + complete( + staging.path(), + id, + "success", + &files.iter().map(|(_, p)| p.clone()).collect::>(), + )?; + let dest = root.join("runs").join(id); + fs::rename(staging.path(), &dest).map_err(|e| e.to_string())?; + File::open(root.join("runs")) + .and_then(|f| f.sync_all()) + .map_err(|e| e.to_string())?; + switch_latest(root, &Path::new("runs").join(id))?; + // The pointer is the commit point; bookkeeping failures cannot undo it. + if let Err(e) = attempt(root, id, "success") { + crate::logging::emit( + crate::logging::Level::Warn, + "publication_bookkeeping_failed", + || json!({"error":e}), + ); + } + Ok(()) +} +pub(crate) fn complete( + out: &Path, + id: &str, + status: &str, + files: &[PathBuf], +) -> Result<(), String> { + write_json( + &out.join("result-complete.json"), + &json!({"runId":id,"resultStatus":status,"finishedAt":now(),"files":files}), + ) +} diff --git a/src/cli/validate.rs b/src/cli/validate.rs index 11a9661..e35fc23 100644 --- a/src/cli/validate.rs +++ b/src/cli/validate.rs @@ -1,228 +1,227 @@ -//! Parse validation options into typed runtime configuration. - +//! Execute one validation attempt, then commit complete successful outputs. +use super::{config::Config, publication}; +use serde_json::{Value, json}; use std::fs::{self, File, OpenOptions}; use std::io::{BufRead, BufReader, BufWriter, Write}; use std::path::{Path, PathBuf}; -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum SyncMode { - Auto, - Snapshot, - Delta, +pub(super) fn run(config: Config) -> Result { + // A separate process group makes the complete attempt cancellable, including + // blocking native validation and rsync descendants. Fork occurs before workers. + super::super::daemon::process::supervise(config.run_timeout_secs, || run_once(&config)) + .inspect_err(|error| { + let _ = record_failure(&config, "interrupted", error); + }) } - -pub(crate) const VALIDATE_OPTIONS: &[&str] = &[ - "tal", - "log-level", - "log-format", - "log-progress-interval-secs", - "ta", - "tal-id", - "ta-constraints", - "out", - "ccr-out", - "rrdp-state-dir", - "rrdp-sync-mode", - "max-ca-depth", - "http-timeout-secs", - "http-root-cert", - "parallel-max-repo-sync-workers-global", - "parallel-phase2-object-workers", - "parallel-phase2-worker-queue-capacity", - "parallel-repo-worker-queue-capacity", -]; -pub(crate) const VALIDATE_REPEATABLE: &[&str] = - &["tal", "ta", "tal-id", "ta-constraints", "http-root-cert"]; - -pub(super) fn run(args: &[String]) -> Result<(), String> { - let options = super::Options::parse(args, VALIDATE_OPTIONS, VALIDATE_REPEATABLE)?; - let level = resolve_setting( - options.values("log-level"), - std::env::var("PANDA_RPKI_LOG_LEVEL").ok(), - "info", +fn record_failure(config: &Config, id: &str, error: &str) -> Result<(), String> { + if let Some(out) = &config.out + && out.is_dir() + { + let _ = fs::remove_file(out.join("result-complete.json")); + } + if let Some(root) = &config.publish_dir { + publication::attempt(root, id, "failed")?; + } + crate::logging::emit( + crate::logging::Level::Error, + "validation_failed", + || json!({"error":error}), ); - let format = resolve_setting( - options.values("log-format"), - std::env::var("PANDA_RPKI_LOG_FORMAT").ok(), - "text", - ); - crate::logging::configure(&level, &format)?; - let progress_interval = resolve_setting( - options.values("log-progress-interval-secs"), - std::env::var("PANDA_RPKI_LOG_PROGRESS_INTERVAL_SECS").ok(), - "10", - ) - .parse::() - .map_err(|_| { - "invalid log-progress-interval-secs: expected a nonnegative integer".to_string() - })?; - let run_id = - std::env::var("PANDA_RPKI_RUN_ID").unwrap_or_else(|_| uuid::Uuid::new_v4().to_string()); - let run_seq = std::env::var("PANDA_RPKI_RUN_SEQ") - .ok() - .and_then(|value| value.parse().ok()); - crate::logging::set_context(&run_id, run_seq); - let tal_paths = options.values("tal"); - let ta_paths = options.values("ta"); - if tal_paths.is_empty() || ta_paths.is_empty() || tal_paths.len() != ta_paths.len() { - return Err("--tal and --ta must be repeated as equal-length positional pairs".to_string()); - } - let requested_ids = options.values("tal-id"); - if !requested_ids.is_empty() && requested_ids.len() != tal_paths.len() { - return Err("--tal-id must be supplied once for each --tal when used".to_string()); - } - let max_ca_depth = options.parse_or("max-ca-depth", 64_usize)?; - let timeout = options.parse_or("http-timeout-secs", super::DEFAULT_HTTP_TIMEOUT_SECS)?; - let workers = options - .optional_parse("parallel-phase2-object-workers")? - .unwrap_or(super::DEFAULT_WORKER_COUNT); - let repo_workers = options - .optional_parse("parallel-max-repo-sync-workers-global")? - .unwrap_or(super::DEFAULT_REPO_SYNC_WORKER_COUNT); - let phase2_queue = options.parse_or( - "parallel-phase2-worker-queue-capacity", - super::DEFAULT_WORKER_QUEUE_CAPACITY, - )?; - let repo_queue = options.parse_or( - "parallel-repo-worker-queue-capacity", - super::DEFAULT_WORKER_QUEUE_CAPACITY, - )?; - if workers == 0 || repo_workers == 0 || phase2_queue == 0 || repo_queue == 0 { - return Err("worker counts and queue capacities must be greater than zero".to_string()); - } - - let out = PathBuf::from(options.required("out")?); - fs::create_dir_all(&out) - .map_err(|error| format!("create output directory {}: {error}", out.display()))?; - let state_root = options - .values("rrdp-state-dir") - .first() - .map(PathBuf::from) - .unwrap_or_else(|| out.join(".state")); - fs::create_dir_all(&state_root) - .map_err(|error| format!("create state directory {}: {error}", state_root.display()))?; - let db_path = state_root.join("repository-db"); - super::check_state_root(&state_root)?; - let sync_mode = parse_sync_mode(options.values("rrdp-sync-mode"))?; - if sync_mode == SyncMode::Snapshot && directory_has_entries(&db_path)? { - return Err(format!( - "snapshot mode requires a fresh fast-path state directory; existing database: {}", - db_path.display() - )); - } - if sync_mode == SyncMode::Delta && !directory_has_entries(&db_path)? { - return Err(format!( - "delta mode requires an existing fast-path state database: {}", - db_path.display() - )); - } - - use crate::runtime::RunConfig; - use crate::scheduler::config::{ParallelPhase1Config, ParallelPhase2Config}; - use crate::scheduler::types::TalInputSpec; - let mut tal_inputs: Vec<_> = tal_paths - .iter() - .zip(ta_paths) - .map(|(tal, ta)| TalInputSpec::from_file_path_with_ta(tal, ta)) - .collect(); - for (input, id) in tal_inputs.iter_mut().zip(requested_ids) { - input.tal_id = id.clone(); - } - let tal_ids: Vec = tal_inputs - .iter() - .map(|input| input.tal_id.clone()) - .collect(); - let ta_constraints = crate::ta_constraints::TaConstraintsByTal::load_for_tals( - &tal_inputs, - options.values("ta-constraints"), - )?; - let config = RunConfig { - db_path, - tal_paths: tal_paths.iter().map(PathBuf::from).collect(), - ta_paths: ta_paths.iter().map(PathBuf::from).collect(), - tal_path: tal_paths.first().map(PathBuf::from), - ta_path: ta_paths.first().map(PathBuf::from), - tal_inputs, - ta_constraints, - parallel_phase1_config: ParallelPhase1Config { - max_repo_sync_workers_global: repo_workers, - max_pending_repo_results: repo_queue.max(1024), - ..Default::default() - }, - parallel_phase2_config: ParallelPhase2Config { - object_workers: workers, - worker_queue_capacity: phase2_queue, - ..Default::default() - }, - max_ca_depth, - http_timeout_secs: timeout, - http_root_cert_paths: options - .values("http-root-cert") - .iter() - .map(PathBuf::from) - .collect(), - rsync_timeout_secs: 30, - summary_out_path: Some(out.join("summary.json")), - vrps_csv_out_path: Some(out.join(".vrps-source.csv")), - vaps_csv_out_path: Some(out.join("vaps.csv")), - analysis_out_path: Some(out.join("analysis")), - analyze: true, - ccr_out_path: options.values("ccr-out").first().map(PathBuf::from), - ..Default::default() - }; - - let started = std::time::Instant::now(); - let observer = crate::logging::observer::Observer::start(progress_interval)?; - crate::logging::emit(crate::logging::Level::Info, "validation_started", || { - serde_json::json!({ - "tal_count": tal_paths.len(), "tal_ids": tal_ids, "phase2_workers": workers, "repo_workers": repo_workers, - "sync_mode": format!("{sync_mode:?}").to_lowercase(), - }) - }); - let result = crate::runtime::run_config(config).and_then(|()| { - crate::logging::observer::phase("vrp_csv_conversion"); - let fast_vrps = out.join(".vrps-source.csv"); - crate::logging::artifact::write("vrps", &out.join("vrps.csv"), || { - convert_vrp_csv(&fast_vrps, &out.join("vrps.csv")) - }) - }); - observer.finish(result.is_ok()); - result.inspect_err(|error| { - crate::logging::emit(crate::logging::Level::Error, "validation_failed", || { - serde_json::json!({ - "elapsed_ms": started.elapsed().as_millis(), - "error": error, - }) - }); - })?; - - crate::logging::emit(crate::logging::Level::Info, "validation_completed", || { - serde_json::json!({ - "elapsed_ms": started.elapsed().as_millis(), "phase2_workers": workers, "repo_workers": repo_workers, - }) - }); Ok(()) } - -fn parse_sync_mode(values: &[String]) -> Result { - match values.first().map(String::as_str).unwrap_or("auto") { - "auto" => Ok(SyncMode::Auto), - "snapshot" => Ok(SyncMode::Snapshot), - "delta" => Ok(SyncMode::Delta), - other => Err(format!( - "--rrdp-sync-mode must be auto, snapshot, or delta; got {other}" - )), +fn run_once(config: &Config) -> Result { + let out = config.out.as_ref().ok_or("out required")?; + let state = config.rrdp_state_dir.as_ref().ok_or("state required")?; + let mut paths = vec![out.clone(), state.clone()]; + if let Some(p) = &config.publish_dir { + paths.push(p.clone()); } + let _locks = publication::Locks::acquire(&paths)?; + fs::create_dir_all(out).map_err(|e| e.to_string())?; + fs::create_dir_all(state).map_err(|e| e.to_string())?; + match fs::remove_file(out.join("result-complete.json")) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e.to_string()), + } + let internal_context = std::env::var_os("PANDA_RPKI_INTERNAL_CONTEXT").is_some(); + let run_id = if internal_context { + std::env::var("PANDA_RPKI_RUN_ID").unwrap_or_else(|_| uuid::Uuid::new_v4().to_string()) + } else { + uuid::Uuid::new_v4().to_string() + }; + let run_seq = internal_context + .then(|| std::env::var("PANDA_RPKI_RUN_SEQ").ok()) + .flatten() + .and_then(|value| value.parse::().ok()); + crate::logging::set_context(&run_id, run_seq); + let result = execute(config, &run_id); + if let Err(error) = &result { + let _ = record_failure(config, &run_id, error); + } + result } - -fn resolve_setting(cli_values: &[String], env_value: Option, default: &str) -> String { - cli_values - .first() - .cloned() - .or(env_value) - .unwrap_or_else(|| default.to_string()) +fn execute(config: &Config, run_id: &str) -> Result { + let out = config.out.as_ref().unwrap(); + let state = config.rrdp_state_dir.as_ref().unwrap(); + super::check_state_root(state)?; + let db_path = state.join("repository-db"); + let populated = directory_has_entries(&db_path)?; + if config.rrdp_sync_mode == "snapshot" && populated { + return Err("snapshot mode requires a fresh state directory".into()); + } + if config.rrdp_sync_mode == "delta" && !populated { + return Err("delta mode requires an existing state database".into()); + } + let observer = crate::logging::observer::Observer::start(config.log_progress_interval_secs)?; + let start = std::time::Instant::now(); + crate::logging::emit( + crate::logging::Level::Info, + "validation_started", + || json!({"tal_count":config.anchors.len(),"sync_mode":config.rrdp_sync_mode}), + ); + let (inputs, anchors) = super::anchors::resolve(config)?; + let failed = config.anchors.len() - inputs.len(); + if inputs.is_empty() { + publication::write_json( + &out.join("summary.json"), + &json!({"resultStatus":"failed","trustAnchors":anchors,"taFailedCount":failed,"publicationStatus":"skipped"}), + )?; + observer.finish(false); + return Err("all trust anchors failed".into()); + } + let overrides: Vec = config + .anchors + .iter() + .filter(|a| inputs.iter().any(|i| i.tal_id == a.id)) + .filter_map(|a| { + a.constraints + .as_ref() + .map(|p| format!("{}={}", a.id, p.display())) + }) + .collect(); + let constraints = + crate::ta_constraints::TaConstraintsByTal::load_for_tals(&inputs, &overrides)?; + // All runtime artifacts are first written under an isolated attempt directory. + let staging = tempfile::Builder::new() + .prefix(".validation-") + .tempdir_in(out) + .map_err(|e| e.to_string())?; + let root = staging.path(); + let runtime = crate::runtime::RunConfig { + db_path, + tal_inputs: inputs, + ta_constraints: constraints, + parallel_phase1_config: crate::scheduler::config::ParallelPhase1Config { + max_repo_sync_workers_global: config.parallel_max_repo_sync_workers_global, + max_pending_repo_results: config.parallel_repo_worker_queue_capacity, + ..Default::default() + }, + parallel_phase2_config: crate::scheduler::config::ParallelPhase2Config { + object_workers: config.parallel_phase2_object_workers, + worker_queue_capacity: config.parallel_phase2_worker_queue_capacity, + ..Default::default() + }, + max_ca_depth: config.max_ca_depth, + http_timeout_secs: config.http_timeout_secs, + http_root_cert_paths: config.http_root_cert.clone(), + rsync_timeout_secs: 30, + summary_out_path: Some(root.join("summary.json")), + vrps_csv_out_path: Some(root.join(".vrps-source.csv")), + vaps_csv_out_path: Some(root.join("vaps.csv")), + analysis_out_path: Some(root.join("analysis")), + analyze: true, + ccr_out_path: config.ccr_out.as_ref().map(|_| root.join("result.ccr")), + ..Default::default() + }; + let result = crate::runtime::run_config(runtime) + .and_then(|()| convert_vrp_csv(&root.join(".vrps-source.csv"), &root.join("vrps.csv"))); + observer.finish(result.is_ok()); + result?; + let mut summary: Value = + serde_json::from_slice(&fs::read(root.join("summary.json")).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + let repository_failed = summary["repositoryFailedCount"].as_u64().unwrap_or(0); + let cached = anchors.iter().filter(|a| a["source"] == "cache").count(); + let status = if failed > 0 || repository_failed > 0 || cached > 0 { + "partial" + } else { + "success" + }; + summary["resultStatus"] = json!(status); + summary["taFailedCount"] = json!(failed); + summary["taCacheFallbackCount"] = json!(cached); + summary["trustAnchors"] = json!(anchors); + summary["publicationStatus"] = json!(if config.publish_dir.is_some() { + "skipped" + } else { + "disabled" + }); + publication::write_json(&root.join("summary.json"), &summary)?; + let names = [ + "vrps.csv", + "vaps.csv", + "summary.json", + "stage-timing.json", + "analysis/timing.json", + ]; + let mut files = Vec::new(); + for name in names { + let source = root.join(name); + if !source.is_file() { + return Err(format!("missing expected artifact {name}")); + } + let dest = out.join(name); + publication::write_atomic(&dest, &fs::read(&source).map_err(|e| e.to_string())?)?; + files.push((dest, PathBuf::from(name))); + } + if let Some(dest) = &config.ccr_out { + publication::write_atomic( + dest, + &fs::read(root.join("result.ccr")).map_err(|e| e.to_string())?, + )?; + files.push((dest.clone(), PathBuf::from("result.ccr"))); + } + // The completion marker lives in `out`; use paths relative to that directory + // for its local artifacts and retain an absolute path only for an explicitly + // configured CCR outside `out`. + let completed_files = files + .iter() + .map(|(path, relative)| { + if path.starts_with(out) { + relative.clone() + } else { + path.clone() + } + }) + .collect::>(); + publication::complete(out, run_id, status, &completed_files)?; + if let Some(publish) = &config.publish_dir { + if status == "success" { + publication::publish(publish, run_id, &files, &summary)?; + summary["publicationStatus"] = json!("published"); + if let Err(e) = publication::write_json(&out.join("summary.json"), &summary) { + crate::logging::emit( + crate::logging::Level::Warn, + "publication_bookkeeping_failed", + || json!({"error":e}), + ); + } + } else { + publication::attempt(publish, run_id, status)?; + } + } + crate::logging::emit( + crate::logging::Level::Info, + "validation_completed", + || json!({"elapsed_ms":start.elapsed().as_millis(),"result_status":status}), + ); + Ok(if status == "partial" && config.require_complete { + 3 + } else { + 0 + }) } - fn directory_has_entries(path: &Path) -> Result { if !path.exists() { return Ok(false); @@ -280,21 +279,3 @@ fn convert_vrp_csv(input: &Path, output: &Path) -> Result<(), String> { .flush() .map_err(|error| format!("flush Panda RPKI VRP CSV: {error}")) } - -#[cfg(test)] -mod tests { - use super::resolve_setting; - - #[test] - fn logging_setting_precedence_is_cli_then_env_then_default() { - assert_eq!( - resolve_setting(&["debug".to_string()], Some("error".to_string()), "info"), - "debug" - ); - assert_eq!( - resolve_setting(&[], Some("json".to_string()), "text"), - "json" - ); - assert_eq!(resolve_setting(&[], None, "info"), "info"); - } -} diff --git a/src/daemon/mod.rs b/src/daemon/mod.rs index d24e2ea..b2ba445 100644 --- a/src/daemon/mod.rs +++ b/src/daemon/mod.rs @@ -2,10 +2,10 @@ //! Reuses its atomic status/JSONL writers, serial loop and retention model. //! The adapter runs the public validate command with durable protocol state. mod metrics; -mod process; +pub(crate) mod process; mod storage; -use crate::cli::{Options, VALIDATE_OPTIONS, VALIDATE_REPEATABLE}; +use crate::cli::config::Config as RuntimeConfig; use crate::logging::{Level, emit}; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; @@ -15,20 +15,6 @@ use std::process::{Command, Stdio}; use std::time::Instant; use storage::{append_json_line, write_json_pretty}; -const OWNED: &[&str] = &["out", "ccr-out", "rrdp-state-dir", "rrdp-sync-mode"]; -const CONTROLLER: &[&str] = &[ - "state-root", - "interval-secs", - "max-runs", - "retain-runs", - "run-timeout-secs", - "shutdown-grace-secs", -]; - -pub(crate) fn usage() -> &'static str { - "DAEMON:\n panda-rpki daemon --state-root [--interval-secs ] [--max-runs ] [--retain-runs ] [--run-timeout-secs ] [--shutdown-grace-secs ] -- --tal --ta [validation options]\n\nDefaults: interval=60s after each run, unlimited runs, retain=10, timeout=0 (disabled), shutdown grace=30s. Reuses protocol state across runs/restarts; automatically exports CCR and CSV per run. Output/state/sync-mode options are controller-owned." -} - struct Config { root: PathBuf, interval: u64, @@ -36,86 +22,20 @@ struct Config { retain: usize, timeout: u64, grace: u64, - child_args: Vec, - log_level: String, - log_format: String, + runtime: RuntimeConfig, } - -fn parse(args: &[String]) -> Result { - let split = args - .iter() - .position(|arg| arg == "--") - .ok_or("daemon requires -- before validation options")?; - let options = Options::parse(&args[..split], CONTROLLER, &[])?; - let child_args = args[split + 1..].to_vec(); - let child = Options::parse(&child_args, VALIDATE_OPTIONS, VALIDATE_REPEATABLE)?; - for name in OWNED { - if !child.values(name).is_empty() { - return Err(format!("daemon manages --{name}; use --state-root")); - } +impl Config { + fn new(runtime: RuntimeConfig) -> Result { + Ok(Self { + root: runtime.state_root.clone().ok_or("state-root required")?, + interval: runtime.interval_secs, + max_runs: runtime.max_runs, + retain: runtime.retain_runs, + timeout: runtime.run_timeout_secs, + grace: runtime.shutdown_grace_secs, + runtime, + }) } - let count = child.values("tal").len(); - if count == 0 || child.values("ta").len() != count { - return Err("--tal and --ta must be repeated as equal-length positional pairs".into()); - } - if !child.values("tal-id").is_empty() && child.values("tal-id").len() != count { - return Err("--tal-id must be supplied once for each --tal when used".into()); - } - for name in [ - "parallel-phase2-object-workers", - "parallel-max-repo-sync-workers-global", - "parallel-phase2-worker-queue-capacity", - "parallel-repo-worker-queue-capacity", - ] { - if child.parse_or(name, 1usize)? == 0 { - return Err(format!("--{name} must be greater than zero")); - } - } - child.parse_or("max-ca-depth", 64usize)?; - child.parse_or("http-timeout-secs", 300u64)?; - for (name, variable, default) in [ - ("log-level", "PANDA_RPKI_LOG_LEVEL", "info"), - ("log-format", "PANDA_RPKI_LOG_FORMAT", "text"), - ] { - let value = child - .values(name) - .first() - .cloned() - .or_else(|| std::env::var(variable).ok()) - .unwrap_or(default.into()); - if name == "log-level" { - crate::logging::parse_level(&value)?; - } else if !["text", "json"].contains(&value.as_str()) { - return Err(format!("invalid log format: {value}")); - } - } - let level = child - .values("log-level") - .first() - .cloned() - .or_else(|| std::env::var("PANDA_RPKI_LOG_LEVEL").ok()) - .unwrap_or("info".into()); - let format = child - .values("log-format") - .first() - .cloned() - .or_else(|| std::env::var("PANDA_RPKI_LOG_FORMAT").ok()) - .unwrap_or("text".into()); - let config = Config { - root: PathBuf::from(options.required("state-root")?), - interval: options.parse_or("interval-secs", 60u64)?, - max_runs: options.optional_parse("max-runs")?, - retain: options.parse_or("retain-runs", 10usize)?, - timeout: options.parse_or("run-timeout-secs", 0u64)?, - grace: options.parse_or("shutdown-grace-secs", 30u64)?, - child_args, - log_level: level, - log_format: format, - }; - if config.root.as_os_str().is_empty() || config.max_runs == Some(0) || config.retain == 0 { - return Err("state-root must be non-empty; max-runs and retain-runs must be > 0".into()); - } - Ok(config) } #[derive(Default, Serialize, Deserialize)] @@ -187,9 +107,9 @@ fn apply_retention(runs: &Path, retain: usize) -> Result, String> { && let Ok(summary) = read_json(&entry.path().join("run-summary.json")) && summary["controller"] == "panda-rpki-daemon-v1" && summary["run_seq"] == seq - && ["success", "failed"] + && ["success", "partial", "failed"] .iter() - .any(|s| summary["status"] == *s) + .any(|s| summary["status"] == *s || summary["resultStatus"] == *s) { dirs.push((seq, entry.path(), name)); } @@ -197,7 +117,14 @@ fn apply_retention(runs: &Path, retain: usize) -> Result, String> { dirs.sort_by_key(|x| x.0); let remove_count = dirs.len().saturating_sub(retain); let mut removed = Vec::new(); + let latest = runs + .parent() + .and_then(|p| fs::read_link(p.join("latest")).ok()) + .and_then(|p| p.file_name().map(|s| s.to_os_string())); for (_, path, name) in dirs.into_iter().take(remove_count) { + if latest.as_deref() == path.file_name() { + continue; + } fs::remove_dir_all(&path).map_err(|e| format!("remove old run {}: {e}", path.display()))?; removed.push(name); } @@ -214,20 +141,8 @@ fn status(root: &Path, state: &str, seq: u64, completed: u64) -> Result<(), Stri ) } -pub(crate) fn run(args: &[String]) -> Result<(), String> { - if args.iter().any(|a| a == "--help" || a == "-h") { - return Err(crate::cli::usage()); - } - let mut config = parse(args)?; - let progress_interval = - std::env::var("PANDA_RPKI_LOG_PROGRESS_INTERVAL_SECS").unwrap_or_else(|_| "10".into()); - Options::parse(&config.child_args, VALIDATE_OPTIONS, VALIDATE_REPEATABLE)? - .values("log-progress-interval-secs") - .first() - .unwrap_or(&progress_interval) - .parse::() - .map_err(|_| "invalid log-progress-interval-secs".to_string())?; - crate::logging::configure(&config.log_level, &config.log_format)?; +pub(crate) fn run(runtime: RuntimeConfig) -> Result { + let mut config = Config::new(runtime)?; emit(Level::Info, "daemon_started", || { json!({"interval_secs": config.interval, "retain_runs": config.retain, "run_timeout_secs": config.timeout, "state_root": config.root}) @@ -256,7 +171,7 @@ pub(crate) fn run(args: &[String]) -> Result<(), String> { result } -fn controller(config: &Config) -> Result<(), String> { +fn controller(config: &Config) -> Result { let root = &config.root; let runs = root.join("runs"); let state = root.join("state"); @@ -280,6 +195,7 @@ fn controller(config: &Config) -> Result<(), String> { let executable = std::env::current_exe().map_err(|e| e.to_string())?; let mut completed = 0u64; let mut failures = 0u64; + let mut partials = 0u64; status(root, "starting", lifecycle.last_seq, completed)?; emit( Level::Info, @@ -333,18 +249,17 @@ fn controller(config: &Config) -> Result<(), String> { command.env("LC_ALL", "C"); command .env("PANDA_RPKI_RUN_ID", &run_id) + .env("PANDA_RPKI_INTERNAL_CONTEXT", "daemon") .env("PANDA_RPKI_RUN_SEQ", seq.to_string()); + let config_path = out.join("validator-config.toml"); + crate::cli::publication::write_atomic( + &config_path, + config.runtime.child_config(&out, &state, mode)?.as_bytes(), + )?; command .arg("validate") - .args(&config.child_args) - .arg("--out") - .arg(&out) - .arg("--ccr-out") - .arg(out.join("result.ccr")) - .arg("--rrdp-state-dir") - .arg(&state) - .arg("--rrdp-sync-mode") - .arg(mode) + .arg("--config") + .arg(&config_path) .stdin(Stdio::null()) .stdout(File::create(out.join("process-output.txt")).map_err(|e| e.to_string())?) .stderr(File::create(out.join("validator.log")).map_err(|e| e.to_string())?); @@ -352,15 +267,27 @@ fn controller(config: &Config) -> Result<(), String> { let (exit_code, failure) = match result { Ok((exit, reason)) => ( exit.code(), - reason - .map(str::to_string) - .or_else(|| (!exit.success()).then(|| format!("validator exited: {exit}"))), + reason.map(str::to_string).or_else(|| { + (!exit.success() && exit.code() != Some(3)) + .then(|| format!("validator exited: {exit}")) + }), ), Err(error) => (None, Some(error)), }; let summary = read_json(&out.join("summary.json")); let failure = failure.or_else(|| summary.as_ref().err().cloned()); - let success = failure.is_none(); + let result_status = if failure.is_some() { + "failed" + } else { + summary + .as_ref() + .ok() + .and_then(|v| v["resultStatus"].as_str()) + .unwrap_or("failed") + }; + let success = result_status == "success"; + let partial = result_status == "partial"; + meta["resultStatus"] = json!(result_status); meta["status"] = json!(if success { "success" } else { "failed" }); meta["finished_at"] = json!(now()); meta["wall_ms"] = json!(start.elapsed().as_millis() as u64); @@ -376,7 +303,25 @@ fn controller(config: &Config) -> Result<(), String> { lifecycle.last_success = success; write_json_pretty(&lifecycle_path, &lifecycle)?; completed += 1; - failures += u64::from(!success); + failures += u64::from(!success && !partial); + partials += u64::from(partial); + if success { + crate::cli::publication::switch_latest( + root, + &Path::new("runs").join(out.file_name().unwrap()), + )?; + } + crate::cli::publication::attempt( + root, + &run_id, + if success { + "success" + } else if partial { + "partial" + } else { + "failed" + }, + )?; let removed = apply_retention(&runs, config.retain)?; if !removed.is_empty() { emit( @@ -407,9 +352,13 @@ fn controller(config: &Config) -> Result<(), String> { || json!({"runs_completed": completed, "failed_runs": failures, "stop_requested": process::stopping()}), ); if failures > 0 { - Err(format!("daemon completed with {failures} failed run(s)")) + Ok(2) } else { - Ok(()) + Ok(if config.runtime.require_complete && partials > 0 { + 3 + } else { + 0 + }) } } diff --git a/src/daemon/process.rs b/src/daemon/process.rs index ebc5904..cc7a4c1 100644 --- a/src/daemon/process.rs +++ b/src/daemon/process.rs @@ -39,19 +39,24 @@ impl Drop for Signals { } } -struct ChildGuard(Child); +struct ChildGuard { + child: Child, + reaped: bool, +} impl ChildGuard { #[allow(unsafe_code)] fn signal_group(&self, signal: libc::c_int) { // SAFETY: child is launched in its own group, with pgid equal to child PID. - unsafe { libc::kill(-(self.0.id() as libc::pid_t), signal) }; + unsafe { libc::kill(-(self.child.id() as libc::pid_t), signal) }; } } impl Drop for ChildGuard { fn drop(&mut self) { // Reap every child and terminate residual descendants on all error paths. - self.signal_group(libc::SIGKILL); - let _ = self.0.wait(); + if !self.reaped { + self.signal_group(libc::SIGKILL); + let _ = self.child.wait(); + } } } @@ -62,16 +67,17 @@ pub(super) fn execute( ) -> Result<(ExitStatus, Option<&'static str>), String> { use std::os::unix::process::CommandExt; command.process_group(0); - let mut child = ChildGuard( - command + let mut child = ChildGuard { + child: command .spawn() .map_err(|e| format!("spawn validator: {e}"))?, - ); + reaped: false, + }; let start = Instant::now(); crate::logging::emit( crate::logging::Level::Info, "daemon_child_started", - || serde_json::json!({"pid": child.0.id()}), + || serde_json::json!({"pid": child.child.id()}), ); let mut stop_at = None; let mut terminated_at = None; @@ -79,10 +85,11 @@ pub(super) fn execute( let mut killed = false; loop { if let Some(status) = child - .0 + .child .try_wait() .map_err(|e| format!("wait validator: {e}"))? { + child.reaped = true; return Ok((status, reason)); } if stopping() { @@ -126,3 +133,71 @@ pub(super) fn sleep(seconds: u64) { std::thread::sleep(Duration::from_millis(100)); } } + +/// Fork before starting validator worker threads; supervise the whole process group. +#[allow(unsafe_code)] +pub(crate) fn supervise( + timeout: u64, + run: impl FnOnce() -> Result, +) -> Result { + let _signals = Signals::install()?; + // SAFETY: called only by the single-threaded CLI before validation workers start. + let pid = unsafe { libc::fork() }; + if pid < 0 { + return Err(std::io::Error::last_os_error().to_string()); + } + if pid == 0 { + unsafe { + libc::setpgid(0, 0); + libc::signal(libc::SIGTERM, libc::SIG_DFL); + libc::signal(libc::SIGINT, libc::SIG_DFL); + } + let code = match run() { + Ok(c) => c, + Err(e) => { + eprintln!("panda-rpki: {e}"); + 2 + } + }; + unsafe { libc::_exit(code) }; + } + unsafe { + libc::setpgid(pid, pid); + } + let start = Instant::now(); + let mut terminated = None; + let mut status = 0; + loop { + let waited = unsafe { libc::waitpid(pid, &mut status, libc::WNOHANG) }; + if waited == pid { + if terminated.is_some() { + return Err("validation timed out or was interrupted".into()); + } + return Ok(if libc::WIFEXITED(status) { + libc::WEXITSTATUS(status) + } else { + 2 + }); + } + if waited < 0 { + let e = std::io::Error::last_os_error(); + if e.kind() != std::io::ErrorKind::Interrupted { + return Err(e.to_string()); + } + } + if terminated.is_none() + && (stopping() || (timeout > 0 && start.elapsed().as_secs() >= timeout)) + { + unsafe { + libc::kill(-pid, libc::SIGTERM); + } + terminated = Some(Instant::now()); + } + if terminated.is_some_and(|t| t.elapsed() >= Duration::from_secs(2)) { + unsafe { + libc::kill(-pid, libc::SIGKILL); + } + } + std::thread::sleep(Duration::from_millis(25)); + } +} diff --git a/src/daemon/tests.rs b/src/daemon/tests.rs index 5496bc4..cb095e1 100644 --- a/src/daemon/tests.rs +++ b/src/daemon/tests.rs @@ -1,113 +1,76 @@ use super::*; -fn args(extra: &[&str]) -> Vec { - [ - vec!["--state-root", "data"], - extra.to_vec(), - vec!["--", "--tal", "a.tal", "--ta", "a.cer"], - ] - .concat() - .into_iter() - .map(str::to_owned) - .collect() -} - #[test] -fn defaults_and_invalid_controller_options() { - let config = parse(&args(&[])).unwrap(); - assert_eq!( - ( - config.interval, - config.max_runs, - config.retain, - config.timeout, - config.grace - ), - (60, None, 10, 0, 30) - ); - for bad in [ - vec!["--max-runs", "0"], - vec!["--retain-runs", "0"], - vec!["--interval-secs", "-1"], - vec!["--run-timeout-secs", "bad"], - vec!["--interval-secs", "2", "--interval-secs", "3"], - ] { - assert!(parse(&args(&bad)).is_err()); - } - let config = parse(&args(&["--interval-secs", "0", "--max-runs", "3"])).unwrap(); - assert_eq!((config.interval, config.max_runs), (0, Some(3))); -} - -#[test] -fn rejects_child_overrides_and_unknown_flags() { - for flag in [ - "--out", - "--rrdp-state-dir", - "--rrdp-sync-mode", - "--ccr-out", - "--unknown", - ] { - let mut argv = args(&[]); - argv.extend([flag.into(), "some-value".into()]); - assert!(parse(&argv).is_err(), "{flag}"); - } - let mut argv = args(&[]); - argv.extend(["--parallel-phase2-object-workers".into(), "0".into()]); - assert!(parse(&argv).is_err()); -} - -#[test] -fn protocol_state_selects_snapshot_delta_and_recovery() { +fn sync_mode_selects_snapshot_delta_and_recovery() { let root = tempfile::tempdir().unwrap(); assert_eq!(sync_mode(root.path(), false).unwrap(), "snapshot"); + let db = root.path().join("repository-db"); fs::create_dir(&db).unwrap(); assert_eq!(sync_mode(root.path(), true).unwrap(), "snapshot"); + fs::write(db.join("CURRENT"), b"x").unwrap(); assert_eq!(sync_mode(root.path(), true).unwrap(), "delta"); assert_eq!(sync_mode(root.path(), false).unwrap(), "auto"); } #[test] -fn retention_preserves_unrelated_incomplete_and_symlinked_runs() { +fn retention_keeps_incomplete_and_latest_runs() { let root = tempfile::tempdir().unwrap(); - for seq in 1..=3 { - let path = root.path().join(format!("run_{seq:06}")); + let runs = root.path().join("runs"); + fs::create_dir(&runs).unwrap(); + for (seq, result_status) in [(1, "success"), (2, "partial"), (3, "failed")] { + let path = runs.join(format!("run_{seq:06}")); fs::create_dir(&path).unwrap(); write_json_pretty( &path.join("run-summary.json"), &json!({ - "controller": "panda-rpki-daemon-v1", "run_seq": seq, "status": "success" + "controller": "panda-rpki-daemon-v1", + "run_seq": seq, + "status": if result_status == "success" { "success" } else { "failed" }, + "resultStatus": result_status, }), ) .unwrap(); } - fs::create_dir(root.path().join("unrelated")).unwrap(); - fs::create_dir(root.path().join("run_000004")).unwrap(); - let outside = tempfile::tempdir().unwrap(); - std::os::unix::fs::symlink(outside.path(), root.path().join("run_000005")).unwrap(); - assert_eq!( - apply_retention(root.path(), 1).unwrap(), - ["run_000001", "run_000002"] - ); - for name in ["run_000003", "run_000004", "run_000005", "unrelated"] { - assert!(root.path().join(name).exists()); - } - assert!(regular_directory(&root.path().join("run_000005")).is_err()); + std::os::unix::fs::symlink("runs/run_000001", root.path().join("latest")).unwrap(); + + // Only completed runs are candidates; partial is completed evidence and is retained + // alongside the latest pointer even when the retention limit is one. + assert_eq!(apply_retention(&runs, 1).unwrap(), ["run_000002"]); + assert!(runs.join("run_000001").exists()); + assert!(!runs.join("run_000002").exists()); + assert!(runs.join("run_000003").exists()); } #[test] -fn atomic_json_writers_and_state_lock() { +fn retention_ignores_unrelated_paths_and_symlinks() { + let root = tempfile::tempdir().unwrap(); + let runs = root.path().join("runs"); + fs::create_dir(&runs).unwrap(); + let path = runs.join("run_000001"); + fs::create_dir(&path).unwrap(); + write_json_pretty( + &path.join("run-summary.json"), + &json!({"controller": "other", "run_seq": 1, "status": "success"}), + ) + .unwrap(); + fs::create_dir(runs.join("unrelated")).unwrap(); + + let outside = tempfile::tempdir().unwrap(); + std::os::unix::fs::symlink(outside.path(), runs.join("run_000002")).unwrap(); + + assert!(apply_retention(&runs, 0).unwrap().is_empty()); + assert!(runs.join("run_000001").exists()); + assert!(runs.join("run_000002").exists()); + assert!(runs.join("unrelated").exists()); +} + +#[test] +fn atomic_json_writer_replaces_content() { let root = tempfile::tempdir().unwrap(); let path = root.path().join("state.json"); write_json_pretty(&path, &json!({"seq": 1})).unwrap(); write_json_pretty(&path, &json!({"seq": 2})).unwrap(); assert_eq!(read_json(&path).unwrap()["seq"], 2); - let lines = root.path().join("summary.jsonl"); - append_json_line(&lines, &json!({"seq": 1})).unwrap(); - append_json_line(&lines, &json!({"seq": 2})).unwrap(); - assert_eq!(fs::read_to_string(lines).unwrap().lines().count(), 2); - let file = File::open(&path).unwrap(); - file.try_lock().unwrap(); - assert!(File::open(path).unwrap().try_lock().is_err()); } diff --git a/src/main.rs b/src/main.rs index d41c302..7191b64 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,16 +1,10 @@ fn main() { - match panda_rpki::cli::run(std::env::args().skip(1)) { - Ok(()) => {} - Err(error) if error == panda_rpki::cli::usage() => { - println!("{error}"); - } + let code = match panda_rpki::cli::run(std::env::args().skip(1)) { + Ok(code) => code, Err(error) => { - panda_rpki::logging::emit( - panda_rpki::logging::Level::Error, - "command_failed", - || serde_json::json!({"error": error}), - ); - std::process::exit(2); + eprintln!("panda-rpki: {error}"); + 2 } - } + }; + std::process::exit(code); } diff --git a/src/repository/sync/rrdp/notification_sync.rs b/src/repository/sync/rrdp/notification_sync.rs index 8dd8fc7..b9340b5 100644 --- a/src/repository/sync/rrdp/notification_sync.rs +++ b/src/repository/sync/rrdp/notification_sync.rs @@ -106,15 +106,22 @@ fn sync_from_notification_inner( .as_ref() .filter(|s| s.session_id == notif.session_id.to_string()); let log_selection = |mode: &str, reason: &str| { - crate::logging::emit(crate::logging::Level::Debug, "rrdp_sync_selected", || serde_json::json!({ - "notify_uri": notification_uri, "mode": mode, "reason": reason, - "session_id": notif.session_id.to_string(), "serial": notif.serial, - "previous_serial": state.as_ref().map(|previous| previous.serial), - "previous_session_id": state.as_ref().map(|previous| &previous.session_id), - })); + crate::logging::emit(crate::logging::Level::Debug, "rrdp_sync_selected", || { + serde_json::json!({ + "notify_uri": notification_uri, "mode": mode, "reason": reason, + "session_id": notif.session_id.to_string(), "serial": notif.serial, + "previous_serial": state.as_ref().map(|previous| previous.serial), + "previous_session_id": state.as_ref().map(|previous| &previous.session_id), + }) + }); + }; + let mut snapshot_reason = if state.is_none() { + "no_local_state" + } else if same_session_state.is_none() { + "session_changed" + } else { + "delta_chain_unavailable" }; - let mut snapshot_reason = if state.is_none() { "no_local_state" } - else if same_session_state.is_none() { "session_changed" } else { "delta_chain_unavailable" }; if let Some(s) = same_session_state { if s.serial == notif.serial { @@ -191,9 +198,12 @@ fn sync_from_notification_inner( ¬ification_origin, ) { Ok(bytes) => { - crate::logging::progress::emit!("rrdp_download_completed", serde_json::json!({ - "resource": "delta", "bytes": bytes.len(), "notify_uri": notification_uri, - })); + crate::logging::progress::emit!( + "rrdp_download_completed", + serde_json::json!({ + "resource": "delta", "bytes": bytes.len(), "notify_uri": notification_uri, + }) + ); if let Some(t) = timing.as_ref() { t.record_count("rrdp_delta_fetch_ok_total", 1); t.record_count("rrdp_delta_bytes_total", bytes.len() as u64); @@ -243,7 +253,11 @@ fn sync_from_notification_inner( Ok(n) => applied_total += n, Err(error) => { snapshot_reason = "delta_apply_failed"; - crate::logging::emit(crate::logging::Level::Warn, "rrdp_delta_fallback", || serde_json::json!({"notify_uri": notification_uri, "error": error.to_string(), "serial": serial})); + crate::logging::emit( + crate::logging::Level::Warn, + "rrdp_delta_fallback", + || serde_json::json!({"notify_uri": notification_uri, "error": error.to_string(), "serial": serial}), + ); ok = false; break; } @@ -320,9 +334,12 @@ fn sync_from_notification_inner( ¬ification_origin, ) { Ok(v) => { - crate::logging::progress::emit!("rrdp_download_completed", serde_json::json!({ - "resource": "snapshot", "bytes": v.1, "notify_uri": notification_uri, - })); + crate::logging::progress::emit!( + "rrdp_download_completed", + serde_json::json!({ + "resource": "snapshot", "bytes": v.1, "notify_uri": notification_uri, + }) + ); if let Some(t) = timing.as_ref() { t.record_count("rrdp_snapshot_fetch_ok_total", 1); t.record_count("rrdp_snapshot_bytes_total", v.1); diff --git a/src/repository/sync/rrdp/snapshot_sync.rs b/src/repository/sync/rrdp/snapshot_sync.rs index b2500d5..33d9844 100644 --- a/src/repository/sync/rrdp/snapshot_sync.rs +++ b/src/repository/sync/rrdp/snapshot_sync.rs @@ -80,10 +80,12 @@ fn sync_from_notification_snapshot_inner( }; drop(_parse_step); drop(_parse_total); - crate::logging::emit(crate::logging::Level::Debug, "rrdp_sync_selected", || serde_json::json!({ - "notify_uri": notification_uri, "mode": "snapshot", "reason": "snapshot_requested", - "session_id": notif.session_id.to_string(), "serial": notif.serial, - })); + crate::logging::emit(crate::logging::Level::Debug, "rrdp_sync_selected", || { + serde_json::json!({ + "notify_uri": notification_uri, "mode": "snapshot", "reason": "snapshot_requested", + "session_id": notif.session_id.to_string(), "serial": notif.serial, + }) + }); let _fetch_step = timing .as_ref() @@ -100,9 +102,12 @@ fn sync_from_notification_snapshot_inner( ¬ification_origin, ) { Ok(v) => { - crate::logging::progress::emit!("rrdp_download_completed", serde_json::json!({ - "resource": "snapshot", "bytes": v.1, "notify_uri": notification_uri, - })); + crate::logging::progress::emit!( + "rrdp_download_completed", + serde_json::json!({ + "resource": "snapshot", "bytes": v.1, "notify_uri": notification_uri, + }) + ); if let Some(t) = timing.as_ref() { t.record_count("rrdp_snapshot_fetch_ok_total", 1); t.record_count("rrdp_snapshot_bytes_total", v.1); diff --git a/src/runtime/post_validation.rs b/src/runtime/post_validation.rs index feeb34d..54847cb 100644 --- a/src/runtime/post_validation.rs +++ b/src/runtime/post_validation.rs @@ -40,26 +40,62 @@ fn print_summary_from_shared(validation_time: time::OffsetDateTime, shared: &Pos }) }); if warning_count > 0 { - let mut categories: std::collections::BTreeMap<&str, (usize, Vec)> = std::collections::BTreeMap::new(); - let warnings = shared.tree_warnings.iter().map(|warning| (warning.category.as_str(), warning.message.as_str(), warning.context.as_deref())) - .chain(shared.publication_points.iter().flat_map(|pp| pp.warnings.iter()).map(|warning| (warning.category.as_str(), warning.message.as_str(), warning.context.as_deref()))); + let mut categories: std::collections::BTreeMap<&str, (usize, Vec)> = + std::collections::BTreeMap::new(); + let warnings = shared + .tree_warnings + .iter() + .map(|warning| { + ( + warning.category.as_str(), + warning.message.as_str(), + warning.context.as_deref(), + ) + }) + .chain( + shared + .publication_points + .iter() + .flat_map(|pp| pp.warnings.iter()) + .map(|warning| { + ( + warning.category.as_str(), + warning.message.as_str(), + warning.context.as_deref(), + ) + }), + ); for (category, message, context) in warnings { let entry = categories.entry(category).or_default(); entry.0 += 1; let mut text = crate::logging::redact_text(message); let truncated = text.len() > 1024; let mut boundary = text.len().min(1024); - while !text.is_char_boundary(boundary) { boundary -= 1; } + while !text.is_char_boundary(boundary) { + boundary -= 1; + } text.truncate(boundary); if entry.1.len() < 3 { - entry.1.push(serde_json::json!({"message": text, "truncated": truncated})); + entry + .1 + .push(serde_json::json!({"message": text, "truncated": truncated})); } - crate::logging::emit(crate::logging::Level::Debug, "validation_warning_detail", || serde_json::json!({ - "category": category, "message": message, "context": context, - })); + crate::logging::emit( + crate::logging::Level::Debug, + "validation_warning_detail", + || { + serde_json::json!({ + "category": category, "message": message, "context": context, + }) + }, + ); } for (category, (count, examples)) in categories { - crate::logging::emit(crate::logging::Level::Info, "validation_warning_category", || serde_json::json!({"category": category, "count": count, "examples": examples})); + crate::logging::emit( + crate::logging::Level::Info, + "validation_warning_category", + || serde_json::json!({"category": category, "count": count, "examples": examples}), + ); } crate::logging::emit(crate::logging::Level::Warn, "validation_warnings", || { serde_json::json!({ diff --git a/src/runtime/run.rs b/src/runtime/run.rs index 23c665c..bb8ce17 100644 --- a/src/runtime/run.rs +++ b/src/runtime/run.rs @@ -369,7 +369,9 @@ pub(crate) fn run_config(args: RunConfig) -> Result<(), String> { shared.publication_points.len() as u64, ); let timing_json_path = out_dir.join("timing.json"); - crate::logging::artifact::write("timing", &timing_json_path, || t.write_json(&timing_json_path, 20))?; + crate::logging::artifact::write("timing", &timing_json_path, || { + t.write_json(&timing_json_path, 20) + })?; } // Write the compatibility summary after analysis timing has been flushed. @@ -378,13 +380,15 @@ pub(crate) fn run_config(args: RunConfig) -> Result<(), String> { // point audit, so the final summary can use those authoritative counts. if let Some(path) = args.summary_out_path.as_deref() { let analysis_counts = timing.as_ref().map(|(_, handle)| handle.counts_snapshot()); - crate::logging::artifact::write("summary", path, || write_summary( - path, - &shared, - &args, - validation_ms, - analysis_counts.as_ref(), - ))?; + crate::logging::artifact::write("summary", path, || { + write_summary( + path, + &shared, + &args, + validation_ms, + analysis_counts.as_ref(), + ) + })?; } print_summary_from_shared(validation_time, &shared); @@ -393,6 +397,8 @@ pub(crate) fn run_config(args: RunConfig) -> Result<(), String> { #[derive(Serialize)] struct ValidationSummary { + #[serde(rename = "repositoryFailedCount")] + repository_failed_count: usize, backend: &'static str, worker_count: usize, repo_sync_worker_count: usize, @@ -428,6 +434,17 @@ fn write_summary( use std::collections::BTreeSet; let mut summary = ValidationSummary { + repository_failed_count: shared + .publication_points + .iter() + .filter(|p| p.repo_sync_error.is_some()) + .map(|p| { + p.rrdp_notification_uri + .as_deref() + .unwrap_or(&p.rsync_base_uri) + }) + .collect::>() + .len(), backend: "panda-rpki", worker_count: args.parallel_phase2_config.object_workers, repo_sync_worker_count: args.parallel_phase1_config.max_repo_sync_workers_global, diff --git a/src/ta_constraints/implementation.rs b/src/ta_constraints/implementation.rs index 30c7e4c..49b03a9 100644 --- a/src/ta_constraints/implementation.rs +++ b/src/ta_constraints/implementation.rs @@ -143,25 +143,23 @@ impl TaConstraintsByTal { for spec in explicit_specs { let (tal_id, path) = spec .split_once('=') - .ok_or_else(|| format!("--ta-constraints expects =, got '{spec}'"))?; + .ok_or_else(|| format!("constraints expects =, got '{spec}'"))?; let tal_id = tal_id.trim(); let path = path.trim(); if tal_id.is_empty() || path.is_empty() { return Err(format!( - "--ta-constraints expects non-empty =, got '{spec}'" + "constraints expects non-empty =, got '{spec}'" )); } if !tal_ids.contains(tal_id) { - return Err(format!( - "--ta-constraints references unknown TAL id '{tal_id}'" - )); + return Err(format!("constraints references unknown TAL id '{tal_id}'")); } if explicit_paths .insert(tal_id.to_string(), PathBuf::from(path)) .is_some() { return Err(format!( - "--ta-constraints specifies TAL id '{tal_id}' more than once" + "constraints specifies TAL id '{tal_id}' more than once" )); } } diff --git a/src/validation/manifest/models_and_process.rs b/src/validation/manifest/models_and_process.rs index 47ac23f..1ba4d95 100644 --- a/src/validation/manifest/models_and_process.rs +++ b/src/validation/manifest/models_and_process.rs @@ -212,9 +212,13 @@ impl ManifestFreshError { use crate::output::report::WarningCategory; match self { Self::Storage(_) | Self::RepoSyncFailed { .. } => WarningCategory::Unclassified, - Self::IssuerCaLoadFailed { .. } | Self::EeCertPath(_) | Self::EeResources(_) => WarningCategory::Certificate, - Self::ManifestOutsidePublicationPoint { .. } | Self::ManifestNumberNotIncreasing { .. } - | Self::ThisUpdateNotIncreasing { .. } | Self::HashMismatch { .. } => WarningCategory::SecurityPolicy, + Self::IssuerCaLoadFailed { .. } | Self::EeCertPath(_) | Self::EeResources(_) => { + WarningCategory::Certificate + } + Self::ManifestOutsidePublicationPoint { .. } + | Self::ManifestNumberNotIncreasing { .. } + | Self::ThisUpdateNotIncreasing { .. } + | Self::HashMismatch { .. } => WarningCategory::SecurityPolicy, _ => WarningCategory::ManifestCrl, } } diff --git a/src/validation/run_tree_from_tal/discovery.rs b/src/validation/run_tree_from_tal/discovery.rs index a9d3935..ae09eff 100644 --- a/src/validation/run_tree_from_tal/discovery.rs +++ b/src/validation/run_tree_from_tal/discovery.rs @@ -273,7 +273,11 @@ fn discover_multiple_roots_from_tal_inputs( let mut roots = Vec::with_capacity(tal_inputs.len()); crate::logging::observer::phase("trust_anchor_discovery"); for tal_input in tal_inputs { - crate::logging::emit(crate::logging::Level::Info, "trust_anchor_started", || serde_json::json!({"tal_id": tal_input.tal_id})); + crate::logging::emit( + crate::logging::Level::Info, + "trust_anchor_started", + || serde_json::json!({"tal_id": tal_input.tal_id}), + ); let discovery = match root_discovery_from_tal_input( tal_input, http_fetcher, @@ -295,7 +299,11 @@ fn discover_multiple_roots_from_tal_inputs( continue; } Err(error) => { - crate::logging::emit(crate::logging::Level::Error, "trust_anchor_failed", || serde_json::json!({"tal_id": tal_input.tal_id, "error": error.to_string()})); + crate::logging::emit( + crate::logging::Level::Error, + "trust_anchor_failed", + || serde_json::json!({"tal_id": tal_input.tal_id, "error": error.to_string()}), + ); return Err(error.into()); } }; @@ -310,7 +318,11 @@ fn discover_multiple_roots_from_tal_inputs( discovery, root_handle, }); - crate::logging::emit(crate::logging::Level::Info, "trust_anchor_completed", || serde_json::json!({"tal_id": tal_input.tal_id})); + crate::logging::emit( + crate::logging::Level::Info, + "trust_anchor_completed", + || serde_json::json!({"tal_id": tal_input.tal_id}), + ); } if roots.is_empty() { return Err(RunTreeFromTalError::Setup( diff --git a/src/validation/tree_parallel/phase2.rs b/src/validation/tree_parallel/phase2.rs index a9037b6..ea44551 100644 --- a/src/validation/tree_parallel/phase2.rs +++ b/src/validation/tree_parallel/phase2.rs @@ -164,17 +164,19 @@ pub fn run_tree_parallel_phase2_audit_multi_root( config, ); - crate::logging::observer::update(finished.len() as u64, || serde_json::json!({ - "publication_points_completed": finished.len(), - "publication_points_started": instances_started, - "ca_queue": ca_queue.len(), "ready_queue": ready_queue.len(), - "waiting_repository_identities": ca_waiting_repo_by_identity.len(), - "publication_points_inflight": inflight_publication_points.len(), - "pending_object_dispatch": pending_roa_dispatch.len(), - "pending_finalization": pending_finalization.len(), - "finalization_inflight": finalize_inflight, - "staging_inflight": staging_inflight, - })); + crate::logging::observer::update(finished.len() as u64, || { + serde_json::json!({ + "publication_points_completed": finished.len(), + "publication_points_started": instances_started, + "ca_queue": ca_queue.len(), "ready_queue": ready_queue.len(), + "waiting_repository_identities": ca_waiting_repo_by_identity.len(), + "publication_points_inflight": inflight_publication_points.len(), + "pending_object_dispatch": pending_roa_dispatch.len(), + "pending_finalization": pending_finalization.len(), + "finalization_inflight": finalize_inflight, + "staging_inflight": staging_inflight, + }) + }); let repo_poll_timeout = event_poll_timeout( &ca_queue, &ready_queue, diff --git a/src/validation/tree_runner/audit_projection.rs b/src/validation/tree_runner/audit_projection.rs index 07a6dae..1d4e9a7 100644 --- a/src/validation/tree_runner/audit_projection.rs +++ b/src/validation/tree_runner/audit_projection.rs @@ -127,7 +127,9 @@ fn build_publication_point_audit_from_failed_fetch( .map(AuditWarning::from) .collect::>(); warnings.push(AuditWarning::from( - &Warning::new(fresh_error.to_string()).with_category(fresh_error.warning_category()).with_context(&ca.manifest_rsync_uri), + &Warning::new(fresh_error.to_string()) + .with_category(fresh_error.warning_category()) + .with_context(&ca.manifest_rsync_uri), )); PublicationPointAudit { node_id: None, diff --git a/tests/cli_errors.rs b/tests/cli_errors.rs index 1da7e28..679b750 100644 --- a/tests/cli_errors.rs +++ b/tests/cli_errors.rs @@ -1,15 +1,12 @@ use std::process::Command; fn command() -> Command { - let mut cmd = Command::new(env!("CARGO_BIN_EXE_panda-rpki")); - cmd.env_remove("PANDA_RPKI_LOG_LEVEL") - .env_remove("PANDA_RPKI_LOG_FORMAT"); - cmd + Command::new(env!("CARGO_BIN_EXE_panda-rpki")) } #[test] -fn help_is_successful_and_goes_to_stdout() { - for args in [vec![], vec!["--help"], vec!["-h"]] { +fn top_level_help_and_version_are_successful() { + for args in [vec!["--help"], vec!["-h"], vec!["--version"], vec!["-V"]] { let result = command().args(args).output().unwrap(); assert!(result.status.success()); assert!(String::from_utf8_lossy(&result.stdout).contains("panda-rpki")); @@ -18,20 +15,26 @@ fn help_is_successful_and_goes_to_stdout() { } #[test] -fn invalid_command_returns_two_without_stdout() { - let result = command().arg("not-a-command").output().unwrap(); - assert_eq!(result.status.code(), Some(2)); - assert!(result.stdout.is_empty()); - assert!(String::from_utf8_lossy(&result.stderr).contains("unknown command")); +fn subcommand_help_does_not_read_a_config() { + for mode in ["validate", "daemon"] { + let result = command() + .args([mode, "--help", "--config", "missing.toml"]) + .output() + .unwrap(); + assert!(result.status.success()); + assert!(String::from_utf8_lossy(&result.stdout).contains("--config FILE")); + assert!(result.stderr.is_empty()); + } } #[test] -fn malformed_validation_arguments_do_not_create_outputs() { +fn missing_config_and_removed_options_are_rejected_without_outputs() { let directory = tempfile::tempdir().unwrap(); for args in [ - vec!["validate", "--out", "output"], - vec!["validate", "--unknown", "value"], - vec!["validate", "--out"], + vec!["validate"], + vec!["validate", "--tal", "input.tal"], + vec!["daemon", "--state-root", "data"], + vec!["validate", "--config", "missing.toml", "--unknown"], ] { let result = command() .current_dir(directory.path()) @@ -40,25 +43,51 @@ fn malformed_validation_arguments_do_not_create_outputs() { .unwrap(); assert_eq!(result.status.code(), Some(2)); assert!(result.stdout.is_empty()); + assert!(String::from_utf8_lossy(&result.stderr).contains("panda-rpki:")); assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 0); } } #[test] -fn missing_daemon_state_root_fails_without_creating_state() { +fn check_config_is_read_only_for_invalid_inputs() { let directory = tempfile::tempdir().unwrap(); + std::fs::write( + directory.path().join("validate.toml"), + r#" +config-version = 1 +mode = "validate" +out = "output" + +[[anchors]] +id = "missing" +tal = "missing.tal" +"#, + ) + .unwrap(); + let result = command() .current_dir(directory.path()) - .args([ - "daemon", - "--", - "--tal", - "missing.tal", - "--ta", - "missing.cer", - ]) + .args(["validate", "--config", "validate.toml", "--check-config"]) .output() .unwrap(); assert_eq!(result.status.code(), Some(2)); - assert_eq!(std::fs::read_dir(directory.path()).unwrap().count(), 0); + assert!(!directory.path().join("output").exists()); + + let result = command() + .current_dir(directory.path()) + .args(["validate", "--config", "validate.toml", "--print-config"]) + .output() + .unwrap(); + assert_eq!(result.status.code(), Some(2)); + assert!(!directory.path().join("output").exists()); +} + +#[test] +fn config_mode_and_mutually_exclusive_inspection_flags_are_rejected() { + let result = command() + .args(["validate", "--check-config", "--print-config"]) + .output() + .unwrap(); + assert_eq!(result.status.code(), Some(2)); + assert!(String::from_utf8_lossy(&result.stderr).contains("mutually exclusive")); } diff --git a/tests/logging_contract.rs b/tests/logging_contract.rs index 82eb841..7da07c1 100644 --- a/tests/logging_contract.rs +++ b/tests/logging_contract.rs @@ -1,70 +1,15 @@ use std::process::Command; #[test] -fn default_format_is_human_readable_text() { +fn environment_log_overrides_are_ignored() { let result = Command::new(env!("CARGO_BIN_EXE_panda-rpki")) - .env_remove("PANDA_RPKI_LOG_FORMAT") - .env_remove("PANDA_RPKI_LOG_LEVEL") - .args(["validate"]) + .env("PANDA_RPKI_LOG_LEVEL", "invalid") + .env("PANDA_RPKI_LOG_FORMAT", "invalid") + .args(["validate", "--log-level", "debug"]) .output() .unwrap(); - let diagnostic = String::from_utf8(result.stderr).unwrap(); assert_eq!(result.status.code(), Some(2)); - assert!(diagnostic.contains("ERROR Command failed")); - assert!(!diagnostic.contains("command_failed")); - let call_line = include_str!("../src/main.rs") - .lines() - .position(|line| line.contains("panda_rpki::logging::emit(")) - .unwrap() - + 1; - assert!(diagnostic.ends_with(&format!("[src/main.rs:{call_line}]\n"))); - assert!(result.stdout.is_empty()); -} - -#[test] -fn invalid_progress_interval_is_rejected_without_outputs() { - for value in ["-1", "wrong", "18446744073709551616"] { - let result = Command::new(env!("CARGO_BIN_EXE_panda-rpki")) - .args(["validate", "--log-progress-interval-secs", value]) - .output() - .unwrap(); - assert_eq!(result.status.code(), Some(2)); - assert!(result.stdout.is_empty()); - assert!(String::from_utf8_lossy(&result.stderr).contains("log-progress-interval-secs")); - } -} - -#[test] -fn progress_cli_overrides_invalid_environment() { - let result = Command::new(env!("CARGO_BIN_EXE_panda-rpki")) - .env("PANDA_RPKI_LOG_PROGRESS_INTERVAL_SECS", "invalid") - .args([ - "validate", - "--log-progress-interval-secs", - "0", - "--log-format", - "json", - ]) - .output() - .unwrap(); - let record: serde_json::Value = serde_json::from_slice(&result.stderr).unwrap(); - let call_line = include_str!("../src/main.rs") - .lines() - .position(|line| line.contains("panda_rpki::logging::emit(")) - .unwrap() - + 1; - assert_eq!(record["source"]["file"], "src/main.rs"); - assert_eq!(record["source"]["line"], call_line); - assert!( - record["fields"]["error"] - .as_str() - .unwrap() - .contains("--tal") - ); - assert!( - !record["fields"]["error"] - .as_str() - .unwrap() - .contains("progress") - ); + let stderr = String::from_utf8_lossy(&result.stderr); + assert!(stderr.contains("unsupported or repeated option: --log-level")); + assert!(!stderr.contains("invalid log level")); } diff --git a/tests/support/test_daemon.py b/tests/support/test_daemon.py index 8b358e6..81fa9ef 100644 --- a/tests/support/test_daemon.py +++ b/tests/support/test_daemon.py @@ -1,6 +1,5 @@ -"""Real CLI regression: HTTPS snapshot/delta, restart, interval, lock and signals.""" +"""Real CLI regression for the forced-TOML validate and daemon contracts.""" import json -import os from pathlib import Path import signal import socket @@ -28,6 +27,62 @@ def state_is(root, state): return path.exists() and read(path)["state"] == state +def validate_config(path, fixture, out, state, mode="auto", ccr=None, timeout=2, + progress=10): + ccr = ccr or (out / "result.ccr") + path.write_text( + f'''config-version = 1 +mode = "validate" +out = {str(out)!r} +rrdp-state-dir = {str(state)!r} +rrdp-sync-mode = "{mode}" +ccr-out = {str(ccr)!r} +http-timeout-secs = {timeout} +http-root-cert = [{str(fixture / "certs/rrdp-ca.pem")!r}] +parallel-max-repo-sync-workers-global = 2 +parallel-phase2-object-workers = 2 +log-level = "info" +log-format = "json" +log-progress-interval-secs = {progress} + +[[anchors]] +id = "custom" +tal = {str(fixture / "tal/custom.tal")!r} +ta = {str(fixture / "ta/custom-ta.cer")!r} +''', + encoding="utf-8", + ) + + +def daemon_config(path, fixture, root, max_runs=None, interval=2, retain=3, + timeout=0, grace=30, tal=None, auto=False, progress=10): + tal = tal or (fixture / "tal/custom.tal") + ta_line = "" if auto else f'ta = {str(fixture / "ta/custom-ta.cer")!r}\n' + max_line = "" if max_runs is None else f"max-runs = {max_runs}\n" + path.write_text( + f'''config-version = 1 +mode = "daemon" +state-root = {str(root)!r} +interval-secs = {interval} +{max_line}retain-runs = {retain} +run-timeout-secs = {timeout} +shutdown-grace-secs = {grace} +http-timeout-secs = 2 +http-root-cert = [{str(fixture / "certs/rrdp-ca.pem")!r}] +parallel-max-repo-sync-workers-global = 2 +parallel-phase2-object-workers = 2 +log-level = "info" +log-format = "json" +log-progress-interval-secs = {progress} + +[[anchors]] +id = "custom" +tal = {str(tal)!r} +{ta_line}''', + encoding="utf-8", + ) + + def run(binary): support = Path(__file__).resolve().parent with tempfile.TemporaryDirectory() as tmp: @@ -36,41 +91,44 @@ def run(binary): with socket.socket() as sock: sock.bind(("127.0.0.1", 0)) port = sock.getsockname()[1] - subprocess.run([sys.executable, str(support / "generate_repository.py"), - "--rrdp-host", "127.0.0.1", "--rrdp-port", str(port), - # Key generation is CPU-heavy under instrumented CI builds. - "--output", str(fixture)], check=True, timeout=120) + subprocess.run( + [sys.executable, str(support / "generate_repository.py"), + "--rrdp-host", "127.0.0.1", "--rrdp-port", str(port), + "--output", str(fixture)], check=True, timeout=120) active = fixture / "active" active.symlink_to("cases/baseline-v1/http") - server = subprocess.Popen([sys.executable, str(support / "serve_repository.py"), - str(active), str(port), str(fixture / "certs/rrdp-server.pem"), - str(fixture / "certs/rrdp-server.key")], - stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) + server = subprocess.Popen( + [sys.executable, str(support / "serve_repository.py"), str(active), + str(port), str(fixture / "certs/rrdp-server.pem"), + str(fixture / "certs/rrdp-server.key")], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) processes = [] logs = [] try: assert server.stdout.readline().strip() == "ready" - child_args = ["--tal", str(fixture / "tal/custom.tal"), "--ta", str(fixture / "ta/custom-ta.cer"), - "--http-root-cert", str(fixture / "certs/rrdp-ca.pem"), - "--http-timeout-secs", "2", "--log-format", "json"] - def launch(root, options, inputs=None): + def launch(root, *, max_runs=None, interval=2, retain=3, timeout=0, + grace=30, tal=None, auto=False, progress=10): + config = base / f"daemon-{len(logs)}.toml" + daemon_config(config, fixture, root, max_runs=max_runs, + interval=interval, retain=retain, timeout=timeout, + grace=grace, tal=tal, auto=auto, progress=progress) log = (base / f"controller-{len(logs)}.log").open("w") logs.append(log) - proc = subprocess.Popen([binary, "daemon", "--state-root", str(root), *options, - "--", *(child_args if inputs is None else inputs)], - stdout=log, stderr=log) + proc = subprocess.Popen( + [binary, "daemon", "--config", str(config)], + stdout=log, stderr=log) processes.append(proc) return proc root = base / "normal" - proc = launch(root, ["--interval-secs", "2", "--max-runs", "2", "--retain-runs", "2"]) + proc = launch(root, max_runs=2, interval=2, retain=2) wait_for(lambda: state_is(root, "sleeping"), "snapshot completion") first = read(root / "runs/run_000001/run-summary.json") assert first["status"] == "success" and first["sync_mode"] == "snapshot", first assert first["summary"]["vrps"] == 1, first - # Single controller even in the interval when RocksDB itself is closed. - rival = launch(root, ["--max-runs", "1"]) + + rival = launch(root, max_runs=1) assert rival.wait(timeout=10) == 2 active.unlink() active.symlink_to("cases/baseline-v2/http") @@ -80,101 +138,69 @@ def run(binary): assert second["summary"]["vrps"] == 2, second timing = read(root / "runs/run_000002/stage-timing.json") assert timing["analysis_counts"]["rrdp_delta_ops_applied_total"] > 0 - from datetime import datetime - gap = (datetime.fromisoformat(second["started_at"].replace("Z", "+00:00")) - - datetime.fromisoformat(first["finished_at"].replace("Z", "+00:00"))).total_seconds() - assert gap >= 1.9, gap + assert (root / "latest").is_symlink() for seq in (1, 2): out = root / f"runs/run_{seq:06}" - assert not (out / "stderr.log").exists() - assert not (out / "stdout.log").exists() - assert (out / "process-output.txt").read_bytes() == b"" - records = [json.loads(line) for line in (out / "validator.log").read_text().splitlines()] + assert (out / "validator-config.toml").is_file() + assert (out / "result-complete.json").is_file() + records = [json.loads(line) for line in + (out / "validator.log").read_text().splitlines()] assert records[-1]["event"] == "validation_completed", records[-1] - assert all(record["source"]["line"] > 0 and record["source"]["file"].endswith(".rs") for record in records) - assert all(record["source"]["file"] not in ("src/logging.rs", "src/logging/progress.rs") for record in records) assert all(record["fields"]["run_seq"] == seq for record in records) - assert len({record["fields"]["run_id"] for record in records}) == 1 - assert any(record["event"] == "artifact_write_completed" for record in records) assert len((out / "vrps.csv").read_text().splitlines()) - 1 == seq subprocess.run(["openssl", "asn1parse", "-inform", "DER", "-in", str(out / "result.ccr"), "-noout"], check=True) - # Restart appends sequence and keeps protocol state; retention only drops completed runs. + (root / "runs/operator-notes").mkdir() - (root / "runs/run_000003").mkdir() # Simulated interrupted run evidence. - restart = launch(root, ["--interval-secs", "0", "--max-runs", "1", "--retain-runs", "1"]) + (root / "runs/run_000003").mkdir() + restart = launch(root, max_runs=1, interval=0, retain=1) assert restart.wait(timeout=30) == 0 fourth = read(root / "runs/run_000004/run-summary.json") assert fourth["sync_mode"] == "auto" and fourth["summary"]["vrps"] == 2 assert not (root / "runs/run_000001").exists() - assert (root / "runs/operator-notes").is_dir() and (root / "runs/run_000003").is_dir() + assert (root / "runs/operator-notes").is_dir() + assert (root / "runs/run_000003").is_dir() - sleeper = launch(root, ["--interval-secs", "600"]) - wait_for(lambda: state_is(root, "sleeping"), "daemon sleep") - sleeper.send_signal(signal.SIGTERM) - assert sleeper.wait(timeout=5) == 0 - assert state_is(root, "exited") - - # A real validator failure is recorded, retried, and returned as nonzero at max-runs. + # A malformed output destination is rejected during read-only config validation. + bad_config = base / "bad.toml" + validate_config(bad_config, fixture, base / "bad-output", base / "bad-state", + ccr=base) output_failure = subprocess.run( - [binary, "validate", *child_args, "--out", str(base / "bad-output"), "--ccr-out", str(base)], - capture_output=True, text=True, timeout=30, - ) + [binary, "validate", "--config", str(bad_config)], + capture_output=True, text=True, timeout=30) assert output_failure.returncode == 2 and not output_failure.stdout - failure_records = [json.loads(line) for line in output_failure.stderr.splitlines()] - assert any(record["event"] == "artifact_write_failed" and record["fields"]["artifact_type"] == "ccr" for record in failure_records) - assert any(record["event"] == "validation_stage_failed" for record in failure_records) - assert not any(record["event"] == "validation_completed" for record in failure_records) + assert "ccr-out must be a file" in output_failure.stderr + + # A valid TAL with an unreachable automatic TA URI creates failed daemon runs. + bad_tal = base / "unreachable.tal" + tal_text = (fixture / "tal/custom.tal").read_text() + bad_tal.write_text( + tal_text + .replace("rpki-local-rsync:873", "127.0.0.1:1") + .replace(f"https://127.0.0.1:{port}", "https://127.0.0.1:1") + ) failed = base / "failed" - bad_inputs = ["--tal", str(base / "missing.tal"), "--ta", str(base / "missing.cer")] - failure = launch(failed, ["--max-runs", "2", "--interval-secs", "0"], bad_inputs) - assert failure.wait(timeout=20) == 2 + failure = launch(failed, max_runs=2, interval=0, tal=bad_tal, auto=True) + assert failure.wait(timeout=30) == 2 for seq in (1, 2): assert read(failed / f"runs/run_{seq:06}/run-summary.json")["status"] == "failed" - # Suspend the HTTPS server so the actual validator cannot complete a request. + # Whole-run timeout and SIGTERM are exercised while the RRDP server is stopped. server.send_signal(signal.SIGSTOP) - slow_inputs = list(child_args) - slow_inputs[slow_inputs.index("--http-timeout-secs") + 1] = "30" - slow_inputs += ["--log-progress-interval-secs", "1"] - progress_root = base / "progress" - progress = launch(progress_root, ["--max-runs", "1", "--run-timeout-secs", "20"], slow_inputs) - progress_log = progress_root / "runs/run_000001/validator.log" + timeout_root = base / "timeout" + timeout_proc = launch(timeout_root, max_runs=1, timeout=2, progress=0) + assert timeout_proc.wait(timeout=15) == 2 + timeout_run = read(timeout_root / "runs/run_000001/run-summary.json") + assert timeout_run["error"] == "timeout", timeout_run - def has_progress(): - if not progress_log.exists(): - return False - lines = progress_log.read_text().splitlines() - return sum('"event":"validation_progress"' in line for line in lines) >= 2 - - wait_for(has_progress, "periodic progress during blocked network", timeout=15) - server.send_signal(signal.SIGCONT) - assert progress.wait(timeout=30) == 0 - progress_records = [json.loads(line) for line in progress_log.read_text().splitlines()] - assert progress_records[-1]["event"] == "validation_completed" - assert any(record["event"] == "validation_progress" and record["fields"]["repository_attempts_inflight"] > 0 for record in progress_records) - server.send_signal(signal.SIGSTOP) - hung = base / "timeout" - disabled_inputs = list(slow_inputs) - disabled_inputs[-1] = "0" - timeout = launch(hung, ["--max-runs", "1", "--run-timeout-secs", "2"], disabled_inputs) - assert timeout.wait(timeout=10) == 2 - assert read(hung / "runs/run_000001/run-summary.json")["error"] == "timeout" - assert '"event":"validation_progress"' not in (hung / "runs/run_000001/validator.log").read_text() interrupted = base / "interrupted" - active_proc = launch(interrupted, ["--shutdown-grace-secs", "0"]) + active_proc = launch(interrupted, grace=0) wait_for(lambda: state_is(interrupted, "running"), "active validator") - time.sleep(0.2) active_proc.send_signal(signal.SIGTERM) - assert active_proc.wait(timeout=10) == 2 + assert active_proc.wait(timeout=15) == 2 assert read(interrupted / "runs/run_000001/run-summary.json")["error"] == "interrupted" server.send_signal(signal.SIGCONT) - recovered = launch(interrupted, ["--max-runs", "1", "--interval-secs", "0"]) - assert recovered.wait(timeout=30) == 0 - resumed = read(interrupted / "runs/run_000002/run-summary.json") - assert resumed["status"] == "success" and resumed["summary"]["vrps"] == 2 - assert resumed["sync_mode"] in ("auto", "snapshot") - print("Real daemon E2E passed: snapshot/delta/restart/interval/retention/lock/failure/timeout/SIGTERM") + print("Real daemon E2E passed: TOML config/snapshot/delta/restart/retention/lock/timeout/SIGTERM") except BaseException: for log in logs: log.flush() diff --git a/tests/synthetic_docker_e2e.sh b/tests/synthetic_docker_e2e.sh index 6436ec1..057f4ef 100755 --- a/tests/synthetic_docker_e2e.sh +++ b/tests/synthetic_docker_e2e.sh @@ -47,18 +47,32 @@ fi run_validator() { local out="$1" local mode="$2" + local config="$work_root/validate-$mode.toml" + cat >"$config" <"$daemon_config" <"$work_root/daemon-container.log" 2>&1 & daemon_job=$! python3 - "$work_root/daemon" "$fixture" <<'PY'