From 2a0b5067bd2be071cd1d1b30e739bcebf9842f7a Mon Sep 17 00:00:00 2001 From: yuyr Date: Wed, 2 Sep 2026 11:48:36 +0800 Subject: [PATCH] =?UTF-8?q?20260902=20=E5=AE=8C=E6=88=90=E5=85=AC=E5=BC=80?= =?UTF-8?q?=E6=A0=91=E6=B8=85=E7=90=86=E4=B8=8E=E7=83=AD=E7=82=B9=E6=8B=86?= =?UTF-8?q?=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 17 + .gitignore | 1 + .readthedocs.yaml | 14 + CONTRIBUTING.md | 20 +- LICENSE | 28 + README.md | 123 +- SECURITY.md | 19 +- crates/panda-rpki-validator/Cargo.toml | 1 + crates/panda-rpki-validator/src/cir/export.rs | 1202 +--- .../src/cir/export/build_and_write.rs | 224 + .../src/cir/export/models_and_collect.rs | 192 + .../src/cir/export/tests.rs | 780 +++ crates/panda-rpki-validator/src/cli.rs | 2860 +------- .../src/cli/parse_args.rs | 819 +++ .../src/cli/post_validation.rs | 619 ++ crates/panda-rpki-validator/src/cli/report.rs | 218 + .../src/cli/report_tasks.rs | 180 + crates/panda-rpki-validator/src/cli/run.rs | 777 +++ crates/panda-rpki-validator/src/cli/tests.rs | 2222 +----- .../src/cli/tests_parts/parse_core.rs | 977 +++ .../src/cli/tests_parts/parse_options.rs | 683 ++ .../src/cli/tests_parts/report_helpers.rs | 281 + .../src/cli/tests_parts/report_tasks.rs | 281 + crates/panda-rpki-validator/src/cli/types.rs | 135 + crates/panda-rpki-validator/src/cli/usage.rs | 116 + .../src/crypto_sig_cache.rs | 32 +- crates/panda-rpki-validator/src/daemon.rs | 2211 +----- .../panda-rpki-validator/src/daemon/args.rs | 332 + .../src/daemon/daemon_run.rs | 120 + .../src/daemon/run_child.rs | 92 + .../src/daemon/run_metrics.rs | 404 ++ .../panda-rpki-validator/src/daemon/status.rs | 299 + .../panda-rpki-validator/src/daemon/tests.rs | 785 +++ .../panda-rpki-validator/src/daemon/types.rs | 175 + .../panda-rpki-validator/src/data_model/rc.rs | 1848 +---- .../data_model/rc/certificate_validation.rs | 249 + .../src/data_model/rc/parsed_validation.rs | 348 + .../src/data_model/rc/parsing.rs | 586 ++ .../src/data_model/rc/strict_name_tests.rs | 102 + .../src/data_model/rc/types.rs | 564 ++ .../src/data_model/signed_object.rs | 1616 +---- .../data_model/signed_object/cms_reader.rs | 181 + .../signed_object/parsed_profile.rs | 542 ++ .../data_model/signed_object/signed_attrs.rs | 344 + .../signed_object/signed_object_impl.rs | 133 + .../src/data_model/signed_object/tests.rs | 116 + .../data_model/signed_object/types_errors.rs | 300 + crates/panda-rpki-validator/src/fetch/http.rs | 2 +- .../src/fetch/rsync_system.rs | 2 +- crates/panda-rpki-validator/src/lib.rs | 14 +- .../src/parallel/config.rs | 2 +- .../src/parallel/dead_repo_blacklist.rs | 4 +- .../src/parallel/dead_repo_blacklist_tests.rs | 2 +- .../src/parallel/repo_runtime.rs | 1500 +--- .../src/parallel/repo_runtime/outcome.rs | 58 + .../parallel/repo_runtime/phase1_runtime.rs | 372 + .../repo_runtime/runtime_trait_impl.rs | 173 + .../src/parallel/repo_runtime/tests.rs | 798 +++ .../parallel/repo_runtime/types_and_trait.rs | 101 + .../src/parallel/repo_scheduler.rs | 2166 +----- .../src/parallel/repo_scheduler/repo_state.rs | 199 + .../src/parallel/repo_scheduler/tests_repo.rs | 231 + .../repo_scheduler/tests_transport.rs | 864 +++ .../repo_scheduler/transport_state.rs | 871 +++ .../src/parallel/repo_worker.rs | 1248 +--- .../src/parallel/repo_worker/executors.rs | 384 ++ .../src/parallel/repo_worker/pools.rs | 255 + .../src/parallel/repo_worker/tests.rs | 599 ++ .../src/parallel/run_coordinator.rs | 8 +- .../src/replay/delta_archive.rs | 1152 +--- .../src/replay/delta_archive/loaders.rs | 270 + .../src/replay/delta_archive/models.rs | 344 + .../src/replay/delta_archive/tests.rs | 537 ++ crates/panda-rpki-validator/src/storage.rs | 4844 +------------ .../src/storage/batch_helpers.rs | 69 + .../src/storage/memory.rs | 199 + .../src/storage/models_core.rs | 758 ++ .../src/storage/models_publication.rs | 628 ++ .../src/storage/models_summary.rs | 730 ++ .../src/storage/models_vcir.rs | 548 ++ .../src/storage/store_child_cache.rs | 300 + .../src/storage/store_lifecycle.rs | 199 + .../src/storage/store_publication_cache.rs | 509 ++ .../src/storage/store_repository.rs | 363 + .../src/storage/store_transport_rrdp.rs | 332 + .../src/storage/store_vcir.rs | 233 + .../panda-rpki-validator/src/storage/tests.rs | 2466 +------ .../src/storage/tests_parts/child_cache.rs | 357 + .../storage/tests_parts/helpers_and_models.rs | 310 + .../src/storage/tests_parts/object_loading.rs | 266 + .../src/storage/tests_parts/projection.rs | 473 ++ .../src/storage/tests_parts/repository.rs | 418 ++ .../src/storage/tests_parts/rrdp.rs | 258 + .../src/storage/tests_parts/vcir.rs | 386 ++ .../src/storage/verification.rs | 32 + .../src/sync/repo/tests.rs | 1336 +--- .../src/sync/repo/tests_parts/delta_replay.rs | 279 + .../repo/tests_parts/fallback_and_replay.rs | 344 + .../sync/repo/tests_parts/setup_and_sync.rs | 713 ++ crates/panda-rpki-validator/src/sync/rrdp.rs | 1333 +--- .../src/sync/rrdp/delta.rs | 272 + .../src/sync/rrdp/models_and_parsing.rs | 461 ++ .../src/sync/rrdp/notification_sync.rs | 357 + .../src/sync/rrdp/parse_helpers.rs | 87 + .../src/sync/rrdp/snapshot_sync.rs | 157 + .../src/sync/rrdp/tests.rs | 1321 +--- .../src/sync/rrdp/tests_parts/delta_apply.rs | 359 + .../src/sync/rrdp/tests_parts/edge_cases.rs | 214 + .../src/sync/rrdp/tests_parts/parsing.rs | 367 + .../src/sync/rrdp/tests_parts/sync.rs | 381 + .../src/ta_constraints.rs | 1138 +-- .../src/ta_constraints/implementation.rs | 779 +++ .../src/ta_constraints/tests.rs | 357 + .../src/validation/ca_path.rs | 2414 +------ .../validation/ca_path/certificate_checks.rs | 145 + .../src/validation/ca_path/increment.rs | 15 + .../src/validation/ca_path/ip_resources.rs | 350 + .../validation/ca_path/resource_resolution.rs | 445 ++ .../src/validation/ca_path/tests.rs | 962 +++ .../ca_path/types_and_validation.rs | 464 ++ .../src/validation/cert_path.rs | 2 +- .../src/validation/manifest.rs | 1813 +---- .../src/validation/manifest/helpers.rs | 96 + .../validation/manifest/models_and_process.rs | 878 +++ .../src/validation/manifest/tests.rs | 833 +++ .../src/validation/objects.rs | 5682 +-------------- .../src/validation/objects/cache.rs | 871 +++ .../validation/objects/object_validation.rs | 398 ++ .../validation/objects/parallel_processing.rs | 406 ++ .../src/validation/objects/parallel_stage.rs | 805 +++ .../validation/objects/resource_validation.rs | 825 +++ .../validation/objects/serial_processing.rs | 683 ++ .../src/validation/objects/tests/cache.rs | 908 +++ .../src/validation/objects/tests/output.rs | 117 + .../objects/tests/resource_helpers.rs | 654 ++ .../src/validation/run_tree_from_tal.rs | 3005 +------- .../validation/run_tree_from_tal/discovery.rs | 558 ++ .../validation/run_tree_from_tal/phase1.rs | 365 + .../run_tree_from_tal/replay_delta.rs | 119 + .../run_tree_from_tal/replay_setup.rs | 314 + .../validation/run_tree_from_tal/serial.rs | 407 ++ .../run_tree_from_tal/serial_replay.rs | 602 ++ .../src/validation/run_tree_from_tal/tests.rs | 627 ++ .../src/validation/tree_parallel.rs | 3816 +--------- .../src/validation/tree_parallel/dispatch.rs | 468 ++ .../src/validation/tree_parallel/finalize.rs | 520 ++ .../src/validation/tree_parallel/phase2.rs | 445 ++ .../validation/tree_parallel/ready_stage.rs | 849 +++ .../src/validation/tree_parallel/state.rs | 597 ++ .../tree_parallel/tests/backpressure.rs | 76 + .../tree_parallel/tests/control_loop.rs | 502 ++ .../validation/tree_parallel/tests/stage.rs | 346 + .../src/validation/tree_runner.rs | 6139 +---------------- .../tree_runner/audit_projection.rs | 732 ++ .../validation/tree_runner/cache_methods.rs | 538 ++ .../tree_runner/cache_reuse_methods.rs | 481 ++ .../src/validation/tree_runner/cache_types.rs | 339 + .../tree_runner/child_validation.rs | 176 + .../src/validation/tree_runner/discovery.rs | 332 + .../tree_runner/discovery/body_loop.rs | 923 +++ .../tree_runner/discovery/wrappers.rs | 48 + .../src/validation/tree_runner/labels.rs | 114 + .../tree_runner/publication_point_runner.rs | 731 ++ .../src/validation/tree_runner/tests.rs | 5877 +--------------- .../tree_runner/tests/cache_basics.rs | 897 +++ .../tree_runner/tests/cache_behaviour.rs | 718 ++ .../tree_runner/tests/fixture_repro.rs | 444 ++ .../validation/tree_runner/tests/helpers.rs | 803 +++ .../tree_runner/tests/projection.rs | 904 +++ .../tree_runner/tests/projection_tail.rs | 515 ++ .../tree_runner/tests/publication_cache.rs | 887 +++ .../tree_runner/tests/runner_behaviour.rs | 694 ++ .../src/validation/tree_runner/types.rs | 101 + .../validation/tree_runner/vcir_outputs.rs | 933 +++ .../tree_runner/vcir_persistence.rs | 688 ++ .../afrinic-current-ipv4-deny.constraints | 2 +- .../afrinic-full-ipv4-deny.constraints | 2 +- .../local-custom-allow.constraints | 2 +- .../local-custom-deny.constraints | 2 +- .../ripe-ncc-afrinic-deny.constraints | 2 +- deploy/docker/.env.example | 4 +- docker/base-images.toml | 5 +- docker/validator-runtime.Dockerfile | 2 +- docs/Makefile | 18 + docs/architecture.md | 21 - docs/make.bat | 19 + docs/output-abi.md | 61 - docs/requirements.txt | 2 + docs/source/architecture.rst | 25 + docs/source/cli.rst | 22 + docs/source/conf.py | 17 + docs/source/configuration.rst | 32 + docs/source/contributing.rst | 11 + docs/source/development.rst | 17 + docs/source/getting-started.rst | 46 + docs/source/index.rst | 27 + docs/source/operations.rst | 19 + docs/source/output-abi.rst | 19 + docs/source/security.rst | 11 + docs/source/testing.rst | 33 + fixtures/manifest.toml | 3 +- fixtures/minimal/README.md | 18 +- migration/allowlist.toml | 77 - provenance/source-baseline.toml | 17 - scripts/ci/check_public_tree.sh | 30 + scripts/docker/build_image.sh | 8 +- scripts/runtime/run_validator.sh | 5 +- tests/compat/README.md | 63 +- tests/compat/baseline-manifest.toml | 3 +- 209 files changed, 59875 insertions(+), 59423 deletions(-) create mode 100644 .readthedocs.yaml create mode 100644 LICENSE create mode 100644 crates/panda-rpki-validator/src/cir/export/build_and_write.rs create mode 100644 crates/panda-rpki-validator/src/cir/export/models_and_collect.rs create mode 100644 crates/panda-rpki-validator/src/cir/export/tests.rs create mode 100644 crates/panda-rpki-validator/src/cli/parse_args.rs create mode 100644 crates/panda-rpki-validator/src/cli/post_validation.rs create mode 100644 crates/panda-rpki-validator/src/cli/report.rs create mode 100644 crates/panda-rpki-validator/src/cli/report_tasks.rs create mode 100644 crates/panda-rpki-validator/src/cli/run.rs create mode 100644 crates/panda-rpki-validator/src/cli/tests_parts/parse_core.rs create mode 100644 crates/panda-rpki-validator/src/cli/tests_parts/parse_options.rs create mode 100644 crates/panda-rpki-validator/src/cli/tests_parts/report_helpers.rs create mode 100644 crates/panda-rpki-validator/src/cli/tests_parts/report_tasks.rs create mode 100644 crates/panda-rpki-validator/src/cli/types.rs create mode 100644 crates/panda-rpki-validator/src/cli/usage.rs create mode 100644 crates/panda-rpki-validator/src/daemon/args.rs create mode 100644 crates/panda-rpki-validator/src/daemon/daemon_run.rs create mode 100644 crates/panda-rpki-validator/src/daemon/run_child.rs create mode 100644 crates/panda-rpki-validator/src/daemon/run_metrics.rs create mode 100644 crates/panda-rpki-validator/src/daemon/status.rs create mode 100644 crates/panda-rpki-validator/src/daemon/tests.rs create mode 100644 crates/panda-rpki-validator/src/daemon/types.rs create mode 100644 crates/panda-rpki-validator/src/data_model/rc/certificate_validation.rs create mode 100644 crates/panda-rpki-validator/src/data_model/rc/parsed_validation.rs create mode 100644 crates/panda-rpki-validator/src/data_model/rc/parsing.rs create mode 100644 crates/panda-rpki-validator/src/data_model/rc/strict_name_tests.rs create mode 100644 crates/panda-rpki-validator/src/data_model/rc/types.rs create mode 100644 crates/panda-rpki-validator/src/data_model/signed_object/cms_reader.rs create mode 100644 crates/panda-rpki-validator/src/data_model/signed_object/parsed_profile.rs create mode 100644 crates/panda-rpki-validator/src/data_model/signed_object/signed_attrs.rs create mode 100644 crates/panda-rpki-validator/src/data_model/signed_object/signed_object_impl.rs create mode 100644 crates/panda-rpki-validator/src/data_model/signed_object/tests.rs create mode 100644 crates/panda-rpki-validator/src/data_model/signed_object/types_errors.rs create mode 100644 crates/panda-rpki-validator/src/parallel/repo_runtime/outcome.rs create mode 100644 crates/panda-rpki-validator/src/parallel/repo_runtime/phase1_runtime.rs create mode 100644 crates/panda-rpki-validator/src/parallel/repo_runtime/runtime_trait_impl.rs create mode 100644 crates/panda-rpki-validator/src/parallel/repo_runtime/tests.rs create mode 100644 crates/panda-rpki-validator/src/parallel/repo_runtime/types_and_trait.rs create mode 100644 crates/panda-rpki-validator/src/parallel/repo_scheduler/repo_state.rs create mode 100644 crates/panda-rpki-validator/src/parallel/repo_scheduler/tests_repo.rs create mode 100644 crates/panda-rpki-validator/src/parallel/repo_scheduler/tests_transport.rs create mode 100644 crates/panda-rpki-validator/src/parallel/repo_scheduler/transport_state.rs create mode 100644 crates/panda-rpki-validator/src/parallel/repo_worker/executors.rs create mode 100644 crates/panda-rpki-validator/src/parallel/repo_worker/pools.rs create mode 100644 crates/panda-rpki-validator/src/parallel/repo_worker/tests.rs create mode 100644 crates/panda-rpki-validator/src/replay/delta_archive/loaders.rs create mode 100644 crates/panda-rpki-validator/src/replay/delta_archive/models.rs create mode 100644 crates/panda-rpki-validator/src/replay/delta_archive/tests.rs create mode 100644 crates/panda-rpki-validator/src/storage/batch_helpers.rs create mode 100644 crates/panda-rpki-validator/src/storage/memory.rs create mode 100644 crates/panda-rpki-validator/src/storage/models_core.rs create mode 100644 crates/panda-rpki-validator/src/storage/models_publication.rs create mode 100644 crates/panda-rpki-validator/src/storage/models_summary.rs create mode 100644 crates/panda-rpki-validator/src/storage/models_vcir.rs create mode 100644 crates/panda-rpki-validator/src/storage/store_child_cache.rs create mode 100644 crates/panda-rpki-validator/src/storage/store_lifecycle.rs create mode 100644 crates/panda-rpki-validator/src/storage/store_publication_cache.rs create mode 100644 crates/panda-rpki-validator/src/storage/store_repository.rs create mode 100644 crates/panda-rpki-validator/src/storage/store_transport_rrdp.rs create mode 100644 crates/panda-rpki-validator/src/storage/store_vcir.rs create mode 100644 crates/panda-rpki-validator/src/storage/tests_parts/child_cache.rs create mode 100644 crates/panda-rpki-validator/src/storage/tests_parts/helpers_and_models.rs create mode 100644 crates/panda-rpki-validator/src/storage/tests_parts/object_loading.rs create mode 100644 crates/panda-rpki-validator/src/storage/tests_parts/projection.rs create mode 100644 crates/panda-rpki-validator/src/storage/tests_parts/repository.rs create mode 100644 crates/panda-rpki-validator/src/storage/tests_parts/rrdp.rs create mode 100644 crates/panda-rpki-validator/src/storage/tests_parts/vcir.rs create mode 100644 crates/panda-rpki-validator/src/storage/verification.rs create mode 100644 crates/panda-rpki-validator/src/sync/repo/tests_parts/delta_replay.rs create mode 100644 crates/panda-rpki-validator/src/sync/repo/tests_parts/fallback_and_replay.rs create mode 100644 crates/panda-rpki-validator/src/sync/repo/tests_parts/setup_and_sync.rs create mode 100644 crates/panda-rpki-validator/src/sync/rrdp/delta.rs create mode 100644 crates/panda-rpki-validator/src/sync/rrdp/models_and_parsing.rs create mode 100644 crates/panda-rpki-validator/src/sync/rrdp/notification_sync.rs create mode 100644 crates/panda-rpki-validator/src/sync/rrdp/parse_helpers.rs create mode 100644 crates/panda-rpki-validator/src/sync/rrdp/snapshot_sync.rs create mode 100644 crates/panda-rpki-validator/src/sync/rrdp/tests_parts/delta_apply.rs create mode 100644 crates/panda-rpki-validator/src/sync/rrdp/tests_parts/edge_cases.rs create mode 100644 crates/panda-rpki-validator/src/sync/rrdp/tests_parts/parsing.rs create mode 100644 crates/panda-rpki-validator/src/sync/rrdp/tests_parts/sync.rs create mode 100644 crates/panda-rpki-validator/src/ta_constraints/implementation.rs create mode 100644 crates/panda-rpki-validator/src/ta_constraints/tests.rs create mode 100644 crates/panda-rpki-validator/src/validation/ca_path/certificate_checks.rs create mode 100644 crates/panda-rpki-validator/src/validation/ca_path/increment.rs create mode 100644 crates/panda-rpki-validator/src/validation/ca_path/ip_resources.rs create mode 100644 crates/panda-rpki-validator/src/validation/ca_path/resource_resolution.rs create mode 100644 crates/panda-rpki-validator/src/validation/ca_path/tests.rs create mode 100644 crates/panda-rpki-validator/src/validation/ca_path/types_and_validation.rs create mode 100644 crates/panda-rpki-validator/src/validation/manifest/helpers.rs create mode 100644 crates/panda-rpki-validator/src/validation/manifest/models_and_process.rs create mode 100644 crates/panda-rpki-validator/src/validation/manifest/tests.rs create mode 100644 crates/panda-rpki-validator/src/validation/objects/cache.rs create mode 100644 crates/panda-rpki-validator/src/validation/objects/object_validation.rs create mode 100644 crates/panda-rpki-validator/src/validation/objects/parallel_processing.rs create mode 100644 crates/panda-rpki-validator/src/validation/objects/parallel_stage.rs create mode 100644 crates/panda-rpki-validator/src/validation/objects/resource_validation.rs create mode 100644 crates/panda-rpki-validator/src/validation/objects/serial_processing.rs create mode 100644 crates/panda-rpki-validator/src/validation/objects/tests/cache.rs create mode 100644 crates/panda-rpki-validator/src/validation/objects/tests/output.rs create mode 100644 crates/panda-rpki-validator/src/validation/objects/tests/resource_helpers.rs create mode 100644 crates/panda-rpki-validator/src/validation/run_tree_from_tal/discovery.rs create mode 100644 crates/panda-rpki-validator/src/validation/run_tree_from_tal/phase1.rs create mode 100644 crates/panda-rpki-validator/src/validation/run_tree_from_tal/replay_delta.rs create mode 100644 crates/panda-rpki-validator/src/validation/run_tree_from_tal/replay_setup.rs create mode 100644 crates/panda-rpki-validator/src/validation/run_tree_from_tal/serial.rs create mode 100644 crates/panda-rpki-validator/src/validation/run_tree_from_tal/serial_replay.rs create mode 100644 crates/panda-rpki-validator/src/validation/run_tree_from_tal/tests.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_parallel/dispatch.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_parallel/finalize.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_parallel/phase2.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_parallel/ready_stage.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_parallel/state.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_parallel/tests/backpressure.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_parallel/tests/control_loop.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_parallel/tests/stage.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_runner/audit_projection.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_runner/cache_methods.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_runner/cache_reuse_methods.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_runner/cache_types.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_runner/child_validation.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_runner/discovery.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_runner/discovery/body_loop.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_runner/discovery/wrappers.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_runner/labels.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_runner/publication_point_runner.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_runner/tests/cache_basics.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_runner/tests/cache_behaviour.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_runner/tests/fixture_repro.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_runner/tests/helpers.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_runner/tests/projection.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_runner/tests/projection_tail.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_runner/tests/publication_cache.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_runner/tests/runner_behaviour.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_runner/types.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_runner/vcir_outputs.rs create mode 100644 crates/panda-rpki-validator/src/validation/tree_runner/vcir_persistence.rs create mode 100644 docs/Makefile delete mode 100644 docs/architecture.md create mode 100644 docs/make.bat delete mode 100644 docs/output-abi.md create mode 100644 docs/requirements.txt create mode 100644 docs/source/architecture.rst create mode 100644 docs/source/cli.rst create mode 100644 docs/source/conf.py create mode 100644 docs/source/configuration.rst create mode 100644 docs/source/contributing.rst create mode 100644 docs/source/development.rst create mode 100644 docs/source/getting-started.rst create mode 100644 docs/source/index.rst create mode 100644 docs/source/operations.rst create mode 100644 docs/source/output-abi.rst create mode 100644 docs/source/security.rst create mode 100644 docs/source/testing.rst delete mode 100644 migration/allowlist.toml delete mode 100644 provenance/source-baseline.toml create mode 100644 scripts/ci/check_public_tree.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a71364a..e4926aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,23 @@ on: pull_request: jobs: + public-tree: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: bash scripts/ci/check_public_tree.sh + + docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python -m pip install -r docs/requirements.txt + - run: sphinx-build -W --keep-going -b html docs/source docs/_build/html + - run: sphinx-build -W --keep-going -b linkcheck docs/source docs/_build/linkcheck + rust: runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index bb7541e..1c962d7 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ /logs/ /tmp/ /docker-out/ +/docs/_build/ /provenance/generated/ .env .env.* diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..1a2ba75 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,14 @@ +version: 2 + +build: + os: ubuntu-24.04 + tools: + python: "3.12" + +sphinx: + configuration: docs/source/conf.py + fail_on_warning: true + +python: + install: + - requirements: docs/requirements.txt diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e4e9298..11affd3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,10 +1,16 @@ # Contributing to panda-rpki -This repository is private staging for the `panda-rpki-validator` extraction. -External contributions are not enabled while ownership, license and contribution -governance are pending. +panda-rpki is preparing its contributor process. The source is licensed under +the BSD 3-Clause License in [LICENSE](LICENSE); maintainers, code-of-conduct +policy, and public security-reporting channel must be published before +external contributions are accepted. -During M3–M6, changes must preserve `docs/output-abi.md`, update the relevant -allowlist/provenance entry, and pass the locked Rust build and Docker smoke -checks. Do not add live RIR data, private state, credentials, internal hosts, -or copied history. +For local development, follow the workflow in docs/source/development.rst: +format Rust code, run the locked test suite and Clippy, build the documentation, +and add focused tests for every behaviour change. Do not submit credentials, +private state, live customer data, or material whose redistribution terms are +unknown. + +The output contract is part of the runtime interface. Changes to run artifacts, +container entrypoints, persistent-volume layout, or CSV/CIR/CCR semantics +require compatibility tests and a documented migration decision. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..53aff33 --- /dev/null +++ b/LICENSE @@ -0,0 +1,28 @@ +BSD 3-Clause License + +Copyright (c) 2026, panda-rpki contributors + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md index abf8f87..1cc4abd 100644 --- a/README.md +++ b/README.md @@ -1,100 +1,57 @@ # panda-rpki -`panda-rpki` is the staging workspace for the independently distributable RPKI -synchronization and validation component. The first component is -`panda-rpki-validator` (Cargo package and CLI); its Rust import path is -`panda_rpki_validator`. +panda-rpki is an RPKI synchronization and validation runtime written in Rust. +The first component, panda-rpki-validator, fetches RPKI repositories through +RRDP or rsync, validates the resulting object graph, and writes a structured +run directory containing reports, payload CSV files, and canonical artifacts. -This directory is currently a **private staging repository**. It has no copied -history from the existing private `rpki` repository. The normal synchronization -and validation path has now been extracted, together with its daemon and -Docker lifecycle wrapper. The first M5 canonical snapshot/delta baseline and -amd64/arm64 staging image checks are complete; remaining profile/performance -gates still block any public release. Do not use this staging build as a -production validator. +The project is currently a pre-release. Its command-line interface, output +directory contract, container runtime, and supported deployment practices are +documented in the Sphinx source tree at docs/source. The source is provided +under the BSD 3-Clause License; public support and release policies will be +published before a production release. -## Current milestone +## Quick start -M4/M5 contain the first functional extraction from source commit -`74cbebbd3334ac0063761c1a97a88ee000cc2a57`: normal RRDP/rsync synchronization, -RPKI validation, RocksDB state, CIR/CCR/report/CSV outputs, the run daemon, and -the snapshot/delta lifecycle wrapper. The `verification-only` mode is -intentionally deferred. The project license is intentionally **TBD**; no -external contributions or public release are accepted until ownership and -licensing are approved. +Build and test the validator from a checkout: -## Build and test the staging component + cargo build --locked -p panda-rpki-validator + cargo test --locked -p panda-rpki-validator + cargo fmt --all --check + cargo clippy --locked -p panda-rpki-validator --all-targets -- -D warnings -```bash -cargo build --locked -p panda-rpki-validator -cargo test --locked -p panda-rpki-validator -cargo fmt --all --check -cargo clippy --locked -p panda-rpki-validator --all-targets -- -D warnings -./scripts/docker/build_image.sh --arch amd64 --allow-dirty --no-save -``` +Build a local amd64 runtime image: -The extracted test suite currently passes `735` tests with `1` ignored test in -the normal profile. The APNIC offline snapshot/delta profile has also passed -the M5 canonical comparator and a five-run pinned release smoke baseline; the -full cache/fallback/failure/profile matrix is still pending. + ./scripts/docker/build_image.sh --arch amd64 --allow-dirty + ./scripts/docker/verify_image.sh \ + --image panda-rpki-validator:0.1.0-dirty-amd64 -For a deterministic single-RIR run against the checked-in test repository, -build the binaries first and invoke the lifecycle wrapper with -`RPKI_EXTRA_ARGS=--disable-rrdp --rsync-local-dir ...`; the wrapper writes the -normal run ABI under `RUN_ROOT/runs/run_0001/`. +The checked-in TAL and trust-anchor files are bootstrap inputs, not a bundled +offline repository snapshot. A deterministic offline run requires a reviewed +replay archive supplied by the operator. The runtime wrapper writes normal run +artifacts below RUN_ROOT/runs/run_0001. -## Docker smoke +## Documentation -```bash -./scripts/docker/build_image.sh --arch amd64 --allow-dirty -./scripts/docker/verify_image.sh \ - --image panda-rpki-validator:0.1.0-dirty-amd64 -``` +The documentation website is built with Sphinx and the Read the Docs Sphinx +Theme. Before the hosted site is configured, build it locally: -The runtime image contains both `panda-rpki-validator` and -`panda-rpki-validator-daemon`, the complete normal-run wrapper, and the -redistributable TAL/TA fixtures. It uses one persistent data root mounted at -`/var/lib/panda-rpki-validator`; `run`, `run-validator`, and `daemon` are -entrypoint subcommands. The wrapper preserves the output contract documented -in -[`docs/output-abi.md`](docs/output-abi.md). + python -m pip install -r docs/requirements.txt + sphinx-build -W --keep-going -b html docs/source docs/_build/html -For native or container A/B output checks, run the ABI verifier and canonical -comparator against two retained run directories: +Start with docs/source/getting-started.rst. It links to configuration, CLI, +operations, testing, development, architecture, and output-contract material. -```bash -tests/compat/verify_run_abi.sh /path/to/run_0001 -tests/compat/compare_runs.py \ - /path/to/original/runs/run_0001 \ - /path/to/panda/runs/run_0001 -``` +This project is licensed under the [BSD 3-Clause License](LICENSE). -The comparator has matched the original runtime for the APNIC offline -snapshot/delta baseline. Five serial release samples on that small single-RIR -profile stay within the current wall-time gate under a pinned CPU. A live APNIC -run also produced an identical decoded CCR state (MFT/VRP/VAP/TA/RK); only the -time-bearing `producedAt` byte differed in the raw DER. A concurrent all-RIR -(`all5`) run is retained as network/fallback evidence, but its source was not an -atomic snapshot (the original image timed out on the ARIN RRDP notification), -so its different CCR state is not a code-parity verdict. See the detailed -[`M5 live RIR comparison report`](../specs/develop/20260901/m5_live_rir_image_artifact_comparison_milestone_report.md). -The follow-up remote-231 serial all5 run (old image first, then this image; -one snapshot plus three warm deltas per image with cache/prefetch/parallel -flags) also completed 4/4 runs on each side and retained full artifacts. Its -live CCR/CIR and timing differences are documented separately and are not a -frozen-input parity result: [`remote-231 all5 cache/prefetch report`](../specs/develop/20260901_2/m5_live_all5_cache_prefetch_serial_4run_milestone_report.md). -The new image also completed a separate remote-231 all5 long sequence of one -snapshot plus ten deltas; the timing and cache counters are retained as live -health evidence, not as a frozen-input parity gate: [`one snapshot + ten delta -timing report`](../specs/develop/20260901_2/m5_new_image_all5_snapshot_10delta_timing_report.md). -Cache, RRDP/fallback, constraints, replay and failure-path profiles remain in -M5. +## Safety and support -## Repository status +RPKI inputs are network- and attacker-controlled data. Run the validator with +a dedicated writable data directory and follow the resource and retention +guidance in the operations documentation. Do not place credentials, production +state, or private trust-anchor material in this repository or a container +image. -- `verification-only`: deferred to a separate backlog item. -- License and copyright owner: TBD; do not add a speculative `LICENSE` file. -- Staging Git remote: `https://git.nasp.fit/yuyr/panda-rpki.git` (private - staging target selected); public visibility, registry prefix, signing identity, - and release tags remain unselected until release governance is complete. -- Source provenance baseline: `provenance/source-baseline.toml`. +The public security-reporting and support channels are not yet assigned. +Until they are published, do not disclose a suspected vulnerability in a public +issue or commit. diff --git a/SECURITY.md b/SECURITY.md index b54d660..9b741aa 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,12 +1,13 @@ # Security policy -This project is not public yet and does not have a public vulnerability intake -address. Do not publish vulnerability details in an issue or commit. Report -urgent findings to the project owner through the private project channel and -include the affected commit, image tag/digest, reproduction steps, and whether -state or output data is exposed. +panda-rpki processes untrusted network data and is not yet a supported +production release. A public security-reporting channel, response targets, and +supported release branches have not been assigned. -The public contact, response SLA, supported release branches, and disclosure -process will be added after ownership and license approval. Container images -must never contain credentials, production state, private registry settings, -or internal hostnames. +Do not disclose suspected vulnerabilities in a public issue, pull request, or +commit until a public reporting address is published. Release artifacts and +container images must never include credentials, production state, private +registry configuration, or internal host names. + +The first supported release will publish a security contact, disclosure +process, update policy, and supported-version table in the documentation site. diff --git a/crates/panda-rpki-validator/Cargo.toml b/crates/panda-rpki-validator/Cargo.toml index 1733d42..43ed115 100644 --- a/crates/panda-rpki-validator/Cargo.toml +++ b/crates/panda-rpki-validator/Cargo.toml @@ -4,6 +4,7 @@ version = "0.1.0" edition = "2024" rust-version = "1.92" description = "Panda RPKI synchronization and validation runtime" +license = "BSD-3-Clause" publish = false [lib] diff --git a/crates/panda-rpki-validator/src/cir/export.rs b/crates/panda-rpki-validator/src/cir/export.rs index e5b6d6c..b47b0be 100644 --- a/crates/panda-rpki-validator/src/cir/export.rs +++ b/crates/panda-rpki-validator/src/cir/export.rs @@ -15,1203 +15,9 @@ use crate::current_repo_index::CurrentRepoObject; use crate::data_model::ta::TrustAnchor; use crate::storage::RocksStore; -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct CirExportTiming { - pub build_cir_ms: u64, - pub write_cir_ms: u64, - pub total_ms: u64, -} - -#[derive(Debug, thiserror::Error)] -pub enum CirExportError { - #[error("CIR TAL URI must be http(s), got: {0}")] - InvalidTalUri(String), - - #[error("TAL does not contain any rsync TA URI; CIR replay scheme A requires one")] - MissingTaRsyncUri, - - #[error("CIR model validation failed: {0}")] - Validate(String), - - #[error("CIR consumed audit has conflicting hashes for {rsync_uri}: {first} vs {second}")] - ConflictingObjectHash { - rsync_uri: String, - first: String, - second: String, - }, - - #[error("encode CIR failed: {0}")] - Encode(#[from] CirEncodeError), - - #[error("static pool export failed: {0}")] - StaticPool(#[from] CirStaticPoolError), - - #[error("write CIR file failed: {0}: {1}")] - Write(String, String), -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct CirRawStoreExportSummary { - pub unique_hashes: usize, - pub written_entries: usize, - pub reused_entries: usize, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct CirExportSummary { - pub object_count: usize, - pub trust_anchor_count: usize, - pub timing: CirExportTiming, -} - -#[derive(Clone, Copy, Debug)] -pub struct CirTrustAnchorBinding<'a> { - pub trust_anchor: &'a TrustAnchor, - pub tal_uri: &'a str, -} - -fn is_sha256_hex(value: &str) -> bool { - value.len() == 64 && value.as_bytes().iter().all(u8::is_ascii_hexdigit) -} - -fn insert_consumed_object_hash( - objects: &mut BTreeMap, - rsync_uri: &str, - sha256_hex: &str, -) -> Result<(), CirExportError> { - if !rsync_uri.starts_with("rsync://") || !is_sha256_hex(sha256_hex) { - return Ok(()); - } - - let normalized = sha256_hex.to_ascii_lowercase(); - if let Some(existing) = objects.get(rsync_uri) { - if existing != &normalized { - return Err(CirExportError::ConflictingObjectHash { - rsync_uri: rsync_uri.to_string(), - first: existing.clone(), - second: normalized, - }); - } - return Ok(()); - } - - objects.insert(rsync_uri.to_string(), normalized); - Ok(()) -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum CirObjectSection { - Fresh, - Cached, -} - -fn publication_point_is_cached_fallback(pp: &PublicationPointAudit) -> bool { - pp.source == "vcir_current_instance" || pp.repo_terminal_state == "fallback_current_instance" -} - -fn publication_point_cir_entries<'a>( - pp: &'a PublicationPointAudit, - section: CirObjectSection, -) -> &'a [ObjectAuditEntry] { - match (section, publication_point_is_cached_fallback(pp)) { - (CirObjectSection::Fresh, false) => &pp.objects, - (CirObjectSection::Cached, true) => &pp.objects, - _ => &[], - } -} - -fn collect_cir_objects_from_validation_audit( - publication_points: &[PublicationPointAudit], - section: CirObjectSection, -) -> Result, CirExportError> { - let mut objects = BTreeMap::new(); - for pp in publication_points { - for obj in publication_point_cir_entries(pp, section) { - if !matches!(obj.result, AuditObjectResult::Ok | AuditObjectResult::Error) { - continue; - } - insert_consumed_object_hash(&mut objects, &obj.rsync_uri, &obj.sha256_hex)?; - } - } - Ok(objects) -} - -fn collect_rejected_objects_from_validation_audit( - publication_points: &[PublicationPointAudit], - section: CirObjectSection, -) -> Vec { - let mut rejected_objects = publication_points - .iter() - .flat_map(|pp| publication_point_cir_entries(pp, section).iter()) - .filter(|item| item.result == AuditObjectResult::Error) - .filter(|item| item.rsync_uri.starts_with("rsync://")) - .map(|item| CirRejectedObject { - object_uri: item.rsync_uri.clone(), - reason: item.detail.clone(), - }) - .collect::>(); - rejected_objects.sort_by(|a, b| a.object_uri.cmp(&b.object_uri)); - rejected_objects.dedup_by(|a, b| a.object_uri == b.object_uri); - rejected_objects -} - -fn cir_objects_from_hash_map(objects: BTreeMap) -> Vec { - objects - .into_iter() - .map(|(rsync_uri, sha256_hex)| CirObject { - rsync_uri, - sha256: hex::decode(sha256_hex).expect("validated hex"), - }) - .collect() -} - -fn canonical_ta_rsync_uri(trust_anchor: &TrustAnchor) -> Result { - if let Some(uri) = &trust_anchor.resolved_ta_uri - && uri.scheme() == "rsync" - { - return Ok(uri.as_str().to_string()); - } - trust_anchor - .tal - .ta_uris - .iter() - .filter(|uri| uri.scheme() == "rsync") - .map(|uri| uri.as_str().to_string()) - .min() - .ok_or(CirExportError::MissingTaRsyncUri) -} - -fn build_cir_trust_anchors( - tal_bindings: &[CirTrustAnchorBinding<'_>], -) -> Result, CirExportError> { - for binding in tal_bindings { - if !(binding.tal_uri.starts_with("https://") || binding.tal_uri.starts_with("http://")) { - return Err(CirExportError::InvalidTalUri(binding.tal_uri.to_string())); - } - } - - let mut trust_anchors = Vec::with_capacity(tal_bindings.len()); - for binding in tal_bindings { - let ta_rsync_uri = canonical_ta_rsync_uri(binding.trust_anchor)?; - let ta_certificate_der = binding.trust_anchor.ta_certificate.raw_der.clone(); - trust_anchors.push(CirTrustAnchor { - ta_rsync_uri, - tal_uri: binding.tal_uri.to_string(), - tal_bytes: binding.trust_anchor.tal.raw.clone(), - ta_certificate_sha256: crate::cir::model::sha256(&ta_certificate_der), - ta_certificate_der, - }); - } - trust_anchors.sort_by(|a, b| a.ta_rsync_uri.cmp(&b.ta_rsync_uri)); - Ok(trust_anchors) -} - -pub fn build_cir_from_run( - store: &RocksStore, - trust_anchor: &TrustAnchor, - tal_uri: &str, - validation_time: time::OffsetDateTime, - publication_points: &[PublicationPointAudit], -) -> Result { - build_cir_from_run_multi( - store, - &[CirTrustAnchorBinding { - trust_anchor, - tal_uri, - }], - validation_time, - publication_points, - None, - ) -} - -pub fn build_cir_from_run_multi( - _store: &RocksStore, - tal_bindings: &[CirTrustAnchorBinding<'_>], - validation_time: time::OffsetDateTime, - publication_points: &[PublicationPointAudit], - _current_repo_objects: Option<&[CurrentRepoObject]>, -) -> Result { - let fresh_objects = - collect_cir_objects_from_validation_audit(publication_points, CirObjectSection::Fresh)?; - let cached_objects = - collect_cir_objects_from_validation_audit(publication_points, CirObjectSection::Cached)?; - - let trust_anchors = build_cir_trust_anchors(tal_bindings)?; - - let fresh_rejected_objects = - collect_rejected_objects_from_validation_audit(publication_points, CirObjectSection::Fresh); - let cached_rejected_objects = collect_rejected_objects_from_validation_audit( - publication_points, - CirObjectSection::Cached, - ); - - let cir = CanonicalInputRepresentation::new_v4( - validation_time, - cir_objects_from_hash_map(fresh_objects), - cir_objects_from_hash_map(cached_objects), - trust_anchors, - fresh_rejected_objects, - cached_rejected_objects, - ); - cir.validate().map_err(CirExportError::Validate)?; - Ok(cir) -} - -pub fn build_cir_from_input_snapshot_multi( - tal_bindings: &[CirTrustAnchorBinding<'_>], - validation_time: time::OffsetDateTime, - input: CirInputSnapshot, -) -> Result { - let trust_anchors = build_cir_trust_anchors(tal_bindings)?; - Ok(CanonicalInputRepresentation::new_v4( - validation_time, - input.fresh_validated_objects, - input.cached_validated_objects, - trust_anchors, - input.fresh_rejected_objects, - input.cached_rejected_objects, - )) -} - -pub fn write_cir_file( - path: &Path, - cir: &CanonicalInputRepresentation, -) -> Result<(), CirExportError> { - let der = encode_cir(cir)?; - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent) - .map_err(|e| CirExportError::Write(path.display().to_string(), e.to_string()))?; - } - std::fs::write(path, der) - .map_err(|e| CirExportError::Write(path.display().to_string(), e.to_string())) -} - -pub fn export_cir_static_pool( - store: &RocksStore, - static_root: &Path, - capture_date_utc: time::Date, - cir: &CanonicalInputRepresentation, - trust_anchors: &[&TrustAnchor], -) -> Result { - let _ = trust_anchors; - let hashes = cir - .validated_objects() - .map(|item| hex::encode(&item.sha256)) - .collect::>(); - export_hashes_from_store(store, static_root, capture_date_utc, &hashes).map_err(Into::into) -} - -pub fn export_cir_raw_store( - store: &RocksStore, - raw_store_path: &Path, - cir: &CanonicalInputRepresentation, - trust_anchors: &[&TrustAnchor], -) -> Result { - let _ = trust_anchors; - let unique: BTreeSet = cir - .validated_objects() - .map(|item| hex::encode(&item.sha256)) - .collect(); - - let written_entries = 0usize; - let mut reused_entries = 0usize; - for sha256_hex in &unique { - if store - .get_blob_bytes(sha256_hex) - .map_err(|e| { - CirExportError::Write(raw_store_path.display().to_string(), e.to_string()) - })? - .is_some() - { - reused_entries += 1; - continue; - } - return Err(CirExportError::Write( - raw_store_path.display().to_string(), - format!("raw store missing object for sha256={sha256_hex}"), - )); - } - - Ok(CirRawStoreExportSummary { - unique_hashes: unique.len(), - written_entries, - reused_entries, - }) -} - -pub fn export_cir_from_run( - store: &RocksStore, - trust_anchor: &TrustAnchor, - tal_uri: &str, - validation_time: time::OffsetDateTime, - publication_points: &[PublicationPointAudit], - cir_out: &Path, - capture_date_utc: time::Date, -) -> Result { - export_cir_from_run_multi( - store, - &[CirTrustAnchorBinding { - trust_anchor, - tal_uri, - }], - validation_time, - publication_points, - cir_out, - capture_date_utc, - None, - ) -} - -pub fn export_cir_from_run_multi( - store: &RocksStore, - tal_bindings: &[CirTrustAnchorBinding<'_>], - validation_time: time::OffsetDateTime, - publication_points: &[PublicationPointAudit], - cir_out: &Path, - capture_date_utc: time::Date, - current_repo_objects: Option<&[CurrentRepoObject]>, -) -> Result { - let _ = capture_date_utc; - let total_started = std::time::Instant::now(); - - let started = std::time::Instant::now(); - let cir = build_cir_from_run_multi( - store, - tal_bindings, - validation_time, - publication_points, - current_repo_objects, - )?; - let build_cir_ms = started.elapsed().as_millis() as u64; - - let _ = store; - - let started = std::time::Instant::now(); - write_cir_file(cir_out, &cir)?; - let write_cir_ms = started.elapsed().as_millis() as u64; - - Ok(CirExportSummary { - object_count: cir.validated_object_count(), - trust_anchor_count: cir.trust_anchors.len(), - timing: CirExportTiming { - build_cir_ms, - write_cir_ms, - total_ms: total_started.elapsed().as_millis() as u64, - }, - }) -} - -pub fn export_cir_from_input_snapshot_multi( - tal_bindings: &[CirTrustAnchorBinding<'_>], - validation_time: time::OffsetDateTime, - input: CirInputSnapshot, - cir_out: &Path, -) -> Result { - let total_started = std::time::Instant::now(); - - let started = std::time::Instant::now(); - let cir = build_cir_from_input_snapshot_multi(tal_bindings, validation_time, input)?; - let build_cir_ms = started.elapsed().as_millis() as u64; - - let started = std::time::Instant::now(); - write_cir_file(cir_out, &cir)?; - let write_cir_ms = started.elapsed().as_millis() as u64; - - Ok(CirExportSummary { - object_count: cir.validated_object_count(), - trust_anchor_count: cir.trust_anchors.len(), - timing: CirExportTiming { - build_cir_ms, - write_cir_ms, - total_ms: total_started.elapsed().as_millis() as u64, - }, - }) -} +include!("export/models_and_collect.rs"); +include!("export/build_and_write.rs"); #[cfg(test)] -mod tests { - use super::*; - use crate::cir::decode::decode_cir; - use crate::data_model::ta::TrustAnchor; - use crate::data_model::tal::Tal; - use crate::storage::{RawByHashEntry, RocksStore}; - - fn sample_time() -> time::OffsetDateTime { - time::OffsetDateTime::parse( - "2026-04-07T12:34:56Z", - &time::format_description::well_known::Rfc3339, - ) - .unwrap() - } - - fn sample_date() -> time::Date { - time::Date::from_calendar_date(2026, time::Month::April, 7).unwrap() - } - - fn sample_trust_anchor() -> TrustAnchor { - let base = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); - let tal_bytes = - std::fs::read(base.join("tests/fixtures/tal/apnic-rfc7730-https.tal")).unwrap(); - let ta_der = std::fs::read(base.join("tests/fixtures/ta/apnic-ta.cer")).unwrap(); - let tal = Tal::decode_bytes(&tal_bytes).unwrap(); - TrustAnchor::bind_der(tal, &ta_der, None).unwrap() - } - - fn sample_arin_trust_anchor() -> TrustAnchor { - let base = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); - let tal_bytes = std::fs::read(base.join("tests/fixtures/tal/arin.tal")).unwrap(); - let ta_der = std::fs::read(base.join("tests/fixtures/ta/arin-ta.cer")).unwrap(); - let tal = Tal::decode_bytes(&tal_bytes).unwrap(); - TrustAnchor::bind_der(tal, &ta_der, None).unwrap() - } - - fn sample_trust_anchor_without_rsync_uri() -> TrustAnchor { - let mut ta = sample_trust_anchor(); - ta.tal.ta_uris.retain(|uri| uri.scheme() != "rsync"); - ta - } - - fn sha256_hex(bytes: &[u8]) -> String { - use sha2::{Digest, Sha256}; - hex::encode(Sha256::digest(bytes)) - } - - fn audit_entry( - uri: &str, - hash: &str, - kind: crate::audit::AuditObjectKind, - result: crate::audit::AuditObjectResult, - detail: Option<&str>, - ) -> crate::audit::ObjectAuditEntry { - crate::audit::ObjectAuditEntry { - rsync_uri: uri.to_string(), - sha256_hex: hash.to_string(), - kind, - result, - detail: detail.map(ToString::to_string), - } - } - - #[test] - fn build_cir_from_run_collects_consumed_audit_objects_and_tal() { - let td = tempfile::tempdir().unwrap(); - let store = RocksStore::open(td.path()).unwrap(); - let bytes = b"object-a".to_vec(); - let hash = sha256_hex(&bytes); - let publication_points = vec![PublicationPointAudit { - objects: vec![audit_entry( - "rsync://example.test/repo/a.cer", - &hash, - crate::audit::AuditObjectKind::Certificate, - crate::audit::AuditObjectResult::Ok, - None, - )], - ..PublicationPointAudit::default() - }]; - - let ta = sample_trust_anchor(); - let cir = build_cir_from_run( - &store, - &ta, - "https://example.test/root.tal", - sample_time(), - &publication_points, - ) - .expect("build cir"); - assert_eq!(cir.version, crate::cir::model::CIR_VERSION_V4); - assert_eq!(cir.trust_anchors.len(), 1); - assert_eq!( - cir.trust_anchors[0].tal_uri, - "https://example.test/root.tal" - ); - assert!( - cir.fresh_validated_objects - .iter() - .any(|item| item.rsync_uri == "rsync://example.test/repo/a.cer") - ); - assert!( - !cir.fresh_validated_objects - .iter() - .any(|item| item.rsync_uri == cir.trust_anchors[0].ta_rsync_uri) - ); - assert!(!cir.trust_anchors[0].ta_certificate_der.is_empty()); - } - - #[test] - fn export_cir_from_run_writes_der_and_static_pool() { - let td = tempfile::tempdir().unwrap(); - let store_dir = td.path().join("db"); - let out_dir = td.path().join("out"); - let _static_root = td.path().join("static"); - let store = RocksStore::open(&store_dir).unwrap(); - - let bytes = b"object-b".to_vec(); - let hash = sha256_hex(&bytes); - let mut raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone()); - raw.origin_uris - .push("rsync://example.test/repo/b.roa".into()); - store.put_raw_by_hash_entry(&raw).unwrap(); - let publication_points = vec![PublicationPointAudit { - objects: vec![audit_entry( - "rsync://example.test/repo/b.roa", - &hash, - crate::audit::AuditObjectKind::Roa, - crate::audit::AuditObjectResult::Ok, - None, - )], - ..PublicationPointAudit::default() - }]; - - let ta = sample_trust_anchor(); - let cir_path = out_dir.join("example.cir"); - let summary = export_cir_from_run( - &store, - &ta, - "https://example.test/root.tal", - sample_time(), - &publication_points, - &cir_path, - sample_date(), - ) - .expect("export cir"); - assert_eq!(summary.trust_anchor_count, 1); - assert_eq!(summary.object_count, 1); - assert!(summary.timing.total_ms >= summary.timing.build_cir_ms); - - let der = std::fs::read(&cir_path).unwrap(); - let cir = decode_cir(&der).unwrap(); - assert_eq!( - cir.trust_anchors[0].tal_uri, - "https://example.test/root.tal" - ); - } - - #[test] - fn export_cir_from_run_uses_raw_store_backend_without_pool_export() { - let td = tempfile::tempdir().unwrap(); - let store_dir = td.path().join("db"); - let raw_store = td.path().join("raw-store.db"); - let out_dir = td.path().join("out"); - let store = RocksStore::open_with_external_raw_store(&store_dir, &raw_store).unwrap(); - - let bytes = b"object-d".to_vec(); - let hash = sha256_hex(&bytes); - let mut raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone()); - raw.origin_uris - .push("rsync://example.test/repo/d.roa".into()); - store.put_raw_by_hash_entry(&raw).unwrap(); - let publication_points = vec![PublicationPointAudit { - objects: vec![audit_entry( - "rsync://example.test/repo/d.roa", - &hash, - crate::audit::AuditObjectKind::Roa, - crate::audit::AuditObjectResult::Ok, - None, - )], - ..PublicationPointAudit::default() - }]; - - let ta = sample_trust_anchor(); - let cir_path = out_dir.join("example.cir"); - let summary = export_cir_from_run( - &store, - &ta, - "https://example.test/root.tal", - sample_time(), - &publication_points, - &cir_path, - sample_date(), - ) - .expect("export cir"); - assert_eq!(summary.object_count, 1); - assert!(raw_store.exists()); - assert!(cir_path.exists()); - } - - #[test] - fn export_cir_from_run_does_not_write_ta_bytes_to_repo_bytes_store() { - let td = tempfile::tempdir().unwrap(); - let store = RocksStore::open_with_external_repo_bytes( - &td.path().join("db"), - &td.path().join("repo-bytes.db"), - ) - .unwrap(); - - let ta = sample_trust_anchor(); - let ta_hash = sha256_hex(&ta.ta_certificate.raw_der); - let cir_path = td.path().join("out").join("example.cir"); - export_cir_from_run( - &store, - &ta, - "https://example.test/root.tal", - sample_time(), - &[], - &cir_path, - sample_date(), - ) - .expect("export cir"); - - assert_eq!(store.get_blob_bytes(&ta_hash).unwrap(), None); - } - - #[test] - fn build_cir_from_run_includes_consumed_vcir_current_instance_objects_from_audit() { - let td = tempfile::tempdir().unwrap(); - let store = RocksStore::open(td.path()).unwrap(); - let ta = sample_trust_anchor(); - - let mut pp = PublicationPointAudit { - source: "vcir_current_instance".to_string(), - ..PublicationPointAudit::default() - }; - pp.objects.push(crate::audit::ObjectAuditEntry { - rsync_uri: "rsync://example.test/repo/fallback.mft".to_string(), - sha256_hex: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - .to_string(), - kind: crate::audit::AuditObjectKind::Manifest, - result: crate::audit::AuditObjectResult::Ok, - detail: None, - }); - pp.objects.push(crate::audit::ObjectAuditEntry { - rsync_uri: "rsync://example.test/repo/fallback.roa".to_string(), - sha256_hex: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" - .to_string(), - kind: crate::audit::AuditObjectKind::Roa, - result: crate::audit::AuditObjectResult::Ok, - detail: None, - }); - - let cir = build_cir_from_run( - &store, - &ta, - "https://example.test/root.tal", - sample_time(), - &[pp], - ) - .expect("build cir"); - - assert!( - cir.cached_validated_objects - .iter() - .any(|item| item.rsync_uri == "rsync://example.test/repo/fallback.mft") - ); - assert!( - cir.cached_validated_objects - .iter() - .any(|item| item.rsync_uri == "rsync://example.test/repo/fallback.roa") - ); - } - - #[test] - fn build_cir_from_run_multi_ignores_current_repo_superfluous_objects() { - let td = tempfile::tempdir().unwrap(); - let store = RocksStore::open(td.path()).unwrap(); - let ta1 = sample_trust_anchor(); - let ta2 = sample_arin_trust_anchor(); - let current_repo_objects = vec![crate::current_repo_index::CurrentRepoObject { - rsync_uri: "rsync://example.test/repo/superfluous.roa".to_string(), - current_hash_hex: "11".repeat(32), - repository_source: "https://rrdp.example.test/notification.xml".to_string(), - object_type: Some("roa".to_string()), - }]; - let publication_points = vec![PublicationPointAudit { - objects: vec![audit_entry( - "rsync://example.test/repo/consumed.roa", - &"22".repeat(32), - crate::audit::AuditObjectKind::Roa, - crate::audit::AuditObjectResult::Ok, - None, - )], - ..PublicationPointAudit::default() - }]; - - let cir = build_cir_from_run_multi( - &store, - &[ - CirTrustAnchorBinding { - trust_anchor: &ta1, - tal_uri: "https://example.test/apnic.tal", - }, - CirTrustAnchorBinding { - trust_anchor: &ta2, - tal_uri: "https://example.test/arin.tal", - }, - ], - sample_time(), - &publication_points, - Some(¤t_repo_objects), - ) - .expect("build cir from consumed audit objects"); - - assert_eq!(cir.trust_anchors.len(), 2); - assert!( - cir.fresh_validated_objects - .iter() - .any(|item| item.rsync_uri == "rsync://example.test/repo/consumed.roa") - ); - assert!( - !cir.fresh_validated_objects - .iter() - .any(|item| item.rsync_uri == "rsync://example.test/repo/superfluous.roa"), - "current repo objects must not be included unless validation consumed them", - ); - for trust_anchor in &cir.trust_anchors { - assert!( - !cir.fresh_validated_objects - .iter() - .any(|item| item.rsync_uri == trust_anchor.ta_rsync_uri), - "trust anchor rsync objects must not be included in CIR.objects", - ); - } - } - - #[test] - fn build_cir_from_run_multi_sorts_trust_anchors_by_ta_rsync_uri() { - let td = tempfile::tempdir().unwrap(); - let store = RocksStore::open(td.path()).unwrap(); - let apnic = sample_trust_anchor(); - let arin = sample_arin_trust_anchor(); - - let cir = build_cir_from_run_multi( - &store, - &[ - CirTrustAnchorBinding { - trust_anchor: &apnic, - tal_uri: "https://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer", - }, - CirTrustAnchorBinding { - trust_anchor: &arin, - tal_uri: "https://rrdp.arin.net/arin-rpki-ta.cer", - }, - ], - sample_time(), - &[], - Some(&[]), - ) - .expect("build cir with unsorted input bindings"); - - assert_eq!( - cir.trust_anchors - .iter() - .map(|trust_anchor| trust_anchor.ta_rsync_uri.as_str()) - .collect::>(), - vec![ - "rsync://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer", - "rsync://rpki.arin.net/repository/arin-rpki-ta.cer", - ] - ); - } - - #[test] - fn build_cir_from_run_multi_exports_rejected_objects_from_error_audit_only() { - let td = tempfile::tempdir().unwrap(); - let store = RocksStore::open(td.path()).unwrap(); - let ta = sample_trust_anchor(); - let publication_points = vec![PublicationPointAudit { - objects: vec![ - crate::audit::ObjectAuditEntry { - rsync_uri: "rsync://example.test/repo/a.roa".to_string(), - sha256_hex: "11".repeat(32), - kind: crate::audit::AuditObjectKind::Roa, - result: crate::audit::AuditObjectResult::Error, - detail: Some("invalid roa".to_string()), - }, - crate::audit::ObjectAuditEntry { - rsync_uri: "rsync://example.test/repo/b.asa".to_string(), - sha256_hex: "22".repeat(32), - kind: crate::audit::AuditObjectKind::Aspa, - result: crate::audit::AuditObjectResult::Skipped, - detail: Some("skipped".to_string()), - }, - crate::audit::ObjectAuditEntry { - rsync_uri: "rsync://example.test/repo/c.roa".to_string(), - sha256_hex: "33".repeat(32), - kind: crate::audit::AuditObjectKind::Roa, - result: crate::audit::AuditObjectResult::Error, - detail: Some("second rejected roa".to_string()), - }, - ], - ..PublicationPointAudit::default() - }]; - - let cir = build_cir_from_run_multi( - &store, - &[CirTrustAnchorBinding { - trust_anchor: &ta, - tal_uri: "https://example.test/root.tal", - }], - sample_time(), - &publication_points, - None, - ) - .expect("build cir"); - - assert_eq!(cir.rejected_object_count(), 2); - assert_eq!( - cir.fresh_rejected_objects[0].object_uri, - "rsync://example.test/repo/a.roa" - ); - assert_eq!( - cir.fresh_rejected_objects[0].reason.as_deref(), - Some("invalid roa") - ); - assert_eq!( - cir.fresh_rejected_objects[1].object_uri, - "rsync://example.test/repo/c.roa" - ); - assert!( - cir.fresh_validated_objects - .iter() - .any(|item| item.rsync_uri == "rsync://example.test/repo/a.roa"), - "rejected audit objects were still consumed as validation input", - ); - assert!( - cir.fresh_validated_objects - .iter() - .any(|item| item.rsync_uri == "rsync://example.test/repo/c.roa"), - "rejected audit objects were still consumed as validation input", - ); - assert!( - !cir.fresh_validated_objects - .iter() - .any(|item| item.rsync_uri == "rsync://example.test/repo/b.asa"), - "skipped audit objects are not considered consumed input", - ); - } - - #[test] - fn build_cir_from_run_multi_records_crl_expired_manifest_in_objects_and_rejects() { - let td = tempfile::tempdir().unwrap(); - let store = RocksStore::open(td.path()).unwrap(); - let ta = sample_trust_anchor(); - let manifest_uri = "rsync://example.test/repo/expired-crl.mft"; - let reject_reason = "manifest embedded EE certificate path validation failed: CRL not valid at validation_time (RFC 5280 §6.3.3(g); RFC 5280 §5.1.2.4-§5.1.2.5; RFC 6487 §5)"; - let publication_points = vec![PublicationPointAudit { - objects: vec![crate::audit::ObjectAuditEntry { - rsync_uri: manifest_uri.to_string(), - sha256_hex: "44".repeat(32), - kind: crate::audit::AuditObjectKind::Manifest, - result: crate::audit::AuditObjectResult::Error, - detail: Some(reject_reason.to_string()), - }], - ..PublicationPointAudit::default() - }]; - - let cir = build_cir_from_run_multi( - &store, - &[CirTrustAnchorBinding { - trust_anchor: &ta, - tal_uri: "https://example.test/root.tal", - }], - sample_time(), - &publication_points, - None, - ) - .expect("build cir"); - - assert!( - cir.fresh_validated_objects - .iter() - .any(|item| item.rsync_uri == manifest_uri), - "manifest rejected because issuer CRL is expired was still read as validation input", - ); - assert_eq!(cir.rejected_object_count(), 1); - assert_eq!(cir.fresh_rejected_objects[0].object_uri, manifest_uri); - assert_eq!( - cir.fresh_rejected_objects[0].reason.as_deref(), - Some(reject_reason) - ); - assert_eq!( - cir.reject_list_sha256, - crate::cir::model::compute_reject_list_sha256([manifest_uri].into_iter()) - ); - } - - #[test] - fn build_cir_from_run_multi_excludes_manifest_locked_files_when_manifest_is_rejected() { - let td = tempfile::tempdir().unwrap(); - let store = RocksStore::open(td.path()).unwrap(); - let ta = sample_trust_anchor(); - let manifest_uri = "rsync://example.test/repo/rejected.mft"; - let roa_uri = "rsync://example.test/repo/listed.roa"; - let crl_uri = "rsync://example.test/repo/listed.crl"; - let publication_points = vec![PublicationPointAudit { - objects: vec![ - crate::audit::ObjectAuditEntry { - rsync_uri: manifest_uri.to_string(), - sha256_hex: "44".repeat(32), - kind: crate::audit::AuditObjectKind::Manifest, - result: crate::audit::AuditObjectResult::Error, - detail: Some("manifest EE cert path rejected".to_string()), - }, - crate::audit::ObjectAuditEntry { - rsync_uri: roa_uri.to_string(), - sha256_hex: "55".repeat(32), - kind: crate::audit::AuditObjectKind::Roa, - result: crate::audit::AuditObjectResult::Skipped, - detail: Some("manifest rejected before locked object validation".to_string()), - }, - crate::audit::ObjectAuditEntry { - rsync_uri: crl_uri.to_string(), - sha256_hex: "66".repeat(32), - kind: crate::audit::AuditObjectKind::Crl, - result: crate::audit::AuditObjectResult::Skipped, - detail: Some("manifest rejected before locked object validation".to_string()), - }, - ], - ..PublicationPointAudit::default() - }]; - - let cir = build_cir_from_run_multi( - &store, - &[CirTrustAnchorBinding { - trust_anchor: &ta, - tal_uri: "https://example.test/root.tal", - }], - sample_time(), - &publication_points, - None, - ) - .expect("build cir"); - - assert!( - cir.fresh_validated_objects - .iter() - .any(|item| item.rsync_uri == manifest_uri), - "rejected manifest is still a current-run validation input", - ); - assert!( - !cir.fresh_validated_objects - .iter() - .any(|item| item.rsync_uri == roa_uri), - "ROA listed by a rejected manifest must not enter CIR objects", - ); - assert!( - !cir.fresh_validated_objects - .iter() - .any(|item| item.rsync_uri == crl_uri), - "CRL listed by a rejected manifest must not enter CIR objects", - ); - assert_eq!(cir.rejected_object_count(), 1); - assert_eq!(cir.fresh_rejected_objects[0].object_uri, manifest_uri); - } - - #[test] - fn build_cir_from_run_multi_reject_digest_ignores_reason_text() { - let td = tempfile::tempdir().unwrap(); - let store = RocksStore::open(td.path()).unwrap(); - let ta = sample_trust_anchor(); - - let mk_pp = |detail: &str| PublicationPointAudit { - objects: vec![crate::audit::ObjectAuditEntry { - rsync_uri: "rsync://example.test/repo/a.roa".to_string(), - sha256_hex: "11".repeat(32), - kind: crate::audit::AuditObjectKind::Roa, - result: crate::audit::AuditObjectResult::Error, - detail: Some(detail.to_string()), - }], - ..PublicationPointAudit::default() - }; - - let cir_a = build_cir_from_run_multi( - &store, - &[CirTrustAnchorBinding { - trust_anchor: &ta, - tal_uri: "https://example.test/root.tal", - }], - sample_time(), - &[mk_pp("reason-a")], - None, - ) - .expect("build cir a"); - let cir_b = build_cir_from_run_multi( - &store, - &[CirTrustAnchorBinding { - trust_anchor: &ta, - tal_uri: "https://example.test/root.tal", - }], - sample_time(), - &[mk_pp("reason-b")], - None, - ) - .expect("build cir b"); - - assert_eq!(cir_a.reject_list_sha256, cir_b.reject_list_sha256); - assert_ne!( - cir_a.fresh_rejected_objects[0].reason, - cir_b.fresh_rejected_objects[0].reason - ); - } - - #[test] - fn build_cir_from_run_multi_rejects_invalid_tal_uri_and_missing_rsync_ta_uri() { - let td = tempfile::tempdir().unwrap(); - let store = RocksStore::open(td.path()).unwrap(); - - let err = build_cir_from_run_multi( - &store, - &[CirTrustAnchorBinding { - trust_anchor: &sample_trust_anchor(), - tal_uri: "file:///not-supported.tal", - }], - sample_time(), - &[], - None, - ) - .expect_err("non-http tal uri must fail"); - assert!(matches!(err, CirExportError::InvalidTalUri(_)), "{err}"); - - let err = build_cir_from_run_multi( - &store, - &[CirTrustAnchorBinding { - trust_anchor: &sample_trust_anchor_without_rsync_uri(), - tal_uri: "https://example.test/root.tal", - }], - sample_time(), - &[], - None, - ) - .expect_err("missing rsync ta uri must fail"); - assert!(matches!(err, CirExportError::MissingTaRsyncUri), "{err}"); - } - - #[test] - fn build_cir_from_input_snapshot_multi_preserves_online_sections() { - let ta = sample_trust_anchor(); - let cir = build_cir_from_input_snapshot_multi( - &[CirTrustAnchorBinding { - trust_anchor: &ta, - tal_uri: "https://example.test/root.tal", - }], - sample_time(), - CirInputSnapshot { - fresh_validated_objects: vec![CirObject { - rsync_uri: "rsync://example.test/repo/fresh.roa".to_string(), - sha256: vec![0x11; 32], - }], - cached_validated_objects: vec![CirObject { - rsync_uri: "rsync://example.test/repo/cached.roa".to_string(), - sha256: vec![0x22; 32], - }], - fresh_rejected_objects: vec![CirRejectedObject { - object_uri: "rsync://example.test/repo/fresh.roa".to_string(), - reason: Some("fresh rejected".to_string()), - }], - cached_rejected_objects: vec![CirRejectedObject { - object_uri: "rsync://example.test/repo/cached.roa".to_string(), - reason: None, - }], - }, - ) - .expect("build CIR from online input"); - - cir.validate().expect("online CIR snapshot is valid"); - assert_eq!(cir.fresh_validated_objects.len(), 1); - assert_eq!(cir.cached_validated_objects.len(), 1); - assert_eq!(cir.fresh_rejected_objects.len(), 1); - assert_eq!(cir.cached_rejected_objects.len(), 1); - } - - #[test] - fn export_cir_static_pool_writes_repository_objects_only() { - let td = tempfile::tempdir().unwrap(); - let store = RocksStore::open(&td.path().join("db")).unwrap(); - let static_root = td.path().join("static"); - let ta1 = sample_trust_anchor(); - let ta2 = sample_arin_trust_anchor(); - - let object_bytes = b"object-z".to_vec(); - let hash = sha256_hex(&object_bytes); - let mut raw = RawByHashEntry::from_bytes(hash.clone(), object_bytes.clone()); - raw.origin_uris - .push("rsync://example.test/repo/z.roa".into()); - store.put_raw_by_hash_entry(&raw).unwrap(); - let publication_points = vec![PublicationPointAudit { - objects: vec![audit_entry( - "rsync://example.test/repo/z.roa", - &hash, - crate::audit::AuditObjectKind::Roa, - crate::audit::AuditObjectResult::Ok, - None, - )], - ..PublicationPointAudit::default() - }]; - - let cir = build_cir_from_run_multi( - &store, - &[ - CirTrustAnchorBinding { - trust_anchor: &ta1, - tal_uri: "https://example.test/apnic.tal", - }, - CirTrustAnchorBinding { - trust_anchor: &ta2, - tal_uri: "https://example.test/arin.tal", - }, - ], - sample_time(), - &publication_points, - None, - ) - .expect("build cir"); - - let summary = - export_cir_static_pool(&store, &static_root, sample_date(), &cir, &[&ta1, &ta2]) - .expect("export static pool"); - assert_eq!(summary.unique_hashes, 1); - assert_eq!(summary.written_files, 1); - for trust_anchor in &cir.trust_anchors { - let ta_hash = hex::encode(&trust_anchor.ta_certificate_sha256); - assert!( - !crate::cir::static_pool::static_pool_path(&static_root, sample_date(), &ta_hash) - .expect("static pool ta path") - .exists() - ); - } - } - - #[test] - fn export_cir_raw_store_reports_missing_non_ta_object_only() { - let td = tempfile::tempdir().unwrap(); - let raw_store_path = td.path().join("raw-store.db"); - let store = - RocksStore::open_with_external_raw_store(&td.path().join("db"), &raw_store_path) - .unwrap(); - let ta1 = sample_trust_anchor(); - let ta2 = sample_arin_trust_anchor(); - - let cir_only_tas = build_cir_from_run_multi( - &store, - &[ - CirTrustAnchorBinding { - trust_anchor: &ta1, - tal_uri: "https://example.test/apnic.tal", - }, - CirTrustAnchorBinding { - trust_anchor: &ta2, - tal_uri: "https://example.test/arin.tal", - }, - ], - sample_time(), - &[], - Some(&[]), - ) - .expect("build cir with tas only"); - - let summary = export_cir_raw_store(&store, &raw_store_path, &cir_only_tas, &[&ta1, &ta2]) - .expect("export raw store"); - assert_eq!(summary.unique_hashes, 0); - assert_eq!(summary.written_entries, 0); - assert_eq!(summary.reused_entries, 0); - - let mut cir_missing_object = cir_only_tas.clone(); - cir_missing_object.fresh_validated_objects.push(CirObject { - rsync_uri: "rsync://example.test/repo/missing.roa".to_string(), - sha256: vec![0x44; 32], - }); - let err = export_cir_raw_store(&store, &raw_store_path, &cir_missing_object, &[&ta1, &ta2]) - .expect_err("missing non-ta object must fail"); - assert!(matches!(err, CirExportError::Write(_, _)), "{err}"); - } -} +#[path = "export/tests.rs"] +mod tests; diff --git a/crates/panda-rpki-validator/src/cir/export/build_and_write.rs b/crates/panda-rpki-validator/src/cir/export/build_and_write.rs new file mode 100644 index 0000000..eece8ef --- /dev/null +++ b/crates/panda-rpki-validator/src/cir/export/build_and_write.rs @@ -0,0 +1,224 @@ +// CIR construction, serialization, and export entry points. + +pub fn build_cir_from_run( + store: &RocksStore, + trust_anchor: &TrustAnchor, + tal_uri: &str, + validation_time: time::OffsetDateTime, + publication_points: &[PublicationPointAudit], +) -> Result { + build_cir_from_run_multi( + store, + &[CirTrustAnchorBinding { + trust_anchor, + tal_uri, + }], + validation_time, + publication_points, + None, + ) +} + +pub fn build_cir_from_run_multi( + _store: &RocksStore, + tal_bindings: &[CirTrustAnchorBinding<'_>], + validation_time: time::OffsetDateTime, + publication_points: &[PublicationPointAudit], + _current_repo_objects: Option<&[CurrentRepoObject]>, +) -> Result { + let fresh_objects = + collect_cir_objects_from_validation_audit(publication_points, CirObjectSection::Fresh)?; + let cached_objects = + collect_cir_objects_from_validation_audit(publication_points, CirObjectSection::Cached)?; + + let trust_anchors = build_cir_trust_anchors(tal_bindings)?; + + let fresh_rejected_objects = + collect_rejected_objects_from_validation_audit(publication_points, CirObjectSection::Fresh); + let cached_rejected_objects = collect_rejected_objects_from_validation_audit( + publication_points, + CirObjectSection::Cached, + ); + + let cir = CanonicalInputRepresentation::new_v4( + validation_time, + cir_objects_from_hash_map(fresh_objects), + cir_objects_from_hash_map(cached_objects), + trust_anchors, + fresh_rejected_objects, + cached_rejected_objects, + ); + cir.validate().map_err(CirExportError::Validate)?; + Ok(cir) +} + +pub fn build_cir_from_input_snapshot_multi( + tal_bindings: &[CirTrustAnchorBinding<'_>], + validation_time: time::OffsetDateTime, + input: CirInputSnapshot, +) -> Result { + let trust_anchors = build_cir_trust_anchors(tal_bindings)?; + Ok(CanonicalInputRepresentation::new_v4( + validation_time, + input.fresh_validated_objects, + input.cached_validated_objects, + trust_anchors, + input.fresh_rejected_objects, + input.cached_rejected_objects, + )) +} + +pub fn write_cir_file( + path: &Path, + cir: &CanonicalInputRepresentation, +) -> Result<(), CirExportError> { + let der = encode_cir(cir)?; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| CirExportError::Write(path.display().to_string(), e.to_string()))?; + } + std::fs::write(path, der) + .map_err(|e| CirExportError::Write(path.display().to_string(), e.to_string())) +} + +pub fn export_cir_static_pool( + store: &RocksStore, + static_root: &Path, + capture_date_utc: time::Date, + cir: &CanonicalInputRepresentation, + trust_anchors: &[&TrustAnchor], +) -> Result { + let _ = trust_anchors; + let hashes = cir + .validated_objects() + .map(|item| hex::encode(&item.sha256)) + .collect::>(); + export_hashes_from_store(store, static_root, capture_date_utc, &hashes).map_err(Into::into) +} + +pub fn export_cir_raw_store( + store: &RocksStore, + raw_store_path: &Path, + cir: &CanonicalInputRepresentation, + trust_anchors: &[&TrustAnchor], +) -> Result { + let _ = trust_anchors; + let unique: BTreeSet = cir + .validated_objects() + .map(|item| hex::encode(&item.sha256)) + .collect(); + + let written_entries = 0usize; + let mut reused_entries = 0usize; + for sha256_hex in &unique { + if store + .get_blob_bytes(sha256_hex) + .map_err(|e| { + CirExportError::Write(raw_store_path.display().to_string(), e.to_string()) + })? + .is_some() + { + reused_entries += 1; + continue; + } + return Err(CirExportError::Write( + raw_store_path.display().to_string(), + format!("raw store missing object for sha256={sha256_hex}"), + )); + } + + Ok(CirRawStoreExportSummary { + unique_hashes: unique.len(), + written_entries, + reused_entries, + }) +} + +pub fn export_cir_from_run( + store: &RocksStore, + trust_anchor: &TrustAnchor, + tal_uri: &str, + validation_time: time::OffsetDateTime, + publication_points: &[PublicationPointAudit], + cir_out: &Path, + capture_date_utc: time::Date, +) -> Result { + export_cir_from_run_multi( + store, + &[CirTrustAnchorBinding { + trust_anchor, + tal_uri, + }], + validation_time, + publication_points, + cir_out, + capture_date_utc, + None, + ) +} + +pub fn export_cir_from_run_multi( + store: &RocksStore, + tal_bindings: &[CirTrustAnchorBinding<'_>], + validation_time: time::OffsetDateTime, + publication_points: &[PublicationPointAudit], + cir_out: &Path, + capture_date_utc: time::Date, + current_repo_objects: Option<&[CurrentRepoObject]>, +) -> Result { + let _ = capture_date_utc; + let total_started = std::time::Instant::now(); + + let started = std::time::Instant::now(); + let cir = build_cir_from_run_multi( + store, + tal_bindings, + validation_time, + publication_points, + current_repo_objects, + )?; + let build_cir_ms = started.elapsed().as_millis() as u64; + + let _ = store; + + let started = std::time::Instant::now(); + write_cir_file(cir_out, &cir)?; + let write_cir_ms = started.elapsed().as_millis() as u64; + + Ok(CirExportSummary { + object_count: cir.validated_object_count(), + trust_anchor_count: cir.trust_anchors.len(), + timing: CirExportTiming { + build_cir_ms, + write_cir_ms, + total_ms: total_started.elapsed().as_millis() as u64, + }, + }) +} + +pub fn export_cir_from_input_snapshot_multi( + tal_bindings: &[CirTrustAnchorBinding<'_>], + validation_time: time::OffsetDateTime, + input: CirInputSnapshot, + cir_out: &Path, +) -> Result { + let total_started = std::time::Instant::now(); + + let started = std::time::Instant::now(); + let cir = build_cir_from_input_snapshot_multi(tal_bindings, validation_time, input)?; + let build_cir_ms = started.elapsed().as_millis() as u64; + + let started = std::time::Instant::now(); + write_cir_file(cir_out, &cir)?; + let write_cir_ms = started.elapsed().as_millis() as u64; + + Ok(CirExportSummary { + object_count: cir.validated_object_count(), + trust_anchor_count: cir.trust_anchors.len(), + timing: CirExportTiming { + build_cir_ms, + write_cir_ms, + total_ms: total_started.elapsed().as_millis() as u64, + }, + }) +} diff --git a/crates/panda-rpki-validator/src/cir/export/models_and_collect.rs b/crates/panda-rpki-validator/src/cir/export/models_and_collect.rs new file mode 100644 index 0000000..fef0a4f --- /dev/null +++ b/crates/panda-rpki-validator/src/cir/export/models_and_collect.rs @@ -0,0 +1,192 @@ +// CIR export data types and audit-object collection helpers. + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CirExportTiming { + pub build_cir_ms: u64, + pub write_cir_ms: u64, + pub total_ms: u64, +} + +#[derive(Debug, thiserror::Error)] +pub enum CirExportError { + #[error("CIR TAL URI must be http(s), got: {0}")] + InvalidTalUri(String), + + #[error("TAL does not contain any rsync TA URI; CIR replay scheme A requires one")] + MissingTaRsyncUri, + + #[error("CIR model validation failed: {0}")] + Validate(String), + + #[error("CIR consumed audit has conflicting hashes for {rsync_uri}: {first} vs {second}")] + ConflictingObjectHash { + rsync_uri: String, + first: String, + second: String, + }, + + #[error("encode CIR failed: {0}")] + Encode(#[from] CirEncodeError), + + #[error("static pool export failed: {0}")] + StaticPool(#[from] CirStaticPoolError), + + #[error("write CIR file failed: {0}: {1}")] + Write(String, String), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CirRawStoreExportSummary { + pub unique_hashes: usize, + pub written_entries: usize, + pub reused_entries: usize, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CirExportSummary { + pub object_count: usize, + pub trust_anchor_count: usize, + pub timing: CirExportTiming, +} + +#[derive(Clone, Copy, Debug)] +pub struct CirTrustAnchorBinding<'a> { + pub trust_anchor: &'a TrustAnchor, + pub tal_uri: &'a str, +} + +fn is_sha256_hex(value: &str) -> bool { + value.len() == 64 && value.as_bytes().iter().all(u8::is_ascii_hexdigit) +} + +fn insert_consumed_object_hash( + objects: &mut BTreeMap, + rsync_uri: &str, + sha256_hex: &str, +) -> Result<(), CirExportError> { + if !rsync_uri.starts_with("rsync://") || !is_sha256_hex(sha256_hex) { + return Ok(()); + } + + let normalized = sha256_hex.to_ascii_lowercase(); + if let Some(existing) = objects.get(rsync_uri) { + if existing != &normalized { + return Err(CirExportError::ConflictingObjectHash { + rsync_uri: rsync_uri.to_string(), + first: existing.clone(), + second: normalized, + }); + } + return Ok(()); + } + + objects.insert(rsync_uri.to_string(), normalized); + Ok(()) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CirObjectSection { + Fresh, + Cached, +} + +fn publication_point_is_cached_fallback(pp: &PublicationPointAudit) -> bool { + pp.source == "vcir_current_instance" || pp.repo_terminal_state == "fallback_current_instance" +} + +fn publication_point_cir_entries<'a>( + pp: &'a PublicationPointAudit, + section: CirObjectSection, +) -> &'a [ObjectAuditEntry] { + match (section, publication_point_is_cached_fallback(pp)) { + (CirObjectSection::Fresh, false) => &pp.objects, + (CirObjectSection::Cached, true) => &pp.objects, + _ => &[], + } +} + +fn collect_cir_objects_from_validation_audit( + publication_points: &[PublicationPointAudit], + section: CirObjectSection, +) -> Result, CirExportError> { + let mut objects = BTreeMap::new(); + for pp in publication_points { + for obj in publication_point_cir_entries(pp, section) { + if !matches!(obj.result, AuditObjectResult::Ok | AuditObjectResult::Error) { + continue; + } + insert_consumed_object_hash(&mut objects, &obj.rsync_uri, &obj.sha256_hex)?; + } + } + Ok(objects) +} + +fn collect_rejected_objects_from_validation_audit( + publication_points: &[PublicationPointAudit], + section: CirObjectSection, +) -> Vec { + let mut rejected_objects = publication_points + .iter() + .flat_map(|pp| publication_point_cir_entries(pp, section).iter()) + .filter(|item| item.result == AuditObjectResult::Error) + .filter(|item| item.rsync_uri.starts_with("rsync://")) + .map(|item| CirRejectedObject { + object_uri: item.rsync_uri.clone(), + reason: item.detail.clone(), + }) + .collect::>(); + rejected_objects.sort_by(|a, b| a.object_uri.cmp(&b.object_uri)); + rejected_objects.dedup_by(|a, b| a.object_uri == b.object_uri); + rejected_objects +} + +fn cir_objects_from_hash_map(objects: BTreeMap) -> Vec { + objects + .into_iter() + .map(|(rsync_uri, sha256_hex)| CirObject { + rsync_uri, + sha256: hex::decode(sha256_hex).expect("validated hex"), + }) + .collect() +} + +fn canonical_ta_rsync_uri(trust_anchor: &TrustAnchor) -> Result { + if let Some(uri) = &trust_anchor.resolved_ta_uri + && uri.scheme() == "rsync" + { + return Ok(uri.as_str().to_string()); + } + trust_anchor + .tal + .ta_uris + .iter() + .filter(|uri| uri.scheme() == "rsync") + .map(|uri| uri.as_str().to_string()) + .min() + .ok_or(CirExportError::MissingTaRsyncUri) +} + +fn build_cir_trust_anchors( + tal_bindings: &[CirTrustAnchorBinding<'_>], +) -> Result, CirExportError> { + for binding in tal_bindings { + if !(binding.tal_uri.starts_with("https://") || binding.tal_uri.starts_with("http://")) { + return Err(CirExportError::InvalidTalUri(binding.tal_uri.to_string())); + } + } + + let mut trust_anchors = Vec::with_capacity(tal_bindings.len()); + for binding in tal_bindings { + let ta_rsync_uri = canonical_ta_rsync_uri(binding.trust_anchor)?; + let ta_certificate_der = binding.trust_anchor.ta_certificate.raw_der.clone(); + trust_anchors.push(CirTrustAnchor { + ta_rsync_uri, + tal_uri: binding.tal_uri.to_string(), + tal_bytes: binding.trust_anchor.tal.raw.clone(), + ta_certificate_sha256: crate::cir::model::sha256(&ta_certificate_der), + ta_certificate_der, + }); + } + trust_anchors.sort_by(|a, b| a.ta_rsync_uri.cmp(&b.ta_rsync_uri)); + Ok(trust_anchors) +} diff --git a/crates/panda-rpki-validator/src/cir/export/tests.rs b/crates/panda-rpki-validator/src/cir/export/tests.rs new file mode 100644 index 0000000..915fab6 --- /dev/null +++ b/crates/panda-rpki-validator/src/cir/export/tests.rs @@ -0,0 +1,780 @@ +// CIR export and raw-store integration tests. + +use super::*; +use crate::cir::decode::decode_cir; +use crate::data_model::ta::TrustAnchor; +use crate::data_model::tal::Tal; +use crate::storage::{RawByHashEntry, RocksStore}; + +fn sample_time() -> time::OffsetDateTime { + time::OffsetDateTime::parse( + "2026-04-07T12:34:56Z", + &time::format_description::well_known::Rfc3339, + ) + .unwrap() +} + +fn sample_date() -> time::Date { + time::Date::from_calendar_date(2026, time::Month::April, 7).unwrap() +} + +fn sample_trust_anchor() -> TrustAnchor { + let base = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let tal_bytes = std::fs::read(base.join("tests/fixtures/tal/apnic-rfc7730-https.tal")).unwrap(); + let ta_der = std::fs::read(base.join("tests/fixtures/ta/apnic-ta.cer")).unwrap(); + let tal = Tal::decode_bytes(&tal_bytes).unwrap(); + TrustAnchor::bind_der(tal, &ta_der, None).unwrap() +} + +fn sample_arin_trust_anchor() -> TrustAnchor { + let base = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let tal_bytes = std::fs::read(base.join("tests/fixtures/tal/arin.tal")).unwrap(); + let ta_der = std::fs::read(base.join("tests/fixtures/ta/arin-ta.cer")).unwrap(); + let tal = Tal::decode_bytes(&tal_bytes).unwrap(); + TrustAnchor::bind_der(tal, &ta_der, None).unwrap() +} + +fn sample_trust_anchor_without_rsync_uri() -> TrustAnchor { + let mut ta = sample_trust_anchor(); + ta.tal.ta_uris.retain(|uri| uri.scheme() != "rsync"); + ta +} + +fn sha256_hex(bytes: &[u8]) -> String { + use sha2::{Digest, Sha256}; + hex::encode(Sha256::digest(bytes)) +} + +fn audit_entry( + uri: &str, + hash: &str, + kind: crate::audit::AuditObjectKind, + result: crate::audit::AuditObjectResult, + detail: Option<&str>, +) -> crate::audit::ObjectAuditEntry { + crate::audit::ObjectAuditEntry { + rsync_uri: uri.to_string(), + sha256_hex: hash.to_string(), + kind, + result, + detail: detail.map(ToString::to_string), + } +} + +#[test] +fn build_cir_from_run_collects_consumed_audit_objects_and_tal() { + let td = tempfile::tempdir().unwrap(); + let store = RocksStore::open(td.path()).unwrap(); + let bytes = b"object-a".to_vec(); + let hash = sha256_hex(&bytes); + let publication_points = vec![PublicationPointAudit { + objects: vec![audit_entry( + "rsync://example.test/repo/a.cer", + &hash, + crate::audit::AuditObjectKind::Certificate, + crate::audit::AuditObjectResult::Ok, + None, + )], + ..PublicationPointAudit::default() + }]; + + let ta = sample_trust_anchor(); + let cir = build_cir_from_run( + &store, + &ta, + "https://example.test/root.tal", + sample_time(), + &publication_points, + ) + .expect("build cir"); + assert_eq!(cir.version, crate::cir::model::CIR_VERSION_V4); + assert_eq!(cir.trust_anchors.len(), 1); + assert_eq!( + cir.trust_anchors[0].tal_uri, + "https://example.test/root.tal" + ); + assert!( + cir.fresh_validated_objects + .iter() + .any(|item| item.rsync_uri == "rsync://example.test/repo/a.cer") + ); + assert!( + !cir.fresh_validated_objects + .iter() + .any(|item| item.rsync_uri == cir.trust_anchors[0].ta_rsync_uri) + ); + assert!(!cir.trust_anchors[0].ta_certificate_der.is_empty()); +} + +#[test] +fn export_cir_from_run_writes_der_and_static_pool() { + let td = tempfile::tempdir().unwrap(); + let store_dir = td.path().join("db"); + let out_dir = td.path().join("out"); + let _static_root = td.path().join("static"); + let store = RocksStore::open(&store_dir).unwrap(); + + let bytes = b"object-b".to_vec(); + let hash = sha256_hex(&bytes); + let mut raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone()); + raw.origin_uris + .push("rsync://example.test/repo/b.roa".into()); + store.put_raw_by_hash_entry(&raw).unwrap(); + let publication_points = vec![PublicationPointAudit { + objects: vec![audit_entry( + "rsync://example.test/repo/b.roa", + &hash, + crate::audit::AuditObjectKind::Roa, + crate::audit::AuditObjectResult::Ok, + None, + )], + ..PublicationPointAudit::default() + }]; + + let ta = sample_trust_anchor(); + let cir_path = out_dir.join("example.cir"); + let summary = export_cir_from_run( + &store, + &ta, + "https://example.test/root.tal", + sample_time(), + &publication_points, + &cir_path, + sample_date(), + ) + .expect("export cir"); + assert_eq!(summary.trust_anchor_count, 1); + assert_eq!(summary.object_count, 1); + assert!(summary.timing.total_ms >= summary.timing.build_cir_ms); + + let der = std::fs::read(&cir_path).unwrap(); + let cir = decode_cir(&der).unwrap(); + assert_eq!( + cir.trust_anchors[0].tal_uri, + "https://example.test/root.tal" + ); +} + +#[test] +fn export_cir_from_run_uses_raw_store_backend_without_pool_export() { + let td = tempfile::tempdir().unwrap(); + let store_dir = td.path().join("db"); + let raw_store = td.path().join("raw-store.db"); + let out_dir = td.path().join("out"); + let store = RocksStore::open_with_external_raw_store(&store_dir, &raw_store).unwrap(); + + let bytes = b"object-d".to_vec(); + let hash = sha256_hex(&bytes); + let mut raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone()); + raw.origin_uris + .push("rsync://example.test/repo/d.roa".into()); + store.put_raw_by_hash_entry(&raw).unwrap(); + let publication_points = vec![PublicationPointAudit { + objects: vec![audit_entry( + "rsync://example.test/repo/d.roa", + &hash, + crate::audit::AuditObjectKind::Roa, + crate::audit::AuditObjectResult::Ok, + None, + )], + ..PublicationPointAudit::default() + }]; + + let ta = sample_trust_anchor(); + let cir_path = out_dir.join("example.cir"); + let summary = export_cir_from_run( + &store, + &ta, + "https://example.test/root.tal", + sample_time(), + &publication_points, + &cir_path, + sample_date(), + ) + .expect("export cir"); + assert_eq!(summary.object_count, 1); + assert!(raw_store.exists()); + assert!(cir_path.exists()); +} + +#[test] +fn export_cir_from_run_does_not_write_ta_bytes_to_repo_bytes_store() { + let td = tempfile::tempdir().unwrap(); + let store = RocksStore::open_with_external_repo_bytes( + &td.path().join("db"), + &td.path().join("repo-bytes.db"), + ) + .unwrap(); + + let ta = sample_trust_anchor(); + let ta_hash = sha256_hex(&ta.ta_certificate.raw_der); + let cir_path = td.path().join("out").join("example.cir"); + export_cir_from_run( + &store, + &ta, + "https://example.test/root.tal", + sample_time(), + &[], + &cir_path, + sample_date(), + ) + .expect("export cir"); + + assert_eq!(store.get_blob_bytes(&ta_hash).unwrap(), None); +} + +#[test] +fn build_cir_from_run_includes_consumed_vcir_current_instance_objects_from_audit() { + let td = tempfile::tempdir().unwrap(); + let store = RocksStore::open(td.path()).unwrap(); + let ta = sample_trust_anchor(); + + let mut pp = PublicationPointAudit { + source: "vcir_current_instance".to_string(), + ..PublicationPointAudit::default() + }; + pp.objects.push(crate::audit::ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/fallback.mft".to_string(), + sha256_hex: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_string(), + kind: crate::audit::AuditObjectKind::Manifest, + result: crate::audit::AuditObjectResult::Ok, + detail: None, + }); + pp.objects.push(crate::audit::ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/fallback.roa".to_string(), + sha256_hex: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string(), + kind: crate::audit::AuditObjectKind::Roa, + result: crate::audit::AuditObjectResult::Ok, + detail: None, + }); + + let cir = build_cir_from_run( + &store, + &ta, + "https://example.test/root.tal", + sample_time(), + &[pp], + ) + .expect("build cir"); + + assert!( + cir.cached_validated_objects + .iter() + .any(|item| item.rsync_uri == "rsync://example.test/repo/fallback.mft") + ); + assert!( + cir.cached_validated_objects + .iter() + .any(|item| item.rsync_uri == "rsync://example.test/repo/fallback.roa") + ); +} + +#[test] +fn build_cir_from_run_multi_ignores_current_repo_superfluous_objects() { + let td = tempfile::tempdir().unwrap(); + let store = RocksStore::open(td.path()).unwrap(); + let ta1 = sample_trust_anchor(); + let ta2 = sample_arin_trust_anchor(); + let current_repo_objects = vec![crate::current_repo_index::CurrentRepoObject { + rsync_uri: "rsync://example.test/repo/superfluous.roa".to_string(), + current_hash_hex: "11".repeat(32), + repository_source: "https://rrdp.example.test/notification.xml".to_string(), + object_type: Some("roa".to_string()), + }]; + let publication_points = vec![PublicationPointAudit { + objects: vec![audit_entry( + "rsync://example.test/repo/consumed.roa", + &"22".repeat(32), + crate::audit::AuditObjectKind::Roa, + crate::audit::AuditObjectResult::Ok, + None, + )], + ..PublicationPointAudit::default() + }]; + + let cir = build_cir_from_run_multi( + &store, + &[ + CirTrustAnchorBinding { + trust_anchor: &ta1, + tal_uri: "https://example.test/apnic.tal", + }, + CirTrustAnchorBinding { + trust_anchor: &ta2, + tal_uri: "https://example.test/arin.tal", + }, + ], + sample_time(), + &publication_points, + Some(¤t_repo_objects), + ) + .expect("build cir from consumed audit objects"); + + assert_eq!(cir.trust_anchors.len(), 2); + assert!( + cir.fresh_validated_objects + .iter() + .any(|item| item.rsync_uri == "rsync://example.test/repo/consumed.roa") + ); + assert!( + !cir.fresh_validated_objects + .iter() + .any(|item| item.rsync_uri == "rsync://example.test/repo/superfluous.roa"), + "current repo objects must not be included unless validation consumed them", + ); + for trust_anchor in &cir.trust_anchors { + assert!( + !cir.fresh_validated_objects + .iter() + .any(|item| item.rsync_uri == trust_anchor.ta_rsync_uri), + "trust anchor rsync objects must not be included in CIR.objects", + ); + } +} + +#[test] +fn build_cir_from_run_multi_sorts_trust_anchors_by_ta_rsync_uri() { + let td = tempfile::tempdir().unwrap(); + let store = RocksStore::open(td.path()).unwrap(); + let apnic = sample_trust_anchor(); + let arin = sample_arin_trust_anchor(); + + let cir = build_cir_from_run_multi( + &store, + &[ + CirTrustAnchorBinding { + trust_anchor: &apnic, + tal_uri: "https://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer", + }, + CirTrustAnchorBinding { + trust_anchor: &arin, + tal_uri: "https://rrdp.arin.net/arin-rpki-ta.cer", + }, + ], + sample_time(), + &[], + Some(&[]), + ) + .expect("build cir with unsorted input bindings"); + + assert_eq!( + cir.trust_anchors + .iter() + .map(|trust_anchor| trust_anchor.ta_rsync_uri.as_str()) + .collect::>(), + vec![ + "rsync://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer", + "rsync://rpki.arin.net/repository/arin-rpki-ta.cer", + ] + ); +} + +#[test] +fn build_cir_from_run_multi_exports_rejected_objects_from_error_audit_only() { + let td = tempfile::tempdir().unwrap(); + let store = RocksStore::open(td.path()).unwrap(); + let ta = sample_trust_anchor(); + let publication_points = vec![PublicationPointAudit { + objects: vec![ + crate::audit::ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/a.roa".to_string(), + sha256_hex: "11".repeat(32), + kind: crate::audit::AuditObjectKind::Roa, + result: crate::audit::AuditObjectResult::Error, + detail: Some("invalid roa".to_string()), + }, + crate::audit::ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/b.asa".to_string(), + sha256_hex: "22".repeat(32), + kind: crate::audit::AuditObjectKind::Aspa, + result: crate::audit::AuditObjectResult::Skipped, + detail: Some("skipped".to_string()), + }, + crate::audit::ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/c.roa".to_string(), + sha256_hex: "33".repeat(32), + kind: crate::audit::AuditObjectKind::Roa, + result: crate::audit::AuditObjectResult::Error, + detail: Some("second rejected roa".to_string()), + }, + ], + ..PublicationPointAudit::default() + }]; + + let cir = build_cir_from_run_multi( + &store, + &[CirTrustAnchorBinding { + trust_anchor: &ta, + tal_uri: "https://example.test/root.tal", + }], + sample_time(), + &publication_points, + None, + ) + .expect("build cir"); + + assert_eq!(cir.rejected_object_count(), 2); + assert_eq!( + cir.fresh_rejected_objects[0].object_uri, + "rsync://example.test/repo/a.roa" + ); + assert_eq!( + cir.fresh_rejected_objects[0].reason.as_deref(), + Some("invalid roa") + ); + assert_eq!( + cir.fresh_rejected_objects[1].object_uri, + "rsync://example.test/repo/c.roa" + ); + assert!( + cir.fresh_validated_objects + .iter() + .any(|item| item.rsync_uri == "rsync://example.test/repo/a.roa"), + "rejected audit objects were still consumed as validation input", + ); + assert!( + cir.fresh_validated_objects + .iter() + .any(|item| item.rsync_uri == "rsync://example.test/repo/c.roa"), + "rejected audit objects were still consumed as validation input", + ); + assert!( + !cir.fresh_validated_objects + .iter() + .any(|item| item.rsync_uri == "rsync://example.test/repo/b.asa"), + "skipped audit objects are not considered consumed input", + ); +} + +#[test] +fn build_cir_from_run_multi_records_crl_expired_manifest_in_objects_and_rejects() { + let td = tempfile::tempdir().unwrap(); + let store = RocksStore::open(td.path()).unwrap(); + let ta = sample_trust_anchor(); + let manifest_uri = "rsync://example.test/repo/expired-crl.mft"; + let reject_reason = "manifest embedded EE certificate path validation failed: CRL not valid at validation_time (RFC 5280 §6.3.3(g); RFC 5280 §5.1.2.4-§5.1.2.5; RFC 6487 §5)"; + let publication_points = vec![PublicationPointAudit { + objects: vec![crate::audit::ObjectAuditEntry { + rsync_uri: manifest_uri.to_string(), + sha256_hex: "44".repeat(32), + kind: crate::audit::AuditObjectKind::Manifest, + result: crate::audit::AuditObjectResult::Error, + detail: Some(reject_reason.to_string()), + }], + ..PublicationPointAudit::default() + }]; + + let cir = build_cir_from_run_multi( + &store, + &[CirTrustAnchorBinding { + trust_anchor: &ta, + tal_uri: "https://example.test/root.tal", + }], + sample_time(), + &publication_points, + None, + ) + .expect("build cir"); + + assert!( + cir.fresh_validated_objects + .iter() + .any(|item| item.rsync_uri == manifest_uri), + "manifest rejected because issuer CRL is expired was still read as validation input", + ); + assert_eq!(cir.rejected_object_count(), 1); + assert_eq!(cir.fresh_rejected_objects[0].object_uri, manifest_uri); + assert_eq!( + cir.fresh_rejected_objects[0].reason.as_deref(), + Some(reject_reason) + ); + assert_eq!( + cir.reject_list_sha256, + crate::cir::model::compute_reject_list_sha256([manifest_uri].into_iter()) + ); +} + +#[test] +fn build_cir_from_run_multi_excludes_manifest_locked_files_when_manifest_is_rejected() { + let td = tempfile::tempdir().unwrap(); + let store = RocksStore::open(td.path()).unwrap(); + let ta = sample_trust_anchor(); + let manifest_uri = "rsync://example.test/repo/rejected.mft"; + let roa_uri = "rsync://example.test/repo/listed.roa"; + let crl_uri = "rsync://example.test/repo/listed.crl"; + let publication_points = vec![PublicationPointAudit { + objects: vec![ + crate::audit::ObjectAuditEntry { + rsync_uri: manifest_uri.to_string(), + sha256_hex: "44".repeat(32), + kind: crate::audit::AuditObjectKind::Manifest, + result: crate::audit::AuditObjectResult::Error, + detail: Some("manifest EE cert path rejected".to_string()), + }, + crate::audit::ObjectAuditEntry { + rsync_uri: roa_uri.to_string(), + sha256_hex: "55".repeat(32), + kind: crate::audit::AuditObjectKind::Roa, + result: crate::audit::AuditObjectResult::Skipped, + detail: Some("manifest rejected before locked object validation".to_string()), + }, + crate::audit::ObjectAuditEntry { + rsync_uri: crl_uri.to_string(), + sha256_hex: "66".repeat(32), + kind: crate::audit::AuditObjectKind::Crl, + result: crate::audit::AuditObjectResult::Skipped, + detail: Some("manifest rejected before locked object validation".to_string()), + }, + ], + ..PublicationPointAudit::default() + }]; + + let cir = build_cir_from_run_multi( + &store, + &[CirTrustAnchorBinding { + trust_anchor: &ta, + tal_uri: "https://example.test/root.tal", + }], + sample_time(), + &publication_points, + None, + ) + .expect("build cir"); + + assert!( + cir.fresh_validated_objects + .iter() + .any(|item| item.rsync_uri == manifest_uri), + "rejected manifest is still a current-run validation input", + ); + assert!( + !cir.fresh_validated_objects + .iter() + .any(|item| item.rsync_uri == roa_uri), + "ROA listed by a rejected manifest must not enter CIR objects", + ); + assert!( + !cir.fresh_validated_objects + .iter() + .any(|item| item.rsync_uri == crl_uri), + "CRL listed by a rejected manifest must not enter CIR objects", + ); + assert_eq!(cir.rejected_object_count(), 1); + assert_eq!(cir.fresh_rejected_objects[0].object_uri, manifest_uri); +} + +#[test] +fn build_cir_from_run_multi_reject_digest_ignores_reason_text() { + let td = tempfile::tempdir().unwrap(); + let store = RocksStore::open(td.path()).unwrap(); + let ta = sample_trust_anchor(); + + let mk_pp = |detail: &str| PublicationPointAudit { + objects: vec![crate::audit::ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/a.roa".to_string(), + sha256_hex: "11".repeat(32), + kind: crate::audit::AuditObjectKind::Roa, + result: crate::audit::AuditObjectResult::Error, + detail: Some(detail.to_string()), + }], + ..PublicationPointAudit::default() + }; + + let cir_a = build_cir_from_run_multi( + &store, + &[CirTrustAnchorBinding { + trust_anchor: &ta, + tal_uri: "https://example.test/root.tal", + }], + sample_time(), + &[mk_pp("reason-a")], + None, + ) + .expect("build cir a"); + let cir_b = build_cir_from_run_multi( + &store, + &[CirTrustAnchorBinding { + trust_anchor: &ta, + tal_uri: "https://example.test/root.tal", + }], + sample_time(), + &[mk_pp("reason-b")], + None, + ) + .expect("build cir b"); + + assert_eq!(cir_a.reject_list_sha256, cir_b.reject_list_sha256); + assert_ne!( + cir_a.fresh_rejected_objects[0].reason, + cir_b.fresh_rejected_objects[0].reason + ); +} + +#[test] +fn build_cir_from_run_multi_rejects_invalid_tal_uri_and_missing_rsync_ta_uri() { + let td = tempfile::tempdir().unwrap(); + let store = RocksStore::open(td.path()).unwrap(); + + let err = build_cir_from_run_multi( + &store, + &[CirTrustAnchorBinding { + trust_anchor: &sample_trust_anchor(), + tal_uri: "file:///not-supported.tal", + }], + sample_time(), + &[], + None, + ) + .expect_err("non-http tal uri must fail"); + assert!(matches!(err, CirExportError::InvalidTalUri(_)), "{err}"); + + let err = build_cir_from_run_multi( + &store, + &[CirTrustAnchorBinding { + trust_anchor: &sample_trust_anchor_without_rsync_uri(), + tal_uri: "https://example.test/root.tal", + }], + sample_time(), + &[], + None, + ) + .expect_err("missing rsync ta uri must fail"); + assert!(matches!(err, CirExportError::MissingTaRsyncUri), "{err}"); +} + +#[test] +fn build_cir_from_input_snapshot_multi_preserves_online_sections() { + let ta = sample_trust_anchor(); + let cir = build_cir_from_input_snapshot_multi( + &[CirTrustAnchorBinding { + trust_anchor: &ta, + tal_uri: "https://example.test/root.tal", + }], + sample_time(), + CirInputSnapshot { + fresh_validated_objects: vec![CirObject { + rsync_uri: "rsync://example.test/repo/fresh.roa".to_string(), + sha256: vec![0x11; 32], + }], + cached_validated_objects: vec![CirObject { + rsync_uri: "rsync://example.test/repo/cached.roa".to_string(), + sha256: vec![0x22; 32], + }], + fresh_rejected_objects: vec![CirRejectedObject { + object_uri: "rsync://example.test/repo/fresh.roa".to_string(), + reason: Some("fresh rejected".to_string()), + }], + cached_rejected_objects: vec![CirRejectedObject { + object_uri: "rsync://example.test/repo/cached.roa".to_string(), + reason: None, + }], + }, + ) + .expect("build CIR from online input"); + + cir.validate().expect("online CIR snapshot is valid"); + assert_eq!(cir.fresh_validated_objects.len(), 1); + assert_eq!(cir.cached_validated_objects.len(), 1); + assert_eq!(cir.fresh_rejected_objects.len(), 1); + assert_eq!(cir.cached_rejected_objects.len(), 1); +} + +#[test] +fn export_cir_static_pool_writes_repository_objects_only() { + let td = tempfile::tempdir().unwrap(); + let store = RocksStore::open(&td.path().join("db")).unwrap(); + let static_root = td.path().join("static"); + let ta1 = sample_trust_anchor(); + let ta2 = sample_arin_trust_anchor(); + + let object_bytes = b"object-z".to_vec(); + let hash = sha256_hex(&object_bytes); + let mut raw = RawByHashEntry::from_bytes(hash.clone(), object_bytes.clone()); + raw.origin_uris + .push("rsync://example.test/repo/z.roa".into()); + store.put_raw_by_hash_entry(&raw).unwrap(); + let publication_points = vec![PublicationPointAudit { + objects: vec![audit_entry( + "rsync://example.test/repo/z.roa", + &hash, + crate::audit::AuditObjectKind::Roa, + crate::audit::AuditObjectResult::Ok, + None, + )], + ..PublicationPointAudit::default() + }]; + + let cir = build_cir_from_run_multi( + &store, + &[ + CirTrustAnchorBinding { + trust_anchor: &ta1, + tal_uri: "https://example.test/apnic.tal", + }, + CirTrustAnchorBinding { + trust_anchor: &ta2, + tal_uri: "https://example.test/arin.tal", + }, + ], + sample_time(), + &publication_points, + None, + ) + .expect("build cir"); + + let summary = export_cir_static_pool(&store, &static_root, sample_date(), &cir, &[&ta1, &ta2]) + .expect("export static pool"); + assert_eq!(summary.unique_hashes, 1); + assert_eq!(summary.written_files, 1); + for trust_anchor in &cir.trust_anchors { + let ta_hash = hex::encode(&trust_anchor.ta_certificate_sha256); + assert!( + !crate::cir::static_pool::static_pool_path(&static_root, sample_date(), &ta_hash) + .expect("static pool ta path") + .exists() + ); + } +} + +#[test] +fn export_cir_raw_store_reports_missing_non_ta_object_only() { + let td = tempfile::tempdir().unwrap(); + let raw_store_path = td.path().join("raw-store.db"); + let store = + RocksStore::open_with_external_raw_store(&td.path().join("db"), &raw_store_path).unwrap(); + let ta1 = sample_trust_anchor(); + let ta2 = sample_arin_trust_anchor(); + + let cir_only_tas = build_cir_from_run_multi( + &store, + &[ + CirTrustAnchorBinding { + trust_anchor: &ta1, + tal_uri: "https://example.test/apnic.tal", + }, + CirTrustAnchorBinding { + trust_anchor: &ta2, + tal_uri: "https://example.test/arin.tal", + }, + ], + sample_time(), + &[], + Some(&[]), + ) + .expect("build cir with tas only"); + + let summary = export_cir_raw_store(&store, &raw_store_path, &cir_only_tas, &[&ta1, &ta2]) + .expect("export raw store"); + assert_eq!(summary.unique_hashes, 0); + assert_eq!(summary.written_entries, 0); + assert_eq!(summary.reused_entries, 0); + + let mut cir_missing_object = cir_only_tas.clone(); + cir_missing_object.fresh_validated_objects.push(CirObject { + rsync_uri: "rsync://example.test/repo/missing.roa".to_string(), + sha256: vec![0x44; 32], + }); + let err = export_cir_raw_store(&store, &raw_store_path, &cir_missing_object, &[&ta1, &ta2]) + .expect_err("missing non-ta object must fail"); + assert!(matches!(err, CirExportError::Write(_, _)), "{err}"); +} diff --git a/crates/panda-rpki-validator/src/cli.rs b/crates/panda-rpki-validator/src/cli.rs index da1f8fd..d14c9eb 100644 --- a/crates/panda-rpki-validator/src/cli.rs +++ b/crates/panda-rpki-validator/src/cli.rs @@ -1,15 +1,13 @@ mod output; +mod report_tasks; -use crate::ccr::{ - CcrAccumulator, CcrBuildBreakdown, build_ccr_from_run_with_breakdown, write_ccr_file, -}; +use crate::ccr::{CcrAccumulator, CcrBuildBreakdown}; use crate::cir::{CirTrustAnchorBinding, export_cir_from_input_snapshot_multi}; use std::path::{Path, PathBuf}; use crate::analysis::timing::{ DurationStats, TimingHandle, TimingMeta, TimingMetaUpdate, TopDurationEntry, }; -use crate::audit::AuditRepoSyncStats; #[cfg(test)] use crate::audit::{ AspaOutput, AuditReportV2, AuditRunMeta, AuditWarning, TreeSummary, VrpOutput, @@ -42,2855 +40,21 @@ use crate::validation::run_tree_from_tal::{ use crate::validation::tree::{DEFAULT_MAX_CA_DEPTH, TreeRunConfig}; #[cfg(test)] use output::write_json; -use output::{ - ReportJsonFormat, run_compare_view_task, write_report_json_from_shared, write_stage_timing, +use output::{ReportJsonFormat, run_compare_view_task, write_stage_timing}; +use report_tasks::{ + ReportTaskOutput, build_repo_sync_stats, effective_cir_tal_uris_for_discoveries, + resolve_cir_export_tal_uris, run_ccr_task, run_report_task, }; use serde::Serialize; use std::collections::HashMap; use std::sync::Arc; -#[derive(Clone, Debug, PartialEq, Eq, Serialize)] -struct RunStageTiming { - validation_ms: u64, - enable_roa_validation_cache: bool, - enable_child_certificate_validation_cache: bool, - publication_point_cache_observe_only: bool, - enable_publication_point_validation_cache: bool, - crypto_signature_cache_observe: Option, - enable_transport_request_prefetch: bool, - report_build_ms: u64, - report_write_ms: Option, - ccr_build_ms: Option, - ccr_build_breakdown: Option, - ccr_write_ms: Option, - compare_view_build_ms: Option, - compare_view_write_ms: Option, - cir_build_cir_ms: Option, - cir_write_cir_ms: Option, - cir_total_ms: Option, - total_ms: u64, - publication_points: usize, - repo_sync_ms_total: u64, - publication_point_repo_sync_ms_total: u64, - download_event_count: u64, - rrdp_download_ms_total: u64, - rsync_download_ms_total: u64, - download_bytes_total: u64, - roa_validation_cache: crate::validation::objects::RoaValidationCacheStats, - analysis_counts: HashMap, - analysis_phases: HashMap, - analysis_top_publication_points: Vec, - analysis_top_publication_point_steps: Vec, - analysis_top_publication_point_cache_steps: Vec, - vcir_storage_summary_ms: Option, - vcir_storage: Option, - publication_point_cache_index_load: Option, - publication_point_cache_index_refresh: Option, - memory_telemetry: Option, -} - -fn record_memory_checkpoint( - checkpoints: &mut Vec, - label: &str, - total_started: &std::time::Instant, - store: &RocksStore, -) { - checkpoints.push(MemoryTelemetryCheckpoint { - label: label.to_string(), - elapsed_ms: total_started.elapsed().as_millis() as u64, - process: crate::memory_telemetry::process_memory_snapshot(label), - rocksdb: store.memory_snapshot(), - }); -} - -fn memory_trim_probe_enabled() -> bool { - std::env::var("RPKI_MEMORY_TRIM_PROBE") - .map(|value| matches!(value.as_str(), "1" | "true" | "TRUE" | "yes" | "YES")) - .unwrap_or(false) -} - -fn vcir_storage_summary_enabled() -> bool { - std::env::var("RPKI_VCIR_STORAGE_SUMMARY") - .map(|value| matches!(value.as_str(), "1" | "true" | "TRUE" | "yes" | "YES")) - .unwrap_or(false) -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct CliArgs { - pub validation_contract_out_path: Option, - pub tal_urls: Vec, - pub tal_paths: Vec, - pub ta_paths: Vec, - pub tal_url: Option, - pub tal_path: Option, - pub ta_path: Option, - pub parallel_phase1_config: ParallelPhase1Config, - pub parallel_phase2_config: ParallelPhase2Config, - pub tal_inputs: Vec, - pub ta_constraints: TaConstraintsByTal, - - pub db_path: PathBuf, - pub raw_store_db: Option, - pub repo_bytes_db: Option, - pub policy_path: Option, - pub strict_policy: Option, - pub resource_validation_mode: Option, - pub report_json_path: Option, - pub report_json_compact: bool, - pub skip_report_build: bool, - pub skip_vcir_persist: bool, - pub enable_roa_validation_cache: bool, - pub enable_child_certificate_validation_cache: bool, - pub publication_point_cache_observe_only: bool, - pub enable_publication_point_validation_cache: bool, - pub crypto_signature_cache_observe_only: bool, - pub enable_crypto_signature_cache: bool, - pub enable_transport_request_prefetch: bool, - pub ccr_out_path: Option, - pub vrps_csv_out_path: Option, - pub vaps_csv_out_path: Option, - pub compare_view_trust_anchor: Option, - pub cir_enabled: bool, - pub cir_out_path: Option, - pub cir_static_root: Option, - pub cir_tal_uris: Vec, - pub cir_tal_uri: Option, - pub payload_replay_archive: Option, - pub payload_replay_locks: Option, - pub payload_base_archive: Option, - pub payload_base_locks: Option, - pub payload_base_validation_time: Option, - pub payload_delta_archive: Option, - pub payload_delta_locks: Option, - pub memory_trim_after_validation: bool, - - pub rsync_local_dir: Option, - pub disable_rrdp: bool, - pub rsync_command: Option, - - pub http_timeout_secs: u64, - pub http_root_cert_paths: Vec, - pub rsync_timeout_secs: u64, - pub rsync_mirror_root: Option, - pub rsync_scope_policy: RsyncScopePolicy, - - pub max_ca_depth: usize, - pub max_instances: Option, - pub validation_time: Option, - - pub analyze: bool, - pub analysis_out_path: Option, - pub profile_cpu: bool, -} - -fn usage() -> String { - let bin = "panda-rpki-validator"; - format!( - "\ -Usage: - {bin} --db --tal-url [--tal-url ...] [options] - {bin} --db --tal-path --ta-path [--tal-path --ta-path ...] [options] - -Options: - --validation-contract-out - Write the effective normal-run validation contract - --db RocksDB directory path (required) - --raw-store-db External raw-by-hash store DB path (optional) - --repo-bytes-db External repo object bytes DB path (optional) - --policy Policy TOML path (optional) - --ta-constraints = - Apply local EE-resource constraints to one TAL (repeatable); adjacent .constraints files are auto-discovered - --strict [policies] Enable strict policies (default all; comma list: name,cms-der,signed-attrs; none disables) - --resource-validation-mode - Resource certificate validation mode (default: validation-update-03) - --report-json Write full audit report as JSON (optional) - --report-json-compact Write report JSON without pretty-printing (requires --report-json) - --skip-report-build Skip full audit report construction when --report-json is not requested - --skip-vcir-persist Skip VCIR persistence/projection building for compare-only runs - --enable-roa-validation-cache - Reuse accepted ROA validation outputs from previous VCIR records (default: off) - --enable-child-certificate-validation-cache - Experimental: reuse validated child certificate discovery results - --publication-point-cache-observe-only - Evaluate publication-point cache eligibility without changing results - --enable-publication-point-validation-cache - Experimental: reuse complete publication-point validation projections - --crypto-signature-cache-observe-only - Measure crypto signature cache key hit rates and verify durations - without changing validation behavior (default: off) - --enable-crypto-signature-cache - Experimental: skip cryptographic signature verification on cache - hits (positive conclusions only; default: off) - --enable-transport-request-prefetch - Experimental: prefetch previous run transport repo requests before tree traversal - --ccr-out Write CCR DER ContentInfo to this path (optional) - --vrps-csv-out Write VRP compare-view CSV directly from validation output (optional; requires --vaps-csv-out) - --vaps-csv-out Write VAP compare-view CSV directly from validation output (optional; requires --vrps-csv-out) - --compare-view-trust-anchor - Trust-anchor label used by direct compare-view CSV output (default: unknown) - --cir-enable Export CIR after the run completes - --cir-out Write CIR DER to this path (requires --cir-enable) - --cir-static-root Deprecated; CIR export no longer exports object pools - --cir-tal-uri Override TAL URI for CIR export (repeatable in multi-TAL mode) - --payload-replay-archive Use local payload replay archive root (offline replay mode) - --payload-replay-locks Use local payload replay locks.json (offline replay mode) - --payload-base-archive Use local base payload archive root (offline delta replay) - --payload-base-locks Use local base locks.json (offline delta replay) - --payload-base-validation-time Validation time for the base bootstrap inside offline delta replay - --payload-delta-archive Use local delta payload archive root (offline delta replay) - --payload-delta-locks Use local locks-delta.json (offline delta replay) - --memory-trim-after-validation Call malloc_trim(0) after validation/report memory checkpoints (Linux glibc only; default off) - - --tal-url TAL URL (repeatable; URL mode) - --tal-path TAL file path (repeatable; file mode) - --ta-path TA certificate DER file path (repeatable in file mode; pairs with --tal-path by position) - --parallel-max-repo-sync-workers-global - Phase 1 global repo sync worker budget (default: 4) - --parallel-max-inflight-snapshot-bytes-global - Phase 1 inflight snapshot byte budget (default: 512MiB) - --parallel-max-pending-repo-results - Phase 1 pending repo result budget (default: 1024) - --parallel-phase2-object-workers - Phase 2 object worker count (default: 8) - --parallel-phase2-worker-queue-capacity - Phase 2 per-worker object queue capacity (default: 256) - --parallel-phase2-ready-batch-size - Phase 2 ready publication points processed per scheduler turn (default: 256) - --parallel-phase2-ready-batch-wall-time-budget-ms - Phase 2 ready staging wall-time budget per scheduler turn (default: 100) - --parallel-phase2-result-drain-batch-size - Phase 2 object results drained per scheduler turn (default: 2048) - --parallel-phase2-finalize-batch-size - Legacy Phase 2 scheduler finalize budget; dedicated finalize worker ignores it (default: 256) - --parallel-phase2-finalize-batch-wall-time-budget-ms - Legacy Phase 2 scheduler finalize time budget; dedicated finalize worker ignores it (default: 100) - --parallel-phase2-finalize-queue-capacity - Phase 2 dedicated finalize worker queue capacity (default: 32768) - --control-plane-stage-workers - Experimental: Phase 2 ready publication point stage worker count; - 0 disables the stage pool and keeps inline staging (default: 0) - --dead-repo-blacklist - Enable the dead-repo transport blacklist persisted at this JSON - path (default: disabled). Blacklisted rrdp repos skip straight to - rsync; dual-blacklisted repos terminate instantly. - --dead-repo-blacklist-fail-threshold - Consecutive runs with transport-class fetch failure before a - (repo, transport) entry is blacklisted (default: 3) - - --rsync-local-dir Use LocalDirRsyncFetcher rooted at this directory (offline tests) - --disable-rrdp Disable RRDP and synchronize only via rsync - --rsync-command Use this rsync command instead of the default rsync binary - --http-timeout-secs HTTP fetch timeout seconds (default: 20) - --http-root-cert Extra PEM root certificate trusted by HTTPS fetches (repeatable) - --rsync-timeout-secs rsync I/O timeout seconds (default: 60) - --rsync-mirror-root Persist rsync mirrors under this directory (default: disabled) - --rsync-scope rsync scope policy: host, publication-point, or module-root (default: module-root) - --max-ca-depth Maximum CA depth from a trust anchor (root = 0, default: {DEFAULT_MAX_CA_DEPTH}) - --max-depth Deprecated alias for --max-ca-depth - --max-instances Max number of CA instances to process - --validation-time Validation time in RFC3339 (default: now UTC) - --analyze Write timing analysis JSON under target/live/analyze// - --analysis-out Write timing analysis JSON under this directory (implies --analyze) - --profile-cpu (Requires build feature 'profile') Write CPU flamegraph under analyze dir - - --help Show this help -" - ) -} - -pub fn parse_args(argv: &[String]) -> Result { - let mut validation_contract_out_path: Option = None; - let mut tal_urls: Vec = Vec::new(); - let mut tal_paths: Vec = Vec::new(); - let mut ta_paths: Vec = Vec::new(); - let mut ta_constraint_specs: Vec = Vec::new(); - let mut parallel_phase1_cfg = ParallelPhase1Config::default(); - let mut parallel_phase2_cfg = ParallelPhase2Config::default(); - let mut dead_repo_blacklist_path: Option = None; - let mut dead_repo_blacklist_fail_threshold: Option = None; - - let mut db_path: Option = None; - let mut raw_store_db: Option = None; - let mut repo_bytes_db: Option = None; - let mut policy_path: Option = None; - let mut strict_policy: Option = None; - let mut resource_validation_mode: Option = None; - let mut report_json_path: Option = None; - let mut report_json_compact: bool = false; - let mut skip_report_build: bool = false; - let mut skip_vcir_persist: bool = false; - let mut enable_roa_validation_cache: bool = false; - let mut enable_child_certificate_validation_cache: bool = false; - let mut publication_point_cache_observe_only: bool = false; - let mut crypto_signature_cache_observe_only: bool = false; - let mut enable_crypto_signature_cache: bool = false; - let mut enable_publication_point_validation_cache: bool = false; - let mut enable_transport_request_prefetch: bool = false; - let mut ccr_out_path: Option = None; - let mut vrps_csv_out_path: Option = None; - let mut vaps_csv_out_path: Option = None; - let mut compare_view_trust_anchor: Option = None; - let mut cir_enabled: bool = false; - let mut cir_out_path: Option = None; - let mut cir_static_root: Option = None; - let mut cir_tal_uris: Vec = Vec::new(); - let mut cir_tal_uri: Option = None; - let mut payload_replay_archive: Option = None; - let mut payload_replay_locks: Option = None; - let mut payload_base_archive: Option = None; - let mut payload_base_locks: Option = None; - let mut payload_base_validation_time: Option = None; - let mut payload_delta_archive: Option = None; - let mut payload_delta_locks: Option = None; - let mut memory_trim_after_validation = false; - - let mut rsync_local_dir: Option = None; - let mut disable_rrdp: bool = false; - let mut rsync_command: Option = None; - let mut http_timeout_secs: u64 = 30; - let mut http_root_cert_paths: Vec = Vec::new(); - let mut rsync_timeout_secs: u64 = 30; - let mut rsync_mirror_root: Option = None; - let mut rsync_scope_policy = RsyncScopePolicy::default(); - let mut max_ca_depth: Option = None; - let mut max_ca_depth_option: Option<&'static str> = None; - let mut max_instances: Option = None; - let mut validation_time: Option = None; - let mut analyze: bool = false; - let mut analysis_out_path: Option = None; - let mut profile_cpu: bool = false; - - let mut i = 1usize; - while i < argv.len() { - let arg = argv[i].as_str(); - match arg { - "--help" | "-h" => return Err(usage()), - "--validation-contract-out" => { - i += 1; - validation_contract_out_path = Some(PathBuf::from( - argv.get(i) - .ok_or("--validation-contract-out requires a value")?, - )); - } - "--tal-url" => { - i += 1; - let v = argv.get(i).ok_or("--tal-url requires a value")?; - tal_urls.push(v.clone()); - } - "--tal-path" => { - i += 1; - let v = argv.get(i).ok_or("--tal-path requires a value")?; - tal_paths.push(PathBuf::from(v)); - } - "--ta-path" => { - i += 1; - let v = argv.get(i).ok_or("--ta-path requires a value")?; - ta_paths.push(PathBuf::from(v)); - } - "--ta-constraints" => { - i += 1; - let v = argv - .get(i) - .ok_or("--ta-constraints requires =")?; - ta_constraint_specs.push(v.clone()); - } - "--parallel-max-repo-sync-workers-global" => { - i += 1; - let v = argv - .get(i) - .ok_or("--parallel-max-repo-sync-workers-global requires a value")?; - parallel_phase1_cfg.max_repo_sync_workers_global = v - .parse::() - .map_err(|_| format!("invalid --parallel-max-repo-sync-workers-global: {v}"))?; - } - "--parallel-max-inflight-snapshot-bytes-global" => { - i += 1; - let v = argv - .get(i) - .ok_or("--parallel-max-inflight-snapshot-bytes-global requires a value")?; - parallel_phase1_cfg.max_inflight_snapshot_bytes_global = - v.parse::().map_err(|_| { - format!("invalid --parallel-max-inflight-snapshot-bytes-global: {v}") - })?; - } - "--parallel-max-pending-repo-results" => { - i += 1; - let v = argv - .get(i) - .ok_or("--parallel-max-pending-repo-results requires a value")?; - parallel_phase1_cfg.max_pending_repo_results = v - .parse::() - .map_err(|_| format!("invalid --parallel-max-pending-repo-results: {v}"))?; - } - "--parallel-phase2-object-workers" => { - i += 1; - let v = argv - .get(i) - .ok_or("--parallel-phase2-object-workers requires a value")?; - parallel_phase2_cfg.object_workers = v - .parse::() - .map_err(|_| format!("invalid --parallel-phase2-object-workers: {v}"))?; - } - "--parallel-phase2-worker-queue-capacity" => { - i += 1; - let v = argv - .get(i) - .ok_or("--parallel-phase2-worker-queue-capacity requires a value")?; - parallel_phase2_cfg.worker_queue_capacity = v - .parse::() - .map_err(|_| format!("invalid --parallel-phase2-worker-queue-capacity: {v}"))?; - } - "--parallel-phase2-ready-batch-size" => { - i += 1; - let v = argv - .get(i) - .ok_or("--parallel-phase2-ready-batch-size requires a value")?; - parallel_phase2_cfg.ready_batch_size = v - .parse::() - .map_err(|_| format!("invalid --parallel-phase2-ready-batch-size: {v}"))?; - } - "--parallel-phase2-ready-batch-wall-time-budget-ms" => { - i += 1; - let v = argv - .get(i) - .ok_or("--parallel-phase2-ready-batch-wall-time-budget-ms requires a value")?; - parallel_phase2_cfg.ready_batch_wall_time_budget_ms = - v.parse::().map_err(|_| { - format!("invalid --parallel-phase2-ready-batch-wall-time-budget-ms: {v}") - })?; - } - "--parallel-phase2-result-drain-batch-size" => { - i += 1; - let v = argv - .get(i) - .ok_or("--parallel-phase2-result-drain-batch-size requires a value")?; - parallel_phase2_cfg.object_result_drain_batch_size = - v.parse::().map_err(|_| { - format!("invalid --parallel-phase2-result-drain-batch-size: {v}") - })?; - } - "--parallel-phase2-finalize-batch-size" => { - i += 1; - let v = argv - .get(i) - .ok_or("--parallel-phase2-finalize-batch-size requires a value")?; - parallel_phase2_cfg.publication_point_finalize_batch_size = v - .parse::() - .map_err(|_| format!("invalid --parallel-phase2-finalize-batch-size: {v}"))?; - } - "--parallel-phase2-finalize-batch-wall-time-budget-ms" => { - i += 1; - let v = argv.get(i).ok_or( - "--parallel-phase2-finalize-batch-wall-time-budget-ms requires a value", - )?; - parallel_phase2_cfg.publication_point_finalize_wall_time_budget_ms = - v.parse::().map_err(|_| { - format!("invalid --parallel-phase2-finalize-batch-wall-time-budget-ms: {v}") - })?; - } - "--parallel-phase2-finalize-queue-capacity" => { - i += 1; - let v = argv - .get(i) - .ok_or("--parallel-phase2-finalize-queue-capacity requires a value")?; - parallel_phase2_cfg.publication_point_finalize_queue_capacity = - v.parse::().map_err(|_| { - format!("invalid --parallel-phase2-finalize-queue-capacity: {v}") - })?; - } - "--control-plane-stage-workers" => { - i += 1; - let v = argv - .get(i) - .ok_or("--control-plane-stage-workers requires a value")?; - parallel_phase2_cfg.stage_workers = v - .parse::() - .map_err(|_| format!("invalid --control-plane-stage-workers: {v}"))?; - } - "--dead-repo-blacklist" => { - i += 1; - let v = argv - .get(i) - .ok_or("--dead-repo-blacklist requires a value")?; - dead_repo_blacklist_path = Some(PathBuf::from(v)); - } - "--dead-repo-blacklist-fail-threshold" => { - i += 1; - let v = argv - .get(i) - .ok_or("--dead-repo-blacklist-fail-threshold requires a value")?; - dead_repo_blacklist_fail_threshold = - Some(v.parse::().map_err(|_| { - format!("invalid --dead-repo-blacklist-fail-threshold: {v}") - })?); - } - "--db" => { - i += 1; - let v = argv.get(i).ok_or("--db requires a value")?; - db_path = Some(PathBuf::from(v)); - } - "--raw-store-db" => { - i += 1; - let v = argv.get(i).ok_or("--raw-store-db requires a value")?; - raw_store_db = Some(PathBuf::from(v)); - } - "--repo-bytes-db" => { - i += 1; - let v = argv.get(i).ok_or("--repo-bytes-db requires a value")?; - repo_bytes_db = Some(PathBuf::from(v)); - } - "--policy" => { - i += 1; - let v = argv.get(i).ok_or("--policy requires a value")?; - policy_path = Some(PathBuf::from(v)); - } - "--strict" => { - let next = argv.get(i + 1).map(String::as_str); - let spec = next.filter(|v| !v.starts_with("--")); - if spec.is_some() { - i += 1; - } - strict_policy = Some(StrictPolicy::parse_cli_spec(spec)?); - } - _ if arg.starts_with("--strict=") => { - let spec = arg.strip_prefix("--strict=").expect("prefix checked"); - strict_policy = Some(StrictPolicy::parse_cli_spec(Some(spec))?); - } - "--resource-validation-mode" => { - i += 1; - let v = argv - .get(i) - .ok_or("--resource-validation-mode requires a value")?; - resource_validation_mode = Some(ResourceValidationMode::parse_cli_value(v)?); - } - _ if arg.starts_with("--resource-validation-mode=") => { - let spec = arg - .strip_prefix("--resource-validation-mode=") - .expect("prefix checked"); - resource_validation_mode = Some(ResourceValidationMode::parse_cli_value(spec)?); - } - "--report-json" => { - i += 1; - let v = argv.get(i).ok_or("--report-json requires a value")?; - report_json_path = Some(PathBuf::from(v)); - } - "--report-json-compact" => { - report_json_compact = true; - } - "--skip-report-build" => { - skip_report_build = true; - } - "--skip-vcir-persist" => { - skip_vcir_persist = true; - } - "--enable-roa-validation-cache" => { - enable_roa_validation_cache = true; - } - "--enable-child-certificate-validation-cache" => { - enable_child_certificate_validation_cache = true; - } - "--publication-point-cache-observe-only" => { - publication_point_cache_observe_only = true; - } - "--enable-publication-point-validation-cache" => { - enable_publication_point_validation_cache = true; - } - "--crypto-signature-cache-observe-only" => { - crypto_signature_cache_observe_only = true; - } - "--enable-crypto-signature-cache" => { - enable_crypto_signature_cache = true; - } - "--enable-transport-request-prefetch" => { - enable_transport_request_prefetch = true; - } - "--ccr-out" => { - i += 1; - let v = argv.get(i).ok_or("--ccr-out requires a value")?; - ccr_out_path = Some(PathBuf::from(v)); - } - "--vrps-csv-out" => { - i += 1; - let v = argv.get(i).ok_or("--vrps-csv-out requires a value")?; - vrps_csv_out_path = Some(PathBuf::from(v)); - } - "--vaps-csv-out" => { - i += 1; - let v = argv.get(i).ok_or("--vaps-csv-out requires a value")?; - vaps_csv_out_path = Some(PathBuf::from(v)); - } - "--compare-view-trust-anchor" => { - i += 1; - let v = argv - .get(i) - .ok_or("--compare-view-trust-anchor requires a value")?; - compare_view_trust_anchor = Some(v.clone()); - } - "--cir-enable" => { - cir_enabled = true; - } - "--cir-out" => { - i += 1; - let v = argv.get(i).ok_or("--cir-out requires a value")?; - cir_out_path = Some(PathBuf::from(v)); - } - "--cir-static-root" => { - i += 1; - let v = argv.get(i).ok_or("--cir-static-root requires a value")?; - cir_static_root = Some(PathBuf::from(v)); - } - "--cir-tal-uri" => { - i += 1; - let v = argv.get(i).ok_or("--cir-tal-uri requires a value")?; - cir_tal_uris.push(v.clone()); - cir_tal_uri = cir_tal_uris.first().cloned(); - } - "--payload-replay-archive" => { - i += 1; - let v = argv - .get(i) - .ok_or("--payload-replay-archive requires a value")?; - payload_replay_archive = Some(PathBuf::from(v)); - } - "--payload-replay-locks" => { - i += 1; - let v = argv - .get(i) - .ok_or("--payload-replay-locks requires a value")?; - payload_replay_locks = Some(PathBuf::from(v)); - } - "--payload-base-archive" => { - i += 1; - let v = argv - .get(i) - .ok_or("--payload-base-archive requires a value")?; - payload_base_archive = Some(PathBuf::from(v)); - } - "--payload-base-locks" => { - i += 1; - let v = argv.get(i).ok_or("--payload-base-locks requires a value")?; - payload_base_locks = Some(PathBuf::from(v)); - } - "--payload-base-validation-time" => { - i += 1; - let v = argv - .get(i) - .ok_or("--payload-base-validation-time requires a value")?; - use time::format_description::well_known::Rfc3339; - let t = time::OffsetDateTime::parse(v, &Rfc3339).map_err(|e| { - format!("invalid --payload-base-validation-time (RFC3339 expected): {e}") - })?; - payload_base_validation_time = Some(t); - } - "--payload-delta-archive" => { - i += 1; - let v = argv - .get(i) - .ok_or("--payload-delta-archive requires a value")?; - payload_delta_archive = Some(PathBuf::from(v)); - } - "--payload-delta-locks" => { - i += 1; - let v = argv - .get(i) - .ok_or("--payload-delta-locks requires a value")?; - payload_delta_locks = Some(PathBuf::from(v)); - } - "--memory-trim-after-validation" => { - memory_trim_after_validation = true; - } - "--rsync-local-dir" => { - i += 1; - let v = argv.get(i).ok_or("--rsync-local-dir requires a value")?; - rsync_local_dir = Some(PathBuf::from(v)); - } - "--disable-rrdp" => { - disable_rrdp = true; - } - "--rsync-command" => { - i += 1; - let v = argv.get(i).ok_or("--rsync-command requires a value")?; - rsync_command = Some(PathBuf::from(v)); - } - "--http-timeout-secs" => { - i += 1; - let v = argv.get(i).ok_or("--http-timeout-secs requires a value")?; - http_timeout_secs = v - .parse::() - .map_err(|_| format!("invalid --http-timeout-secs: {v}"))?; - } - "--http-root-cert" => { - i += 1; - let v = argv.get(i).ok_or("--http-root-cert requires a value")?; - http_root_cert_paths.push(PathBuf::from(v)); - } - "--rsync-timeout-secs" => { - i += 1; - let v = argv.get(i).ok_or("--rsync-timeout-secs requires a value")?; - rsync_timeout_secs = v - .parse::() - .map_err(|_| format!("invalid --rsync-timeout-secs: {v}"))?; - } - "--rsync-mirror-root" => { - i += 1; - let v = argv.get(i).ok_or("--rsync-mirror-root requires a value")?; - rsync_mirror_root = Some(PathBuf::from(v)); - } - "--rsync-scope" => { - i += 1; - let v = argv.get(i).ok_or("--rsync-scope requires a value")?; - rsync_scope_policy = RsyncScopePolicy::parse_cli_value(v)?; - } - "--max-ca-depth" | "--max-depth" => { - i += 1; - let option = arg; - let v = argv - .get(i) - .ok_or_else(|| format!("{option} requires a value"))?; - if let Some(previous_option) = max_ca_depth_option { - return Err(format!( - "{option} cannot be combined with {previous_option}; use only --max-ca-depth" - )); - } - max_ca_depth = Some( - v.parse::() - .map_err(|_| format!("invalid {option}: {v}"))?, - ); - max_ca_depth_option = Some(match option { - "--max-ca-depth" => "--max-ca-depth", - "--max-depth" => "--max-depth", - _ => unreachable!("matched CA depth option"), - }); - } - "--max-instances" => { - i += 1; - let v = argv.get(i).ok_or("--max-instances requires a value")?; - max_instances = Some( - v.parse::() - .map_err(|_| format!("invalid --max-instances: {v}"))?, - ); - } - "--validation-time" => { - i += 1; - let v = argv.get(i).ok_or("--validation-time requires a value")?; - use time::format_description::well_known::Rfc3339; - let t = time::OffsetDateTime::parse(v, &Rfc3339) - .map_err(|e| format!("invalid --validation-time (RFC3339 expected): {e}"))?; - validation_time = Some(t); - } - "--analyze" => { - analyze = true; - } - "--analysis-out" => { - i += 1; - let v = argv.get(i).ok_or("--analysis-out requires a value")?; - analyze = true; - analysis_out_path = Some(PathBuf::from(v)); - } - "--profile-cpu" => { - profile_cpu = true; - } - _ => return Err(format!("unknown argument: {arg}\n\n{}", usage())), - } - i += 1; - } - - let db_path = db_path.ok_or_else(|| format!("--db is required\n\n{}", usage()))?; - - let tal_mode_count = (!tal_urls.is_empty()) as u8 + (!tal_paths.is_empty()) as u8; - if tal_mode_count != 1 { - return Err(format!( - "must specify either one-or-more --tal-url or one-or-more --tal-path/--ta-path pairs\n\n{}", - usage() - )); - } - if parallel_phase2_cfg.object_workers == 0 { - return Err(format!( - "--parallel-phase2-object-workers must be > 0\n\n{}", - usage() - )); - } - if parallel_phase2_cfg.worker_queue_capacity == 0 { - return Err(format!( - "--parallel-phase2-worker-queue-capacity must be > 0\n\n{}", - usage() - )); - } - if parallel_phase2_cfg.ready_batch_size == 0 { - return Err(format!( - "--parallel-phase2-ready-batch-size must be > 0\n\n{}", - usage() - )); - } - if parallel_phase2_cfg.ready_batch_wall_time_budget_ms == 0 { - return Err(format!( - "--parallel-phase2-ready-batch-wall-time-budget-ms must be > 0\n\n{}", - usage() - )); - } - if parallel_phase2_cfg.object_result_drain_batch_size == 0 { - return Err(format!( - "--parallel-phase2-result-drain-batch-size must be > 0\n\n{}", - usage() - )); - } - if parallel_phase2_cfg.publication_point_finalize_batch_size == 0 { - return Err(format!( - "--parallel-phase2-finalize-batch-size must be > 0\n\n{}", - usage() - )); - } - if parallel_phase2_cfg.publication_point_finalize_wall_time_budget_ms == 0 { - return Err(format!( - "--parallel-phase2-finalize-batch-wall-time-budget-ms must be > 0\n\n{}", - usage() - )); - } - if parallel_phase2_cfg.publication_point_finalize_queue_capacity == 0 { - return Err(format!( - "--parallel-phase2-finalize-queue-capacity must be > 0\n\n{}", - usage() - )); - } - if !tal_urls.is_empty() && !ta_paths.is_empty() { - return Err(format!( - "--ta-path cannot be used with --tal-url mode\n\n{}", - usage() - )); - } - if !tal_paths.is_empty() { - if !ta_paths.is_empty() { - if ta_paths.len() != tal_paths.len() { - return Err(format!( - "--tal-path and --ta-path counts must match in file mode\n\n{}", - usage() - )); - } - } else if ta_paths.is_empty() && !disable_rrdp { - return Err(format!( - "--tal-path requires --ta-path unless --disable-rrdp is set\n\n{}", - usage() - )); - } - } - let tal_url = tal_urls.first().cloned(); - let tal_path = tal_paths.first().cloned(); - let ta_path = ta_paths.first().cloned(); - if cir_enabled && cir_out_path.is_none() { - return Err(format!("--cir-enable requires --cir-out\n\n{}", usage())); - } - if report_json_compact && report_json_path.is_none() { - return Err(format!( - "--report-json-compact requires --report-json\n\n{}", - usage() - )); - } - if skip_report_build && report_json_path.is_some() { - return Err(format!( - "--skip-report-build cannot be combined with --report-json\n\n{}", - usage() - )); - } - if vrps_csv_out_path.is_some() != vaps_csv_out_path.is_some() { - return Err(format!( - "--vrps-csv-out and --vaps-csv-out must be provided together\n\n{}", - usage() - )); - } - if compare_view_trust_anchor.is_some() && vrps_csv_out_path.is_none() { - return Err(format!( - "--compare-view-trust-anchor requires --vrps-csv-out/--vaps-csv-out\n\n{}", - usage() - )); - } - if cir_static_root.is_some() { - return Err(format!( - "--cir-static-root is no longer supported; CIR export now writes only .cir files\n\n{}", - usage() - )); - } - if !cir_enabled && (cir_out_path.is_some() || !cir_tal_uris.is_empty()) { - return Err(format!( - "--cir-out/--cir-tal-uri require --cir-enable\n\n{}", - usage() - )); - } - if cir_enabled && !cir_tal_uris.is_empty() { - let expected = if !tal_paths.is_empty() { - tal_paths.len() - } else { - tal_urls.len() - }; - if cir_tal_uris.len() != expected { - return Err(format!( - "--cir-tal-uri count must match TAL input count when provided\n\n{}", - usage() - )); - } - } - if cir_enabled && !tal_paths.is_empty() && cir_tal_uris.is_empty() { - return Err(format!( - "CIR export in --tal-path mode requires --cir-tal-uri for each TAL\n\n{}", - usage() - )); - } - - let replay_mode_count = - payload_replay_archive.is_some() as u8 + payload_replay_locks.is_some() as u8; - if replay_mode_count == 1 { - return Err(format!( - "--payload-replay-archive and --payload-replay-locks must be provided together - -{}", - usage() - )); - } - - let delta_mode_count = payload_base_archive.is_some() as u8 - + payload_base_locks.is_some() as u8 - + payload_delta_archive.is_some() as u8 - + payload_delta_locks.is_some() as u8; - if delta_mode_count > 0 && delta_mode_count < 4 { - return Err(format!( - "--payload-base-archive, --payload-base-locks, --payload-delta-archive and --payload-delta-locks must be provided together - -{}", - usage() - )); - } - if replay_mode_count == 2 && delta_mode_count == 4 { - return Err(format!( - "snapshot replay mode and delta replay mode are mutually exclusive - -{}", - usage() - )); - } - if replay_mode_count == 2 { - if tal_url.is_some() { - return Err(format!( - "payload replay mode requires --tal-path and --ta-path; --tal-url is not supported - -{}", - usage() - )); - } - if tal_path.is_none() || ta_path.is_none() { - return Err(format!( - "payload replay mode requires --tal-path and --ta-path - -{}", - usage() - )); - } - if rsync_local_dir.is_some() { - return Err(format!( - "payload replay mode cannot be combined with --rsync-local-dir - -{}", - usage() - )); - } - } - if delta_mode_count == 4 { - if tal_url.is_some() { - return Err(format!( - "payload delta replay mode requires --tal-path and --ta-path; --tal-url is not supported - -{}", - usage() - )); - } - if tal_path.is_none() || ta_path.is_none() { - return Err(format!( - "payload delta replay mode requires --tal-path and --ta-path - -{}", - usage() - )); - } - if rsync_local_dir.is_some() { - return Err(format!( - "payload delta replay mode cannot be combined with --rsync-local-dir - -{}", - usage() - )); - } - } - - let mut tal_inputs = Vec::new(); - if !tal_urls.is_empty() { - tal_inputs.extend(tal_urls.iter().cloned().map(TalInputSpec::from_url)); - } else if !tal_paths.is_empty() { - if ta_paths.len() == tal_paths.len() { - tal_inputs.extend(tal_paths.iter().cloned().zip(ta_paths.iter().cloned()).map( - |(tal_path, ta_path)| TalInputSpec::from_file_path_with_ta(tal_path, ta_path), - )); - } else { - tal_inputs.extend(tal_paths.iter().cloned().map(TalInputSpec::from_file_path)); - } - } - let ta_constraints = TaConstraintsByTal::load_for_tals(&tal_inputs, &ta_constraint_specs)?; - - if dead_repo_blacklist_fail_threshold.is_some() && dead_repo_blacklist_path.is_none() { - return Err( - "--dead-repo-blacklist-fail-threshold requires --dead-repo-blacklist".to_string(), - ); - } - if let Some(path) = dead_repo_blacklist_path { - let fail_threshold = dead_repo_blacklist_fail_threshold.unwrap_or(3); - if fail_threshold == 0 { - return Err("--dead-repo-blacklist-fail-threshold must be >= 1".to_string()); - } - parallel_phase1_cfg.dead_repo_blacklist = Some( - crate::parallel::dead_repo_blacklist::DeadRepoBlacklistConfig { - path, - fail_threshold, - capacity: - crate::parallel::dead_repo_blacklist::DEAD_REPO_BLACKLIST_DEFAULT_CAPACITY, - }, - ); - } - - Ok(CliArgs { - validation_contract_out_path, - tal_urls, - tal_paths, - ta_paths, - tal_url, - tal_path, - ta_path, - parallel_phase1_config: parallel_phase1_cfg, - parallel_phase2_config: parallel_phase2_cfg, - tal_inputs, - ta_constraints, - db_path, - raw_store_db, - repo_bytes_db, - policy_path, - strict_policy, - resource_validation_mode, - report_json_path, - report_json_compact, - skip_report_build, - skip_vcir_persist, - enable_roa_validation_cache, - enable_child_certificate_validation_cache, - publication_point_cache_observe_only, - enable_publication_point_validation_cache, - crypto_signature_cache_observe_only, - enable_crypto_signature_cache, - enable_transport_request_prefetch, - ccr_out_path, - vrps_csv_out_path, - vaps_csv_out_path, - compare_view_trust_anchor, - cir_enabled, - cir_out_path, - cir_static_root, - cir_tal_uris, - cir_tal_uri, - payload_replay_archive, - payload_replay_locks, - payload_base_archive, - payload_base_locks, - payload_base_validation_time, - payload_delta_archive, - payload_delta_locks, - memory_trim_after_validation, - rsync_local_dir, - disable_rrdp, - rsync_command, - http_timeout_secs, - http_root_cert_paths, - rsync_timeout_secs, - rsync_mirror_root, - rsync_scope_policy, - max_ca_depth: max_ca_depth.unwrap_or(DEFAULT_MAX_CA_DEPTH), - max_instances, - validation_time, - analyze, - analysis_out_path, - profile_cpu, - }) -} - -fn read_policy(path: Option<&Path>) -> Result { - match path { - None => Ok(Policy::default()), - Some(p) => { - let s = std::fs::read_to_string(p) - .map_err(|e| format!("read policy file failed: {}: {e}", p.display()))?; - Policy::from_toml_str(&s).map_err(|e| e.to_string()) - } - } -} - -fn unique_rrdp_repos_from_publication_points( - publication_points: &[crate::audit::PublicationPointAudit], -) -> usize { - use std::collections::HashSet; - let mut set: HashSet<&str> = HashSet::new(); - for pp in publication_points { - if let Some(u) = pp.rrdp_notification_uri.as_deref() { - set.insert(u); - } - } - set.len() -} - -#[cfg(test)] -fn unique_rrdp_repos(report: &AuditReportV2) -> usize { - unique_rrdp_repos_from_publication_points(&report.publication_points) -} - -#[cfg(test)] -fn print_summary(report: &AuditReportV2) { - let rrdp_repos = unique_rrdp_repos(report); - println!("RPKI stage2 serial run summary"); - println!( - "validation_time={}", - report.meta.validation_time_rfc3339_utc - ); - println!( - "publication_points_processed={} publication_points_failed={}", - report.tree.instances_processed, report.tree.instances_failed - ); - println!("rrdp_repos_unique={rrdp_repos}"); - println!("vrps={}", report.vrps.len()); - println!("aspas={}", report.aspas.len()); - println!( - "audit_publication_points={}", - report.publication_points.len() - ); - println!( - "warnings_total={}", - report.tree.warnings.len() - + report - .publication_points - .iter() - .map(|pp| pp.warnings.len()) - .sum::() - ); -} - -fn print_summary_from_shared(validation_time: time::OffsetDateTime, shared: &PostValidationShared) { - use time::format_description::well_known::Rfc3339; - let validation_time_rfc3339_utc = validation_time - .to_offset(time::UtcOffset::UTC) - .format(&Rfc3339) - .expect("format validation_time"); - let rrdp_repos = unique_rrdp_repos_from_publication_points(shared.publication_points.as_ref()); - println!("RPKI stage2 serial run summary"); - println!("validation_time={validation_time_rfc3339_utc}"); - println!( - "publication_points_processed={} publication_points_failed={}", - shared.instances_processed, shared.instances_failed - ); - println!("rrdp_repos_unique={rrdp_repos}"); - println!("vrps={}", shared.vrps.len()); - println!("aspas={}", shared.aspas.len()); - println!( - "audit_publication_points={}", - shared.publication_points.len() - ); - println!( - "warnings_total={}", - shared.tree_warnings.len() - + shared - .publication_points - .iter() - .map(|pp| pp.warnings.len()) - .sum::() - ); -} - -#[derive(Clone, Debug, PartialEq, Eq)] -struct PostValidationShared { - discovery: crate::validation::from_tal::DiscoveredRootCaInstance, - discoveries: Arc<[crate::validation::from_tal::DiscoveredRootCaInstance]>, - successful_tal_inputs: Arc<[TalInputSpec]>, - instances_processed: usize, - instances_failed: usize, - tree_warnings: Arc<[crate::report::Warning]>, - vrps: Arc<[crate::validation::objects::Vrp]>, - aspas: Arc<[crate::validation::objects::AspaAttestation]>, - router_keys: Arc<[crate::validation::objects::RouterKeyPayload]>, - publication_points: Arc<[crate::audit::PublicationPointAudit]>, - roa_cache_stats: crate::validation::objects::RoaValidationCacheStats, - downloads: Arc<[crate::audit::AuditDownloadEvent]>, - download_stats: crate::audit::AuditDownloadStats, - current_repo_objects: Arc<[crate::current_repo_index::CurrentRepoObject]>, - ccr_accumulator: Option, - cir_input: crate::cir::CirInputSnapshot, -} - -impl PostValidationShared { - fn from_run_output(out: RunTreeFromTalAuditOutput) -> Self { - let RunTreeFromTalAuditOutput { - discovery, - discoveries, - successful_tal_inputs, - tree, - publication_points, - roa_cache_stats, - downloads, - download_stats, - current_repo_objects, - ccr_accumulator, - cir_input, - } = out; - let crate::validation::tree::TreeRunOutput { - instances_processed, - instances_failed, - warnings, - vrps, - aspas, - router_keys, - } = tree; - - Self { - discovery, - discoveries: discoveries.into(), - successful_tal_inputs: successful_tal_inputs.into(), - instances_processed, - instances_failed, - tree_warnings: warnings.into(), - vrps: vrps.into(), - aspas: aspas.into(), - router_keys: router_keys.into(), - publication_points: publication_points.into(), - roa_cache_stats, - downloads: downloads.into(), - download_stats, - current_repo_objects: current_repo_objects.into(), - ccr_accumulator, - cir_input, - } - } - - fn trust_anchors(&self) -> Vec { - if self.discoveries.is_empty() { - vec![self.discovery.trust_anchor.clone()] - } else { - self.discoveries - .iter() - .map(|item| item.trust_anchor.clone()) - .collect() - } - } -} - -#[derive(Default)] -struct ObjectGraphSectionBuilder { - name: String, - item_count: u64, - shallow_bytes: u64, - heap_bytes: u64, - string_count: u64, - string_bytes: u64, - string_capacity_bytes: u64, - vec_count: u64, - vec_heap_bytes: u64, - vec_capacity_bytes: u64, - details: Vec, -} - -impl ObjectGraphSectionBuilder { - fn new(name: impl Into) -> Self { - Self { - name: name.into(), - ..Self::default() - } - } - - fn items(&mut self, count: usize, item_size: usize) { - self.item_count += count as u64; - self.shallow_bytes += (count as u64) * (item_size as u64); - } - - fn heap_bytes(&mut self, value: usize) { - self.heap_bytes += value as u64; - } - - fn string(&mut self, value: &str) { - self.string_count += 1; - self.string_bytes += value.len() as u64; - self.string_capacity_bytes += value.len() as u64; - self.heap_bytes += value.len() as u64; - } - - fn owned_string(&mut self, value: &String) { - self.string_count += 1; - self.string_bytes += value.len() as u64; - self.string_capacity_bytes += value.capacity() as u64; - self.heap_bytes += value.capacity() as u64; - } - - fn optional_string(&mut self, value: Option<&String>) { - if let Some(value) = value { - self.owned_string(value); - } - } - - fn vec_header_with_capacity(&mut self, len: usize, capacity: usize, element_size: usize) { - self.vec_count += 1; - let payload_bytes = len * element_size; - let capacity_bytes = capacity * element_size; - self.vec_heap_bytes += payload_bytes as u64; - self.vec_capacity_bytes += capacity_bytes as u64; - self.heap_bytes += capacity_bytes as u64; - } - - fn byte_vec_owned(&mut self, value: &Vec) { - self.vec_header_with_capacity(value.len(), value.capacity(), std::mem::size_of::()); - } - - fn string_vec_owned(&mut self, values: &Vec) { - self.vec_header_with_capacity( - values.len(), - values.capacity(), - std::mem::size_of::(), - ); - for value in values { - self.owned_string(value); - } - } - - fn metric(&mut self, name: impl Into, value: u64) { - self.details.push(ObjectGraphMemoryMetric { - name: name.into(), - value, - }); - } - - fn finish(self) -> ObjectGraphMemorySection { - let estimated_bytes = self.shallow_bytes + self.heap_bytes; - ObjectGraphMemorySection { - name: self.name, - item_count: self.item_count, - shallow_bytes: self.shallow_bytes, - heap_bytes: self.heap_bytes, - estimated_bytes, - string_count: self.string_count, - string_bytes: self.string_bytes, - string_capacity_bytes: self.string_capacity_bytes, - vec_count: self.vec_count, - vec_heap_bytes: self.vec_heap_bytes, - vec_capacity_bytes: self.vec_capacity_bytes, - details: self.details, - } - } -} - -fn estimate_shared_object_graph(shared: &PostValidationShared) -> ObjectGraphMemorySummary { - let mut sections = Vec::new(); - sections.push(estimate_publication_points_graph( - shared.publication_points.as_ref(), - )); - sections.push(estimate_vrps_graph(shared.vrps.as_ref())); - sections.push(estimate_aspas_graph(shared.aspas.as_ref())); - sections.push(estimate_router_keys_graph(shared.router_keys.as_ref())); - sections.push(estimate_warnings_graph( - "tree_warnings", - shared.tree_warnings.as_ref(), - )); - sections.push(estimate_downloads_graph(shared.downloads.as_ref())); - sections.push(estimate_current_repo_objects_graph( - shared.current_repo_objects.as_ref(), - )); - sections.push(estimate_trust_anchor_graph(shared)); - sections.push(estimate_ccr_accumulator_graph( - shared.ccr_accumulator.as_ref(), - )); - - let total_estimated_bytes = sections - .iter() - .map(|section| section.estimated_bytes) - .sum::(); - ObjectGraphMemorySummary { - captured_at_label: "after_validation".to_string(), - total_estimated_bytes, - sections, - notes: vec![ - "Estimated bytes are Rust object graph approximations based on struct sizes and owned String/Vec payload lengths.".to_string(), - "The estimate intentionally excludes allocator metadata, fragmentation, freed-but-retained arenas, RocksDB C++ heap, and transient worker allocations.".to_string(), - "Large RSS minus this estimate points to allocator retention or structures not yet modeled by this telemetry.".to_string(), - ], - } -} - -fn estimate_publication_points_graph( - publication_points: &[crate::audit::PublicationPointAudit], -) -> ObjectGraphMemorySection { - let mut builder = ObjectGraphSectionBuilder::new("publication_points"); - builder.items( - publication_points.len(), - std::mem::size_of::(), - ); - builder.metric("publication_point_count", publication_points.len() as u64); - let mut object_count = 0u64; - let mut pp_warning_count = 0u64; - let mut pp_discovered_from_count = 0u64; - let mut object_detail_count = 0u64; - - for pp in publication_points { - builder.owned_string(&pp.rsync_base_uri); - builder.owned_string(&pp.manifest_rsync_uri); - builder.owned_string(&pp.publication_point_rsync_uri); - builder.optional_string(pp.rrdp_notification_uri.as_ref()); - builder.owned_string(&pp.source); - builder.optional_string(pp.repo_sync_source.as_ref()); - builder.optional_string(pp.repo_sync_phase.as_ref()); - builder.optional_string(pp.repo_sync_error.as_ref()); - builder.owned_string(&pp.repo_terminal_state); - builder.owned_string(&pp.this_update_rfc3339_utc); - builder.owned_string(&pp.next_update_rfc3339_utc); - builder.owned_string(&pp.verified_at_rfc3339_utc); - - if let Some(discovered_from) = &pp.discovered_from { - pp_discovered_from_count += 1; - builder.heap_bytes(std::mem::size_of::()); - builder.owned_string(&discovered_from.parent_manifest_rsync_uri); - builder.owned_string(&discovered_from.child_ca_certificate_rsync_uri); - builder.owned_string(&discovered_from.child_ca_certificate_sha256_hex); - } - - pp_warning_count += pp.warnings.len() as u64; - builder.vec_header_with_capacity( - pp.warnings.len(), - pp.warnings.capacity(), - std::mem::size_of::(), - ); - for warning in &pp.warnings { - builder.owned_string(&warning.message); - builder.string_vec_owned(&warning.rfc_refs); - builder.optional_string(warning.context.as_ref()); - } - - object_count += pp.objects.len() as u64; - builder.vec_header_with_capacity( - pp.objects.len(), - pp.objects.capacity(), - std::mem::size_of::(), - ); - for object in &pp.objects { - builder.owned_string(&object.rsync_uri); - builder.owned_string(&object.sha256_hex); - if object.detail.is_some() { - object_detail_count += 1; - } - builder.optional_string(object.detail.as_ref()); - } - } - - builder.metric("object_audit_entry_count", object_count); - builder.metric("publication_point_warning_count", pp_warning_count); - builder.metric( - "publication_point_discovered_from_count", - pp_discovered_from_count, - ); - builder.metric("object_detail_count", object_detail_count); - builder.finish() -} - -fn estimate_vrps_graph(vrps: &[crate::validation::objects::Vrp]) -> ObjectGraphMemorySection { - let mut builder = ObjectGraphSectionBuilder::new("vrps"); - builder.items( - vrps.len(), - std::mem::size_of::(), - ); - builder.metric("vrp_count", vrps.len() as u64); - builder.finish() -} - -fn estimate_aspas_graph( - aspas: &[crate::validation::objects::AspaAttestation], -) -> ObjectGraphMemorySection { - let mut builder = ObjectGraphSectionBuilder::new("aspas"); - builder.items( - aspas.len(), - std::mem::size_of::(), - ); - let mut providers_total = 0u64; - for aspa in aspas { - providers_total += aspa.provider_as_ids.len() as u64; - builder.vec_header_with_capacity( - aspa.provider_as_ids.len(), - aspa.provider_as_ids.capacity(), - std::mem::size_of::(), - ); - } - builder.metric("aspa_count", aspas.len() as u64); - builder.metric("provider_asn_count", providers_total); - builder.finish() -} - -fn estimate_router_keys_graph( - router_keys: &[crate::validation::objects::RouterKeyPayload], -) -> ObjectGraphMemorySection { - let mut builder = ObjectGraphSectionBuilder::new("router_keys"); - builder.items( - router_keys.len(), - std::mem::size_of::(), - ); - for router_key in router_keys { - builder.byte_vec_owned(&router_key.ski); - builder.byte_vec_owned(&router_key.spki_der); - builder.owned_string(&router_key.source_object_uri); - builder.owned_string(&router_key.source_object_hash); - builder.owned_string(&router_key.source_ee_cert_hash); - } - builder.metric("router_key_count", router_keys.len() as u64); - builder.finish() -} - -fn estimate_warnings_graph( - name: &str, - warnings: &[crate::report::Warning], -) -> ObjectGraphMemorySection { - let mut builder = ObjectGraphSectionBuilder::new(name); - builder.items( - warnings.len(), - std::mem::size_of::(), - ); - for warning in warnings { - builder.owned_string(&warning.message); - builder.vec_header_with_capacity( - warning.rfc_refs.len(), - warning.rfc_refs.capacity(), - std::mem::size_of::(), - ); - builder.optional_string(warning.context.as_ref()); - } - builder.metric("warning_count", warnings.len() as u64); - builder.finish() -} - -fn estimate_downloads_graph( - downloads: &[crate::audit::AuditDownloadEvent], -) -> ObjectGraphMemorySection { - let mut builder = ObjectGraphSectionBuilder::new("downloads"); - builder.items( - downloads.len(), - std::mem::size_of::(), - ); - let mut error_count = 0u64; - let mut bytes_count = 0u64; - let mut objects_stat_count = 0u64; - for event in downloads { - builder.owned_string(&event.uri); - builder.owned_string(&event.started_at_rfc3339_utc); - builder.owned_string(&event.finished_at_rfc3339_utc); - if event.error.is_some() { - error_count += 1; - } - if event.bytes.is_some() { - bytes_count += 1; - } - if event.objects.is_some() { - objects_stat_count += 1; - } - builder.optional_string(event.error.as_ref()); - } - builder.metric("download_event_count", downloads.len() as u64); - builder.metric("download_error_count", error_count); - builder.metric("download_bytes_field_count", bytes_count); - builder.metric("download_objects_stat_count", objects_stat_count); - builder.finish() -} - -fn estimate_current_repo_objects_graph( - objects: &[crate::current_repo_index::CurrentRepoObject], -) -> ObjectGraphMemorySection { - let mut builder = ObjectGraphSectionBuilder::new("current_repo_objects"); - builder.items( - objects.len(), - std::mem::size_of::(), - ); - let mut object_type_count = 0u64; - for object in objects { - builder.owned_string(&object.rsync_uri); - builder.owned_string(&object.current_hash_hex); - builder.owned_string(&object.repository_source); - if object.object_type.is_some() { - object_type_count += 1; - } - builder.optional_string(object.object_type.as_ref()); - } - builder.metric("current_repo_object_count", objects.len() as u64); - builder.metric("current_repo_object_type_count", object_type_count); - builder.finish() -} - -fn estimate_trust_anchor_graph(shared: &PostValidationShared) -> ObjectGraphMemorySection { - let mut builder = ObjectGraphSectionBuilder::new("trust_anchors_and_tal_inputs"); - builder.items( - 1, - std::mem::size_of::(), - ); - estimate_discovered_root(&mut builder, &shared.discovery); - builder.items( - shared.discoveries.len(), - std::mem::size_of::(), - ); - for discovery in shared.discoveries.iter() { - estimate_discovered_root(&mut builder, discovery); - } - builder.items( - shared.successful_tal_inputs.len(), - std::mem::size_of::(), - ); - for tal_input in shared.successful_tal_inputs.iter() { - estimate_tal_input(&mut builder, tal_input); - } - builder.metric("discoveries_count", shared.discoveries.len() as u64); - builder.metric( - "successful_tal_inputs_count", - shared.successful_tal_inputs.len() as u64, - ); - builder.finish() -} - -fn estimate_discovered_root( - builder: &mut ObjectGraphSectionBuilder, - discovery: &crate::validation::from_tal::DiscoveredRootCaInstance, -) { - builder.optional_string(discovery.tal_url.as_ref()); - estimate_trust_anchor(builder, &discovery.trust_anchor); - builder.owned_string(&discovery.ca_instance.rsync_base_uri); - builder.owned_string(&discovery.ca_instance.manifest_rsync_uri); - builder.owned_string(&discovery.ca_instance.publication_point_rsync_uri); - builder.optional_string(discovery.ca_instance.rrdp_notification_uri.as_ref()); -} - -fn estimate_trust_anchor( - builder: &mut ObjectGraphSectionBuilder, - trust_anchor: &crate::data_model::ta::TrustAnchor, -) { - builder.byte_vec_owned(&trust_anchor.tal.raw); - builder.string_vec_owned(&trust_anchor.tal.comments); - builder.vec_header_with_capacity( - trust_anchor.tal.ta_uris.len(), - trust_anchor.tal.ta_uris.capacity(), - std::mem::size_of::(), - ); - for uri in &trust_anchor.tal.ta_uris { - builder.string(uri.as_str()); - } - builder.byte_vec_owned(&trust_anchor.tal.subject_public_key_info_der); - builder.byte_vec_owned(&trust_anchor.ta_certificate.raw_der); - if let Some(uri) = &trust_anchor.resolved_ta_uri { - builder.string(uri.as_str()); - } -} - -fn estimate_tal_input(builder: &mut ObjectGraphSectionBuilder, tal_input: &TalInputSpec) { - builder.owned_string(&tal_input.tal_id); - builder.owned_string(&tal_input.rir_id); - match &tal_input.source { - crate::parallel::types::TalSource::Url(url) => builder.owned_string(url), - crate::parallel::types::TalSource::DerBytes { - tal_url, - tal_bytes, - ta_der, - } => { - builder.owned_string(tal_url); - builder.byte_vec_owned(tal_bytes); - builder.byte_vec_owned(ta_der); - } - crate::parallel::types::TalSource::FilePath(path) => { - builder.string(&path.to_string_lossy()); - } - crate::parallel::types::TalSource::FilePathWithTa { tal_path, ta_path } => { - builder.string(&tal_path.to_string_lossy()); - builder.string(&ta_path.to_string_lossy()); - } - } -} - -fn estimate_ccr_accumulator_graph( - accumulator: Option<&CcrAccumulator>, -) -> ObjectGraphMemorySection { - let mut builder = ObjectGraphSectionBuilder::new("ccr_accumulator"); - if let Some(accumulator) = accumulator { - builder.items(1, std::mem::size_of::()); - let stats = accumulator.memory_stats(); - builder.heap_bytes(stats.estimated_heap_bytes as usize); - builder.metric("trust_anchor_count", stats.trust_anchor_count); - builder.metric("manifest_count", stats.manifest_count); - builder.metric("string_bytes", stats.string_bytes); - builder.metric("string_capacity_bytes", stats.string_capacity_bytes); - builder.metric("vec_payload_bytes", stats.vec_payload_bytes); - builder.metric("vec_capacity_bytes", stats.vec_capacity_bytes); - builder.metric("locations_der_count", stats.locations_der_count); - builder.metric("subordinate_ski_count", stats.subordinate_ski_count); - builder.metric("btree_key_capacity_bytes", stats.btree_key_capacity_bytes); - builder.metric("btree_entry_shallow_bytes", stats.btree_entry_shallow_bytes); - } else { - builder.metric("manifest_count", 0); - } - builder.finish() -} - -#[cfg(test)] -fn build_report( - policy: &Policy, - validation_time: time::OffsetDateTime, - shared: &PostValidationShared, -) -> AuditReportV2 { - use time::format_description::well_known::Rfc3339; - let validation_time_rfc3339_utc = validation_time - .to_offset(time::UtcOffset::UTC) - .format(&Rfc3339) - .expect("format validation_time"); - - let vrps = shared - .vrps - .iter() - .map(|v| VrpOutput { - asn: v.asn, - prefix: format_roa_ip_prefix(&v.prefix), - max_length: v.max_length, - }) - .collect::>(); - - let aspas = shared - .aspas - .iter() - .map(|a| AspaOutput { - customer_as_id: a.customer_as_id, - provider_as_ids: a.provider_as_ids.clone(), - }) - .collect::>(); - - let repo_sync_stats = build_repo_sync_stats(shared.publication_points.as_ref()); - - AuditReportV2 { - format_version: 2, - meta: AuditRunMeta { - validation_time_rfc3339_utc, - }, - policy: policy.clone(), - tree: TreeSummary { - instances_processed: shared.instances_processed, - instances_failed: shared.instances_failed, - warnings: shared - .tree_warnings - .iter() - .map(AuditWarning::from) - .collect(), - }, - publication_points: shared.publication_points.iter().cloned().collect(), - vrps, - aspas, - downloads: shared.downloads.iter().cloned().collect(), - download_stats: shared.download_stats.clone(), - repo_sync_stats, - query_audit: None, - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -struct ReportTaskOutput { - report_build_ms: u64, - report_write_ms: Option, -} - -impl ReportTaskOutput { - fn skipped() -> Self { - Self { - report_build_ms: 0, - report_write_ms: None, - } - } -} - -fn run_report_task( - policy: &Policy, - validation_time: time::OffsetDateTime, - shared: &PostValidationShared, - report_json_path: Option<&Path>, - report_json_format: ReportJsonFormat, -) -> Result { - if let Some(path) = report_json_path { - let timing = write_report_json_from_shared( - path, - policy, - validation_time, - shared, - report_json_format, - )?; - Ok(ReportTaskOutput { - report_build_ms: timing.build_ms, - report_write_ms: Some(timing.write_ms), - }) - } else { - Ok(ReportTaskOutput::skipped()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -struct CcrTaskOutput { - ccr_build_ms: Option, - ccr_build_breakdown: Option, - ccr_write_ms: Option, -} - -fn run_ccr_task( - store: &RocksStore, - shared: &PostValidationShared, - ccr_out_path: Option<&Path>, - produced_at: time::OffsetDateTime, -) -> Result { - let mut ccr_build_ms = None; - let mut ccr_build_breakdown = None; - let mut ccr_write_ms = None; - if let Some(path) = ccr_out_path { - let started = std::time::Instant::now(); - let (ccr, build_breakdown) = if let Some(accumulator) = shared.ccr_accumulator.as_ref() { - ( - accumulator - .finish( - produced_at, - shared.vrps.as_ref(), - shared.aspas.as_ref(), - shared.router_keys.as_ref(), - ) - .map_err(|e| e.to_string())?, - None, - ) - } else { - let trust_anchors = shared.trust_anchors(); - let (ccr, build_breakdown) = build_ccr_from_run_with_breakdown( - store, - &trust_anchors, - shared.vrps.as_ref(), - shared.aspas.as_ref(), - shared.router_keys.as_ref(), - produced_at, - ) - .map_err(|e| e.to_string())?; - (ccr, Some(build_breakdown)) - }; - ccr_build_ms = Some(started.elapsed().as_millis() as u64); - ccr_build_breakdown = build_breakdown; - let started = std::time::Instant::now(); - write_ccr_file(path, &ccr).map_err(|e| e.to_string())?; - ccr_write_ms = Some(started.elapsed().as_millis() as u64); - eprintln!("wrote CCR: {}", path.display()); - } - - Ok(CcrTaskOutput { - ccr_build_ms, - ccr_build_breakdown, - ccr_write_ms, - }) -} - -fn resolve_cir_export_tal_uris(args: &CliArgs) -> Result, String> { - if !args.cir_tal_uris.is_empty() { - return Ok(args.cir_tal_uris.clone()); - } - if !args.tal_urls.is_empty() { - return Ok(args.tal_urls.clone()); - } - Err("CIR export requires TAL URI source(s)".to_string()) -} - -fn effective_cir_tal_uris_for_discoveries( - args: &CliArgs, - shared: &PostValidationShared, - cir_tal_uris: Vec, -) -> Result, String> { - if shared.successful_tal_inputs.is_empty() { - return Ok(cir_tal_uris); - } - if cir_tal_uris.len() == shared.discoveries.len() { - return Ok(cir_tal_uris); - } - if cir_tal_uris.len() != args.tal_inputs.len() { - return Ok(cir_tal_uris); - } - - let mut mapped = Vec::with_capacity(shared.successful_tal_inputs.len()); - for successful in shared.successful_tal_inputs.iter() { - let input_index = args - .tal_inputs - .iter() - .position(|candidate| candidate == successful) - .ok_or_else(|| { - format!( - "successful TAL '{}' was not found in original TAL input list", - successful.tal_id - ) - })?; - mapped.push(cir_tal_uris[input_index].clone()); - } - Ok(mapped) -} - -fn build_repo_sync_stats( - publication_points: &[crate::audit::PublicationPointAudit], -) -> AuditRepoSyncStats { - let mut stats = AuditRepoSyncStats { - publication_points_total: publication_points.len() as u64, - ..AuditRepoSyncStats::default() - }; - - for pp in publication_points { - let duration = pp.repo_sync_duration_ms.unwrap_or(0); - if let Some(phase) = pp.repo_sync_phase.as_ref() { - let entry = stats.by_phase.entry(phase.clone()).or_default(); - entry.count += 1; - entry.duration_ms_total += duration; - } - let entry = stats - .by_terminal_state - .entry(pp.repo_terminal_state.clone()) - .or_default(); - entry.count += 1; - entry.duration_ms_total += duration; - } - - stats -} - -fn run_online_validation_with_fetchers( - store: Arc, - policy: &Policy, - args: &CliArgs, - http: &H, - rsync: &R, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, - collect_current_repo_objects: bool, - timing: Option<&TimingHandle>, -) -> Result -where - H: crate::sync::rrdp::Fetcher + Clone + 'static, - R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, -{ - // The multi-TAL entry point preserves the TAL id supplied by the CLI. - // A single-file TAL may otherwise derive its id from the embedded TA URI, - // which is intentionally different from the local filename used for - // adjacent .constraints discovery. - if args.tal_inputs.len() > 1 || !policy.ta_constraints.is_empty() { - return if let Some(t) = timing { - run_tree_from_multiple_tals_parallel_phase2_audit_with_timing( - store, - policy, - args.tal_inputs.clone(), - http, - rsync, - validation_time, - config, - args.parallel_phase1_config.clone(), - args.parallel_phase2_config.clone(), - collect_current_repo_objects, - t, - ) - } else { - run_tree_from_multiple_tals_parallel_phase2_audit( - store, - policy, - args.tal_inputs.clone(), - http, - rsync, - validation_time, - config, - args.parallel_phase1_config.clone(), - args.parallel_phase2_config.clone(), - collect_current_repo_objects, - ) - } - .map_err(|e| e.to_string()); - } - - match ( - args.tal_url.as_ref(), - args.tal_path.as_ref(), - args.ta_path.as_ref(), - ) { - (Some(url), _, _) => if let Some(t) = timing { - run_tree_from_tal_url_parallel_phase2_audit_with_timing( - store, - policy, - url, - http, - rsync, - validation_time, - config, - args.parallel_phase1_config.clone(), - args.parallel_phase2_config.clone(), - collect_current_repo_objects, - t, - ) - } else { - run_tree_from_tal_url_parallel_phase2_audit( - store, - policy, - url, - http, - rsync, - validation_time, - config, - args.parallel_phase1_config.clone(), - args.parallel_phase2_config.clone(), - collect_current_repo_objects, - ) - } - .map_err(|e| e.to_string()), - (None, Some(tal_path), Some(ta_path)) => { - let tal_bytes = std::fs::read(tal_path) - .map_err(|e| format!("read tal failed: {}: {e}", tal_path.display()))?; - let ta_der = std::fs::read(ta_path) - .map_err(|e| format!("read ta failed: {}: {e}", ta_path.display()))?; - if let Some(t) = timing { - run_tree_from_tal_and_ta_der_parallel_phase2_audit_with_timing( - store, - policy, - &tal_bytes, - &ta_der, - None, - http, - rsync, - validation_time, - config, - args.parallel_phase1_config.clone(), - args.parallel_phase2_config.clone(), - collect_current_repo_objects, - t, - ) - } else { - run_tree_from_tal_and_ta_der_parallel_phase2_audit( - store, - policy, - &tal_bytes, - &ta_der, - None, - http, - rsync, - validation_time, - config, - args.parallel_phase1_config.clone(), - args.parallel_phase2_config.clone(), - collect_current_repo_objects, - ) - } - .map_err(|e| e.to_string()) - } - (None, Some(tal_path), None) => { - let tal_bytes = std::fs::read(tal_path) - .map_err(|e| format!("read tal failed: {}: {e}", tal_path.display()))?; - let tal_uri = args.cir_tal_uri.clone(); - if let Some(t) = timing { - crate::validation::run_tree_from_tal::run_tree_from_tal_bytes_serial_audit_with_timing( - store.as_ref(), - policy, - &tal_bytes, - tal_uri, - http, - rsync, - validation_time, - config, - t, - ) - .map_err(|e| e.to_string()) - } else { - crate::validation::run_tree_from_tal::run_tree_from_tal_bytes_serial_audit( - store.as_ref(), - policy, - &tal_bytes, - tal_uri, - http, - rsync, - validation_time, - config, - ) - .map_err(|e| e.to_string()) - } - } - _ => unreachable!("validated by parse_args"), - } -} - -pub fn run(argv: &[String]) -> Result<(), String> { - let mut args = parse_args(argv)?; - let mut policy = read_policy(args.policy_path.as_deref())?; - if let Some(strict_policy) = args.strict_policy { - policy.strict = strict_policy; - } - if let Some(resource_validation_mode) = args.resource_validation_mode { - policy.resource_validation_mode = resource_validation_mode; - } - if args.disable_rrdp { - policy.sync_preference = crate::policy::SyncPreference::RsyncOnly; - } - policy.ta_constraints = args.ta_constraints.clone(); - for warning in policy.ta_constraints.configuration_warnings() { - eprintln!("warning: {warning}"); - } - let validation_time = args - .validation_time - .unwrap_or_else(time::OffsetDateTime::now_utc); - let validation_time = - time::OffsetDateTime::from_unix_timestamp(validation_time.unix_timestamp()) - .map_err(|error| format!("normalize validation time failed: {error}"))?; - let http_root_certificates_pem = args - .http_root_cert_paths - .iter() - .map(|path| { - std::fs::read(path) - .map_err(|e| format!("read HTTP root certificate failed: {}: {e}", path.display())) - }) - .collect::, _>>()?; - - let store = if args.raw_store_db.is_some() || args.repo_bytes_db.is_some() { - Arc::new( - RocksStore::open_with_external_stores( - &args.db_path, - args.raw_store_db.as_deref(), - args.repo_bytes_db.as_deref(), - ) - .map_err(|e| e.to_string())?, - ) - } else { - Arc::new(RocksStore::open(&args.db_path).map_err(|e| e.to_string())?) - }; - let config = TreeRunConfig { - max_depth: Some(args.max_ca_depth), - max_instances: args.max_instances, - compact_audit: args.skip_report_build - && args.report_json_path.is_none() - && !args.cir_enabled, - persist_vcir: !args.skip_vcir_persist, - build_ccr_accumulator: args.ccr_out_path.is_some(), - enable_roa_validation_cache: args.enable_roa_validation_cache, - enable_child_certificate_validation_cache: args.enable_child_certificate_validation_cache, - publication_point_cache_observe_only: args.publication_point_cache_observe_only, - enable_publication_point_validation_cache: args.enable_publication_point_validation_cache, - enable_transport_request_prefetch: args.enable_transport_request_prefetch, - }; - let replay_mode = args.payload_replay_archive.is_some(); - let delta_replay_mode = args.payload_base_archive.is_some(); - - use time::format_description::well_known::Rfc3339; - let mut timing: Option<(std::path::PathBuf, TimingHandle)> = None; - if args.analyze { - let recorded_at_utc_rfc3339 = time::OffsetDateTime::now_utc() - .to_offset(time::UtcOffset::UTC) - .format(&Rfc3339) - .map_err(|e| format!("format recorded_at_utc failed: {e}"))?; - let validation_time_utc_rfc3339 = validation_time - .to_offset(time::UtcOffset::UTC) - .format(&Rfc3339) - .map_err(|e| format!("format validation_time failed: {e}"))?; - - let ts_compact = { - let fmt = time::format_description::parse("[year][month][day]T[hour][minute][second]Z") - .map_err(|e| format!("format description parse failed: {e}"))?; - time::OffsetDateTime::now_utc() - .format(&fmt) - .map_err(|e| format!("format timestamp failed: {e}"))? - }; - - let out_dir = args.analysis_out_path.clone().unwrap_or_else(|| { - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("target") - .join("live") - .join("analyze") - .join(ts_compact) - }); - std::fs::create_dir_all(&out_dir) - .map_err(|e| format!("create analyze out dir failed: {}: {e}", out_dir.display()))?; - - let handle = TimingHandle::new(TimingMeta { - recorded_at_utc_rfc3339, - validation_time_utc_rfc3339, - tal_url: None, - db_path: None, - }); - handle.set_meta(TimingMetaUpdate { - tal_url: args.tal_url.as_deref(), - db_path: Some(args.db_path.to_string_lossy().as_ref()), - }); - timing = Some((out_dir, handle)); - } - - if args.profile_cpu && !args.analyze { - return Err("--profile-cpu requires --analyze".to_string()); - } - - #[cfg(not(feature = "profile"))] - if args.profile_cpu { - return Err("CPU profiling requires building with: --features profile".to_string()); - } - - #[cfg(feature = "profile")] - let mut profiler_guard: Option> = if args.profile_cpu { - Some( - pprof::ProfilerGuard::new(100) - .map_err(|e| format!("pprof ProfilerGuard init failed: {e}"))?, - ) - } else { - None - }; - - let total_started = std::time::Instant::now(); - let mut memory_checkpoints: Vec = Vec::new(); - let mut malloc_trim_probes: Vec = Vec::new(); - let enable_memory_trim_probe = memory_trim_probe_enabled() || args.memory_trim_after_validation; - record_memory_checkpoint( - &mut memory_checkpoints, - "after_store_open", - &total_started, - store.as_ref(), - ); - let validation_started = std::time::Instant::now(); - let crypto_sig_cache = - if args.crypto_signature_cache_observe_only || args.enable_crypto_signature_cache { - let cache_file = crate::crypto_sig_cache::default_cache_file_path(&args.db_path); - let cache = Arc::new(crate::crypto_sig_cache::CryptoSigCache::load_or_rebuild( - cache_file, - args.enable_crypto_signature_cache, - )); - crate::crypto_sig_cache::install_global(Arc::clone(&cache)); - Some(cache) - } else { - None - }; - let collect_current_repo_objects = false; - let out = if delta_replay_mode { - let tal_path = args - .tal_path - .as_ref() - .expect("validated by parse_args for delta replay mode"); - let ta_path = args - .ta_path - .as_ref() - .expect("validated by parse_args for delta replay mode"); - let base_archive = args - .payload_base_archive - .as_ref() - .expect("validated by parse_args for delta replay mode"); - let base_locks = args - .payload_base_locks - .as_ref() - .expect("validated by parse_args for delta replay mode"); - let base_validation_time = args.payload_base_validation_time.unwrap_or(validation_time); - let delta_archive = args - .payload_delta_archive - .as_ref() - .expect("validated by parse_args for delta replay mode"); - let delta_locks = args - .payload_delta_locks - .as_ref() - .expect("validated by parse_args for delta replay mode"); - let tal_bytes = std::fs::read(tal_path) - .map_err(|e| format!("read tal failed: {}: {e}", tal_path.display()))?; - let ta_der = std::fs::read(ta_path) - .map_err(|e| format!("read ta failed: {}: {e}", ta_path.display()))?; - if let Some((_, t)) = timing.as_ref() { - run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit_with_timing( - store.as_ref(), - &policy, - &tal_bytes, - &ta_der, - None, - base_archive, - base_locks, - delta_archive, - delta_locks, - base_validation_time, - validation_time, - &config, - t, - ) - .map_err(|e| e.to_string())? - } else { - run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit( - store.as_ref(), - &policy, - &tal_bytes, - &ta_der, - None, - base_archive, - base_locks, - delta_archive, - delta_locks, - base_validation_time, - validation_time, - &config, - ) - .map_err(|e| e.to_string())? - } - } else if replay_mode { - let tal_path = args - .tal_path - .as_ref() - .expect("validated by parse_args for replay mode"); - let ta_path = args - .ta_path - .as_ref() - .expect("validated by parse_args for replay mode"); - let archive_root = args - .payload_replay_archive - .as_ref() - .expect("validated by parse_args for replay mode"); - let locks_path = args - .payload_replay_locks - .as_ref() - .expect("validated by parse_args for replay mode"); - let tal_bytes = std::fs::read(tal_path) - .map_err(|e| format!("read tal failed: {}: {e}", tal_path.display()))?; - let ta_der = std::fs::read(ta_path) - .map_err(|e| format!("read ta failed: {}: {e}", ta_path.display()))?; - if let Some((_, t)) = timing.as_ref() { - run_tree_from_tal_and_ta_der_payload_replay_serial_audit_with_timing( - store.as_ref(), - &policy, - &tal_bytes, - &ta_der, - None, - archive_root, - locks_path, - validation_time, - &config, - t, - ) - .map_err(|e| e.to_string())? - } else { - run_tree_from_tal_and_ta_der_payload_replay_serial_audit( - store.as_ref(), - &policy, - &tal_bytes, - &ta_der, - None, - archive_root, - locks_path, - validation_time, - &config, - ) - .map_err(|e| e.to_string())? - } - } else if let Some(dir) = args.rsync_local_dir.as_ref() { - let http = BlockingHttpFetcher::new(HttpFetcherConfig { - timeout: std::time::Duration::from_secs(args.http_timeout_secs.max(1)), - extra_root_certificates_pem: http_root_certificates_pem.clone(), - ..HttpFetcherConfig::default() - }) - .map_err(|e| e.to_string())?; - let rsync = LocalDirRsyncFetcher::new(dir); - run_online_validation_with_fetchers( - Arc::clone(&store), - &policy, - &args, - &http, - &rsync, - validation_time, - &config, - collect_current_repo_objects, - timing.as_ref().map(|(_, t)| t), - )? - } else { - let http = BlockingHttpFetcher::new(HttpFetcherConfig { - timeout: std::time::Duration::from_secs(args.http_timeout_secs.max(1)), - extra_root_certificates_pem: http_root_certificates_pem.clone(), - ..HttpFetcherConfig::default() - }) - .map_err(|e| e.to_string())?; - let rsync = SystemRsyncFetcher::new(SystemRsyncConfig { - rsync_bin: args - .rsync_command - .clone() - .unwrap_or_else(|| PathBuf::from("rsync")), - timeout: std::time::Duration::from_secs(args.rsync_timeout_secs.max(1)), - mirror_root: args.rsync_mirror_root.clone(), - scope_policy: args.rsync_scope_policy, - ..SystemRsyncConfig::default() - }); - run_online_validation_with_fetchers( - Arc::clone(&store), - &policy, - &args, - &http, - &rsync, - validation_time, - &config, - collect_current_repo_objects, - timing.as_ref().map(|(_, t)| t), - )? - }; - - let validation_ms = validation_started.elapsed().as_millis() as u64; - let mut shared = PostValidationShared::from_run_output(out); - let vcir_storage_summary_enabled = vcir_storage_summary_enabled(); - let vcir_storage_summary_started = std::time::Instant::now(); - let vcir_storage = if config.persist_vcir && vcir_storage_summary_enabled { - Some( - store - .summarize_vcir_storage() - .map_err(|e| format!("summarize VCIR storage failed: {e}"))?, - ) - } else { - None - }; - let vcir_storage_summary_ms = (config.persist_vcir && vcir_storage_summary_enabled) - .then(|| vcir_storage_summary_started.elapsed().as_millis() as u64); - record_memory_checkpoint( - &mut memory_checkpoints, - "after_validation", - &total_started, - store.as_ref(), - ); - if enable_memory_trim_probe { - malloc_trim_probes.push(crate::memory_telemetry::malloc_trim_probe()); - record_memory_checkpoint( - &mut memory_checkpoints, - "after_validation_malloc_trim", - &total_started, - store.as_ref(), - ); - } - - if let Some((_out_dir, t)) = timing.as_ref() { - t.record_count("instances_processed", shared.instances_processed as u64); - t.record_count("instances_failed", shared.instances_failed as u64); - } - - let publication_points = shared.publication_points.len(); - let publication_point_repo_sync_ms_total: u64 = shared - .publication_points - .iter() - .map(|pp| pp.repo_sync_duration_ms.unwrap_or(0)) - .sum(); - let download_event_count = shared.download_stats.events_total; - let rrdp_download_ms_total: u64 = ["rrdp_notification", "rrdp_snapshot", "rrdp_delta"] - .iter() - .map(|key| { - shared - .download_stats - .by_kind - .get(*key) - .map(|item| item.duration_ms_total) - .unwrap_or(0) - }) - .sum(); - let rsync_download_ms_total = shared - .download_stats - .by_kind - .get("rsync") - .map(|item| item.duration_ms_total) - .unwrap_or(0); - let repo_sync_ms_total = rrdp_download_ms_total + rsync_download_ms_total; - let download_bytes_total: u64 = shared - .download_stats - .by_kind - .values() - .map(|item| item.bytes_total.unwrap_or(0)) - .sum(); - - #[cfg(feature = "profile")] - let profiler_report = if let Some(guard) = profiler_guard.take() { - Some( - guard - .report() - .build() - .map_err(|e| format!("pprof report build failed: {e}"))?, - ) - } else { - None - }; - - let report_json_format = if args.report_json_compact { - ReportJsonFormat::Compact - } else { - ReportJsonFormat::Pretty - }; - let ccr_produced_at = time::OffsetDateTime::now_utc(); - let compare_view_trust_anchor = args - .compare_view_trust_anchor - .as_deref() - .unwrap_or("unknown"); - let cir_tal_uris = if args.cir_enabled { - Some(effective_cir_tal_uris_for_discoveries( - &args, - &shared, - resolve_cir_export_tal_uris(&args)?, - )?) - } else { - None - }; - let cir_out_path = if args.cir_enabled { - Some( - args.cir_out_path - .as_deref() - .expect("validated by parse_args for cir"), - ) - } else { - None - }; - // Take the CIR input snapshot before the output stage so the CIR export can - // run inside the same scoped-thread group as report/ccr/compare_view; no - // other output task reads `shared.cir_input`. - let cir_input_owned = args - .cir_enabled - .then(|| std::mem::take(&mut shared.cir_input)); - let (report_result, ccr_result, compare_view_result, cir_result) = - std::thread::scope(|scope| { - // Reborrow `shared` as a plain reference so the scoped output tasks - // (incl. the `move` CIR task) capture the reference instead of - // moving fields out of the owned value. - let shared = &shared; - let report_handle = if args.skip_report_build { - None - } else { - Some(scope.spawn(|| { - run_report_task( - &policy, - validation_time, - shared, - args.report_json_path.as_deref(), - report_json_format, - ) - })) - }; - let ccr_handle = scope.spawn(|| { - run_ccr_task( - store.as_ref(), - shared, - args.ccr_out_path.as_deref(), - ccr_produced_at, - ) - }); - let compare_view_handle = scope.spawn(|| { - run_compare_view_task( - shared, - args.vrps_csv_out_path.as_deref(), - args.vaps_csv_out_path.as_deref(), - compare_view_trust_anchor, - ) - }); - let cir_handle = match (cir_tal_uris.as_ref(), cir_out_path, cir_input_owned) { - (Some(cir_tal_uris), Some(cir_out_path), Some(cir_input)) => { - Some(scope.spawn(move || { - if cir_tal_uris.len() != shared.discoveries.len() { - return Err(format!( - "CIR export TAL URI count ({}) does not match discovery count ({})", - cir_tal_uris.len(), - shared.discoveries.len() - )); - } - let tal_bindings = shared - .discoveries - .iter() - .zip(cir_tal_uris.iter()) - .map(|(discovery, tal_uri)| CirTrustAnchorBinding { - trust_anchor: &discovery.trust_anchor, - tal_uri: tal_uri.as_str(), - }) - .collect::>(); - export_cir_from_input_snapshot_multi( - &tal_bindings, - validation_time, - cir_input, - cir_out_path, - ) - .map_err(|e| e.to_string()) - })) - } - _ => None, - }; - let report_result = match report_handle { - Some(handle) => handle - .join() - .map_err(|_| "report task panicked".to_string()) - .and_then(|result| result), - None => Ok(ReportTaskOutput::skipped()), - }; - let ccr_result = ccr_handle - .join() - .map_err(|_| "ccr task panicked".to_string()) - .and_then(|result| result); - let compare_view_result = compare_view_handle - .join() - .map_err(|_| "compare view task panicked".to_string()) - .and_then(|result| result); - let cir_result = cir_handle.map(|handle| { - handle - .join() - .map_err(|_| "cir task panicked".to_string()) - .and_then(|result| result) - }); - (report_result, ccr_result, compare_view_result, cir_result) - }); - let report_output = report_result?; - let ccr_output = ccr_result?; - let compare_view_output = compare_view_result?; - let cir_summary = match cir_result { - Some(result) => Some(result?), - None => None, - }; - record_memory_checkpoint( - &mut memory_checkpoints, - "after_report_and_ccr", - &total_started, - store.as_ref(), - ); - if enable_memory_trim_probe { - malloc_trim_probes.push(crate::memory_telemetry::malloc_trim_probe()); - record_memory_checkpoint( - &mut memory_checkpoints, - "after_report_and_ccr_malloc_trim", - &total_started, - store.as_ref(), - ); - } - let report_build_ms = report_output.report_build_ms; - let report_write_ms = report_output.report_write_ms; - let ccr_build_ms = ccr_output.ccr_build_ms; - let ccr_build_breakdown = ccr_output.ccr_build_breakdown; - let ccr_write_ms = ccr_output.ccr_write_ms; - let compare_view_build_ms = compare_view_output.build_ms; - let compare_view_write_ms = compare_view_output.write_ms; - record_memory_checkpoint( - &mut memory_checkpoints, - "after_compare_view", - &total_started, - store.as_ref(), - ); - - let mut cir_build_cir_ms = None; - let mut cir_write_cir_ms = None; - let mut cir_total_ms = None; - if let Some(summary) = cir_summary { - cir_build_cir_ms = Some(summary.timing.build_cir_ms); - cir_write_cir_ms = Some(summary.timing.write_cir_ms); - cir_total_ms = Some(summary.timing.total_ms); - eprintln!( - "wrote CIR: {} (objects={}, trust_anchors={}, build_cir_ms={}, write_cir_ms={}, total_ms={})", - cir_out_path - .expect("cir path present when cir enabled") - .display(), - summary.object_count, - summary.trust_anchor_count, - summary.timing.build_cir_ms, - summary.timing.write_cir_ms, - summary.timing.total_ms - ); - record_memory_checkpoint( - &mut memory_checkpoints, - "after_cir", - &total_started, - store.as_ref(), - ); - } - record_memory_checkpoint( - &mut memory_checkpoints, - "before_stage_timing", - &total_started, - store.as_ref(), - ); - let publication_point_cache_index_refresh = if args.enable_publication_point_validation_cache - || args.publication_point_cache_observe_only - { - match store.refresh_publication_point_cache_mmap_index() { - Ok(stats) => stats, - Err(e) => { - crate::progress_log::emit( - "publication_point_cache_mmap_index_refresh", - serde_json::json!({ - "state": "failed", - "error": e.to_string(), - }), - ); - None - } - } - } else { - None - }; - let publication_point_cache_index_load = store.publication_point_cache_mmap_index_load_stats(); - let crypto_signature_cache_observe = crypto_sig_cache.as_ref().map(|cache| { - if let Err(e) = cache.persist() { - crate::progress_log::emit( - "crypto_signature_cache_persist", - serde_json::json!({ - "state": "failed", - "error": e, - }), - ); - } - crate::crypto_sig_cache::clear_global(); - cache.summary() - }); - if let (Some((_, t)), Some(summary)) = - (timing.as_ref(), crypto_signature_cache_observe.as_ref()) - { - let (mut calls, mut would_hit, mut new_keys, mut executed, mut skipped) = - (0u64, 0u64, 0u64, 0u64, 0u64); - for stats in summary.per_point.values() { - calls += stats.calls; - would_hit += stats.would_hit; - new_keys += stats.new_keys; - executed += stats.verify_executed; - skipped += stats.verify_skipped; - } - t.record_count("crypto_signature_cache_observe_calls", calls); - t.record_count("crypto_signature_cache_observe_would_hit", would_hit); - t.record_count("crypto_signature_cache_observe_new_keys", new_keys); - t.record_count("crypto_signature_cache_verify_executed", executed); - t.record_count("crypto_signature_cache_verify_skipped", skipped); - } - let timing_report_snapshot = timing - .as_ref() - .map(|(_, handle)| handle.report_snapshot(50)); - let stage_timing = RunStageTiming { - validation_ms, - enable_roa_validation_cache: args.enable_roa_validation_cache, - enable_child_certificate_validation_cache: args.enable_child_certificate_validation_cache, - publication_point_cache_observe_only: args.publication_point_cache_observe_only, - enable_publication_point_validation_cache: args.enable_publication_point_validation_cache, - crypto_signature_cache_observe, - enable_transport_request_prefetch: args.enable_transport_request_prefetch, - report_build_ms, - report_write_ms, - ccr_build_ms, - ccr_build_breakdown, - ccr_write_ms, - compare_view_build_ms, - compare_view_write_ms, - cir_build_cir_ms, - cir_write_cir_ms, - cir_total_ms, - total_ms: total_started.elapsed().as_millis() as u64, - publication_points, - repo_sync_ms_total, - publication_point_repo_sync_ms_total, - download_event_count, - rrdp_download_ms_total, - rsync_download_ms_total, - download_bytes_total, - roa_validation_cache: shared.roa_cache_stats.clone(), - analysis_counts: timing - .as_ref() - .map(|(_, handle)| handle.counts_snapshot()) - .unwrap_or_default(), - analysis_phases: timing_report_snapshot - .as_ref() - .map(|report| report.phases.clone()) - .unwrap_or_default(), - analysis_top_publication_points: timing_report_snapshot - .as_ref() - .map(|report| report.top_publication_points.clone()) - .unwrap_or_default(), - analysis_top_publication_point_steps: timing_report_snapshot - .as_ref() - .map(|report| report.top_publication_point_steps.clone()) - .unwrap_or_default(), - analysis_top_publication_point_cache_steps: timing_report_snapshot - .as_ref() - .map(|report| { - report - .top_publication_point_steps - .iter() - .filter(|entry| entry.key.contains("::publication_point_cache_")) - .cloned() - .collect() - }) - .unwrap_or_default(), - vcir_storage_summary_ms, - vcir_storage, - publication_point_cache_index_load, - publication_point_cache_index_refresh, - memory_telemetry: Some(MemoryTelemetrySummary { - checkpoints: memory_checkpoints, - object_graph: Some(estimate_shared_object_graph(&shared)), - malloc_trim_probes, - }), - }; - let stage_timing_anchor_path = args - .report_json_path - .as_deref() - .or(args.ccr_out_path.as_deref()) - .or(args.vrps_csv_out_path.as_deref()); - write_stage_timing(stage_timing_anchor_path, &stage_timing)?; - - // Finalize the normal-run contract only after all validation outputs have - // been written successfully. This prevents incomplete runs from leaving a - // contract that looks usable to downstream replay tooling. - if let Some(path) = args.validation_contract_out_path.as_deref() { - let mut contract = crate::contract::ValidationContract::for_current_binary( - validation_time, - policy.clone(), - args.max_ca_depth, - args.max_instances, - crate::contract::ValidationCacheContract { - publication_point: args.enable_publication_point_validation_cache, - roa: args.enable_roa_validation_cache, - child_certificate: args.enable_child_certificate_validation_cache, - transport_prefetch: args.enable_transport_request_prefetch, - crypto_signature: args.enable_crypto_signature_cache, - }, - args.rsync_scope_policy, - )?; - if !policy.ta_constraints.is_empty() { - contract.ta_constraints_fingerprint = - Some(policy.ta_constraints.fingerprint_sha256_hex()); - } - crate::contract::write_validation_contract(path, &contract)?; - } - - if let Some((out_dir, t)) = timing.as_ref() { - t.record_count("vrps", shared.vrps.len() as u64); - t.record_count("aspas", shared.aspas.len() as u64); - t.record_count( - "audit_publication_points", - shared.publication_points.len() as u64, - ); - let timing_json_path = out_dir.join("timing.json"); - t.write_json(&timing_json_path, 20)?; - eprintln!("analysis: wrote {}", timing_json_path.display()); - } - - #[cfg(feature = "profile")] - if let (Some((out_dir, _)), Some(report)) = (timing.as_ref(), profiler_report) { - let svg_path = out_dir.join("flamegraph.svg"); - let svg_file = std::fs::File::create(&svg_path) - .map_err(|e| format!("create flamegraph failed: {}: {e}", svg_path.display()))?; - report - .flamegraph(svg_file) - .map_err(|e| format!("write flamegraph failed: {e}"))?; - eprintln!("analysis: wrote {}", svg_path.display()); - - let pb_path = out_dir.join("pprof.pb.gz"); - let pprof_profile = report - .pprof() - .map_err(|e| format!("pprof export failed: {e}"))?; - use pprof::protos::Message; - let mut body = Vec::with_capacity(pprof_profile.encoded_len()); - pprof_profile - .encode(&mut body) - .map_err(|e| format!("pprof encode failed: {e}"))?; - let gz = flate2::write::GzEncoder::new( - std::fs::File::create(&pb_path) - .map_err(|e| format!("create pprof.pb.gz failed: {}: {e}", pb_path.display()))?, - flate2::Compression::default(), - ); - let mut gz = gz; - use std::io::Write; - gz.write_all(&body) - .map_err(|e| format!("write pprof.pb.gz failed: {e}"))?; - gz.finish() - .map_err(|e| format!("finish pprof.pb.gz failed: {e}"))?; - eprintln!("analysis: wrote {}", pb_path.display()); - } - - print_summary_from_shared(validation_time, &shared); - Ok(()) -} +include!("cli/types.rs"); +include!("cli/usage.rs"); +include!("cli/parse_args.rs"); +include!("cli/post_validation.rs"); +include!("cli/report.rs"); +include!("cli/run.rs"); #[cfg(test)] #[path = "cli/tests.rs"] diff --git a/crates/panda-rpki-validator/src/cli/parse_args.rs b/crates/panda-rpki-validator/src/cli/parse_args.rs new file mode 100644 index 0000000..d0c9c0d --- /dev/null +++ b/crates/panda-rpki-validator/src/cli/parse_args.rs @@ -0,0 +1,819 @@ +// Command-line argument parsing and validation. + +pub fn parse_args(argv: &[String]) -> Result { + let mut validation_contract_out_path: Option = None; + let mut tal_urls: Vec = Vec::new(); + let mut tal_paths: Vec = Vec::new(); + let mut ta_paths: Vec = Vec::new(); + let mut ta_constraint_specs: Vec = Vec::new(); + let mut parallel_phase1_cfg = ParallelPhase1Config::default(); + let mut parallel_phase2_cfg = ParallelPhase2Config::default(); + let mut dead_repo_blacklist_path: Option = None; + let mut dead_repo_blacklist_fail_threshold: Option = None; + + let mut db_path: Option = None; + let mut raw_store_db: Option = None; + let mut repo_bytes_db: Option = None; + let mut policy_path: Option = None; + let mut strict_policy: Option = None; + let mut resource_validation_mode: Option = None; + let mut report_json_path: Option = None; + let mut report_json_compact: bool = false; + let mut skip_report_build: bool = false; + let mut skip_vcir_persist: bool = false; + let mut enable_roa_validation_cache: bool = false; + let mut enable_child_certificate_validation_cache: bool = false; + let mut publication_point_cache_observe_only: bool = false; + let mut crypto_signature_cache_observe_only: bool = false; + let mut enable_crypto_signature_cache: bool = false; + let mut enable_publication_point_validation_cache: bool = false; + let mut enable_transport_request_prefetch: bool = false; + let mut ccr_out_path: Option = None; + let mut vrps_csv_out_path: Option = None; + let mut vaps_csv_out_path: Option = None; + let mut compare_view_trust_anchor: Option = None; + let mut cir_enabled: bool = false; + let mut cir_out_path: Option = None; + let mut cir_static_root: Option = None; + let mut cir_tal_uris: Vec = Vec::new(); + let mut cir_tal_uri: Option = None; + let mut payload_replay_archive: Option = None; + let mut payload_replay_locks: Option = None; + let mut payload_base_archive: Option = None; + let mut payload_base_locks: Option = None; + let mut payload_base_validation_time: Option = None; + let mut payload_delta_archive: Option = None; + let mut payload_delta_locks: Option = None; + let mut memory_trim_after_validation = false; + + let mut rsync_local_dir: Option = None; + let mut disable_rrdp: bool = false; + let mut rsync_command: Option = None; + let mut http_timeout_secs: u64 = 30; + let mut http_root_cert_paths: Vec = Vec::new(); + let mut rsync_timeout_secs: u64 = 30; + let mut rsync_mirror_root: Option = None; + let mut rsync_scope_policy = RsyncScopePolicy::default(); + let mut max_ca_depth: Option = None; + let mut max_ca_depth_option: Option<&'static str> = None; + let mut max_instances: Option = None; + let mut validation_time: Option = None; + let mut analyze: bool = false; + let mut analysis_out_path: Option = None; + let mut profile_cpu: bool = false; + + let mut i = 1usize; + while i < argv.len() { + let arg = argv[i].as_str(); + match arg { + "--help" | "-h" => return Err(usage()), + "--validation-contract-out" => { + i += 1; + validation_contract_out_path = Some(PathBuf::from( + argv.get(i) + .ok_or("--validation-contract-out requires a value")?, + )); + } + "--tal-url" => { + i += 1; + let v = argv.get(i).ok_or("--tal-url requires a value")?; + tal_urls.push(v.clone()); + } + "--tal-path" => { + i += 1; + let v = argv.get(i).ok_or("--tal-path requires a value")?; + tal_paths.push(PathBuf::from(v)); + } + "--ta-path" => { + i += 1; + let v = argv.get(i).ok_or("--ta-path requires a value")?; + ta_paths.push(PathBuf::from(v)); + } + "--ta-constraints" => { + i += 1; + let v = argv + .get(i) + .ok_or("--ta-constraints requires =")?; + ta_constraint_specs.push(v.clone()); + } + "--parallel-max-repo-sync-workers-global" => { + i += 1; + let v = argv + .get(i) + .ok_or("--parallel-max-repo-sync-workers-global requires a value")?; + parallel_phase1_cfg.max_repo_sync_workers_global = v + .parse::() + .map_err(|_| format!("invalid --parallel-max-repo-sync-workers-global: {v}"))?; + } + "--parallel-max-inflight-snapshot-bytes-global" => { + i += 1; + let v = argv + .get(i) + .ok_or("--parallel-max-inflight-snapshot-bytes-global requires a value")?; + parallel_phase1_cfg.max_inflight_snapshot_bytes_global = + v.parse::().map_err(|_| { + format!("invalid --parallel-max-inflight-snapshot-bytes-global: {v}") + })?; + } + "--parallel-max-pending-repo-results" => { + i += 1; + let v = argv + .get(i) + .ok_or("--parallel-max-pending-repo-results requires a value")?; + parallel_phase1_cfg.max_pending_repo_results = v + .parse::() + .map_err(|_| format!("invalid --parallel-max-pending-repo-results: {v}"))?; + } + "--parallel-phase2-object-workers" => { + i += 1; + let v = argv + .get(i) + .ok_or("--parallel-phase2-object-workers requires a value")?; + parallel_phase2_cfg.object_workers = v + .parse::() + .map_err(|_| format!("invalid --parallel-phase2-object-workers: {v}"))?; + } + "--parallel-phase2-worker-queue-capacity" => { + i += 1; + let v = argv + .get(i) + .ok_or("--parallel-phase2-worker-queue-capacity requires a value")?; + parallel_phase2_cfg.worker_queue_capacity = v + .parse::() + .map_err(|_| format!("invalid --parallel-phase2-worker-queue-capacity: {v}"))?; + } + "--parallel-phase2-ready-batch-size" => { + i += 1; + let v = argv + .get(i) + .ok_or("--parallel-phase2-ready-batch-size requires a value")?; + parallel_phase2_cfg.ready_batch_size = v + .parse::() + .map_err(|_| format!("invalid --parallel-phase2-ready-batch-size: {v}"))?; + } + "--parallel-phase2-ready-batch-wall-time-budget-ms" => { + i += 1; + let v = argv + .get(i) + .ok_or("--parallel-phase2-ready-batch-wall-time-budget-ms requires a value")?; + parallel_phase2_cfg.ready_batch_wall_time_budget_ms = + v.parse::().map_err(|_| { + format!("invalid --parallel-phase2-ready-batch-wall-time-budget-ms: {v}") + })?; + } + "--parallel-phase2-result-drain-batch-size" => { + i += 1; + let v = argv + .get(i) + .ok_or("--parallel-phase2-result-drain-batch-size requires a value")?; + parallel_phase2_cfg.object_result_drain_batch_size = + v.parse::().map_err(|_| { + format!("invalid --parallel-phase2-result-drain-batch-size: {v}") + })?; + } + "--parallel-phase2-finalize-batch-size" => { + i += 1; + let v = argv + .get(i) + .ok_or("--parallel-phase2-finalize-batch-size requires a value")?; + parallel_phase2_cfg.publication_point_finalize_batch_size = v + .parse::() + .map_err(|_| format!("invalid --parallel-phase2-finalize-batch-size: {v}"))?; + } + "--parallel-phase2-finalize-batch-wall-time-budget-ms" => { + i += 1; + let v = argv.get(i).ok_or( + "--parallel-phase2-finalize-batch-wall-time-budget-ms requires a value", + )?; + parallel_phase2_cfg.publication_point_finalize_wall_time_budget_ms = + v.parse::().map_err(|_| { + format!("invalid --parallel-phase2-finalize-batch-wall-time-budget-ms: {v}") + })?; + } + "--parallel-phase2-finalize-queue-capacity" => { + i += 1; + let v = argv + .get(i) + .ok_or("--parallel-phase2-finalize-queue-capacity requires a value")?; + parallel_phase2_cfg.publication_point_finalize_queue_capacity = + v.parse::().map_err(|_| { + format!("invalid --parallel-phase2-finalize-queue-capacity: {v}") + })?; + } + "--control-plane-stage-workers" => { + i += 1; + let v = argv + .get(i) + .ok_or("--control-plane-stage-workers requires a value")?; + parallel_phase2_cfg.stage_workers = v + .parse::() + .map_err(|_| format!("invalid --control-plane-stage-workers: {v}"))?; + } + "--dead-repo-blacklist" => { + i += 1; + let v = argv + .get(i) + .ok_or("--dead-repo-blacklist requires a value")?; + dead_repo_blacklist_path = Some(PathBuf::from(v)); + } + "--dead-repo-blacklist-fail-threshold" => { + i += 1; + let v = argv + .get(i) + .ok_or("--dead-repo-blacklist-fail-threshold requires a value")?; + dead_repo_blacklist_fail_threshold = + Some(v.parse::().map_err(|_| { + format!("invalid --dead-repo-blacklist-fail-threshold: {v}") + })?); + } + "--db" => { + i += 1; + let v = argv.get(i).ok_or("--db requires a value")?; + db_path = Some(PathBuf::from(v)); + } + "--raw-store-db" => { + i += 1; + let v = argv.get(i).ok_or("--raw-store-db requires a value")?; + raw_store_db = Some(PathBuf::from(v)); + } + "--repo-bytes-db" => { + i += 1; + let v = argv.get(i).ok_or("--repo-bytes-db requires a value")?; + repo_bytes_db = Some(PathBuf::from(v)); + } + "--policy" => { + i += 1; + let v = argv.get(i).ok_or("--policy requires a value")?; + policy_path = Some(PathBuf::from(v)); + } + "--strict" => { + let next = argv.get(i + 1).map(String::as_str); + let spec = next.filter(|v| !v.starts_with("--")); + if spec.is_some() { + i += 1; + } + strict_policy = Some(StrictPolicy::parse_cli_spec(spec)?); + } + _ if arg.starts_with("--strict=") => { + let spec = arg.strip_prefix("--strict=").expect("prefix checked"); + strict_policy = Some(StrictPolicy::parse_cli_spec(Some(spec))?); + } + "--resource-validation-mode" => { + i += 1; + let v = argv + .get(i) + .ok_or("--resource-validation-mode requires a value")?; + resource_validation_mode = Some(ResourceValidationMode::parse_cli_value(v)?); + } + _ if arg.starts_with("--resource-validation-mode=") => { + let spec = arg + .strip_prefix("--resource-validation-mode=") + .expect("prefix checked"); + resource_validation_mode = Some(ResourceValidationMode::parse_cli_value(spec)?); + } + "--report-json" => { + i += 1; + let v = argv.get(i).ok_or("--report-json requires a value")?; + report_json_path = Some(PathBuf::from(v)); + } + "--report-json-compact" => { + report_json_compact = true; + } + "--skip-report-build" => { + skip_report_build = true; + } + "--skip-vcir-persist" => { + skip_vcir_persist = true; + } + "--enable-roa-validation-cache" => { + enable_roa_validation_cache = true; + } + "--enable-child-certificate-validation-cache" => { + enable_child_certificate_validation_cache = true; + } + "--publication-point-cache-observe-only" => { + publication_point_cache_observe_only = true; + } + "--enable-publication-point-validation-cache" => { + enable_publication_point_validation_cache = true; + } + "--crypto-signature-cache-observe-only" => { + crypto_signature_cache_observe_only = true; + } + "--enable-crypto-signature-cache" => { + enable_crypto_signature_cache = true; + } + "--enable-transport-request-prefetch" => { + enable_transport_request_prefetch = true; + } + "--ccr-out" => { + i += 1; + let v = argv.get(i).ok_or("--ccr-out requires a value")?; + ccr_out_path = Some(PathBuf::from(v)); + } + "--vrps-csv-out" => { + i += 1; + let v = argv.get(i).ok_or("--vrps-csv-out requires a value")?; + vrps_csv_out_path = Some(PathBuf::from(v)); + } + "--vaps-csv-out" => { + i += 1; + let v = argv.get(i).ok_or("--vaps-csv-out requires a value")?; + vaps_csv_out_path = Some(PathBuf::from(v)); + } + "--compare-view-trust-anchor" => { + i += 1; + let v = argv + .get(i) + .ok_or("--compare-view-trust-anchor requires a value")?; + compare_view_trust_anchor = Some(v.clone()); + } + "--cir-enable" => { + cir_enabled = true; + } + "--cir-out" => { + i += 1; + let v = argv.get(i).ok_or("--cir-out requires a value")?; + cir_out_path = Some(PathBuf::from(v)); + } + "--cir-static-root" => { + i += 1; + let v = argv.get(i).ok_or("--cir-static-root requires a value")?; + cir_static_root = Some(PathBuf::from(v)); + } + "--cir-tal-uri" => { + i += 1; + let v = argv.get(i).ok_or("--cir-tal-uri requires a value")?; + cir_tal_uris.push(v.clone()); + cir_tal_uri = cir_tal_uris.first().cloned(); + } + "--payload-replay-archive" => { + i += 1; + let v = argv + .get(i) + .ok_or("--payload-replay-archive requires a value")?; + payload_replay_archive = Some(PathBuf::from(v)); + } + "--payload-replay-locks" => { + i += 1; + let v = argv + .get(i) + .ok_or("--payload-replay-locks requires a value")?; + payload_replay_locks = Some(PathBuf::from(v)); + } + "--payload-base-archive" => { + i += 1; + let v = argv + .get(i) + .ok_or("--payload-base-archive requires a value")?; + payload_base_archive = Some(PathBuf::from(v)); + } + "--payload-base-locks" => { + i += 1; + let v = argv.get(i).ok_or("--payload-base-locks requires a value")?; + payload_base_locks = Some(PathBuf::from(v)); + } + "--payload-base-validation-time" => { + i += 1; + let v = argv + .get(i) + .ok_or("--payload-base-validation-time requires a value")?; + use time::format_description::well_known::Rfc3339; + let t = time::OffsetDateTime::parse(v, &Rfc3339).map_err(|e| { + format!("invalid --payload-base-validation-time (RFC3339 expected): {e}") + })?; + payload_base_validation_time = Some(t); + } + "--payload-delta-archive" => { + i += 1; + let v = argv + .get(i) + .ok_or("--payload-delta-archive requires a value")?; + payload_delta_archive = Some(PathBuf::from(v)); + } + "--payload-delta-locks" => { + i += 1; + let v = argv + .get(i) + .ok_or("--payload-delta-locks requires a value")?; + payload_delta_locks = Some(PathBuf::from(v)); + } + "--memory-trim-after-validation" => { + memory_trim_after_validation = true; + } + "--rsync-local-dir" => { + i += 1; + let v = argv.get(i).ok_or("--rsync-local-dir requires a value")?; + rsync_local_dir = Some(PathBuf::from(v)); + } + "--disable-rrdp" => { + disable_rrdp = true; + } + "--rsync-command" => { + i += 1; + let v = argv.get(i).ok_or("--rsync-command requires a value")?; + rsync_command = Some(PathBuf::from(v)); + } + "--http-timeout-secs" => { + i += 1; + let v = argv.get(i).ok_or("--http-timeout-secs requires a value")?; + http_timeout_secs = v + .parse::() + .map_err(|_| format!("invalid --http-timeout-secs: {v}"))?; + } + "--http-root-cert" => { + i += 1; + let v = argv.get(i).ok_or("--http-root-cert requires a value")?; + http_root_cert_paths.push(PathBuf::from(v)); + } + "--rsync-timeout-secs" => { + i += 1; + let v = argv.get(i).ok_or("--rsync-timeout-secs requires a value")?; + rsync_timeout_secs = v + .parse::() + .map_err(|_| format!("invalid --rsync-timeout-secs: {v}"))?; + } + "--rsync-mirror-root" => { + i += 1; + let v = argv.get(i).ok_or("--rsync-mirror-root requires a value")?; + rsync_mirror_root = Some(PathBuf::from(v)); + } + "--rsync-scope" => { + i += 1; + let v = argv.get(i).ok_or("--rsync-scope requires a value")?; + rsync_scope_policy = RsyncScopePolicy::parse_cli_value(v)?; + } + "--max-ca-depth" | "--max-depth" => { + i += 1; + let option = arg; + let v = argv + .get(i) + .ok_or_else(|| format!("{option} requires a value"))?; + if let Some(previous_option) = max_ca_depth_option { + return Err(format!( + "{option} cannot be combined with {previous_option}; use only --max-ca-depth" + )); + } + max_ca_depth = Some( + v.parse::() + .map_err(|_| format!("invalid {option}: {v}"))?, + ); + max_ca_depth_option = Some(match option { + "--max-ca-depth" => "--max-ca-depth", + "--max-depth" => "--max-depth", + _ => unreachable!("matched CA depth option"), + }); + } + "--max-instances" => { + i += 1; + let v = argv.get(i).ok_or("--max-instances requires a value")?; + max_instances = Some( + v.parse::() + .map_err(|_| format!("invalid --max-instances: {v}"))?, + ); + } + "--validation-time" => { + i += 1; + let v = argv.get(i).ok_or("--validation-time requires a value")?; + use time::format_description::well_known::Rfc3339; + let t = time::OffsetDateTime::parse(v, &Rfc3339) + .map_err(|e| format!("invalid --validation-time (RFC3339 expected): {e}"))?; + validation_time = Some(t); + } + "--analyze" => { + analyze = true; + } + "--analysis-out" => { + i += 1; + let v = argv.get(i).ok_or("--analysis-out requires a value")?; + analyze = true; + analysis_out_path = Some(PathBuf::from(v)); + } + "--profile-cpu" => { + profile_cpu = true; + } + _ => return Err(format!("unknown argument: {arg}\n\n{}", usage())), + } + i += 1; + } + + let db_path = db_path.ok_or_else(|| format!("--db is required\n\n{}", usage()))?; + + let tal_mode_count = (!tal_urls.is_empty()) as u8 + (!tal_paths.is_empty()) as u8; + if tal_mode_count != 1 { + return Err(format!( + "must specify either one-or-more --tal-url or one-or-more --tal-path/--ta-path pairs\n\n{}", + usage() + )); + } + if parallel_phase2_cfg.object_workers == 0 { + return Err(format!( + "--parallel-phase2-object-workers must be > 0\n\n{}", + usage() + )); + } + if parallel_phase2_cfg.worker_queue_capacity == 0 { + return Err(format!( + "--parallel-phase2-worker-queue-capacity must be > 0\n\n{}", + usage() + )); + } + if parallel_phase2_cfg.ready_batch_size == 0 { + return Err(format!( + "--parallel-phase2-ready-batch-size must be > 0\n\n{}", + usage() + )); + } + if parallel_phase2_cfg.ready_batch_wall_time_budget_ms == 0 { + return Err(format!( + "--parallel-phase2-ready-batch-wall-time-budget-ms must be > 0\n\n{}", + usage() + )); + } + if parallel_phase2_cfg.object_result_drain_batch_size == 0 { + return Err(format!( + "--parallel-phase2-result-drain-batch-size must be > 0\n\n{}", + usage() + )); + } + if parallel_phase2_cfg.publication_point_finalize_batch_size == 0 { + return Err(format!( + "--parallel-phase2-finalize-batch-size must be > 0\n\n{}", + usage() + )); + } + if parallel_phase2_cfg.publication_point_finalize_wall_time_budget_ms == 0 { + return Err(format!( + "--parallel-phase2-finalize-batch-wall-time-budget-ms must be > 0\n\n{}", + usage() + )); + } + if parallel_phase2_cfg.publication_point_finalize_queue_capacity == 0 { + return Err(format!( + "--parallel-phase2-finalize-queue-capacity must be > 0\n\n{}", + usage() + )); + } + if !tal_urls.is_empty() && !ta_paths.is_empty() { + return Err(format!( + "--ta-path cannot be used with --tal-url mode\n\n{}", + usage() + )); + } + if !tal_paths.is_empty() { + if !ta_paths.is_empty() { + if ta_paths.len() != tal_paths.len() { + return Err(format!( + "--tal-path and --ta-path counts must match in file mode\n\n{}", + usage() + )); + } + } else if ta_paths.is_empty() && !disable_rrdp { + return Err(format!( + "--tal-path requires --ta-path unless --disable-rrdp is set\n\n{}", + usage() + )); + } + } + let tal_url = tal_urls.first().cloned(); + let tal_path = tal_paths.first().cloned(); + let ta_path = ta_paths.first().cloned(); + if cir_enabled && cir_out_path.is_none() { + return Err(format!("--cir-enable requires --cir-out\n\n{}", usage())); + } + if report_json_compact && report_json_path.is_none() { + return Err(format!( + "--report-json-compact requires --report-json\n\n{}", + usage() + )); + } + if skip_report_build && report_json_path.is_some() { + return Err(format!( + "--skip-report-build cannot be combined with --report-json\n\n{}", + usage() + )); + } + if vrps_csv_out_path.is_some() != vaps_csv_out_path.is_some() { + return Err(format!( + "--vrps-csv-out and --vaps-csv-out must be provided together\n\n{}", + usage() + )); + } + if compare_view_trust_anchor.is_some() && vrps_csv_out_path.is_none() { + return Err(format!( + "--compare-view-trust-anchor requires --vrps-csv-out/--vaps-csv-out\n\n{}", + usage() + )); + } + if cir_static_root.is_some() { + return Err(format!( + "--cir-static-root is no longer supported; CIR export now writes only .cir files\n\n{}", + usage() + )); + } + if !cir_enabled && (cir_out_path.is_some() || !cir_tal_uris.is_empty()) { + return Err(format!( + "--cir-out/--cir-tal-uri require --cir-enable\n\n{}", + usage() + )); + } + if cir_enabled && !cir_tal_uris.is_empty() { + let expected = if !tal_paths.is_empty() { + tal_paths.len() + } else { + tal_urls.len() + }; + if cir_tal_uris.len() != expected { + return Err(format!( + "--cir-tal-uri count must match TAL input count when provided\n\n{}", + usage() + )); + } + } + if cir_enabled && !tal_paths.is_empty() && cir_tal_uris.is_empty() { + return Err(format!( + "CIR export in --tal-path mode requires --cir-tal-uri for each TAL\n\n{}", + usage() + )); + } + + let replay_mode_count = + payload_replay_archive.is_some() as u8 + payload_replay_locks.is_some() as u8; + if replay_mode_count == 1 { + return Err(format!( + "--payload-replay-archive and --payload-replay-locks must be provided together + +{}", + usage() + )); + } + + let delta_mode_count = payload_base_archive.is_some() as u8 + + payload_base_locks.is_some() as u8 + + payload_delta_archive.is_some() as u8 + + payload_delta_locks.is_some() as u8; + if delta_mode_count > 0 && delta_mode_count < 4 { + return Err(format!( + "--payload-base-archive, --payload-base-locks, --payload-delta-archive and --payload-delta-locks must be provided together + +{}", + usage() + )); + } + if replay_mode_count == 2 && delta_mode_count == 4 { + return Err(format!( + "snapshot replay mode and delta replay mode are mutually exclusive + +{}", + usage() + )); + } + if replay_mode_count == 2 { + if tal_url.is_some() { + return Err(format!( + "payload replay mode requires --tal-path and --ta-path; --tal-url is not supported + +{}", + usage() + )); + } + if tal_path.is_none() || ta_path.is_none() { + return Err(format!( + "payload replay mode requires --tal-path and --ta-path + +{}", + usage() + )); + } + if rsync_local_dir.is_some() { + return Err(format!( + "payload replay mode cannot be combined with --rsync-local-dir + +{}", + usage() + )); + } + } + if delta_mode_count == 4 { + if tal_url.is_some() { + return Err(format!( + "payload delta replay mode requires --tal-path and --ta-path; --tal-url is not supported + +{}", + usage() + )); + } + if tal_path.is_none() || ta_path.is_none() { + return Err(format!( + "payload delta replay mode requires --tal-path and --ta-path + +{}", + usage() + )); + } + if rsync_local_dir.is_some() { + return Err(format!( + "payload delta replay mode cannot be combined with --rsync-local-dir + +{}", + usage() + )); + } + } + + let mut tal_inputs = Vec::new(); + if !tal_urls.is_empty() { + tal_inputs.extend(tal_urls.iter().cloned().map(TalInputSpec::from_url)); + } else if !tal_paths.is_empty() { + if ta_paths.len() == tal_paths.len() { + tal_inputs.extend(tal_paths.iter().cloned().zip(ta_paths.iter().cloned()).map( + |(tal_path, ta_path)| TalInputSpec::from_file_path_with_ta(tal_path, ta_path), + )); + } else { + tal_inputs.extend(tal_paths.iter().cloned().map(TalInputSpec::from_file_path)); + } + } + let ta_constraints = TaConstraintsByTal::load_for_tals(&tal_inputs, &ta_constraint_specs)?; + + if dead_repo_blacklist_fail_threshold.is_some() && dead_repo_blacklist_path.is_none() { + return Err( + "--dead-repo-blacklist-fail-threshold requires --dead-repo-blacklist".to_string(), + ); + } + if let Some(path) = dead_repo_blacklist_path { + let fail_threshold = dead_repo_blacklist_fail_threshold.unwrap_or(3); + if fail_threshold == 0 { + return Err("--dead-repo-blacklist-fail-threshold must be >= 1".to_string()); + } + parallel_phase1_cfg.dead_repo_blacklist = Some( + crate::parallel::dead_repo_blacklist::DeadRepoBlacklistConfig { + path, + fail_threshold, + capacity: + crate::parallel::dead_repo_blacklist::DEAD_REPO_BLACKLIST_DEFAULT_CAPACITY, + }, + ); + } + + Ok(CliArgs { + validation_contract_out_path, + tal_urls, + tal_paths, + ta_paths, + tal_url, + tal_path, + ta_path, + parallel_phase1_config: parallel_phase1_cfg, + parallel_phase2_config: parallel_phase2_cfg, + tal_inputs, + ta_constraints, + db_path, + raw_store_db, + repo_bytes_db, + policy_path, + strict_policy, + resource_validation_mode, + report_json_path, + report_json_compact, + skip_report_build, + skip_vcir_persist, + enable_roa_validation_cache, + enable_child_certificate_validation_cache, + publication_point_cache_observe_only, + enable_publication_point_validation_cache, + crypto_signature_cache_observe_only, + enable_crypto_signature_cache, + enable_transport_request_prefetch, + ccr_out_path, + vrps_csv_out_path, + vaps_csv_out_path, + compare_view_trust_anchor, + cir_enabled, + cir_out_path, + cir_static_root, + cir_tal_uris, + cir_tal_uri, + payload_replay_archive, + payload_replay_locks, + payload_base_archive, + payload_base_locks, + payload_base_validation_time, + payload_delta_archive, + payload_delta_locks, + memory_trim_after_validation, + rsync_local_dir, + disable_rrdp, + rsync_command, + http_timeout_secs, + http_root_cert_paths, + rsync_timeout_secs, + rsync_mirror_root, + rsync_scope_policy, + max_ca_depth: max_ca_depth.unwrap_or(DEFAULT_MAX_CA_DEPTH), + max_instances, + validation_time, + analyze, + analysis_out_path, + profile_cpu, + }) +} diff --git a/crates/panda-rpki-validator/src/cli/post_validation.rs b/crates/panda-rpki-validator/src/cli/post_validation.rs new file mode 100644 index 0000000..151f11c --- /dev/null +++ b/crates/panda-rpki-validator/src/cli/post_validation.rs @@ -0,0 +1,619 @@ +// Shared post-validation state and memory/report graph estimation. + +fn read_policy(path: Option<&Path>) -> Result { + match path { + None => Ok(Policy::default()), + Some(p) => { + let s = std::fs::read_to_string(p) + .map_err(|e| format!("read policy file failed: {}: {e}", p.display()))?; + Policy::from_toml_str(&s).map_err(|e| e.to_string()) + } + } +} + +fn unique_rrdp_repos_from_publication_points( + publication_points: &[crate::audit::PublicationPointAudit], +) -> usize { + use std::collections::HashSet; + let mut set: HashSet<&str> = HashSet::new(); + for pp in publication_points { + if let Some(u) = pp.rrdp_notification_uri.as_deref() { + set.insert(u); + } + } + set.len() +} + +#[cfg(test)] +fn unique_rrdp_repos(report: &AuditReportV2) -> usize { + unique_rrdp_repos_from_publication_points(&report.publication_points) +} + +#[cfg(test)] +fn print_summary(report: &AuditReportV2) { + let rrdp_repos = unique_rrdp_repos(report); + println!("RPKI validation run summary"); + println!( + "validation_time={}", + report.meta.validation_time_rfc3339_utc + ); + println!( + "publication_points_processed={} publication_points_failed={}", + report.tree.instances_processed, report.tree.instances_failed + ); + println!("rrdp_repos_unique={rrdp_repos}"); + println!("vrps={}", report.vrps.len()); + println!("aspas={}", report.aspas.len()); + println!( + "audit_publication_points={}", + report.publication_points.len() + ); + println!( + "warnings_total={}", + report.tree.warnings.len() + + report + .publication_points + .iter() + .map(|pp| pp.warnings.len()) + .sum::() + ); +} + +fn print_summary_from_shared(validation_time: time::OffsetDateTime, shared: &PostValidationShared) { + use time::format_description::well_known::Rfc3339; + let validation_time_rfc3339_utc = validation_time + .to_offset(time::UtcOffset::UTC) + .format(&Rfc3339) + .expect("format validation_time"); + let rrdp_repos = unique_rrdp_repos_from_publication_points(shared.publication_points.as_ref()); + println!("RPKI validation run summary"); + println!("validation_time={validation_time_rfc3339_utc}"); + println!( + "publication_points_processed={} publication_points_failed={}", + shared.instances_processed, shared.instances_failed + ); + println!("rrdp_repos_unique={rrdp_repos}"); + println!("vrps={}", shared.vrps.len()); + println!("aspas={}", shared.aspas.len()); + println!( + "audit_publication_points={}", + shared.publication_points.len() + ); + println!( + "warnings_total={}", + shared.tree_warnings.len() + + shared + .publication_points + .iter() + .map(|pp| pp.warnings.len()) + .sum::() + ); +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct PostValidationShared { + discovery: crate::validation::from_tal::DiscoveredRootCaInstance, + discoveries: Arc<[crate::validation::from_tal::DiscoveredRootCaInstance]>, + successful_tal_inputs: Arc<[TalInputSpec]>, + instances_processed: usize, + instances_failed: usize, + tree_warnings: Arc<[crate::report::Warning]>, + vrps: Arc<[crate::validation::objects::Vrp]>, + aspas: Arc<[crate::validation::objects::AspaAttestation]>, + router_keys: Arc<[crate::validation::objects::RouterKeyPayload]>, + publication_points: Arc<[crate::audit::PublicationPointAudit]>, + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats, + downloads: Arc<[crate::audit::AuditDownloadEvent]>, + download_stats: crate::audit::AuditDownloadStats, + current_repo_objects: Arc<[crate::current_repo_index::CurrentRepoObject]>, + ccr_accumulator: Option, + cir_input: crate::cir::CirInputSnapshot, +} + +impl PostValidationShared { + fn from_run_output(out: RunTreeFromTalAuditOutput) -> Self { + let RunTreeFromTalAuditOutput { + discovery, + discoveries, + successful_tal_inputs, + tree, + publication_points, + roa_cache_stats, + downloads, + download_stats, + current_repo_objects, + ccr_accumulator, + cir_input, + } = out; + let crate::validation::tree::TreeRunOutput { + instances_processed, + instances_failed, + warnings, + vrps, + aspas, + router_keys, + } = tree; + + Self { + discovery, + discoveries: discoveries.into(), + successful_tal_inputs: successful_tal_inputs.into(), + instances_processed, + instances_failed, + tree_warnings: warnings.into(), + vrps: vrps.into(), + aspas: aspas.into(), + router_keys: router_keys.into(), + publication_points: publication_points.into(), + roa_cache_stats, + downloads: downloads.into(), + download_stats, + current_repo_objects: current_repo_objects.into(), + ccr_accumulator, + cir_input, + } + } + + fn trust_anchors(&self) -> Vec { + if self.discoveries.is_empty() { + vec![self.discovery.trust_anchor.clone()] + } else { + self.discoveries + .iter() + .map(|item| item.trust_anchor.clone()) + .collect() + } + } +} + +#[derive(Default)] +struct ObjectGraphSectionBuilder { + name: String, + item_count: u64, + shallow_bytes: u64, + heap_bytes: u64, + string_count: u64, + string_bytes: u64, + string_capacity_bytes: u64, + vec_count: u64, + vec_heap_bytes: u64, + vec_capacity_bytes: u64, + details: Vec, +} + +impl ObjectGraphSectionBuilder { + fn new(name: impl Into) -> Self { + Self { + name: name.into(), + ..Self::default() + } + } + + fn items(&mut self, count: usize, item_size: usize) { + self.item_count += count as u64; + self.shallow_bytes += (count as u64) * (item_size as u64); + } + + fn heap_bytes(&mut self, value: usize) { + self.heap_bytes += value as u64; + } + + fn string(&mut self, value: &str) { + self.string_count += 1; + self.string_bytes += value.len() as u64; + self.string_capacity_bytes += value.len() as u64; + self.heap_bytes += value.len() as u64; + } + + fn owned_string(&mut self, value: &String) { + self.string_count += 1; + self.string_bytes += value.len() as u64; + self.string_capacity_bytes += value.capacity() as u64; + self.heap_bytes += value.capacity() as u64; + } + + fn optional_string(&mut self, value: Option<&String>) { + if let Some(value) = value { + self.owned_string(value); + } + } + + fn vec_header_with_capacity(&mut self, len: usize, capacity: usize, element_size: usize) { + self.vec_count += 1; + let payload_bytes = len * element_size; + let capacity_bytes = capacity * element_size; + self.vec_heap_bytes += payload_bytes as u64; + self.vec_capacity_bytes += capacity_bytes as u64; + self.heap_bytes += capacity_bytes as u64; + } + + fn byte_vec_owned(&mut self, value: &Vec) { + self.vec_header_with_capacity(value.len(), value.capacity(), std::mem::size_of::()); + } + + fn string_vec_owned(&mut self, values: &Vec) { + self.vec_header_with_capacity( + values.len(), + values.capacity(), + std::mem::size_of::(), + ); + for value in values { + self.owned_string(value); + } + } + + fn metric(&mut self, name: impl Into, value: u64) { + self.details.push(ObjectGraphMemoryMetric { + name: name.into(), + value, + }); + } + + fn finish(self) -> ObjectGraphMemorySection { + let estimated_bytes = self.shallow_bytes + self.heap_bytes; + ObjectGraphMemorySection { + name: self.name, + item_count: self.item_count, + shallow_bytes: self.shallow_bytes, + heap_bytes: self.heap_bytes, + estimated_bytes, + string_count: self.string_count, + string_bytes: self.string_bytes, + string_capacity_bytes: self.string_capacity_bytes, + vec_count: self.vec_count, + vec_heap_bytes: self.vec_heap_bytes, + vec_capacity_bytes: self.vec_capacity_bytes, + details: self.details, + } + } +} + +fn estimate_shared_object_graph(shared: &PostValidationShared) -> ObjectGraphMemorySummary { + let mut sections = Vec::new(); + sections.push(estimate_publication_points_graph( + shared.publication_points.as_ref(), + )); + sections.push(estimate_vrps_graph(shared.vrps.as_ref())); + sections.push(estimate_aspas_graph(shared.aspas.as_ref())); + sections.push(estimate_router_keys_graph(shared.router_keys.as_ref())); + sections.push(estimate_warnings_graph( + "tree_warnings", + shared.tree_warnings.as_ref(), + )); + sections.push(estimate_downloads_graph(shared.downloads.as_ref())); + sections.push(estimate_current_repo_objects_graph( + shared.current_repo_objects.as_ref(), + )); + sections.push(estimate_trust_anchor_graph(shared)); + sections.push(estimate_ccr_accumulator_graph( + shared.ccr_accumulator.as_ref(), + )); + + let total_estimated_bytes = sections + .iter() + .map(|section| section.estimated_bytes) + .sum::(); + ObjectGraphMemorySummary { + captured_at_label: "after_validation".to_string(), + total_estimated_bytes, + sections, + notes: vec![ + "Estimated bytes are Rust object graph approximations based on struct sizes and owned String/Vec payload lengths.".to_string(), + "The estimate intentionally excludes allocator metadata, fragmentation, freed-but-retained arenas, RocksDB C++ heap, and transient worker allocations.".to_string(), + "Large RSS minus this estimate points to allocator retention or structures not yet modeled by this telemetry.".to_string(), + ], + } +} + +fn estimate_publication_points_graph( + publication_points: &[crate::audit::PublicationPointAudit], +) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new("publication_points"); + builder.items( + publication_points.len(), + std::mem::size_of::(), + ); + builder.metric("publication_point_count", publication_points.len() as u64); + let mut object_count = 0u64; + let mut pp_warning_count = 0u64; + let mut pp_discovered_from_count = 0u64; + let mut object_detail_count = 0u64; + + for pp in publication_points { + builder.owned_string(&pp.rsync_base_uri); + builder.owned_string(&pp.manifest_rsync_uri); + builder.owned_string(&pp.publication_point_rsync_uri); + builder.optional_string(pp.rrdp_notification_uri.as_ref()); + builder.owned_string(&pp.source); + builder.optional_string(pp.repo_sync_source.as_ref()); + builder.optional_string(pp.repo_sync_phase.as_ref()); + builder.optional_string(pp.repo_sync_error.as_ref()); + builder.owned_string(&pp.repo_terminal_state); + builder.owned_string(&pp.this_update_rfc3339_utc); + builder.owned_string(&pp.next_update_rfc3339_utc); + builder.owned_string(&pp.verified_at_rfc3339_utc); + + if let Some(discovered_from) = &pp.discovered_from { + pp_discovered_from_count += 1; + builder.heap_bytes(std::mem::size_of::()); + builder.owned_string(&discovered_from.parent_manifest_rsync_uri); + builder.owned_string(&discovered_from.child_ca_certificate_rsync_uri); + builder.owned_string(&discovered_from.child_ca_certificate_sha256_hex); + } + + pp_warning_count += pp.warnings.len() as u64; + builder.vec_header_with_capacity( + pp.warnings.len(), + pp.warnings.capacity(), + std::mem::size_of::(), + ); + for warning in &pp.warnings { + builder.owned_string(&warning.message); + builder.string_vec_owned(&warning.rfc_refs); + builder.optional_string(warning.context.as_ref()); + } + + object_count += pp.objects.len() as u64; + builder.vec_header_with_capacity( + pp.objects.len(), + pp.objects.capacity(), + std::mem::size_of::(), + ); + for object in &pp.objects { + builder.owned_string(&object.rsync_uri); + builder.owned_string(&object.sha256_hex); + if object.detail.is_some() { + object_detail_count += 1; + } + builder.optional_string(object.detail.as_ref()); + } + } + + builder.metric("object_audit_entry_count", object_count); + builder.metric("publication_point_warning_count", pp_warning_count); + builder.metric( + "publication_point_discovered_from_count", + pp_discovered_from_count, + ); + builder.metric("object_detail_count", object_detail_count); + builder.finish() +} + +fn estimate_vrps_graph(vrps: &[crate::validation::objects::Vrp]) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new("vrps"); + builder.items( + vrps.len(), + std::mem::size_of::(), + ); + builder.metric("vrp_count", vrps.len() as u64); + builder.finish() +} + +fn estimate_aspas_graph( + aspas: &[crate::validation::objects::AspaAttestation], +) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new("aspas"); + builder.items( + aspas.len(), + std::mem::size_of::(), + ); + let mut providers_total = 0u64; + for aspa in aspas { + providers_total += aspa.provider_as_ids.len() as u64; + builder.vec_header_with_capacity( + aspa.provider_as_ids.len(), + aspa.provider_as_ids.capacity(), + std::mem::size_of::(), + ); + } + builder.metric("aspa_count", aspas.len() as u64); + builder.metric("provider_asn_count", providers_total); + builder.finish() +} + +fn estimate_router_keys_graph( + router_keys: &[crate::validation::objects::RouterKeyPayload], +) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new("router_keys"); + builder.items( + router_keys.len(), + std::mem::size_of::(), + ); + for router_key in router_keys { + builder.byte_vec_owned(&router_key.ski); + builder.byte_vec_owned(&router_key.spki_der); + builder.owned_string(&router_key.source_object_uri); + builder.owned_string(&router_key.source_object_hash); + builder.owned_string(&router_key.source_ee_cert_hash); + } + builder.metric("router_key_count", router_keys.len() as u64); + builder.finish() +} + +fn estimate_warnings_graph( + name: &str, + warnings: &[crate::report::Warning], +) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new(name); + builder.items( + warnings.len(), + std::mem::size_of::(), + ); + for warning in warnings { + builder.owned_string(&warning.message); + builder.vec_header_with_capacity( + warning.rfc_refs.len(), + warning.rfc_refs.capacity(), + std::mem::size_of::(), + ); + builder.optional_string(warning.context.as_ref()); + } + builder.metric("warning_count", warnings.len() as u64); + builder.finish() +} + +fn estimate_downloads_graph( + downloads: &[crate::audit::AuditDownloadEvent], +) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new("downloads"); + builder.items( + downloads.len(), + std::mem::size_of::(), + ); + let mut error_count = 0u64; + let mut bytes_count = 0u64; + let mut objects_stat_count = 0u64; + for event in downloads { + builder.owned_string(&event.uri); + builder.owned_string(&event.started_at_rfc3339_utc); + builder.owned_string(&event.finished_at_rfc3339_utc); + if event.error.is_some() { + error_count += 1; + } + if event.bytes.is_some() { + bytes_count += 1; + } + if event.objects.is_some() { + objects_stat_count += 1; + } + builder.optional_string(event.error.as_ref()); + } + builder.metric("download_event_count", downloads.len() as u64); + builder.metric("download_error_count", error_count); + builder.metric("download_bytes_field_count", bytes_count); + builder.metric("download_objects_stat_count", objects_stat_count); + builder.finish() +} + +fn estimate_current_repo_objects_graph( + objects: &[crate::current_repo_index::CurrentRepoObject], +) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new("current_repo_objects"); + builder.items( + objects.len(), + std::mem::size_of::(), + ); + let mut object_type_count = 0u64; + for object in objects { + builder.owned_string(&object.rsync_uri); + builder.owned_string(&object.current_hash_hex); + builder.owned_string(&object.repository_source); + if object.object_type.is_some() { + object_type_count += 1; + } + builder.optional_string(object.object_type.as_ref()); + } + builder.metric("current_repo_object_count", objects.len() as u64); + builder.metric("current_repo_object_type_count", object_type_count); + builder.finish() +} + +fn estimate_trust_anchor_graph(shared: &PostValidationShared) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new("trust_anchors_and_tal_inputs"); + builder.items( + 1, + std::mem::size_of::(), + ); + estimate_discovered_root(&mut builder, &shared.discovery); + builder.items( + shared.discoveries.len(), + std::mem::size_of::(), + ); + for discovery in shared.discoveries.iter() { + estimate_discovered_root(&mut builder, discovery); + } + builder.items( + shared.successful_tal_inputs.len(), + std::mem::size_of::(), + ); + for tal_input in shared.successful_tal_inputs.iter() { + estimate_tal_input(&mut builder, tal_input); + } + builder.metric("discoveries_count", shared.discoveries.len() as u64); + builder.metric( + "successful_tal_inputs_count", + shared.successful_tal_inputs.len() as u64, + ); + builder.finish() +} + +fn estimate_discovered_root( + builder: &mut ObjectGraphSectionBuilder, + discovery: &crate::validation::from_tal::DiscoveredRootCaInstance, +) { + builder.optional_string(discovery.tal_url.as_ref()); + estimate_trust_anchor(builder, &discovery.trust_anchor); + builder.owned_string(&discovery.ca_instance.rsync_base_uri); + builder.owned_string(&discovery.ca_instance.manifest_rsync_uri); + builder.owned_string(&discovery.ca_instance.publication_point_rsync_uri); + builder.optional_string(discovery.ca_instance.rrdp_notification_uri.as_ref()); +} + +fn estimate_trust_anchor( + builder: &mut ObjectGraphSectionBuilder, + trust_anchor: &crate::data_model::ta::TrustAnchor, +) { + builder.byte_vec_owned(&trust_anchor.tal.raw); + builder.string_vec_owned(&trust_anchor.tal.comments); + builder.vec_header_with_capacity( + trust_anchor.tal.ta_uris.len(), + trust_anchor.tal.ta_uris.capacity(), + std::mem::size_of::(), + ); + for uri in &trust_anchor.tal.ta_uris { + builder.string(uri.as_str()); + } + builder.byte_vec_owned(&trust_anchor.tal.subject_public_key_info_der); + builder.byte_vec_owned(&trust_anchor.ta_certificate.raw_der); + if let Some(uri) = &trust_anchor.resolved_ta_uri { + builder.string(uri.as_str()); + } +} + +fn estimate_tal_input(builder: &mut ObjectGraphSectionBuilder, tal_input: &TalInputSpec) { + builder.owned_string(&tal_input.tal_id); + builder.owned_string(&tal_input.rir_id); + match &tal_input.source { + crate::parallel::types::TalSource::Url(url) => builder.owned_string(url), + crate::parallel::types::TalSource::DerBytes { + tal_url, + tal_bytes, + ta_der, + } => { + builder.owned_string(tal_url); + builder.byte_vec_owned(tal_bytes); + builder.byte_vec_owned(ta_der); + } + crate::parallel::types::TalSource::FilePath(path) => { + builder.string(&path.to_string_lossy()); + } + crate::parallel::types::TalSource::FilePathWithTa { tal_path, ta_path } => { + builder.string(&tal_path.to_string_lossy()); + builder.string(&ta_path.to_string_lossy()); + } + } +} + +fn estimate_ccr_accumulator_graph( + accumulator: Option<&CcrAccumulator>, +) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new("ccr_accumulator"); + if let Some(accumulator) = accumulator { + builder.items(1, std::mem::size_of::()); + let stats = accumulator.memory_stats(); + builder.heap_bytes(stats.estimated_heap_bytes as usize); + builder.metric("trust_anchor_count", stats.trust_anchor_count); + builder.metric("manifest_count", stats.manifest_count); + builder.metric("string_bytes", stats.string_bytes); + builder.metric("string_capacity_bytes", stats.string_capacity_bytes); + builder.metric("vec_payload_bytes", stats.vec_payload_bytes); + builder.metric("vec_capacity_bytes", stats.vec_capacity_bytes); + builder.metric("locations_der_count", stats.locations_der_count); + builder.metric("subordinate_ski_count", stats.subordinate_ski_count); + builder.metric("btree_key_capacity_bytes", stats.btree_key_capacity_bytes); + builder.metric("btree_entry_shallow_bytes", stats.btree_entry_shallow_bytes); + } else { + builder.metric("manifest_count", 0); + } + builder.finish() +} diff --git a/crates/panda-rpki-validator/src/cli/report.rs b/crates/panda-rpki-validator/src/cli/report.rs new file mode 100644 index 0000000..ee0c75d --- /dev/null +++ b/crates/panda-rpki-validator/src/cli/report.rs @@ -0,0 +1,218 @@ +// Audit report construction and online validation orchestration. + +#[cfg(test)] +fn build_report( + policy: &Policy, + validation_time: time::OffsetDateTime, + shared: &PostValidationShared, +) -> AuditReportV2 { + use time::format_description::well_known::Rfc3339; + let validation_time_rfc3339_utc = validation_time + .to_offset(time::UtcOffset::UTC) + .format(&Rfc3339) + .expect("format validation_time"); + + let vrps = shared + .vrps + .iter() + .map(|v| VrpOutput { + asn: v.asn, + prefix: format_roa_ip_prefix(&v.prefix), + max_length: v.max_length, + }) + .collect::>(); + + let aspas = shared + .aspas + .iter() + .map(|a| AspaOutput { + customer_as_id: a.customer_as_id, + provider_as_ids: a.provider_as_ids.clone(), + }) + .collect::>(); + + let repo_sync_stats = build_repo_sync_stats(shared.publication_points.as_ref()); + + AuditReportV2 { + format_version: 2, + meta: AuditRunMeta { + validation_time_rfc3339_utc, + }, + policy: policy.clone(), + tree: TreeSummary { + instances_processed: shared.instances_processed, + instances_failed: shared.instances_failed, + warnings: shared + .tree_warnings + .iter() + .map(AuditWarning::from) + .collect(), + }, + publication_points: shared.publication_points.iter().cloned().collect(), + vrps, + aspas, + downloads: shared.downloads.iter().cloned().collect(), + download_stats: shared.download_stats.clone(), + repo_sync_stats, + query_audit: None, + } +} + +fn run_online_validation_with_fetchers( + store: Arc, + policy: &Policy, + args: &CliArgs, + http: &H, + rsync: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + collect_current_repo_objects: bool, + timing: Option<&TimingHandle>, +) -> Result +where + H: crate::sync::rrdp::Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + // The multi-TAL entry point preserves the TAL id supplied by the CLI. + // A single-file TAL may otherwise derive its id from the embedded TA URI, + // which is intentionally different from the local filename used for + // adjacent .constraints discovery. + if args.tal_inputs.len() > 1 || !policy.ta_constraints.is_empty() { + return if let Some(t) = timing { + run_tree_from_multiple_tals_parallel_phase2_audit_with_timing( + store, + policy, + args.tal_inputs.clone(), + http, + rsync, + validation_time, + config, + args.parallel_phase1_config.clone(), + args.parallel_phase2_config.clone(), + collect_current_repo_objects, + t, + ) + } else { + run_tree_from_multiple_tals_parallel_phase2_audit( + store, + policy, + args.tal_inputs.clone(), + http, + rsync, + validation_time, + config, + args.parallel_phase1_config.clone(), + args.parallel_phase2_config.clone(), + collect_current_repo_objects, + ) + } + .map_err(|e| e.to_string()); + } + + match ( + args.tal_url.as_ref(), + args.tal_path.as_ref(), + args.ta_path.as_ref(), + ) { + (Some(url), _, _) => if let Some(t) = timing { + run_tree_from_tal_url_parallel_phase2_audit_with_timing( + store, + policy, + url, + http, + rsync, + validation_time, + config, + args.parallel_phase1_config.clone(), + args.parallel_phase2_config.clone(), + collect_current_repo_objects, + t, + ) + } else { + run_tree_from_tal_url_parallel_phase2_audit( + store, + policy, + url, + http, + rsync, + validation_time, + config, + args.parallel_phase1_config.clone(), + args.parallel_phase2_config.clone(), + collect_current_repo_objects, + ) + } + .map_err(|e| e.to_string()), + (None, Some(tal_path), Some(ta_path)) => { + let tal_bytes = std::fs::read(tal_path) + .map_err(|e| format!("read tal failed: {}: {e}", tal_path.display()))?; + let ta_der = std::fs::read(ta_path) + .map_err(|e| format!("read ta failed: {}: {e}", ta_path.display()))?; + if let Some(t) = timing { + run_tree_from_tal_and_ta_der_parallel_phase2_audit_with_timing( + store, + policy, + &tal_bytes, + &ta_der, + None, + http, + rsync, + validation_time, + config, + args.parallel_phase1_config.clone(), + args.parallel_phase2_config.clone(), + collect_current_repo_objects, + t, + ) + } else { + run_tree_from_tal_and_ta_der_parallel_phase2_audit( + store, + policy, + &tal_bytes, + &ta_der, + None, + http, + rsync, + validation_time, + config, + args.parallel_phase1_config.clone(), + args.parallel_phase2_config.clone(), + collect_current_repo_objects, + ) + } + .map_err(|e| e.to_string()) + } + (None, Some(tal_path), None) => { + let tal_bytes = std::fs::read(tal_path) + .map_err(|e| format!("read tal failed: {}: {e}", tal_path.display()))?; + let tal_uri = args.cir_tal_uri.clone(); + if let Some(t) = timing { + crate::validation::run_tree_from_tal::run_tree_from_tal_bytes_serial_audit_with_timing( + store.as_ref(), + policy, + &tal_bytes, + tal_uri, + http, + rsync, + validation_time, + config, + t, + ) + .map_err(|e| e.to_string()) + } else { + crate::validation::run_tree_from_tal::run_tree_from_tal_bytes_serial_audit( + store.as_ref(), + policy, + &tal_bytes, + tal_uri, + http, + rsync, + validation_time, + config, + ) + .map_err(|e| e.to_string()) + } + } + _ => unreachable!("validated by parse_args"), + } +} diff --git a/crates/panda-rpki-validator/src/cli/report_tasks.rs b/crates/panda-rpki-validator/src/cli/report_tasks.rs new file mode 100644 index 0000000..21761df --- /dev/null +++ b/crates/panda-rpki-validator/src/cli/report_tasks.rs @@ -0,0 +1,180 @@ +//! Isolated CLI output tasks. +//! +//! Report, CCR, CIR input selection, and repository-sync aggregation have no +//! dependency on network execution. Keeping them here lets the CLI +//! orchestrator own scheduling while these functions retain focused tests. + +use std::path::Path; + +use crate::audit::AuditRepoSyncStats; +use crate::ccr::{CcrBuildBreakdown, build_ccr_from_run_with_breakdown, write_ccr_file}; +use crate::policy::Policy; +use crate::storage::RocksStore; + +use super::output::{ReportJsonFormat, write_report_json_from_shared}; +use super::{CliArgs, PostValidationShared}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct ReportTaskOutput { + pub(super) report_build_ms: u64, + pub(super) report_write_ms: Option, +} + +impl ReportTaskOutput { + pub(super) fn skipped() -> Self { + Self { + report_build_ms: 0, + report_write_ms: None, + } + } +} + +pub(super) fn run_report_task( + policy: &Policy, + validation_time: time::OffsetDateTime, + shared: &PostValidationShared, + report_json_path: Option<&Path>, + report_json_format: ReportJsonFormat, +) -> Result { + if let Some(path) = report_json_path { + let timing = write_report_json_from_shared( + path, + policy, + validation_time, + shared, + report_json_format, + )?; + Ok(ReportTaskOutput { + report_build_ms: timing.build_ms, + report_write_ms: Some(timing.write_ms), + }) + } else { + Ok(ReportTaskOutput::skipped()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct CcrTaskOutput { + pub(super) ccr_build_ms: Option, + pub(super) ccr_build_breakdown: Option, + pub(super) ccr_write_ms: Option, +} + +pub(super) fn run_ccr_task( + store: &RocksStore, + shared: &PostValidationShared, + ccr_out_path: Option<&Path>, + produced_at: time::OffsetDateTime, +) -> Result { + let mut ccr_build_ms = None; + let mut ccr_build_breakdown = None; + let mut ccr_write_ms = None; + if let Some(path) = ccr_out_path { + let started = std::time::Instant::now(); + let (ccr, build_breakdown) = if let Some(accumulator) = shared.ccr_accumulator.as_ref() { + ( + accumulator + .finish( + produced_at, + shared.vrps.as_ref(), + shared.aspas.as_ref(), + shared.router_keys.as_ref(), + ) + .map_err(|e| e.to_string())?, + None, + ) + } else { + let trust_anchors = shared.trust_anchors(); + let (ccr, build_breakdown) = build_ccr_from_run_with_breakdown( + store, + &trust_anchors, + shared.vrps.as_ref(), + shared.aspas.as_ref(), + shared.router_keys.as_ref(), + produced_at, + ) + .map_err(|e| e.to_string())?; + (ccr, Some(build_breakdown)) + }; + ccr_build_ms = Some(started.elapsed().as_millis() as u64); + ccr_build_breakdown = build_breakdown; + let started = std::time::Instant::now(); + write_ccr_file(path, &ccr).map_err(|e| e.to_string())?; + ccr_write_ms = Some(started.elapsed().as_millis() as u64); + eprintln!("wrote CCR: {}", path.display()); + } + + Ok(CcrTaskOutput { + ccr_build_ms, + ccr_build_breakdown, + ccr_write_ms, + }) +} + +pub(super) fn resolve_cir_export_tal_uris(args: &CliArgs) -> Result, String> { + if !args.cir_tal_uris.is_empty() { + return Ok(args.cir_tal_uris.clone()); + } + if !args.tal_urls.is_empty() { + return Ok(args.tal_urls.clone()); + } + Err("CIR export requires TAL URI source(s)".to_string()) +} + +pub(super) fn effective_cir_tal_uris_for_discoveries( + args: &CliArgs, + shared: &PostValidationShared, + cir_tal_uris: Vec, +) -> Result, String> { + if shared.successful_tal_inputs.is_empty() { + return Ok(cir_tal_uris); + } + if cir_tal_uris.len() == shared.discoveries.len() { + return Ok(cir_tal_uris); + } + if cir_tal_uris.len() != args.tal_inputs.len() { + return Ok(cir_tal_uris); + } + + let mut mapped = Vec::with_capacity(shared.successful_tal_inputs.len()); + for successful in shared.successful_tal_inputs.iter() { + let input_index = args + .tal_inputs + .iter() + .position(|candidate| candidate == successful) + .ok_or_else(|| { + format!( + "successful TAL '{}' was not found in original TAL input list", + successful.tal_id + ) + })?; + mapped.push(cir_tal_uris[input_index].clone()); + } + Ok(mapped) +} + +pub(super) fn build_repo_sync_stats( + publication_points: &[crate::audit::PublicationPointAudit], +) -> AuditRepoSyncStats { + let mut stats = AuditRepoSyncStats { + publication_points_total: publication_points.len() as u64, + ..AuditRepoSyncStats::default() + }; + + for pp in publication_points { + let duration = pp.repo_sync_duration_ms.unwrap_or(0); + if let Some(phase) = pp.repo_sync_phase.as_ref() { + let entry = stats.by_phase.entry(phase.clone()).or_default(); + entry.count += 1; + entry.duration_ms_total += duration; + } + let entry = stats + .by_terminal_state + .entry(pp.repo_terminal_state.clone()) + .or_default(); + entry.count += 1; + entry.duration_ms_total += duration; + } + + stats +} diff --git a/crates/panda-rpki-validator/src/cli/run.rs b/crates/panda-rpki-validator/src/cli/run.rs new file mode 100644 index 0000000..885c49a --- /dev/null +++ b/crates/panda-rpki-validator/src/cli/run.rs @@ -0,0 +1,777 @@ +// Top-level CLI execution pipeline. + +pub fn run(argv: &[String]) -> Result<(), String> { + let mut args = parse_args(argv)?; + let mut policy = read_policy(args.policy_path.as_deref())?; + if let Some(strict_policy) = args.strict_policy { + policy.strict = strict_policy; + } + if let Some(resource_validation_mode) = args.resource_validation_mode { + policy.resource_validation_mode = resource_validation_mode; + } + if args.disable_rrdp { + policy.sync_preference = crate::policy::SyncPreference::RsyncOnly; + } + policy.ta_constraints = args.ta_constraints.clone(); + for warning in policy.ta_constraints.configuration_warnings() { + eprintln!("warning: {warning}"); + } + let validation_time = args + .validation_time + .unwrap_or_else(time::OffsetDateTime::now_utc); + let validation_time = + time::OffsetDateTime::from_unix_timestamp(validation_time.unix_timestamp()) + .map_err(|error| format!("normalize validation time failed: {error}"))?; + let http_root_certificates_pem = args + .http_root_cert_paths + .iter() + .map(|path| { + std::fs::read(path) + .map_err(|e| format!("read HTTP root certificate failed: {}: {e}", path.display())) + }) + .collect::, _>>()?; + + let store = if args.raw_store_db.is_some() || args.repo_bytes_db.is_some() { + Arc::new( + RocksStore::open_with_external_stores( + &args.db_path, + args.raw_store_db.as_deref(), + args.repo_bytes_db.as_deref(), + ) + .map_err(|e| e.to_string())?, + ) + } else { + Arc::new(RocksStore::open(&args.db_path).map_err(|e| e.to_string())?) + }; + let config = TreeRunConfig { + max_depth: Some(args.max_ca_depth), + max_instances: args.max_instances, + compact_audit: args.skip_report_build + && args.report_json_path.is_none() + && !args.cir_enabled, + persist_vcir: !args.skip_vcir_persist, + build_ccr_accumulator: args.ccr_out_path.is_some(), + enable_roa_validation_cache: args.enable_roa_validation_cache, + enable_child_certificate_validation_cache: args.enable_child_certificate_validation_cache, + publication_point_cache_observe_only: args.publication_point_cache_observe_only, + enable_publication_point_validation_cache: args.enable_publication_point_validation_cache, + enable_transport_request_prefetch: args.enable_transport_request_prefetch, + }; + let replay_mode = args.payload_replay_archive.is_some(); + let delta_replay_mode = args.payload_base_archive.is_some(); + + use time::format_description::well_known::Rfc3339; + let mut timing: Option<(std::path::PathBuf, TimingHandle)> = None; + if args.analyze { + let recorded_at_utc_rfc3339 = time::OffsetDateTime::now_utc() + .to_offset(time::UtcOffset::UTC) + .format(&Rfc3339) + .map_err(|e| format!("format recorded_at_utc failed: {e}"))?; + let validation_time_utc_rfc3339 = validation_time + .to_offset(time::UtcOffset::UTC) + .format(&Rfc3339) + .map_err(|e| format!("format validation_time failed: {e}"))?; + + let ts_compact = { + let fmt = time::format_description::parse("[year][month][day]T[hour][minute][second]Z") + .map_err(|e| format!("format description parse failed: {e}"))?; + time::OffsetDateTime::now_utc() + .format(&fmt) + .map_err(|e| format!("format timestamp failed: {e}"))? + }; + + let out_dir = args.analysis_out_path.clone().unwrap_or_else(|| { + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("target") + .join("live") + .join("analyze") + .join(ts_compact) + }); + std::fs::create_dir_all(&out_dir) + .map_err(|e| format!("create analyze out dir failed: {}: {e}", out_dir.display()))?; + + let handle = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339, + validation_time_utc_rfc3339, + tal_url: None, + db_path: None, + }); + handle.set_meta(TimingMetaUpdate { + tal_url: args.tal_url.as_deref(), + db_path: Some(args.db_path.to_string_lossy().as_ref()), + }); + timing = Some((out_dir, handle)); + } + + if args.profile_cpu && !args.analyze { + return Err("--profile-cpu requires --analyze".to_string()); + } + + #[cfg(not(feature = "profile"))] + if args.profile_cpu { + return Err("CPU profiling requires building with: --features profile".to_string()); + } + + #[cfg(feature = "profile")] + let mut profiler_guard: Option> = if args.profile_cpu { + Some( + pprof::ProfilerGuard::new(100) + .map_err(|e| format!("pprof ProfilerGuard init failed: {e}"))?, + ) + } else { + None + }; + + let total_started = std::time::Instant::now(); + let mut memory_checkpoints: Vec = Vec::new(); + let mut malloc_trim_probes: Vec = Vec::new(); + let enable_memory_trim_probe = memory_trim_probe_enabled() || args.memory_trim_after_validation; + record_memory_checkpoint( + &mut memory_checkpoints, + "after_store_open", + &total_started, + store.as_ref(), + ); + let validation_started = std::time::Instant::now(); + let crypto_sig_cache = + if args.crypto_signature_cache_observe_only || args.enable_crypto_signature_cache { + let cache_file = crate::crypto_sig_cache::default_cache_file_path(&args.db_path); + let cache = Arc::new(crate::crypto_sig_cache::CryptoSigCache::load_or_rebuild( + cache_file, + args.enable_crypto_signature_cache, + )); + crate::crypto_sig_cache::install_global(Arc::clone(&cache)); + Some(cache) + } else { + None + }; + let collect_current_repo_objects = false; + let out = if delta_replay_mode { + let tal_path = args + .tal_path + .as_ref() + .expect("validated by parse_args for delta replay mode"); + let ta_path = args + .ta_path + .as_ref() + .expect("validated by parse_args for delta replay mode"); + let base_archive = args + .payload_base_archive + .as_ref() + .expect("validated by parse_args for delta replay mode"); + let base_locks = args + .payload_base_locks + .as_ref() + .expect("validated by parse_args for delta replay mode"); + let base_validation_time = args.payload_base_validation_time.unwrap_or(validation_time); + let delta_archive = args + .payload_delta_archive + .as_ref() + .expect("validated by parse_args for delta replay mode"); + let delta_locks = args + .payload_delta_locks + .as_ref() + .expect("validated by parse_args for delta replay mode"); + let tal_bytes = std::fs::read(tal_path) + .map_err(|e| format!("read tal failed: {}: {e}", tal_path.display()))?; + let ta_der = std::fs::read(ta_path) + .map_err(|e| format!("read ta failed: {}: {e}", ta_path.display()))?; + if let Some((_, t)) = timing.as_ref() { + run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit_with_timing( + store.as_ref(), + &policy, + &tal_bytes, + &ta_der, + None, + base_archive, + base_locks, + delta_archive, + delta_locks, + base_validation_time, + validation_time, + &config, + t, + ) + .map_err(|e| e.to_string())? + } else { + run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit( + store.as_ref(), + &policy, + &tal_bytes, + &ta_der, + None, + base_archive, + base_locks, + delta_archive, + delta_locks, + base_validation_time, + validation_time, + &config, + ) + .map_err(|e| e.to_string())? + } + } else if replay_mode { + let tal_path = args + .tal_path + .as_ref() + .expect("validated by parse_args for replay mode"); + let ta_path = args + .ta_path + .as_ref() + .expect("validated by parse_args for replay mode"); + let archive_root = args + .payload_replay_archive + .as_ref() + .expect("validated by parse_args for replay mode"); + let locks_path = args + .payload_replay_locks + .as_ref() + .expect("validated by parse_args for replay mode"); + let tal_bytes = std::fs::read(tal_path) + .map_err(|e| format!("read tal failed: {}: {e}", tal_path.display()))?; + let ta_der = std::fs::read(ta_path) + .map_err(|e| format!("read ta failed: {}: {e}", ta_path.display()))?; + if let Some((_, t)) = timing.as_ref() { + run_tree_from_tal_and_ta_der_payload_replay_serial_audit_with_timing( + store.as_ref(), + &policy, + &tal_bytes, + &ta_der, + None, + archive_root, + locks_path, + validation_time, + &config, + t, + ) + .map_err(|e| e.to_string())? + } else { + run_tree_from_tal_and_ta_der_payload_replay_serial_audit( + store.as_ref(), + &policy, + &tal_bytes, + &ta_der, + None, + archive_root, + locks_path, + validation_time, + &config, + ) + .map_err(|e| e.to_string())? + } + } else if let Some(dir) = args.rsync_local_dir.as_ref() { + let http = BlockingHttpFetcher::new(HttpFetcherConfig { + timeout: std::time::Duration::from_secs(args.http_timeout_secs.max(1)), + extra_root_certificates_pem: http_root_certificates_pem.clone(), + ..HttpFetcherConfig::default() + }) + .map_err(|e| e.to_string())?; + let rsync = LocalDirRsyncFetcher::new(dir); + run_online_validation_with_fetchers( + Arc::clone(&store), + &policy, + &args, + &http, + &rsync, + validation_time, + &config, + collect_current_repo_objects, + timing.as_ref().map(|(_, t)| t), + )? + } else { + let http = BlockingHttpFetcher::new(HttpFetcherConfig { + timeout: std::time::Duration::from_secs(args.http_timeout_secs.max(1)), + extra_root_certificates_pem: http_root_certificates_pem.clone(), + ..HttpFetcherConfig::default() + }) + .map_err(|e| e.to_string())?; + let rsync = SystemRsyncFetcher::new(SystemRsyncConfig { + rsync_bin: args + .rsync_command + .clone() + .unwrap_or_else(|| PathBuf::from("rsync")), + timeout: std::time::Duration::from_secs(args.rsync_timeout_secs.max(1)), + mirror_root: args.rsync_mirror_root.clone(), + scope_policy: args.rsync_scope_policy, + ..SystemRsyncConfig::default() + }); + run_online_validation_with_fetchers( + Arc::clone(&store), + &policy, + &args, + &http, + &rsync, + validation_time, + &config, + collect_current_repo_objects, + timing.as_ref().map(|(_, t)| t), + )? + }; + + let validation_ms = validation_started.elapsed().as_millis() as u64; + let mut shared = PostValidationShared::from_run_output(out); + let vcir_storage_summary_enabled = vcir_storage_summary_enabled(); + let vcir_storage_summary_started = std::time::Instant::now(); + let vcir_storage = if config.persist_vcir && vcir_storage_summary_enabled { + Some( + store + .summarize_vcir_storage() + .map_err(|e| format!("summarize VCIR storage failed: {e}"))?, + ) + } else { + None + }; + let vcir_storage_summary_ms = (config.persist_vcir && vcir_storage_summary_enabled) + .then(|| vcir_storage_summary_started.elapsed().as_millis() as u64); + record_memory_checkpoint( + &mut memory_checkpoints, + "after_validation", + &total_started, + store.as_ref(), + ); + if enable_memory_trim_probe { + malloc_trim_probes.push(crate::memory_telemetry::malloc_trim_probe()); + record_memory_checkpoint( + &mut memory_checkpoints, + "after_validation_malloc_trim", + &total_started, + store.as_ref(), + ); + } + + if let Some((_out_dir, t)) = timing.as_ref() { + t.record_count("instances_processed", shared.instances_processed as u64); + t.record_count("instances_failed", shared.instances_failed as u64); + } + + let publication_points = shared.publication_points.len(); + let publication_point_repo_sync_ms_total: u64 = shared + .publication_points + .iter() + .map(|pp| pp.repo_sync_duration_ms.unwrap_or(0)) + .sum(); + let download_event_count = shared.download_stats.events_total; + let rrdp_download_ms_total: u64 = ["rrdp_notification", "rrdp_snapshot", "rrdp_delta"] + .iter() + .map(|key| { + shared + .download_stats + .by_kind + .get(*key) + .map(|item| item.duration_ms_total) + .unwrap_or(0) + }) + .sum(); + let rsync_download_ms_total = shared + .download_stats + .by_kind + .get("rsync") + .map(|item| item.duration_ms_total) + .unwrap_or(0); + let repo_sync_ms_total = rrdp_download_ms_total + rsync_download_ms_total; + let download_bytes_total: u64 = shared + .download_stats + .by_kind + .values() + .map(|item| item.bytes_total.unwrap_or(0)) + .sum(); + + #[cfg(feature = "profile")] + let profiler_report = if let Some(guard) = profiler_guard.take() { + Some( + guard + .report() + .build() + .map_err(|e| format!("pprof report build failed: {e}"))?, + ) + } else { + None + }; + + let report_json_format = if args.report_json_compact { + ReportJsonFormat::Compact + } else { + ReportJsonFormat::Pretty + }; + let ccr_produced_at = time::OffsetDateTime::now_utc(); + let compare_view_trust_anchor = args + .compare_view_trust_anchor + .as_deref() + .unwrap_or("unknown"); + let cir_tal_uris = if args.cir_enabled { + Some(effective_cir_tal_uris_for_discoveries( + &args, + &shared, + resolve_cir_export_tal_uris(&args)?, + )?) + } else { + None + }; + let cir_out_path = if args.cir_enabled { + Some( + args.cir_out_path + .as_deref() + .expect("validated by parse_args for cir"), + ) + } else { + None + }; + // Take the CIR input snapshot before the output stage so the CIR export can + // run inside the same scoped-thread group as report/ccr/compare_view; no + // other output task reads `shared.cir_input`. + let cir_input_owned = args + .cir_enabled + .then(|| std::mem::take(&mut shared.cir_input)); + let (report_result, ccr_result, compare_view_result, cir_result) = + std::thread::scope(|scope| { + // Reborrow `shared` as a plain reference so the scoped output tasks + // (incl. the `move` CIR task) capture the reference instead of + // moving fields out of the owned value. + let shared = &shared; + let report_handle = if args.skip_report_build { + None + } else { + Some(scope.spawn(|| { + run_report_task( + &policy, + validation_time, + shared, + args.report_json_path.as_deref(), + report_json_format, + ) + })) + }; + let ccr_handle = scope.spawn(|| { + run_ccr_task( + store.as_ref(), + shared, + args.ccr_out_path.as_deref(), + ccr_produced_at, + ) + }); + let compare_view_handle = scope.spawn(|| { + run_compare_view_task( + shared, + args.vrps_csv_out_path.as_deref(), + args.vaps_csv_out_path.as_deref(), + compare_view_trust_anchor, + ) + }); + let cir_handle = match (cir_tal_uris.as_ref(), cir_out_path, cir_input_owned) { + (Some(cir_tal_uris), Some(cir_out_path), Some(cir_input)) => { + Some(scope.spawn(move || { + if cir_tal_uris.len() != shared.discoveries.len() { + return Err(format!( + "CIR export TAL URI count ({}) does not match discovery count ({})", + cir_tal_uris.len(), + shared.discoveries.len() + )); + } + let tal_bindings = shared + .discoveries + .iter() + .zip(cir_tal_uris.iter()) + .map(|(discovery, tal_uri)| CirTrustAnchorBinding { + trust_anchor: &discovery.trust_anchor, + tal_uri: tal_uri.as_str(), + }) + .collect::>(); + export_cir_from_input_snapshot_multi( + &tal_bindings, + validation_time, + cir_input, + cir_out_path, + ) + .map_err(|e| e.to_string()) + })) + } + _ => None, + }; + let report_result = match report_handle { + Some(handle) => handle + .join() + .map_err(|_| "report task panicked".to_string()) + .and_then(|result| result), + None => Ok(ReportTaskOutput::skipped()), + }; + let ccr_result = ccr_handle + .join() + .map_err(|_| "ccr task panicked".to_string()) + .and_then(|result| result); + let compare_view_result = compare_view_handle + .join() + .map_err(|_| "compare view task panicked".to_string()) + .and_then(|result| result); + let cir_result = cir_handle.map(|handle| { + handle + .join() + .map_err(|_| "cir task panicked".to_string()) + .and_then(|result| result) + }); + (report_result, ccr_result, compare_view_result, cir_result) + }); + let report_output = report_result?; + let ccr_output = ccr_result?; + let compare_view_output = compare_view_result?; + let cir_summary = match cir_result { + Some(result) => Some(result?), + None => None, + }; + record_memory_checkpoint( + &mut memory_checkpoints, + "after_report_and_ccr", + &total_started, + store.as_ref(), + ); + if enable_memory_trim_probe { + malloc_trim_probes.push(crate::memory_telemetry::malloc_trim_probe()); + record_memory_checkpoint( + &mut memory_checkpoints, + "after_report_and_ccr_malloc_trim", + &total_started, + store.as_ref(), + ); + } + let report_build_ms = report_output.report_build_ms; + let report_write_ms = report_output.report_write_ms; + let ccr_build_ms = ccr_output.ccr_build_ms; + let ccr_build_breakdown = ccr_output.ccr_build_breakdown; + let ccr_write_ms = ccr_output.ccr_write_ms; + let compare_view_build_ms = compare_view_output.build_ms; + let compare_view_write_ms = compare_view_output.write_ms; + record_memory_checkpoint( + &mut memory_checkpoints, + "after_compare_view", + &total_started, + store.as_ref(), + ); + + let mut cir_build_cir_ms = None; + let mut cir_write_cir_ms = None; + let mut cir_total_ms = None; + if let Some(summary) = cir_summary { + cir_build_cir_ms = Some(summary.timing.build_cir_ms); + cir_write_cir_ms = Some(summary.timing.write_cir_ms); + cir_total_ms = Some(summary.timing.total_ms); + eprintln!( + "wrote CIR: {} (objects={}, trust_anchors={}, build_cir_ms={}, write_cir_ms={}, total_ms={})", + cir_out_path + .expect("cir path present when cir enabled") + .display(), + summary.object_count, + summary.trust_anchor_count, + summary.timing.build_cir_ms, + summary.timing.write_cir_ms, + summary.timing.total_ms + ); + record_memory_checkpoint( + &mut memory_checkpoints, + "after_cir", + &total_started, + store.as_ref(), + ); + } + record_memory_checkpoint( + &mut memory_checkpoints, + "before_stage_timing", + &total_started, + store.as_ref(), + ); + let publication_point_cache_index_refresh = if args.enable_publication_point_validation_cache + || args.publication_point_cache_observe_only + { + match store.refresh_publication_point_cache_mmap_index() { + Ok(stats) => stats, + Err(e) => { + crate::progress_log::emit( + "publication_point_cache_mmap_index_refresh", + serde_json::json!({ + "state": "failed", + "error": e.to_string(), + }), + ); + None + } + } + } else { + None + }; + let publication_point_cache_index_load = store.publication_point_cache_mmap_index_load_stats(); + let crypto_signature_cache_observe = crypto_sig_cache.as_ref().map(|cache| { + if let Err(e) = cache.persist() { + crate::progress_log::emit( + "crypto_signature_cache_persist", + serde_json::json!({ + "state": "failed", + "error": e, + }), + ); + } + crate::crypto_sig_cache::clear_global(); + cache.summary() + }); + if let (Some((_, t)), Some(summary)) = + (timing.as_ref(), crypto_signature_cache_observe.as_ref()) + { + let (mut calls, mut would_hit, mut new_keys, mut executed, mut skipped) = + (0u64, 0u64, 0u64, 0u64, 0u64); + for stats in summary.per_point.values() { + calls += stats.calls; + would_hit += stats.would_hit; + new_keys += stats.new_keys; + executed += stats.verify_executed; + skipped += stats.verify_skipped; + } + t.record_count("crypto_signature_cache_observe_calls", calls); + t.record_count("crypto_signature_cache_observe_would_hit", would_hit); + t.record_count("crypto_signature_cache_observe_new_keys", new_keys); + t.record_count("crypto_signature_cache_verify_executed", executed); + t.record_count("crypto_signature_cache_verify_skipped", skipped); + } + let timing_report_snapshot = timing + .as_ref() + .map(|(_, handle)| handle.report_snapshot(50)); + let stage_timing = RunStageTiming { + validation_ms, + enable_roa_validation_cache: args.enable_roa_validation_cache, + enable_child_certificate_validation_cache: args.enable_child_certificate_validation_cache, + publication_point_cache_observe_only: args.publication_point_cache_observe_only, + enable_publication_point_validation_cache: args.enable_publication_point_validation_cache, + crypto_signature_cache_observe, + enable_transport_request_prefetch: args.enable_transport_request_prefetch, + report_build_ms, + report_write_ms, + ccr_build_ms, + ccr_build_breakdown, + ccr_write_ms, + compare_view_build_ms, + compare_view_write_ms, + cir_build_cir_ms, + cir_write_cir_ms, + cir_total_ms, + total_ms: total_started.elapsed().as_millis() as u64, + publication_points, + repo_sync_ms_total, + publication_point_repo_sync_ms_total, + download_event_count, + rrdp_download_ms_total, + rsync_download_ms_total, + download_bytes_total, + roa_validation_cache: shared.roa_cache_stats.clone(), + analysis_counts: timing + .as_ref() + .map(|(_, handle)| handle.counts_snapshot()) + .unwrap_or_default(), + analysis_phases: timing_report_snapshot + .as_ref() + .map(|report| report.phases.clone()) + .unwrap_or_default(), + analysis_top_publication_points: timing_report_snapshot + .as_ref() + .map(|report| report.top_publication_points.clone()) + .unwrap_or_default(), + analysis_top_publication_point_steps: timing_report_snapshot + .as_ref() + .map(|report| report.top_publication_point_steps.clone()) + .unwrap_or_default(), + analysis_top_publication_point_cache_steps: timing_report_snapshot + .as_ref() + .map(|report| { + report + .top_publication_point_steps + .iter() + .filter(|entry| entry.key.contains("::publication_point_cache_")) + .cloned() + .collect() + }) + .unwrap_or_default(), + vcir_storage_summary_ms, + vcir_storage, + publication_point_cache_index_load, + publication_point_cache_index_refresh, + memory_telemetry: Some(MemoryTelemetrySummary { + checkpoints: memory_checkpoints, + object_graph: Some(estimate_shared_object_graph(&shared)), + malloc_trim_probes, + }), + }; + let stage_timing_anchor_path = args + .report_json_path + .as_deref() + .or(args.ccr_out_path.as_deref()) + .or(args.vrps_csv_out_path.as_deref()); + write_stage_timing(stage_timing_anchor_path, &stage_timing)?; + + // Finalize the normal-run contract only after all validation outputs have + // been written successfully. This prevents incomplete runs from leaving a + // contract that looks usable to downstream replay tooling. + if let Some(path) = args.validation_contract_out_path.as_deref() { + let mut contract = crate::contract::ValidationContract::for_current_binary( + validation_time, + policy.clone(), + args.max_ca_depth, + args.max_instances, + crate::contract::ValidationCacheContract { + publication_point: args.enable_publication_point_validation_cache, + roa: args.enable_roa_validation_cache, + child_certificate: args.enable_child_certificate_validation_cache, + transport_prefetch: args.enable_transport_request_prefetch, + crypto_signature: args.enable_crypto_signature_cache, + }, + args.rsync_scope_policy, + )?; + if !policy.ta_constraints.is_empty() { + contract.ta_constraints_fingerprint = + Some(policy.ta_constraints.fingerprint_sha256_hex()); + } + crate::contract::write_validation_contract(path, &contract)?; + } + + if let Some((out_dir, t)) = timing.as_ref() { + t.record_count("vrps", shared.vrps.len() as u64); + t.record_count("aspas", shared.aspas.len() as u64); + t.record_count( + "audit_publication_points", + shared.publication_points.len() as u64, + ); + let timing_json_path = out_dir.join("timing.json"); + t.write_json(&timing_json_path, 20)?; + eprintln!("analysis: wrote {}", timing_json_path.display()); + } + + #[cfg(feature = "profile")] + if let (Some((out_dir, _)), Some(report)) = (timing.as_ref(), profiler_report) { + let svg_path = out_dir.join("flamegraph.svg"); + let svg_file = std::fs::File::create(&svg_path) + .map_err(|e| format!("create flamegraph failed: {}: {e}", svg_path.display()))?; + report + .flamegraph(svg_file) + .map_err(|e| format!("write flamegraph failed: {e}"))?; + eprintln!("analysis: wrote {}", svg_path.display()); + + let pb_path = out_dir.join("pprof.pb.gz"); + let pprof_profile = report + .pprof() + .map_err(|e| format!("pprof export failed: {e}"))?; + use pprof::protos::Message; + let mut body = Vec::with_capacity(pprof_profile.encoded_len()); + pprof_profile + .encode(&mut body) + .map_err(|e| format!("pprof encode failed: {e}"))?; + let gz = flate2::write::GzEncoder::new( + std::fs::File::create(&pb_path) + .map_err(|e| format!("create pprof.pb.gz failed: {}: {e}", pb_path.display()))?, + flate2::Compression::default(), + ); + let mut gz = gz; + use std::io::Write; + gz.write_all(&body) + .map_err(|e| format!("write pprof.pb.gz failed: {e}"))?; + gz.finish() + .map_err(|e| format!("finish pprof.pb.gz failed: {e}"))?; + eprintln!("analysis: wrote {}", pb_path.display()); + } + + print_summary_from_shared(validation_time, &shared); + Ok(()) +} diff --git a/crates/panda-rpki-validator/src/cli/tests.rs b/crates/panda-rpki-validator/src/cli/tests.rs index 5a7f702..b542b49 100644 --- a/crates/panda-rpki-validator/src/cli/tests.rs +++ b/crates/panda-rpki-validator/src/cli/tests.rs @@ -1,2217 +1,5 @@ -use super::*; -use crate::data_model::oid::OID_AD_SIGNED_OBJECT; -use crate::data_model::rc::AccessDescription; -use crate::memory_telemetry::{ - MemoryTelemetryCheckpoint, MemoryTelemetrySummary, ProcessMemorySnapshot, -}; -use crate::policy::ResourceValidationMode; -use crate::storage::{ - RocksDbMemorySnapshot, RocksDbMemoryTotals, VcirCcrProjectionSizeBreakdown, - VcirChildResourceSizeBreakdown, VcirCoreFieldSizeBreakdown, VcirFieldSizeBreakdown, - VcirStorageEntrySummary, VcirStorageSummary, -}; - -#[test] -fn parse_help_returns_usage() { - let argv = vec!["rpki".to_string(), "--help".to_string()]; - let err = parse_args(&argv).unwrap_err(); - assert!(err.contains("Usage:"), "{err}"); - assert!(err.contains("--db"), "{err}"); - assert!(err.contains("--rsync-mirror-root"), "{err}"); - assert!(err.contains("--rsync-scope"), "{err}"); - assert!(err.contains("--parallel-phase2-object-workers"), "{err}"); - assert!(err.contains("--memory-trim-after-validation"), "{err}"); - assert!(err.contains("--enable-roa-validation-cache"), "{err}"); - assert!(err.contains("--resource-validation-mode"), "{err}"); - assert!(err.contains("--ta-constraints"), "{err}"); - assert!(err.contains("--max-ca-depth"), "{err}"); - assert!(err.contains("default: 32"), "{err}"); - assert!( - err.contains("--publication-point-cache-observe-only"), - "{err}" - ); - assert!( - err.contains("--enable-publication-point-validation-cache"), - "{err}" - ); - assert!(!err.contains("--parallel-phase1"), "{err}"); - assert!(!err.contains("--parallel-phase2 "), "{err}"); -} - -#[test] -fn parse_accepts_explicit_ta_constraints_for_known_tal() { - let dir = tempfile::tempdir().expect("tmpdir"); - let constraints_path = dir.path().join("example.constraints"); - std::fs::write(&constraints_path, "allow 192.0.2.0/24\n").expect("write constraints"); - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "example.tal".to_string(), - "--ta-path".to_string(), - "example.cer".to_string(), - "--ta-constraints".to_string(), - format!("example={}", constraints_path.display()), - ]; - let args = parse_args(&argv).expect("parse args"); - assert!(!args.ta_constraints.is_empty()); -} - -#[test] -fn parse_rejects_unknown_argument() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--nope".to_string(), - ]; - let err = parse_args(&argv).unwrap_err(); - assert!(err.contains("unknown argument"), "{err}"); -} - -#[test] -fn parse_accepts_normal_validation_contract_output() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/root.tal".to_string(), - "--validation-contract-out".to_string(), - "out/validation-contract.json".to_string(), - ]; - let args = parse_args(&argv).expect("parse normal contract output"); - assert_eq!( - args.validation_contract_out_path, - Some(PathBuf::from("out/validation-contract.json")) - ); -} - -#[test] -fn parse_rejects_both_tal_url_and_tal_path() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--tal-path".to_string(), - "x.tal".to_string(), - ]; - let err = parse_args(&argv).unwrap_err(); - assert!( - err.contains("one-or-more --tal-url or one-or-more --tal-path/--ta-path pairs"), - "{err}" - ); -} - -#[test] -fn parse_rejects_invalid_max_depth() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--max-depth".to_string(), - "nope".to_string(), - ]; - let err = parse_args(&argv).unwrap_err(); - assert!(err.contains("invalid --max-depth"), "{err}"); -} - -#[test] -fn parse_rejects_invalid_max_ca_depth() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--max-ca-depth".to_string(), - "nope".to_string(), - ]; - let err = parse_args(&argv).unwrap_err(); - assert!(err.contains("invalid --max-ca-depth"), "{err}"); -} - -#[test] -fn parse_rejects_both_ca_depth_options() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--max-ca-depth".to_string(), - "32".to_string(), - "--max-depth".to_string(), - "12".to_string(), - ]; - let err = parse_args(&argv).unwrap_err(); - assert!(err.contains("cannot be combined"), "{err}"); -} - -#[test] -fn parse_defaults_max_ca_depth_to_32() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - ]; - let args = parse_args(&argv).expect("parse"); - assert_eq!(args.max_ca_depth, 32); -} - -#[test] -fn parse_accepts_ccr_out_path() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "x.tal".to_string(), - "--ta-path".to_string(), - "x.cer".to_string(), - "--rsync-local-dir".to_string(), - "repo".to_string(), - "--ccr-out".to_string(), - "out/example.ccr".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert_eq!( - args.ccr_out_path.as_deref(), - Some(std::path::Path::new("out/example.ccr")) - ); -} - -#[test] -fn parse_accepts_memory_trim_after_validation() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "x.tal".to_string(), - "--ta-path".to_string(), - "x.cer".to_string(), - "--memory-trim-after-validation".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert!(args.memory_trim_after_validation); -} - -#[test] -fn parse_disables_memory_trim_after_validation_by_default() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "x.tal".to_string(), - "--ta-path".to_string(), - "x.cer".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert!(!args.memory_trim_after_validation); -} - -#[test] -fn parse_accepts_enable_roa_validation_cache() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "x.tal".to_string(), - "--ta-path".to_string(), - "x.cer".to_string(), - "--enable-roa-validation-cache".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert!(args.enable_roa_validation_cache); -} - -#[test] -fn parse_accepts_enable_child_certificate_validation_cache() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "x.tal".to_string(), - "--ta-path".to_string(), - "x.cer".to_string(), - "--enable-child-certificate-validation-cache".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert!(args.enable_child_certificate_validation_cache); -} - -#[test] -fn parse_accepts_publication_point_cache_flags() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "x.tal".to_string(), - "--ta-path".to_string(), - "x.cer".to_string(), - "--publication-point-cache-observe-only".to_string(), - "--enable-publication-point-validation-cache".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert!(args.publication_point_cache_observe_only); - assert!(args.enable_publication_point_validation_cache); -} - -#[test] -fn parse_disables_crypto_signature_cache_observe_only_by_default() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "x.tal".to_string(), - "--ta-path".to_string(), - "x.cer".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert!(!args.crypto_signature_cache_observe_only); - assert!(!args.enable_crypto_signature_cache); -} - -#[test] -fn parse_accepts_crypto_signature_cache_observe_only() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "x.tal".to_string(), - "--ta-path".to_string(), - "x.cer".to_string(), - "--crypto-signature-cache-observe-only".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert!(args.crypto_signature_cache_observe_only); - assert!(!args.enable_crypto_signature_cache); -} - -#[test] -fn parse_accepts_enable_crypto_signature_cache() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "x.tal".to_string(), - "--ta-path".to_string(), - "x.cer".to_string(), - "--enable-crypto-signature-cache".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert!(args.enable_crypto_signature_cache); - assert!(!args.crypto_signature_cache_observe_only); -} - -#[test] -fn parse_accepts_enable_transport_request_prefetch() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "x.tal".to_string(), - "--ta-path".to_string(), - "x.cer".to_string(), - "--enable-transport-request-prefetch".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert!(args.enable_transport_request_prefetch); -} - -#[test] -fn parse_disables_roa_validation_cache_by_default() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "x.tal".to_string(), - "--ta-path".to_string(), - "x.cer".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert!(!args.enable_roa_validation_cache); - assert!(!args.enable_child_certificate_validation_cache); - assert!(!args.publication_point_cache_observe_only); - assert!(!args.enable_publication_point_validation_cache); - assert!(!args.enable_transport_request_prefetch); -} - -#[test] -fn parse_accepts_analysis_out_and_implies_analyze() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "x.tal".to_string(), - "--ta-path".to_string(), - "x.cer".to_string(), - "--analysis-out".to_string(), - "run/analyze".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert!(args.analyze); - assert_eq!( - args.analysis_out_path.as_deref(), - Some(std::path::Path::new("run/analyze")) - ); -} - -#[test] -fn parse_accepts_report_json_compact_when_report_json_is_set() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--report-json".to_string(), - "out/report.json".to_string(), - "--report-json-compact".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert_eq!( - args.report_json_path.as_deref(), - Some(std::path::Path::new("out/report.json")) - ); - assert!(args.report_json_compact); -} - -#[test] -fn parse_accepts_rsync_scope_policy() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--rsync-scope".to_string(), - "module-root".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert_eq!(args.rsync_scope_policy, RsyncScopePolicy::ModuleRoot); -} - -#[test] -fn parse_accepts_host_rsync_scope_policy() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--rsync-scope".to_string(), - "host".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert_eq!(args.rsync_scope_policy, RsyncScopePolicy::Host); -} - -#[test] -fn parse_accepts_repeatable_http_root_cert_paths() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--http-root-cert".to_string(), - "local-ca-1.pem".to_string(), - "--http-root-cert".to_string(), - "local-ca-2.pem".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert_eq!( - args.http_root_cert_paths, - vec![ - PathBuf::from("local-ca-1.pem"), - PathBuf::from("local-ca-2.pem") - ] - ); -} - -#[test] -fn parse_rejects_invalid_rsync_scope_policy() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--rsync-scope".to_string(), - "wide".to_string(), - ]; - let err = parse_args(&argv).expect_err("invalid rsync scope should fail"); - assert!(err.contains("invalid --rsync-scope"), "{err}"); -} - -#[test] -fn parse_accepts_strict_policy_list() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--strict".to_string(), - "name,cms-der".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert_eq!( - args.strict_policy, - Some(StrictPolicy { - name: true, - cms_der: true, - signed_attrs: false, - }) - ); -} - -#[test] -fn parse_accepts_strict_without_value_as_all() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--strict".to_string(), - "--report-json".to_string(), - "out/report.json".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert_eq!(args.strict_policy, Some(StrictPolicy::all())); -} - -#[test] -fn parse_accepts_resource_validation_mode() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--resource-validation-mode".to_string(), - "rfc6487".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert_eq!( - args.resource_validation_mode, - Some(ResourceValidationMode::Rfc6487) - ); -} - -#[test] -fn parse_rejects_unknown_resource_validation_mode() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--resource-validation-mode".to_string(), - "bogus".to_string(), - ]; - let err = parse_args(&argv).expect_err("unknown mode should fail"); - assert!(err.contains("unknown resource validation mode"), "{err}"); -} - -#[test] -fn parse_rejects_unknown_strict_policy() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--strict=unknown".to_string(), - ]; - let err = parse_args(&argv).expect_err("unknown strict policy should fail"); - assert!(err.contains("unknown strict policy"), "{err}"); -} - -#[test] -fn effective_cir_tal_uris_filters_skipped_multi_tal_inputs() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "afrinic.tal".to_string(), - "--ta-path".to_string(), - "afrinic.cer".to_string(), - "--tal-path".to_string(), - "apnic.tal".to_string(), - "--ta-path".to_string(), - "apnic.cer".to_string(), - "--tal-path".to_string(), - "arin.tal".to_string(), - "--ta-path".to_string(), - "arin.cer".to_string(), - "--cir-enable".to_string(), - "--cir-out".to_string(), - "out.cir".to_string(), - "--cir-tal-uri".to_string(), - "https://example.test/afrinic.cer".to_string(), - "--cir-tal-uri".to_string(), - "https://example.test/apnic.cer".to_string(), - "--cir-tal-uri".to_string(), - "https://example.test/arin.cer".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - let mut shared = synthetic_post_validation_shared(); - shared.discoveries = vec![shared.discovery.clone(), shared.discovery.clone()].into(); - shared.successful_tal_inputs = - vec![args.tal_inputs[0].clone(), args.tal_inputs[2].clone()].into(); - - let effective = effective_cir_tal_uris_for_discoveries( - &args, - &shared, - resolve_cir_export_tal_uris(&args).expect("resolve cir tal uris"), - ) - .expect("map effective cir tal uris"); - - assert_eq!( - effective, - vec![ - "https://example.test/afrinic.cer".to_string(), - "https://example.test/arin.cer".to_string(), - ] - ); -} - -#[test] -fn parse_rejects_report_json_compact_without_report_json() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--report-json-compact".to_string(), - ]; - let err = parse_args(&argv).expect_err("compact flag without report path should fail"); - assert!( - err.contains("--report-json-compact requires --report-json"), - "{err}" - ); -} - -#[test] -fn parse_accepts_skip_report_build_without_report_json() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--ccr-out".to_string(), - "out/result.ccr".to_string(), - "--skip-report-build".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert!(args.skip_report_build); - assert_eq!( - args.ccr_out_path.as_deref(), - Some(std::path::Path::new("out/result.ccr")) - ); -} - -#[test] -fn parse_accepts_skip_vcir_persist() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--skip-vcir-persist".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert!(args.skip_vcir_persist); -} - -#[test] -fn parse_rejects_skip_report_build_with_report_json() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--report-json".to_string(), - "out/report.json".to_string(), - "--skip-report-build".to_string(), - ]; - let err = parse_args(&argv).expect_err("skip report build with report path should fail"); - assert!( - err.contains("--skip-report-build cannot be combined with --report-json"), - "{err}" - ); -} - -#[test] -fn parse_accepts_direct_compare_view_csv_outputs() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--vrps-csv-out".to_string(), - "out/vrps.csv".to_string(), - "--vaps-csv-out".to_string(), - "out/vaps.csv".to_string(), - "--compare-view-trust-anchor".to_string(), - "unknown".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert_eq!( - args.vrps_csv_out_path.as_deref(), - Some(std::path::Path::new("out/vrps.csv")) - ); - assert_eq!( - args.vaps_csv_out_path.as_deref(), - Some(std::path::Path::new("out/vaps.csv")) - ); - assert_eq!(args.compare_view_trust_anchor.as_deref(), Some("unknown")); -} - -#[test] -fn parse_rejects_partial_direct_compare_view_csv_outputs() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--vrps-csv-out".to_string(), - "out/vrps.csv".to_string(), - ]; - let err = parse_args(&argv).expect_err("partial direct compare view output should fail"); - assert!( - err.contains("--vrps-csv-out and --vaps-csv-out must be provided together"), - "{err}" - ); -} - -#[test] -fn parse_accepts_external_raw_store_db() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--raw-store-db".to_string(), - "raw-store.db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert_eq!( - args.raw_store_db.as_deref(), - Some(std::path::Path::new("raw-store.db")) - ); -} - -#[test] -fn parse_accepts_external_repo_bytes_db() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--repo-bytes-db".to_string(), - "repo-bytes.db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert_eq!( - args.repo_bytes_db.as_deref(), - Some(std::path::Path::new("repo-bytes.db")) - ); -} - -#[test] -fn parse_accepts_cir_enable_with_raw_store_backend() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--raw-store-db".to_string(), - "raw-store.db".to_string(), - "--tal-path".to_string(), - "x.tal".to_string(), - "--ta-path".to_string(), - "x.cer".to_string(), - "--rsync-local-dir".to_string(), - "repo".to_string(), - "--cir-enable".to_string(), - "--cir-out".to_string(), - "out/example.cir".to_string(), - "--cir-tal-uri".to_string(), - "https://example.test/root.tal".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert!(args.cir_enabled); - assert_eq!( - args.raw_store_db.as_deref(), - Some(std::path::Path::new("raw-store.db")) - ); - assert_eq!(args.cir_static_root, None); -} - -#[test] -fn parse_accepts_cir_enable_with_required_paths_and_tal_override() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "x.tal".to_string(), - "--ta-path".to_string(), - "x.cer".to_string(), - "--rsync-local-dir".to_string(), - "repo".to_string(), - "--cir-enable".to_string(), - "--cir-out".to_string(), - "out/example.cir".to_string(), - "--cir-tal-uri".to_string(), - "https://example.test/root.tal".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert!(args.cir_enabled); - assert_eq!( - args.cir_out_path.as_deref(), - Some(std::path::Path::new("out/example.cir")) - ); - assert_eq!( - args.cir_tal_uri.as_deref(), - Some("https://example.test/root.tal") - ); - assert_eq!( - args.cir_tal_uris, - vec!["https://example.test/root.tal".to_string()] - ); -} - -#[test] -fn parse_rejects_deprecated_cir_static_root() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/root.tal".to_string(), - "--cir-enable".to_string(), - "--cir-out".to_string(), - "out/example.cir".to_string(), - "--cir-static-root".to_string(), - "out/static".to_string(), - ]; - let err = parse_args(&argv).expect_err("cir-static-root should be rejected"); - assert!(err.contains("no longer supported"), "{err}"); -} - -#[test] -fn parse_accepts_default_parallel_config_and_phase2_overrides() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/root.tal".to_string(), - "--parallel-phase2-object-workers".to_string(), - "3".to_string(), - "--parallel-phase2-worker-queue-capacity".to_string(), - "17".to_string(), - "--parallel-phase2-ready-batch-size".to_string(), - "31".to_string(), - "--parallel-phase2-ready-batch-wall-time-budget-ms".to_string(), - "43".to_string(), - "--parallel-phase2-result-drain-batch-size".to_string(), - "37".to_string(), - "--parallel-phase2-finalize-batch-size".to_string(), - "41".to_string(), - "--parallel-phase2-finalize-batch-wall-time-budget-ms".to_string(), - "47".to_string(), - "--parallel-phase2-finalize-queue-capacity".to_string(), - "8192".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert_eq!(args.parallel_phase2_config.object_workers, 3); - assert_eq!(args.parallel_phase2_config.worker_queue_capacity, 17); - assert_eq!(args.parallel_phase2_config.ready_batch_size, 31); - assert_eq!( - args.parallel_phase2_config.ready_batch_wall_time_budget_ms, - 43 - ); - assert_eq!( - args.parallel_phase2_config.object_result_drain_batch_size, - 37 - ); - assert_eq!( - args.parallel_phase2_config - .publication_point_finalize_batch_size, - 41 - ); - assert_eq!( - args.parallel_phase2_config - .publication_point_finalize_wall_time_budget_ms, - 47 - ); - assert_eq!( - args.parallel_phase2_config - .publication_point_finalize_queue_capacity, - 8192 - ); - assert_eq!(args.parallel_phase1_config, ParallelPhase1Config::default()); -} - -#[test] -fn parse_rejects_zero_phase2_ready_batch_size() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/root.tal".to_string(), - "--parallel-phase2-ready-batch-size".to_string(), - "0".to_string(), - ]; - let err = parse_args(&argv).expect_err("zero ready batch must fail"); - assert!(err.contains("--parallel-phase2-ready-batch-size"), "{err}"); -} - -#[test] -fn parse_rejects_zero_phase2_result_drain_batch_size() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/root.tal".to_string(), - "--parallel-phase2-result-drain-batch-size".to_string(), - "0".to_string(), - ]; - let err = parse_args(&argv).expect_err("zero result drain batch must fail"); - assert!( - err.contains("--parallel-phase2-result-drain-batch-size"), - "{err}" - ); -} - -#[test] -fn parse_rejects_zero_phase2_ready_batch_wall_time_budget_ms() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/root.tal".to_string(), - "--parallel-phase2-ready-batch-wall-time-budget-ms".to_string(), - "0".to_string(), - ]; - let err = parse_args(&argv).expect_err("zero ready time budget must fail"); - assert!( - err.contains("--parallel-phase2-ready-batch-wall-time-budget-ms"), - "{err}" - ); -} - -#[test] -fn parse_rejects_zero_phase2_finalize_batch_size() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/root.tal".to_string(), - "--parallel-phase2-finalize-batch-size".to_string(), - "0".to_string(), - ]; - let err = parse_args(&argv).expect_err("zero finalize batch must fail"); - assert!( - err.contains("--parallel-phase2-finalize-batch-size"), - "{err}" - ); -} - -#[test] -fn parse_rejects_zero_phase2_finalize_batch_wall_time_budget_ms() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/root.tal".to_string(), - "--parallel-phase2-finalize-batch-wall-time-budget-ms".to_string(), - "0".to_string(), - ]; - let err = parse_args(&argv).expect_err("zero finalize time budget must fail"); - assert!( - err.contains("--parallel-phase2-finalize-batch-wall-time-budget-ms"), - "{err}" - ); -} - -#[test] -fn parse_rejects_zero_phase2_finalize_queue_capacity() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/root.tal".to_string(), - "--parallel-phase2-finalize-queue-capacity".to_string(), - "0".to_string(), - ]; - let err = parse_args(&argv).expect_err("zero finalize queue capacity must fail"); - assert!( - err.contains("--parallel-phase2-finalize-queue-capacity"), - "{err}" - ); -} - -#[test] -fn parse_rejects_removed_parallel_enable_flags() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/root.tal".to_string(), - "--parallel-phase1".to_string(), - ]; - let err = parse_args(&argv).expect_err("removed phase flag should fail"); - assert!(err.contains("unknown argument: --parallel-phase1"), "{err}"); - - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/root.tal".to_string(), - "--parallel-phase2".to_string(), - ]; - let err = parse_args(&argv).expect_err("removed phase flag should fail"); - assert!(err.contains("unknown argument: --parallel-phase2"), "{err}"); -} - -#[test] -fn parse_accepts_multi_tal_cir_overrides_in_file_mode() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "apnic.tal".to_string(), - "--ta-path".to_string(), - "apnic.cer".to_string(), - "--tal-path".to_string(), - "arin.tal".to_string(), - "--ta-path".to_string(), - "arin.cer".to_string(), - "--rsync-local-dir".to_string(), - "repo".to_string(), - "--cir-enable".to_string(), - "--cir-out".to_string(), - "out/example.cir".to_string(), - "--cir-tal-uri".to_string(), - "https://example.test/apnic.tal".to_string(), - "--cir-tal-uri".to_string(), - "https://example.test/arin.tal".to_string(), - ]; - let args = parse_args(&argv).expect("parse args"); - assert_eq!( - args.cir_tal_uris, - vec![ - "https://example.test/apnic.tal".to_string(), - "https://example.test/arin.tal".to_string() - ] - ); -} - -#[test] -fn parse_rejects_incomplete_or_invalid_cir_flags() { - let argv_missing = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/root.tal".to_string(), - "--cir-enable".to_string(), - ]; - let err = parse_args(&argv_missing).unwrap_err(); - assert!(err.contains("--cir-enable requires --cir-out"), "{err}"); - - let argv_needs_enable = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/root.tal".to_string(), - "--cir-out".to_string(), - "out/example.cir".to_string(), - ]; - let err = parse_args(&argv_needs_enable).unwrap_err(); - assert!(err.contains("require --cir-enable"), "{err}"); - - let argv_offline_missing_uri = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "x.tal".to_string(), - "--ta-path".to_string(), - "x.cer".to_string(), - "--rsync-local-dir".to_string(), - "repo".to_string(), - "--cir-enable".to_string(), - "--cir-out".to_string(), - "out/example.cir".to_string(), - ]; - let err = parse_args(&argv_offline_missing_uri).unwrap_err(); - assert!(err.contains("requires --cir-tal-uri"), "{err}"); -} - -#[test] -fn parse_rejects_invalid_validation_time() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--validation-time".to_string(), - "not-a-time".to_string(), - ]; - let err = parse_args(&argv).unwrap_err(); - assert!(err.contains("invalid --validation-time"), "{err}"); -} - -#[test] -fn parse_rejects_invalid_max_instances() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--max-instances".to_string(), - "nope".to_string(), - ]; - let err = parse_args(&argv).unwrap_err(); - assert!(err.contains("invalid --max-instances"), "{err}"); -} - -#[test] -fn parse_rejects_missing_value_for_db() { - let argv = vec!["rpki".to_string(), "--db".to_string()]; - let err = parse_args(&argv).unwrap_err(); - assert!(err.contains("--db requires a value"), "{err}"); -} - -#[test] -fn parse_rejects_missing_value_for_tal_url() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - ]; - let err = parse_args(&argv).unwrap_err(); - assert!(err.contains("--tal-url requires a value"), "{err}"); -} - -#[test] -fn parse_rejects_missing_db() { - let argv = vec!["rpki".to_string(), "--tal-url".to_string(), "x".to_string()]; - let err = parse_args(&argv).unwrap_err(); - assert!(err.contains("--db is required"), "{err}"); -} - -#[test] -fn parse_rejects_missing_tal_mode() { - let argv = vec!["rpki".to_string(), "--db".to_string(), "db".to_string()]; - let err = parse_args(&argv).unwrap_err(); - assert!( - err.contains("--tal-url") || err.contains("--tal-path"), - "{err}" - ); -} - -#[test] -fn parse_accepts_tal_url_mode() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - ]; - let args = parse_args(&argv).expect("parse"); - assert_eq!(args.tal_url.as_deref(), Some("https://example.test/x.tal")); - assert_eq!( - args.tal_urls, - vec!["https://example.test/x.tal".to_string()] - ); - assert!(args.tal_path.is_none()); - assert!(args.ta_path.is_none()); - assert_eq!(args.tal_inputs.len(), 1); - assert_eq!(args.tal_inputs[0].tal_id, "x"); - assert_eq!(args.parallel_phase1_config, ParallelPhase1Config::default()); - assert_eq!(args.parallel_phase2_config, ParallelPhase2Config::default()); -} - -#[test] -fn parse_accepts_multi_tal_without_parallel_flags() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/arin.tal".to_string(), - "--tal-url".to_string(), - "https://example.test/apnic.tal".to_string(), - "--tal-url".to_string(), - "https://example.test/ripe.tal".to_string(), - "--parallel-max-repo-sync-workers-global".to_string(), - "8".to_string(), - ]; - let args = parse_args(&argv).expect("parse"); - assert_eq!(args.tal_urls.len(), 3); - assert_eq!(args.tal_inputs.len(), 3); - assert_eq!(args.tal_inputs[0].tal_id, "arin"); - assert_eq!(args.tal_inputs[1].tal_id, "apnic"); - assert_eq!(args.tal_inputs[2].tal_id, "ripe"); - assert_eq!(args.parallel_phase1_config.max_repo_sync_workers_global, 8); -} - -#[test] -fn parse_accepts_multi_tal_urls_by_default() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/arin.tal".to_string(), - "--tal-url".to_string(), - "https://example.test/apnic.tal".to_string(), - ]; - let args = parse_args(&argv).expect("parse"); - assert_eq!(args.tal_urls.len(), 2); - assert_eq!(args.tal_inputs.len(), 2); -} - -#[test] -fn parse_accepts_offline_mode_requires_ta() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "a.tal".to_string(), - "--ta-path".to_string(), - "ta.cer".to_string(), - "--max-depth".to_string(), - "0".to_string(), - ]; - let args = parse_args(&argv).expect("parse"); - assert_eq!(args.tal_paths, vec![PathBuf::from("a.tal")]); - assert_eq!(args.ta_paths, vec![PathBuf::from("ta.cer")]); - assert_eq!(args.tal_path.as_deref(), Some(Path::new("a.tal"))); - assert_eq!(args.ta_path.as_deref(), Some(Path::new("ta.cer"))); - assert_eq!(args.max_ca_depth, 0); -} - -#[test] -fn parse_accepts_multiple_tal_path_pairs_by_default() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "apnic.tal".to_string(), - "--ta-path".to_string(), - "apnic-ta.cer".to_string(), - "--tal-path".to_string(), - "arin.tal".to_string(), - "--ta-path".to_string(), - "arin-ta.cer".to_string(), - ]; - let args = parse_args(&argv).expect("parse"); - assert_eq!(args.tal_paths.len(), 2); - assert_eq!(args.ta_paths.len(), 2); - assert_eq!(args.tal_inputs.len(), 2); -} - -#[test] -fn parse_rejects_mixed_tal_url_and_tal_path_modes() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/arin.tal".to_string(), - "--tal-path".to_string(), - "apnic.tal".to_string(), - "--ta-path".to_string(), - "apnic-ta.cer".to_string(), - ]; - let err = parse_args(&argv).unwrap_err(); - assert!( - err.contains( - "must specify either one-or-more --tal-url or one-or-more --tal-path/--ta-path pairs" - ), - "{err}" - ); -} - -#[test] -fn parse_rejects_mismatched_tal_path_and_ta_path_counts() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "apnic.tal".to_string(), - "--tal-path".to_string(), - "arin.tal".to_string(), - "--ta-path".to_string(), - "apnic-ta.cer".to_string(), - ]; - let err = parse_args(&argv).unwrap_err(); - assert!( - err.contains("--tal-path and --ta-path counts must match"), - "{err}" - ); -} - -#[test] -fn parse_accepts_tal_path_without_ta_when_disable_rrdp_is_set() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "a.tal".to_string(), - "--disable-rrdp".to_string(), - "--rsync-command".to_string(), - "/tmp/fake-rsync".to_string(), - ]; - let args = parse_args(&argv).expect("parse"); - assert_eq!(args.tal_path.as_deref(), Some(Path::new("a.tal"))); - assert!(args.ta_path.is_none()); - assert!(args.disable_rrdp); - assert_eq!( - args.rsync_command.as_deref(), - Some(Path::new("/tmp/fake-rsync")) - ); -} - -#[test] -fn parse_accepts_multiple_tal_paths_without_ta_when_disable_rrdp() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "a.tal".to_string(), - "--tal-path".to_string(), - "b.tal".to_string(), - "--disable-rrdp".to_string(), - "--rsync-command".to_string(), - "/tmp/fake-rsync".to_string(), - ]; - let args = parse_args(&argv).expect("parse"); - assert_eq!( - args.tal_paths, - vec![PathBuf::from("a.tal"), PathBuf::from("b.tal")] - ); - assert!(args.ta_paths.is_empty()); - assert_eq!(args.tal_inputs.len(), 2); - assert!(args.disable_rrdp); -} - -#[test] -fn parse_accepts_payload_delta_replay_mode_with_offline_tal_and_ta() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "a.tal".to_string(), - "--ta-path".to_string(), - "ta.cer".to_string(), - "--payload-base-archive".to_string(), - "base-archive".to_string(), - "--payload-base-locks".to_string(), - "base-locks.json".to_string(), - "--payload-delta-archive".to_string(), - "delta-archive".to_string(), - "--payload-delta-locks".to_string(), - "delta-locks.json".to_string(), - ]; - let args = parse_args(&argv).expect("parse delta replay mode"); - assert_eq!( - args.payload_base_archive.as_deref(), - Some(Path::new("base-archive")) - ); - assert_eq!( - args.payload_base_locks.as_deref(), - Some(Path::new("base-locks.json")) - ); - assert_eq!( - args.payload_delta_archive.as_deref(), - Some(Path::new("delta-archive")) - ); - assert_eq!( - args.payload_delta_locks.as_deref(), - Some(Path::new("delta-locks.json")) - ); -} - -#[test] -fn parse_rejects_partial_payload_delta_arguments_and_mutual_exclusion() { - let argv_partial = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "a.tal".to_string(), - "--ta-path".to_string(), - "ta.cer".to_string(), - "--payload-base-archive".to_string(), - "base-archive".to_string(), - ]; - let err = parse_args(&argv_partial).unwrap_err(); - assert!(err.contains("must be provided together"), "{err}"); - - let argv_both = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "a.tal".to_string(), - "--ta-path".to_string(), - "ta.cer".to_string(), - "--payload-replay-archive".to_string(), - "archive".to_string(), - "--payload-replay-locks".to_string(), - "locks.json".to_string(), - "--payload-base-archive".to_string(), - "base-archive".to_string(), - "--payload-base-locks".to_string(), - "base-locks.json".to_string(), - "--payload-delta-archive".to_string(), - "delta-archive".to_string(), - "--payload-delta-locks".to_string(), - "delta-locks.json".to_string(), - ]; - let err = parse_args(&argv_both).unwrap_err(); - assert!(err.contains("mutually exclusive"), "{err}"); -} - -#[test] -fn parse_rejects_payload_delta_with_tal_url_or_rsync_local_dir() { - let argv_url = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--payload-base-archive".to_string(), - "base-archive".to_string(), - "--payload-base-locks".to_string(), - "base-locks.json".to_string(), - "--payload-delta-archive".to_string(), - "delta-archive".to_string(), - "--payload-delta-locks".to_string(), - "delta-locks.json".to_string(), - ]; - let err = parse_args(&argv_url).unwrap_err(); - assert!(err.contains("--tal-url is not supported"), "{err}"); - - let argv_rsync = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "a.tal".to_string(), - "--ta-path".to_string(), - "ta.cer".to_string(), - "--payload-base-archive".to_string(), - "base-archive".to_string(), - "--payload-base-locks".to_string(), - "base-locks.json".to_string(), - "--payload-delta-archive".to_string(), - "delta-archive".to_string(), - "--payload-delta-locks".to_string(), - "delta-locks.json".to_string(), - "--rsync-local-dir".to_string(), - "repo".to_string(), - ]; - let err = parse_args(&argv_rsync).unwrap_err(); - assert!( - err.contains("payload delta replay mode cannot be combined with --rsync-local-dir"), - "{err}" - ); -} - -#[test] -fn parse_accepts_payload_replay_mode_with_offline_tal_and_ta() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "a.tal".to_string(), - "--ta-path".to_string(), - "ta.cer".to_string(), - "--payload-replay-archive".to_string(), - "archive".to_string(), - "--payload-replay-locks".to_string(), - "locks.json".to_string(), - ]; - let args = parse_args(&argv).expect("parse replay mode"); - assert_eq!( - args.payload_replay_archive.as_deref(), - Some(Path::new("archive")) - ); - assert_eq!( - args.payload_replay_locks.as_deref(), - Some(Path::new("locks.json")) - ); -} - -#[test] -fn parse_rejects_partial_payload_replay_arguments() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "a.tal".to_string(), - "--ta-path".to_string(), - "ta.cer".to_string(), - "--payload-replay-archive".to_string(), - "archive".to_string(), - ]; - let err = parse_args(&argv).unwrap_err(); - assert!(err.contains("must be provided together"), "{err}"); -} - -#[test] -fn parse_rejects_payload_replay_with_tal_url_or_rsync_local_dir() { - let argv_url = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--payload-replay-archive".to_string(), - "archive".to_string(), - "--payload-replay-locks".to_string(), - "locks.json".to_string(), - ]; - let err = parse_args(&argv_url).unwrap_err(); - assert!(err.contains("--tal-url is not supported"), "{err}"); - - let argv_rsync = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-path".to_string(), - "a.tal".to_string(), - "--ta-path".to_string(), - "ta.cer".to_string(), - "--payload-replay-archive".to_string(), - "archive".to_string(), - "--payload-replay-locks".to_string(), - "locks.json".to_string(), - "--rsync-local-dir".to_string(), - "repo".to_string(), - ]; - let err = parse_args(&argv_rsync).unwrap_err(); - assert!( - err.contains("cannot be combined with --rsync-local-dir"), - "{err}" - ); -} - -#[test] -fn parse_accepts_validation_time_rfc3339() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--validation-time".to_string(), - "2026-01-01T00:00:00Z".to_string(), - ]; - let args = parse_args(&argv).expect("parse"); - assert!(args.validation_time.is_some()); -} - -#[test] -fn parse_rejects_removed_revalidate_only_flag() { - let argv = vec![ - "rpki".to_string(), - "--db".to_string(), - "db".to_string(), - "--tal-url".to_string(), - "https://example.test/x.tal".to_string(), - "--revalidate-only".to_string(), - ]; - let err = parse_args(&argv).unwrap_err(); - assert!(err.contains("unknown argument: --revalidate-only"), "{err}"); -} - -#[test] -fn read_policy_accepts_valid_toml() { - let dir = tempfile::tempdir().expect("tmpdir"); - let p = dir.path().join("policy.toml"); - std::fs::write( - &p, - "signed_object_failure_policy = \"drop_publication_point\"\n", - ) - .expect("write policy"); - - let policy = read_policy(Some(&p)).expect("parse policy"); - assert_eq!( - policy.signed_object_failure_policy, - crate::policy::SignedObjectFailurePolicy::DropPublicationPoint - ); - assert_eq!( - policy.resource_validation_mode, - ResourceValidationMode::ValidationUpdate03 - ); - assert_eq!(policy.strict, StrictPolicy::default()); -} - -#[test] -fn read_policy_accepts_strict_table() { - let dir = tempfile::tempdir().expect("tmpdir"); - let p = dir.path().join("policy.toml"); - std::fs::write( - &p, - r#" - [strict] - name = true - cms_der = true - "#, - ) - .expect("write policy"); - - let policy = read_policy(Some(&p)).expect("parse policy"); - assert_eq!( - policy.strict, - StrictPolicy { - name: true, - cms_der: true, - signed_attrs: false, - } - ); -} - -#[test] -fn read_policy_accepts_resource_validation_mode() { - let dir = tempfile::tempdir().expect("tmpdir"); - let p = dir.path().join("policy.toml"); - std::fs::write(&p, "resource_validation_mode = \"rfc6487\"\n").expect("write policy"); - - let policy = read_policy(Some(&p)).expect("parse policy"); - assert_eq!( - policy.resource_validation_mode, - ResourceValidationMode::Rfc6487 - ); -} - -#[test] -fn read_policy_reports_missing_file() { - let dir = tempfile::tempdir().expect("tmpdir"); - let p = dir.path().join("missing.toml"); - let err = read_policy(Some(&p)).unwrap_err(); - assert!(err.contains("read policy file failed"), "{err}"); -} - -fn synthetic_post_validation_shared() -> PostValidationShared { - let tal_bytes = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/tal/apnic-rfc7730-https.tal"), - ) - .expect("read tal fixture"); - let ta_der = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/ta/apnic-ta.cer"), - ) - .expect("read ta fixture"); - - let discovery = crate::validation::from_tal::discover_root_ca_instance_from_tal_and_ta_der( - &tal_bytes, &ta_der, None, - ) - .expect("discover root"); - - let tree = crate::validation::tree::TreeRunOutput { - instances_processed: 1, - instances_failed: 0, - warnings: vec![ - crate::report::Warning::new("synthetic warning") - .with_rfc_refs(&[crate::report::RfcRef("RFC 6487 §4.8.8.1")]) - .with_context("rsync://example.test/repo/pp/"), - ], - vrps: vec![ - crate::validation::objects::Vrp { - asn: 64496, - prefix: crate::data_model::roa::IpPrefix { - afi: crate::data_model::roa::RoaAfi::Ipv4, - prefix_len: 24, - addr: [192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - }, - max_length: 24, - }, - crate::validation::objects::Vrp { - asn: 64497, - prefix: crate::data_model::roa::IpPrefix { - afi: crate::data_model::roa::RoaAfi::Ipv6, - prefix_len: 48, - addr: [0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - }, - max_length: 64, - }, - ], - aspas: vec![crate::validation::objects::AspaAttestation { - customer_as_id: 64496, - provider_as_ids: vec![64497, 64498], - }], - router_keys: Vec::new(), - }; - - let mut pp1 = crate::audit::PublicationPointAudit::default(); - pp1.source = "fresh".to_string(); - pp1.rrdp_notification_uri = Some("https://example.test/n1.xml".to_string()); - pp1.manifest_rsync_uri = "rsync://example.test/repo/pp1/manifest.mft".to_string(); - pp1.objects.push(crate::audit::ObjectAuditEntry { - rsync_uri: "rsync://example.test/repo/pp1/a.roa".to_string(), - sha256_hex: "11".repeat(32), - kind: crate::audit::AuditObjectKind::Roa, - result: crate::audit::AuditObjectResult::Ok, - detail: None, - }); - let mut pp2 = crate::audit::PublicationPointAudit::default(); - pp2.source = "fresh".to_string(); - pp2.rrdp_notification_uri = Some("https://example.test/n1.xml".to_string()); - let mut pp3 = crate::audit::PublicationPointAudit::default(); - pp3.source = "fresh".to_string(); - pp3.rrdp_notification_uri = Some("https://example.test/n2.xml".to_string()); - - let out = crate::validation::run_tree_from_tal::RunTreeFromTalAuditOutput { - discovery: discovery.clone(), - discoveries: vec![discovery], - successful_tal_inputs: Vec::new(), - tree, - publication_points: vec![pp1, pp2, pp3], - roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), - downloads: Vec::new(), - download_stats: crate::audit::AuditDownloadStats::default(), - current_repo_objects: Vec::new(), - ccr_accumulator: None, - cir_input: crate::cir::CirInputSnapshot::default(), - }; - PostValidationShared::from_run_output(out) -} - -fn sample_cli_ccr_accumulator() -> CcrAccumulator { - let tal_bytes = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/tal/apnic-rfc7730-https.tal"), - ) - .expect("read tal fixture"); - let ta_der = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/ta/apnic-ta.cer"), - ) - .expect("read ta fixture"); - let discovery = crate::validation::from_tal::discover_root_ca_instance_from_tal_and_ta_der( - &tal_bytes, &ta_der, None, - ) - .expect("discover root"); - let mut accumulator = CcrAccumulator::new(vec![discovery.trust_anchor.clone()]); - let manifest_uri = "rsync://example.test/repo/current.mft".to_string(); - let projection = crate::storage::VcirCcrManifestProjection { - manifest_rsync_uri: manifest_uri.clone(), - manifest_sha256: vec![0x44; 32], - manifest_size: 2048, - manifest_ee_aki: vec![0x55; 20], - manifest_number_be: vec![1], - manifest_this_update: crate::storage::PackTime::from_utc_offset_datetime( - time::OffsetDateTime::now_utc(), - ), - manifest_sia_locations_der: vec![ - crate::ccr::manifest_location::encode_access_description_der(&AccessDescription { - access_method_oid: OID_AD_SIGNED_OBJECT.to_string(), - access_location: manifest_uri, - }) - .expect("encode signedObject"), - ], - subordinate_skis: vec![vec![0x33; 20]], - }; - accumulator - .append_manifest_projection(&projection) - .expect("append manifest projection"); - accumulator -} - -#[test] -fn build_report_and_helpers_work_on_synthetic_output() { - let shared = synthetic_post_validation_shared(); - let policy = Policy::default(); - let validation_time = time::OffsetDateTime::now_utc(); - let report = build_report(&policy, validation_time, &shared); - - assert_eq!(unique_rrdp_repos(&report), 2); - assert_eq!(report.vrps.len(), 2); - assert_eq!(report.aspas.len(), 1); - - print_summary(&report); -} - -#[test] -fn run_report_task_and_stage_timing_work() { - let shared = synthetic_post_validation_shared(); - let policy = Policy::default(); - let validation_time = time::OffsetDateTime::now_utc(); - let dir = tempfile::tempdir().expect("tmpdir"); - let report_path = dir.path().join("report.json"); - let report_output = run_report_task( - &policy, - validation_time, - &shared, - Some(&report_path), - ReportJsonFormat::Compact, - ) - .expect("run report task"); - - assert!(report_output.report_write_ms.is_some()); - - let report_json = std::fs::read_to_string(&report_path).expect("read report json"); - assert!(!report_json.contains('\n'), "{report_json}"); - let report: serde_json::Value = - serde_json::from_str(&report_json).expect("parse compact report json"); - assert_eq!(report["vrps"].as_array().unwrap().len(), 2); - assert_eq!(report["aspas"].as_array().unwrap().len(), 1); - assert_eq!(report["queryAudit"]["status"].as_str(), Some("complete")); - assert!(report["queryAudit"]["eventsCount"].as_u64().unwrap() > 0); - let events_path = dir.path().join( - report["queryAudit"]["eventsPath"] - .as_str() - .expect("events path"), - ); - let events = std::fs::read_to_string(events_path).expect("read validation events"); - assert!( - events - .lines() - .any(|line| line.contains("\"eventType\":\"object\"")) - ); - - let stage_timing = RunStageTiming { - validation_ms: 1, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - crypto_signature_cache_observe: None, - enable_transport_request_prefetch: false, - report_build_ms: report_output.report_build_ms, - report_write_ms: report_output.report_write_ms, - ccr_build_ms: Some(2), - ccr_build_breakdown: None, - ccr_write_ms: Some(3), - compare_view_build_ms: Some(4), - compare_view_write_ms: Some(5), - cir_build_cir_ms: Some(6), - cir_write_cir_ms: Some(7), - cir_total_ms: Some(8), - total_ms: 9, - publication_points: shared.publication_points.len(), - repo_sync_ms_total: 10, - publication_point_repo_sync_ms_total: 11, - download_event_count: 12, - rrdp_download_ms_total: 13, - rsync_download_ms_total: 14, - download_bytes_total: 15, - roa_validation_cache: crate::validation::objects::RoaValidationCacheStats::default(), - analysis_counts: std::collections::HashMap::new(), - analysis_phases: std::collections::HashMap::new(), - analysis_top_publication_points: Vec::new(), - analysis_top_publication_point_steps: Vec::new(), - analysis_top_publication_point_cache_steps: Vec::new(), - vcir_storage_summary_ms: Some(16), - vcir_storage: Some(VcirStorageSummary { - entry_count: 2, - vcir_value_bytes: 100, - vcir_value_bytes_max: 60, - vcir_value_bytes_max_manifest_rsync_uri: Some( - "rsync://example.test/repo/max.mft".to_string(), - ), - core_fields: VcirCoreFieldSizeBreakdown { - manifest_rsync_uri_bytes: 10, - ..VcirCoreFieldSizeBreakdown::default() - }, - ccr_projection: VcirCcrProjectionSizeBreakdown { - manifest_sha256_bytes: 32, - ..VcirCcrProjectionSizeBreakdown::default() - }, - child_resources: VcirChildResourceSizeBreakdown { - effective_ip_resource_cbor_bytes: 12, - effective_as_resource_cbor_bytes: 6, - }, - field_sizes: VcirFieldSizeBreakdown { - local_output_count: 1, - local_output_payload_json_bytes: 70, - local_output_payload_typed_body_bytes: 20, - ..VcirFieldSizeBreakdown::default() - }, - local_output_old_projection_bytes: 80, - local_output_typed_projection_bytes: 30, - local_output_projection_saved_bytes: 50, - top_entries_by_vcir_value_bytes: vec![VcirStorageEntrySummary { - manifest_rsync_uri: "rsync://example.test/repo/max.mft".to_string(), - vcir_value_bytes: 60, - local_vrp_count: 1, - local_aspa_count: 0, - local_router_key_count: 0, - accepted_object_count: 1, - rejected_object_count: 0, - child_count: 0, - core_fields: VcirCoreFieldSizeBreakdown::default(), - ccr_projection: VcirCcrProjectionSizeBreakdown::default(), - child_resources: VcirChildResourceSizeBreakdown::default(), - field_sizes: VcirFieldSizeBreakdown::default(), - local_output_old_projection_bytes: 1, - local_output_typed_projection_bytes: 1, - local_output_projection_saved_bytes: 0, - }], - }), - publication_point_cache_index_load: None, - publication_point_cache_index_refresh: None, - memory_telemetry: None, - }; - write_stage_timing(Some(&report_path), &stage_timing).expect("write stage timing"); - let stage_timing_json = - std::fs::read_to_string(dir.path().join("stage-timing.json")).expect("read timing"); - assert!(stage_timing_json.contains("\"validation_ms\"")); - assert!(stage_timing_json.contains("\"ccr_build_ms\"")); - assert!(stage_timing_json.contains("\"vcir_storage\"")); - assert!(stage_timing_json.contains("\"local_output_projection_saved_bytes\"")); - - let ccr_path = dir.path().join("result.ccr"); - write_stage_timing(Some(&ccr_path), &stage_timing).expect("write stage timing via ccr path"); - assert!( - dir.path().join("stage-timing.json").exists(), - "stage timing should use parent directory of the anchor path" - ); - - let skipped = ReportTaskOutput::skipped(); - assert_eq!(skipped.report_build_ms, 0); - assert!(skipped.report_write_ms.is_none()); -} - -#[test] -fn stage_timing_serializes_memory_telemetry() { - let dir = tempfile::tempdir().expect("tmpdir"); - let report_path = dir.path().join("report.json"); - let stage_timing = RunStageTiming { - validation_ms: 1, - enable_roa_validation_cache: true, - enable_child_certificate_validation_cache: true, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - crypto_signature_cache_observe: None, - enable_transport_request_prefetch: true, - report_build_ms: 2, - report_write_ms: None, - ccr_build_ms: None, - ccr_build_breakdown: None, - ccr_write_ms: None, - compare_view_build_ms: None, - compare_view_write_ms: None, - cir_build_cir_ms: None, - cir_write_cir_ms: None, - cir_total_ms: None, - total_ms: 3, - publication_points: 4, - repo_sync_ms_total: 5, - publication_point_repo_sync_ms_total: 6, - download_event_count: 7, - rrdp_download_ms_total: 8, - rsync_download_ms_total: 9, - download_bytes_total: 10, - roa_validation_cache: crate::validation::objects::RoaValidationCacheStats { - hit_roas: 2, - ..crate::validation::objects::RoaValidationCacheStats::default() - }, - analysis_counts: std::collections::HashMap::from([( - "roa_validation_cache_hit_roas".to_string(), - 2, - )]), - analysis_phases: std::collections::HashMap::new(), - analysis_top_publication_points: Vec::new(), - analysis_top_publication_point_steps: Vec::new(), - analysis_top_publication_point_cache_steps: Vec::new(), - vcir_storage_summary_ms: None, - vcir_storage: None, - publication_point_cache_index_load: None, - publication_point_cache_index_refresh: None, - memory_telemetry: Some(MemoryTelemetrySummary { - checkpoints: vec![MemoryTelemetryCheckpoint { - label: "after_validation".to_string(), - elapsed_ms: 11, - process: ProcessMemorySnapshot { - label: "after_validation".to_string(), - vm_rss_kb: Some(12), - vm_size_kb: None, - vm_data_kb: None, - vm_swap_kb: None, - rss_anon_kb: Some(13), - rss_file_kb: None, - rss_shmem_kb: None, - threads: Some(14), - fd_count: Some(15), - smaps_rollup: None, - smaps_mapping_summary: None, - errors: Vec::new(), - }, - rocksdb: RocksDbMemorySnapshot { - databases: Vec::new(), - totals: RocksDbMemoryTotals { - cur_size_all_mem_tables: 16, - size_all_mem_tables: 17, - estimate_table_readers_mem: 18, - block_cache_capacity: 19, - block_cache_usage: 20, - block_cache_pinned_usage: 21, - }, - }, - }], - object_graph: None, - malloc_trim_probes: Vec::new(), - }), - }; - - write_stage_timing(Some(&report_path), &stage_timing).expect("write stage timing"); - let value: serde_json::Value = serde_json::from_str( - &std::fs::read_to_string(dir.path().join("stage-timing.json")).unwrap(), - ) - .expect("parse stage timing json"); - let checkpoint = &value["memory_telemetry"]["checkpoints"][0]; - assert_eq!(checkpoint["label"], "after_validation"); - assert_eq!(checkpoint["process"]["vm_rss_kb"], 12); - assert_eq!( - checkpoint["rocksdb"]["totals"]["cur_size_all_mem_tables"], - 16 - ); - assert_eq!(value["analysis_counts"]["roa_validation_cache_hit_roas"], 2); - assert_eq!(value["roa_validation_cache"]["hit_roas"], 2); - assert_eq!(value["publication_point_cache_observe_only"], false); - assert_eq!(value["enable_publication_point_validation_cache"], false); - assert!( - value["memory_telemetry"] - .as_object() - .expect("memory telemetry object") - .get("malloc_trim_probes") - .is_none() - ); -} - -#[test] -fn shared_object_graph_estimate_counts_audit_and_outputs() { - let mut shared = synthetic_post_validation_shared(); - let mut publication_points = shared - .publication_points - .iter() - .cloned() - .collect::>(); - publication_points[0].rsync_base_uri = "rsync://example.test/repo/".to_string(); - publication_points[0].manifest_rsync_uri = "rsync://example.test/repo/a.mft".to_string(); - publication_points[0].publication_point_rsync_uri = "rsync://example.test/repo/".to_string(); - publication_points[0].objects = vec![crate::audit::ObjectAuditEntry { - rsync_uri: "rsync://example.test/repo/a.roa".to_string(), - sha256_hex: "11".repeat(32), - kind: crate::audit::AuditObjectKind::Roa, - result: crate::audit::AuditObjectResult::Ok, - detail: None, - }]; - shared.publication_points = publication_points.into(); - - let graph = estimate_shared_object_graph(&shared); - let publication_points_section = graph - .sections - .iter() - .find(|section| section.name == "publication_points") - .expect("publication points section"); - let object_count = publication_points_section - .details - .iter() - .find(|metric| metric.name == "object_audit_entry_count") - .expect("object count metric"); - assert_eq!(object_count.value, 1); - assert!(publication_points_section.estimated_bytes > 0); - - let vrps_section = graph - .sections - .iter() - .find(|section| section.name == "vrps") - .expect("vrps section"); - assert_eq!(vrps_section.item_count, 2); - assert!(graph.total_estimated_bytes >= publication_points_section.estimated_bytes); -} - -#[test] -fn run_compare_view_task_writes_csv_from_shared_output() { - let shared = synthetic_post_validation_shared(); - let dir = tempfile::tempdir().expect("tmpdir"); - let vrps_path = dir.path().join("vrps.csv"); - let vaps_path = dir.path().join("vaps.csv"); - - let output = run_compare_view_task(&shared, Some(&vrps_path), Some(&vaps_path), "unknown") - .expect("write direct compare views"); - - assert!(output.build_ms.is_some()); - assert!(output.write_ms.is_some()); - let vrps_csv = std::fs::read_to_string(vrps_path).expect("read vrps csv"); - let vaps_csv = std::fs::read_to_string(vaps_path).expect("read vaps csv"); - assert!(vrps_csv.contains("ASN,IP Prefix,Max Length,Trust Anchor")); - assert!(vrps_csv.contains("AS64496,192.0.2.0/24,24,unknown")); - assert!(vrps_csv.contains("AS64497,2001:db8::/48,64,unknown")); - assert!(vaps_csv.contains("Customer ASN,Providers,Trust Anchor")); - assert!(vaps_csv.contains("AS64496,AS64497;AS64498,unknown")); -} - -#[test] -fn run_ccr_task_uses_accumulator_when_phase2_output_contains_reuse_sources() { - let mut shared = synthetic_post_validation_shared(); - shared.ccr_accumulator = Some(sample_cli_ccr_accumulator()); - let mut publication_points = shared - .publication_points - .iter() - .cloned() - .collect::>(); - publication_points[1].source = "vcir_current_instance".to_string(); - publication_points[2].source = "failed_no_cache".to_string(); - shared.publication_points = publication_points.into(); - let dir = tempfile::tempdir().expect("tmpdir"); - let ccr_path = dir.path().join("result.ccr"); - let store = RocksStore::open(&dir.path().join("db")).expect("open empty store"); - - let output = run_ccr_task( - &store, - &shared, - Some(&ccr_path), - time::OffsetDateTime::now_utc(), - ) - .expect("run ccr task"); - - assert!(output.ccr_build_ms.is_some()); - assert!(output.ccr_build_breakdown.is_none()); - let der = std::fs::read(&ccr_path).expect("read ccr"); - let ci = crate::ccr::decode_content_info(&der).expect("decode ccr"); - assert_eq!( - ci.content - .mfts - .as_ref() - .map(|manifest_state| manifest_state.mis.len()), - Some(1) - ); -} - -#[test] -fn write_json_writes_report() { - let report = AuditReportV2 { - format_version: 2, - meta: AuditRunMeta { - validation_time_rfc3339_utc: "2026-01-01T00:00:00Z".to_string(), - }, - policy: Policy::default(), - tree: TreeSummary { - instances_processed: 0, - instances_failed: 0, - warnings: Vec::new(), - }, - publication_points: Vec::new(), - vrps: Vec::new(), - aspas: Vec::new(), - downloads: Vec::new(), - download_stats: crate::audit::AuditDownloadStats::default(), - repo_sync_stats: crate::audit::AuditRepoSyncStats::default(), - query_audit: None, - }; - - let dir = tempfile::tempdir().expect("tmpdir"); - let pretty_path = dir.path().join("report-pretty.json"); - write_json(&pretty_path, &report, ReportJsonFormat::Pretty).expect("write pretty json"); - let pretty = std::fs::read_to_string(&pretty_path).expect("read pretty report"); - assert!(pretty.contains("\"format_version\"")); - assert!(pretty.contains("\"policy\"")); - assert!(pretty.contains("\n \"format_version\""), "{pretty}"); - - let compact_path = dir.path().join("report-compact.json"); - write_json(&compact_path, &report, ReportJsonFormat::Compact).expect("write compact json"); - let compact = std::fs::read_to_string(&compact_path).expect("read compact report"); - assert!(compact.contains("\"format_version\"")); - assert!(compact.contains("\"policy\"")); - assert!(!compact.contains('\n'), "{compact}"); -} - -#[test] -fn build_repo_sync_stats_aggregates_phase_and_terminal_state() { - let mut pp1 = crate::audit::PublicationPointAudit::default(); - pp1.repo_sync_phase = Some("rrdp_ok".to_string()); - pp1.repo_sync_duration_ms = Some(10); - pp1.repo_terminal_state = "fresh".to_string(); - - let mut pp2 = crate::audit::PublicationPointAudit::default(); - pp2.repo_sync_phase = Some("rrdp_failed_rsync_failed".to_string()); - pp2.repo_sync_duration_ms = Some(20); - pp2.repo_terminal_state = "failed_no_cache".to_string(); - - let mut pp3 = crate::audit::PublicationPointAudit::default(); - pp3.repo_sync_phase = Some("rrdp_failed_rsync_failed".to_string()); - pp3.repo_sync_duration_ms = Some(30); - pp3.repo_terminal_state = "failed_no_cache".to_string(); - - let stats = build_repo_sync_stats(&[pp1, pp2, pp3]); - assert_eq!(stats.publication_points_total, 3); - assert_eq!(stats.by_phase["rrdp_ok"].count, 1); - assert_eq!(stats.by_phase["rrdp_ok"].duration_ms_total, 10); - assert_eq!(stats.by_phase["rrdp_failed_rsync_failed"].count, 2); - assert_eq!( - stats.by_phase["rrdp_failed_rsync_failed"].duration_ms_total, - 50 - ); - assert_eq!(stats.by_terminal_state["fresh"].count, 1); - assert_eq!(stats.by_terminal_state["failed_no_cache"].count, 2); - assert_eq!( - stats.by_terminal_state["failed_no_cache"].duration_ms_total, - 50 - ); -} +// CLI tests are grouped by parser, report, and task behavior. +include!("tests_parts/parse_core.rs"); +include!("tests_parts/parse_options.rs"); +include!("tests_parts/report_helpers.rs"); +include!("tests_parts/report_tasks.rs"); diff --git a/crates/panda-rpki-validator/src/cli/tests_parts/parse_core.rs b/crates/panda-rpki-validator/src/cli/tests_parts/parse_core.rs new file mode 100644 index 0000000..1520abc --- /dev/null +++ b/crates/panda-rpki-validator/src/cli/tests_parts/parse_core.rs @@ -0,0 +1,977 @@ +// CLI test group: parse core. + +use super::*; +use crate::data_model::oid::OID_AD_SIGNED_OBJECT; +use crate::data_model::rc::AccessDescription; +use crate::memory_telemetry::{ + MemoryTelemetryCheckpoint, MemoryTelemetrySummary, ProcessMemorySnapshot, +}; +use crate::policy::ResourceValidationMode; +use crate::storage::{ + RocksDbMemorySnapshot, RocksDbMemoryTotals, VcirCcrProjectionSizeBreakdown, + VcirChildResourceSizeBreakdown, VcirCoreFieldSizeBreakdown, VcirFieldSizeBreakdown, + VcirStorageEntrySummary, VcirStorageSummary, +}; + +#[test] +fn parse_help_returns_usage() { + let argv = vec!["rpki".to_string(), "--help".to_string()]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("Usage:"), "{err}"); + assert!(err.contains("--db"), "{err}"); + assert!(err.contains("--rsync-mirror-root"), "{err}"); + assert!(err.contains("--rsync-scope"), "{err}"); + assert!(err.contains("--parallel-phase2-object-workers"), "{err}"); + assert!(err.contains("--memory-trim-after-validation"), "{err}"); + assert!(err.contains("--enable-roa-validation-cache"), "{err}"); + assert!(err.contains("--resource-validation-mode"), "{err}"); + assert!(err.contains("--ta-constraints"), "{err}"); + assert!(err.contains("--max-ca-depth"), "{err}"); + assert!(err.contains("default: 32"), "{err}"); + assert!( + err.contains("--publication-point-cache-observe-only"), + "{err}" + ); + assert!( + err.contains("--enable-publication-point-validation-cache"), + "{err}" + ); + assert!(!err.contains("--parallel-phase1"), "{err}"); + assert!(!err.contains("--parallel-phase2 "), "{err}"); +} + +#[test] +fn parse_accepts_explicit_ta_constraints_for_known_tal() { + let dir = tempfile::tempdir().expect("tmpdir"); + let constraints_path = dir.path().join("example.constraints"); + std::fs::write(&constraints_path, "allow 192.0.2.0/24\n").expect("write constraints"); + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "example.tal".to_string(), + "--ta-path".to_string(), + "example.cer".to_string(), + "--ta-constraints".to_string(), + format!("example={}", constraints_path.display()), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(!args.ta_constraints.is_empty()); +} + +#[test] +fn parse_rejects_unknown_argument() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--nope".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("unknown argument"), "{err}"); +} + +#[test] +fn parse_accepts_normal_validation_contract_output() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--validation-contract-out".to_string(), + "out/validation-contract.json".to_string(), + ]; + let args = parse_args(&argv).expect("parse normal contract output"); + assert_eq!( + args.validation_contract_out_path, + Some(PathBuf::from("out/validation-contract.json")) + ); +} + +#[test] +fn parse_rejects_both_tal_url_and_tal_path() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!( + err.contains("one-or-more --tal-url or one-or-more --tal-path/--ta-path pairs"), + "{err}" + ); +} + +#[test] +fn parse_rejects_invalid_max_depth() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--max-depth".to_string(), + "nope".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("invalid --max-depth"), "{err}"); +} + +#[test] +fn parse_rejects_invalid_max_ca_depth() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--max-ca-depth".to_string(), + "nope".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("invalid --max-ca-depth"), "{err}"); +} + +#[test] +fn parse_rejects_both_ca_depth_options() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--max-ca-depth".to_string(), + "32".to_string(), + "--max-depth".to_string(), + "12".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("cannot be combined"), "{err}"); +} + +#[test] +fn parse_defaults_max_ca_depth_to_32() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!(args.max_ca_depth, 32); +} + +#[test] +fn parse_accepts_ccr_out_path() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--rsync-local-dir".to_string(), + "repo".to_string(), + "--ccr-out".to_string(), + "out/example.ccr".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!( + args.ccr_out_path.as_deref(), + Some(std::path::Path::new("out/example.ccr")) + ); +} + +#[test] +fn parse_accepts_memory_trim_after_validation() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--memory-trim-after-validation".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.memory_trim_after_validation); +} + +#[test] +fn parse_disables_memory_trim_after_validation_by_default() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(!args.memory_trim_after_validation); +} + +#[test] +fn parse_accepts_enable_roa_validation_cache() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--enable-roa-validation-cache".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.enable_roa_validation_cache); +} + +#[test] +fn parse_accepts_enable_child_certificate_validation_cache() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--enable-child-certificate-validation-cache".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.enable_child_certificate_validation_cache); +} + +#[test] +fn parse_accepts_publication_point_cache_flags() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--publication-point-cache-observe-only".to_string(), + "--enable-publication-point-validation-cache".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.publication_point_cache_observe_only); + assert!(args.enable_publication_point_validation_cache); +} + +#[test] +fn parse_disables_crypto_signature_cache_observe_only_by_default() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(!args.crypto_signature_cache_observe_only); + assert!(!args.enable_crypto_signature_cache); +} + +#[test] +fn parse_accepts_crypto_signature_cache_observe_only() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--crypto-signature-cache-observe-only".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.crypto_signature_cache_observe_only); + assert!(!args.enable_crypto_signature_cache); +} + +#[test] +fn parse_accepts_enable_crypto_signature_cache() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--enable-crypto-signature-cache".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.enable_crypto_signature_cache); + assert!(!args.crypto_signature_cache_observe_only); +} + +#[test] +fn parse_accepts_enable_transport_request_prefetch() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--enable-transport-request-prefetch".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.enable_transport_request_prefetch); +} + +#[test] +fn parse_disables_roa_validation_cache_by_default() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(!args.enable_roa_validation_cache); + assert!(!args.enable_child_certificate_validation_cache); + assert!(!args.publication_point_cache_observe_only); + assert!(!args.enable_publication_point_validation_cache); + assert!(!args.enable_transport_request_prefetch); +} + +#[test] +fn parse_accepts_analysis_out_and_implies_analyze() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--analysis-out".to_string(), + "run/analyze".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.analyze); + assert_eq!( + args.analysis_out_path.as_deref(), + Some(std::path::Path::new("run/analyze")) + ); +} + +#[test] +fn parse_accepts_report_json_compact_when_report_json_is_set() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--report-json".to_string(), + "out/report.json".to_string(), + "--report-json-compact".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!( + args.report_json_path.as_deref(), + Some(std::path::Path::new("out/report.json")) + ); + assert!(args.report_json_compact); +} + +#[test] +fn parse_accepts_rsync_scope_policy() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--rsync-scope".to_string(), + "module-root".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!(args.rsync_scope_policy, RsyncScopePolicy::ModuleRoot); +} + +#[test] +fn parse_accepts_host_rsync_scope_policy() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--rsync-scope".to_string(), + "host".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!(args.rsync_scope_policy, RsyncScopePolicy::Host); +} + +#[test] +fn parse_accepts_repeatable_http_root_cert_paths() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--http-root-cert".to_string(), + "local-ca-1.pem".to_string(), + "--http-root-cert".to_string(), + "local-ca-2.pem".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!( + args.http_root_cert_paths, + vec![ + PathBuf::from("local-ca-1.pem"), + PathBuf::from("local-ca-2.pem") + ] + ); +} + +#[test] +fn parse_rejects_invalid_rsync_scope_policy() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--rsync-scope".to_string(), + "wide".to_string(), + ]; + let err = parse_args(&argv).expect_err("invalid rsync scope should fail"); + assert!(err.contains("invalid --rsync-scope"), "{err}"); +} + +#[test] +fn parse_accepts_strict_policy_list() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--strict".to_string(), + "name,cms-der".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!( + args.strict_policy, + Some(StrictPolicy { + name: true, + cms_der: true, + signed_attrs: false, + }) + ); +} + +#[test] +fn parse_accepts_strict_without_value_as_all() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--strict".to_string(), + "--report-json".to_string(), + "out/report.json".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!(args.strict_policy, Some(StrictPolicy::all())); +} + +#[test] +fn parse_accepts_resource_validation_mode() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--resource-validation-mode".to_string(), + "rfc6487".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!( + args.resource_validation_mode, + Some(ResourceValidationMode::Rfc6487) + ); +} + +#[test] +fn parse_rejects_unknown_resource_validation_mode() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--resource-validation-mode".to_string(), + "bogus".to_string(), + ]; + let err = parse_args(&argv).expect_err("unknown mode should fail"); + assert!(err.contains("unknown resource validation mode"), "{err}"); +} + +#[test] +fn parse_rejects_unknown_strict_policy() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--strict=unknown".to_string(), + ]; + let err = parse_args(&argv).expect_err("unknown strict policy should fail"); + assert!(err.contains("unknown strict policy"), "{err}"); +} + +#[test] +fn effective_cir_tal_uris_filters_skipped_multi_tal_inputs() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "afrinic.tal".to_string(), + "--ta-path".to_string(), + "afrinic.cer".to_string(), + "--tal-path".to_string(), + "apnic.tal".to_string(), + "--ta-path".to_string(), + "apnic.cer".to_string(), + "--tal-path".to_string(), + "arin.tal".to_string(), + "--ta-path".to_string(), + "arin.cer".to_string(), + "--cir-enable".to_string(), + "--cir-out".to_string(), + "out.cir".to_string(), + "--cir-tal-uri".to_string(), + "https://example.test/afrinic.cer".to_string(), + "--cir-tal-uri".to_string(), + "https://example.test/apnic.cer".to_string(), + "--cir-tal-uri".to_string(), + "https://example.test/arin.cer".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + let mut shared = synthetic_post_validation_shared(); + shared.discoveries = vec![shared.discovery.clone(), shared.discovery.clone()].into(); + shared.successful_tal_inputs = + vec![args.tal_inputs[0].clone(), args.tal_inputs[2].clone()].into(); + + let effective = effective_cir_tal_uris_for_discoveries( + &args, + &shared, + resolve_cir_export_tal_uris(&args).expect("resolve cir tal uris"), + ) + .expect("map effective cir tal uris"); + + assert_eq!( + effective, + vec![ + "https://example.test/afrinic.cer".to_string(), + "https://example.test/arin.cer".to_string(), + ] + ); +} + +#[test] +fn parse_rejects_report_json_compact_without_report_json() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--report-json-compact".to_string(), + ]; + let err = parse_args(&argv).expect_err("compact flag without report path should fail"); + assert!( + err.contains("--report-json-compact requires --report-json"), + "{err}" + ); +} + +#[test] +fn parse_accepts_skip_report_build_without_report_json() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--ccr-out".to_string(), + "out/result.ccr".to_string(), + "--skip-report-build".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.skip_report_build); + assert_eq!( + args.ccr_out_path.as_deref(), + Some(std::path::Path::new("out/result.ccr")) + ); +} + +#[test] +fn parse_accepts_skip_vcir_persist() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--skip-vcir-persist".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.skip_vcir_persist); +} + +#[test] +fn parse_rejects_skip_report_build_with_report_json() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--report-json".to_string(), + "out/report.json".to_string(), + "--skip-report-build".to_string(), + ]; + let err = parse_args(&argv).expect_err("skip report build with report path should fail"); + assert!( + err.contains("--skip-report-build cannot be combined with --report-json"), + "{err}" + ); +} + +#[test] +fn parse_accepts_direct_compare_view_csv_outputs() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--vrps-csv-out".to_string(), + "out/vrps.csv".to_string(), + "--vaps-csv-out".to_string(), + "out/vaps.csv".to_string(), + "--compare-view-trust-anchor".to_string(), + "unknown".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!( + args.vrps_csv_out_path.as_deref(), + Some(std::path::Path::new("out/vrps.csv")) + ); + assert_eq!( + args.vaps_csv_out_path.as_deref(), + Some(std::path::Path::new("out/vaps.csv")) + ); + assert_eq!(args.compare_view_trust_anchor.as_deref(), Some("unknown")); +} + +#[test] +fn parse_rejects_partial_direct_compare_view_csv_outputs() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--vrps-csv-out".to_string(), + "out/vrps.csv".to_string(), + ]; + let err = parse_args(&argv).expect_err("partial direct compare view output should fail"); + assert!( + err.contains("--vrps-csv-out and --vaps-csv-out must be provided together"), + "{err}" + ); +} + +#[test] +fn parse_accepts_external_raw_store_db() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--raw-store-db".to_string(), + "raw-store.db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!( + args.raw_store_db.as_deref(), + Some(std::path::Path::new("raw-store.db")) + ); +} + +#[test] +fn parse_accepts_external_repo_bytes_db() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--repo-bytes-db".to_string(), + "repo-bytes.db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!( + args.repo_bytes_db.as_deref(), + Some(std::path::Path::new("repo-bytes.db")) + ); +} + +#[test] +fn parse_accepts_cir_enable_with_raw_store_backend() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--raw-store-db".to_string(), + "raw-store.db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--rsync-local-dir".to_string(), + "repo".to_string(), + "--cir-enable".to_string(), + "--cir-out".to_string(), + "out/example.cir".to_string(), + "--cir-tal-uri".to_string(), + "https://example.test/root.tal".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.cir_enabled); + assert_eq!( + args.raw_store_db.as_deref(), + Some(std::path::Path::new("raw-store.db")) + ); + assert_eq!(args.cir_static_root, None); +} + +#[test] +fn parse_accepts_cir_enable_with_required_paths_and_tal_override() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--rsync-local-dir".to_string(), + "repo".to_string(), + "--cir-enable".to_string(), + "--cir-out".to_string(), + "out/example.cir".to_string(), + "--cir-tal-uri".to_string(), + "https://example.test/root.tal".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.cir_enabled); + assert_eq!( + args.cir_out_path.as_deref(), + Some(std::path::Path::new("out/example.cir")) + ); + assert_eq!( + args.cir_tal_uri.as_deref(), + Some("https://example.test/root.tal") + ); + assert_eq!( + args.cir_tal_uris, + vec!["https://example.test/root.tal".to_string()] + ); +} + +#[test] +fn parse_rejects_deprecated_cir_static_root() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--cir-enable".to_string(), + "--cir-out".to_string(), + "out/example.cir".to_string(), + "--cir-static-root".to_string(), + "out/static".to_string(), + ]; + let err = parse_args(&argv).expect_err("cir-static-root should be rejected"); + assert!(err.contains("no longer supported"), "{err}"); +} + +#[test] +fn parse_accepts_default_parallel_config_and_phase2_overrides() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase2-object-workers".to_string(), + "3".to_string(), + "--parallel-phase2-worker-queue-capacity".to_string(), + "17".to_string(), + "--parallel-phase2-ready-batch-size".to_string(), + "31".to_string(), + "--parallel-phase2-ready-batch-wall-time-budget-ms".to_string(), + "43".to_string(), + "--parallel-phase2-result-drain-batch-size".to_string(), + "37".to_string(), + "--parallel-phase2-finalize-batch-size".to_string(), + "41".to_string(), + "--parallel-phase2-finalize-batch-wall-time-budget-ms".to_string(), + "47".to_string(), + "--parallel-phase2-finalize-queue-capacity".to_string(), + "8192".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!(args.parallel_phase2_config.object_workers, 3); + assert_eq!(args.parallel_phase2_config.worker_queue_capacity, 17); + assert_eq!(args.parallel_phase2_config.ready_batch_size, 31); + assert_eq!( + args.parallel_phase2_config.ready_batch_wall_time_budget_ms, + 43 + ); + assert_eq!( + args.parallel_phase2_config.object_result_drain_batch_size, + 37 + ); + assert_eq!( + args.parallel_phase2_config + .publication_point_finalize_batch_size, + 41 + ); + assert_eq!( + args.parallel_phase2_config + .publication_point_finalize_wall_time_budget_ms, + 47 + ); + assert_eq!( + args.parallel_phase2_config + .publication_point_finalize_queue_capacity, + 8192 + ); + assert_eq!(args.parallel_phase1_config, ParallelPhase1Config::default()); +} + +#[test] +fn parse_rejects_zero_phase2_ready_batch_size() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase2-ready-batch-size".to_string(), + "0".to_string(), + ]; + let err = parse_args(&argv).expect_err("zero ready batch must fail"); + assert!(err.contains("--parallel-phase2-ready-batch-size"), "{err}"); +} + +#[test] +fn parse_rejects_zero_phase2_result_drain_batch_size() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase2-result-drain-batch-size".to_string(), + "0".to_string(), + ]; + let err = parse_args(&argv).expect_err("zero result drain batch must fail"); + assert!( + err.contains("--parallel-phase2-result-drain-batch-size"), + "{err}" + ); +} + +#[test] +fn parse_rejects_zero_phase2_ready_batch_wall_time_budget_ms() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase2-ready-batch-wall-time-budget-ms".to_string(), + "0".to_string(), + ]; + let err = parse_args(&argv).expect_err("zero ready time budget must fail"); + assert!( + err.contains("--parallel-phase2-ready-batch-wall-time-budget-ms"), + "{err}" + ); +} + +#[test] +fn parse_rejects_zero_phase2_finalize_batch_size() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase2-finalize-batch-size".to_string(), + "0".to_string(), + ]; + let err = parse_args(&argv).expect_err("zero finalize batch must fail"); + assert!( + err.contains("--parallel-phase2-finalize-batch-size"), + "{err}" + ); +} + +#[test] +fn parse_rejects_zero_phase2_finalize_batch_wall_time_budget_ms() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase2-finalize-batch-wall-time-budget-ms".to_string(), + "0".to_string(), + ]; + let err = parse_args(&argv).expect_err("zero finalize time budget must fail"); + assert!( + err.contains("--parallel-phase2-finalize-batch-wall-time-budget-ms"), + "{err}" + ); +} diff --git a/crates/panda-rpki-validator/src/cli/tests_parts/parse_options.rs b/crates/panda-rpki-validator/src/cli/tests_parts/parse_options.rs new file mode 100644 index 0000000..7d641ab --- /dev/null +++ b/crates/panda-rpki-validator/src/cli/tests_parts/parse_options.rs @@ -0,0 +1,683 @@ +// CLI test group: parse options. + +#[test] +fn parse_rejects_zero_phase2_finalize_queue_capacity() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase2-finalize-queue-capacity".to_string(), + "0".to_string(), + ]; + let err = parse_args(&argv).expect_err("zero finalize queue capacity must fail"); + assert!( + err.contains("--parallel-phase2-finalize-queue-capacity"), + "{err}" + ); +} + +#[test] +fn parse_rejects_removed_parallel_enable_flags() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase1".to_string(), + ]; + let err = parse_args(&argv).expect_err("removed phase flag should fail"); + assert!(err.contains("unknown argument: --parallel-phase1"), "{err}"); + + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase2".to_string(), + ]; + let err = parse_args(&argv).expect_err("removed phase flag should fail"); + assert!(err.contains("unknown argument: --parallel-phase2"), "{err}"); +} + +#[test] +fn parse_accepts_multi_tal_cir_overrides_in_file_mode() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "apnic.tal".to_string(), + "--ta-path".to_string(), + "apnic.cer".to_string(), + "--tal-path".to_string(), + "arin.tal".to_string(), + "--ta-path".to_string(), + "arin.cer".to_string(), + "--rsync-local-dir".to_string(), + "repo".to_string(), + "--cir-enable".to_string(), + "--cir-out".to_string(), + "out/example.cir".to_string(), + "--cir-tal-uri".to_string(), + "https://example.test/apnic.tal".to_string(), + "--cir-tal-uri".to_string(), + "https://example.test/arin.tal".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!( + args.cir_tal_uris, + vec![ + "https://example.test/apnic.tal".to_string(), + "https://example.test/arin.tal".to_string() + ] + ); +} + +#[test] +fn parse_rejects_incomplete_or_invalid_cir_flags() { + let argv_missing = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--cir-enable".to_string(), + ]; + let err = parse_args(&argv_missing).unwrap_err(); + assert!(err.contains("--cir-enable requires --cir-out"), "{err}"); + + let argv_needs_enable = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--cir-out".to_string(), + "out/example.cir".to_string(), + ]; + let err = parse_args(&argv_needs_enable).unwrap_err(); + assert!(err.contains("require --cir-enable"), "{err}"); + + let argv_offline_missing_uri = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--rsync-local-dir".to_string(), + "repo".to_string(), + "--cir-enable".to_string(), + "--cir-out".to_string(), + "out/example.cir".to_string(), + ]; + let err = parse_args(&argv_offline_missing_uri).unwrap_err(); + assert!(err.contains("requires --cir-tal-uri"), "{err}"); +} + +#[test] +fn parse_rejects_invalid_validation_time() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--validation-time".to_string(), + "not-a-time".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("invalid --validation-time"), "{err}"); +} + +#[test] +fn parse_rejects_invalid_max_instances() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--max-instances".to_string(), + "nope".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("invalid --max-instances"), "{err}"); +} + +#[test] +fn parse_rejects_missing_value_for_db() { + let argv = vec!["rpki".to_string(), "--db".to_string()]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("--db requires a value"), "{err}"); +} + +#[test] +fn parse_rejects_missing_value_for_tal_url() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("--tal-url requires a value"), "{err}"); +} + +#[test] +fn parse_rejects_missing_db() { + let argv = vec!["rpki".to_string(), "--tal-url".to_string(), "x".to_string()]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("--db is required"), "{err}"); +} + +#[test] +fn parse_rejects_missing_tal_mode() { + let argv = vec!["rpki".to_string(), "--db".to_string(), "db".to_string()]; + let err = parse_args(&argv).unwrap_err(); + assert!( + err.contains("--tal-url") || err.contains("--tal-path"), + "{err}" + ); +} + +#[test] +fn parse_accepts_tal_url_mode() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!(args.tal_url.as_deref(), Some("https://example.test/x.tal")); + assert_eq!( + args.tal_urls, + vec!["https://example.test/x.tal".to_string()] + ); + assert!(args.tal_path.is_none()); + assert!(args.ta_path.is_none()); + assert_eq!(args.tal_inputs.len(), 1); + assert_eq!(args.tal_inputs[0].tal_id, "x"); + assert_eq!(args.parallel_phase1_config, ParallelPhase1Config::default()); + assert_eq!(args.parallel_phase2_config, ParallelPhase2Config::default()); +} + +#[test] +fn parse_accepts_multi_tal_without_parallel_flags() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/arin.tal".to_string(), + "--tal-url".to_string(), + "https://example.test/apnic.tal".to_string(), + "--tal-url".to_string(), + "https://example.test/ripe.tal".to_string(), + "--parallel-max-repo-sync-workers-global".to_string(), + "8".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!(args.tal_urls.len(), 3); + assert_eq!(args.tal_inputs.len(), 3); + assert_eq!(args.tal_inputs[0].tal_id, "arin"); + assert_eq!(args.tal_inputs[1].tal_id, "apnic"); + assert_eq!(args.tal_inputs[2].tal_id, "ripe"); + assert_eq!(args.parallel_phase1_config.max_repo_sync_workers_global, 8); +} + +#[test] +fn parse_accepts_multi_tal_urls_by_default() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/arin.tal".to_string(), + "--tal-url".to_string(), + "https://example.test/apnic.tal".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!(args.tal_urls.len(), 2); + assert_eq!(args.tal_inputs.len(), 2); +} + +#[test] +fn parse_accepts_offline_mode_requires_ta() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--ta-path".to_string(), + "ta.cer".to_string(), + "--max-depth".to_string(), + "0".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!(args.tal_paths, vec![PathBuf::from("a.tal")]); + assert_eq!(args.ta_paths, vec![PathBuf::from("ta.cer")]); + assert_eq!(args.tal_path.as_deref(), Some(Path::new("a.tal"))); + assert_eq!(args.ta_path.as_deref(), Some(Path::new("ta.cer"))); + assert_eq!(args.max_ca_depth, 0); +} + +#[test] +fn parse_accepts_multiple_tal_path_pairs_by_default() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "apnic.tal".to_string(), + "--ta-path".to_string(), + "apnic-ta.cer".to_string(), + "--tal-path".to_string(), + "arin.tal".to_string(), + "--ta-path".to_string(), + "arin-ta.cer".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!(args.tal_paths.len(), 2); + assert_eq!(args.ta_paths.len(), 2); + assert_eq!(args.tal_inputs.len(), 2); +} + +#[test] +fn parse_rejects_mixed_tal_url_and_tal_path_modes() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/arin.tal".to_string(), + "--tal-path".to_string(), + "apnic.tal".to_string(), + "--ta-path".to_string(), + "apnic-ta.cer".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!( + err.contains( + "must specify either one-or-more --tal-url or one-or-more --tal-path/--ta-path pairs" + ), + "{err}" + ); +} + +#[test] +fn parse_rejects_mismatched_tal_path_and_ta_path_counts() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "apnic.tal".to_string(), + "--tal-path".to_string(), + "arin.tal".to_string(), + "--ta-path".to_string(), + "apnic-ta.cer".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!( + err.contains("--tal-path and --ta-path counts must match"), + "{err}" + ); +} + +#[test] +fn parse_accepts_tal_path_without_ta_when_disable_rrdp_is_set() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--disable-rrdp".to_string(), + "--rsync-command".to_string(), + "/tmp/fake-rsync".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!(args.tal_path.as_deref(), Some(Path::new("a.tal"))); + assert!(args.ta_path.is_none()); + assert!(args.disable_rrdp); + assert_eq!( + args.rsync_command.as_deref(), + Some(Path::new("/tmp/fake-rsync")) + ); +} + +#[test] +fn parse_accepts_multiple_tal_paths_without_ta_when_disable_rrdp() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--tal-path".to_string(), + "b.tal".to_string(), + "--disable-rrdp".to_string(), + "--rsync-command".to_string(), + "/tmp/fake-rsync".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!( + args.tal_paths, + vec![PathBuf::from("a.tal"), PathBuf::from("b.tal")] + ); + assert!(args.ta_paths.is_empty()); + assert_eq!(args.tal_inputs.len(), 2); + assert!(args.disable_rrdp); +} + +#[test] +fn parse_accepts_payload_delta_replay_mode_with_offline_tal_and_ta() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--ta-path".to_string(), + "ta.cer".to_string(), + "--payload-base-archive".to_string(), + "base-archive".to_string(), + "--payload-base-locks".to_string(), + "base-locks.json".to_string(), + "--payload-delta-archive".to_string(), + "delta-archive".to_string(), + "--payload-delta-locks".to_string(), + "delta-locks.json".to_string(), + ]; + let args = parse_args(&argv).expect("parse delta replay mode"); + assert_eq!( + args.payload_base_archive.as_deref(), + Some(Path::new("base-archive")) + ); + assert_eq!( + args.payload_base_locks.as_deref(), + Some(Path::new("base-locks.json")) + ); + assert_eq!( + args.payload_delta_archive.as_deref(), + Some(Path::new("delta-archive")) + ); + assert_eq!( + args.payload_delta_locks.as_deref(), + Some(Path::new("delta-locks.json")) + ); +} + +#[test] +fn parse_rejects_partial_payload_delta_arguments_and_mutual_exclusion() { + let argv_partial = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--ta-path".to_string(), + "ta.cer".to_string(), + "--payload-base-archive".to_string(), + "base-archive".to_string(), + ]; + let err = parse_args(&argv_partial).unwrap_err(); + assert!(err.contains("must be provided together"), "{err}"); + + let argv_both = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--ta-path".to_string(), + "ta.cer".to_string(), + "--payload-replay-archive".to_string(), + "archive".to_string(), + "--payload-replay-locks".to_string(), + "locks.json".to_string(), + "--payload-base-archive".to_string(), + "base-archive".to_string(), + "--payload-base-locks".to_string(), + "base-locks.json".to_string(), + "--payload-delta-archive".to_string(), + "delta-archive".to_string(), + "--payload-delta-locks".to_string(), + "delta-locks.json".to_string(), + ]; + let err = parse_args(&argv_both).unwrap_err(); + assert!(err.contains("mutually exclusive"), "{err}"); +} + +#[test] +fn parse_rejects_payload_delta_with_tal_url_or_rsync_local_dir() { + let argv_url = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--payload-base-archive".to_string(), + "base-archive".to_string(), + "--payload-base-locks".to_string(), + "base-locks.json".to_string(), + "--payload-delta-archive".to_string(), + "delta-archive".to_string(), + "--payload-delta-locks".to_string(), + "delta-locks.json".to_string(), + ]; + let err = parse_args(&argv_url).unwrap_err(); + assert!(err.contains("--tal-url is not supported"), "{err}"); + + let argv_rsync = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--ta-path".to_string(), + "ta.cer".to_string(), + "--payload-base-archive".to_string(), + "base-archive".to_string(), + "--payload-base-locks".to_string(), + "base-locks.json".to_string(), + "--payload-delta-archive".to_string(), + "delta-archive".to_string(), + "--payload-delta-locks".to_string(), + "delta-locks.json".to_string(), + "--rsync-local-dir".to_string(), + "repo".to_string(), + ]; + let err = parse_args(&argv_rsync).unwrap_err(); + assert!( + err.contains("payload delta replay mode cannot be combined with --rsync-local-dir"), + "{err}" + ); +} + +#[test] +fn parse_accepts_payload_replay_mode_with_offline_tal_and_ta() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--ta-path".to_string(), + "ta.cer".to_string(), + "--payload-replay-archive".to_string(), + "archive".to_string(), + "--payload-replay-locks".to_string(), + "locks.json".to_string(), + ]; + let args = parse_args(&argv).expect("parse replay mode"); + assert_eq!( + args.payload_replay_archive.as_deref(), + Some(Path::new("archive")) + ); + assert_eq!( + args.payload_replay_locks.as_deref(), + Some(Path::new("locks.json")) + ); +} + +#[test] +fn parse_rejects_partial_payload_replay_arguments() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--ta-path".to_string(), + "ta.cer".to_string(), + "--payload-replay-archive".to_string(), + "archive".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("must be provided together"), "{err}"); +} + +#[test] +fn parse_rejects_payload_replay_with_tal_url_or_rsync_local_dir() { + let argv_url = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--payload-replay-archive".to_string(), + "archive".to_string(), + "--payload-replay-locks".to_string(), + "locks.json".to_string(), + ]; + let err = parse_args(&argv_url).unwrap_err(); + assert!(err.contains("--tal-url is not supported"), "{err}"); + + let argv_rsync = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--ta-path".to_string(), + "ta.cer".to_string(), + "--payload-replay-archive".to_string(), + "archive".to_string(), + "--payload-replay-locks".to_string(), + "locks.json".to_string(), + "--rsync-local-dir".to_string(), + "repo".to_string(), + ]; + let err = parse_args(&argv_rsync).unwrap_err(); + assert!( + err.contains("cannot be combined with --rsync-local-dir"), + "{err}" + ); +} + +#[test] +fn parse_accepts_validation_time_rfc3339() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--validation-time".to_string(), + "2026-01-01T00:00:00Z".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert!(args.validation_time.is_some()); +} + +#[test] +fn parse_rejects_removed_revalidate_only_flag() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--revalidate-only".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("unknown argument: --revalidate-only"), "{err}"); +} + +#[test] +fn read_policy_accepts_valid_toml() { + let dir = tempfile::tempdir().expect("tmpdir"); + let p = dir.path().join("policy.toml"); + std::fs::write( + &p, + "signed_object_failure_policy = \"drop_publication_point\"\n", + ) + .expect("write policy"); + + let policy = read_policy(Some(&p)).expect("parse policy"); + assert_eq!( + policy.signed_object_failure_policy, + crate::policy::SignedObjectFailurePolicy::DropPublicationPoint + ); + assert_eq!( + policy.resource_validation_mode, + ResourceValidationMode::ValidationUpdate03 + ); + assert_eq!(policy.strict, StrictPolicy::default()); +} + +#[test] +fn read_policy_accepts_strict_table() { + let dir = tempfile::tempdir().expect("tmpdir"); + let p = dir.path().join("policy.toml"); + std::fs::write( + &p, + r#" + [strict] + name = true + cms_der = true + "#, + ) + .expect("write policy"); + + let policy = read_policy(Some(&p)).expect("parse policy"); + assert_eq!( + policy.strict, + StrictPolicy { + name: true, + cms_der: true, + signed_attrs: false, + } + ); +} + +#[test] +fn read_policy_accepts_resource_validation_mode() { + let dir = tempfile::tempdir().expect("tmpdir"); + let p = dir.path().join("policy.toml"); + std::fs::write(&p, "resource_validation_mode = \"rfc6487\"\n").expect("write policy"); + + let policy = read_policy(Some(&p)).expect("parse policy"); + assert_eq!( + policy.resource_validation_mode, + ResourceValidationMode::Rfc6487 + ); +} + +#[test] +fn read_policy_reports_missing_file() { + let dir = tempfile::tempdir().expect("tmpdir"); + let p = dir.path().join("missing.toml"); + let err = read_policy(Some(&p)).unwrap_err(); + assert!(err.contains("read policy file failed"), "{err}"); +} diff --git a/crates/panda-rpki-validator/src/cli/tests_parts/report_helpers.rs b/crates/panda-rpki-validator/src/cli/tests_parts/report_helpers.rs new file mode 100644 index 0000000..8fd3a13 --- /dev/null +++ b/crates/panda-rpki-validator/src/cli/tests_parts/report_helpers.rs @@ -0,0 +1,281 @@ +// CLI test group: report helpers. + +fn synthetic_post_validation_shared() -> PostValidationShared { + let tal_bytes = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/tal/apnic-rfc7730-https.tal"), + ) + .expect("read tal fixture"); + let ta_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/ta/apnic-ta.cer"), + ) + .expect("read ta fixture"); + + let discovery = crate::validation::from_tal::discover_root_ca_instance_from_tal_and_ta_der( + &tal_bytes, &ta_der, None, + ) + .expect("discover root"); + + let tree = crate::validation::tree::TreeRunOutput { + instances_processed: 1, + instances_failed: 0, + warnings: vec![ + crate::report::Warning::new("synthetic warning") + .with_rfc_refs(&[crate::report::RfcRef("RFC 6487 §4.8.8.1")]) + .with_context("rsync://example.test/repo/pp/"), + ], + vrps: vec![ + crate::validation::objects::Vrp { + asn: 64496, + prefix: crate::data_model::roa::IpPrefix { + afi: crate::data_model::roa::RoaAfi::Ipv4, + prefix_len: 24, + addr: [192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + }, + max_length: 24, + }, + crate::validation::objects::Vrp { + asn: 64497, + prefix: crate::data_model::roa::IpPrefix { + afi: crate::data_model::roa::RoaAfi::Ipv6, + prefix_len: 48, + addr: [0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + }, + max_length: 64, + }, + ], + aspas: vec![crate::validation::objects::AspaAttestation { + customer_as_id: 64496, + provider_as_ids: vec![64497, 64498], + }], + router_keys: Vec::new(), + }; + + let mut pp1 = crate::audit::PublicationPointAudit::default(); + pp1.source = "fresh".to_string(); + pp1.rrdp_notification_uri = Some("https://example.test/n1.xml".to_string()); + pp1.manifest_rsync_uri = "rsync://example.test/repo/pp1/manifest.mft".to_string(); + pp1.objects.push(crate::audit::ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/pp1/a.roa".to_string(), + sha256_hex: "11".repeat(32), + kind: crate::audit::AuditObjectKind::Roa, + result: crate::audit::AuditObjectResult::Ok, + detail: None, + }); + let mut pp2 = crate::audit::PublicationPointAudit::default(); + pp2.source = "fresh".to_string(); + pp2.rrdp_notification_uri = Some("https://example.test/n1.xml".to_string()); + let mut pp3 = crate::audit::PublicationPointAudit::default(); + pp3.source = "fresh".to_string(); + pp3.rrdp_notification_uri = Some("https://example.test/n2.xml".to_string()); + + let out = crate::validation::run_tree_from_tal::RunTreeFromTalAuditOutput { + discovery: discovery.clone(), + discoveries: vec![discovery], + successful_tal_inputs: Vec::new(), + tree, + publication_points: vec![pp1, pp2, pp3], + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + downloads: Vec::new(), + download_stats: crate::audit::AuditDownloadStats::default(), + current_repo_objects: Vec::new(), + ccr_accumulator: None, + cir_input: crate::cir::CirInputSnapshot::default(), + }; + PostValidationShared::from_run_output(out) +} + +fn sample_cli_ccr_accumulator() -> CcrAccumulator { + let tal_bytes = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/tal/apnic-rfc7730-https.tal"), + ) + .expect("read tal fixture"); + let ta_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/ta/apnic-ta.cer"), + ) + .expect("read ta fixture"); + let discovery = crate::validation::from_tal::discover_root_ca_instance_from_tal_and_ta_der( + &tal_bytes, &ta_der, None, + ) + .expect("discover root"); + let mut accumulator = CcrAccumulator::new(vec![discovery.trust_anchor.clone()]); + let manifest_uri = "rsync://example.test/repo/current.mft".to_string(); + let projection = crate::storage::VcirCcrManifestProjection { + manifest_rsync_uri: manifest_uri.clone(), + manifest_sha256: vec![0x44; 32], + manifest_size: 2048, + manifest_ee_aki: vec![0x55; 20], + manifest_number_be: vec![1], + manifest_this_update: crate::storage::PackTime::from_utc_offset_datetime( + time::OffsetDateTime::now_utc(), + ), + manifest_sia_locations_der: vec![ + crate::ccr::manifest_location::encode_access_description_der(&AccessDescription { + access_method_oid: OID_AD_SIGNED_OBJECT.to_string(), + access_location: manifest_uri, + }) + .expect("encode signedObject"), + ], + subordinate_skis: vec![vec![0x33; 20]], + }; + accumulator + .append_manifest_projection(&projection) + .expect("append manifest projection"); + accumulator +} + +#[test] +fn build_report_and_helpers_work_on_synthetic_output() { + let shared = synthetic_post_validation_shared(); + let policy = Policy::default(); + let validation_time = time::OffsetDateTime::now_utc(); + let report = build_report(&policy, validation_time, &shared); + + assert_eq!(unique_rrdp_repos(&report), 2); + assert_eq!(report.vrps.len(), 2); + assert_eq!(report.aspas.len(), 1); + + print_summary(&report); +} + +#[test] +fn run_report_task_and_stage_timing_work() { + let shared = synthetic_post_validation_shared(); + let policy = Policy::default(); + let validation_time = time::OffsetDateTime::now_utc(); + let dir = tempfile::tempdir().expect("tmpdir"); + let report_path = dir.path().join("report.json"); + let report_output = run_report_task( + &policy, + validation_time, + &shared, + Some(&report_path), + ReportJsonFormat::Compact, + ) + .expect("run report task"); + + assert!(report_output.report_write_ms.is_some()); + + let report_json = std::fs::read_to_string(&report_path).expect("read report json"); + assert!(!report_json.contains('\n'), "{report_json}"); + let report: serde_json::Value = + serde_json::from_str(&report_json).expect("parse compact report json"); + assert_eq!(report["vrps"].as_array().unwrap().len(), 2); + assert_eq!(report["aspas"].as_array().unwrap().len(), 1); + assert_eq!(report["queryAudit"]["status"].as_str(), Some("complete")); + assert!(report["queryAudit"]["eventsCount"].as_u64().unwrap() > 0); + let events_path = dir.path().join( + report["queryAudit"]["eventsPath"] + .as_str() + .expect("events path"), + ); + let events = std::fs::read_to_string(events_path).expect("read validation events"); + assert!( + events + .lines() + .any(|line| line.contains("\"eventType\":\"object\"")) + ); + + let stage_timing = RunStageTiming { + validation_ms: 1, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + crypto_signature_cache_observe: None, + enable_transport_request_prefetch: false, + report_build_ms: report_output.report_build_ms, + report_write_ms: report_output.report_write_ms, + ccr_build_ms: Some(2), + ccr_build_breakdown: None, + ccr_write_ms: Some(3), + compare_view_build_ms: Some(4), + compare_view_write_ms: Some(5), + cir_build_cir_ms: Some(6), + cir_write_cir_ms: Some(7), + cir_total_ms: Some(8), + total_ms: 9, + publication_points: shared.publication_points.len(), + repo_sync_ms_total: 10, + publication_point_repo_sync_ms_total: 11, + download_event_count: 12, + rrdp_download_ms_total: 13, + rsync_download_ms_total: 14, + download_bytes_total: 15, + roa_validation_cache: crate::validation::objects::RoaValidationCacheStats::default(), + analysis_counts: std::collections::HashMap::new(), + analysis_phases: std::collections::HashMap::new(), + analysis_top_publication_points: Vec::new(), + analysis_top_publication_point_steps: Vec::new(), + analysis_top_publication_point_cache_steps: Vec::new(), + vcir_storage_summary_ms: Some(16), + vcir_storage: Some(VcirStorageSummary { + entry_count: 2, + vcir_value_bytes: 100, + vcir_value_bytes_max: 60, + vcir_value_bytes_max_manifest_rsync_uri: Some( + "rsync://example.test/repo/max.mft".to_string(), + ), + core_fields: VcirCoreFieldSizeBreakdown { + manifest_rsync_uri_bytes: 10, + ..VcirCoreFieldSizeBreakdown::default() + }, + ccr_projection: VcirCcrProjectionSizeBreakdown { + manifest_sha256_bytes: 32, + ..VcirCcrProjectionSizeBreakdown::default() + }, + child_resources: VcirChildResourceSizeBreakdown { + effective_ip_resource_cbor_bytes: 12, + effective_as_resource_cbor_bytes: 6, + }, + field_sizes: VcirFieldSizeBreakdown { + local_output_count: 1, + local_output_payload_json_bytes: 70, + local_output_payload_typed_body_bytes: 20, + ..VcirFieldSizeBreakdown::default() + }, + local_output_old_projection_bytes: 80, + local_output_typed_projection_bytes: 30, + local_output_projection_saved_bytes: 50, + top_entries_by_vcir_value_bytes: vec![VcirStorageEntrySummary { + manifest_rsync_uri: "rsync://example.test/repo/max.mft".to_string(), + vcir_value_bytes: 60, + local_vrp_count: 1, + local_aspa_count: 0, + local_router_key_count: 0, + accepted_object_count: 1, + rejected_object_count: 0, + child_count: 0, + core_fields: VcirCoreFieldSizeBreakdown::default(), + ccr_projection: VcirCcrProjectionSizeBreakdown::default(), + child_resources: VcirChildResourceSizeBreakdown::default(), + field_sizes: VcirFieldSizeBreakdown::default(), + local_output_old_projection_bytes: 1, + local_output_typed_projection_bytes: 1, + local_output_projection_saved_bytes: 0, + }], + }), + publication_point_cache_index_load: None, + publication_point_cache_index_refresh: None, + memory_telemetry: None, + }; + write_stage_timing(Some(&report_path), &stage_timing).expect("write stage timing"); + let stage_timing_json = + std::fs::read_to_string(dir.path().join("stage-timing.json")).expect("read timing"); + assert!(stage_timing_json.contains("\"validation_ms\"")); + assert!(stage_timing_json.contains("\"ccr_build_ms\"")); + assert!(stage_timing_json.contains("\"vcir_storage\"")); + assert!(stage_timing_json.contains("\"local_output_projection_saved_bytes\"")); + + let ccr_path = dir.path().join("result.ccr"); + write_stage_timing(Some(&ccr_path), &stage_timing).expect("write stage timing via ccr path"); + assert!( + dir.path().join("stage-timing.json").exists(), + "stage timing should use parent directory of the anchor path" + ); + + let skipped = ReportTaskOutput::skipped(); + assert_eq!(skipped.report_build_ms, 0); + assert!(skipped.report_write_ms.is_none()); +} diff --git a/crates/panda-rpki-validator/src/cli/tests_parts/report_tasks.rs b/crates/panda-rpki-validator/src/cli/tests_parts/report_tasks.rs new file mode 100644 index 0000000..d2fe18e --- /dev/null +++ b/crates/panda-rpki-validator/src/cli/tests_parts/report_tasks.rs @@ -0,0 +1,281 @@ +// CLI test group: report tasks. + +#[test] +fn stage_timing_serializes_memory_telemetry() { + let dir = tempfile::tempdir().expect("tmpdir"); + let report_path = dir.path().join("report.json"); + let stage_timing = RunStageTiming { + validation_ms: 1, + enable_roa_validation_cache: true, + enable_child_certificate_validation_cache: true, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + crypto_signature_cache_observe: None, + enable_transport_request_prefetch: true, + report_build_ms: 2, + report_write_ms: None, + ccr_build_ms: None, + ccr_build_breakdown: None, + ccr_write_ms: None, + compare_view_build_ms: None, + compare_view_write_ms: None, + cir_build_cir_ms: None, + cir_write_cir_ms: None, + cir_total_ms: None, + total_ms: 3, + publication_points: 4, + repo_sync_ms_total: 5, + publication_point_repo_sync_ms_total: 6, + download_event_count: 7, + rrdp_download_ms_total: 8, + rsync_download_ms_total: 9, + download_bytes_total: 10, + roa_validation_cache: crate::validation::objects::RoaValidationCacheStats { + hit_roas: 2, + ..crate::validation::objects::RoaValidationCacheStats::default() + }, + analysis_counts: std::collections::HashMap::from([( + "roa_validation_cache_hit_roas".to_string(), + 2, + )]), + analysis_phases: std::collections::HashMap::new(), + analysis_top_publication_points: Vec::new(), + analysis_top_publication_point_steps: Vec::new(), + analysis_top_publication_point_cache_steps: Vec::new(), + vcir_storage_summary_ms: None, + vcir_storage: None, + publication_point_cache_index_load: None, + publication_point_cache_index_refresh: None, + memory_telemetry: Some(MemoryTelemetrySummary { + checkpoints: vec![MemoryTelemetryCheckpoint { + label: "after_validation".to_string(), + elapsed_ms: 11, + process: ProcessMemorySnapshot { + label: "after_validation".to_string(), + vm_rss_kb: Some(12), + vm_size_kb: None, + vm_data_kb: None, + vm_swap_kb: None, + rss_anon_kb: Some(13), + rss_file_kb: None, + rss_shmem_kb: None, + threads: Some(14), + fd_count: Some(15), + smaps_rollup: None, + smaps_mapping_summary: None, + errors: Vec::new(), + }, + rocksdb: RocksDbMemorySnapshot { + databases: Vec::new(), + totals: RocksDbMemoryTotals { + cur_size_all_mem_tables: 16, + size_all_mem_tables: 17, + estimate_table_readers_mem: 18, + block_cache_capacity: 19, + block_cache_usage: 20, + block_cache_pinned_usage: 21, + }, + }, + }], + object_graph: None, + malloc_trim_probes: Vec::new(), + }), + }; + + write_stage_timing(Some(&report_path), &stage_timing).expect("write stage timing"); + let value: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(dir.path().join("stage-timing.json")).unwrap(), + ) + .expect("parse stage timing json"); + let checkpoint = &value["memory_telemetry"]["checkpoints"][0]; + assert_eq!(checkpoint["label"], "after_validation"); + assert_eq!(checkpoint["process"]["vm_rss_kb"], 12); + assert_eq!( + checkpoint["rocksdb"]["totals"]["cur_size_all_mem_tables"], + 16 + ); + assert_eq!(value["analysis_counts"]["roa_validation_cache_hit_roas"], 2); + assert_eq!(value["roa_validation_cache"]["hit_roas"], 2); + assert_eq!(value["publication_point_cache_observe_only"], false); + assert_eq!(value["enable_publication_point_validation_cache"], false); + assert!( + value["memory_telemetry"] + .as_object() + .expect("memory telemetry object") + .get("malloc_trim_probes") + .is_none() + ); +} + +#[test] +fn shared_object_graph_estimate_counts_audit_and_outputs() { + let mut shared = synthetic_post_validation_shared(); + let mut publication_points = shared + .publication_points + .iter() + .cloned() + .collect::>(); + publication_points[0].rsync_base_uri = "rsync://example.test/repo/".to_string(); + publication_points[0].manifest_rsync_uri = "rsync://example.test/repo/a.mft".to_string(); + publication_points[0].publication_point_rsync_uri = "rsync://example.test/repo/".to_string(); + publication_points[0].objects = vec![crate::audit::ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/a.roa".to_string(), + sha256_hex: "11".repeat(32), + kind: crate::audit::AuditObjectKind::Roa, + result: crate::audit::AuditObjectResult::Ok, + detail: None, + }]; + shared.publication_points = publication_points.into(); + + let graph = estimate_shared_object_graph(&shared); + let publication_points_section = graph + .sections + .iter() + .find(|section| section.name == "publication_points") + .expect("publication points section"); + let object_count = publication_points_section + .details + .iter() + .find(|metric| metric.name == "object_audit_entry_count") + .expect("object count metric"); + assert_eq!(object_count.value, 1); + assert!(publication_points_section.estimated_bytes > 0); + + let vrps_section = graph + .sections + .iter() + .find(|section| section.name == "vrps") + .expect("vrps section"); + assert_eq!(vrps_section.item_count, 2); + assert!(graph.total_estimated_bytes >= publication_points_section.estimated_bytes); +} + +#[test] +fn run_compare_view_task_writes_csv_from_shared_output() { + let shared = synthetic_post_validation_shared(); + let dir = tempfile::tempdir().expect("tmpdir"); + let vrps_path = dir.path().join("vrps.csv"); + let vaps_path = dir.path().join("vaps.csv"); + + let output = run_compare_view_task(&shared, Some(&vrps_path), Some(&vaps_path), "unknown") + .expect("write direct compare views"); + + assert!(output.build_ms.is_some()); + assert!(output.write_ms.is_some()); + let vrps_csv = std::fs::read_to_string(vrps_path).expect("read vrps csv"); + let vaps_csv = std::fs::read_to_string(vaps_path).expect("read vaps csv"); + assert!(vrps_csv.contains("ASN,IP Prefix,Max Length,Trust Anchor")); + assert!(vrps_csv.contains("AS64496,192.0.2.0/24,24,unknown")); + assert!(vrps_csv.contains("AS64497,2001:db8::/48,64,unknown")); + assert!(vaps_csv.contains("Customer ASN,Providers,Trust Anchor")); + assert!(vaps_csv.contains("AS64496,AS64497;AS64498,unknown")); +} + +#[test] +fn run_ccr_task_uses_accumulator_when_phase2_output_contains_reuse_sources() { + let mut shared = synthetic_post_validation_shared(); + shared.ccr_accumulator = Some(sample_cli_ccr_accumulator()); + let mut publication_points = shared + .publication_points + .iter() + .cloned() + .collect::>(); + publication_points[1].source = "vcir_current_instance".to_string(); + publication_points[2].source = "failed_no_cache".to_string(); + shared.publication_points = publication_points.into(); + let dir = tempfile::tempdir().expect("tmpdir"); + let ccr_path = dir.path().join("result.ccr"); + let store = RocksStore::open(&dir.path().join("db")).expect("open empty store"); + + let output = run_ccr_task( + &store, + &shared, + Some(&ccr_path), + time::OffsetDateTime::now_utc(), + ) + .expect("run ccr task"); + + assert!(output.ccr_build_ms.is_some()); + assert!(output.ccr_build_breakdown.is_none()); + let der = std::fs::read(&ccr_path).expect("read ccr"); + let ci = crate::ccr::decode_content_info(&der).expect("decode ccr"); + assert_eq!( + ci.content + .mfts + .as_ref() + .map(|manifest_state| manifest_state.mis.len()), + Some(1) + ); +} + +#[test] +fn write_json_writes_report() { + let report = AuditReportV2 { + format_version: 2, + meta: AuditRunMeta { + validation_time_rfc3339_utc: "2026-01-01T00:00:00Z".to_string(), + }, + policy: Policy::default(), + tree: TreeSummary { + instances_processed: 0, + instances_failed: 0, + warnings: Vec::new(), + }, + publication_points: Vec::new(), + vrps: Vec::new(), + aspas: Vec::new(), + downloads: Vec::new(), + download_stats: crate::audit::AuditDownloadStats::default(), + repo_sync_stats: crate::audit::AuditRepoSyncStats::default(), + query_audit: None, + }; + + let dir = tempfile::tempdir().expect("tmpdir"); + let pretty_path = dir.path().join("report-pretty.json"); + write_json(&pretty_path, &report, ReportJsonFormat::Pretty).expect("write pretty json"); + let pretty = std::fs::read_to_string(&pretty_path).expect("read pretty report"); + assert!(pretty.contains("\"format_version\"")); + assert!(pretty.contains("\"policy\"")); + assert!(pretty.contains("\n \"format_version\""), "{pretty}"); + + let compact_path = dir.path().join("report-compact.json"); + write_json(&compact_path, &report, ReportJsonFormat::Compact).expect("write compact json"); + let compact = std::fs::read_to_string(&compact_path).expect("read compact report"); + assert!(compact.contains("\"format_version\"")); + assert!(compact.contains("\"policy\"")); + assert!(!compact.contains('\n'), "{compact}"); +} + +#[test] +fn build_repo_sync_stats_aggregates_phase_and_terminal_state() { + let mut pp1 = crate::audit::PublicationPointAudit::default(); + pp1.repo_sync_phase = Some("rrdp_ok".to_string()); + pp1.repo_sync_duration_ms = Some(10); + pp1.repo_terminal_state = "fresh".to_string(); + + let mut pp2 = crate::audit::PublicationPointAudit::default(); + pp2.repo_sync_phase = Some("rrdp_failed_rsync_failed".to_string()); + pp2.repo_sync_duration_ms = Some(20); + pp2.repo_terminal_state = "failed_no_cache".to_string(); + + let mut pp3 = crate::audit::PublicationPointAudit::default(); + pp3.repo_sync_phase = Some("rrdp_failed_rsync_failed".to_string()); + pp3.repo_sync_duration_ms = Some(30); + pp3.repo_terminal_state = "failed_no_cache".to_string(); + + let stats = build_repo_sync_stats(&[pp1, pp2, pp3]); + assert_eq!(stats.publication_points_total, 3); + assert_eq!(stats.by_phase["rrdp_ok"].count, 1); + assert_eq!(stats.by_phase["rrdp_ok"].duration_ms_total, 10); + assert_eq!(stats.by_phase["rrdp_failed_rsync_failed"].count, 2); + assert_eq!( + stats.by_phase["rrdp_failed_rsync_failed"].duration_ms_total, + 50 + ); + assert_eq!(stats.by_terminal_state["fresh"].count, 1); + assert_eq!(stats.by_terminal_state["failed_no_cache"].count, 2); + assert_eq!( + stats.by_terminal_state["failed_no_cache"].duration_ms_total, + 50 + ); +} diff --git a/crates/panda-rpki-validator/src/cli/types.rs b/crates/panda-rpki-validator/src/cli/types.rs new file mode 100644 index 0000000..68bc3ef --- /dev/null +++ b/crates/panda-rpki-validator/src/cli/types.rs @@ -0,0 +1,135 @@ +// CLI timing and argument data types. + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +struct RunStageTiming { + validation_ms: u64, + enable_roa_validation_cache: bool, + enable_child_certificate_validation_cache: bool, + publication_point_cache_observe_only: bool, + enable_publication_point_validation_cache: bool, + crypto_signature_cache_observe: Option, + enable_transport_request_prefetch: bool, + report_build_ms: u64, + report_write_ms: Option, + ccr_build_ms: Option, + ccr_build_breakdown: Option, + ccr_write_ms: Option, + compare_view_build_ms: Option, + compare_view_write_ms: Option, + cir_build_cir_ms: Option, + cir_write_cir_ms: Option, + cir_total_ms: Option, + total_ms: u64, + publication_points: usize, + repo_sync_ms_total: u64, + publication_point_repo_sync_ms_total: u64, + download_event_count: u64, + rrdp_download_ms_total: u64, + rsync_download_ms_total: u64, + download_bytes_total: u64, + roa_validation_cache: crate::validation::objects::RoaValidationCacheStats, + analysis_counts: HashMap, + analysis_phases: HashMap, + analysis_top_publication_points: Vec, + analysis_top_publication_point_steps: Vec, + analysis_top_publication_point_cache_steps: Vec, + vcir_storage_summary_ms: Option, + vcir_storage: Option, + publication_point_cache_index_load: Option, + publication_point_cache_index_refresh: Option, + memory_telemetry: Option, +} + +fn record_memory_checkpoint( + checkpoints: &mut Vec, + label: &str, + total_started: &std::time::Instant, + store: &RocksStore, +) { + checkpoints.push(MemoryTelemetryCheckpoint { + label: label.to_string(), + elapsed_ms: total_started.elapsed().as_millis() as u64, + process: crate::memory_telemetry::process_memory_snapshot(label), + rocksdb: store.memory_snapshot(), + }); +} + +fn memory_trim_probe_enabled() -> bool { + std::env::var("RPKI_MEMORY_TRIM_PROBE") + .map(|value| matches!(value.as_str(), "1" | "true" | "TRUE" | "yes" | "YES")) + .unwrap_or(false) +} + +fn vcir_storage_summary_enabled() -> bool { + std::env::var("RPKI_VCIR_STORAGE_SUMMARY") + .map(|value| matches!(value.as_str(), "1" | "true" | "TRUE" | "yes" | "YES")) + .unwrap_or(false) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CliArgs { + pub validation_contract_out_path: Option, + pub tal_urls: Vec, + pub tal_paths: Vec, + pub ta_paths: Vec, + pub tal_url: Option, + pub tal_path: Option, + pub ta_path: Option, + pub parallel_phase1_config: ParallelPhase1Config, + pub parallel_phase2_config: ParallelPhase2Config, + pub tal_inputs: Vec, + pub ta_constraints: TaConstraintsByTal, + + pub db_path: PathBuf, + pub raw_store_db: Option, + pub repo_bytes_db: Option, + pub policy_path: Option, + pub strict_policy: Option, + pub resource_validation_mode: Option, + pub report_json_path: Option, + pub report_json_compact: bool, + pub skip_report_build: bool, + pub skip_vcir_persist: bool, + pub enable_roa_validation_cache: bool, + pub enable_child_certificate_validation_cache: bool, + pub publication_point_cache_observe_only: bool, + pub enable_publication_point_validation_cache: bool, + pub crypto_signature_cache_observe_only: bool, + pub enable_crypto_signature_cache: bool, + pub enable_transport_request_prefetch: bool, + pub ccr_out_path: Option, + pub vrps_csv_out_path: Option, + pub vaps_csv_out_path: Option, + pub compare_view_trust_anchor: Option, + pub cir_enabled: bool, + pub cir_out_path: Option, + pub cir_static_root: Option, + pub cir_tal_uris: Vec, + pub cir_tal_uri: Option, + pub payload_replay_archive: Option, + pub payload_replay_locks: Option, + pub payload_base_archive: Option, + pub payload_base_locks: Option, + pub payload_base_validation_time: Option, + pub payload_delta_archive: Option, + pub payload_delta_locks: Option, + pub memory_trim_after_validation: bool, + + pub rsync_local_dir: Option, + pub disable_rrdp: bool, + pub rsync_command: Option, + + pub http_timeout_secs: u64, + pub http_root_cert_paths: Vec, + pub rsync_timeout_secs: u64, + pub rsync_mirror_root: Option, + pub rsync_scope_policy: RsyncScopePolicy, + + pub max_ca_depth: usize, + pub max_instances: Option, + pub validation_time: Option, + + pub analyze: bool, + pub analysis_out_path: Option, + pub profile_cpu: bool, +} diff --git a/crates/panda-rpki-validator/src/cli/usage.rs b/crates/panda-rpki-validator/src/cli/usage.rs new file mode 100644 index 0000000..2658696 --- /dev/null +++ b/crates/panda-rpki-validator/src/cli/usage.rs @@ -0,0 +1,116 @@ +// Command-line usage text. + +fn usage() -> String { + let bin = "panda-rpki-validator"; + format!( + "\ +Usage: + {bin} --db --tal-url [--tal-url ...] [options] + {bin} --db --tal-path --ta-path [--tal-path --ta-path ...] [options] + +Options: + --validation-contract-out + Write the effective normal-run validation contract + --db RocksDB directory path (required) + --raw-store-db External raw-by-hash store DB path (optional) + --repo-bytes-db External repo object bytes DB path (optional) + --policy Policy TOML path (optional) + --ta-constraints = + Apply local EE-resource constraints to one TAL (repeatable); adjacent .constraints files are auto-discovered + --strict [policies] Enable strict policies (default all; comma list: name,cms-der,signed-attrs; none disables) + --resource-validation-mode + Resource certificate validation mode (default: validation-update-03) + --report-json Write full audit report as JSON (optional) + --report-json-compact Write report JSON without pretty-printing (requires --report-json) + --skip-report-build Skip full audit report construction when --report-json is not requested + --skip-vcir-persist Skip VCIR persistence/projection building for compare-only runs + --enable-roa-validation-cache + Reuse accepted ROA validation outputs from previous VCIR records (default: off) + --enable-child-certificate-validation-cache + Experimental: reuse validated child certificate discovery results + --publication-point-cache-observe-only + Evaluate publication-point cache eligibility without changing results + --enable-publication-point-validation-cache + Experimental: reuse complete publication-point validation projections + --crypto-signature-cache-observe-only + Measure crypto signature cache key hit rates and verify durations + without changing validation behavior (default: off) + --enable-crypto-signature-cache + Experimental: skip cryptographic signature verification on cache + hits (positive conclusions only; default: off) + --enable-transport-request-prefetch + Experimental: prefetch previous run transport repo requests before tree traversal + --ccr-out Write CCR DER ContentInfo to this path (optional) + --vrps-csv-out Write VRP compare-view CSV directly from validation output (optional; requires --vaps-csv-out) + --vaps-csv-out Write VAP compare-view CSV directly from validation output (optional; requires --vrps-csv-out) + --compare-view-trust-anchor + Trust-anchor label used by direct compare-view CSV output (default: unknown) + --cir-enable Export CIR after the run completes + --cir-out Write CIR DER to this path (requires --cir-enable) + --cir-static-root Deprecated; CIR export no longer exports object pools + --cir-tal-uri Override TAL URI for CIR export (repeatable in multi-TAL mode) + --payload-replay-archive Use local payload replay archive root (offline replay mode) + --payload-replay-locks Use local payload replay locks.json (offline replay mode) + --payload-base-archive Use local base payload archive root (offline delta replay) + --payload-base-locks Use local base locks.json (offline delta replay) + --payload-base-validation-time Validation time for the base bootstrap inside offline delta replay + --payload-delta-archive Use local delta payload archive root (offline delta replay) + --payload-delta-locks Use local locks-delta.json (offline delta replay) + --memory-trim-after-validation Call malloc_trim(0) after validation/report memory checkpoints (Linux glibc only; default off) + + --tal-url TAL URL (repeatable; URL mode) + --tal-path TAL file path (repeatable; file mode) + --ta-path TA certificate DER file path (repeatable in file mode; pairs with --tal-path by position) + --parallel-max-repo-sync-workers-global + Phase 1 global repo sync worker budget (default: 4) + --parallel-max-inflight-snapshot-bytes-global + Phase 1 inflight snapshot byte budget (default: 512MiB) + --parallel-max-pending-repo-results + Phase 1 pending repo result budget (default: 1024) + --parallel-phase2-object-workers + Phase 2 object worker count (default: 8) + --parallel-phase2-worker-queue-capacity + Phase 2 per-worker object queue capacity (default: 256) + --parallel-phase2-ready-batch-size + Phase 2 ready publication points processed per scheduler turn (default: 256) + --parallel-phase2-ready-batch-wall-time-budget-ms + Phase 2 ready staging wall-time budget per scheduler turn (default: 100) + --parallel-phase2-result-drain-batch-size + Phase 2 object results drained per scheduler turn (default: 2048) + --parallel-phase2-finalize-batch-size + Legacy Phase 2 scheduler finalize budget; dedicated finalize worker ignores it (default: 256) + --parallel-phase2-finalize-batch-wall-time-budget-ms + Legacy Phase 2 scheduler finalize time budget; dedicated finalize worker ignores it (default: 100) + --parallel-phase2-finalize-queue-capacity + Phase 2 dedicated finalize worker queue capacity (default: 32768) + --control-plane-stage-workers + Experimental: Phase 2 ready publication point stage worker count; + 0 disables the stage pool and keeps inline staging (default: 0) + --dead-repo-blacklist + Enable the dead-repo transport blacklist persisted at this JSON + path (default: disabled). Blacklisted rrdp repos skip straight to + rsync; dual-blacklisted repos terminate instantly. + --dead-repo-blacklist-fail-threshold + Consecutive runs with transport-class fetch failure before a + (repo, transport) entry is blacklisted (default: 3) + + --rsync-local-dir Use LocalDirRsyncFetcher rooted at this directory (offline tests) + --disable-rrdp Disable RRDP and synchronize only via rsync + --rsync-command Use this rsync command instead of the default rsync binary + --http-timeout-secs HTTP fetch timeout seconds (default: 20) + --http-root-cert Extra PEM root certificate trusted by HTTPS fetches (repeatable) + --rsync-timeout-secs rsync I/O timeout seconds (default: 60) + --rsync-mirror-root Persist rsync mirrors under this directory (default: disabled) + --rsync-scope rsync scope policy: host, publication-point, or module-root (default: module-root) + --max-ca-depth Maximum CA depth from a trust anchor (root = 0, default: {DEFAULT_MAX_CA_DEPTH}) + --max-depth Deprecated alias for --max-ca-depth + --max-instances Max number of CA instances to process + --validation-time Validation time in RFC3339 (default: now UTC) + --analyze Write timing analysis JSON under target/live/analyze// + --analysis-out Write timing analysis JSON under this directory (implies --analyze) + --profile-cpu (Requires build feature 'profile') Write CPU flamegraph under analyze dir + + --help Show this help +" + ) +} diff --git a/crates/panda-rpki-validator/src/crypto_sig_cache.rs b/crates/panda-rpki-validator/src/crypto_sig_cache.rs index c66f94c..939b377 100644 --- a/crates/panda-rpki-validator/src/crypto_sig_cache.rs +++ b/crates/panda-rpki-validator/src/crypto_sig_cache.rs @@ -1,4 +1,4 @@ -//! Crypto signature verification cache (feature #126). +//! Crypto signature verification cache. //! //! Caches the context-free cryptographic fact "this signature over these exact bytes verifies //! under this public key" so that later runs can skip the RSA verification entirely. Only @@ -8,14 +8,14 @@ //! //! Modes (CLI): //! -//! - `--crypto-signature-cache-observe-only` (M1): compute keys, maintain the persistent set, +//! - `--crypto-signature-cache-observe-only`: compute keys, maintain the persistent set, //! record hit/miss/timing statistics, always run the real verification. -//! - `--enable-crypto-signature-cache` (M2): on a cache hit, skip the real verification and +//! - `--enable-crypto-signature-cache`: on a cache hit, skip the real verification and //! return success; on a miss, run the real verification and store the positive conclusion. //! Statistics are recorded the same way; `verify_skipped`/`verify_executed` report what //! actually happened. Both flags may be combined (reuse + statistics). //! -//! The five choke points (see the development plan for the full caller inventory): +//! The five verification choke points are: //! //! - `CmsSignedObject`: `data_model::signed_object` CMS verify (ROA / ASPA / manifest on the //! main validation path; GBR objects are not signature-verified by the tree runner). @@ -30,9 +30,9 @@ //! Trust-anchor self-signature verification is deliberately excluded (a handful of calls per //! run). //! -//! Cache key (M1 decision: verify-input bytes, not full object bytes — the key material is in +//! Cache key: verify-input bytes, not full object bytes. The key material is in //! hand inside each choke point, while the repo-bytes content hash is only available at the -//! upper processing layer): +//! upper processing layer. //! //! ```text //! key = SHA-256( "rpki-crypto-sig-cache-key-v1" | point-tag | "sha256WithRSAEncryption" @@ -46,18 +46,18 @@ //! Storage: //! //! - Single small file next to the work DB (`.crypto-sig-cache`), never inside -//! the work DB, not touched by periodic snapshot reset (#091/#092): the reset paths only +//! the work DB, not touched by periodic snapshot reset: the reset paths only //! remove/replace the work DB directory and the PP/child-cert cache artifacts, not siblings //! of their own choosing. //! - Binary format: 8-byte magic `RPKICSC2` | u64 record-count | records of -//! key[32] | verified_at_unix_secs(i64) | crypto_impl_version(u32) | reserved(u32). -//! A missing, corrupt, or magic-mismatched file (including the M1 `RPKICSC1` seen-set +//! key\[32\] | verified_at_unix_secs(i64) | crypto_impl_version(u32) | reserved(u32). +//! A missing, corrupt, or magic-mismatched file (including the legacy `RPKICSC1` seen-set //! format) is silently rebuilt empty. Writes are atomic (write-then-rename). -//! - The cache is persisted once at the end of the validation stage (same point as the M1 -//! seen-set). If a run fails mid-validation, entries added during that run are lost; +//! - The cache is persisted once at the end of the validation stage. If a run fails +//! mid-validation, entries added during that run are lost; //! previously persisted entries are unaffected. -//! - Size cap: approximate上限 `capacity` entries (default 2,000,000; measured M1 APNIC -//! ~148k keys/run, all5 estimated ~750k). When an insert would exceed the cap, the whole +//! - Size cap: approximately `capacity` entries (default 2,000,000). When an insert +//! would exceed the cap, the whole //! cache is cleared and rebuilt from scratch (full-rebuild eviction — the simplest policy; //! concurrent inserts may briefly overshoot the cap by a few entries). Each eviction is //! counted in `evictions_total`. @@ -222,8 +222,8 @@ pub struct CryptoSigCache { impl CryptoSigCache { /// Load the cache from `cache_file`; start empty when the file is missing, corrupt, or - /// has an unknown schema header. `reuse_enabled` selects the M2 hit-skips-verify - /// behavior; when false the cache is observe-only (M1 semantics). + /// has an unknown schema header. `reuse_enabled` selects hit-skips-verify + /// behavior; when false the cache is observe-only. pub fn load_or_rebuild(cache_file: PathBuf, reuse_enabled: bool) -> Self { Self::load_or_rebuild_with_capacity(cache_file, reuse_enabled, DEFAULT_CAPACITY) } @@ -760,7 +760,7 @@ mod tests { let cache = CryptoSigCache::load_or_rebuild(path.clone(), true); assert_eq!(cache.summary().entries_loaded, 0); - // M1 seen-set format (RPKICSC1) must not load as a v2 cache: cold start. + // Legacy seen-set format (RPKICSC1) must not load as a v2 cache: cold start. let mut bytes = Vec::new(); bytes.extend_from_slice(b"RPKICSC1"); bytes.extend_from_slice(&1u64.to_le_bytes()); diff --git a/crates/panda-rpki-validator/src/daemon.rs b/crates/panda-rpki-validator/src/daemon.rs index 4f9095b..b4ecb24 100644 --- a/crates/panda-rpki-validator/src/daemon.rs +++ b/crates/panda-rpki-validator/src/daemon.rs @@ -9,2208 +9,13 @@ use std::time::Duration; use crate::parallel::dead_repo_blacklist::DeadRepoBlacklist; use crate::parallel::types::RepoTransportMode; -#[derive(Clone, Debug, PartialEq, Eq)] -struct Args { - state_root: PathBuf, - rpki_bin: PathBuf, - interval_secs: u64, - max_runs: Option, - retain_runs: usize, - status_json: Option, - summary_jsonl: Option, - work_db: PathBuf, - repo_bytes_db: Option, - raw_store_db: Option, - db_stats_bin: Option, - db_stats_exact_every: Option, - time_bin: Option, - /// Dead-repo blacklist (#141): enables daemon-side health checks and - /// auto-injects the child `--dead-repo-blacklist` flag when absent. - dead_repo_blacklist: Option, - dead_repo_blacklist_fail_threshold: Option, - dead_repo_health_check_interval_secs: u64, - dead_repo_probe_rsync_bin: PathBuf, - child_args: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -struct RunContext { - seq: u64, - run_id: String, - run_dir: PathBuf, -} - -#[derive(Clone, Debug, Serialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -enum DaemonState { - Starting, - Idle, - Running, - Collecting, - Sleeping, - Exited, -} - -#[derive(Clone, Debug, Serialize, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -struct DaemonStatus { - state: DaemonState, - updated_at_rfc3339_utc: String, - runs_completed: u64, - max_runs: Option, - current_run_seq: Option, - current_run_id: Option, - last_run_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - dead_repo_blacklist: Option, -} - -#[derive(Clone, Debug, Serialize, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -struct DeadRepoBlacklistStatus { - path: String, - entries: usize, - blacklisted: usize, - last_health_check_at_rfc3339_utc: Option, -} - -/// Mutable daemon-side health check bookkeeping (#141). -#[derive(Clone, Debug, Default, PartialEq, Eq)] -struct DeadRepoHealthRuntime { - last_health_check_at: Option, -} - -#[derive(Clone, Debug, Serialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -enum RunStatus { - Success, - Failed, - SpawnFailed, -} - -#[derive(Clone, Debug, Serialize, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -struct ArtifactInfo { - path: String, - size_bytes: u64, -} - -#[derive(Clone, Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct ProcessMetrics { - time_wrapper_used: bool, - time_output_path: Option, - user_seconds: Option, - system_seconds: Option, - cpu_percent: Option, - elapsed_raw: Option, - max_rss_kb: Option, - exit_status_from_time: Option, - parse_error: Option, -} - -#[derive(Clone, Debug, Serialize, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -struct ReportCounts { - vrps: usize, - aspas: usize, - publication_points: usize, - rrdp_repos_unique: Option, - tree_instances_processed: Option, - tree_instances_failed: Option, - warnings: usize, -} - -#[derive(Clone, Debug, Serialize, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -struct PathFileStats { - label: String, - path: String, - exists: bool, - is_dir: bool, - total_size_bytes: u64, - file_count: u64, - dir_count: u64, -} - -#[derive(Clone, Debug, Serialize, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -struct DbStatsSummary { - mode: String, - db_path: String, - output_path: Option, - stderr_path: Option, - status: String, - exit_code: Option, - error: Option, - metrics: BTreeMap, -} - -#[derive(Clone, Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct RunSummary { - run_seq: u64, - run_id: String, - run_dir: String, - started_at_rfc3339_utc: String, - finished_at_rfc3339_utc: String, - wall_ms: u64, - status: RunStatus, - exit_code: Option, - exit_status: Option, - error: Option, - rpki_bin: String, - child_args: Vec, - stdout_path: String, - stderr_path: String, - process_metrics: Option, - stage_timing: Option, - report_counts: Option, - repo_sync_stats: Option, - path_stats: Vec, - db_stats: Vec, - retention_deleted_runs: Vec, - artifacts: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - dead_repo_blacklist: Option, -} - -#[derive(Clone, Debug, Serialize, PartialEq, Eq)] -#[serde(rename_all = "camelCase")] -struct DeadRepoBlacklistRunSummary { - path: String, - entries: usize, - blacklisted: usize, -} - -fn usage() -> String { - let bin = "rpki_daemon"; - format!( - "\ -Usage: - {bin} --state-root --rpki-bin [options] -- - -Options: - --state-root Persistent daemon root containing state/, runs/, status, and JSONL summary - --rpki-bin rpki child binary to execute for each run - --interval-secs Sleep seconds between runs (default: 60) - --max-runs Stop after n runs (default: run forever) - --retain-runs Keep only the latest n run directories (default: 10) - --status-json Override status JSON path (default: /daemon-status.json) - --summary-jsonl Override summary JSONL path (default: /daemon-runs.jsonl) - --work-db Work DB path for metrics (default: /state/work-db) - --repo-bytes-db Repo bytes DB path for file metrics (default: /state/repo-bytes.db) - --raw-store-db Raw store DB path for file metrics (optional) - --db-stats-bin db_stats binary path (default: sibling db_stats next to this executable when present) - --db-stats-exact-every - Run db_stats --exact every n runs (default: disabled) - --time-bin GNU time binary for child process metrics (default: /usr/bin/time when present) - --no-time-wrapper Disable GNU time wrapper - --dead-repo-blacklist - Enable the dead-repo transport blacklist (#141): daemon probes - blacklisted entries between runs and removes revived repos; - auto-injects the child flag when not already present - --dead-repo-blacklist-fail-threshold - Admission threshold forwarded to the child when its args do not - already set it (default: child default 3) - --dead-repo-health-check-interval-secs - Minimum seconds between health check sweeps (default: 600) - --dead-repo-probe-rsync-bin - rsync binary used for health probes (default: rsync) - --help Show this help - -Child argument placeholders: - {{state_root}} Daemon state root - {{run_out}} Current run output directory - {{run_id}} Current run id, e.g. 000001-20260428T090000Z - {{run_seq}} Current run sequence number -" - ) -} - -fn default_time_bin() -> Option { - let path = PathBuf::from("/usr/bin/time"); - if path.is_file() { Some(path) } else { None } -} - -fn default_db_stats_bin() -> Option { - let mut path = std::env::current_exe().ok()?; - path.set_file_name("db_stats"); - if path.is_file() { Some(path) } else { None } -} - -fn parse_args(argv: &[String]) -> Result { - if argv.iter().any(|arg| arg == "--help" || arg == "-h") { - return Err(usage()); - } - - let mut state_root: Option = None; - let mut rpki_bin: Option = None; - let mut interval_secs = 60u64; - let mut max_runs = None; - let mut retain_runs = 10usize; - let mut status_json: Option = None; - let mut summary_jsonl: Option = None; - let mut work_db: Option = None; - let mut repo_bytes_db: Option = None; - let mut raw_store_db: Option = None; - let mut db_stats_bin: Option = None; - let mut db_stats_exact_every = None; - let mut time_bin = default_time_bin(); - let mut no_time_wrapper = false; - let mut dead_repo_blacklist: Option = None; - let mut dead_repo_blacklist_fail_threshold: Option = None; - let mut dead_repo_health_check_interval_secs = 600u64; - let mut dead_repo_probe_rsync_bin = PathBuf::from("rsync"); - - let mut i = 1usize; - while i < argv.len() { - match argv[i].as_str() { - "--" => { - let mut child_args = argv[i + 1..].to_vec(); - let state_root = - state_root.ok_or_else(|| format!("--state-root is required\n\n{}", usage()))?; - let work_db = work_db.unwrap_or_else(|| state_root.join("state").join("work-db")); - let repo_bytes_db = - repo_bytes_db.or_else(|| Some(state_root.join("state").join("repo-bytes.db"))); - if no_time_wrapper { - time_bin = None; - } - // Dead-repo blacklist (#141): forward enablement to the child - // unless its args already carry the flag explicitly. - if let Some(path) = dead_repo_blacklist.as_ref() { - if !child_args.iter().any(|arg| arg == "--dead-repo-blacklist") { - child_args.push("--dead-repo-blacklist".to_string()); - child_args.push(path_string(path)); - } - if let Some(threshold) = dead_repo_blacklist_fail_threshold { - if !child_args - .iter() - .any(|arg| arg == "--dead-repo-blacklist-fail-threshold") - { - child_args.push("--dead-repo-blacklist-fail-threshold".to_string()); - child_args.push(threshold.to_string()); - } - } - } - let args = Args { - state_root, - rpki_bin: rpki_bin - .ok_or_else(|| format!("--rpki-bin is required\n\n{}", usage()))?, - interval_secs, - max_runs, - retain_runs, - status_json, - summary_jsonl, - work_db, - repo_bytes_db, - raw_store_db, - db_stats_bin, - db_stats_exact_every, - time_bin, - dead_repo_blacklist, - dead_repo_blacklist_fail_threshold, - dead_repo_health_check_interval_secs, - dead_repo_probe_rsync_bin, - child_args, - }; - return validate_args(args); - } - "--state-root" => { - i += 1; - state_root = Some(PathBuf::from(value_at(argv, i, "--state-root")?)); - } - "--rpki-bin" => { - i += 1; - rpki_bin = Some(PathBuf::from(value_at(argv, i, "--rpki-bin")?)); - } - "--interval-secs" => { - i += 1; - interval_secs = - parse_u64(value_at(argv, i, "--interval-secs")?, "--interval-secs")?; - } - "--max-runs" => { - i += 1; - let parsed = parse_u64(value_at(argv, i, "--max-runs")?, "--max-runs")?; - if parsed == 0 { - return Err("--max-runs must be > 0".to_string()); - } - max_runs = Some(parsed); - } - "--retain-runs" => { - i += 1; - let parsed = parse_usize(value_at(argv, i, "--retain-runs")?, "--retain-runs")?; - if parsed == 0 { - return Err("--retain-runs must be > 0".to_string()); - } - retain_runs = parsed; - } - "--status-json" => { - i += 1; - status_json = Some(PathBuf::from(value_at(argv, i, "--status-json")?)); - } - "--summary-jsonl" => { - i += 1; - summary_jsonl = Some(PathBuf::from(value_at(argv, i, "--summary-jsonl")?)); - } - "--work-db" => { - i += 1; - work_db = Some(PathBuf::from(value_at(argv, i, "--work-db")?)); - } - "--repo-bytes-db" => { - i += 1; - repo_bytes_db = Some(PathBuf::from(value_at(argv, i, "--repo-bytes-db")?)); - } - "--raw-store-db" => { - i += 1; - raw_store_db = Some(PathBuf::from(value_at(argv, i, "--raw-store-db")?)); - } - "--db-stats-bin" => { - i += 1; - db_stats_bin = Some(PathBuf::from(value_at(argv, i, "--db-stats-bin")?)); - } - "--db-stats-exact-every" => { - i += 1; - let parsed = parse_u64( - value_at(argv, i, "--db-stats-exact-every")?, - "--db-stats-exact-every", - )?; - if parsed == 0 { - return Err("--db-stats-exact-every must be > 0".to_string()); - } - db_stats_exact_every = Some(parsed); - } - "--time-bin" => { - i += 1; - time_bin = Some(PathBuf::from(value_at(argv, i, "--time-bin")?)); - } - "--no-time-wrapper" => { - no_time_wrapper = true; - } - "--dead-repo-blacklist" => { - i += 1; - dead_repo_blacklist = - Some(PathBuf::from(value_at(argv, i, "--dead-repo-blacklist")?)); - } - "--dead-repo-blacklist-fail-threshold" => { - i += 1; - let parsed = parse_u64( - value_at(argv, i, "--dead-repo-blacklist-fail-threshold")?, - "--dead-repo-blacklist-fail-threshold", - )?; - if parsed == 0 || parsed > u32::MAX as u64 { - return Err( - "--dead-repo-blacklist-fail-threshold must be in 1..=u32::MAX".to_string(), - ); - } - dead_repo_blacklist_fail_threshold = Some(parsed as u32); - } - "--dead-repo-health-check-interval-secs" => { - i += 1; - dead_repo_health_check_interval_secs = parse_u64( - value_at(argv, i, "--dead-repo-health-check-interval-secs")?, - "--dead-repo-health-check-interval-secs", - )?; - } - "--dead-repo-probe-rsync-bin" => { - i += 1; - dead_repo_probe_rsync_bin = - PathBuf::from(value_at(argv, i, "--dead-repo-probe-rsync-bin")?); - } - other => return Err(format!("unknown argument: {other}\n\n{}", usage())), - } - i += 1; - } - - Err(format!("missing -- before child rpki args\n\n{}", usage())) -} - -fn validate_args(args: Args) -> Result { - if args.child_args.is_empty() { - return Err(format!( - "child rpki args are required after --\n\n{}", - usage() - )); - } - if args.dead_repo_blacklist_fail_threshold.is_some() && args.dead_repo_blacklist.is_none() { - return Err( - "--dead-repo-blacklist-fail-threshold requires --dead-repo-blacklist".to_string(), - ); - } - Ok(args) -} - -fn value_at<'a>(argv: &'a [String], index: usize, flag: &str) -> Result<&'a str, String> { - argv.get(index) - .map(String::as_str) - .ok_or_else(|| format!("{flag} requires a value")) -} - -fn parse_u64(raw: &str, flag: &str) -> Result { - raw.parse::() - .map_err(|_| format!("invalid {flag}: {raw}")) -} - -fn parse_usize(raw: &str, flag: &str) -> Result { - raw.parse::() - .map_err(|_| format!("invalid {flag}: {raw}")) -} - -fn status_path(args: &Args) -> PathBuf { - args.status_json - .clone() - .unwrap_or_else(|| args.state_root.join("daemon-status.json")) -} - -fn summary_jsonl_path(args: &Args) -> PathBuf { - args.summary_jsonl - .clone() - .unwrap_or_else(|| args.state_root.join("daemon-runs.jsonl")) -} - -fn utc_now() -> time::OffsetDateTime { - time::OffsetDateTime::now_utc().to_offset(time::UtcOffset::UTC) -} - -fn format_rfc3339(t: time::OffsetDateTime) -> Result { - t.format(&time::format_description::well_known::Rfc3339) - .map_err(|e| format!("format RFC3339 failed: {e}")) -} - -fn format_compact_utc(t: time::OffsetDateTime) -> String { - format!( - "{:04}{:02}{:02}T{:02}{:02}{:02}Z", - t.year(), - u8::from(t.month()), - t.day(), - t.hour(), - t.minute(), - t.second() - ) -} - -fn write_json_pretty(path: &Path, value: &T) -> Result<(), String> { - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .map_err(|e| format!("create parent dir failed: {}: {e}", parent.display()))?; - } - let bytes = serde_json::to_vec_pretty(value) - .map_err(|e| format!("serialize json failed: {}: {e}", path.display()))?; - // Atomic write (tmp + rename) so concurrent readers never see a torn file. - let mut tmp_name = path - .file_name() - .map(|name| name.to_os_string()) - .unwrap_or_default(); - tmp_name.push(".tmp"); - let tmp_path = path.with_file_name(tmp_name); - fs::write(&tmp_path, &bytes) - .map_err(|e| format!("write json tmp failed: {}: {e}", tmp_path.display()))?; - fs::rename(&tmp_path, path).map_err(|e| { - format!( - "rename json tmp failed: {} -> {}: {e}", - tmp_path.display(), - path.display() - ) - }) -} - -fn append_json_line(path: &Path, value: &T) -> Result<(), String> { - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .map_err(|e| format!("create parent dir failed: {}: {e}", parent.display()))?; - } - let mut file = OpenOptions::new() - .create(true) - .append(true) - .open(path) - .map_err(|e| format!("open jsonl failed: {}: {e}", path.display()))?; - serde_json::to_writer(&mut file, value) - .map_err(|e| format!("write jsonl failed: {}: {e}", path.display()))?; - file.write_all(b"\n") - .map_err(|e| format!("flush jsonl failed: {}: {e}", path.display())) -} - -fn dead_repo_blacklist_status( - args: &Args, - health: &DeadRepoHealthRuntime, -) -> Option { - let path = args.dead_repo_blacklist.as_ref()?; - let (blacklist, _) = DeadRepoBlacklist::load(path); - let last_health_check_at_rfc3339_utc = health - .last_health_check_at - .and_then(|t| format_rfc3339(t).ok()); - Some(DeadRepoBlacklistStatus { - path: path_string(path), - entries: blacklist.len(), - blacklisted: blacklist.blacklisted_len(), - last_health_check_at_rfc3339_utc, - }) -} - -fn write_status( - args: &Args, - state: DaemonState, - runs_completed: u64, - current: Option<&RunContext>, - last_run_id: Option, - health: &DeadRepoHealthRuntime, -) -> Result<(), String> { - let updated_at_rfc3339_utc = format_rfc3339(utc_now())?; - let status = DaemonStatus { - state, - updated_at_rfc3339_utc, - runs_completed, - max_runs: args.max_runs, - current_run_seq: current.map(|ctx| ctx.seq), - current_run_id: current.map(|ctx| ctx.run_id.clone()), - last_run_id, - dead_repo_blacklist: dead_repo_blacklist_status(args, health), - }; - write_json_pretty(&status_path(args), &status) -} - -/// Short-timeout HTTP GET of the RRDP notification file. Any HTTP status -/// (even 4xx) proves the transport is alive; only transport errors keep the -/// entry blacklisted. -fn probe_rrdp_transport(uri: &str) -> bool { - let config = crate::fetch::http::HttpFetcherConfig { - connect_timeout: Duration::from_secs(3), - timeout: Duration::from_secs(6), - large_body_timeout: Duration::from_secs(6), - ..Default::default() - }; - let fetcher = match crate::fetch::http::BlockingHttpFetcher::new(config) { - Ok(fetcher) => fetcher, - Err(_) => return false, - }; - match fetcher.fetch_bytes(uri) { - Ok(_) => true, - Err(err) => err.starts_with("http status"), - } -} - -/// rsync list-only probe: connecting to the daemon and listing the module -/// root is enough to prove the transport is alive. -fn probe_rsync_transport(rsync_bin: &Path, base_uri: &str) -> bool { - Command::new(rsync_bin) - .arg("--contimeout=5") - .arg("--timeout=8") - .arg(base_uri) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .map(|status| status.success()) - .unwrap_or(false) -} - -/// Probe due blacklist entries between runs (#141). Runs in the supervisor -/// loop while no child is active, preserving the single-writer assumption. -fn maybe_run_dead_repo_health_check(args: &Args, health: &mut DeadRepoHealthRuntime) { - let Some(path) = args.dead_repo_blacklist.clone() else { - return; - }; - let now = utc_now(); - if let Some(last) = health.last_health_check_at { - let elapsed = (now - last).whole_seconds(); - if elapsed >= 0 && (elapsed as u64) < args.dead_repo_health_check_interval_secs { - return; - } - } - let (mut blacklist, warning) = DeadRepoBlacklist::load(&path); - if let Some(warning) = warning { - eprintln!("[dead-repo-health] {warning}"); - } - let now_unix = now.unix_timestamp().max(0) as u64; - let due = blacklist.probe_due_entries(now_unix, args.dead_repo_health_check_interval_secs); - if due.is_empty() { - health.last_health_check_at = Some(now); - return; - } - // Probe in parallel: in the docker soak the daemon runs once per run - // (--max-runs 1), so a serial sweep of ~12 dead entries would add up to a - // minute of latency to every run. - let rsync_bin = args.dead_repo_probe_rsync_bin.clone(); - let probe_results: Vec<(RepoTransportMode, String, bool)> = due - .iter() - .map(|entry| (entry.transport, entry.uri.clone())) - .collect::>() - .into_iter() - .map(|(transport, uri)| { - let rsync_bin = rsync_bin.clone(); - std::thread::spawn(move || { - let alive = match transport { - RepoTransportMode::Rrdp => probe_rrdp_transport(&uri), - RepoTransportMode::Rsync => probe_rsync_transport(&rsync_bin, &uri), - }; - (transport, uri, alive) - }) - }) - .filter_map(|handle| handle.join().ok()) - .collect(); - let mut removed = 0usize; - let mut still_dead = 0usize; - for (transport, uri, alive) in &probe_results { - if *alive { - blacklist.record_probe_success(*transport, uri); - removed += 1; - eprintln!( - "[dead-repo-health] removed {} {} (probe ok)", - transport.as_str(), - uri - ); - crate::progress_log::emit( - "dead_repo_blacklist_remove", - serde_json::json!({ - "transport": transport.as_str(), - "uri": uri, - "reason": "health_probe_ok", - }), - ); - } else { - blacklist.record_probe_failure(*transport, uri, now_unix); - still_dead += 1; - eprintln!( - "[dead-repo-health] still dead {} {} (probe failed)", - transport.as_str(), - uri - ); - } - } - if let Err(err) = blacklist.store_atomic(&path, now_unix) { - eprintln!( - "[dead-repo-health] persist blacklist failed: {}: {err}", - path.display() - ); - } - crate::progress_log::emit( - "dead_repo_health_check", - serde_json::json!({ - "probed": probe_results.len(), - "removed": removed, - "still_dead": still_dead, - }), - ); - health.last_health_check_at = Some(now); -} - -fn render_child_args(args: &[String], daemon_args: &Args, ctx: &RunContext) -> Vec { - args.iter() - .map(|arg| { - arg.replace("{state_root}", &path_string(&daemon_args.state_root)) - .replace("{run_out}", &path_string(&ctx.run_dir)) - .replace("{run_id}", &ctx.run_id) - .replace("{run_seq}", &ctx.seq.to_string()) - }) - .collect() -} - -fn render_path_template(path: &Path, daemon_args: &Args, ctx: &RunContext) -> PathBuf { - PathBuf::from( - path_string(path) - .replace("{state_root}", &path_string(&daemon_args.state_root)) - .replace("{run_out}", &path_string(&ctx.run_dir)) - .replace("{run_id}", &ctx.run_id) - .replace("{run_seq}", &ctx.seq.to_string()), - ) -} - -fn path_string(path: &Path) -> String { - path.to_string_lossy().into_owned() -} - -fn make_run_context(args: &Args, seq: u64, now: time::OffsetDateTime) -> RunContext { - let run_id = format!("{seq:06}-{}", format_compact_utc(now)); - let run_dir = args.state_root.join("runs").join(&run_id); - RunContext { - seq, - run_id, - run_dir, - } -} - -fn collect_artifacts(run_dir: &Path) -> Result, String> { - let mut artifacts = Vec::new(); - for entry in fs::read_dir(run_dir) - .map_err(|e| format!("read run dir failed: {}: {e}", run_dir.display()))? - { - let entry = entry.map_err(|e| format!("read run dir entry failed: {e}"))?; - if !entry - .file_type() - .map_err(|e| format!("read file type failed: {}: {e}", entry.path().display()))? - .is_file() - { - continue; - } - let metadata = entry - .metadata() - .map_err(|e| format!("read metadata failed: {}: {e}", entry.path().display()))?; - artifacts.push(ArtifactInfo { - path: path_string(&entry.path()), - size_bytes: metadata.len(), - }); - } - artifacts.sort_by(|a, b| a.path.cmp(&b.path)); - Ok(artifacts) -} - -fn collect_process_metrics(time_wrapper_used: bool, time_output_path: &Path) -> ProcessMetrics { - if !time_wrapper_used { - return ProcessMetrics { - time_wrapper_used, - time_output_path: None, - user_seconds: None, - system_seconds: None, - cpu_percent: None, - elapsed_raw: None, - max_rss_kb: None, - exit_status_from_time: None, - parse_error: None, - }; - } - - let mut metrics = ProcessMetrics { - time_wrapper_used, - time_output_path: Some(path_string(time_output_path)), - user_seconds: None, - system_seconds: None, - cpu_percent: None, - elapsed_raw: None, - max_rss_kb: None, - exit_status_from_time: None, - parse_error: None, - }; - - let text = match fs::read_to_string(time_output_path) { - Ok(text) => text, - Err(err) => { - metrics.parse_error = Some(format!( - "read process time output failed: {}: {err}", - time_output_path.display() - )); - return metrics; - } - }; - - for line in text.lines() { - let line = line.trim(); - if let Some(value) = line.strip_prefix("User time (seconds):") { - metrics.user_seconds = value.trim().parse::().ok(); - } else if let Some(value) = line.strip_prefix("System time (seconds):") { - metrics.system_seconds = value.trim().parse::().ok(); - } else if let Some(value) = line.strip_prefix("Percent of CPU this job got:") { - metrics.cpu_percent = value.trim().trim_end_matches('%').parse::().ok(); - } else if let Some(value) = - line.strip_prefix("Elapsed (wall clock) time (h:mm:ss or m:ss):") - { - metrics.elapsed_raw = Some(value.trim().to_string()); - } else if let Some(value) = line.strip_prefix("Maximum resident set size (kbytes):") { - metrics.max_rss_kb = value.trim().parse::().ok(); - } else if let Some(value) = line.strip_prefix("Exit status:") { - metrics.exit_status_from_time = value.trim().parse::().ok(); - } - } - metrics -} - -fn run_child_once(args: &Args, ctx: &RunContext) -> Result { - fs::create_dir_all(&ctx.run_dir) - .map_err(|e| format!("create run dir failed: {}: {e}", ctx.run_dir.display()))?; - - let started_at = utc_now(); - let started_at_rfc3339_utc = format_rfc3339(started_at)?; - let stdout_path = ctx.run_dir.join("stdout.log"); - let stderr_path = ctx.run_dir.join("stderr.log"); - let stdout = File::create(&stdout_path) - .map_err(|e| format!("create stdout log failed: {}: {e}", stdout_path.display()))?; - let stderr = File::create(&stderr_path) - .map_err(|e| format!("create stderr log failed: {}: {e}", stderr_path.display()))?; - let child_args = render_child_args(&args.child_args, args, ctx); - let time_output_path = ctx.run_dir.join("process-time.txt"); - - let mut command = if let Some(time_bin) = args.time_bin.as_ref() { - let mut command = Command::new(time_bin); - command - .arg("-v") - .arg("-o") - .arg(&time_output_path) - .arg("--") - .arg(&args.rpki_bin) - .args(&child_args); - command - } else { - let mut command = Command::new(&args.rpki_bin); - command.args(&child_args); - command - }; - command - .stdout(Stdio::from(stdout)) - .stderr(Stdio::from(stderr)); - - let (status, exit_code, exit_status, error) = match command.status() { - Ok(status) if status.success() => ( - RunStatus::Success, - status.code(), - Some(status.to_string()), - None, - ), - Ok(status) => ( - RunStatus::Failed, - status.code(), - Some(status.to_string()), - None, - ), - Err(err) => ( - RunStatus::SpawnFailed, - None, - None, - Some(format!("spawn child failed: {err}")), - ), - }; - - let finished_at = utc_now(); - let finished_at_rfc3339_utc = format_rfc3339(finished_at)?; - let wall_ms = (finished_at - started_at).whole_milliseconds().max(0) as u64; - let process_metrics = Some(collect_process_metrics( - args.time_bin.is_some(), - &time_output_path, - )); - let artifacts = collect_artifacts(&ctx.run_dir)?; - let summary = RunSummary { - run_seq: ctx.seq, - run_id: ctx.run_id.clone(), - run_dir: path_string(&ctx.run_dir), - started_at_rfc3339_utc, - finished_at_rfc3339_utc, - wall_ms, - status, - exit_code, - exit_status, - error, - rpki_bin: path_string(&args.rpki_bin), - child_args, - stdout_path: path_string(&stdout_path), - stderr_path: path_string(&stderr_path), - process_metrics, - stage_timing: None, - report_counts: None, - repo_sync_stats: None, - path_stats: Vec::new(), - db_stats: Vec::new(), - retention_deleted_runs: Vec::new(), - artifacts, - dead_repo_blacklist: None, - }; - Ok(summary) -} - -fn apply_retention(runs_root: &Path, retain_runs: usize) -> Result, String> { - if !runs_root.exists() { - return Ok(Vec::new()); - } - let mut dirs = Vec::new(); - for entry in fs::read_dir(runs_root) - .map_err(|e| format!("read runs dir failed: {}: {e}", runs_root.display()))? - { - let entry = entry.map_err(|e| format!("read runs dir entry failed: {e}"))?; - if entry - .file_type() - .map_err(|e| format!("read file type failed: {}: {e}", entry.path().display()))? - .is_dir() - { - dirs.push(entry.path()); - } - } - dirs.sort(); - let remove_count = dirs.len().saturating_sub(retain_runs); - let mut removed = Vec::new(); - for dir in dirs.into_iter().take(remove_count) { - fs::remove_dir_all(&dir) - .map_err(|e| format!("remove old run dir failed: {}: {e}", dir.display()))?; - removed.push(dir); - } - Ok(removed) -} - -fn find_named_file(root: &Path, name: &str) -> Option { - let mut stack = vec![root.to_path_buf()]; - while let Some(dir) = stack.pop() { - let entries = fs::read_dir(&dir).ok()?; - for entry in entries.flatten() { - let path = entry.path(); - let file_type = entry.file_type().ok()?; - if file_type.is_file() && entry.file_name().to_string_lossy() == name { - return Some(path); - } - if file_type.is_dir() { - stack.push(path); - } - } - } - None -} - -fn read_json_value_if_exists(path: &Path) -> Option { - let bytes = fs::read(path).ok()?; - serde_json::from_slice(&bytes).ok() -} - -fn json_array_len(value: &serde_json::Value, key: &str) -> usize { - value - .get(key) - .and_then(serde_json::Value::as_array) - .map(Vec::len) - .unwrap_or(0) -} - -fn parse_stdout_summary(run_dir: &Path) -> Option { - let stdout_path = run_dir.join("stdout.log"); - let text = fs::read_to_string(stdout_path).ok()?; - let mut vrps = None; - let mut aspas = None; - let mut publication_points = None; - let mut rrdp_repos_unique = None; - let mut tree_instances_processed = None; - let mut tree_instances_failed = None; - let mut warnings = None; - - for line in text.lines() { - if let Some(value) = line.strip_prefix("vrps=") { - vrps = value.trim().parse::().ok(); - } else if let Some(value) = line.strip_prefix("aspas=") { - aspas = value.trim().parse::().ok(); - } else if let Some(value) = line.strip_prefix("audit_publication_points=") { - publication_points = value.trim().parse::().ok(); - } else if let Some(value) = line.strip_prefix("rrdp_repos_unique=") { - rrdp_repos_unique = value.trim().parse::().ok(); - } else if let Some(value) = line.strip_prefix("warnings_total=") { - warnings = value.trim().parse::().ok(); - } else if let Some(rest) = line.strip_prefix("publication_points_processed=") { - for token in rest.split_whitespace() { - if let Some(value) = token.strip_prefix("publication_points_failed=") { - tree_instances_failed = value.parse::().ok(); - } else if tree_instances_processed.is_none() { - tree_instances_processed = token.parse::().ok(); - } - } - } - } - - Some(ReportCounts { - vrps: vrps?, - aspas: aspas?, - publication_points: publication_points?, - rrdp_repos_unique, - tree_instances_processed, - tree_instances_failed, - warnings: warnings.unwrap_or(0), - }) -} - -fn parse_report_counts_fallback(report: &serde_json::Value) -> ReportCounts { - let tree = report.get("tree"); - let tree_warnings = tree - .and_then(|tree| tree.get("warnings")) - .and_then(serde_json::Value::as_array) - .map(Vec::len) - .unwrap_or(0); - let pp_warnings = report - .get("publication_points") - .and_then(serde_json::Value::as_array) - .map(|items| { - items - .iter() - .map(|pp| { - pp.get("warnings") - .and_then(serde_json::Value::as_array) - .map(Vec::len) - .unwrap_or(0) - }) - .sum() - }) - .unwrap_or(0); - ReportCounts { - vrps: json_array_len(report, "vrps"), - aspas: json_array_len(report, "aspas"), - publication_points: json_array_len(report, "publication_points"), - rrdp_repos_unique: None, - tree_instances_processed: tree - .and_then(|tree| tree.get("instances_processed")) - .and_then(serde_json::Value::as_u64), - tree_instances_failed: tree - .and_then(|tree| tree.get("instances_failed")) - .and_then(serde_json::Value::as_u64), - warnings: tree_warnings + pp_warnings, - } -} - -fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option { - haystack - .windows(needle.len()) - .position(|window| window == needle) -} - -fn extract_json_object_field(path: &Path, field_name: &str) -> Option { - let bytes = fs::read(path).ok()?; - let needle = format!("\"{field_name}\":"); - let pos = find_subslice(&bytes, needle.as_bytes())?; - let mut i = pos + needle.len(); - while i < bytes.len() && bytes[i].is_ascii_whitespace() { - i += 1; - } - if bytes.get(i).copied()? != b'{' { - return None; - } - let start = i; - let mut depth = 0u32; - let mut in_string = false; - let mut escaped = false; - for (offset, &b) in bytes[start..].iter().enumerate() { - if in_string { - if escaped { - escaped = false; - } else if b == b'\\' { - escaped = true; - } else if b == b'"' { - in_string = false; - } - continue; - } - match b { - b'"' => in_string = true, - b'{' => depth = depth.saturating_add(1), - b'}' => { - depth = depth.saturating_sub(1); - if depth == 0 { - let end = start + offset + 1; - return serde_json::from_slice(&bytes[start..end]).ok(); - } - } - _ => {} - } - } - None -} - -fn parse_report_metadata(run_dir: &Path) -> (Option, Option) { - let Some(report_path) = find_named_file(run_dir, "report.json") else { - return (parse_stdout_summary(run_dir), None); - }; - let counts = parse_stdout_summary(run_dir).or_else(|| { - read_json_value_if_exists(&report_path).map(|report| parse_report_counts_fallback(&report)) - }); - let repo_sync_stats = extract_json_object_field(&report_path, "repo_sync_stats"); - (counts, repo_sync_stats) -} - -fn collect_path_file_stats(label: &str, path: &Path) -> PathFileStats { - let mut stats = PathFileStats { - label: label.to_string(), - path: path_string(path), - exists: path.exists(), - is_dir: path.is_dir(), - total_size_bytes: 0, - file_count: 0, - dir_count: 0, - }; - if !stats.exists { - return stats; - } - if path.is_file() { - if let Ok(metadata) = path.metadata() { - stats.total_size_bytes = metadata.len(); - stats.file_count = 1; - } - return stats; - } - - let mut stack = vec![path.to_path_buf()]; - while let Some(dir) = stack.pop() { - let Ok(entries) = fs::read_dir(&dir) else { - continue; - }; - for entry in entries.flatten() { - let Ok(file_type) = entry.file_type() else { - continue; - }; - if file_type.is_dir() { - stats.dir_count = stats.dir_count.saturating_add(1); - stack.push(entry.path()); - } else if file_type.is_file() { - stats.file_count = stats.file_count.saturating_add(1); - if let Ok(metadata) = entry.metadata() { - stats.total_size_bytes = stats.total_size_bytes.saturating_add(metadata.len()); - } - } - } - } - stats -} - -fn collect_state_path_stats(args: &Args, ctx: &RunContext) -> Vec { - let mut stats = Vec::new(); - stats.push(collect_path_file_stats( - "work_db", - &render_path_template(&args.work_db, args, ctx), - )); - if let Some(path) = args.repo_bytes_db.as_ref() { - stats.push(collect_path_file_stats( - "repo_bytes_db", - &render_path_template(path, args, ctx), - )); - } - if let Some(path) = args.raw_store_db.as_ref() { - stats.push(collect_path_file_stats( - "raw_store_db", - &render_path_template(path, args, ctx), - )); - } - stats -} - -fn parse_key_value_metrics(text: &str) -> BTreeMap { - let mut metrics = BTreeMap::new(); - for line in text.lines() { - let Some((key, value)) = line.split_once('=') else { - continue; - }; - metrics.insert(key.trim().to_string(), value.trim().to_string()); - } - metrics -} - -fn run_db_stats_command( - db_stats_bin: &Path, - db_path: &Path, - run_dir: &Path, - mode: &str, -) -> DbStatsSummary { - let output_path = run_dir.join(format!("db-stats-{mode}.txt")); - let stderr_path = run_dir.join(format!("db-stats-{mode}.stderr.txt")); - let mut summary = DbStatsSummary { - mode: mode.to_string(), - db_path: path_string(db_path), - output_path: Some(path_string(&output_path)), - stderr_path: None, - status: "success".to_string(), - exit_code: None, - error: None, - metrics: BTreeMap::new(), - }; - - if !db_path.exists() { - summary.status = "skipped".to_string(); - summary.error = Some(format!("db path does not exist: {}", db_path.display())); - summary.output_path = None; - return summary; - } - - let mut command = Command::new(db_stats_bin); - command.arg("--db").arg(db_path); - if mode == "exact" { - command.arg("--exact"); - } - match command.output() { - Ok(output) => { - summary.exit_code = output.status.code(); - if !output.status.success() { - summary.status = "failed".to_string(); - } - let stdout_text = String::from_utf8_lossy(&output.stdout).into_owned(); - if let Err(err) = fs::write(&output_path, stdout_text.as_bytes()) { - summary.status = "failed".to_string(); - summary.error = Some(format!( - "write db_stats output failed: {}: {err}", - output_path.display() - )); - } - summary.metrics = parse_key_value_metrics(&stdout_text); - if !output.stderr.is_empty() { - if fs::write(&stderr_path, &output.stderr).is_ok() { - summary.stderr_path = Some(path_string(&stderr_path)); - } - } - if !output.status.success() && summary.error.is_none() { - summary.error = Some(String::from_utf8_lossy(&output.stderr).into_owned()); - } - } - Err(err) => { - summary.status = "spawn_failed".to_string(); - summary.exit_code = None; - summary.output_path = None; - summary.error = Some(format!("spawn db_stats failed: {err}")); - } - } - summary -} - -fn collect_db_stats(args: &Args, ctx: &RunContext) -> Vec { - let work_db = render_path_template(&args.work_db, args, ctx); - let Some(db_stats_bin) = args - .db_stats_bin - .as_ref() - .cloned() - .or_else(default_db_stats_bin) - else { - return vec![DbStatsSummary { - mode: "estimate".to_string(), - db_path: path_string(&work_db), - output_path: None, - stderr_path: None, - status: "skipped".to_string(), - exit_code: None, - error: Some( - "db_stats binary not configured and sibling db_stats was not found".to_string(), - ), - metrics: BTreeMap::new(), - }]; - }; - - let mut stats = Vec::new(); - stats.push(run_db_stats_command( - &db_stats_bin, - &work_db, - &ctx.run_dir, - "estimate", - )); - if args - .db_stats_exact_every - .is_some_and(|every| ctx.seq % every == 0) - { - stats.push(run_db_stats_command( - &db_stats_bin, - &work_db, - &ctx.run_dir, - "exact", - )); - } - stats -} - -fn collect_post_run_metrics(args: &Args, ctx: &RunContext, summary: &mut RunSummary) { - if let Some(path) = find_named_file(&ctx.run_dir, "stage-timing.json") { - summary.stage_timing = read_json_value_if_exists(&path); - } - let (report_counts, repo_sync_stats) = parse_report_metadata(&ctx.run_dir); - summary.report_counts = report_counts; - summary.repo_sync_stats = repo_sync_stats; - summary.path_stats = collect_state_path_stats(args, ctx); - summary.db_stats = collect_db_stats(args, ctx); - summary.artifacts = collect_artifacts(&ctx.run_dir).unwrap_or_default(); - if let Some(path) = args.dead_repo_blacklist.as_ref() { - let (blacklist, _) = DeadRepoBlacklist::load(path); - summary.dead_repo_blacklist = Some(DeadRepoBlacklistRunSummary { - path: path_string(path), - entries: blacklist.len(), - blacklisted: blacklist.blacklisted_len(), - }); - } -} - -fn run_daemon(args: &Args) -> Result<(), String> { - fs::create_dir_all(args.state_root.join("state")).map_err(|e| { - format!( - "create daemon state dir failed: {}: {e}", - args.state_root.join("state").display() - ) - })?; - fs::create_dir_all(args.state_root.join("runs")).map_err(|e| { - format!( - "create daemon runs dir failed: {}: {e}", - args.state_root.join("runs").display() - ) - })?; - - let mut runs_completed = 0u64; - let mut next_seq = 1u64; - let mut last_run_id = None; - let mut health = DeadRepoHealthRuntime::default(); - write_status( - args, - DaemonState::Starting, - runs_completed, - None, - last_run_id.clone(), - &health, - )?; - // Probe once at startup so revived repos are removed before the first run. - maybe_run_dead_repo_health_check(args, &mut health); - - loop { - if args.max_runs.is_some_and(|max| runs_completed >= max) { - break; - } - - write_status( - args, - DaemonState::Idle, - runs_completed, - None, - last_run_id.clone(), - &health, - )?; - let ctx = make_run_context(args, next_seq, utc_now()); - write_status( - args, - DaemonState::Running, - runs_completed, - Some(&ctx), - last_run_id.clone(), - &health, - )?; - let mut summary = run_child_once(args, &ctx)?; - write_status( - args, - DaemonState::Collecting, - runs_completed, - Some(&ctx), - last_run_id.clone(), - &health, - )?; - collect_post_run_metrics(args, &ctx, &mut summary); - let removed = apply_retention(&args.state_root.join("runs"), args.retain_runs)?; - summary.retention_deleted_runs = removed.iter().map(|p| path_string(p)).collect(); - summary.artifacts = collect_artifacts(&ctx.run_dir).unwrap_or_default(); - write_json_pretty(&ctx.run_dir.join("run-summary.json"), &summary)?; - append_json_line(&summary_jsonl_path(args), &summary)?; - runs_completed += 1; - next_seq += 1; - last_run_id = Some(ctx.run_id); - - if args.max_runs.is_some_and(|max| runs_completed >= max) { - break; - } - // Health probes run only while no child is active (single-writer). - maybe_run_dead_repo_health_check(args, &mut health); - write_status( - args, - DaemonState::Sleeping, - runs_completed, - None, - last_run_id.clone(), - &health, - )?; - if args.interval_secs > 0 { - std::thread::sleep(Duration::from_secs(args.interval_secs)); - } - } - - write_status( - args, - DaemonState::Exited, - runs_completed, - None, - last_run_id, - &health, - ) -} - -pub fn main_entry() -> i32 { - let argv: Vec = std::env::args().collect(); - match parse_args(&argv) { - Ok(args) => match run_daemon(&args) { - Ok(()) => 0, - Err(err) => { - eprintln!("{err}"); - 2 - } - }, - Err(err) => { - if argv.iter().any(|a| a == "--help" || a == "-h") { - println!("{err}"); - return 0; - } - eprintln!("{err}"); - 2 - } - } -} +include!("daemon/types.rs"); +include!("daemon/args.rs"); +include!("daemon/status.rs"); +include!("daemon/run_child.rs"); +include!("daemon/run_metrics.rs"); +include!("daemon/daemon_run.rs"); #[cfg(test)] -mod tests { - use super::*; - - fn test_args(state_root: PathBuf) -> Args { - Args { - work_db: state_root.join("state/work-db"), - repo_bytes_db: Some(state_root.join("state/repo-bytes.db")), - state_root, - rpki_bin: PathBuf::from("/bin/true"), - interval_secs: 0, - max_runs: Some(1), - retain_runs: 10, - status_json: None, - summary_jsonl: None, - raw_store_db: None, - db_stats_bin: None, - db_stats_exact_every: None, - time_bin: None, - dead_repo_blacklist: None, - dead_repo_blacklist_fail_threshold: None, - dead_repo_health_check_interval_secs: 600, - dead_repo_probe_rsync_bin: PathBuf::from("rsync"), - child_args: vec!["--version".to_string()], - } - } - - #[test] - fn parse_args_accepts_required_flags_and_child_args() { - let argv = vec![ - "rpki_daemon".to_string(), - "--state-root".to_string(), - "/tmp/daemon".to_string(), - "--rpki-bin".to_string(), - "/bin/echo".to_string(), - "--interval-secs".to_string(), - "0".to_string(), - "--max-runs".to_string(), - "2".to_string(), - "--retain-runs".to_string(), - "3".to_string(), - "--".to_string(), - "--db".to_string(), - "{state_root}/state/work-db".to_string(), - "--report-json".to_string(), - "{run_out}/report.json".to_string(), - ]; - - let args = parse_args(&argv).expect("parse args"); - assert_eq!(args.state_root, PathBuf::from("/tmp/daemon")); - assert_eq!(args.rpki_bin, PathBuf::from("/bin/echo")); - assert_eq!(args.interval_secs, 0); - assert_eq!(args.max_runs, Some(2)); - assert_eq!(args.retain_runs, 3); - assert_eq!(args.work_db, PathBuf::from("/tmp/daemon/state/work-db")); - assert_eq!( - args.repo_bytes_db, - Some(PathBuf::from("/tmp/daemon/state/repo-bytes.db")) - ); - assert_eq!( - args.child_args, - vec![ - "--db", - "{state_root}/state/work-db", - "--report-json", - "{run_out}/report.json" - ] - ); - } - - #[test] - fn usage_and_parse_args_cover_optional_flags_and_errors() { - let help = usage(); - assert!(help.contains("--db-stats-exact-every")); - assert!(help.contains("{run_seq}")); - - let argv = vec![ - "rpki_daemon".to_string(), - "--state-root".to_string(), - "/tmp/daemon".to_string(), - "--rpki-bin".to_string(), - "/bin/echo".to_string(), - "--status-json".to_string(), - "/tmp/status.json".to_string(), - "--summary-jsonl".to_string(), - "/tmp/runs.jsonl".to_string(), - "--work-db".to_string(), - "{state_root}/work".to_string(), - "--repo-bytes-db".to_string(), - "{state_root}/repo-bytes".to_string(), - "--raw-store-db".to_string(), - "{state_root}/raw".to_string(), - "--db-stats-bin".to_string(), - "/bin/echo".to_string(), - "--db-stats-exact-every".to_string(), - "2".to_string(), - "--time-bin".to_string(), - "/usr/bin/time".to_string(), - "--no-time-wrapper".to_string(), - "--".to_string(), - "child".to_string(), - ]; - let args = parse_args(&argv).expect("optional args"); - assert_eq!(args.status_json, Some(PathBuf::from("/tmp/status.json"))); - assert_eq!(args.summary_jsonl, Some(PathBuf::from("/tmp/runs.jsonl"))); - assert_eq!(args.work_db, PathBuf::from("{state_root}/work")); - assert_eq!( - args.repo_bytes_db, - Some(PathBuf::from("{state_root}/repo-bytes")) - ); - assert_eq!(args.raw_store_db, Some(PathBuf::from("{state_root}/raw"))); - assert_eq!(args.db_stats_bin, Some(PathBuf::from("/bin/echo"))); - assert_eq!(args.db_stats_exact_every, Some(2)); - assert_eq!(args.time_bin, None); - - for (argv, expected) in [ - (vec!["rpki_daemon", "--help"], "Usage:"), - ( - vec![ - "rpki_daemon", - "--state-root", - "/tmp/x", - "--rpki-bin", - "/bin/echo", - "--max-runs", - "0", - "--", - "child", - ], - "--max-runs must be > 0", - ), - ( - vec![ - "rpki_daemon", - "--state-root", - "/tmp/x", - "--rpki-bin", - "/bin/echo", - "--retain-runs", - "0", - "--", - "child", - ], - "--retain-runs must be > 0", - ), - ( - vec![ - "rpki_daemon", - "--state-root", - "/tmp/x", - "--rpki-bin", - "/bin/echo", - "--db-stats-exact-every", - "0", - "--", - "child", - ], - "--db-stats-exact-every must be > 0", - ), - (vec!["rpki_daemon", "--unknown"], "unknown argument"), - ( - vec!["rpki_daemon", "--state-root"], - "--state-root requires a value", - ), - (vec!["rpki_daemon"], "missing -- before child rpki args"), - ( - vec![ - "rpki_daemon", - "--state-root", - "/tmp/x", - "--rpki-bin", - "/bin/echo", - "--", - ], - "child rpki args are required", - ), - ] { - let owned: Vec = argv.into_iter().map(str::to_string).collect(); - let err = parse_args(&owned).expect_err("parse should fail"); - assert!(err.contains(expected), "{err}"); - } - } - - #[test] - fn render_child_args_replaces_placeholders() { - let args = Args { - state_root: PathBuf::from("/tmp/root"), - rpki_bin: PathBuf::from("/bin/echo"), - interval_secs: 0, - max_runs: Some(1), - retain_runs: 10, - status_json: None, - summary_jsonl: None, - work_db: PathBuf::from("/tmp/root/state/work-db"), - repo_bytes_db: Some(PathBuf::from("/tmp/root/state/repo-bytes.db")), - raw_store_db: None, - db_stats_bin: None, - db_stats_exact_every: None, - time_bin: None, - dead_repo_blacklist: None, - dead_repo_blacklist_fail_threshold: None, - dead_repo_health_check_interval_secs: 600, - dead_repo_probe_rsync_bin: PathBuf::from("rsync"), - child_args: vec![ - "{state_root}/state/work-db".to_string(), - "{run_out}/result.ccr".to_string(), - "{run_id}".to_string(), - "{run_seq}".to_string(), - ], - }; - let ctx = RunContext { - seq: 7, - run_id: "000007-20260428T090000Z".to_string(), - run_dir: PathBuf::from("/tmp/root/runs/000007-20260428T090000Z"), - }; - - assert_eq!( - render_child_args(&args.child_args, &args, &ctx), - vec![ - "/tmp/root/state/work-db", - "/tmp/root/runs/000007-20260428T090000Z/result.ccr", - "000007-20260428T090000Z", - "7", - ] - ); - } - - #[test] - fn path_json_and_report_helpers_cover_fallbacks_and_nested_stats() { - let td = tempfile::tempdir().expect("tempdir"); - let state_root = td.path().join("daemon"); - let mut args = test_args(state_root.clone()); - args.work_db = PathBuf::from("{state_root}/state/work-db"); - args.repo_bytes_db = Some(PathBuf::from("{state_root}/state/repo-bytes.db")); - args.raw_store_db = Some(PathBuf::from("{state_root}/state/raw-store.db")); - - let now = time::Date::from_calendar_date(2026, time::Month::April, 28) - .expect("date") - .with_hms(9, 0, 0) - .expect("time") - .assume_utc(); - let ctx = make_run_context(&args, 42, now); - assert_eq!(ctx.run_id, "000042-20260428T090000Z"); - - let rendered = render_path_template(Path::new("{run_out}/{run_id}/{run_seq}"), &args, &ctx); - assert!(rendered.ends_with("000042-20260428T090000Z/000042-20260428T090000Z/42")); - - let nested_json = td.path().join("nested/out/status.json"); - write_json_pretty(&nested_json, &serde_json::json!({"ok": true})).expect("write json"); - append_json_line( - &td.path().join("nested/out/runs.jsonl"), - &serde_json::json!({"n": 1}), - ) - .expect("append jsonl"); - - fs::create_dir_all(ctx.run_dir.join("subdir")).expect("subdir"); - fs::write(ctx.run_dir.join("a.txt"), "aaa").expect("file"); - fs::write(ctx.run_dir.join("subdir/ignored.txt"), "bbb").expect("nested file"); - let artifacts = collect_artifacts(&ctx.run_dir).expect("artifacts"); - assert_eq!(artifacts.len(), 1); - assert!(artifacts[0].path.ends_with("a.txt")); - - fs::create_dir_all(state_root.join("state/work-db/nested")).expect("work db"); - fs::write(state_root.join("state/work-db/file.sst"), "abc").expect("sst"); - fs::write(state_root.join("state/work-db/nested/inner.sst"), "def").expect("inner"); - fs::write(state_root.join("state/raw-store.db"), "raw").expect("raw file"); - let file_stats = - collect_path_file_stats("raw_store_db", &state_root.join("state/raw-store.db")); - assert!(file_stats.exists); - assert!(!file_stats.is_dir); - assert_eq!(file_stats.file_count, 1); - let missing_stats = collect_path_file_stats("missing", &state_root.join("missing")); - assert!(!missing_stats.exists); - let state_stats = collect_state_path_stats(&args, &ctx); - assert!( - state_stats - .iter() - .any(|s| s.label == "raw_store_db" && s.exists) - ); - assert!( - state_stats - .iter() - .any(|s| s.label == "work_db" && s.file_count == 2) - ); - - let report_path = td.path().join("report.json"); - fs::write( - &report_path, - r#"{"repo_sync_stats": { "nested": {"text": "a\"b"} }, "after": 1}"#, - ) - .expect("report"); - assert_eq!( - extract_json_object_field(&report_path, "repo_sync_stats").expect("repo stats")["nested"] - ["text"], - "a\"b" - ); - fs::write(&report_path, r#"{"repo_sync_stats": []}"#).expect("report"); - assert!(extract_json_object_field(&report_path, "repo_sync_stats").is_none()); - assert!(extract_json_object_field(&report_path, "missing").is_none()); - - let counts = parse_report_counts_fallback(&serde_json::json!({ - "vrps": [{}, {}], - "aspas": [{}], - "publication_points": [{"warnings": [{}, {}]}, {"warnings": [{}]}], - "tree": {"instances_processed": 2, "instances_failed": 1, "warnings": [{}]} - })); - assert_eq!(counts.vrps, 2); - assert_eq!(counts.aspas, 1); - assert_eq!(counts.publication_points, 2); - assert_eq!(counts.warnings, 4); - assert_eq!(counts.tree_instances_processed, Some(2)); - assert_eq!(counts.tree_instances_failed, Some(1)); - } - - #[test] - fn retention_removes_oldest_run_directories() { - let td = tempfile::tempdir().expect("tempdir"); - let runs = td.path().join("runs"); - fs::create_dir_all(&runs).expect("runs dir"); - for name in [ - "000001-20260428T000001Z", - "000002-20260428T000002Z", - "000003-20260428T000003Z", - ] { - fs::create_dir_all(runs.join(name)).expect("run dir"); - } - - let removed = apply_retention(&runs, 2).expect("retention"); - assert_eq!(removed.len(), 1); - assert!(!runs.join("000001-20260428T000001Z").exists()); - assert!(runs.join("000002-20260428T000002Z").exists()); - assert!(runs.join("000003-20260428T000003Z").exists()); - } - - #[test] - fn retention_empty_root_and_parse_metrics_error_paths_are_reported() { - let td = tempfile::tempdir().expect("tempdir"); - let missing_runs = td.path().join("missing-runs"); - assert!( - apply_retention(&missing_runs, 2) - .expect("empty retention") - .is_empty() - ); - - let disabled = collect_process_metrics(false, &td.path().join("missing-time.txt")); - assert!(!disabled.time_wrapper_used); - assert!(disabled.time_output_path.is_none()); - - let missing = collect_process_metrics(true, &td.path().join("missing-time.txt")); - assert!(missing.time_wrapper_used); - assert!( - missing - .parse_error - .expect("parse error") - .contains("read process time output failed") - ); - } - - #[test] - fn process_metrics_parses_gnu_time_elapsed_line() { - let td = tempfile::tempdir().expect("tempdir"); - let path = td.path().join("time.txt"); - fs::write( - &path, - "User time (seconds): 1.25\nSystem time (seconds): 0.50\nPercent of CPU this job got: 175%\nElapsed (wall clock) time (h:mm:ss or m:ss): 0:01.00\nMaximum resident set size (kbytes): 12345\nExit status: 0\n", - ) - .expect("write time"); - - let metrics = collect_process_metrics(true, &path); - assert_eq!(metrics.user_seconds, Some(1.25)); - assert_eq!(metrics.system_seconds, Some(0.50)); - assert_eq!(metrics.cpu_percent, Some(175.0)); - assert_eq!(metrics.elapsed_raw.as_deref(), Some("0:01.00")); - assert_eq!(metrics.max_rss_kb, Some(12345)); - assert_eq!(metrics.exit_status_from_time, Some(0)); - } - - #[test] - #[cfg(unix)] - fn child_and_db_stats_error_paths_are_reported() { - use std::os::unix::fs::PermissionsExt; - - let td = tempfile::tempdir().expect("tempdir"); - let run_dir = td.path().join("run"); - fs::create_dir_all(&run_dir).expect("run dir"); - let missing_db = td.path().join("missing-db"); - let skipped = - run_db_stats_command(Path::new("/bin/echo"), &missing_db, &run_dir, "estimate"); - assert_eq!(skipped.status, "skipped"); - assert!(skipped.output_path.is_none()); - - let db_path = td.path().join("db"); - fs::create_dir_all(&db_path).expect("db"); - let failing_bin = td.path().join("failing_db_stats.sh"); - fs::write( - &failing_bin, - "#!/bin/sh\necho partial=1\necho db-stats failed >&2\nexit 7\n", - ) - .expect("script"); - let mut permissions = fs::metadata(&failing_bin).expect("metadata").permissions(); - permissions.set_mode(0o755); - fs::set_permissions(&failing_bin, permissions).expect("chmod"); - let failed = run_db_stats_command(&failing_bin, &db_path, &run_dir, "exact"); - assert_eq!(failed.status, "failed"); - assert_eq!(failed.exit_code, Some(7)); - assert_eq!(failed.metrics.get("partial").map(String::as_str), Some("1")); - assert!(failed.stderr_path.is_some()); - assert!(failed.error.expect("stderr").contains("db-stats failed")); - - let spawn_failed = run_db_stats_command( - Path::new("/definitely/not/db_stats"), - &db_path, - &run_dir, - "estimate", - ); - assert_eq!(spawn_failed.status, "spawn_failed"); - assert!( - spawn_failed - .error - .expect("spawn err") - .contains("spawn db_stats failed") - ); - - let metrics = parse_key_value_metrics("alpha=1\nnot-a-pair\nbeta = two\n"); - assert_eq!(metrics.get("alpha").map(String::as_str), Some("1")); - assert_eq!(metrics.get("beta").map(String::as_str), Some("two")); - - let mut args = test_args(td.path().join("daemon")); - args.rpki_bin = PathBuf::from("/bin/sh"); - args.time_bin = Some(PathBuf::from("/usr/bin/time")); - args.child_args = vec![ - "-c".to_string(), - "echo child-out; echo child-err >&2; exit 7".to_string(), - ]; - let ctx = RunContext { - seq: 1, - run_id: "000001-20260428T000000Z".to_string(), - run_dir: args.state_root.join("runs/000001-20260428T000000Z"), - }; - let summary = run_child_once(&args, &ctx).expect("run failing child"); - assert_eq!(summary.status, RunStatus::Failed); - assert_eq!(summary.exit_code, Some(7)); - let process_metrics = summary.process_metrics.expect("process metrics"); - assert!(process_metrics.time_wrapper_used); - assert_eq!(process_metrics.exit_status_from_time, Some(7)); - - let mut spawn_args = test_args(td.path().join("spawn")); - spawn_args.rpki_bin = PathBuf::from("/definitely/not/rpki"); - spawn_args.child_args = vec!["--help".to_string()]; - let spawn_ctx = RunContext { - seq: 1, - run_id: "000001-20260428T000001Z".to_string(), - run_dir: spawn_args.state_root.join("runs/000001-20260428T000001Z"), - }; - let spawn_summary = run_child_once(&spawn_args, &spawn_ctx).expect("spawn summary"); - assert_eq!(spawn_summary.status, RunStatus::SpawnFailed); - assert!( - spawn_summary - .error - .expect("spawn error") - .contains("spawn child failed") - ); - } - - #[test] - fn report_metadata_prefers_stdout_summary_and_extracts_repo_sync_stats() { - let td = tempfile::tempdir().expect("tempdir"); - fs::write( - td.path().join("stdout.log"), - "RPKI stage2 serial run summary\npublication_points_processed=7 publication_points_failed=1\nrrdp_repos_unique=3\nvrps=11\naspas=2\naudit_publication_points=7\nwarnings_total=5\n", - ) - .expect("stdout"); - fs::write( - td.path().join("report.json"), - "{\"large\":[{\"ignored\":\"{}\"}],\"repo_sync_stats\":{\"by_phase\":{\"rrdp_ok\":{\"count\":7}},\"by_terminal_state\":{},\"publication_points_total\":7}}", - ) - .expect("report"); - - let (counts, repo_sync_stats) = parse_report_metadata(td.path()); - let counts = counts.expect("counts"); - assert_eq!(counts.vrps, 11); - assert_eq!(counts.aspas, 2); - assert_eq!(counts.publication_points, 7); - assert_eq!(counts.rrdp_repos_unique, Some(3)); - assert_eq!(counts.tree_instances_processed, Some(7)); - assert_eq!(counts.tree_instances_failed, Some(1)); - assert_eq!(counts.warnings, 5); - assert_eq!( - repo_sync_stats.expect("repo sync")["by_phase"]["rrdp_ok"]["count"].as_u64(), - Some(7) - ); - } - - #[test] - fn daemon_exits_immediately_when_max_runs_already_reached() { - let td = tempfile::tempdir().expect("tempdir"); - let mut args = test_args(td.path().join("daemon")); - args.max_runs = Some(0); - - run_daemon(&args).expect("run daemon"); - - let status_text = - fs::read_to_string(args.state_root.join("daemon-status.json")).expect("status"); - assert!(status_text.contains("\"state\": \"exited\"")); - assert!(status_text.contains("\"runsCompleted\": 0")); - assert!(args.state_root.join("runs").exists()); - } - - #[test] - fn daemon_runs_fake_child_twice_and_writes_summaries() { - let td = tempfile::tempdir().expect("tempdir"); - let args = Args { - state_root: td.path().join("daemon"), - rpki_bin: PathBuf::from("/bin/sh"), - interval_secs: 0, - max_runs: Some(2), - retain_runs: 10, - status_json: None, - summary_jsonl: None, - work_db: td.path().join("daemon/state/work-db"), - repo_bytes_db: Some(td.path().join("daemon/state/repo-bytes.db")), - raw_store_db: None, - db_stats_bin: None, - db_stats_exact_every: None, - time_bin: None, - dead_repo_blacklist: None, - dead_repo_blacklist_fail_threshold: None, - dead_repo_health_check_interval_secs: 600, - dead_repo_probe_rsync_bin: PathBuf::from("rsync"), - child_args: vec![ - "-c".to_string(), - "echo stdout-{run_seq}; echo stderr-{run_seq} >&2; echo marker > {run_out}/marker.txt".to_string(), - ], - }; - - run_daemon(&args).expect("run daemon"); - - let status_text = - fs::read_to_string(args.state_root.join("daemon-status.json")).expect("status"); - assert!(status_text.contains("\"state\": \"exited\"")); - assert!(status_text.contains("\"runsCompleted\": 2")); - - let jsonl = - fs::read_to_string(args.state_root.join("daemon-runs.jsonl")).expect("summary jsonl"); - assert_eq!(jsonl.lines().count(), 2); - - let run_dirs = fs::read_dir(args.state_root.join("runs")) - .expect("runs dir") - .collect::, _>>() - .expect("entries"); - assert_eq!(run_dirs.len(), 2); - for entry in run_dirs { - assert!(entry.path().join("run-summary.json").exists()); - assert!(entry.path().join("marker.txt").exists()); - } - } - - #[test] - fn daemon_sleep_and_retention_deletion_are_recorded() { - let td = tempfile::tempdir().expect("tempdir"); - let mut args = test_args(td.path().join("daemon")); - args.rpki_bin = PathBuf::from("/bin/sh"); - args.interval_secs = 1; - args.max_runs = Some(2); - args.retain_runs = 1; - args.child_args = vec![ - "-c".to_string(), - "echo run-{run_seq}; printf marker > {run_out}/marker.txt".to_string(), - ]; - - run_daemon(&args).expect("run daemon"); - - let jsonl = fs::read_to_string(args.state_root.join("daemon-runs.jsonl")).expect("jsonl"); - assert_eq!(jsonl.lines().count(), 2); - let last: serde_json::Value = - serde_json::from_str(jsonl.lines().last().expect("last line")).expect("summary"); - assert_eq!( - last["retentionDeletedRuns"] - .as_array() - .expect("deleted") - .len(), - 1 - ); - let run_dirs = fs::read_dir(args.state_root.join("runs")) - .expect("runs") - .collect::, _>>() - .expect("entries"); - assert_eq!(run_dirs.len(), 1); - } - - #[test] - #[cfg(unix)] - fn daemon_collects_stage_report_db_and_file_metrics() { - use std::os::unix::fs::PermissionsExt; - - let td = tempfile::tempdir().expect("tempdir"); - let db_stats_bin = td.path().join("fake_db_stats.sh"); - fs::write( - &db_stats_bin, - "#!/bin/sh\nif [ \"$3\" = \"--exact\" ]; then echo mode=exact; else echo mode=estimate; fi\necho total=42\necho db.files.total_size_bytes=123\n", - ) - .expect("fake db_stats"); - let mut permissions = fs::metadata(&db_stats_bin).expect("metadata").permissions(); - permissions.set_mode(0o755); - fs::set_permissions(&db_stats_bin, permissions).expect("chmod"); - - let args = Args { - state_root: td.path().join("daemon"), - rpki_bin: PathBuf::from("/bin/sh"), - interval_secs: 0, - max_runs: Some(1), - retain_runs: 10, - status_json: None, - summary_jsonl: None, - work_db: PathBuf::from("{state_root}/state/work-db"), - repo_bytes_db: Some(PathBuf::from("{state_root}/state/repo-bytes.db")), - raw_store_db: None, - db_stats_bin: Some(db_stats_bin), - db_stats_exact_every: Some(1), - time_bin: None, - dead_repo_blacklist: None, - dead_repo_blacklist_fail_threshold: None, - dead_repo_health_check_interval_secs: 600, - dead_repo_probe_rsync_bin: PathBuf::from("rsync"), - child_args: vec![ - "-c".to_string(), - "mkdir -p {state_root}/state/work-db {state_root}/state/repo-bytes.db; \ - printf x > {state_root}/state/work-db/000001.sst; \ - printf '{\"validation_ms\":7,\"download_event_count\":2}' > {run_out}/stage-timing.json; \ - printf '{\"tree\":{\"instances_processed\":3,\"instances_failed\":1,\"warnings\":[{}]},\"publication_points\":[{},{}],\"vrps\":[{},{}],\"aspas\":[{}],\"repo_sync_stats\":{\"by_phase\":{\"snapshot\":{\"count\":1}}}}' > {run_out}/report.json" - .to_string(), - ], - }; - - run_daemon(&args).expect("run daemon"); - - let run_summary = find_named_file(&args.state_root.join("runs"), "run-summary.json") - .expect("run summary"); - let summary: serde_json::Value = - serde_json::from_slice(&fs::read(run_summary).expect("read run summary")) - .expect("parse run summary"); - - assert_eq!(summary["stageTiming"]["validation_ms"].as_u64(), Some(7)); - assert_eq!(summary["reportCounts"]["vrps"].as_u64(), Some(2)); - assert_eq!(summary["reportCounts"]["aspas"].as_u64(), Some(1)); - assert_eq!( - summary["reportCounts"]["publicationPoints"].as_u64(), - Some(2) - ); - assert_eq!( - summary["repoSyncStats"]["by_phase"]["snapshot"]["count"].as_u64(), - Some(1) - ); - assert_eq!(summary["dbStats"].as_array().expect("db stats").len(), 2); - assert!( - summary["pathStats"] - .as_array() - .expect("path stats") - .iter() - .any(|item| item["label"] == "work_db" && item["exists"] == true) - ); - } - - #[test] - fn dead_repo_blacklist_flag_is_injected_into_child_args() { - let argv = vec![ - "rpki_daemon".to_string(), - "--state-root".to_string(), - "/tmp/daemon".to_string(), - "--rpki-bin".to_string(), - "/bin/echo".to_string(), - "--dead-repo-blacklist".to_string(), - "/tmp/daemon/state/dead-repo-blacklist.json".to_string(), - "--dead-repo-blacklist-fail-threshold".to_string(), - "2".to_string(), - "--".to_string(), - "--db".to_string(), - "{state_root}/state/work-db".to_string(), - ]; - let args = parse_args(&argv).expect("parse"); - assert_eq!( - args.child_args, - vec![ - "--db".to_string(), - "{state_root}/state/work-db".to_string(), - "--dead-repo-blacklist".to_string(), - "/tmp/daemon/state/dead-repo-blacklist.json".to_string(), - "--dead-repo-blacklist-fail-threshold".to_string(), - "2".to_string(), - ] - ); - } - - #[test] - fn dead_repo_blacklist_injection_respects_existing_child_flags() { - let argv = vec![ - "rpki_daemon".to_string(), - "--state-root".to_string(), - "/tmp/daemon".to_string(), - "--rpki-bin".to_string(), - "/bin/echo".to_string(), - "--dead-repo-blacklist".to_string(), - "/tmp/daemon/state/dead-repo-blacklist.json".to_string(), - "--".to_string(), - "--dead-repo-blacklist".to_string(), - "/custom/path.json".to_string(), - ]; - let args = parse_args(&argv).expect("parse"); - assert_eq!( - args.child_args, - vec![ - "--dead-repo-blacklist".to_string(), - "/custom/path.json".to_string(), - ] - ); - } - - #[test] - fn dead_repo_threshold_without_blacklist_is_rejected() { - let argv = vec![ - "rpki_daemon".to_string(), - "--state-root".to_string(), - "/tmp/daemon".to_string(), - "--rpki-bin".to_string(), - "/bin/echo".to_string(), - "--dead-repo-blacklist-fail-threshold".to_string(), - "2".to_string(), - "--".to_string(), - "--version".to_string(), - ]; - let err = parse_args(&argv).expect_err("parse should fail"); - assert!(err.contains("requires --dead-repo-blacklist"), "{err}"); - } - - #[test] - fn health_check_keeps_dead_entry_and_removes_revived_entry() { - use std::io::{Read as _, Write as _}; - use std::net::TcpListener; - - // Revived RRDP host: one-shot HTTP server answering 200. - let listener = TcpListener::bind("127.0.0.1:0").expect("bind"); - let port = listener.local_addr().expect("local addr").port(); - std::thread::spawn(move || { - if let Ok((mut stream, _)) = listener.accept() { - let mut buf = [0u8; 1024]; - let _ = stream.read(&mut buf); - let body = b""; - let _ = stream.write_all( - format!( - "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - body.len() - ) - .as_bytes(), - ); - let _ = stream.write_all(body); - } - }); - - let td = tempfile::tempdir().expect("tempdir"); - let blacklist_path = td.path().join("dead-repo-blacklist.json"); - let revived_uri = format!("http://127.0.0.1:{port}/notification.xml"); - let dead_uri = "http://127.0.0.1:1/notification.xml".to_string(); - let mut blacklist = DeadRepoBlacklist::new(); - for uri in [&revived_uri, &dead_uri] { - blacklist.record_transport_failure(RepoTransportMode::Rrdp, uri, 100, 1, 256); - } - blacklist - .store_atomic(&blacklist_path, 100) - .expect("store blacklist"); - - let mut args = test_args(td.path().join("daemon")); - args.dead_repo_blacklist = Some(blacklist_path.clone()); - args.dead_repo_health_check_interval_secs = 0; - let mut health = DeadRepoHealthRuntime::default(); - maybe_run_dead_repo_health_check(&args, &mut health); - - let (after, warning) = DeadRepoBlacklist::load(&blacklist_path); - assert!(warning.is_none()); - assert!(!after.is_blacklisted(RepoTransportMode::Rrdp, &revived_uri)); - assert!(after.is_blacklisted(RepoTransportMode::Rrdp, &dead_uri)); - assert!(health.last_health_check_at.is_some()); - - // Status JSON exposes the blacklist section. - let status = dead_repo_blacklist_status(&args, &health).expect("status"); - assert_eq!(status.entries, 1); - assert_eq!(status.blacklisted, 1); - assert!(status.last_health_check_at_rfc3339_utc.is_some()); - } -} +#[path = "daemon/tests.rs"] +mod tests; diff --git a/crates/panda-rpki-validator/src/daemon/args.rs b/crates/panda-rpki-validator/src/daemon/args.rs new file mode 100644 index 0000000..70821e5 --- /dev/null +++ b/crates/panda-rpki-validator/src/daemon/args.rs @@ -0,0 +1,332 @@ +// Daemon command-line parsing and JSON/status helpers. + +fn usage() -> String { + let bin = "rpki_daemon"; + format!( + "\ +Usage: + {bin} --state-root --rpki-bin [options] -- + +Options: + --state-root Persistent daemon root containing state/, runs/, status, and JSONL summary + --rpki-bin rpki child binary to execute for each run + --interval-secs Sleep seconds between runs (default: 60) + --max-runs Stop after n runs (default: run forever) + --retain-runs Keep only the latest n run directories (default: 10) + --status-json Override status JSON path (default: /daemon-status.json) + --summary-jsonl Override summary JSONL path (default: /daemon-runs.jsonl) + --work-db Work DB path for metrics (default: /state/work-db) + --repo-bytes-db Repo bytes DB path for file metrics (default: /state/repo-bytes.db) + --raw-store-db Raw store DB path for file metrics (optional) + --db-stats-bin db_stats binary path (default: sibling db_stats next to this executable when present) + --db-stats-exact-every + Run db_stats --exact every n runs (default: disabled) + --time-bin GNU time binary for child process metrics (default: /usr/bin/time when present) + --no-time-wrapper Disable GNU time wrapper + --dead-repo-blacklist + Enable the dead-repository transport blacklist: daemon probes + blacklisted entries between runs and removes revived repos; + auto-injects the child flag when not already present + --dead-repo-blacklist-fail-threshold + Admission threshold forwarded to the child when its args do not + already set it (default: child default 3) + --dead-repo-health-check-interval-secs + Minimum seconds between health check sweeps (default: 600) + --dead-repo-probe-rsync-bin + rsync binary used for health probes (default: rsync) + --help Show this help + +Child argument placeholders: + {{state_root}} Daemon state root + {{run_out}} Current run output directory + {{run_id}} Current run id, e.g. 000001-20260428T090000Z + {{run_seq}} Current run sequence number +" + ) +} + +fn default_time_bin() -> Option { + let path = PathBuf::from("/usr/bin/time"); + if path.is_file() { Some(path) } else { None } +} + +fn default_db_stats_bin() -> Option { + let mut path = std::env::current_exe().ok()?; + path.set_file_name("db_stats"); + if path.is_file() { Some(path) } else { None } +} + +fn parse_args(argv: &[String]) -> Result { + if argv.iter().any(|arg| arg == "--help" || arg == "-h") { + return Err(usage()); + } + + let mut state_root: Option = None; + let mut rpki_bin: Option = None; + let mut interval_secs = 60u64; + let mut max_runs = None; + let mut retain_runs = 10usize; + let mut status_json: Option = None; + let mut summary_jsonl: Option = None; + let mut work_db: Option = None; + let mut repo_bytes_db: Option = None; + let mut raw_store_db: Option = None; + let mut db_stats_bin: Option = None; + let mut db_stats_exact_every = None; + let mut time_bin = default_time_bin(); + let mut no_time_wrapper = false; + let mut dead_repo_blacklist: Option = None; + let mut dead_repo_blacklist_fail_threshold: Option = None; + let mut dead_repo_health_check_interval_secs = 600u64; + let mut dead_repo_probe_rsync_bin = PathBuf::from("rsync"); + + let mut i = 1usize; + while i < argv.len() { + match argv[i].as_str() { + "--" => { + let mut child_args = argv[i + 1..].to_vec(); + let state_root = + state_root.ok_or_else(|| format!("--state-root is required\n\n{}", usage()))?; + let work_db = work_db.unwrap_or_else(|| state_root.join("state").join("work-db")); + let repo_bytes_db = + repo_bytes_db.or_else(|| Some(state_root.join("state").join("repo-bytes.db"))); + if no_time_wrapper { + time_bin = None; + } + // Forward blacklist enablement to the child process. + // unless its args already carry the flag explicitly. + if let Some(path) = dead_repo_blacklist.as_ref() { + if !child_args.iter().any(|arg| arg == "--dead-repo-blacklist") { + child_args.push("--dead-repo-blacklist".to_string()); + child_args.push(path_string(path)); + } + if let Some(threshold) = dead_repo_blacklist_fail_threshold { + if !child_args + .iter() + .any(|arg| arg == "--dead-repo-blacklist-fail-threshold") + { + child_args.push("--dead-repo-blacklist-fail-threshold".to_string()); + child_args.push(threshold.to_string()); + } + } + } + let args = Args { + state_root, + rpki_bin: rpki_bin + .ok_or_else(|| format!("--rpki-bin is required\n\n{}", usage()))?, + interval_secs, + max_runs, + retain_runs, + status_json, + summary_jsonl, + work_db, + repo_bytes_db, + raw_store_db, + db_stats_bin, + db_stats_exact_every, + time_bin, + dead_repo_blacklist, + dead_repo_blacklist_fail_threshold, + dead_repo_health_check_interval_secs, + dead_repo_probe_rsync_bin, + child_args, + }; + return validate_args(args); + } + "--state-root" => { + i += 1; + state_root = Some(PathBuf::from(value_at(argv, i, "--state-root")?)); + } + "--rpki-bin" => { + i += 1; + rpki_bin = Some(PathBuf::from(value_at(argv, i, "--rpki-bin")?)); + } + "--interval-secs" => { + i += 1; + interval_secs = + parse_u64(value_at(argv, i, "--interval-secs")?, "--interval-secs")?; + } + "--max-runs" => { + i += 1; + let parsed = parse_u64(value_at(argv, i, "--max-runs")?, "--max-runs")?; + if parsed == 0 { + return Err("--max-runs must be > 0".to_string()); + } + max_runs = Some(parsed); + } + "--retain-runs" => { + i += 1; + let parsed = parse_usize(value_at(argv, i, "--retain-runs")?, "--retain-runs")?; + if parsed == 0 { + return Err("--retain-runs must be > 0".to_string()); + } + retain_runs = parsed; + } + "--status-json" => { + i += 1; + status_json = Some(PathBuf::from(value_at(argv, i, "--status-json")?)); + } + "--summary-jsonl" => { + i += 1; + summary_jsonl = Some(PathBuf::from(value_at(argv, i, "--summary-jsonl")?)); + } + "--work-db" => { + i += 1; + work_db = Some(PathBuf::from(value_at(argv, i, "--work-db")?)); + } + "--repo-bytes-db" => { + i += 1; + repo_bytes_db = Some(PathBuf::from(value_at(argv, i, "--repo-bytes-db")?)); + } + "--raw-store-db" => { + i += 1; + raw_store_db = Some(PathBuf::from(value_at(argv, i, "--raw-store-db")?)); + } + "--db-stats-bin" => { + i += 1; + db_stats_bin = Some(PathBuf::from(value_at(argv, i, "--db-stats-bin")?)); + } + "--db-stats-exact-every" => { + i += 1; + let parsed = parse_u64( + value_at(argv, i, "--db-stats-exact-every")?, + "--db-stats-exact-every", + )?; + if parsed == 0 { + return Err("--db-stats-exact-every must be > 0".to_string()); + } + db_stats_exact_every = Some(parsed); + } + "--time-bin" => { + i += 1; + time_bin = Some(PathBuf::from(value_at(argv, i, "--time-bin")?)); + } + "--no-time-wrapper" => { + no_time_wrapper = true; + } + "--dead-repo-blacklist" => { + i += 1; + dead_repo_blacklist = + Some(PathBuf::from(value_at(argv, i, "--dead-repo-blacklist")?)); + } + "--dead-repo-blacklist-fail-threshold" => { + i += 1; + let parsed = parse_u64( + value_at(argv, i, "--dead-repo-blacklist-fail-threshold")?, + "--dead-repo-blacklist-fail-threshold", + )?; + if parsed == 0 || parsed > u32::MAX as u64 { + return Err( + "--dead-repo-blacklist-fail-threshold must be in 1..=u32::MAX".to_string(), + ); + } + dead_repo_blacklist_fail_threshold = Some(parsed as u32); + } + "--dead-repo-health-check-interval-secs" => { + i += 1; + dead_repo_health_check_interval_secs = parse_u64( + value_at(argv, i, "--dead-repo-health-check-interval-secs")?, + "--dead-repo-health-check-interval-secs", + )?; + } + "--dead-repo-probe-rsync-bin" => { + i += 1; + dead_repo_probe_rsync_bin = + PathBuf::from(value_at(argv, i, "--dead-repo-probe-rsync-bin")?); + } + other => return Err(format!("unknown argument: {other}\n\n{}", usage())), + } + i += 1; + } + + Err(format!("missing -- before child rpki args\n\n{}", usage())) +} + +fn validate_args(args: Args) -> Result { + if args.child_args.is_empty() { + return Err(format!( + "child rpki args are required after --\n\n{}", + usage() + )); + } + if args.dead_repo_blacklist_fail_threshold.is_some() && args.dead_repo_blacklist.is_none() { + return Err( + "--dead-repo-blacklist-fail-threshold requires --dead-repo-blacklist".to_string(), + ); + } + Ok(args) +} + +fn value_at<'a>(argv: &'a [String], index: usize, flag: &str) -> Result<&'a str, String> { + argv.get(index) + .map(String::as_str) + .ok_or_else(|| format!("{flag} requires a value")) +} + +fn parse_u64(raw: &str, flag: &str) -> Result { + raw.parse::() + .map_err(|_| format!("invalid {flag}: {raw}")) +} + +fn parse_usize(raw: &str, flag: &str) -> Result { + raw.parse::() + .map_err(|_| format!("invalid {flag}: {raw}")) +} + +fn status_path(args: &Args) -> PathBuf { + args.status_json + .clone() + .unwrap_or_else(|| args.state_root.join("daemon-status.json")) +} + +fn summary_jsonl_path(args: &Args) -> PathBuf { + args.summary_jsonl + .clone() + .unwrap_or_else(|| args.state_root.join("daemon-runs.jsonl")) +} + +fn utc_now() -> time::OffsetDateTime { + time::OffsetDateTime::now_utc().to_offset(time::UtcOffset::UTC) +} + +fn format_rfc3339(t: time::OffsetDateTime) -> Result { + t.format(&time::format_description::well_known::Rfc3339) + .map_err(|e| format!("format RFC3339 failed: {e}")) +} + +fn format_compact_utc(t: time::OffsetDateTime) -> String { + format!( + "{:04}{:02}{:02}T{:02}{:02}{:02}Z", + t.year(), + u8::from(t.month()), + t.day(), + t.hour(), + t.minute(), + t.second() + ) +} + +fn write_json_pretty(path: &Path, value: &T) -> Result<(), String> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .map_err(|e| format!("create parent dir failed: {}: {e}", parent.display()))?; + } + let bytes = serde_json::to_vec_pretty(value) + .map_err(|e| format!("serialize json failed: {}: {e}", path.display()))?; + // Atomic write (tmp + rename) so concurrent readers never see a torn file. + let mut tmp_name = path + .file_name() + .map(|name| name.to_os_string()) + .unwrap_or_default(); + tmp_name.push(".tmp"); + let tmp_path = path.with_file_name(tmp_name); + fs::write(&tmp_path, &bytes) + .map_err(|e| format!("write json tmp failed: {}: {e}", tmp_path.display()))?; + fs::rename(&tmp_path, path).map_err(|e| { + format!( + "rename json tmp failed: {} -> {}: {e}", + tmp_path.display(), + path.display() + ) + }) +} diff --git a/crates/panda-rpki-validator/src/daemon/daemon_run.rs b/crates/panda-rpki-validator/src/daemon/daemon_run.rs new file mode 100644 index 0000000..f0d2b96 --- /dev/null +++ b/crates/panda-rpki-validator/src/daemon/daemon_run.rs @@ -0,0 +1,120 @@ +// Daemon loop and public entry point. + +fn run_daemon(args: &Args) -> Result<(), String> { + fs::create_dir_all(args.state_root.join("state")).map_err(|e| { + format!( + "create daemon state dir failed: {}: {e}", + args.state_root.join("state").display() + ) + })?; + fs::create_dir_all(args.state_root.join("runs")).map_err(|e| { + format!( + "create daemon runs dir failed: {}: {e}", + args.state_root.join("runs").display() + ) + })?; + + let mut runs_completed = 0u64; + let mut next_seq = 1u64; + let mut last_run_id = None; + let mut health = DeadRepoHealthRuntime::default(); + write_status( + args, + DaemonState::Starting, + runs_completed, + None, + last_run_id.clone(), + &health, + )?; + // Probe once at startup so revived repos are removed before the first run. + maybe_run_dead_repo_health_check(args, &mut health); + + loop { + if args.max_runs.is_some_and(|max| runs_completed >= max) { + break; + } + + write_status( + args, + DaemonState::Idle, + runs_completed, + None, + last_run_id.clone(), + &health, + )?; + let ctx = make_run_context(args, next_seq, utc_now()); + write_status( + args, + DaemonState::Running, + runs_completed, + Some(&ctx), + last_run_id.clone(), + &health, + )?; + let mut summary = run_child_once(args, &ctx)?; + write_status( + args, + DaemonState::Collecting, + runs_completed, + Some(&ctx), + last_run_id.clone(), + &health, + )?; + collect_post_run_metrics(args, &ctx, &mut summary); + let removed = apply_retention(&args.state_root.join("runs"), args.retain_runs)?; + summary.retention_deleted_runs = removed.iter().map(|p| path_string(p)).collect(); + summary.artifacts = collect_artifacts(&ctx.run_dir).unwrap_or_default(); + write_json_pretty(&ctx.run_dir.join("run-summary.json"), &summary)?; + append_json_line(&summary_jsonl_path(args), &summary)?; + runs_completed += 1; + next_seq += 1; + last_run_id = Some(ctx.run_id); + + if args.max_runs.is_some_and(|max| runs_completed >= max) { + break; + } + // Health probes run only while no child is active (single-writer). + maybe_run_dead_repo_health_check(args, &mut health); + write_status( + args, + DaemonState::Sleeping, + runs_completed, + None, + last_run_id.clone(), + &health, + )?; + if args.interval_secs > 0 { + std::thread::sleep(Duration::from_secs(args.interval_secs)); + } + } + + write_status( + args, + DaemonState::Exited, + runs_completed, + None, + last_run_id, + &health, + ) +} + +pub fn main_entry() -> i32 { + let argv: Vec = std::env::args().collect(); + match parse_args(&argv) { + Ok(args) => match run_daemon(&args) { + Ok(()) => 0, + Err(err) => { + eprintln!("{err}"); + 2 + } + }, + Err(err) => { + if argv.iter().any(|a| a == "--help" || a == "-h") { + println!("{err}"); + return 0; + } + eprintln!("{err}"); + 2 + } + } +} diff --git a/crates/panda-rpki-validator/src/daemon/run_child.rs b/crates/panda-rpki-validator/src/daemon/run_child.rs new file mode 100644 index 0000000..af42615 --- /dev/null +++ b/crates/panda-rpki-validator/src/daemon/run_child.rs @@ -0,0 +1,92 @@ +// One child validation run and summary assembly. + +fn run_child_once(args: &Args, ctx: &RunContext) -> Result { + fs::create_dir_all(&ctx.run_dir) + .map_err(|e| format!("create run dir failed: {}: {e}", ctx.run_dir.display()))?; + + let started_at = utc_now(); + let started_at_rfc3339_utc = format_rfc3339(started_at)?; + let stdout_path = ctx.run_dir.join("stdout.log"); + let stderr_path = ctx.run_dir.join("stderr.log"); + let stdout = File::create(&stdout_path) + .map_err(|e| format!("create stdout log failed: {}: {e}", stdout_path.display()))?; + let stderr = File::create(&stderr_path) + .map_err(|e| format!("create stderr log failed: {}: {e}", stderr_path.display()))?; + let child_args = render_child_args(&args.child_args, args, ctx); + let time_output_path = ctx.run_dir.join("process-time.txt"); + + let mut command = if let Some(time_bin) = args.time_bin.as_ref() { + let mut command = Command::new(time_bin); + command + .arg("-v") + .arg("-o") + .arg(&time_output_path) + .arg("--") + .arg(&args.rpki_bin) + .args(&child_args); + command + } else { + let mut command = Command::new(&args.rpki_bin); + command.args(&child_args); + command + }; + command + .stdout(Stdio::from(stdout)) + .stderr(Stdio::from(stderr)); + + let (status, exit_code, exit_status, error) = match command.status() { + Ok(status) if status.success() => ( + RunStatus::Success, + status.code(), + Some(status.to_string()), + None, + ), + Ok(status) => ( + RunStatus::Failed, + status.code(), + Some(status.to_string()), + None, + ), + Err(err) => ( + RunStatus::SpawnFailed, + None, + None, + Some(format!("spawn child failed: {err}")), + ), + }; + + let finished_at = utc_now(); + let finished_at_rfc3339_utc = format_rfc3339(finished_at)?; + let wall_ms = (finished_at - started_at).whole_milliseconds().max(0) as u64; + let process_metrics = Some(collect_process_metrics( + args.time_bin.is_some(), + &time_output_path, + )); + let artifacts = collect_artifacts(&ctx.run_dir)?; + let summary = RunSummary { + run_seq: ctx.seq, + run_id: ctx.run_id.clone(), + run_dir: path_string(&ctx.run_dir), + started_at_rfc3339_utc, + finished_at_rfc3339_utc, + wall_ms, + status, + exit_code, + exit_status, + error, + rpki_bin: path_string(&args.rpki_bin), + child_args, + stdout_path: path_string(&stdout_path), + stderr_path: path_string(&stderr_path), + process_metrics, + stage_timing: None, + report_counts: None, + repo_sync_stats: None, + path_stats: Vec::new(), + db_stats: Vec::new(), + retention_deleted_runs: Vec::new(), + artifacts, + dead_repo_blacklist: None, + }; + Ok(summary) +} diff --git a/crates/panda-rpki-validator/src/daemon/run_metrics.rs b/crates/panda-rpki-validator/src/daemon/run_metrics.rs new file mode 100644 index 0000000..8f5d7f2 --- /dev/null +++ b/crates/panda-rpki-validator/src/daemon/run_metrics.rs @@ -0,0 +1,404 @@ +// Retention, report parsing, and post-run metric collection. + +fn apply_retention(runs_root: &Path, retain_runs: usize) -> Result, String> { + if !runs_root.exists() { + return Ok(Vec::new()); + } + let mut dirs = Vec::new(); + for entry in fs::read_dir(runs_root) + .map_err(|e| format!("read runs dir failed: {}: {e}", runs_root.display()))? + { + let entry = entry.map_err(|e| format!("read runs dir entry failed: {e}"))?; + if entry + .file_type() + .map_err(|e| format!("read file type failed: {}: {e}", entry.path().display()))? + .is_dir() + { + dirs.push(entry.path()); + } + } + dirs.sort(); + let remove_count = dirs.len().saturating_sub(retain_runs); + let mut removed = Vec::new(); + for dir in dirs.into_iter().take(remove_count) { + fs::remove_dir_all(&dir) + .map_err(|e| format!("remove old run dir failed: {}: {e}", dir.display()))?; + removed.push(dir); + } + Ok(removed) +} + +fn find_named_file(root: &Path, name: &str) -> Option { + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + let entries = fs::read_dir(&dir).ok()?; + for entry in entries.flatten() { + let path = entry.path(); + let file_type = entry.file_type().ok()?; + if file_type.is_file() && entry.file_name().to_string_lossy() == name { + return Some(path); + } + if file_type.is_dir() { + stack.push(path); + } + } + } + None +} + +fn read_json_value_if_exists(path: &Path) -> Option { + let bytes = fs::read(path).ok()?; + serde_json::from_slice(&bytes).ok() +} + +fn json_array_len(value: &serde_json::Value, key: &str) -> usize { + value + .get(key) + .and_then(serde_json::Value::as_array) + .map(Vec::len) + .unwrap_or(0) +} + +fn parse_stdout_summary(run_dir: &Path) -> Option { + let stdout_path = run_dir.join("stdout.log"); + let text = fs::read_to_string(stdout_path).ok()?; + let mut vrps = None; + let mut aspas = None; + let mut publication_points = None; + let mut rrdp_repos_unique = None; + let mut tree_instances_processed = None; + let mut tree_instances_failed = None; + let mut warnings = None; + + for line in text.lines() { + if let Some(value) = line.strip_prefix("vrps=") { + vrps = value.trim().parse::().ok(); + } else if let Some(value) = line.strip_prefix("aspas=") { + aspas = value.trim().parse::().ok(); + } else if let Some(value) = line.strip_prefix("audit_publication_points=") { + publication_points = value.trim().parse::().ok(); + } else if let Some(value) = line.strip_prefix("rrdp_repos_unique=") { + rrdp_repos_unique = value.trim().parse::().ok(); + } else if let Some(value) = line.strip_prefix("warnings_total=") { + warnings = value.trim().parse::().ok(); + } else if let Some(rest) = line.strip_prefix("publication_points_processed=") { + for token in rest.split_whitespace() { + if let Some(value) = token.strip_prefix("publication_points_failed=") { + tree_instances_failed = value.parse::().ok(); + } else if tree_instances_processed.is_none() { + tree_instances_processed = token.parse::().ok(); + } + } + } + } + + Some(ReportCounts { + vrps: vrps?, + aspas: aspas?, + publication_points: publication_points?, + rrdp_repos_unique, + tree_instances_processed, + tree_instances_failed, + warnings: warnings.unwrap_or(0), + }) +} + +fn parse_report_counts_fallback(report: &serde_json::Value) -> ReportCounts { + let tree = report.get("tree"); + let tree_warnings = tree + .and_then(|tree| tree.get("warnings")) + .and_then(serde_json::Value::as_array) + .map(Vec::len) + .unwrap_or(0); + let pp_warnings = report + .get("publication_points") + .and_then(serde_json::Value::as_array) + .map(|items| { + items + .iter() + .map(|pp| { + pp.get("warnings") + .and_then(serde_json::Value::as_array) + .map(Vec::len) + .unwrap_or(0) + }) + .sum() + }) + .unwrap_or(0); + ReportCounts { + vrps: json_array_len(report, "vrps"), + aspas: json_array_len(report, "aspas"), + publication_points: json_array_len(report, "publication_points"), + rrdp_repos_unique: None, + tree_instances_processed: tree + .and_then(|tree| tree.get("instances_processed")) + .and_then(serde_json::Value::as_u64), + tree_instances_failed: tree + .and_then(|tree| tree.get("instances_failed")) + .and_then(serde_json::Value::as_u64), + warnings: tree_warnings + pp_warnings, + } +} + +fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) +} + +fn extract_json_object_field(path: &Path, field_name: &str) -> Option { + let bytes = fs::read(path).ok()?; + let needle = format!("\"{field_name}\":"); + let pos = find_subslice(&bytes, needle.as_bytes())?; + let mut i = pos + needle.len(); + while i < bytes.len() && bytes[i].is_ascii_whitespace() { + i += 1; + } + if bytes.get(i).copied()? != b'{' { + return None; + } + let start = i; + let mut depth = 0u32; + let mut in_string = false; + let mut escaped = false; + for (offset, &b) in bytes[start..].iter().enumerate() { + if in_string { + if escaped { + escaped = false; + } else if b == b'\\' { + escaped = true; + } else if b == b'"' { + in_string = false; + } + continue; + } + match b { + b'"' => in_string = true, + b'{' => depth = depth.saturating_add(1), + b'}' => { + depth = depth.saturating_sub(1); + if depth == 0 { + let end = start + offset + 1; + return serde_json::from_slice(&bytes[start..end]).ok(); + } + } + _ => {} + } + } + None +} + +fn parse_report_metadata(run_dir: &Path) -> (Option, Option) { + let Some(report_path) = find_named_file(run_dir, "report.json") else { + return (parse_stdout_summary(run_dir), None); + }; + let counts = parse_stdout_summary(run_dir).or_else(|| { + read_json_value_if_exists(&report_path).map(|report| parse_report_counts_fallback(&report)) + }); + let repo_sync_stats = extract_json_object_field(&report_path, "repo_sync_stats"); + (counts, repo_sync_stats) +} + +fn collect_path_file_stats(label: &str, path: &Path) -> PathFileStats { + let mut stats = PathFileStats { + label: label.to_string(), + path: path_string(path), + exists: path.exists(), + is_dir: path.is_dir(), + total_size_bytes: 0, + file_count: 0, + dir_count: 0, + }; + if !stats.exists { + return stats; + } + if path.is_file() { + if let Ok(metadata) = path.metadata() { + stats.total_size_bytes = metadata.len(); + stats.file_count = 1; + } + return stats; + } + + let mut stack = vec![path.to_path_buf()]; + while let Some(dir) = stack.pop() { + let Ok(entries) = fs::read_dir(&dir) else { + continue; + }; + for entry in entries.flatten() { + let Ok(file_type) = entry.file_type() else { + continue; + }; + if file_type.is_dir() { + stats.dir_count = stats.dir_count.saturating_add(1); + stack.push(entry.path()); + } else if file_type.is_file() { + stats.file_count = stats.file_count.saturating_add(1); + if let Ok(metadata) = entry.metadata() { + stats.total_size_bytes = stats.total_size_bytes.saturating_add(metadata.len()); + } + } + } + } + stats +} + +fn collect_state_path_stats(args: &Args, ctx: &RunContext) -> Vec { + let mut stats = Vec::new(); + stats.push(collect_path_file_stats( + "work_db", + &render_path_template(&args.work_db, args, ctx), + )); + if let Some(path) = args.repo_bytes_db.as_ref() { + stats.push(collect_path_file_stats( + "repo_bytes_db", + &render_path_template(path, args, ctx), + )); + } + if let Some(path) = args.raw_store_db.as_ref() { + stats.push(collect_path_file_stats( + "raw_store_db", + &render_path_template(path, args, ctx), + )); + } + stats +} + +fn parse_key_value_metrics(text: &str) -> BTreeMap { + let mut metrics = BTreeMap::new(); + for line in text.lines() { + let Some((key, value)) = line.split_once('=') else { + continue; + }; + metrics.insert(key.trim().to_string(), value.trim().to_string()); + } + metrics +} + +fn run_db_stats_command( + db_stats_bin: &Path, + db_path: &Path, + run_dir: &Path, + mode: &str, +) -> DbStatsSummary { + let output_path = run_dir.join(format!("db-stats-{mode}.txt")); + let stderr_path = run_dir.join(format!("db-stats-{mode}.stderr.txt")); + let mut summary = DbStatsSummary { + mode: mode.to_string(), + db_path: path_string(db_path), + output_path: Some(path_string(&output_path)), + stderr_path: None, + status: "success".to_string(), + exit_code: None, + error: None, + metrics: BTreeMap::new(), + }; + + if !db_path.exists() { + summary.status = "skipped".to_string(); + summary.error = Some(format!("db path does not exist: {}", db_path.display())); + summary.output_path = None; + return summary; + } + + let mut command = Command::new(db_stats_bin); + command.arg("--db").arg(db_path); + if mode == "exact" { + command.arg("--exact"); + } + match command.output() { + Ok(output) => { + summary.exit_code = output.status.code(); + if !output.status.success() { + summary.status = "failed".to_string(); + } + let stdout_text = String::from_utf8_lossy(&output.stdout).into_owned(); + if let Err(err) = fs::write(&output_path, stdout_text.as_bytes()) { + summary.status = "failed".to_string(); + summary.error = Some(format!( + "write db_stats output failed: {}: {err}", + output_path.display() + )); + } + summary.metrics = parse_key_value_metrics(&stdout_text); + if !output.stderr.is_empty() { + if fs::write(&stderr_path, &output.stderr).is_ok() { + summary.stderr_path = Some(path_string(&stderr_path)); + } + } + if !output.status.success() && summary.error.is_none() { + summary.error = Some(String::from_utf8_lossy(&output.stderr).into_owned()); + } + } + Err(err) => { + summary.status = "spawn_failed".to_string(); + summary.exit_code = None; + summary.output_path = None; + summary.error = Some(format!("spawn db_stats failed: {err}")); + } + } + summary +} + +fn collect_db_stats(args: &Args, ctx: &RunContext) -> Vec { + let work_db = render_path_template(&args.work_db, args, ctx); + let Some(db_stats_bin) = args + .db_stats_bin + .as_ref() + .cloned() + .or_else(default_db_stats_bin) + else { + return vec![DbStatsSummary { + mode: "estimate".to_string(), + db_path: path_string(&work_db), + output_path: None, + stderr_path: None, + status: "skipped".to_string(), + exit_code: None, + error: Some( + "db_stats binary not configured and sibling db_stats was not found".to_string(), + ), + metrics: BTreeMap::new(), + }]; + }; + + let mut stats = Vec::new(); + stats.push(run_db_stats_command( + &db_stats_bin, + &work_db, + &ctx.run_dir, + "estimate", + )); + if args + .db_stats_exact_every + .is_some_and(|every| ctx.seq % every == 0) + { + stats.push(run_db_stats_command( + &db_stats_bin, + &work_db, + &ctx.run_dir, + "exact", + )); + } + stats +} + +fn collect_post_run_metrics(args: &Args, ctx: &RunContext, summary: &mut RunSummary) { + if let Some(path) = find_named_file(&ctx.run_dir, "stage-timing.json") { + summary.stage_timing = read_json_value_if_exists(&path); + } + let (report_counts, repo_sync_stats) = parse_report_metadata(&ctx.run_dir); + summary.report_counts = report_counts; + summary.repo_sync_stats = repo_sync_stats; + summary.path_stats = collect_state_path_stats(args, ctx); + summary.db_stats = collect_db_stats(args, ctx); + summary.artifacts = collect_artifacts(&ctx.run_dir).unwrap_or_default(); + if let Some(path) = args.dead_repo_blacklist.as_ref() { + let (blacklist, _) = DeadRepoBlacklist::load(path); + summary.dead_repo_blacklist = Some(DeadRepoBlacklistRunSummary { + path: path_string(path), + entries: blacklist.len(), + blacklisted: blacklist.blacklisted_len(), + }); + } +} diff --git a/crates/panda-rpki-validator/src/daemon/status.rs b/crates/panda-rpki-validator/src/daemon/status.rs new file mode 100644 index 0000000..243cf62 --- /dev/null +++ b/crates/panda-rpki-validator/src/daemon/status.rs @@ -0,0 +1,299 @@ +// Health probes, child argument rendering, and process metrics. + +fn append_json_line(path: &Path, value: &T) -> Result<(), String> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .map_err(|e| format!("create parent dir failed: {}: {e}", parent.display()))?; + } + let mut file = OpenOptions::new() + .create(true) + .append(true) + .open(path) + .map_err(|e| format!("open jsonl failed: {}: {e}", path.display()))?; + serde_json::to_writer(&mut file, value) + .map_err(|e| format!("write jsonl failed: {}: {e}", path.display()))?; + file.write_all(b"\n") + .map_err(|e| format!("flush jsonl failed: {}: {e}", path.display())) +} + +fn dead_repo_blacklist_status( + args: &Args, + health: &DeadRepoHealthRuntime, +) -> Option { + let path = args.dead_repo_blacklist.as_ref()?; + let (blacklist, _) = DeadRepoBlacklist::load(path); + let last_health_check_at_rfc3339_utc = health + .last_health_check_at + .and_then(|t| format_rfc3339(t).ok()); + Some(DeadRepoBlacklistStatus { + path: path_string(path), + entries: blacklist.len(), + blacklisted: blacklist.blacklisted_len(), + last_health_check_at_rfc3339_utc, + }) +} + +fn write_status( + args: &Args, + state: DaemonState, + runs_completed: u64, + current: Option<&RunContext>, + last_run_id: Option, + health: &DeadRepoHealthRuntime, +) -> Result<(), String> { + let updated_at_rfc3339_utc = format_rfc3339(utc_now())?; + let status = DaemonStatus { + state, + updated_at_rfc3339_utc, + runs_completed, + max_runs: args.max_runs, + current_run_seq: current.map(|ctx| ctx.seq), + current_run_id: current.map(|ctx| ctx.run_id.clone()), + last_run_id, + dead_repo_blacklist: dead_repo_blacklist_status(args, health), + }; + write_json_pretty(&status_path(args), &status) +} + +/// Short-timeout HTTP GET of the RRDP notification file. Any HTTP status +/// (even 4xx) proves the transport is alive; only transport errors keep the +/// entry blacklisted. +fn probe_rrdp_transport(uri: &str) -> bool { + let config = crate::fetch::http::HttpFetcherConfig { + connect_timeout: Duration::from_secs(3), + timeout: Duration::from_secs(6), + large_body_timeout: Duration::from_secs(6), + ..Default::default() + }; + let fetcher = match crate::fetch::http::BlockingHttpFetcher::new(config) { + Ok(fetcher) => fetcher, + Err(_) => return false, + }; + match fetcher.fetch_bytes(uri) { + Ok(_) => true, + Err(err) => err.starts_with("http status"), + } +} + +/// rsync list-only probe: connecting to the daemon and listing the module +/// root is enough to prove the transport is alive. +fn probe_rsync_transport(rsync_bin: &Path, base_uri: &str) -> bool { + Command::new(rsync_bin) + .arg("--contimeout=5") + .arg("--timeout=8") + .arg(base_uri) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .map(|status| status.success()) + .unwrap_or(false) +} + +/// Probe due blacklist entries between runs. Runs in the supervisor +/// loop while no child is active, preserving the single-writer assumption. +fn maybe_run_dead_repo_health_check(args: &Args, health: &mut DeadRepoHealthRuntime) { + let Some(path) = args.dead_repo_blacklist.clone() else { + return; + }; + let now = utc_now(); + if let Some(last) = health.last_health_check_at { + let elapsed = (now - last).whole_seconds(); + if elapsed >= 0 && (elapsed as u64) < args.dead_repo_health_check_interval_secs { + return; + } + } + let (mut blacklist, warning) = DeadRepoBlacklist::load(&path); + if let Some(warning) = warning { + eprintln!("[dead-repo-health] {warning}"); + } + let now_unix = now.unix_timestamp().max(0) as u64; + let due = blacklist.probe_due_entries(now_unix, args.dead_repo_health_check_interval_secs); + if due.is_empty() { + health.last_health_check_at = Some(now); + return; + } + // Probe in parallel: in the docker soak the daemon runs once per run + // (--max-runs 1), so a serial sweep of ~12 dead entries would add up to a + // minute of latency to every run. + let rsync_bin = args.dead_repo_probe_rsync_bin.clone(); + let probe_results: Vec<(RepoTransportMode, String, bool)> = due + .iter() + .map(|entry| (entry.transport, entry.uri.clone())) + .collect::>() + .into_iter() + .map(|(transport, uri)| { + let rsync_bin = rsync_bin.clone(); + std::thread::spawn(move || { + let alive = match transport { + RepoTransportMode::Rrdp => probe_rrdp_transport(&uri), + RepoTransportMode::Rsync => probe_rsync_transport(&rsync_bin, &uri), + }; + (transport, uri, alive) + }) + }) + .filter_map(|handle| handle.join().ok()) + .collect(); + let mut removed = 0usize; + let mut still_dead = 0usize; + for (transport, uri, alive) in &probe_results { + if *alive { + blacklist.record_probe_success(*transport, uri); + removed += 1; + eprintln!( + "[dead-repo-health] removed {} {} (probe ok)", + transport.as_str(), + uri + ); + crate::progress_log::emit( + "dead_repo_blacklist_remove", + serde_json::json!({ + "transport": transport.as_str(), + "uri": uri, + "reason": "health_probe_ok", + }), + ); + } else { + blacklist.record_probe_failure(*transport, uri, now_unix); + still_dead += 1; + eprintln!( + "[dead-repo-health] still dead {} {} (probe failed)", + transport.as_str(), + uri + ); + } + } + if let Err(err) = blacklist.store_atomic(&path, now_unix) { + eprintln!( + "[dead-repo-health] persist blacklist failed: {}: {err}", + path.display() + ); + } + crate::progress_log::emit( + "dead_repo_health_check", + serde_json::json!({ + "probed": probe_results.len(), + "removed": removed, + "still_dead": still_dead, + }), + ); + health.last_health_check_at = Some(now); +} + +fn render_child_args(args: &[String], daemon_args: &Args, ctx: &RunContext) -> Vec { + args.iter() + .map(|arg| { + arg.replace("{state_root}", &path_string(&daemon_args.state_root)) + .replace("{run_out}", &path_string(&ctx.run_dir)) + .replace("{run_id}", &ctx.run_id) + .replace("{run_seq}", &ctx.seq.to_string()) + }) + .collect() +} + +fn render_path_template(path: &Path, daemon_args: &Args, ctx: &RunContext) -> PathBuf { + PathBuf::from( + path_string(path) + .replace("{state_root}", &path_string(&daemon_args.state_root)) + .replace("{run_out}", &path_string(&ctx.run_dir)) + .replace("{run_id}", &ctx.run_id) + .replace("{run_seq}", &ctx.seq.to_string()), + ) +} + +fn path_string(path: &Path) -> String { + path.to_string_lossy().into_owned() +} + +fn make_run_context(args: &Args, seq: u64, now: time::OffsetDateTime) -> RunContext { + let run_id = format!("{seq:06}-{}", format_compact_utc(now)); + let run_dir = args.state_root.join("runs").join(&run_id); + RunContext { + seq, + run_id, + run_dir, + } +} + +fn collect_artifacts(run_dir: &Path) -> Result, String> { + let mut artifacts = Vec::new(); + for entry in fs::read_dir(run_dir) + .map_err(|e| format!("read run dir failed: {}: {e}", run_dir.display()))? + { + let entry = entry.map_err(|e| format!("read run dir entry failed: {e}"))?; + if !entry + .file_type() + .map_err(|e| format!("read file type failed: {}: {e}", entry.path().display()))? + .is_file() + { + continue; + } + let metadata = entry + .metadata() + .map_err(|e| format!("read metadata failed: {}: {e}", entry.path().display()))?; + artifacts.push(ArtifactInfo { + path: path_string(&entry.path()), + size_bytes: metadata.len(), + }); + } + artifacts.sort_by(|a, b| a.path.cmp(&b.path)); + Ok(artifacts) +} + +fn collect_process_metrics(time_wrapper_used: bool, time_output_path: &Path) -> ProcessMetrics { + if !time_wrapper_used { + return ProcessMetrics { + time_wrapper_used, + time_output_path: None, + user_seconds: None, + system_seconds: None, + cpu_percent: None, + elapsed_raw: None, + max_rss_kb: None, + exit_status_from_time: None, + parse_error: None, + }; + } + + let mut metrics = ProcessMetrics { + time_wrapper_used, + time_output_path: Some(path_string(time_output_path)), + user_seconds: None, + system_seconds: None, + cpu_percent: None, + elapsed_raw: None, + max_rss_kb: None, + exit_status_from_time: None, + parse_error: None, + }; + + let text = match fs::read_to_string(time_output_path) { + Ok(text) => text, + Err(err) => { + metrics.parse_error = Some(format!( + "read process time output failed: {}: {err}", + time_output_path.display() + )); + return metrics; + } + }; + + for line in text.lines() { + let line = line.trim(); + if let Some(value) = line.strip_prefix("User time (seconds):") { + metrics.user_seconds = value.trim().parse::().ok(); + } else if let Some(value) = line.strip_prefix("System time (seconds):") { + metrics.system_seconds = value.trim().parse::().ok(); + } else if let Some(value) = line.strip_prefix("Percent of CPU this job got:") { + metrics.cpu_percent = value.trim().trim_end_matches('%').parse::().ok(); + } else if let Some(value) = + line.strip_prefix("Elapsed (wall clock) time (h:mm:ss or m:ss):") + { + metrics.elapsed_raw = Some(value.trim().to_string()); + } else if let Some(value) = line.strip_prefix("Maximum resident set size (kbytes):") { + metrics.max_rss_kb = value.trim().parse::().ok(); + } else if let Some(value) = line.strip_prefix("Exit status:") { + metrics.exit_status_from_time = value.trim().parse::().ok(); + } + } + metrics +} diff --git a/crates/panda-rpki-validator/src/daemon/tests.rs b/crates/panda-rpki-validator/src/daemon/tests.rs new file mode 100644 index 0000000..7a7944b --- /dev/null +++ b/crates/panda-rpki-validator/src/daemon/tests.rs @@ -0,0 +1,785 @@ +use super::*; + +fn test_args(state_root: PathBuf) -> Args { + Args { + work_db: state_root.join("state/work-db"), + repo_bytes_db: Some(state_root.join("state/repo-bytes.db")), + state_root, + rpki_bin: PathBuf::from("/bin/true"), + interval_secs: 0, + max_runs: Some(1), + retain_runs: 10, + status_json: None, + summary_jsonl: None, + raw_store_db: None, + db_stats_bin: None, + db_stats_exact_every: None, + time_bin: None, + dead_repo_blacklist: None, + dead_repo_blacklist_fail_threshold: None, + dead_repo_health_check_interval_secs: 600, + dead_repo_probe_rsync_bin: PathBuf::from("rsync"), + child_args: vec!["--version".to_string()], + } +} + +#[test] +fn parse_args_accepts_required_flags_and_child_args() { + let argv = vec![ + "rpki_daemon".to_string(), + "--state-root".to_string(), + "/tmp/daemon".to_string(), + "--rpki-bin".to_string(), + "/bin/echo".to_string(), + "--interval-secs".to_string(), + "0".to_string(), + "--max-runs".to_string(), + "2".to_string(), + "--retain-runs".to_string(), + "3".to_string(), + "--".to_string(), + "--db".to_string(), + "{state_root}/state/work-db".to_string(), + "--report-json".to_string(), + "{run_out}/report.json".to_string(), + ]; + + let args = parse_args(&argv).expect("parse args"); + assert_eq!(args.state_root, PathBuf::from("/tmp/daemon")); + assert_eq!(args.rpki_bin, PathBuf::from("/bin/echo")); + assert_eq!(args.interval_secs, 0); + assert_eq!(args.max_runs, Some(2)); + assert_eq!(args.retain_runs, 3); + assert_eq!(args.work_db, PathBuf::from("/tmp/daemon/state/work-db")); + assert_eq!( + args.repo_bytes_db, + Some(PathBuf::from("/tmp/daemon/state/repo-bytes.db")) + ); + assert_eq!( + args.child_args, + vec![ + "--db", + "{state_root}/state/work-db", + "--report-json", + "{run_out}/report.json" + ] + ); +} + +#[test] +fn usage_and_parse_args_cover_optional_flags_and_errors() { + let help = usage(); + assert!(help.contains("--db-stats-exact-every")); + assert!(help.contains("{run_seq}")); + + let argv = vec![ + "rpki_daemon".to_string(), + "--state-root".to_string(), + "/tmp/daemon".to_string(), + "--rpki-bin".to_string(), + "/bin/echo".to_string(), + "--status-json".to_string(), + "/tmp/status.json".to_string(), + "--summary-jsonl".to_string(), + "/tmp/runs.jsonl".to_string(), + "--work-db".to_string(), + "{state_root}/work".to_string(), + "--repo-bytes-db".to_string(), + "{state_root}/repo-bytes".to_string(), + "--raw-store-db".to_string(), + "{state_root}/raw".to_string(), + "--db-stats-bin".to_string(), + "/bin/echo".to_string(), + "--db-stats-exact-every".to_string(), + "2".to_string(), + "--time-bin".to_string(), + "/usr/bin/time".to_string(), + "--no-time-wrapper".to_string(), + "--".to_string(), + "child".to_string(), + ]; + let args = parse_args(&argv).expect("optional args"); + assert_eq!(args.status_json, Some(PathBuf::from("/tmp/status.json"))); + assert_eq!(args.summary_jsonl, Some(PathBuf::from("/tmp/runs.jsonl"))); + assert_eq!(args.work_db, PathBuf::from("{state_root}/work")); + assert_eq!( + args.repo_bytes_db, + Some(PathBuf::from("{state_root}/repo-bytes")) + ); + assert_eq!(args.raw_store_db, Some(PathBuf::from("{state_root}/raw"))); + assert_eq!(args.db_stats_bin, Some(PathBuf::from("/bin/echo"))); + assert_eq!(args.db_stats_exact_every, Some(2)); + assert_eq!(args.time_bin, None); + + for (argv, expected) in [ + (vec!["rpki_daemon", "--help"], "Usage:"), + ( + vec![ + "rpki_daemon", + "--state-root", + "/tmp/x", + "--rpki-bin", + "/bin/echo", + "--max-runs", + "0", + "--", + "child", + ], + "--max-runs must be > 0", + ), + ( + vec![ + "rpki_daemon", + "--state-root", + "/tmp/x", + "--rpki-bin", + "/bin/echo", + "--retain-runs", + "0", + "--", + "child", + ], + "--retain-runs must be > 0", + ), + ( + vec![ + "rpki_daemon", + "--state-root", + "/tmp/x", + "--rpki-bin", + "/bin/echo", + "--db-stats-exact-every", + "0", + "--", + "child", + ], + "--db-stats-exact-every must be > 0", + ), + (vec!["rpki_daemon", "--unknown"], "unknown argument"), + ( + vec!["rpki_daemon", "--state-root"], + "--state-root requires a value", + ), + (vec!["rpki_daemon"], "missing -- before child rpki args"), + ( + vec![ + "rpki_daemon", + "--state-root", + "/tmp/x", + "--rpki-bin", + "/bin/echo", + "--", + ], + "child rpki args are required", + ), + ] { + let owned: Vec = argv.into_iter().map(str::to_string).collect(); + let err = parse_args(&owned).expect_err("parse should fail"); + assert!(err.contains(expected), "{err}"); + } +} + +#[test] +fn render_child_args_replaces_placeholders() { + let args = Args { + state_root: PathBuf::from("/tmp/root"), + rpki_bin: PathBuf::from("/bin/echo"), + interval_secs: 0, + max_runs: Some(1), + retain_runs: 10, + status_json: None, + summary_jsonl: None, + work_db: PathBuf::from("/tmp/root/state/work-db"), + repo_bytes_db: Some(PathBuf::from("/tmp/root/state/repo-bytes.db")), + raw_store_db: None, + db_stats_bin: None, + db_stats_exact_every: None, + time_bin: None, + dead_repo_blacklist: None, + dead_repo_blacklist_fail_threshold: None, + dead_repo_health_check_interval_secs: 600, + dead_repo_probe_rsync_bin: PathBuf::from("rsync"), + child_args: vec![ + "{state_root}/state/work-db".to_string(), + "{run_out}/result.ccr".to_string(), + "{run_id}".to_string(), + "{run_seq}".to_string(), + ], + }; + let ctx = RunContext { + seq: 7, + run_id: "000007-20260428T090000Z".to_string(), + run_dir: PathBuf::from("/tmp/root/runs/000007-20260428T090000Z"), + }; + + assert_eq!( + render_child_args(&args.child_args, &args, &ctx), + vec![ + "/tmp/root/state/work-db", + "/tmp/root/runs/000007-20260428T090000Z/result.ccr", + "000007-20260428T090000Z", + "7", + ] + ); +} + +#[test] +fn path_json_and_report_helpers_cover_fallbacks_and_nested_stats() { + let td = tempfile::tempdir().expect("tempdir"); + let state_root = td.path().join("daemon"); + let mut args = test_args(state_root.clone()); + args.work_db = PathBuf::from("{state_root}/state/work-db"); + args.repo_bytes_db = Some(PathBuf::from("{state_root}/state/repo-bytes.db")); + args.raw_store_db = Some(PathBuf::from("{state_root}/state/raw-store.db")); + + let now = time::Date::from_calendar_date(2026, time::Month::April, 28) + .expect("date") + .with_hms(9, 0, 0) + .expect("time") + .assume_utc(); + let ctx = make_run_context(&args, 42, now); + assert_eq!(ctx.run_id, "000042-20260428T090000Z"); + + let rendered = render_path_template(Path::new("{run_out}/{run_id}/{run_seq}"), &args, &ctx); + assert!(rendered.ends_with("000042-20260428T090000Z/000042-20260428T090000Z/42")); + + let nested_json = td.path().join("nested/out/status.json"); + write_json_pretty(&nested_json, &serde_json::json!({"ok": true})).expect("write json"); + append_json_line( + &td.path().join("nested/out/runs.jsonl"), + &serde_json::json!({"n": 1}), + ) + .expect("append jsonl"); + + fs::create_dir_all(ctx.run_dir.join("subdir")).expect("subdir"); + fs::write(ctx.run_dir.join("a.txt"), "aaa").expect("file"); + fs::write(ctx.run_dir.join("subdir/ignored.txt"), "bbb").expect("nested file"); + let artifacts = collect_artifacts(&ctx.run_dir).expect("artifacts"); + assert_eq!(artifacts.len(), 1); + assert!(artifacts[0].path.ends_with("a.txt")); + + fs::create_dir_all(state_root.join("state/work-db/nested")).expect("work db"); + fs::write(state_root.join("state/work-db/file.sst"), "abc").expect("sst"); + fs::write(state_root.join("state/work-db/nested/inner.sst"), "def").expect("inner"); + fs::write(state_root.join("state/raw-store.db"), "raw").expect("raw file"); + let file_stats = + collect_path_file_stats("raw_store_db", &state_root.join("state/raw-store.db")); + assert!(file_stats.exists); + assert!(!file_stats.is_dir); + assert_eq!(file_stats.file_count, 1); + let missing_stats = collect_path_file_stats("missing", &state_root.join("missing")); + assert!(!missing_stats.exists); + let state_stats = collect_state_path_stats(&args, &ctx); + assert!( + state_stats + .iter() + .any(|s| s.label == "raw_store_db" && s.exists) + ); + assert!( + state_stats + .iter() + .any(|s| s.label == "work_db" && s.file_count == 2) + ); + + let report_path = td.path().join("report.json"); + fs::write( + &report_path, + r#"{"repo_sync_stats": { "nested": {"text": "a\"b"} }, "after": 1}"#, + ) + .expect("report"); + assert_eq!( + extract_json_object_field(&report_path, "repo_sync_stats").expect("repo stats")["nested"]["text"], + "a\"b" + ); + fs::write(&report_path, r#"{"repo_sync_stats": []}"#).expect("report"); + assert!(extract_json_object_field(&report_path, "repo_sync_stats").is_none()); + assert!(extract_json_object_field(&report_path, "missing").is_none()); + + let counts = parse_report_counts_fallback(&serde_json::json!({ + "vrps": [{}, {}], + "aspas": [{}], + "publication_points": [{"warnings": [{}, {}]}, {"warnings": [{}]}], + "tree": {"instances_processed": 2, "instances_failed": 1, "warnings": [{}]} + })); + assert_eq!(counts.vrps, 2); + assert_eq!(counts.aspas, 1); + assert_eq!(counts.publication_points, 2); + assert_eq!(counts.warnings, 4); + assert_eq!(counts.tree_instances_processed, Some(2)); + assert_eq!(counts.tree_instances_failed, Some(1)); +} + +#[test] +fn retention_removes_oldest_run_directories() { + let td = tempfile::tempdir().expect("tempdir"); + let runs = td.path().join("runs"); + fs::create_dir_all(&runs).expect("runs dir"); + for name in [ + "000001-20260428T000001Z", + "000002-20260428T000002Z", + "000003-20260428T000003Z", + ] { + fs::create_dir_all(runs.join(name)).expect("run dir"); + } + + let removed = apply_retention(&runs, 2).expect("retention"); + assert_eq!(removed.len(), 1); + assert!(!runs.join("000001-20260428T000001Z").exists()); + assert!(runs.join("000002-20260428T000002Z").exists()); + assert!(runs.join("000003-20260428T000003Z").exists()); +} + +#[test] +fn retention_empty_root_and_parse_metrics_error_paths_are_reported() { + let td = tempfile::tempdir().expect("tempdir"); + let missing_runs = td.path().join("missing-runs"); + assert!( + apply_retention(&missing_runs, 2) + .expect("empty retention") + .is_empty() + ); + + let disabled = collect_process_metrics(false, &td.path().join("missing-time.txt")); + assert!(!disabled.time_wrapper_used); + assert!(disabled.time_output_path.is_none()); + + let missing = collect_process_metrics(true, &td.path().join("missing-time.txt")); + assert!(missing.time_wrapper_used); + assert!( + missing + .parse_error + .expect("parse error") + .contains("read process time output failed") + ); +} + +#[test] +fn process_metrics_parses_gnu_time_elapsed_line() { + let td = tempfile::tempdir().expect("tempdir"); + let path = td.path().join("time.txt"); + fs::write( + &path, + "User time (seconds): 1.25\nSystem time (seconds): 0.50\nPercent of CPU this job got: 175%\nElapsed (wall clock) time (h:mm:ss or m:ss): 0:01.00\nMaximum resident set size (kbytes): 12345\nExit status: 0\n", + ) + .expect("write time"); + + let metrics = collect_process_metrics(true, &path); + assert_eq!(metrics.user_seconds, Some(1.25)); + assert_eq!(metrics.system_seconds, Some(0.50)); + assert_eq!(metrics.cpu_percent, Some(175.0)); + assert_eq!(metrics.elapsed_raw.as_deref(), Some("0:01.00")); + assert_eq!(metrics.max_rss_kb, Some(12345)); + assert_eq!(metrics.exit_status_from_time, Some(0)); +} + +#[test] +#[cfg(unix)] +fn child_and_db_stats_error_paths_are_reported() { + use std::os::unix::fs::PermissionsExt; + + let td = tempfile::tempdir().expect("tempdir"); + let run_dir = td.path().join("run"); + fs::create_dir_all(&run_dir).expect("run dir"); + let missing_db = td.path().join("missing-db"); + let skipped = run_db_stats_command(Path::new("/bin/echo"), &missing_db, &run_dir, "estimate"); + assert_eq!(skipped.status, "skipped"); + assert!(skipped.output_path.is_none()); + + let db_path = td.path().join("db"); + fs::create_dir_all(&db_path).expect("db"); + let failing_bin = td.path().join("failing_db_stats.sh"); + fs::write( + &failing_bin, + "#!/bin/sh\necho partial=1\necho db-stats failed >&2\nexit 7\n", + ) + .expect("script"); + let mut permissions = fs::metadata(&failing_bin).expect("metadata").permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&failing_bin, permissions).expect("chmod"); + let failed = run_db_stats_command(&failing_bin, &db_path, &run_dir, "exact"); + assert_eq!(failed.status, "failed"); + assert_eq!(failed.exit_code, Some(7)); + assert_eq!(failed.metrics.get("partial").map(String::as_str), Some("1")); + assert!(failed.stderr_path.is_some()); + assert!(failed.error.expect("stderr").contains("db-stats failed")); + + let spawn_failed = run_db_stats_command( + Path::new("/definitely/not/db_stats"), + &db_path, + &run_dir, + "estimate", + ); + assert_eq!(spawn_failed.status, "spawn_failed"); + assert!( + spawn_failed + .error + .expect("spawn err") + .contains("spawn db_stats failed") + ); + + let metrics = parse_key_value_metrics("alpha=1\nnot-a-pair\nbeta = two\n"); + assert_eq!(metrics.get("alpha").map(String::as_str), Some("1")); + assert_eq!(metrics.get("beta").map(String::as_str), Some("two")); + + let mut args = test_args(td.path().join("daemon")); + args.rpki_bin = PathBuf::from("/bin/sh"); + args.time_bin = Some(PathBuf::from("/usr/bin/time")); + args.child_args = vec![ + "-c".to_string(), + "echo child-out; echo child-err >&2; exit 7".to_string(), + ]; + let ctx = RunContext { + seq: 1, + run_id: "000001-20260428T000000Z".to_string(), + run_dir: args.state_root.join("runs/000001-20260428T000000Z"), + }; + let summary = run_child_once(&args, &ctx).expect("run failing child"); + assert_eq!(summary.status, RunStatus::Failed); + assert_eq!(summary.exit_code, Some(7)); + let process_metrics = summary.process_metrics.expect("process metrics"); + assert!(process_metrics.time_wrapper_used); + assert_eq!(process_metrics.exit_status_from_time, Some(7)); + + let mut spawn_args = test_args(td.path().join("spawn")); + spawn_args.rpki_bin = PathBuf::from("/definitely/not/rpki"); + spawn_args.child_args = vec!["--help".to_string()]; + let spawn_ctx = RunContext { + seq: 1, + run_id: "000001-20260428T000001Z".to_string(), + run_dir: spawn_args.state_root.join("runs/000001-20260428T000001Z"), + }; + let spawn_summary = run_child_once(&spawn_args, &spawn_ctx).expect("spawn summary"); + assert_eq!(spawn_summary.status, RunStatus::SpawnFailed); + assert!( + spawn_summary + .error + .expect("spawn error") + .contains("spawn child failed") + ); +} + +#[test] +fn report_metadata_prefers_stdout_summary_and_extracts_repo_sync_stats() { + let td = tempfile::tempdir().expect("tempdir"); + fs::write( + td.path().join("stdout.log"), + "RPKI validation run summary\npublication_points_processed=7 publication_points_failed=1\nrrdp_repos_unique=3\nvrps=11\naspas=2\naudit_publication_points=7\nwarnings_total=5\n", + ) + .expect("stdout"); + fs::write( + td.path().join("report.json"), + "{\"large\":[{\"ignored\":\"{}\"}],\"repo_sync_stats\":{\"by_phase\":{\"rrdp_ok\":{\"count\":7}},\"by_terminal_state\":{},\"publication_points_total\":7}}", + ) + .expect("report"); + + let (counts, repo_sync_stats) = parse_report_metadata(td.path()); + let counts = counts.expect("counts"); + assert_eq!(counts.vrps, 11); + assert_eq!(counts.aspas, 2); + assert_eq!(counts.publication_points, 7); + assert_eq!(counts.rrdp_repos_unique, Some(3)); + assert_eq!(counts.tree_instances_processed, Some(7)); + assert_eq!(counts.tree_instances_failed, Some(1)); + assert_eq!(counts.warnings, 5); + assert_eq!( + repo_sync_stats.expect("repo sync")["by_phase"]["rrdp_ok"]["count"].as_u64(), + Some(7) + ); +} + +#[test] +fn daemon_exits_immediately_when_max_runs_already_reached() { + let td = tempfile::tempdir().expect("tempdir"); + let mut args = test_args(td.path().join("daemon")); + args.max_runs = Some(0); + + run_daemon(&args).expect("run daemon"); + + let status_text = + fs::read_to_string(args.state_root.join("daemon-status.json")).expect("status"); + assert!(status_text.contains("\"state\": \"exited\"")); + assert!(status_text.contains("\"runsCompleted\": 0")); + assert!(args.state_root.join("runs").exists()); +} + +#[test] +fn daemon_runs_fake_child_twice_and_writes_summaries() { + let td = tempfile::tempdir().expect("tempdir"); + let args = Args { + state_root: td.path().join("daemon"), + rpki_bin: PathBuf::from("/bin/sh"), + interval_secs: 0, + max_runs: Some(2), + retain_runs: 10, + status_json: None, + summary_jsonl: None, + work_db: td.path().join("daemon/state/work-db"), + repo_bytes_db: Some(td.path().join("daemon/state/repo-bytes.db")), + raw_store_db: None, + db_stats_bin: None, + db_stats_exact_every: None, + time_bin: None, + dead_repo_blacklist: None, + dead_repo_blacklist_fail_threshold: None, + dead_repo_health_check_interval_secs: 600, + dead_repo_probe_rsync_bin: PathBuf::from("rsync"), + child_args: vec![ + "-c".to_string(), + "echo stdout-{run_seq}; echo stderr-{run_seq} >&2; echo marker > {run_out}/marker.txt" + .to_string(), + ], + }; + + run_daemon(&args).expect("run daemon"); + + let status_text = + fs::read_to_string(args.state_root.join("daemon-status.json")).expect("status"); + assert!(status_text.contains("\"state\": \"exited\"")); + assert!(status_text.contains("\"runsCompleted\": 2")); + + let jsonl = + fs::read_to_string(args.state_root.join("daemon-runs.jsonl")).expect("summary jsonl"); + assert_eq!(jsonl.lines().count(), 2); + + let run_dirs = fs::read_dir(args.state_root.join("runs")) + .expect("runs dir") + .collect::, _>>() + .expect("entries"); + assert_eq!(run_dirs.len(), 2); + for entry in run_dirs { + assert!(entry.path().join("run-summary.json").exists()); + assert!(entry.path().join("marker.txt").exists()); + } +} + +#[test] +fn daemon_sleep_and_retention_deletion_are_recorded() { + let td = tempfile::tempdir().expect("tempdir"); + let mut args = test_args(td.path().join("daemon")); + args.rpki_bin = PathBuf::from("/bin/sh"); + args.interval_secs = 1; + args.max_runs = Some(2); + args.retain_runs = 1; + args.child_args = vec![ + "-c".to_string(), + "echo run-{run_seq}; printf marker > {run_out}/marker.txt".to_string(), + ]; + + run_daemon(&args).expect("run daemon"); + + let jsonl = fs::read_to_string(args.state_root.join("daemon-runs.jsonl")).expect("jsonl"); + assert_eq!(jsonl.lines().count(), 2); + let last: serde_json::Value = + serde_json::from_str(jsonl.lines().last().expect("last line")).expect("summary"); + assert_eq!( + last["retentionDeletedRuns"] + .as_array() + .expect("deleted") + .len(), + 1 + ); + let run_dirs = fs::read_dir(args.state_root.join("runs")) + .expect("runs") + .collect::, _>>() + .expect("entries"); + assert_eq!(run_dirs.len(), 1); +} + +#[test] +#[cfg(unix)] +fn daemon_collects_stage_report_db_and_file_metrics() { + use std::os::unix::fs::PermissionsExt; + + let td = tempfile::tempdir().expect("tempdir"); + let db_stats_bin = td.path().join("fake_db_stats.sh"); + fs::write( + &db_stats_bin, + "#!/bin/sh\nif [ \"$3\" = \"--exact\" ]; then echo mode=exact; else echo mode=estimate; fi\necho total=42\necho db.files.total_size_bytes=123\n", + ) + .expect("fake db_stats"); + let mut permissions = fs::metadata(&db_stats_bin).expect("metadata").permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&db_stats_bin, permissions).expect("chmod"); + + let args = Args { + state_root: td.path().join("daemon"), + rpki_bin: PathBuf::from("/bin/sh"), + interval_secs: 0, + max_runs: Some(1), + retain_runs: 10, + status_json: None, + summary_jsonl: None, + work_db: PathBuf::from("{state_root}/state/work-db"), + repo_bytes_db: Some(PathBuf::from("{state_root}/state/repo-bytes.db")), + raw_store_db: None, + db_stats_bin: Some(db_stats_bin), + db_stats_exact_every: Some(1), + time_bin: None, + dead_repo_blacklist: None, + dead_repo_blacklist_fail_threshold: None, + dead_repo_health_check_interval_secs: 600, + dead_repo_probe_rsync_bin: PathBuf::from("rsync"), + child_args: vec![ + "-c".to_string(), + "mkdir -p {state_root}/state/work-db {state_root}/state/repo-bytes.db; \ + printf x > {state_root}/state/work-db/000001.sst; \ + printf '{\"validation_ms\":7,\"download_event_count\":2}' > {run_out}/stage-timing.json; \ + printf '{\"tree\":{\"instances_processed\":3,\"instances_failed\":1,\"warnings\":[{}]},\"publication_points\":[{},{}],\"vrps\":[{},{}],\"aspas\":[{}],\"repo_sync_stats\":{\"by_phase\":{\"snapshot\":{\"count\":1}}}}' > {run_out}/report.json" + .to_string(), + ], + }; + + run_daemon(&args).expect("run daemon"); + + let run_summary = + find_named_file(&args.state_root.join("runs"), "run-summary.json").expect("run summary"); + let summary: serde_json::Value = + serde_json::from_slice(&fs::read(run_summary).expect("read run summary")) + .expect("parse run summary"); + + assert_eq!(summary["stageTiming"]["validation_ms"].as_u64(), Some(7)); + assert_eq!(summary["reportCounts"]["vrps"].as_u64(), Some(2)); + assert_eq!(summary["reportCounts"]["aspas"].as_u64(), Some(1)); + assert_eq!( + summary["reportCounts"]["publicationPoints"].as_u64(), + Some(2) + ); + assert_eq!( + summary["repoSyncStats"]["by_phase"]["snapshot"]["count"].as_u64(), + Some(1) + ); + assert_eq!(summary["dbStats"].as_array().expect("db stats").len(), 2); + assert!( + summary["pathStats"] + .as_array() + .expect("path stats") + .iter() + .any(|item| item["label"] == "work_db" && item["exists"] == true) + ); +} + +#[test] +fn dead_repo_blacklist_flag_is_injected_into_child_args() { + let argv = vec![ + "rpki_daemon".to_string(), + "--state-root".to_string(), + "/tmp/daemon".to_string(), + "--rpki-bin".to_string(), + "/bin/echo".to_string(), + "--dead-repo-blacklist".to_string(), + "/tmp/daemon/state/dead-repo-blacklist.json".to_string(), + "--dead-repo-blacklist-fail-threshold".to_string(), + "2".to_string(), + "--".to_string(), + "--db".to_string(), + "{state_root}/state/work-db".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!( + args.child_args, + vec![ + "--db".to_string(), + "{state_root}/state/work-db".to_string(), + "--dead-repo-blacklist".to_string(), + "/tmp/daemon/state/dead-repo-blacklist.json".to_string(), + "--dead-repo-blacklist-fail-threshold".to_string(), + "2".to_string(), + ] + ); +} + +#[test] +fn dead_repo_blacklist_injection_respects_existing_child_flags() { + let argv = vec![ + "rpki_daemon".to_string(), + "--state-root".to_string(), + "/tmp/daemon".to_string(), + "--rpki-bin".to_string(), + "/bin/echo".to_string(), + "--dead-repo-blacklist".to_string(), + "/tmp/daemon/state/dead-repo-blacklist.json".to_string(), + "--".to_string(), + "--dead-repo-blacklist".to_string(), + "/custom/path.json".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!( + args.child_args, + vec![ + "--dead-repo-blacklist".to_string(), + "/custom/path.json".to_string(), + ] + ); +} + +#[test] +fn dead_repo_threshold_without_blacklist_is_rejected() { + let argv = vec![ + "rpki_daemon".to_string(), + "--state-root".to_string(), + "/tmp/daemon".to_string(), + "--rpki-bin".to_string(), + "/bin/echo".to_string(), + "--dead-repo-blacklist-fail-threshold".to_string(), + "2".to_string(), + "--".to_string(), + "--version".to_string(), + ]; + let err = parse_args(&argv).expect_err("parse should fail"); + assert!(err.contains("requires --dead-repo-blacklist"), "{err}"); +} + +#[test] +fn health_check_keeps_dead_entry_and_removes_revived_entry() { + use std::io::{Read as _, Write as _}; + use std::net::TcpListener; + + // Revived RRDP host: one-shot HTTP server answering 200. + let listener = TcpListener::bind("127.0.0.1:0").expect("bind"); + let port = listener.local_addr().expect("local addr").port(); + std::thread::spawn(move || { + if let Ok((mut stream, _)) = listener.accept() { + let mut buf = [0u8; 1024]; + let _ = stream.read(&mut buf); + let body = b""; + let _ = stream.write_all( + format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .as_bytes(), + ); + let _ = stream.write_all(body); + } + }); + + let td = tempfile::tempdir().expect("tempdir"); + let blacklist_path = td.path().join("dead-repo-blacklist.json"); + let revived_uri = format!("http://127.0.0.1:{port}/notification.xml"); + let dead_uri = "http://127.0.0.1:1/notification.xml".to_string(); + let mut blacklist = DeadRepoBlacklist::new(); + for uri in [&revived_uri, &dead_uri] { + blacklist.record_transport_failure(RepoTransportMode::Rrdp, uri, 100, 1, 256); + } + blacklist + .store_atomic(&blacklist_path, 100) + .expect("store blacklist"); + + let mut args = test_args(td.path().join("daemon")); + args.dead_repo_blacklist = Some(blacklist_path.clone()); + args.dead_repo_health_check_interval_secs = 0; + let mut health = DeadRepoHealthRuntime::default(); + maybe_run_dead_repo_health_check(&args, &mut health); + + let (after, warning) = DeadRepoBlacklist::load(&blacklist_path); + assert!(warning.is_none()); + assert!(!after.is_blacklisted(RepoTransportMode::Rrdp, &revived_uri)); + assert!(after.is_blacklisted(RepoTransportMode::Rrdp, &dead_uri)); + assert!(health.last_health_check_at.is_some()); + + // Status JSON exposes the blacklist section. + let status = dead_repo_blacklist_status(&args, &health).expect("status"); + assert_eq!(status.entries, 1); + assert_eq!(status.blacklisted, 1); + assert!(status.last_health_check_at_rfc3339_utc.is_some()); +} diff --git a/crates/panda-rpki-validator/src/daemon/types.rs b/crates/panda-rpki-validator/src/daemon/types.rs new file mode 100644 index 0000000..b5e0b02 --- /dev/null +++ b/crates/panda-rpki-validator/src/daemon/types.rs @@ -0,0 +1,175 @@ +// Daemon arguments, state, and summary data types. + +#[derive(Clone, Debug, PartialEq, Eq)] +struct Args { + state_root: PathBuf, + rpki_bin: PathBuf, + interval_secs: u64, + max_runs: Option, + retain_runs: usize, + status_json: Option, + summary_jsonl: Option, + work_db: PathBuf, + repo_bytes_db: Option, + raw_store_db: Option, + db_stats_bin: Option, + db_stats_exact_every: Option, + time_bin: Option, + /// Enables daemon-side dead-repository health checks and + /// auto-injects the child `--dead-repo-blacklist` flag when absent. + dead_repo_blacklist: Option, + dead_repo_blacklist_fail_threshold: Option, + dead_repo_health_check_interval_secs: u64, + dead_repo_probe_rsync_bin: PathBuf, + child_args: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct RunContext { + seq: u64, + run_id: String, + run_dir: PathBuf, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +enum DaemonState { + Starting, + Idle, + Running, + Collecting, + Sleeping, + Exited, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +struct DaemonStatus { + state: DaemonState, + updated_at_rfc3339_utc: String, + runs_completed: u64, + max_runs: Option, + current_run_seq: Option, + current_run_id: Option, + last_run_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + dead_repo_blacklist: Option, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +struct DeadRepoBlacklistStatus { + path: String, + entries: usize, + blacklisted: usize, + last_health_check_at_rfc3339_utc: Option, +} + +/// Mutable daemon-side health-check bookkeeping. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct DeadRepoHealthRuntime { + last_health_check_at: Option, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +enum RunStatus { + Success, + Failed, + SpawnFailed, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +struct ArtifactInfo { + path: String, + size_bytes: u64, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct ProcessMetrics { + time_wrapper_used: bool, + time_output_path: Option, + user_seconds: Option, + system_seconds: Option, + cpu_percent: Option, + elapsed_raw: Option, + max_rss_kb: Option, + exit_status_from_time: Option, + parse_error: Option, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +struct ReportCounts { + vrps: usize, + aspas: usize, + publication_points: usize, + rrdp_repos_unique: Option, + tree_instances_processed: Option, + tree_instances_failed: Option, + warnings: usize, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +struct PathFileStats { + label: String, + path: String, + exists: bool, + is_dir: bool, + total_size_bytes: u64, + file_count: u64, + dir_count: u64, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +struct DbStatsSummary { + mode: String, + db_path: String, + output_path: Option, + stderr_path: Option, + status: String, + exit_code: Option, + error: Option, + metrics: BTreeMap, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct RunSummary { + run_seq: u64, + run_id: String, + run_dir: String, + started_at_rfc3339_utc: String, + finished_at_rfc3339_utc: String, + wall_ms: u64, + status: RunStatus, + exit_code: Option, + exit_status: Option, + error: Option, + rpki_bin: String, + child_args: Vec, + stdout_path: String, + stderr_path: String, + process_metrics: Option, + stage_timing: Option, + report_counts: Option, + repo_sync_stats: Option, + path_stats: Vec, + db_stats: Vec, + retention_deleted_runs: Vec, + artifacts: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + dead_repo_blacklist: Option, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +struct DeadRepoBlacklistRunSummary { + path: String, + entries: usize, + blacklisted: usize, +} diff --git a/crates/panda-rpki-validator/src/data_model/rc.rs b/crates/panda-rpki-validator/src/data_model/rc.rs index 701ea91..e8895ea 100644 --- a/crates/panda-rpki-validator/src/data_model/rc.rs +++ b/crates/panda-rpki-validator/src/data_model/rc.rs @@ -23,1846 +23,8 @@ use crate::data_model::oid::{ OID_SUBJECT_INFO_ACCESS_RAW, OID_SUBJECT_KEY_IDENTIFIER, OID_SUBJECT_KEY_IDENTIFIER_RAW, }; -/// Resource Certificate kind (semantic classification). -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum ResourceCertKind { - Ca, - Ee, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum ResourceCertificateRole { - TrustAnchor, - Ca, - SignedObjectEe, - RouterEe, -} - -/// A parsed RPKI Resource Certificate (RFC 6487) data model. -/// -/// This module intentionally focuses on the semantics needed by Signed Object validation and -/// object-specific EE certificate checks (MFT/ROA/ASPA), as described in -/// `rpki/specs/03_resource_certificate_rc.md`. -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ResourceCertificate { - pub raw_der: Vec, - pub tbs: RpkixTbsCertificate, - pub kind: ResourceCertKind, -} - -pub type ResourceCaCertificate = ResourceCertificate; -pub type ResourceEeCertificate = ResourceCertificate; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RpkixTbsCertificate { - pub version: u32, - pub serial_number: BigUint, - pub signature_algorithm: String, - pub issuer_name: X509NameDer, - pub subject_name: X509NameDer, - pub validity_not_before: UtcTime, - pub validity_not_after: UtcTime, - /// DER encoding of SubjectPublicKeyInfo. - pub subject_public_key_info: Vec, - pub extensions: RcExtensions, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RcExtensions { - pub basic_constraints_ca: bool, - pub basic_constraints: Option, - pub subject_key_identifier: Option>, - /// Authority Key Identifier (AKI) keyIdentifier value. - pub authority_key_identifier: Option>, - /// CRL Distribution Points URIs (fullName). - pub crl_distribution_points_uris: Option>, - /// Authority Information Access (AIA) caIssuers URIs. - pub ca_issuers_uris: Option>, - pub subject_info_access: Option, - pub certificate_policies_oid: Option, - pub certificate_policies: Option, - pub extension_oids: Vec, - - pub ip_resources: Option, - pub as_resources: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct BasicConstraintsProfile { - pub ca: bool, - pub critical: bool, - pub path_len_constraint: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct CertificatePoliciesProfile { - pub policy_oid: String, - pub qualifier_oids: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ResourceCertificateParsed { - pub raw_der: Vec, - pub version: X509Version, - pub serial_number: BigUint, - pub signature_algorithm: AlgorithmIdentifierValue, - pub tbs_signature_algorithm: AlgorithmIdentifierValue, - pub issuer_name: X509NameDer, - pub subject_name: X509NameDer, - pub validity_not_before: Asn1TimeUtc, - pub validity_not_after: Asn1TimeUtc, - /// DER encoding of SubjectPublicKeyInfo. - pub subject_public_key_info: Vec, - pub extensions: RcExtensionsParsed, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct AlgorithmIdentifierValue { - pub oid: String, - pub parameters: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct AlgorithmParametersValue { - pub class: Asn1Class, - pub tag: Asn1Tag, - pub data: Vec, -} - -impl AlgorithmIdentifierValue { - pub fn params_absent_or_null(&self) -> bool { - match &self.parameters { - None => true, - Some(p) if p.class == Asn1Class::Universal && p.tag == Asn1Tag::Null => true, - Some(_p) => false, - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RcExtensionsParsed { - pub basic_constraints: Vec, - pub subject_key_identifier: Vec<(Vec, bool)>, - pub authority_key_identifier: Vec<(AuthorityKeyIdentifierParsed, bool)>, - pub crl_distribution_points: Vec<(CrlDistributionPointsParsed, bool)>, - pub authority_info_access: Vec<(AuthorityInfoAccessParsed, bool)>, - pub subject_info_access: Vec<(SubjectInfoAccessParsed, bool)>, - pub certificate_policies: Vec<(Vec, bool)>, - pub extension_oids: Vec, - pub ip_resources: Vec<(IpResourceSet, bool)>, - pub as_resources: Vec<(AsResourceSet, bool)>, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct AuthorityKeyIdentifierParsed { - pub key_identifier: Option>, - pub has_authority_cert_issuer: bool, - pub has_authority_cert_serial: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct AuthorityInfoAccessParsed { - pub ca_issuers_uris: Vec, - pub ca_issuers_access_location_not_uri: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct CrlDistributionPointsParsed { - pub distribution_points: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct CrlDistributionPointParsed { - pub distribution_point_present: bool, - pub reasons_present: bool, - pub crl_issuer_present: bool, - pub name_relative_to_crl_issuer_present: bool, - pub full_name_uris: Vec, - pub full_name_not_uri: bool, - pub full_name_present: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct SubjectInfoAccessParsed { - pub access_descriptions: Vec, - pub signed_object_uris: Vec, - pub signed_object_access_location_not_uri: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum SubjectInfoAccess { - Ca(SubjectInfoAccessCa), - Ee(SubjectInfoAccessEe), -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct SubjectInfoAccessCa { - pub access_descriptions: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct SubjectInfoAccessEe { - pub signed_object_uris: Vec, - /// The full list of access descriptions as carried in the SIA extension. - pub access_descriptions: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct AccessDescription { - pub access_method_oid: String, - pub access_location: String, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] -pub enum Afi { - Ipv4, - Ipv6, -} - -impl Afi { - pub fn ub(self) -> u16 { - match self { - Afi::Ipv4 => 32, - Afi::Ipv6 => 128, - } - } - - pub fn octets_len(self) -> usize { - match self { - Afi::Ipv4 => 4, - Afi::Ipv6 => 16, - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct IpResourceSet { - pub families: Vec, -} - -impl IpResourceSet { - /// Decode the DER bytes carried inside the X.509 `extnValue` OCTET STRING for - /// `id-pe-ipAddrBlocks` (RFC 3779 / RFC 6487). - pub fn decode_extn_value(extn_value: &[u8]) -> Result { - parse_ip_addr_blocks(extn_value).map_err(|_| IpResourceSetDecodeError::InvalidEncoding) - } - - pub fn is_all_inherit(&self) -> bool { - self.families - .iter() - .all(|f| matches!(f.choice, IpAddressChoice::Inherit)) - } - - pub fn has_any_inherit(&self) -> bool { - self.families - .iter() - .any(|f| matches!(f.choice, IpAddressChoice::Inherit)) - } - - pub fn contains_prefix(&self, prefix: &IpPrefix) -> bool { - self.families.iter().any(|fam| fam.contains_prefix(prefix)) - } -} - -#[derive(Debug, thiserror::Error)] -pub enum IpResourceSetDecodeError { - #[error("invalid ipAddrBlocks encoding (RFC 3779 §2.2.3; RFC 6487 §4.8.10)")] - InvalidEncoding, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct IpAddressFamily { - pub afi: Afi, - pub choice: IpAddressChoice, -} - -impl IpAddressFamily { - pub fn contains_prefix(&self, prefix: &IpPrefix) -> bool { - if self.afi != prefix.afi { - return false; - } - match &self.choice { - IpAddressChoice::Inherit => true, - IpAddressChoice::AddressesOrRanges(items) => items.iter().any(|item| match item { - IpAddressOrRange::Prefix(p) => prefix_covers(p, prefix), - IpAddressOrRange::Range(r) => range_covers_prefix(self.afi, r, prefix), - }), - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub enum IpAddressChoice { - Inherit, - AddressesOrRanges(Vec), -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub enum IpAddressOrRange { - Prefix(IpPrefix), - Range(IpAddressRange), -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct IpAddressRange { - pub min: Vec, - pub max: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] -pub struct IpPrefix { - pub afi: Afi, - pub prefix_len: u16, - pub addr: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct AsResourceSet { - pub asnum: Option, - pub rdi: Option, -} - -impl AsResourceSet { - /// Decode the DER bytes carried inside the X.509 `extnValue` OCTET STRING for - /// `id-pe-autonomousSysIds` (RFC 3779 / RFC 6487). - pub fn decode_extn_value(extn_value: &[u8]) -> Result { - parse_as_identifiers(extn_value).map_err(|_| AsResourceSetDecodeError::InvalidEncoding) - } - - pub fn is_asnum_inherit(&self) -> bool { - matches!(self.asnum, Some(AsIdentifierChoice::Inherit)) - } - - pub fn has_any_range(&self) -> bool { - self.asnum.as_ref().map(|c| c.has_range()).unwrap_or(false) - || self.rdi.as_ref().map(|c| c.has_range()).unwrap_or(false) - } - - pub fn asnum_single_id(&self) -> Option { - match self.asnum.as_ref()? { - AsIdentifierChoice::Inherit => None, - AsIdentifierChoice::AsIdsOrRanges(items) => { - if items.len() != 1 { - return None; - } - match &items[0] { - AsIdOrRange::Id(v) => Some(*v), - AsIdOrRange::Range { .. } => None, - } - } - } - } -} - -#[derive(Debug, thiserror::Error)] -pub enum AsResourceSetDecodeError { - #[error("invalid autonomousSysIds encoding (RFC 3779 §3.2.3; RFC 6487 §4.8.11)")] - InvalidEncoding, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub enum AsIdentifierChoice { - Inherit, - AsIdsOrRanges(Vec), -} - -impl AsIdentifierChoice { - pub fn has_range(&self) -> bool { - match self { - AsIdentifierChoice::Inherit => false, - AsIdentifierChoice::AsIdsOrRanges(items) => { - items.iter().any(|i| matches!(i, AsIdOrRange::Range { .. })) - } - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub enum AsIdOrRange { - Id(u32), - Range { min: u32, max: u32 }, -} - -#[derive(Debug, thiserror::Error)] -pub enum ResourceCertificateParseError { - #[error("X.509 parse error: {0} (RFC 5280 §4.1; RFC 6487 §4)")] - Parse(String), - - #[error("trailing bytes after certificate DER: {0} bytes (DER; RFC 5280 §4.1)")] - TrailingBytes(usize), - - #[error("invalid RFC 3779 IP resources extension encoding (RFC 6487 §4.8.10; RFC 3779 §2.2)")] - InvalidIpResourcesEncoding, - - #[error("invalid RFC 3779 AS resources extension encoding (RFC 6487 §4.8.11; RFC 3779 §3.2)")] - InvalidAsResourcesEncoding, -} - -#[derive(Debug, thiserror::Error)] -pub enum ResourceCertificateProfileError { - #[error("{0}")] - InvalidTimeEncoding(#[from] InvalidTimeEncodingError), - - #[error("certificate version must be v3 (RFC 5280 §4.1; RFC 6487 §4)")] - InvalidVersion, - - #[error("signatureAlgorithm does not match tbsCertificate.signature (RFC 5280 §4.1)")] - SignatureAlgorithmMismatch, - - #[error( - "unsupported signature algorithm (expected sha256WithRSAEncryption {OID_SHA256_WITH_RSA_ENCRYPTION}) (RFC 7935 §2; RFC 6487 §4)" - )] - UnsupportedSignatureAlgorithm, - - #[error("invalid signature algorithm parameters (RFC 5280 §4.1.1.2)")] - InvalidSignatureAlgorithmParameters, - - #[error( - "{role} Name strict validation failed: {detail} (RFC 6487 §4.4; RFC 5280 §4.1.2.4/§4.1.2.6)" - )] - StrictName { role: &'static str, detail: String }, - - #[error("duplicate extension: {0} (RFC 5280 §4.2; RFC 6487 §4.8)")] - DuplicateExtension(&'static str), - - #[error("SubjectKeyIdentifier criticality must be non-critical (RFC 6487 §4.8.2)")] - SkiCriticality, - - #[error("SubjectInfoAccess criticality must be non-critical (RFC 6487 §4.8.8)")] - SiaCriticality, - - #[error("certificatePolicies criticality must be critical (RFC 6487 §4.8.9)")] - CertificatePoliciesCriticality, - - #[error("certificatePolicies must be present (RFC 6487 §4.8.9)")] - CertificatePoliciesMissing, - - #[error( - "certificatePolicies must contain RPKI policy OID {OID_CP_IPADDR_ASNUMBER}, got {0} (RFC 6487 §4.8.9)" - )] - InvalidCertificatePolicy(String), - - #[error("certificatePolicies may contain at most one CPS qualifier (RFC 6487 §4.8.9)")] - CertificatePoliciesTooManyQualifiers, - - #[error( - "certificatePolicies qualifier must be id-qt-cps ({OID_QT_CPS}), got {0} (RFC 6487 §4.8.9)" - )] - CertificatePoliciesInvalidQualifier(String), - - #[error("basicConstraints must be present in CA certificates (RFC 6487 §4.8.1)")] - BasicConstraintsMissing, - - #[error("basicConstraints criticality must be critical in CA certificates (RFC 6487 §4.8.1)")] - BasicConstraintsCriticality, - - #[error("basicConstraints cA must be TRUE in CA certificates (RFC 6487 §4.8.1)")] - BasicConstraintsCaFalse, - - #[error( - "basicConstraints pathLenConstraint must be absent in CA certificates (RFC 6487 §4.8.1)" - )] - BasicConstraintsPathLenPresent, - - #[error("basicConstraints must be absent in EE certificates (RFC 6487 §4.8.1)")] - BasicConstraintsEeMustOmit, - - #[error("extension {oid} is not permitted for {role} resource certificates (RFC 6487 §4.8)")] - DisallowedExtension { role: &'static str, oid: String }, - - #[error("autonomousSysIds RDI field must be absent (RFC 6487 §4.8.11; RFC 3779 §3.2.3)")] - AsResourcesRdiPresent, - - #[error( - "SIA id-ad-signedObject accessLocation must be URI (RFC 6487 §4.8.8.2; RFC 5280 §4.2.2.2)" - )] - SignedObjectSiaNotUri, - - #[error("SIA id-ad-signedObject must include at least one rsync:// URI (RFC 6487 §4.8.8.2)")] - SignedObjectSiaNoRsync, - - #[error("ipAddrBlocks criticality must be critical when present (RFC 6487 §4.8.10)")] - IpResourcesCriticality, - - #[error("autonomousSysIds criticality must be critical when present (RFC 6487 §4.8.11)")] - AsResourcesCriticality, - - #[error( - "authorityKeyIdentifier must be present in non-self-signed certificates (RFC 6487 §4.8.3; RFC 5280 §4.2.1.1)" - )] - AkiMissing, - - #[error( - "authorityKeyIdentifier criticality must be non-critical (RFC 6487 §4.8.3; RFC 5280 §4.2.1.1)" - )] - AkiCriticality, - - #[error( - "authorityKeyIdentifier authorityCertIssuer MUST NOT be present (RFC 6487 §4.8.3; RFC 5280 §4.2.1.1)" - )] - AkiAuthorityCertIssuerPresent, - - #[error( - "authorityKeyIdentifier authorityCertSerialNumber MUST NOT be present (RFC 6487 §4.8.3; RFC 5280 §4.2.1.1)" - )] - AkiAuthorityCertSerialPresent, - - #[error( - "self-signed certificate authorityKeyIdentifier must equal subjectKeyIdentifier when present (RFC 6487 §4.8.3)" - )] - AkiSelfSignedNotEqualSki, - - #[error( - "CRLDistributionPoints must be present in non-self-signed certificates (RFC 6487 §4.8.6; RFC 5280 §4.2.1.13)" - )] - CrlDistributionPointsMissing, - - #[error( - "CRLDistributionPoints criticality must be non-critical (RFC 6487 §4.8.6; RFC 5280 §4.2.1.13)" - )] - CrlDistributionPointsCriticality, - - #[error("CRLDistributionPoints MUST be omitted in self-signed certificates (RFC 6487 §4.8.6)")] - CrlDistributionPointsSelfSignedMustOmit, - - #[error("CRLDistributionPoints must contain exactly one DistributionPoint (RFC 6487 §4.8.6)")] - CrlDistributionPointsNotSingle, - - #[error("CRLDistributionPoints distributionPoint field MUST be present (RFC 6487 §4.8.6)")] - CrlDistributionPointsNoDistributionPoint, - - #[error("CRLDistributionPoints reasons field MUST be omitted (RFC 6487 §4.8.6)")] - CrlDistributionPointsHasReasons, - - #[error("CRLDistributionPoints cRLIssuer field MUST be omitted (RFC 6487 §4.8.6)")] - CrlDistributionPointsHasCrlIssuer, - - #[error( - "CRLDistributionPoints distributionPoint MUST contain fullName and MUST NOT contain nameRelativeToCRLIssuer (RFC 6487 §4.8.6)" - )] - CrlDistributionPointsInvalidName, - - #[error( - "CRLDistributionPoints fullName must contain only URI GeneralNames (RFC 6487 §4.8.6; RFC 5280 §4.2.1.6)" - )] - CrlDistributionPointsFullNameNotUri, - - #[error("CRLDistributionPoints must include at least one rsync:// URI (RFC 6487 §4.8.6)")] - CrlDistributionPointsNoRsync, - - #[error( - "authorityInfoAccess must be present in non-self-signed certificates (RFC 6487 §4.8.7; RFC 5280 §4.2.2.1)" - )] - AuthorityInfoAccessMissing, - - #[error( - "authorityInfoAccess criticality must be non-critical (RFC 6487 §4.8.7; RFC 5280 §4.2.2.1)" - )] - AuthorityInfoAccessCriticality, - - #[error("authorityInfoAccess MUST be omitted in self-signed certificates (RFC 6487 §4.8.7)")] - AuthorityInfoAccessSelfSignedMustOmit, - - #[error( - "authorityInfoAccess id-ad-caIssuers accessLocation must be URI (RFC 6487 §4.8.7; RFC 5280 §4.2.2.1)" - )] - AuthorityInfoAccessCaIssuersNotUri, - - #[error("authorityInfoAccess must include at least one id-ad-caIssuers URI (RFC 6487 §4.8.7)")] - AuthorityInfoAccessMissingCaIssuers, - - #[error("authorityInfoAccess must include at least one rsync:// URI (RFC 6487 §4.8.7)")] - AuthorityInfoAccessNoRsync, -} - -#[derive(Debug, thiserror::Error)] -pub enum ResourceCertificateDecodeError { - #[error("{0}")] - Parse(#[from] ResourceCertificateParseError), - - #[error("{0}")] - Validate(#[from] ResourceCertificateProfileError), -} - -pub type ResourceCertificateError = ResourceCertificateDecodeError; - -impl ResourceCertificate { - /// Parse step of scheme A (`parse → validate → verify`). - pub fn parse_der( - der: &[u8], - ) -> Result { - let (rem, cert) = X509Certificate::from_der(der) - .map_err(|e| ResourceCertificateParseError::Parse(e.to_string()))?; - if !rem.is_empty() { - return Err(ResourceCertificateParseError::TrailingBytes(rem.len())); - } - - let validity_not_before = asn1_time_to_model(cert.validity().not_before); - let validity_not_after = asn1_time_to_model(cert.validity().not_after); - - let subject_public_key_info = cert.tbs_certificate.subject_pki.raw.to_vec(); - - let signature_algorithm = algorithm_identifier_value(&cert.signature_algorithm); - let tbs_signature_algorithm = algorithm_identifier_value(&cert.tbs_certificate.signature); - let extensions = parse_extensions_parse(cert.extensions())?; - - Ok(ResourceCertificateParsed { - raw_der: der.to_vec(), - version: cert.version(), - serial_number: cert.tbs_certificate.serial.clone(), - signature_algorithm, - tbs_signature_algorithm, - issuer_name: X509NameDer(cert.issuer().as_raw().to_vec()), - subject_name: X509NameDer(cert.subject().as_raw().to_vec()), - validity_not_before, - validity_not_after, - subject_public_key_info, - extensions, - }) - } - - /// Profile validate step of scheme A (`parse → validate → verify`). - /// - /// `ResourceCertificate` is already profile-validated when constructed via `decode_der()` / - /// `ResourceCertificateParsed::validate_profile()`. - pub fn validate_profile(&self) -> Result<(), ResourceCertificateProfileError> { - Ok(()) - } - - pub fn validate_rfc6487_profile( - &self, - role: ResourceCertificateRole, - ) -> Result<(), ResourceCertificateProfileError> { - let role_name = match role { - ResourceCertificateRole::TrustAnchor => "trust anchor CA", - ResourceCertificateRole::Ca => "CA", - ResourceCertificateRole::SignedObjectEe => "signed-object EE", - ResourceCertificateRole::RouterEe => "router EE", - }; - let ca_role = matches!( - role, - ResourceCertificateRole::TrustAnchor | ResourceCertificateRole::Ca - ); - - if ca_role { - let constraints = self - .tbs - .extensions - .basic_constraints - .as_ref() - .ok_or(ResourceCertificateProfileError::BasicConstraintsMissing)?; - if !constraints.critical { - return Err(ResourceCertificateProfileError::BasicConstraintsCriticality); - } - if !constraints.ca { - return Err(ResourceCertificateProfileError::BasicConstraintsCaFalse); - } - if constraints.path_len_constraint.is_some() { - return Err(ResourceCertificateProfileError::BasicConstraintsPathLenPresent); - } - } else if self.tbs.extensions.basic_constraints.is_some() { - return Err(ResourceCertificateProfileError::BasicConstraintsEeMustOmit); - } - - for oid in &self.tbs.extensions.extension_oids { - if !is_permitted_extension(oid, role) { - return Err(ResourceCertificateProfileError::DisallowedExtension { - role: role_name, - oid: oid.clone(), - }); - } - } - - let policies = self - .tbs - .extensions - .certificate_policies - .as_ref() - .ok_or(ResourceCertificateProfileError::CertificatePoliciesMissing)?; - if policies.policy_oid != OID_CP_IPADDR_ASNUMBER { - return Err(ResourceCertificateProfileError::InvalidCertificatePolicy( - policies.policy_oid.clone(), - )); - } - if policies.qualifier_oids.len() > 1 { - return Err(ResourceCertificateProfileError::CertificatePoliciesTooManyQualifiers); - } - if let Some(qualifier_oid) = policies.qualifier_oids.first() { - if qualifier_oid != OID_QT_CPS { - return Err( - ResourceCertificateProfileError::CertificatePoliciesInvalidQualifier( - qualifier_oid.clone(), - ), - ); - } - } - - if self - .tbs - .extensions - .as_resources - .as_ref() - .is_some_and(|resources| resources.rdi.is_some()) - { - return Err(ResourceCertificateProfileError::AsResourcesRdiPresent); - } - - Ok(()) - } - - pub fn validate_strict_name_profile(&self) -> Result<(), ResourceCertificateProfileError> { - validate_strict_rpki_name(&self.tbs.issuer_name, "issuer")?; - validate_strict_rpki_name(&self.tbs.subject_name, "subject")?; - Ok(()) - } - - /// Decode a resource certificate (`parse + validate`). - pub fn decode_der(der: &[u8]) -> Result { - Ok(Self::parse_der(der)?.validate_profile()?) - } - - pub fn decode_der_with_strict_name(der: &[u8]) -> Result { - let cert = Self::decode_der(der)?; - cert.validate_strict_name_profile()?; - Ok(cert) - } - - /// Backwards-compatible helper (historical name). - pub fn from_der(der: &[u8]) -> Result { - Self::decode_der(der) - } -} - -fn is_permitted_extension(oid: &str, role: ResourceCertificateRole) -> bool { - matches!( - oid, - OID_BASIC_CONSTRAINTS - | OID_KEY_USAGE - | OID_SUBJECT_KEY_IDENTIFIER - | OID_AUTHORITY_KEY_IDENTIFIER - | OID_CRL_DISTRIBUTION_POINTS - | OID_AUTHORITY_INFO_ACCESS - | OID_SUBJECT_INFO_ACCESS - | OID_CERTIFICATE_POLICIES - | OID_IP_ADDR_BLOCKS - | OID_AUTONOMOUS_SYS_IDS - ) || (role == ResourceCertificateRole::RouterEe && oid == OID_EXTENDED_KEY_USAGE) -} - -fn validate_strict_rpki_name( - name: &X509NameDer, - role: &'static str, -) -> Result<(), ResourceCertificateProfileError> { - let mut name_seq = DerReader::new(name.as_raw()) - .take_sequence() - .map_err(|e| ResourceCertificateProfileError::StrictName { role, detail: e })?; - - let mut common_name_count = 0usize; - let mut serial_number_count = 0usize; - - while !name_seq.is_empty() { - let set_bytes = name_seq - .take_tag(0x31) - .map_err(|e| ResourceCertificateProfileError::StrictName { role, detail: e })?; - let mut rdn_set = DerReader::new(set_bytes); - if rdn_set.is_empty() { - return Err(ResourceCertificateProfileError::StrictName { - role, - detail: "RelativeDistinguishedName SET is empty".to_string(), - }); - } - - while !rdn_set.is_empty() { - let mut attr = rdn_set - .take_sequence() - .map_err(|e| ResourceCertificateProfileError::StrictName { role, detail: e })?; - let oid = attr - .take_tag(0x06) - .map_err(|e| ResourceCertificateProfileError::StrictName { role, detail: e })?; - let (value_tag, _value) = attr - .take_any() - .map_err(|e| ResourceCertificateProfileError::StrictName { role, detail: e })?; - if !attr.is_empty() { - return Err(ResourceCertificateProfileError::StrictName { - role, - detail: "AttributeTypeAndValue must be SEQUENCE of 2".to_string(), - }); - } - - match oid { - // 2.5.4.3 commonName - &[0x55, 0x04, 0x03] => { - common_name_count += 1; - if value_tag != 0x13 { - return Err(ResourceCertificateProfileError::StrictName { - role, - detail: format!( - "commonName must be PrintableString, got tag 0x{value_tag:02X}" - ), - }); - } - } - // 2.5.4.5 serialNumber - &[0x55, 0x04, 0x05] => { - serial_number_count += 1; - if value_tag != 0x13 { - return Err(ResourceCertificateProfileError::StrictName { - role, - detail: format!( - "serialNumber must be PrintableString, got tag 0x{value_tag:02X}" - ), - }); - } - } - _ => {} - } - } - } - - if common_name_count != 1 { - return Err(ResourceCertificateProfileError::StrictName { - role, - detail: format!("commonName must appear exactly once, got {common_name_count}"), - }); - } - if serial_number_count > 1 { - return Err(ResourceCertificateProfileError::StrictName { - role, - detail: format!("serialNumber must appear at most once, got {serial_number_count}"), - }); - } - Ok(()) -} - -#[cfg(test)] -mod strict_name_tests { - use super::*; - - fn name_with_attrs(attrs: &[(&[u8], u8, &[u8])]) -> X509NameDer { - let mut rdns = Vec::new(); - for (oid, tag, value) in attrs { - let mut attr = Vec::new(); - attr.extend(der_tlv(0x06, oid)); - attr.extend(der_tlv(*tag, value)); - let attr = der_tlv(0x30, &attr); - let rdn = der_tlv(0x31, &attr); - rdns.extend(rdn); - } - X509NameDer(der_tlv(0x30, &rdns)) - } - - fn der_tlv(tag: u8, value: &[u8]) -> Vec { - let mut out = vec![tag]; - encode_len(value.len(), &mut out); - out.extend_from_slice(value); - out - } - - fn encode_len(len: usize, out: &mut Vec) { - if len < 0x80 { - out.push(len as u8); - return; - } - let mut bytes = Vec::new(); - let mut value = len; - while value > 0 { - bytes.push((value & 0xFF) as u8); - value >>= 8; - } - bytes.reverse(); - out.push(0x80 | bytes.len() as u8); - out.extend(bytes); - } - - #[test] - fn strict_name_accepts_printable_common_name_and_serial_number() { - let name = name_with_attrs(&[ - (&[0x55, 0x04, 0x03], 0x13, b"CN1"), - (&[0x55, 0x04, 0x05], 0x13, b"SN1"), - ]); - validate_strict_rpki_name(&name, "subject").expect("strict name"); - } - - #[test] - fn strict_name_rejects_utf8_common_name() { - let name = name_with_attrs(&[(&[0x55, 0x04, 0x03], 0x0C, b"CN1")]); - let err = validate_strict_rpki_name(&name, "subject").expect_err("strict name fails"); - assert!(err.to_string().contains("PrintableString"), "{err}"); - } - - #[test] - fn strict_name_rejects_duplicate_common_name() { - let name = name_with_attrs(&[ - (&[0x55, 0x04, 0x03], 0x13, b"CN1"), - (&[0x55, 0x04, 0x03], 0x13, b"CN2"), - ]); - let err = validate_strict_rpki_name(&name, "subject").expect_err("strict name fails"); - assert!(err.to_string().contains("exactly once"), "{err}"); - } - - #[test] - fn profile_rejects_rfc8360_v2_policy_oid() { - let extensions = RcExtensionsParsed { - basic_constraints: vec![BasicConstraintsProfile { - ca: true, - critical: true, - path_len_constraint: None, - }], - subject_key_identifier: Vec::new(), - authority_key_identifier: Vec::new(), - crl_distribution_points: Vec::new(), - authority_info_access: Vec::new(), - subject_info_access: Vec::new(), - certificate_policies: vec![( - vec![CertificatePoliciesProfile { - policy_oid: "1.3.6.1.5.5.7.14.3".to_string(), - qualifier_oids: Vec::new(), - }], - true, - )], - extension_oids: Vec::new(), - ip_resources: Vec::new(), - as_resources: Vec::new(), - }; - - let err = extensions - .validate_profile(true) - .expect_err("v2 policy OID must remain invalid"); - assert!( - matches!(&err, ResourceCertificateProfileError::InvalidCertificatePolicy(oid) if oid == "1.3.6.1.5.5.7.14.3"), - "{err}" - ); - } -} - -impl ResourceCertificateParsed { - pub fn validate_profile(self) -> Result { - let version = match self.version { - X509Version::V3 => 2u32, - _ => return Err(ResourceCertificateProfileError::InvalidVersion), - }; - - self.validity_not_before - .validate_encoding_rfc5280("notBefore")?; - self.validity_not_after - .validate_encoding_rfc5280("notAfter")?; - - if self.signature_algorithm != self.tbs_signature_algorithm { - return Err(ResourceCertificateProfileError::SignatureAlgorithmMismatch); - } - if self.signature_algorithm.oid != OID_SHA256_WITH_RSA_ENCRYPTION { - return Err(ResourceCertificateProfileError::UnsupportedSignatureAlgorithm); - } - if !self.signature_algorithm.params_absent_or_null() { - return Err(ResourceCertificateProfileError::InvalidSignatureAlgorithmParameters); - } - - let is_self_signed = self.issuer_name == self.subject_name; - let extensions = self.extensions.validate_profile(is_self_signed)?; - let kind = if extensions.basic_constraints_ca { - ResourceCertKind::Ca - } else { - ResourceCertKind::Ee - }; - - Ok(ResourceCertificate { - raw_der: self.raw_der, - tbs: RpkixTbsCertificate { - version, - serial_number: self.serial_number, - signature_algorithm: self.signature_algorithm.oid, - issuer_name: self.issuer_name, - subject_name: self.subject_name, - validity_not_before: self.validity_not_before.utc, - validity_not_after: self.validity_not_after.utc, - subject_public_key_info: self.subject_public_key_info, - extensions, - }, - kind, - }) - } -} - -impl RcExtensionsParsed { - pub fn validate_profile( - self, - is_self_signed: bool, - ) -> Result { - // NOTE(perf): `self` is consumed. Prefer moving decoded fields out rather than cloning, - // especially for large resource sets and URI lists. - let RcExtensionsParsed { - basic_constraints, - subject_key_identifier, - authority_key_identifier, - crl_distribution_points, - authority_info_access, - subject_info_access, - certificate_policies, - extension_oids, - ip_resources, - as_resources, - } = self; - - if basic_constraints.len() > 1 { - return Err(ResourceCertificateProfileError::DuplicateExtension( - "basicConstraints", - )); - } - let basic_constraints = basic_constraints.into_iter().next(); - let basic_constraints_ca = basic_constraints.as_ref().is_some_and(|bc| bc.ca); - - let subject_key_identifier = match subject_key_identifier.len() { - 0 => None, - 1 => { - let (ski, critical) = subject_key_identifier.into_iter().next().expect("len==1"); - if critical { - return Err(ResourceCertificateProfileError::SkiCriticality); - } - Some(ski) - } - _ => { - return Err(ResourceCertificateProfileError::DuplicateExtension( - "subjectKeyIdentifier", - )); - } - }; - - let authority_key_identifier = match authority_key_identifier.len() { - 0 => { - if is_self_signed { - None - } else { - return Err(ResourceCertificateProfileError::AkiMissing); - } - } - 1 => { - let (aki, critical) = authority_key_identifier.into_iter().next().expect("len==1"); - if critical { - return Err(ResourceCertificateProfileError::AkiCriticality); - } - if aki.has_authority_cert_issuer { - return Err(ResourceCertificateProfileError::AkiAuthorityCertIssuerPresent); - } - if aki.has_authority_cert_serial { - return Err(ResourceCertificateProfileError::AkiAuthorityCertSerialPresent); - } - let keyid = aki.key_identifier; - if is_self_signed { - if let (Some(keyid), Some(ski)) = - (keyid.as_ref(), subject_key_identifier.as_ref()) - { - if keyid != ski { - return Err(ResourceCertificateProfileError::AkiSelfSignedNotEqualSki); - } - } - } else if keyid.is_none() { - return Err(ResourceCertificateProfileError::AkiMissing); - } - keyid - } - _ => { - return Err(ResourceCertificateProfileError::DuplicateExtension( - "authorityKeyIdentifier", - )); - } - }; - - let crl_distribution_points_uris = match crl_distribution_points.len() { - 0 => { - if is_self_signed { - None - } else { - return Err(ResourceCertificateProfileError::CrlDistributionPointsMissing); - } - } - 1 => { - let (crldp, critical) = crl_distribution_points.into_iter().next().expect("len==1"); - if critical { - return Err(ResourceCertificateProfileError::CrlDistributionPointsCriticality); - } - if is_self_signed { - return Err( - ResourceCertificateProfileError::CrlDistributionPointsSelfSignedMustOmit, - ); - } - if crldp.distribution_points.len() != 1 { - return Err(ResourceCertificateProfileError::CrlDistributionPointsNotSingle); - } - let dp = crldp - .distribution_points - .into_iter() - .next() - .expect("len==1"); - if dp.reasons_present { - return Err(ResourceCertificateProfileError::CrlDistributionPointsHasReasons); - } - if dp.crl_issuer_present { - return Err(ResourceCertificateProfileError::CrlDistributionPointsHasCrlIssuer); - } - if !dp.distribution_point_present { - return Err( - ResourceCertificateProfileError::CrlDistributionPointsNoDistributionPoint, - ); - } - if dp.name_relative_to_crl_issuer_present || !dp.full_name_present { - return Err(ResourceCertificateProfileError::CrlDistributionPointsInvalidName); - } - if dp.full_name_not_uri { - return Err( - ResourceCertificateProfileError::CrlDistributionPointsFullNameNotUri, - ); - } - if !dp.full_name_uris.iter().any(|u| u.starts_with("rsync://")) { - return Err(ResourceCertificateProfileError::CrlDistributionPointsNoRsync); - } - Some(dp.full_name_uris) - } - _ => { - return Err(ResourceCertificateProfileError::DuplicateExtension( - "cRLDistributionPoints", - )); - } - }; - - let ca_issuers_uris = match authority_info_access.len() { - 0 => { - if is_self_signed { - None - } else { - return Err(ResourceCertificateProfileError::AuthorityInfoAccessMissing); - } - } - 1 => { - let (aia, critical) = authority_info_access.into_iter().next().expect("len==1"); - if critical { - return Err(ResourceCertificateProfileError::AuthorityInfoAccessCriticality); - } - if is_self_signed { - return Err( - ResourceCertificateProfileError::AuthorityInfoAccessSelfSignedMustOmit, - ); - } - if aia.ca_issuers_access_location_not_uri { - return Err( - ResourceCertificateProfileError::AuthorityInfoAccessCaIssuersNotUri, - ); - } - if aia.ca_issuers_uris.is_empty() { - return Err( - ResourceCertificateProfileError::AuthorityInfoAccessMissingCaIssuers, - ); - } - if !aia - .ca_issuers_uris - .iter() - .any(|u| u.starts_with("rsync://")) - { - return Err(ResourceCertificateProfileError::AuthorityInfoAccessNoRsync); - } - Some(aia.ca_issuers_uris) - } - _ => { - return Err(ResourceCertificateProfileError::DuplicateExtension( - "authorityInfoAccess", - )); - } - }; - - let subject_info_access = match subject_info_access.len() { - 0 => None, - 1 => { - let (sia, critical) = subject_info_access.into_iter().next().expect("len==1"); - if critical { - return Err(ResourceCertificateProfileError::SiaCriticality); - } - if sia.signed_object_access_location_not_uri { - return Err(ResourceCertificateProfileError::SignedObjectSiaNotUri); - } - if !sia.signed_object_uris.is_empty() - && !sia - .signed_object_uris - .iter() - .any(|u| u.starts_with("rsync://")) - { - return Err(ResourceCertificateProfileError::SignedObjectSiaNoRsync); - } - if sia.signed_object_uris.is_empty() { - Some(SubjectInfoAccess::Ca(SubjectInfoAccessCa { - access_descriptions: sia.access_descriptions, - })) - } else { - Some(SubjectInfoAccess::Ee(SubjectInfoAccessEe { - signed_object_uris: sia.signed_object_uris, - access_descriptions: sia.access_descriptions, - })) - } - } - _ => { - return Err(ResourceCertificateProfileError::DuplicateExtension( - "subjectInfoAccess", - )); - } - }; - - let certificate_policies = match certificate_policies.len() { - 0 => None, - 1 => { - let (policies, critical) = certificate_policies.into_iter().next().expect("len==1"); - if !critical { - return Err(ResourceCertificateProfileError::CertificatePoliciesCriticality); - } - if policies.len() != 1 { - return Err(ResourceCertificateProfileError::InvalidCertificatePolicy( - "expected exactly one policy".into(), - )); - } - let policy = policies.into_iter().next().expect("len==1"); - if policy.policy_oid != OID_CP_IPADDR_ASNUMBER { - return Err(ResourceCertificateProfileError::InvalidCertificatePolicy( - policy.policy_oid, - )); - } - Some(policy) - } - _ => { - return Err(ResourceCertificateProfileError::DuplicateExtension( - "certificatePolicies", - )); - } - }; - - let ip_resources = match ip_resources.len() { - 0 => None, - 1 => { - let (ip, critical) = ip_resources.into_iter().next().expect("len==1"); - if !critical { - return Err(ResourceCertificateProfileError::IpResourcesCriticality); - } - Some(ip) - } - _ => { - return Err(ResourceCertificateProfileError::DuplicateExtension( - "ipAddrBlocks", - )); - } - }; - - let as_resources = match as_resources.len() { - 0 => None, - 1 => { - let (asn, critical) = as_resources.into_iter().next().expect("len==1"); - if !critical { - return Err(ResourceCertificateProfileError::AsResourcesCriticality); - } - Some(asn) - } - _ => { - return Err(ResourceCertificateProfileError::DuplicateExtension( - "autonomousSysIds", - )); - } - }; - - Ok(RcExtensions { - basic_constraints_ca, - basic_constraints, - subject_key_identifier, - authority_key_identifier, - crl_distribution_points_uris, - ca_issuers_uris, - subject_info_access, - certificate_policies_oid: certificate_policies - .as_ref() - .map(|_| OID_CP_IPADDR_ASNUMBER.to_string()), - certificate_policies, - extension_oids, - ip_resources, - as_resources, - }) - } -} - -fn algorithm_identifier_value( - ai: &x509_parser::x509::AlgorithmIdentifier<'_>, -) -> AlgorithmIdentifierValue { - let parameters = ai.parameters.as_ref().map(|p| AlgorithmParametersValue { - class: p.class(), - tag: p.tag(), - data: p.as_bytes().to_vec(), - }); - // NOTE(perf): Avoid `to_id_string()` allocations for the algorithms we expect - // in RPKI resource certificates. Fall back to `to_id_string()` for unexpected - // algorithms (mostly error paths). - let oid = if ai.algorithm.as_bytes() == OID_SHA256_WITH_RSA_ENCRYPTION_RAW { - OID_SHA256_WITH_RSA_ENCRYPTION.to_string() - } else { - ai.algorithm.to_id_string() - }; - AlgorithmIdentifierValue { oid, parameters } -} - -fn parse_extensions_parse( - exts: &[X509Extension<'_>], -) -> Result { - let mut basic_constraints: Vec = Vec::new(); - let mut ski: Vec<(Vec, bool)> = Vec::new(); - let mut aki: Vec<(AuthorityKeyIdentifierParsed, bool)> = Vec::new(); - let mut crldp: Vec<(CrlDistributionPointsParsed, bool)> = Vec::new(); - let mut aia: Vec<(AuthorityInfoAccessParsed, bool)> = Vec::new(); - let mut sia: Vec<(SubjectInfoAccessParsed, bool)> = Vec::new(); - let mut cert_policies: Vec<(Vec, bool)> = Vec::new(); - let mut extension_oids: Vec = Vec::with_capacity(exts.len()); - - let mut ip_resources: Vec<(IpResourceSet, bool)> = Vec::new(); - let mut as_resources: Vec<(AsResourceSet, bool)> = Vec::new(); - - for ext in exts { - let oid = ext.oid.as_bytes(); - extension_oids.push(ext.oid.to_id_string()); - if oid == OID_BASIC_CONSTRAINTS_RAW { - let ParsedExtension::BasicConstraints(bc) = ext.parsed_extension() else { - return Err(ResourceCertificateParseError::Parse( - "basicConstraints parse failed".into(), - )); - }; - basic_constraints.push(BasicConstraintsProfile { - ca: bc.ca, - critical: ext.critical, - path_len_constraint: bc.path_len_constraint, - }); - } else if oid == OID_SUBJECT_KEY_IDENTIFIER_RAW { - let ParsedExtension::SubjectKeyIdentifier(s) = ext.parsed_extension() else { - return Err(ResourceCertificateParseError::Parse( - "subjectKeyIdentifier parse failed".into(), - )); - }; - ski.push((s.0.to_vec(), ext.critical)); - } else if oid == OID_AUTHORITY_KEY_IDENTIFIER_RAW { - let ParsedExtension::AuthorityKeyIdentifier(a) = ext.parsed_extension() else { - return Err(ResourceCertificateParseError::Parse( - "authorityKeyIdentifier parse failed".into(), - )); - }; - aki.push(( - AuthorityKeyIdentifierParsed { - key_identifier: a.key_identifier.as_ref().map(|k| k.0.to_vec()), - has_authority_cert_issuer: a.authority_cert_issuer.is_some(), - has_authority_cert_serial: a.authority_cert_serial.is_some(), - }, - ext.critical, - )); - } else if oid == OID_CRL_DISTRIBUTION_POINTS_RAW { - let ParsedExtension::CRLDistributionPoints(p) = ext.parsed_extension() else { - return Err(ResourceCertificateParseError::Parse( - "cRLDistributionPoints parse failed".into(), - )); - }; - crldp.push((parse_crldp_parse(p)?, ext.critical)); - } else if oid == OID_AUTHORITY_INFO_ACCESS_RAW { - let ParsedExtension::AuthorityInfoAccess(p) = ext.parsed_extension() else { - return Err(ResourceCertificateParseError::Parse( - "authorityInfoAccess parse failed".into(), - )); - }; - aia.push((parse_aia_parse(p.accessdescs.as_slice())?, ext.critical)); - } else if oid == OID_SUBJECT_INFO_ACCESS_RAW { - let ParsedExtension::SubjectInfoAccess(s) = ext.parsed_extension() else { - return Err(ResourceCertificateParseError::Parse( - "subjectInfoAccess parse failed".into(), - )); - }; - sia.push((parse_sia_parse(s.accessdescs.as_slice())?, ext.critical)); - } else if oid == OID_CERTIFICATE_POLICIES_RAW { - let ParsedExtension::CertificatePolicies(cp) = ext.parsed_extension() else { - return Err(ResourceCertificateParseError::Parse( - "certificatePolicies parse failed".into(), - )); - }; - let mut policies: Vec = Vec::with_capacity(cp.len()); - for p in cp.iter() { - let b = p.policy_id.as_bytes(); - let policy_oid = if b == OID_CP_IPADDR_ASNUMBER_RAW { - OID_CP_IPADDR_ASNUMBER.to_string() - } else { - p.policy_id.to_id_string() - }; - let qualifier_oids = p - .policy_qualifiers - .as_ref() - .map(|qualifiers| { - qualifiers - .iter() - .map(|qualifier| qualifier.policy_qualifier_id.to_id_string()) - .collect() - }) - .unwrap_or_default(); - policies.push(CertificatePoliciesProfile { - policy_oid, - qualifier_oids, - }); - } - cert_policies.push((policies, ext.critical)); - } else if oid == OID_IP_ADDR_BLOCKS_RAW { - let parsed = IpResourceSet::decode_extn_value(ext.value) - .map_err(|_e| ResourceCertificateParseError::InvalidIpResourcesEncoding)?; - ip_resources.push((parsed, ext.critical)); - } else if oid == OID_AUTONOMOUS_SYS_IDS_RAW { - let parsed = AsResourceSet::decode_extn_value(ext.value) - .map_err(|_e| ResourceCertificateParseError::InvalidAsResourcesEncoding)?; - as_resources.push((parsed, ext.critical)); - } - } - - Ok(RcExtensionsParsed { - basic_constraints, - subject_key_identifier: ski, - authority_key_identifier: aki, - crl_distribution_points: crldp, - authority_info_access: aia, - subject_info_access: sia, - certificate_policies: cert_policies, - extension_oids, - ip_resources, - as_resources, - }) -} - -fn parse_aia_parse( - access: &[x509_parser::extensions::AccessDescription<'_>], -) -> Result { - let mut ca_issuers_uris: Vec = Vec::new(); - let mut ca_issuers_access_location_not_uri = false; - - for ad in access { - if ad.access_method.as_bytes() != OID_AD_CA_ISSUERS_RAW { - continue; - } - let uri = match &ad.access_location { - x509_parser::extensions::GeneralName::URI(u) => u, - _ => { - ca_issuers_access_location_not_uri = true; - continue; - } - }; - ca_issuers_uris.push(uri.to_string()); - } - - Ok(AuthorityInfoAccessParsed { - ca_issuers_uris, - ca_issuers_access_location_not_uri, - }) -} - -fn parse_crldp_parse( - crldp: &x509_parser::extensions::CRLDistributionPoints<'_>, -) -> Result { - let mut out: Vec = Vec::new(); - for p in crldp.iter() { - let mut full_name_uris: Vec = Vec::new(); - let mut full_name_not_uri = false; - let mut full_name_present = false; - let mut name_relative_to_crl_issuer_present = false; - let mut distribution_point_present = false; - - if let Some(dp) = &p.distribution_point { - distribution_point_present = true; - match dp { - x509_parser::extensions::DistributionPointName::FullName(names) => { - full_name_present = true; - for n in names { - match n { - x509_parser::extensions::GeneralName::URI(u) => { - full_name_uris.push(u.to_string()); - } - _ => { - full_name_not_uri = true; - } - } - } - } - x509_parser::extensions::DistributionPointName::NameRelativeToCRLIssuer(_) => { - name_relative_to_crl_issuer_present = true; - } - } - } - - out.push(CrlDistributionPointParsed { - distribution_point_present, - reasons_present: p.reasons.is_some(), - crl_issuer_present: p.crl_issuer.is_some(), - name_relative_to_crl_issuer_present, - full_name_uris, - full_name_not_uri, - full_name_present, - }); - } - Ok(CrlDistributionPointsParsed { - distribution_points: out, - }) -} - -fn parse_sia_parse( - access: &[x509_parser::extensions::AccessDescription<'_>], -) -> Result { - let mut all = Vec::with_capacity(access.len()); - let mut signed_object_uris: Vec = Vec::new(); - let mut signed_object_access_location_not_uri = false; - - for ad in access { - let access_method_oid = if ad.access_method.as_bytes() == OID_AD_CA_REPOSITORY_RAW { - OID_AD_CA_REPOSITORY.to_string() - } else if ad.access_method.as_bytes() == OID_AD_RPKI_MANIFEST_RAW { - OID_AD_RPKI_MANIFEST.to_string() - } else if ad.access_method.as_bytes() == OID_AD_RPKI_NOTIFY_RAW { - OID_AD_RPKI_NOTIFY.to_string() - } else if ad.access_method.as_bytes() == OID_AD_SIGNED_OBJECT_RAW { - OID_AD_SIGNED_OBJECT.to_string() - } else { - ad.access_method.to_id_string() - }; - let is_signed_object = access_method_oid == OID_AD_SIGNED_OBJECT; - let uri = match &ad.access_location { - x509_parser::extensions::GeneralName::URI(u) => u, - _ => { - if is_signed_object { - signed_object_access_location_not_uri = true; - } - continue; - } - }; - if is_signed_object { - signed_object_uris.push(uri.to_string()); - } - all.push(AccessDescription { - access_method_oid, - access_location: uri.to_string(), - }); - } - - Ok(SubjectInfoAccessParsed { - access_descriptions: all, - signed_object_uris, - signed_object_access_location_not_uri, - }) -} - -fn parse_ip_addr_blocks(ext_value: &[u8]) -> Result { - let (rem, obj) = parse_der(ext_value).map_err(|_| ())?; - if !rem.is_empty() { - return Err(()); - } - let seq = obj.as_sequence().map_err(|_| ())?; - let mut families = Vec::with_capacity(seq.len()); - for fam in seq { - let fam_seq = fam.as_sequence().map_err(|_| ())?; - if fam_seq.len() != 2 { - return Err(()); - } - let af_bytes = fam_seq[0].as_slice().map_err(|_| ())?; - if af_bytes.len() != 2 { - return Err(()); - } - let afi = match af_bytes { - [0x00, 0x01] => Afi::Ipv4, - [0x00, 0x02] => Afi::Ipv6, - _ => return Err(()), - }; - - let choice = match &fam_seq[1].content { - BerObjectContent::Null => IpAddressChoice::Inherit, - BerObjectContent::Sequence(_) => { - let items_seq = fam_seq[1].as_sequence().map_err(|_| ())?; - let mut items = Vec::with_capacity(items_seq.len()); - for item in items_seq { - items.push(parse_ip_address_or_range(afi, item)?); - } - IpAddressChoice::AddressesOrRanges(items) - } - _ => return Err(()), - }; - families.push(IpAddressFamily { afi, choice }); - } - Ok(IpResourceSet { families }) -} - -fn parse_ip_address_or_range(afi: Afi, obj: &DerObject<'_>) -> Result { - match &obj.content { - BerObjectContent::BitString(_, _) => { - Ok(IpAddressOrRange::Prefix(parse_ip_prefix(afi, obj)?)) - } - BerObjectContent::Sequence(_) => { - let seq = obj.as_sequence().map_err(|_| ())?; - if seq.len() != 2 { - return Err(()); - } - let min = parse_ip_address_bound(afi, &seq[0], false)?; - let max = parse_ip_address_bound(afi, &seq[1], true)?; - Ok(IpAddressOrRange::Range(IpAddressRange { min, max })) - } - _ => Err(()), - } -} - -fn parse_ip_prefix(afi: Afi, obj: &DerObject<'_>) -> Result { - let (unused_bits, bytes) = match &obj.content { - BerObjectContent::BitString(unused, bso) => (*unused, bso.data.to_vec()), - _ => return Err(()), - }; - if unused_bits > 7 { - return Err(()); - } - if !bytes.is_empty() && unused_bits != 0 { - let mask = (1u8 << unused_bits) - 1; - if (bytes[bytes.len() - 1] & mask) != 0 { - return Err(()); - } - } else if bytes.is_empty() && unused_bits != 0 { - return Err(()); - } - let prefix_len = (bytes.len() * 8) - .checked_sub(unused_bits as usize) - .ok_or(())? as u16; - if prefix_len > afi.ub() { - return Err(()); - } - let addr = canonicalize_prefix_addr(afi, prefix_len, &bytes); - Ok(IpPrefix { - afi, - prefix_len, - addr, - }) -} - -/// Parse an RFC 3779 `IPAddress` BIT STRING into an address-like byte array. -/// -/// When used as an `IPAddressRange` endpoint, RFC 3779 allows endpoints to be encoded with -/// fewer than `ub` bits. In that case, the missing bits are interpreted as 0s for the lower -/// bound and 1s for the upper bound. This is essential to correctly interpret ranges that -/// are expressed on non-octet boundaries. -fn parse_ip_address_bound( - afi: Afi, - obj: &DerObject<'_>, - fill_remaining_ones: bool, -) -> Result, ()> { - let (unused_bits, bytes) = match &obj.content { - BerObjectContent::BitString(unused, bso) => (*unused, bso.data.to_vec()), - _ => return Err(()), - }; - if unused_bits > 7 { - return Err(()); - } - if !bytes.is_empty() && unused_bits != 0 { - let mask = (1u8 << unused_bits) - 1; - if (bytes[bytes.len() - 1] & mask) != 0 { - return Err(()); - } - } else if bytes.is_empty() && unused_bits != 0 { - return Err(()); - } - - let bit_len: u16 = (bytes.len() * 8) - .checked_sub(unused_bits as usize) - .ok_or(())? - .try_into() - .map_err(|_| ())?; - if bit_len > afi.ub() { - return Err(()); - } - - let mut out = vec![0u8; afi.octets_len()]; - let copy_len = bytes.len().min(out.len()); - out[..copy_len].copy_from_slice(&bytes[..copy_len]); - - if fill_remaining_ones { - if bit_len == 0 { - for b in &mut out { - *b = 0xFF; - } - return Ok(out); - } - - let last_bit = (bit_len - 1) as usize; - let last_byte = last_bit / 8; - let rem = (bit_len % 8) as u8; - - if rem != 0 && last_byte < out.len() { - // Set the (8-rem) trailing bits in the last byte to 1. - let mask: u8 = (1u8 << (8 - rem)) - 1; - out[last_byte] |= mask; - } - for b in out.iter_mut().skip(last_byte + 1) { - *b = 0xFF; - } - } - - Ok(out) -} - -fn parse_as_identifiers(ext_value: &[u8]) -> Result { - let (rem, obj) = parse_der(ext_value).map_err(|_| ())?; - if !rem.is_empty() { - return Err(()); - } - let seq = obj.as_sequence().map_err(|_| ())?; - let mut asnum: Option = None; - let mut rdi: Option = None; - for item in seq { - if item.class() != Class::ContextSpecific { - return Err(()); - } - match item.tag() { - Tag(0) => { - if asnum.is_some() { - return Err(()); - } - let inner = parse_explicit_inner(item)?; - asnum = Some(parse_as_identifier_choice(&inner)?); - } - Tag(1) => { - if rdi.is_some() { - return Err(()); - } - let inner = parse_explicit_inner(item)?; - rdi = Some(parse_as_identifier_choice(&inner)?); - } - _ => return Err(()), - } - } - Ok(AsResourceSet { asnum, rdi }) -} - -fn parse_explicit_inner<'a>(obj: &'a DerObject<'a>) -> Result, ()> { - let inner_der = obj.as_slice().map_err(|_| ())?; - let (rem, inner) = parse_der(inner_der).map_err(|_| ())?; - if !rem.is_empty() { - return Err(()); - } - Ok(inner) -} - -fn parse_as_identifier_choice(obj: &DerObject<'_>) -> Result { - match &obj.content { - BerObjectContent::Null => Ok(AsIdentifierChoice::Inherit), - BerObjectContent::Sequence(_) => { - let seq = obj.as_sequence().map_err(|_| ())?; - let mut items = Vec::with_capacity(seq.len()); - for item in seq { - items.push(parse_as_id_or_range(item)?); - } - Ok(AsIdentifierChoice::AsIdsOrRanges(items)) - } - _ => Err(()), - } -} - -fn parse_as_id_or_range(obj: &DerObject<'_>) -> Result { - match &obj.content { - BerObjectContent::Integer(_) => { - let v = obj.as_u64().map_err(|_| ())?; - if v > u32::MAX as u64 { - return Err(()); - } - Ok(AsIdOrRange::Id(v as u32)) - } - BerObjectContent::Sequence(_) => { - let seq = obj.as_sequence().map_err(|_| ())?; - if seq.len() != 2 { - return Err(()); - } - let min = seq[0].as_u64().map_err(|_| ())?; - let max = seq[1].as_u64().map_err(|_| ())?; - if min > u32::MAX as u64 || max > u32::MAX as u64 || min > max { - return Err(()); - } - Ok(AsIdOrRange::Range { - min: min as u32, - max: max as u32, - }) - } - _ => Err(()), - } -} - -fn canonicalize_prefix_addr(afi: Afi, prefix_len: u16, bytes: &[u8]) -> Vec { - let full_len = afi.octets_len(); - let mut addr = vec![0u8; full_len]; - let copy_len = bytes.len().min(full_len); - addr[..copy_len].copy_from_slice(&bytes[..copy_len]); - - if prefix_len == 0 { - return addr; - } - - let last_prefix_bit = (prefix_len - 1) as usize; - let last_prefix_byte = last_prefix_bit / 8; - let rem = (prefix_len % 8) as u8; - if rem != 0 && last_prefix_byte < addr.len() { - let mask: u8 = 0xFF << (8 - rem); - addr[last_prefix_byte] &= mask; - } - addr -} - -fn prefix_covers(resource: &IpPrefix, subject: &IpPrefix) -> bool { - if resource.afi != subject.afi { - return false; - } - if resource.prefix_len > subject.prefix_len { - return false; - } - let n = resource.prefix_len as usize; - let whole = n / 8; - let rem = (n % 8) as u8; - if resource.addr.len() != subject.addr.len() { - return false; - } - if resource.addr[..whole] != subject.addr[..whole] { - return false; - } - if rem == 0 { - return true; - } - let mask = 0xFFu8 << (8 - rem); - (resource.addr[whole] & mask) == (subject.addr[whole] & mask) -} - -fn prefix_range(afi: Afi, p: &IpPrefix) -> (u128, u128) { - let mut base_bytes = [0u8; 16]; - match afi { - Afi::Ipv4 => { - base_bytes[12..].copy_from_slice(&p.addr[..4]); - } - Afi::Ipv6 => { - base_bytes.copy_from_slice(&p.addr[..16]); - } - } - let base = u128::from_be_bytes(base_bytes); - let host_bits = (afi.ub() - p.prefix_len) as u32; - if host_bits == 0 { - return (base, base); - } - let mask = (1u128 << host_bits) - 1; - (base, base | mask) -} - -fn range_covers_prefix(afi: Afi, r: &IpAddressRange, p: &IpPrefix) -> bool { - let (p_min, p_max) = prefix_range(afi, p); - let r_min = bytes_to_u128(afi, &r.min); - let r_max = bytes_to_u128(afi, &r.max); - r_min <= p_min && p_max <= r_max -} - -fn bytes_to_u128(afi: Afi, bytes: &[u8]) -> u128 { - let mut out = [0u8; 16]; - match afi { - Afi::Ipv4 => { - let copy_len = bytes.len().min(4); - out[12..12 + copy_len].copy_from_slice(&bytes[..copy_len]); - } - Afi::Ipv6 => { - let copy_len = bytes.len().min(16); - out[..copy_len].copy_from_slice(&bytes[..copy_len]); - } - } - u128::from_be_bytes(out) -} +include!("rc/types.rs"); +include!("rc/certificate_validation.rs"); +include!("rc/strict_name_tests.rs"); +include!("rc/parsed_validation.rs"); +include!("rc/parsing.rs"); diff --git a/crates/panda-rpki-validator/src/data_model/rc/certificate_validation.rs b/crates/panda-rpki-validator/src/data_model/rc/certificate_validation.rs new file mode 100644 index 0000000..30bc253 --- /dev/null +++ b/crates/panda-rpki-validator/src/data_model/rc/certificate_validation.rs @@ -0,0 +1,249 @@ +// Resource certificate profile validation and strict-name checks. + +impl ResourceCertificate { + /// Parse step of scheme A (`parse → validate → verify`). + pub fn parse_der( + der: &[u8], + ) -> Result { + let (rem, cert) = X509Certificate::from_der(der) + .map_err(|e| ResourceCertificateParseError::Parse(e.to_string()))?; + if !rem.is_empty() { + return Err(ResourceCertificateParseError::TrailingBytes(rem.len())); + } + + let validity_not_before = asn1_time_to_model(cert.validity().not_before); + let validity_not_after = asn1_time_to_model(cert.validity().not_after); + + let subject_public_key_info = cert.tbs_certificate.subject_pki.raw.to_vec(); + + let signature_algorithm = algorithm_identifier_value(&cert.signature_algorithm); + let tbs_signature_algorithm = algorithm_identifier_value(&cert.tbs_certificate.signature); + let extensions = parse_extensions_parse(cert.extensions())?; + + Ok(ResourceCertificateParsed { + raw_der: der.to_vec(), + version: cert.version(), + serial_number: cert.tbs_certificate.serial.clone(), + signature_algorithm, + tbs_signature_algorithm, + issuer_name: X509NameDer(cert.issuer().as_raw().to_vec()), + subject_name: X509NameDer(cert.subject().as_raw().to_vec()), + validity_not_before, + validity_not_after, + subject_public_key_info, + extensions, + }) + } + + /// Profile validate step of scheme A (`parse → validate → verify`). + /// + /// `ResourceCertificate` is already profile-validated when constructed via `decode_der()` / + /// `ResourceCertificateParsed::validate_profile()`. + pub fn validate_profile(&self) -> Result<(), ResourceCertificateProfileError> { + Ok(()) + } + + pub fn validate_rfc6487_profile( + &self, + role: ResourceCertificateRole, + ) -> Result<(), ResourceCertificateProfileError> { + let role_name = match role { + ResourceCertificateRole::TrustAnchor => "trust anchor CA", + ResourceCertificateRole::Ca => "CA", + ResourceCertificateRole::SignedObjectEe => "signed-object EE", + ResourceCertificateRole::RouterEe => "router EE", + }; + let ca_role = matches!( + role, + ResourceCertificateRole::TrustAnchor | ResourceCertificateRole::Ca + ); + + if ca_role { + let constraints = self + .tbs + .extensions + .basic_constraints + .as_ref() + .ok_or(ResourceCertificateProfileError::BasicConstraintsMissing)?; + if !constraints.critical { + return Err(ResourceCertificateProfileError::BasicConstraintsCriticality); + } + if !constraints.ca { + return Err(ResourceCertificateProfileError::BasicConstraintsCaFalse); + } + if constraints.path_len_constraint.is_some() { + return Err(ResourceCertificateProfileError::BasicConstraintsPathLenPresent); + } + } else if self.tbs.extensions.basic_constraints.is_some() { + return Err(ResourceCertificateProfileError::BasicConstraintsEeMustOmit); + } + + for oid in &self.tbs.extensions.extension_oids { + if !is_permitted_extension(oid, role) { + return Err(ResourceCertificateProfileError::DisallowedExtension { + role: role_name, + oid: oid.clone(), + }); + } + } + + let policies = self + .tbs + .extensions + .certificate_policies + .as_ref() + .ok_or(ResourceCertificateProfileError::CertificatePoliciesMissing)?; + if policies.policy_oid != OID_CP_IPADDR_ASNUMBER { + return Err(ResourceCertificateProfileError::InvalidCertificatePolicy( + policies.policy_oid.clone(), + )); + } + if policies.qualifier_oids.len() > 1 { + return Err(ResourceCertificateProfileError::CertificatePoliciesTooManyQualifiers); + } + if let Some(qualifier_oid) = policies.qualifier_oids.first() { + if qualifier_oid != OID_QT_CPS { + return Err( + ResourceCertificateProfileError::CertificatePoliciesInvalidQualifier( + qualifier_oid.clone(), + ), + ); + } + } + + if self + .tbs + .extensions + .as_resources + .as_ref() + .is_some_and(|resources| resources.rdi.is_some()) + { + return Err(ResourceCertificateProfileError::AsResourcesRdiPresent); + } + + Ok(()) + } + + pub fn validate_strict_name_profile(&self) -> Result<(), ResourceCertificateProfileError> { + validate_strict_rpki_name(&self.tbs.issuer_name, "issuer")?; + validate_strict_rpki_name(&self.tbs.subject_name, "subject")?; + Ok(()) + } + + /// Decode a resource certificate (`parse + validate`). + pub fn decode_der(der: &[u8]) -> Result { + Ok(Self::parse_der(der)?.validate_profile()?) + } + + pub fn decode_der_with_strict_name(der: &[u8]) -> Result { + let cert = Self::decode_der(der)?; + cert.validate_strict_name_profile()?; + Ok(cert) + } + + /// Backwards-compatible helper (historical name). + pub fn from_der(der: &[u8]) -> Result { + Self::decode_der(der) + } +} + +fn is_permitted_extension(oid: &str, role: ResourceCertificateRole) -> bool { + matches!( + oid, + OID_BASIC_CONSTRAINTS + | OID_KEY_USAGE + | OID_SUBJECT_KEY_IDENTIFIER + | OID_AUTHORITY_KEY_IDENTIFIER + | OID_CRL_DISTRIBUTION_POINTS + | OID_AUTHORITY_INFO_ACCESS + | OID_SUBJECT_INFO_ACCESS + | OID_CERTIFICATE_POLICIES + | OID_IP_ADDR_BLOCKS + | OID_AUTONOMOUS_SYS_IDS + ) || (role == ResourceCertificateRole::RouterEe && oid == OID_EXTENDED_KEY_USAGE) +} + +fn validate_strict_rpki_name( + name: &X509NameDer, + role: &'static str, +) -> Result<(), ResourceCertificateProfileError> { + let mut name_seq = DerReader::new(name.as_raw()) + .take_sequence() + .map_err(|e| ResourceCertificateProfileError::StrictName { role, detail: e })?; + + let mut common_name_count = 0usize; + let mut serial_number_count = 0usize; + + while !name_seq.is_empty() { + let set_bytes = name_seq + .take_tag(0x31) + .map_err(|e| ResourceCertificateProfileError::StrictName { role, detail: e })?; + let mut rdn_set = DerReader::new(set_bytes); + if rdn_set.is_empty() { + return Err(ResourceCertificateProfileError::StrictName { + role, + detail: "RelativeDistinguishedName SET is empty".to_string(), + }); + } + + while !rdn_set.is_empty() { + let mut attr = rdn_set + .take_sequence() + .map_err(|e| ResourceCertificateProfileError::StrictName { role, detail: e })?; + let oid = attr + .take_tag(0x06) + .map_err(|e| ResourceCertificateProfileError::StrictName { role, detail: e })?; + let (value_tag, _value) = attr + .take_any() + .map_err(|e| ResourceCertificateProfileError::StrictName { role, detail: e })?; + if !attr.is_empty() { + return Err(ResourceCertificateProfileError::StrictName { + role, + detail: "AttributeTypeAndValue must be SEQUENCE of 2".to_string(), + }); + } + + match oid { + // 2.5.4.3 commonName + &[0x55, 0x04, 0x03] => { + common_name_count += 1; + if value_tag != 0x13 { + return Err(ResourceCertificateProfileError::StrictName { + role, + detail: format!( + "commonName must be PrintableString, got tag 0x{value_tag:02X}" + ), + }); + } + } + // 2.5.4.5 serialNumber + &[0x55, 0x04, 0x05] => { + serial_number_count += 1; + if value_tag != 0x13 { + return Err(ResourceCertificateProfileError::StrictName { + role, + detail: format!( + "serialNumber must be PrintableString, got tag 0x{value_tag:02X}" + ), + }); + } + } + _ => {} + } + } + } + + if common_name_count != 1 { + return Err(ResourceCertificateProfileError::StrictName { + role, + detail: format!("commonName must appear exactly once, got {common_name_count}"), + }); + } + if serial_number_count > 1 { + return Err(ResourceCertificateProfileError::StrictName { + role, + detail: format!("serialNumber must appear at most once, got {serial_number_count}"), + }); + } + Ok(()) +} diff --git a/crates/panda-rpki-validator/src/data_model/rc/parsed_validation.rs b/crates/panda-rpki-validator/src/data_model/rc/parsed_validation.rs new file mode 100644 index 0000000..8c69198 --- /dev/null +++ b/crates/panda-rpki-validator/src/data_model/rc/parsed_validation.rs @@ -0,0 +1,348 @@ +// Parsed certificate and extension profile validation. + +impl ResourceCertificateParsed { + pub fn validate_profile(self) -> Result { + let version = match self.version { + X509Version::V3 => 2u32, + _ => return Err(ResourceCertificateProfileError::InvalidVersion), + }; + + self.validity_not_before + .validate_encoding_rfc5280("notBefore")?; + self.validity_not_after + .validate_encoding_rfc5280("notAfter")?; + + if self.signature_algorithm != self.tbs_signature_algorithm { + return Err(ResourceCertificateProfileError::SignatureAlgorithmMismatch); + } + if self.signature_algorithm.oid != OID_SHA256_WITH_RSA_ENCRYPTION { + return Err(ResourceCertificateProfileError::UnsupportedSignatureAlgorithm); + } + if !self.signature_algorithm.params_absent_or_null() { + return Err(ResourceCertificateProfileError::InvalidSignatureAlgorithmParameters); + } + + let is_self_signed = self.issuer_name == self.subject_name; + let extensions = self.extensions.validate_profile(is_self_signed)?; + let kind = if extensions.basic_constraints_ca { + ResourceCertKind::Ca + } else { + ResourceCertKind::Ee + }; + + Ok(ResourceCertificate { + raw_der: self.raw_der, + tbs: RpkixTbsCertificate { + version, + serial_number: self.serial_number, + signature_algorithm: self.signature_algorithm.oid, + issuer_name: self.issuer_name, + subject_name: self.subject_name, + validity_not_before: self.validity_not_before.utc, + validity_not_after: self.validity_not_after.utc, + subject_public_key_info: self.subject_public_key_info, + extensions, + }, + kind, + }) + } +} + +impl RcExtensionsParsed { + pub fn validate_profile( + self, + is_self_signed: bool, + ) -> Result { + // NOTE(perf): `self` is consumed. Prefer moving decoded fields out rather than cloning, + // especially for large resource sets and URI lists. + let RcExtensionsParsed { + basic_constraints, + subject_key_identifier, + authority_key_identifier, + crl_distribution_points, + authority_info_access, + subject_info_access, + certificate_policies, + extension_oids, + ip_resources, + as_resources, + } = self; + + if basic_constraints.len() > 1 { + return Err(ResourceCertificateProfileError::DuplicateExtension( + "basicConstraints", + )); + } + let basic_constraints = basic_constraints.into_iter().next(); + let basic_constraints_ca = basic_constraints.as_ref().is_some_and(|bc| bc.ca); + + let subject_key_identifier = match subject_key_identifier.len() { + 0 => None, + 1 => { + let (ski, critical) = subject_key_identifier.into_iter().next().expect("len==1"); + if critical { + return Err(ResourceCertificateProfileError::SkiCriticality); + } + Some(ski) + } + _ => { + return Err(ResourceCertificateProfileError::DuplicateExtension( + "subjectKeyIdentifier", + )); + } + }; + + let authority_key_identifier = match authority_key_identifier.len() { + 0 => { + if is_self_signed { + None + } else { + return Err(ResourceCertificateProfileError::AkiMissing); + } + } + 1 => { + let (aki, critical) = authority_key_identifier.into_iter().next().expect("len==1"); + if critical { + return Err(ResourceCertificateProfileError::AkiCriticality); + } + if aki.has_authority_cert_issuer { + return Err(ResourceCertificateProfileError::AkiAuthorityCertIssuerPresent); + } + if aki.has_authority_cert_serial { + return Err(ResourceCertificateProfileError::AkiAuthorityCertSerialPresent); + } + let keyid = aki.key_identifier; + if is_self_signed { + if let (Some(keyid), Some(ski)) = + (keyid.as_ref(), subject_key_identifier.as_ref()) + { + if keyid != ski { + return Err(ResourceCertificateProfileError::AkiSelfSignedNotEqualSki); + } + } + } else if keyid.is_none() { + return Err(ResourceCertificateProfileError::AkiMissing); + } + keyid + } + _ => { + return Err(ResourceCertificateProfileError::DuplicateExtension( + "authorityKeyIdentifier", + )); + } + }; + + let crl_distribution_points_uris = match crl_distribution_points.len() { + 0 => { + if is_self_signed { + None + } else { + return Err(ResourceCertificateProfileError::CrlDistributionPointsMissing); + } + } + 1 => { + let (crldp, critical) = crl_distribution_points.into_iter().next().expect("len==1"); + if critical { + return Err(ResourceCertificateProfileError::CrlDistributionPointsCriticality); + } + if is_self_signed { + return Err( + ResourceCertificateProfileError::CrlDistributionPointsSelfSignedMustOmit, + ); + } + if crldp.distribution_points.len() != 1 { + return Err(ResourceCertificateProfileError::CrlDistributionPointsNotSingle); + } + let dp = crldp + .distribution_points + .into_iter() + .next() + .expect("len==1"); + if dp.reasons_present { + return Err(ResourceCertificateProfileError::CrlDistributionPointsHasReasons); + } + if dp.crl_issuer_present { + return Err(ResourceCertificateProfileError::CrlDistributionPointsHasCrlIssuer); + } + if !dp.distribution_point_present { + return Err( + ResourceCertificateProfileError::CrlDistributionPointsNoDistributionPoint, + ); + } + if dp.name_relative_to_crl_issuer_present || !dp.full_name_present { + return Err(ResourceCertificateProfileError::CrlDistributionPointsInvalidName); + } + if dp.full_name_not_uri { + return Err( + ResourceCertificateProfileError::CrlDistributionPointsFullNameNotUri, + ); + } + if !dp.full_name_uris.iter().any(|u| u.starts_with("rsync://")) { + return Err(ResourceCertificateProfileError::CrlDistributionPointsNoRsync); + } + Some(dp.full_name_uris) + } + _ => { + return Err(ResourceCertificateProfileError::DuplicateExtension( + "cRLDistributionPoints", + )); + } + }; + + let ca_issuers_uris = match authority_info_access.len() { + 0 => { + if is_self_signed { + None + } else { + return Err(ResourceCertificateProfileError::AuthorityInfoAccessMissing); + } + } + 1 => { + let (aia, critical) = authority_info_access.into_iter().next().expect("len==1"); + if critical { + return Err(ResourceCertificateProfileError::AuthorityInfoAccessCriticality); + } + if is_self_signed { + return Err( + ResourceCertificateProfileError::AuthorityInfoAccessSelfSignedMustOmit, + ); + } + if aia.ca_issuers_access_location_not_uri { + return Err( + ResourceCertificateProfileError::AuthorityInfoAccessCaIssuersNotUri, + ); + } + if aia.ca_issuers_uris.is_empty() { + return Err( + ResourceCertificateProfileError::AuthorityInfoAccessMissingCaIssuers, + ); + } + if !aia + .ca_issuers_uris + .iter() + .any(|u| u.starts_with("rsync://")) + { + return Err(ResourceCertificateProfileError::AuthorityInfoAccessNoRsync); + } + Some(aia.ca_issuers_uris) + } + _ => { + return Err(ResourceCertificateProfileError::DuplicateExtension( + "authorityInfoAccess", + )); + } + }; + + let subject_info_access = match subject_info_access.len() { + 0 => None, + 1 => { + let (sia, critical) = subject_info_access.into_iter().next().expect("len==1"); + if critical { + return Err(ResourceCertificateProfileError::SiaCriticality); + } + if sia.signed_object_access_location_not_uri { + return Err(ResourceCertificateProfileError::SignedObjectSiaNotUri); + } + if !sia.signed_object_uris.is_empty() + && !sia + .signed_object_uris + .iter() + .any(|u| u.starts_with("rsync://")) + { + return Err(ResourceCertificateProfileError::SignedObjectSiaNoRsync); + } + if sia.signed_object_uris.is_empty() { + Some(SubjectInfoAccess::Ca(SubjectInfoAccessCa { + access_descriptions: sia.access_descriptions, + })) + } else { + Some(SubjectInfoAccess::Ee(SubjectInfoAccessEe { + signed_object_uris: sia.signed_object_uris, + access_descriptions: sia.access_descriptions, + })) + } + } + _ => { + return Err(ResourceCertificateProfileError::DuplicateExtension( + "subjectInfoAccess", + )); + } + }; + + let certificate_policies = match certificate_policies.len() { + 0 => None, + 1 => { + let (policies, critical) = certificate_policies.into_iter().next().expect("len==1"); + if !critical { + return Err(ResourceCertificateProfileError::CertificatePoliciesCriticality); + } + if policies.len() != 1 { + return Err(ResourceCertificateProfileError::InvalidCertificatePolicy( + "expected exactly one policy".into(), + )); + } + let policy = policies.into_iter().next().expect("len==1"); + if policy.policy_oid != OID_CP_IPADDR_ASNUMBER { + return Err(ResourceCertificateProfileError::InvalidCertificatePolicy( + policy.policy_oid, + )); + } + Some(policy) + } + _ => { + return Err(ResourceCertificateProfileError::DuplicateExtension( + "certificatePolicies", + )); + } + }; + + let ip_resources = match ip_resources.len() { + 0 => None, + 1 => { + let (ip, critical) = ip_resources.into_iter().next().expect("len==1"); + if !critical { + return Err(ResourceCertificateProfileError::IpResourcesCriticality); + } + Some(ip) + } + _ => { + return Err(ResourceCertificateProfileError::DuplicateExtension( + "ipAddrBlocks", + )); + } + }; + + let as_resources = match as_resources.len() { + 0 => None, + 1 => { + let (asn, critical) = as_resources.into_iter().next().expect("len==1"); + if !critical { + return Err(ResourceCertificateProfileError::AsResourcesCriticality); + } + Some(asn) + } + _ => { + return Err(ResourceCertificateProfileError::DuplicateExtension( + "autonomousSysIds", + )); + } + }; + + Ok(RcExtensions { + basic_constraints_ca, + basic_constraints, + subject_key_identifier, + authority_key_identifier, + crl_distribution_points_uris, + ca_issuers_uris, + subject_info_access, + certificate_policies_oid: certificate_policies + .as_ref() + .map(|_| OID_CP_IPADDR_ASNUMBER.to_string()), + certificate_policies, + extension_oids, + ip_resources, + as_resources, + }) + } +} diff --git a/crates/panda-rpki-validator/src/data_model/rc/parsing.rs b/crates/panda-rpki-validator/src/data_model/rc/parsing.rs new file mode 100644 index 0000000..522fd5c --- /dev/null +++ b/crates/panda-rpki-validator/src/data_model/rc/parsing.rs @@ -0,0 +1,586 @@ +// DER parsing helpers for certificate extensions and resources. + +fn algorithm_identifier_value( + ai: &x509_parser::x509::AlgorithmIdentifier<'_>, +) -> AlgorithmIdentifierValue { + let parameters = ai.parameters.as_ref().map(|p| AlgorithmParametersValue { + class: p.class(), + tag: p.tag(), + data: p.as_bytes().to_vec(), + }); + // NOTE(perf): Avoid `to_id_string()` allocations for the algorithms we expect + // in RPKI resource certificates. Fall back to `to_id_string()` for unexpected + // algorithms (mostly error paths). + let oid = if ai.algorithm.as_bytes() == OID_SHA256_WITH_RSA_ENCRYPTION_RAW { + OID_SHA256_WITH_RSA_ENCRYPTION.to_string() + } else { + ai.algorithm.to_id_string() + }; + AlgorithmIdentifierValue { oid, parameters } +} + +fn parse_extensions_parse( + exts: &[X509Extension<'_>], +) -> Result { + let mut basic_constraints: Vec = Vec::new(); + let mut ski: Vec<(Vec, bool)> = Vec::new(); + let mut aki: Vec<(AuthorityKeyIdentifierParsed, bool)> = Vec::new(); + let mut crldp: Vec<(CrlDistributionPointsParsed, bool)> = Vec::new(); + let mut aia: Vec<(AuthorityInfoAccessParsed, bool)> = Vec::new(); + let mut sia: Vec<(SubjectInfoAccessParsed, bool)> = Vec::new(); + let mut cert_policies: Vec<(Vec, bool)> = Vec::new(); + let mut extension_oids: Vec = Vec::with_capacity(exts.len()); + + let mut ip_resources: Vec<(IpResourceSet, bool)> = Vec::new(); + let mut as_resources: Vec<(AsResourceSet, bool)> = Vec::new(); + + for ext in exts { + let oid = ext.oid.as_bytes(); + extension_oids.push(ext.oid.to_id_string()); + if oid == OID_BASIC_CONSTRAINTS_RAW { + let ParsedExtension::BasicConstraints(bc) = ext.parsed_extension() else { + return Err(ResourceCertificateParseError::Parse( + "basicConstraints parse failed".into(), + )); + }; + basic_constraints.push(BasicConstraintsProfile { + ca: bc.ca, + critical: ext.critical, + path_len_constraint: bc.path_len_constraint, + }); + } else if oid == OID_SUBJECT_KEY_IDENTIFIER_RAW { + let ParsedExtension::SubjectKeyIdentifier(s) = ext.parsed_extension() else { + return Err(ResourceCertificateParseError::Parse( + "subjectKeyIdentifier parse failed".into(), + )); + }; + ski.push((s.0.to_vec(), ext.critical)); + } else if oid == OID_AUTHORITY_KEY_IDENTIFIER_RAW { + let ParsedExtension::AuthorityKeyIdentifier(a) = ext.parsed_extension() else { + return Err(ResourceCertificateParseError::Parse( + "authorityKeyIdentifier parse failed".into(), + )); + }; + aki.push(( + AuthorityKeyIdentifierParsed { + key_identifier: a.key_identifier.as_ref().map(|k| k.0.to_vec()), + has_authority_cert_issuer: a.authority_cert_issuer.is_some(), + has_authority_cert_serial: a.authority_cert_serial.is_some(), + }, + ext.critical, + )); + } else if oid == OID_CRL_DISTRIBUTION_POINTS_RAW { + let ParsedExtension::CRLDistributionPoints(p) = ext.parsed_extension() else { + return Err(ResourceCertificateParseError::Parse( + "cRLDistributionPoints parse failed".into(), + )); + }; + crldp.push((parse_crldp_parse(p)?, ext.critical)); + } else if oid == OID_AUTHORITY_INFO_ACCESS_RAW { + let ParsedExtension::AuthorityInfoAccess(p) = ext.parsed_extension() else { + return Err(ResourceCertificateParseError::Parse( + "authorityInfoAccess parse failed".into(), + )); + }; + aia.push((parse_aia_parse(p.accessdescs.as_slice())?, ext.critical)); + } else if oid == OID_SUBJECT_INFO_ACCESS_RAW { + let ParsedExtension::SubjectInfoAccess(s) = ext.parsed_extension() else { + return Err(ResourceCertificateParseError::Parse( + "subjectInfoAccess parse failed".into(), + )); + }; + sia.push((parse_sia_parse(s.accessdescs.as_slice())?, ext.critical)); + } else if oid == OID_CERTIFICATE_POLICIES_RAW { + let ParsedExtension::CertificatePolicies(cp) = ext.parsed_extension() else { + return Err(ResourceCertificateParseError::Parse( + "certificatePolicies parse failed".into(), + )); + }; + let mut policies: Vec = Vec::with_capacity(cp.len()); + for p in cp.iter() { + let b = p.policy_id.as_bytes(); + let policy_oid = if b == OID_CP_IPADDR_ASNUMBER_RAW { + OID_CP_IPADDR_ASNUMBER.to_string() + } else { + p.policy_id.to_id_string() + }; + let qualifier_oids = p + .policy_qualifiers + .as_ref() + .map(|qualifiers| { + qualifiers + .iter() + .map(|qualifier| qualifier.policy_qualifier_id.to_id_string()) + .collect() + }) + .unwrap_or_default(); + policies.push(CertificatePoliciesProfile { + policy_oid, + qualifier_oids, + }); + } + cert_policies.push((policies, ext.critical)); + } else if oid == OID_IP_ADDR_BLOCKS_RAW { + let parsed = IpResourceSet::decode_extn_value(ext.value) + .map_err(|_e| ResourceCertificateParseError::InvalidIpResourcesEncoding)?; + ip_resources.push((parsed, ext.critical)); + } else if oid == OID_AUTONOMOUS_SYS_IDS_RAW { + let parsed = AsResourceSet::decode_extn_value(ext.value) + .map_err(|_e| ResourceCertificateParseError::InvalidAsResourcesEncoding)?; + as_resources.push((parsed, ext.critical)); + } + } + + Ok(RcExtensionsParsed { + basic_constraints, + subject_key_identifier: ski, + authority_key_identifier: aki, + crl_distribution_points: crldp, + authority_info_access: aia, + subject_info_access: sia, + certificate_policies: cert_policies, + extension_oids, + ip_resources, + as_resources, + }) +} + +fn parse_aia_parse( + access: &[x509_parser::extensions::AccessDescription<'_>], +) -> Result { + let mut ca_issuers_uris: Vec = Vec::new(); + let mut ca_issuers_access_location_not_uri = false; + + for ad in access { + if ad.access_method.as_bytes() != OID_AD_CA_ISSUERS_RAW { + continue; + } + let uri = match &ad.access_location { + x509_parser::extensions::GeneralName::URI(u) => u, + _ => { + ca_issuers_access_location_not_uri = true; + continue; + } + }; + ca_issuers_uris.push(uri.to_string()); + } + + Ok(AuthorityInfoAccessParsed { + ca_issuers_uris, + ca_issuers_access_location_not_uri, + }) +} + +fn parse_crldp_parse( + crldp: &x509_parser::extensions::CRLDistributionPoints<'_>, +) -> Result { + let mut out: Vec = Vec::new(); + for p in crldp.iter() { + let mut full_name_uris: Vec = Vec::new(); + let mut full_name_not_uri = false; + let mut full_name_present = false; + let mut name_relative_to_crl_issuer_present = false; + let mut distribution_point_present = false; + + if let Some(dp) = &p.distribution_point { + distribution_point_present = true; + match dp { + x509_parser::extensions::DistributionPointName::FullName(names) => { + full_name_present = true; + for n in names { + match n { + x509_parser::extensions::GeneralName::URI(u) => { + full_name_uris.push(u.to_string()); + } + _ => { + full_name_not_uri = true; + } + } + } + } + x509_parser::extensions::DistributionPointName::NameRelativeToCRLIssuer(_) => { + name_relative_to_crl_issuer_present = true; + } + } + } + + out.push(CrlDistributionPointParsed { + distribution_point_present, + reasons_present: p.reasons.is_some(), + crl_issuer_present: p.crl_issuer.is_some(), + name_relative_to_crl_issuer_present, + full_name_uris, + full_name_not_uri, + full_name_present, + }); + } + Ok(CrlDistributionPointsParsed { + distribution_points: out, + }) +} + +fn parse_sia_parse( + access: &[x509_parser::extensions::AccessDescription<'_>], +) -> Result { + let mut all = Vec::with_capacity(access.len()); + let mut signed_object_uris: Vec = Vec::new(); + let mut signed_object_access_location_not_uri = false; + + for ad in access { + let access_method_oid = if ad.access_method.as_bytes() == OID_AD_CA_REPOSITORY_RAW { + OID_AD_CA_REPOSITORY.to_string() + } else if ad.access_method.as_bytes() == OID_AD_RPKI_MANIFEST_RAW { + OID_AD_RPKI_MANIFEST.to_string() + } else if ad.access_method.as_bytes() == OID_AD_RPKI_NOTIFY_RAW { + OID_AD_RPKI_NOTIFY.to_string() + } else if ad.access_method.as_bytes() == OID_AD_SIGNED_OBJECT_RAW { + OID_AD_SIGNED_OBJECT.to_string() + } else { + ad.access_method.to_id_string() + }; + let is_signed_object = access_method_oid == OID_AD_SIGNED_OBJECT; + let uri = match &ad.access_location { + x509_parser::extensions::GeneralName::URI(u) => u, + _ => { + if is_signed_object { + signed_object_access_location_not_uri = true; + } + continue; + } + }; + if is_signed_object { + signed_object_uris.push(uri.to_string()); + } + all.push(AccessDescription { + access_method_oid, + access_location: uri.to_string(), + }); + } + + Ok(SubjectInfoAccessParsed { + access_descriptions: all, + signed_object_uris, + signed_object_access_location_not_uri, + }) +} + +fn parse_ip_addr_blocks(ext_value: &[u8]) -> Result { + let (rem, obj) = parse_der(ext_value).map_err(|_| ())?; + if !rem.is_empty() { + return Err(()); + } + let seq = obj.as_sequence().map_err(|_| ())?; + let mut families = Vec::with_capacity(seq.len()); + for fam in seq { + let fam_seq = fam.as_sequence().map_err(|_| ())?; + if fam_seq.len() != 2 { + return Err(()); + } + let af_bytes = fam_seq[0].as_slice().map_err(|_| ())?; + if af_bytes.len() != 2 { + return Err(()); + } + let afi = match af_bytes { + [0x00, 0x01] => Afi::Ipv4, + [0x00, 0x02] => Afi::Ipv6, + _ => return Err(()), + }; + + let choice = match &fam_seq[1].content { + BerObjectContent::Null => IpAddressChoice::Inherit, + BerObjectContent::Sequence(_) => { + let items_seq = fam_seq[1].as_sequence().map_err(|_| ())?; + let mut items = Vec::with_capacity(items_seq.len()); + for item in items_seq { + items.push(parse_ip_address_or_range(afi, item)?); + } + IpAddressChoice::AddressesOrRanges(items) + } + _ => return Err(()), + }; + families.push(IpAddressFamily { afi, choice }); + } + Ok(IpResourceSet { families }) +} + +fn parse_ip_address_or_range(afi: Afi, obj: &DerObject<'_>) -> Result { + match &obj.content { + BerObjectContent::BitString(_, _) => { + Ok(IpAddressOrRange::Prefix(parse_ip_prefix(afi, obj)?)) + } + BerObjectContent::Sequence(_) => { + let seq = obj.as_sequence().map_err(|_| ())?; + if seq.len() != 2 { + return Err(()); + } + let min = parse_ip_address_bound(afi, &seq[0], false)?; + let max = parse_ip_address_bound(afi, &seq[1], true)?; + Ok(IpAddressOrRange::Range(IpAddressRange { min, max })) + } + _ => Err(()), + } +} + +fn parse_ip_prefix(afi: Afi, obj: &DerObject<'_>) -> Result { + let (unused_bits, bytes) = match &obj.content { + BerObjectContent::BitString(unused, bso) => (*unused, bso.data.to_vec()), + _ => return Err(()), + }; + if unused_bits > 7 { + return Err(()); + } + if !bytes.is_empty() && unused_bits != 0 { + let mask = (1u8 << unused_bits) - 1; + if (bytes[bytes.len() - 1] & mask) != 0 { + return Err(()); + } + } else if bytes.is_empty() && unused_bits != 0 { + return Err(()); + } + let prefix_len = (bytes.len() * 8) + .checked_sub(unused_bits as usize) + .ok_or(())? as u16; + if prefix_len > afi.ub() { + return Err(()); + } + let addr = canonicalize_prefix_addr(afi, prefix_len, &bytes); + Ok(IpPrefix { + afi, + prefix_len, + addr, + }) +} + +/// Parse an RFC 3779 `IPAddress` BIT STRING into an address-like byte array. +/// +/// When used as an `IPAddressRange` endpoint, RFC 3779 allows endpoints to be encoded with +/// fewer than `ub` bits. In that case, the missing bits are interpreted as 0s for the lower +/// bound and 1s for the upper bound. This is essential to correctly interpret ranges that +/// are expressed on non-octet boundaries. +fn parse_ip_address_bound( + afi: Afi, + obj: &DerObject<'_>, + fill_remaining_ones: bool, +) -> Result, ()> { + let (unused_bits, bytes) = match &obj.content { + BerObjectContent::BitString(unused, bso) => (*unused, bso.data.to_vec()), + _ => return Err(()), + }; + if unused_bits > 7 { + return Err(()); + } + if !bytes.is_empty() && unused_bits != 0 { + let mask = (1u8 << unused_bits) - 1; + if (bytes[bytes.len() - 1] & mask) != 0 { + return Err(()); + } + } else if bytes.is_empty() && unused_bits != 0 { + return Err(()); + } + + let bit_len: u16 = (bytes.len() * 8) + .checked_sub(unused_bits as usize) + .ok_or(())? + .try_into() + .map_err(|_| ())?; + if bit_len > afi.ub() { + return Err(()); + } + + let mut out = vec![0u8; afi.octets_len()]; + let copy_len = bytes.len().min(out.len()); + out[..copy_len].copy_from_slice(&bytes[..copy_len]); + + if fill_remaining_ones { + if bit_len == 0 { + for b in &mut out { + *b = 0xFF; + } + return Ok(out); + } + + let last_bit = (bit_len - 1) as usize; + let last_byte = last_bit / 8; + let rem = (bit_len % 8) as u8; + + if rem != 0 && last_byte < out.len() { + // Set the (8-rem) trailing bits in the last byte to 1. + let mask: u8 = (1u8 << (8 - rem)) - 1; + out[last_byte] |= mask; + } + for b in out.iter_mut().skip(last_byte + 1) { + *b = 0xFF; + } + } + + Ok(out) +} + +fn parse_as_identifiers(ext_value: &[u8]) -> Result { + let (rem, obj) = parse_der(ext_value).map_err(|_| ())?; + if !rem.is_empty() { + return Err(()); + } + let seq = obj.as_sequence().map_err(|_| ())?; + let mut asnum: Option = None; + let mut rdi: Option = None; + for item in seq { + if item.class() != Class::ContextSpecific { + return Err(()); + } + match item.tag() { + Tag(0) => { + if asnum.is_some() { + return Err(()); + } + let inner = parse_explicit_inner(item)?; + asnum = Some(parse_as_identifier_choice(&inner)?); + } + Tag(1) => { + if rdi.is_some() { + return Err(()); + } + let inner = parse_explicit_inner(item)?; + rdi = Some(parse_as_identifier_choice(&inner)?); + } + _ => return Err(()), + } + } + Ok(AsResourceSet { asnum, rdi }) +} + +fn parse_explicit_inner<'a>(obj: &'a DerObject<'a>) -> Result, ()> { + let inner_der = obj.as_slice().map_err(|_| ())?; + let (rem, inner) = parse_der(inner_der).map_err(|_| ())?; + if !rem.is_empty() { + return Err(()); + } + Ok(inner) +} + +fn parse_as_identifier_choice(obj: &DerObject<'_>) -> Result { + match &obj.content { + BerObjectContent::Null => Ok(AsIdentifierChoice::Inherit), + BerObjectContent::Sequence(_) => { + let seq = obj.as_sequence().map_err(|_| ())?; + let mut items = Vec::with_capacity(seq.len()); + for item in seq { + items.push(parse_as_id_or_range(item)?); + } + Ok(AsIdentifierChoice::AsIdsOrRanges(items)) + } + _ => Err(()), + } +} + +fn parse_as_id_or_range(obj: &DerObject<'_>) -> Result { + match &obj.content { + BerObjectContent::Integer(_) => { + let v = obj.as_u64().map_err(|_| ())?; + if v > u32::MAX as u64 { + return Err(()); + } + Ok(AsIdOrRange::Id(v as u32)) + } + BerObjectContent::Sequence(_) => { + let seq = obj.as_sequence().map_err(|_| ())?; + if seq.len() != 2 { + return Err(()); + } + let min = seq[0].as_u64().map_err(|_| ())?; + let max = seq[1].as_u64().map_err(|_| ())?; + if min > u32::MAX as u64 || max > u32::MAX as u64 || min > max { + return Err(()); + } + Ok(AsIdOrRange::Range { + min: min as u32, + max: max as u32, + }) + } + _ => Err(()), + } +} + +fn canonicalize_prefix_addr(afi: Afi, prefix_len: u16, bytes: &[u8]) -> Vec { + let full_len = afi.octets_len(); + let mut addr = vec![0u8; full_len]; + let copy_len = bytes.len().min(full_len); + addr[..copy_len].copy_from_slice(&bytes[..copy_len]); + + if prefix_len == 0 { + return addr; + } + + let last_prefix_bit = (prefix_len - 1) as usize; + let last_prefix_byte = last_prefix_bit / 8; + let rem = (prefix_len % 8) as u8; + if rem != 0 && last_prefix_byte < addr.len() { + let mask: u8 = 0xFF << (8 - rem); + addr[last_prefix_byte] &= mask; + } + addr +} + +fn prefix_covers(resource: &IpPrefix, subject: &IpPrefix) -> bool { + if resource.afi != subject.afi { + return false; + } + if resource.prefix_len > subject.prefix_len { + return false; + } + let n = resource.prefix_len as usize; + let whole = n / 8; + let rem = (n % 8) as u8; + if resource.addr.len() != subject.addr.len() { + return false; + } + if resource.addr[..whole] != subject.addr[..whole] { + return false; + } + if rem == 0 { + return true; + } + let mask = 0xFFu8 << (8 - rem); + (resource.addr[whole] & mask) == (subject.addr[whole] & mask) +} + +fn prefix_range(afi: Afi, p: &IpPrefix) -> (u128, u128) { + let mut base_bytes = [0u8; 16]; + match afi { + Afi::Ipv4 => { + base_bytes[12..].copy_from_slice(&p.addr[..4]); + } + Afi::Ipv6 => { + base_bytes.copy_from_slice(&p.addr[..16]); + } + } + let base = u128::from_be_bytes(base_bytes); + let host_bits = (afi.ub() - p.prefix_len) as u32; + if host_bits == 0 { + return (base, base); + } + let mask = (1u128 << host_bits) - 1; + (base, base | mask) +} + +fn range_covers_prefix(afi: Afi, r: &IpAddressRange, p: &IpPrefix) -> bool { + let (p_min, p_max) = prefix_range(afi, p); + let r_min = bytes_to_u128(afi, &r.min); + let r_max = bytes_to_u128(afi, &r.max); + r_min <= p_min && p_max <= r_max +} + +fn bytes_to_u128(afi: Afi, bytes: &[u8]) -> u128 { + let mut out = [0u8; 16]; + match afi { + Afi::Ipv4 => { + let copy_len = bytes.len().min(4); + out[12..12 + copy_len].copy_from_slice(&bytes[..copy_len]); + } + Afi::Ipv6 => { + let copy_len = bytes.len().min(16); + out[..copy_len].copy_from_slice(&bytes[..copy_len]); + } + } + u128::from_be_bytes(out) +} diff --git a/crates/panda-rpki-validator/src/data_model/rc/strict_name_tests.rs b/crates/panda-rpki-validator/src/data_model/rc/strict_name_tests.rs new file mode 100644 index 0000000..22ab765 --- /dev/null +++ b/crates/panda-rpki-validator/src/data_model/rc/strict_name_tests.rs @@ -0,0 +1,102 @@ +// Strict RPKI name profile tests. + +#[cfg(test)] +mod strict_name_tests { + use super::*; + + fn name_with_attrs(attrs: &[(&[u8], u8, &[u8])]) -> X509NameDer { + let mut rdns = Vec::new(); + for (oid, tag, value) in attrs { + let mut attr = Vec::new(); + attr.extend(der_tlv(0x06, oid)); + attr.extend(der_tlv(*tag, value)); + let attr = der_tlv(0x30, &attr); + let rdn = der_tlv(0x31, &attr); + rdns.extend(rdn); + } + X509NameDer(der_tlv(0x30, &rdns)) + } + + fn der_tlv(tag: u8, value: &[u8]) -> Vec { + let mut out = vec![tag]; + encode_len(value.len(), &mut out); + out.extend_from_slice(value); + out + } + + fn encode_len(len: usize, out: &mut Vec) { + if len < 0x80 { + out.push(len as u8); + return; + } + let mut bytes = Vec::new(); + let mut value = len; + while value > 0 { + bytes.push((value & 0xFF) as u8); + value >>= 8; + } + bytes.reverse(); + out.push(0x80 | bytes.len() as u8); + out.extend(bytes); + } + + #[test] + fn strict_name_accepts_printable_common_name_and_serial_number() { + let name = name_with_attrs(&[ + (&[0x55, 0x04, 0x03], 0x13, b"CN1"), + (&[0x55, 0x04, 0x05], 0x13, b"SN1"), + ]); + validate_strict_rpki_name(&name, "subject").expect("strict name"); + } + + #[test] + fn strict_name_rejects_utf8_common_name() { + let name = name_with_attrs(&[(&[0x55, 0x04, 0x03], 0x0C, b"CN1")]); + let err = validate_strict_rpki_name(&name, "subject").expect_err("strict name fails"); + assert!(err.to_string().contains("PrintableString"), "{err}"); + } + + #[test] + fn strict_name_rejects_duplicate_common_name() { + let name = name_with_attrs(&[ + (&[0x55, 0x04, 0x03], 0x13, b"CN1"), + (&[0x55, 0x04, 0x03], 0x13, b"CN2"), + ]); + let err = validate_strict_rpki_name(&name, "subject").expect_err("strict name fails"); + assert!(err.to_string().contains("exactly once"), "{err}"); + } + + #[test] + fn profile_rejects_rfc8360_v2_policy_oid() { + let extensions = RcExtensionsParsed { + basic_constraints: vec![BasicConstraintsProfile { + ca: true, + critical: true, + path_len_constraint: None, + }], + subject_key_identifier: Vec::new(), + authority_key_identifier: Vec::new(), + crl_distribution_points: Vec::new(), + authority_info_access: Vec::new(), + subject_info_access: Vec::new(), + certificate_policies: vec![( + vec![CertificatePoliciesProfile { + policy_oid: "1.3.6.1.5.5.7.14.3".to_string(), + qualifier_oids: Vec::new(), + }], + true, + )], + extension_oids: Vec::new(), + ip_resources: Vec::new(), + as_resources: Vec::new(), + }; + + let err = extensions + .validate_profile(true) + .expect_err("v2 policy OID must remain invalid"); + assert!( + matches!(&err, ResourceCertificateProfileError::InvalidCertificatePolicy(oid) if oid == "1.3.6.1.5.5.7.14.3"), + "{err}" + ); + } +} diff --git a/crates/panda-rpki-validator/src/data_model/rc/types.rs b/crates/panda-rpki-validator/src/data_model/rc/types.rs new file mode 100644 index 0000000..b1a24f6 --- /dev/null +++ b/crates/panda-rpki-validator/src/data_model/rc/types.rs @@ -0,0 +1,564 @@ +// Resource certificate and resource-set model types. + +/// Resource Certificate kind (semantic classification). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ResourceCertKind { + Ca, + Ee, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ResourceCertificateRole { + TrustAnchor, + Ca, + SignedObjectEe, + RouterEe, +} + +/// A parsed RPKI Resource Certificate (RFC 6487) data model. +/// +/// This module intentionally focuses on the semantics needed by Signed Object validation and +/// object-specific EE certificate checks (MFT/ROA/ASPA), as described in +/// `rpki/specs/03_resource_certificate_rc.md`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ResourceCertificate { + pub raw_der: Vec, + pub tbs: RpkixTbsCertificate, + pub kind: ResourceCertKind, +} + +pub type ResourceCaCertificate = ResourceCertificate; +pub type ResourceEeCertificate = ResourceCertificate; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RpkixTbsCertificate { + pub version: u32, + pub serial_number: BigUint, + pub signature_algorithm: String, + pub issuer_name: X509NameDer, + pub subject_name: X509NameDer, + pub validity_not_before: UtcTime, + pub validity_not_after: UtcTime, + /// DER encoding of SubjectPublicKeyInfo. + pub subject_public_key_info: Vec, + pub extensions: RcExtensions, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RcExtensions { + pub basic_constraints_ca: bool, + pub basic_constraints: Option, + pub subject_key_identifier: Option>, + /// Authority Key Identifier (AKI) keyIdentifier value. + pub authority_key_identifier: Option>, + /// CRL Distribution Points URIs (fullName). + pub crl_distribution_points_uris: Option>, + /// Authority Information Access (AIA) caIssuers URIs. + pub ca_issuers_uris: Option>, + pub subject_info_access: Option, + pub certificate_policies_oid: Option, + pub certificate_policies: Option, + pub extension_oids: Vec, + + pub ip_resources: Option, + pub as_resources: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BasicConstraintsProfile { + pub ca: bool, + pub critical: bool, + pub path_len_constraint: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CertificatePoliciesProfile { + pub policy_oid: String, + pub qualifier_oids: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ResourceCertificateParsed { + pub raw_der: Vec, + pub version: X509Version, + pub serial_number: BigUint, + pub signature_algorithm: AlgorithmIdentifierValue, + pub tbs_signature_algorithm: AlgorithmIdentifierValue, + pub issuer_name: X509NameDer, + pub subject_name: X509NameDer, + pub validity_not_before: Asn1TimeUtc, + pub validity_not_after: Asn1TimeUtc, + /// DER encoding of SubjectPublicKeyInfo. + pub subject_public_key_info: Vec, + pub extensions: RcExtensionsParsed, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AlgorithmIdentifierValue { + pub oid: String, + pub parameters: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AlgorithmParametersValue { + pub class: Asn1Class, + pub tag: Asn1Tag, + pub data: Vec, +} + +impl AlgorithmIdentifierValue { + pub fn params_absent_or_null(&self) -> bool { + match &self.parameters { + None => true, + Some(p) if p.class == Asn1Class::Universal && p.tag == Asn1Tag::Null => true, + Some(_p) => false, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RcExtensionsParsed { + pub basic_constraints: Vec, + pub subject_key_identifier: Vec<(Vec, bool)>, + pub authority_key_identifier: Vec<(AuthorityKeyIdentifierParsed, bool)>, + pub crl_distribution_points: Vec<(CrlDistributionPointsParsed, bool)>, + pub authority_info_access: Vec<(AuthorityInfoAccessParsed, bool)>, + pub subject_info_access: Vec<(SubjectInfoAccessParsed, bool)>, + pub certificate_policies: Vec<(Vec, bool)>, + pub extension_oids: Vec, + pub ip_resources: Vec<(IpResourceSet, bool)>, + pub as_resources: Vec<(AsResourceSet, bool)>, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AuthorityKeyIdentifierParsed { + pub key_identifier: Option>, + pub has_authority_cert_issuer: bool, + pub has_authority_cert_serial: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AuthorityInfoAccessParsed { + pub ca_issuers_uris: Vec, + pub ca_issuers_access_location_not_uri: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CrlDistributionPointsParsed { + pub distribution_points: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CrlDistributionPointParsed { + pub distribution_point_present: bool, + pub reasons_present: bool, + pub crl_issuer_present: bool, + pub name_relative_to_crl_issuer_present: bool, + pub full_name_uris: Vec, + pub full_name_not_uri: bool, + pub full_name_present: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SubjectInfoAccessParsed { + pub access_descriptions: Vec, + pub signed_object_uris: Vec, + pub signed_object_access_location_not_uri: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SubjectInfoAccess { + Ca(SubjectInfoAccessCa), + Ee(SubjectInfoAccessEe), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SubjectInfoAccessCa { + pub access_descriptions: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SubjectInfoAccessEe { + pub signed_object_uris: Vec, + /// The full list of access descriptions as carried in the SIA extension. + pub access_descriptions: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AccessDescription { + pub access_method_oid: String, + pub access_location: String, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +pub enum Afi { + Ipv4, + Ipv6, +} + +impl Afi { + pub fn ub(self) -> u16 { + match self { + Afi::Ipv4 => 32, + Afi::Ipv6 => 128, + } + } + + pub fn octets_len(self) -> usize { + match self { + Afi::Ipv4 => 4, + Afi::Ipv6 => 16, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct IpResourceSet { + pub families: Vec, +} + +impl IpResourceSet { + /// Decode the DER bytes carried inside the X.509 `extnValue` OCTET STRING for + /// `id-pe-ipAddrBlocks` (RFC 3779 / RFC 6487). + pub fn decode_extn_value(extn_value: &[u8]) -> Result { + parse_ip_addr_blocks(extn_value).map_err(|_| IpResourceSetDecodeError::InvalidEncoding) + } + + pub fn is_all_inherit(&self) -> bool { + self.families + .iter() + .all(|f| matches!(f.choice, IpAddressChoice::Inherit)) + } + + pub fn has_any_inherit(&self) -> bool { + self.families + .iter() + .any(|f| matches!(f.choice, IpAddressChoice::Inherit)) + } + + pub fn contains_prefix(&self, prefix: &IpPrefix) -> bool { + self.families.iter().any(|fam| fam.contains_prefix(prefix)) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum IpResourceSetDecodeError { + #[error("invalid ipAddrBlocks encoding (RFC 3779 §2.2.3; RFC 6487 §4.8.10)")] + InvalidEncoding, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct IpAddressFamily { + pub afi: Afi, + pub choice: IpAddressChoice, +} + +impl IpAddressFamily { + pub fn contains_prefix(&self, prefix: &IpPrefix) -> bool { + if self.afi != prefix.afi { + return false; + } + match &self.choice { + IpAddressChoice::Inherit => true, + IpAddressChoice::AddressesOrRanges(items) => items.iter().any(|item| match item { + IpAddressOrRange::Prefix(p) => prefix_covers(p, prefix), + IpAddressOrRange::Range(r) => range_covers_prefix(self.afi, r, prefix), + }), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum IpAddressChoice { + Inherit, + AddressesOrRanges(Vec), +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum IpAddressOrRange { + Prefix(IpPrefix), + Range(IpAddressRange), +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct IpAddressRange { + pub min: Vec, + pub max: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub struct IpPrefix { + pub afi: Afi, + pub prefix_len: u16, + pub addr: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct AsResourceSet { + pub asnum: Option, + pub rdi: Option, +} + +impl AsResourceSet { + /// Decode the DER bytes carried inside the X.509 `extnValue` OCTET STRING for + /// `id-pe-autonomousSysIds` (RFC 3779 / RFC 6487). + pub fn decode_extn_value(extn_value: &[u8]) -> Result { + parse_as_identifiers(extn_value).map_err(|_| AsResourceSetDecodeError::InvalidEncoding) + } + + pub fn is_asnum_inherit(&self) -> bool { + matches!(self.asnum, Some(AsIdentifierChoice::Inherit)) + } + + pub fn has_any_range(&self) -> bool { + self.asnum.as_ref().map(|c| c.has_range()).unwrap_or(false) + || self.rdi.as_ref().map(|c| c.has_range()).unwrap_or(false) + } + + pub fn asnum_single_id(&self) -> Option { + match self.asnum.as_ref()? { + AsIdentifierChoice::Inherit => None, + AsIdentifierChoice::AsIdsOrRanges(items) => { + if items.len() != 1 { + return None; + } + match &items[0] { + AsIdOrRange::Id(v) => Some(*v), + AsIdOrRange::Range { .. } => None, + } + } + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum AsResourceSetDecodeError { + #[error("invalid autonomousSysIds encoding (RFC 3779 §3.2.3; RFC 6487 §4.8.11)")] + InvalidEncoding, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum AsIdentifierChoice { + Inherit, + AsIdsOrRanges(Vec), +} + +impl AsIdentifierChoice { + pub fn has_range(&self) -> bool { + match self { + AsIdentifierChoice::Inherit => false, + AsIdentifierChoice::AsIdsOrRanges(items) => { + items.iter().any(|i| matches!(i, AsIdOrRange::Range { .. })) + } + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum AsIdOrRange { + Id(u32), + Range { min: u32, max: u32 }, +} + +#[derive(Debug, thiserror::Error)] +pub enum ResourceCertificateParseError { + #[error("X.509 parse error: {0} (RFC 5280 §4.1; RFC 6487 §4)")] + Parse(String), + + #[error("trailing bytes after certificate DER: {0} bytes (DER; RFC 5280 §4.1)")] + TrailingBytes(usize), + + #[error("invalid RFC 3779 IP resources extension encoding (RFC 6487 §4.8.10; RFC 3779 §2.2)")] + InvalidIpResourcesEncoding, + + #[error("invalid RFC 3779 AS resources extension encoding (RFC 6487 §4.8.11; RFC 3779 §3.2)")] + InvalidAsResourcesEncoding, +} + +#[derive(Debug, thiserror::Error)] +pub enum ResourceCertificateProfileError { + #[error("{0}")] + InvalidTimeEncoding(#[from] InvalidTimeEncodingError), + + #[error("certificate version must be v3 (RFC 5280 §4.1; RFC 6487 §4)")] + InvalidVersion, + + #[error("signatureAlgorithm does not match tbsCertificate.signature (RFC 5280 §4.1)")] + SignatureAlgorithmMismatch, + + #[error( + "unsupported signature algorithm (expected sha256WithRSAEncryption {OID_SHA256_WITH_RSA_ENCRYPTION}) (RFC 7935 §2; RFC 6487 §4)" + )] + UnsupportedSignatureAlgorithm, + + #[error("invalid signature algorithm parameters (RFC 5280 §4.1.1.2)")] + InvalidSignatureAlgorithmParameters, + + #[error( + "{role} Name strict validation failed: {detail} (RFC 6487 §4.4; RFC 5280 §4.1.2.4/§4.1.2.6)" + )] + StrictName { role: &'static str, detail: String }, + + #[error("duplicate extension: {0} (RFC 5280 §4.2; RFC 6487 §4.8)")] + DuplicateExtension(&'static str), + + #[error("SubjectKeyIdentifier criticality must be non-critical (RFC 6487 §4.8.2)")] + SkiCriticality, + + #[error("SubjectInfoAccess criticality must be non-critical (RFC 6487 §4.8.8)")] + SiaCriticality, + + #[error("certificatePolicies criticality must be critical (RFC 6487 §4.8.9)")] + CertificatePoliciesCriticality, + + #[error("certificatePolicies must be present (RFC 6487 §4.8.9)")] + CertificatePoliciesMissing, + + #[error( + "certificatePolicies must contain RPKI policy OID {OID_CP_IPADDR_ASNUMBER}, got {0} (RFC 6487 §4.8.9)" + )] + InvalidCertificatePolicy(String), + + #[error("certificatePolicies may contain at most one CPS qualifier (RFC 6487 §4.8.9)")] + CertificatePoliciesTooManyQualifiers, + + #[error( + "certificatePolicies qualifier must be id-qt-cps ({OID_QT_CPS}), got {0} (RFC 6487 §4.8.9)" + )] + CertificatePoliciesInvalidQualifier(String), + + #[error("basicConstraints must be present in CA certificates (RFC 6487 §4.8.1)")] + BasicConstraintsMissing, + + #[error("basicConstraints criticality must be critical in CA certificates (RFC 6487 §4.8.1)")] + BasicConstraintsCriticality, + + #[error("basicConstraints cA must be TRUE in CA certificates (RFC 6487 §4.8.1)")] + BasicConstraintsCaFalse, + + #[error( + "basicConstraints pathLenConstraint must be absent in CA certificates (RFC 6487 §4.8.1)" + )] + BasicConstraintsPathLenPresent, + + #[error("basicConstraints must be absent in EE certificates (RFC 6487 §4.8.1)")] + BasicConstraintsEeMustOmit, + + #[error("extension {oid} is not permitted for {role} resource certificates (RFC 6487 §4.8)")] + DisallowedExtension { role: &'static str, oid: String }, + + #[error("autonomousSysIds RDI field must be absent (RFC 6487 §4.8.11; RFC 3779 §3.2.3)")] + AsResourcesRdiPresent, + + #[error( + "SIA id-ad-signedObject accessLocation must be URI (RFC 6487 §4.8.8.2; RFC 5280 §4.2.2.2)" + )] + SignedObjectSiaNotUri, + + #[error("SIA id-ad-signedObject must include at least one rsync:// URI (RFC 6487 §4.8.8.2)")] + SignedObjectSiaNoRsync, + + #[error("ipAddrBlocks criticality must be critical when present (RFC 6487 §4.8.10)")] + IpResourcesCriticality, + + #[error("autonomousSysIds criticality must be critical when present (RFC 6487 §4.8.11)")] + AsResourcesCriticality, + + #[error( + "authorityKeyIdentifier must be present in non-self-signed certificates (RFC 6487 §4.8.3; RFC 5280 §4.2.1.1)" + )] + AkiMissing, + + #[error( + "authorityKeyIdentifier criticality must be non-critical (RFC 6487 §4.8.3; RFC 5280 §4.2.1.1)" + )] + AkiCriticality, + + #[error( + "authorityKeyIdentifier authorityCertIssuer MUST NOT be present (RFC 6487 §4.8.3; RFC 5280 §4.2.1.1)" + )] + AkiAuthorityCertIssuerPresent, + + #[error( + "authorityKeyIdentifier authorityCertSerialNumber MUST NOT be present (RFC 6487 §4.8.3; RFC 5280 §4.2.1.1)" + )] + AkiAuthorityCertSerialPresent, + + #[error( + "self-signed certificate authorityKeyIdentifier must equal subjectKeyIdentifier when present (RFC 6487 §4.8.3)" + )] + AkiSelfSignedNotEqualSki, + + #[error( + "CRLDistributionPoints must be present in non-self-signed certificates (RFC 6487 §4.8.6; RFC 5280 §4.2.1.13)" + )] + CrlDistributionPointsMissing, + + #[error( + "CRLDistributionPoints criticality must be non-critical (RFC 6487 §4.8.6; RFC 5280 §4.2.1.13)" + )] + CrlDistributionPointsCriticality, + + #[error("CRLDistributionPoints MUST be omitted in self-signed certificates (RFC 6487 §4.8.6)")] + CrlDistributionPointsSelfSignedMustOmit, + + #[error("CRLDistributionPoints must contain exactly one DistributionPoint (RFC 6487 §4.8.6)")] + CrlDistributionPointsNotSingle, + + #[error("CRLDistributionPoints distributionPoint field MUST be present (RFC 6487 §4.8.6)")] + CrlDistributionPointsNoDistributionPoint, + + #[error("CRLDistributionPoints reasons field MUST be omitted (RFC 6487 §4.8.6)")] + CrlDistributionPointsHasReasons, + + #[error("CRLDistributionPoints cRLIssuer field MUST be omitted (RFC 6487 §4.8.6)")] + CrlDistributionPointsHasCrlIssuer, + + #[error( + "CRLDistributionPoints distributionPoint MUST contain fullName and MUST NOT contain nameRelativeToCRLIssuer (RFC 6487 §4.8.6)" + )] + CrlDistributionPointsInvalidName, + + #[error( + "CRLDistributionPoints fullName must contain only URI GeneralNames (RFC 6487 §4.8.6; RFC 5280 §4.2.1.6)" + )] + CrlDistributionPointsFullNameNotUri, + + #[error("CRLDistributionPoints must include at least one rsync:// URI (RFC 6487 §4.8.6)")] + CrlDistributionPointsNoRsync, + + #[error( + "authorityInfoAccess must be present in non-self-signed certificates (RFC 6487 §4.8.7; RFC 5280 §4.2.2.1)" + )] + AuthorityInfoAccessMissing, + + #[error( + "authorityInfoAccess criticality must be non-critical (RFC 6487 §4.8.7; RFC 5280 §4.2.2.1)" + )] + AuthorityInfoAccessCriticality, + + #[error("authorityInfoAccess MUST be omitted in self-signed certificates (RFC 6487 §4.8.7)")] + AuthorityInfoAccessSelfSignedMustOmit, + + #[error( + "authorityInfoAccess id-ad-caIssuers accessLocation must be URI (RFC 6487 §4.8.7; RFC 5280 §4.2.2.1)" + )] + AuthorityInfoAccessCaIssuersNotUri, + + #[error("authorityInfoAccess must include at least one id-ad-caIssuers URI (RFC 6487 §4.8.7)")] + AuthorityInfoAccessMissingCaIssuers, + + #[error("authorityInfoAccess must include at least one rsync:// URI (RFC 6487 §4.8.7)")] + AuthorityInfoAccessNoRsync, +} + +#[derive(Debug, thiserror::Error)] +pub enum ResourceCertificateDecodeError { + #[error("{0}")] + Parse(#[from] ResourceCertificateParseError), + + #[error("{0}")] + Validate(#[from] ResourceCertificateProfileError), +} + +pub type ResourceCertificateError = ResourceCertificateDecodeError; diff --git a/crates/panda-rpki-validator/src/data_model/signed_object.rs b/crates/panda-rpki-validator/src/data_model/signed_object.rs index 524f0c0..0fe82b7 100644 --- a/crates/panda-rpki-validator/src/data_model/signed_object.rs +++ b/crates/panda-rpki-validator/src/data_model/signed_object.rs @@ -16,1614 +16,12 @@ use x509_parser::prelude::X509Certificate; use x509_parser::public_key::PublicKey; use x509_parser::x509::SubjectPublicKeyInfo; -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum EeKeyUsageSummary { - DigitalSignatureOnly, - Missing, - NotCritical, - InvalidBits, - ParseError(String), -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ResourceEeCertificate { - pub raw_der: Vec, - pub subject_key_identifier: Vec, - pub spki_der: Vec, - pub rsa_public_modulus: Vec, - pub rsa_public_exponent: Vec, - pub tbs_certificate_der: Vec, - pub signature_bytes: Vec, - pub key_usage_summary: EeKeyUsageSummary, - pub sia_signed_object_uris: Vec, - pub resource_cert: ResourceCertificate, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RpkiSignedObject { - pub raw_der: Vec, - pub content_info_content_type: String, - pub signed_data: SignedDataProfiled, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct SignedDataProfiled { - pub version: u32, - pub digest_algorithms: Vec, - pub encap_content_info: EncapsulatedContentInfo, - pub certificates: Vec, - pub crls_present: bool, - pub signer_infos: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct EncapsulatedContentInfo { - pub econtent_type: String, - pub econtent: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct SignerInfoProfiled { - pub version: u32, - pub sid_ski: Vec, - pub digest_algorithm: String, - pub signature_algorithm: String, - pub signed_attrs: SignedAttrsProfiled, - pub unsigned_attrs_present: bool, - pub signature: Vec, - pub signed_attrs_der_for_signature: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct SignedAttrsProfiled { - pub content_type: String, - pub message_digest: Vec, - pub signing_time: Asn1TimeUtc, - pub other_attrs_present: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RpkiSignedObjectParsed { - pub raw_der: Vec, - pub content_info_content_type: String, - pub signed_data: SignedDataParsed, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct SignedDataParsed { - pub version: u64, - pub digest_algorithms: Vec, - pub encap_content_info: EncapsulatedContentInfoParsed, - pub certificates: Option>>, - pub crls_present: bool, - pub signer_infos: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct AlgorithmIdentifierParsed { - pub oid: String, - pub params_ok: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct EncapsulatedContentInfoParsed { - pub econtent_type: String, - pub econtent: Option>, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct SignerInfoParsed { - pub version: u64, - pub sid: SignerIdentifierParsed, - pub digest_algorithm: AlgorithmIdentifierParsed, - pub signature_algorithm: AlgorithmIdentifierParsed, - pub signed_attrs_content: Option>, - pub signed_attrs_der_for_signature: Option>, - pub unsigned_attrs_present: bool, - pub signature: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum SignerIdentifierParsed { - SubjectKeyIdentifier(Vec), - Other, -} - -#[derive(Debug, thiserror::Error)] -pub enum SignedObjectParseError { - #[error("DER parse error: {0} (RFC 6488 §2; RFC 6488 §3(1l); RFC 5652 §3/§5)")] - Parse(String), - - #[error("trailing bytes after DER object: {0} bytes (DER; RFC 6488 §3(1l))")] - TrailingBytes(usize), -} - -#[derive(Debug, thiserror::Error)] -pub enum SignedObjectValidateError { - #[error( - "ContentInfo.contentType must be SignedData ({OID_SIGNED_DATA}), got {0} (RFC 6488 §3(1a); RFC 5652 §3)" - )] - InvalidContentInfoContentType(String), - - #[error( - "SignedData.version must be 3, got {0} (RFC 6488 §2.1.1; RFC 6488 §3(1b); RFC 5652 §5.1)" - )] - InvalidSignedDataVersion(u64), - - #[error( - "SignedData.digestAlgorithms must contain exactly one AlgorithmIdentifier, got {0} (RFC 6488 §2.1.2; RFC 6488 §3(1b); RFC 5652 §5.1)" - )] - InvalidDigestAlgorithmsCount(usize), - - #[error( - "digest algorithm must be id-sha256 ({OID_SHA256}), got {0} (RFC 6488 §2.1.2; RFC 6488 §3(1b); RFC 7935 §2)" - )] - InvalidDigestAlgorithm(String), - - #[error("SignedData.certificates MUST be present (RFC 6488 §3(1c); RFC 5652 §5.1)")] - CertificatesMissing, - - #[error( - "SignedData.certificates must contain exactly one EE certificate, got {0} (RFC 6488 §3(1c))" - )] - InvalidCertificatesCount(usize), - - #[error("SignedData.crls MUST be omitted (RFC 6488 §3(1d))")] - CrlsPresent, - - #[error( - "SignedData.signerInfos must contain exactly one SignerInfo, got {0} (RFC 6488 §2.1; RFC 6488 §3(1e); RFC 5652 §5.1)" - )] - InvalidSignerInfosCount(usize), - - #[error("SignerInfo.version must be 3, got {0} (RFC 6488 §3(1e); RFC 5652 §5.3)")] - InvalidSignerInfoVersion(u64), - - #[error("SignerInfo.sid must be subjectKeyIdentifier [0] (RFC 6488 §3(1c); RFC 5652 §5.3)")] - InvalidSignerIdentifier, - - #[error( - "SignerInfo.digestAlgorithm must be id-sha256 ({OID_SHA256}), got {0} (RFC 6488 §3(1j); RFC 7935 §2)" - )] - InvalidSignerInfoDigestAlgorithm(String), - - #[error("SignerInfo.signedAttrs MUST be present (RFC 9589 §4; RFC 6488 §3(1f))")] - SignedAttrsMissing, - - #[error("SignerInfo.unsignedAttrs MUST be omitted (RFC 6488 §3(1i))")] - UnsignedAttrsPresent, - - #[error( - "SignerInfo.signatureAlgorithm must be rsaEncryption ({OID_RSA_ENCRYPTION}) or \ -sha256WithRSAEncryption ({OID_SHA256_WITH_RSA_ENCRYPTION}), got {0} (RFC 6488 §3(1k); RFC 7935 §2)" - )] - InvalidSignatureAlgorithm(String), - - #[error( - "SignerInfo.signatureAlgorithm parameters must be absent or NULL (RFC 5280 §4.1.1.2; RFC 7935 §2)" - )] - InvalidSignatureAlgorithmParameters, - - #[error("signedAttrs contains unsupported attribute OID {0} (RFC 9589 §4; RFC 6488 §2.1.6.4)")] - UnsupportedSignedAttribute(String), - - #[error("signedAttrs contains duplicate attribute OID {0} (RFC 6488 §2.1.6.4; RFC 9589 §4)")] - DuplicateSignedAttribute(String), - - #[error("signedAttrs parse error: {0} (RFC 5652 §5.3; RFC 6488 §3(1f); RFC 9589 §4)")] - SignedAttrsParse(String), - - #[error( - "signedAttrs attribute {oid} attrValues must contain exactly one value, got {count} (RFC 6488 §2.1.6.4; RFC 5652 §5.3)" - )] - InvalidSignedAttributeValuesCount { oid: String, count: usize }, - - #[error( - "signedAttrs missing content-type attribute (RFC 9589 §4; RFC 5652 §11.1; RFC 6488 §2.1.6.4)" - )] - SignedAttrsContentTypeMissing, - - #[error( - "signedAttrs missing message-digest attribute (RFC 9589 §4; RFC 5652 §11.2; RFC 6488 §2.1.6.4)" - )] - SignedAttrsMessageDigestMissing, - - #[error( - "signedAttrs missing signing-time attribute (RFC 9589 §4; RFC 5652 §11.3; RFC 6488 §2.1.6.4)" - )] - SignedAttrsSigningTimeMissing, - - #[error( - "signedAttrs.content-type attrValues must equal eContentType ({econtent_type}), got {attr_content_type} (RFC 6488 §3(1h); RFC 9589 §4)" - )] - ContentTypeAttrMismatch { - econtent_type: String, - attr_content_type: String, - }, - - #[error("EncapsulatedContentInfo.eContent MUST be present (RFC 6488 §2.1.3; RFC 5652 §5.2)")] - EContentMissing, - - #[error( - "signedAttrs.message-digest does not match SHA-256(eContent) (RFC 6488 §3(1f); RFC 5652 §11.2)" - )] - MessageDigestMismatch, - - #[error("EE certificate parse error: {0} (RFC 6488 §3(1c); RFC 6487 §4)")] - EeCertificateParse(String), - - #[error( - "EE certificate missing SubjectKeyIdentifier extension (RFC 6488 §3(1c); RFC 6487 §4.8.2)" - )] - EeCertificateMissingSki, - - #[error( - "EE certificate missing SubjectInfoAccess extension ({OID_SUBJECT_INFO_ACCESS}) (RFC 6487 §4.8.8.2)" - )] - EeCertificateMissingSia, - - #[error( - "EE certificate SIA missing id-ad-signedObject access method ({OID_AD_SIGNED_OBJECT}) (RFC 6487 §4.8.8.2)" - )] - EeCertificateMissingSignedObjectSia, - - #[error( - "EE certificate SIA id-ad-signedObject accessLocation must be a URI (RFC 6487 §4.8.8.2; RFC 5280 §4.2.2.2)" - )] - EeCertificateSignedObjectSiaNotUri, - - #[error( - "EE certificate SIA id-ad-signedObject must include at least one rsync:// URI (RFC 6487 §4.8.8.2)" - )] - EeCertificateSignedObjectSiaNoRsync, - - #[error( - "SignerInfo.sid SKI does not match EE certificate SKI (RFC 6488 §3(1c); RFC 5652 §5.3)" - )] - SidSkiMismatch, - - #[error( - "invalid signing-time attribute value (expected UTCTime or GeneralizedTime) (RFC 5652 §11.3; RFC 9589 §4)" - )] - InvalidSigningTimeValue, -} - -#[derive(Debug, thiserror::Error)] -pub enum SignedObjectDecodeError { - #[error("SignedObject parse error: {0}")] - Parse(#[from] SignedObjectParseError), - - #[error("SignedObject validate error: {0}")] - Validate(#[from] SignedObjectValidateError), -} - -#[derive(Debug, thiserror::Error)] -pub enum SignedObjectVerifyError { - #[error("EE SubjectPublicKeyInfo parse error: {0} (RFC 5280 §4.1.2.7)")] - EeSpkiParse(String), - - #[error("trailing bytes after EE SubjectPublicKeyInfo DER: {0} bytes (DER; RFC 5280 §4.1.2.7)")] - EeSpkiTrailingBytes(usize), - - #[error("unsupported EE public key algorithm (only RSA supported in M3) (RFC 7935 §2)")] - UnsupportedEePublicKeyAlgorithm, - - #[error("EE RSA public exponent invalid (RFC 8017 §A.1.1; RFC 7935 §2)")] - InvalidEeRsaExponent, - - #[error("signature verification failed (RFC 6488 §3(2)-(3); RFC 5652 §5.3; RFC 7935 §2)")] - InvalidSignature, -} - -impl RpkiSignedObject { - /// Parse a DER-encoded RPKI Signed Object (CMS ContentInfo wrapping SignedData). - /// - /// This performs encoding/structure parsing only. Profile constraints are enforced by - /// `RpkiSignedObjectParsed::validate_profile`. - pub fn parse_der(der: &[u8]) -> Result { - parse_signed_object_content_info(der, der, CmsParseMode::BerCompatible) - } - - pub fn parse_der_strict_cms( - der: &[u8], - ) -> Result { - parse_signed_object_content_info(der, der, CmsParseMode::DerStrict) - } - - /// Return the strict-DER CMS parse error for an object that was otherwise - /// accepted through the normal BER-compatible CMS parser. - /// - /// Callers must only surface this as a compatibility warning after normal - /// decoding and validation have succeeded; a strict parse failure alone - /// does not prove that an arbitrary byte string is an RPKI signed object. - pub fn strict_cms_der_error(der: &[u8]) -> Option { - Self::parse_der_strict_cms(der).err() - } - - /// Decode a DER-encoded RPKI Signed Object (CMS ContentInfo wrapping SignedData) and enforce - /// the profile constraints from RFC 6488 §2-§3 and RFC 9589 §4. - pub fn decode_der(der: &[u8]) -> Result { - let parsed = Self::parse_der(der)?; - Ok(parsed.validate_profile()?) - } - - pub fn decode_der_with_strict_options( - der: &[u8], - strict_cms_der: bool, - strict_name: bool, - ) -> Result { - let parsed = if strict_cms_der { - Self::parse_der_strict_cms(der)? - } else { - Self::parse_der(der)? - }; - Ok(parsed.validate_profile_with_strict_name(strict_name)?) - } - - /// Scheme-A naming for signature verification. - pub fn verify(&self) -> Result<(), SignedObjectVerifyError> { - self.verify_signature() - } - - /// Verify the CMS signature using the embedded EE certificate public key. - pub fn verify_signature(&self) -> Result<(), SignedObjectVerifyError> { - let ee = &self.signed_data.certificates[0]; - let signer = &self.signed_data.signer_infos[0]; - crate::crypto_sig_cache::verify_with_cache( - crate::crypto_sig_cache::CryptoSigVerifyPoint::CmsSignedObject, - &signer.signed_attrs_der_for_signature, - &signer.signature, - &ee.spki_der, - || { - self.verify_signature_with_rsa_components( - &ee.rsa_public_modulus, - &ee.rsa_public_exponent, - ) - }, - ) - } - - /// Verify the CMS signature using a DER-encoded SubjectPublicKeyInfo. - pub fn verify_signature_with_ee_spki_der( - &self, - ee_spki_der: &[u8], - ) -> Result<(), SignedObjectVerifyError> { - let (rem, spki) = SubjectPublicKeyInfo::from_der(ee_spki_der) - .map_err(|e| SignedObjectVerifyError::EeSpkiParse(e.to_string()))?; - if !rem.is_empty() { - return Err(SignedObjectVerifyError::EeSpkiTrailingBytes(rem.len())); - } - self.verify_signature_with_ee_spki(&spki) - } - - /// Verify the CMS signature using a parsed SubjectPublicKeyInfo. - pub fn verify_signature_with_ee_spki( - &self, - ee_spki: &SubjectPublicKeyInfo<'_>, - ) -> Result<(), SignedObjectVerifyError> { - let pk = ee_spki - .parsed() - .map_err(|_e| SignedObjectVerifyError::UnsupportedEePublicKeyAlgorithm)?; - let (n, e) = match pk { - PublicKey::RSA(rsa) => { - let n = strip_leading_zeros(rsa.modulus).to_vec(); - let e = strip_leading_zeros(rsa.exponent).to_vec(); - let _exp = rsa - .try_exponent() - .map_err(|_e| SignedObjectVerifyError::InvalidEeRsaExponent)?; - (n, e) - } - _ => return Err(SignedObjectVerifyError::UnsupportedEePublicKeyAlgorithm), - }; - - let signer = &self.signed_data.signer_infos[0]; - crate::crypto_sig_cache::verify_with_cache( - crate::crypto_sig_cache::CryptoSigVerifyPoint::CmsSignedObject, - &signer.signed_attrs_der_for_signature, - &signer.signature, - ee_spki.raw, - || self.verify_signature_with_rsa_components(n.as_slice(), e.as_slice()), - ) - } - - fn verify_signature_with_rsa_components( - &self, - modulus: &[u8], - exponent: &[u8], - ) -> Result<(), SignedObjectVerifyError> { - let signer = &self.signed_data.signer_infos[0]; - let msg = &signer.signed_attrs_der_for_signature; - - let pk = ring::signature::RsaPublicKeyComponents { - n: modulus, - e: exponent, - }; - pk.verify( - &ring::signature::RSA_PKCS1_2048_8192_SHA256, - msg, - &signer.signature, - ) - .map_err(|_e| SignedObjectVerifyError::InvalidSignature) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum CmsParseMode { - BerCompatible, - DerStrict, -} - -struct CmsReader<'a> { - buf: &'a [u8], - mode: CmsParseMode, -} - -impl<'a> CmsReader<'a> { - fn new(buf: &'a [u8], mode: CmsParseMode) -> Self { - Self { buf, mode } - } - - fn is_empty(&self) -> bool { - self.buf.is_empty() - } - - fn remaining_len(&self) -> usize { - self.buf.len() - } - - fn peek_tag(&self) -> Result { - let (_rem, any) = parse_any(self.buf, self.mode)?; - header_to_single_byte_tag(&any.header) - } - - fn take_any(&mut self) -> Result<(u8, &'a [u8]), String> { - let (rem, any) = parse_any(self.buf, self.mode)?; - let tag = header_to_single_byte_tag(&any.header)?; - self.buf = rem; - Ok((tag, any.data)) - } - - fn take_any_full(&mut self) -> Result<(u8, &'a [u8], &'a [u8]), String> { - let (rem, any) = parse_any(self.buf, self.mode)?; - let consumed = self.buf.len() - rem.len(); - let full = &self.buf[..consumed]; - let tag = header_to_single_byte_tag(&any.header)?; - self.buf = rem; - Ok((tag, full, any.data)) - } - - fn skip_any(&mut self) -> Result<(), String> { - let _ = self.take_any()?; - Ok(()) - } - - fn take_tag(&mut self, expected_tag: u8) -> Result<&'a [u8], String> { - let (tag, value) = self.take_any()?; - if tag != expected_tag { - return Err(format!( - "unexpected tag: got 0x{tag:02X}, expected 0x{expected_tag:02X}" - )); - } - Ok(value) - } - - fn take_sequence(&mut self) -> Result, String> { - let value = self.take_tag(0x30)?; - Ok(CmsReader::new(value, self.mode)) - } - - fn take_octet_string(&mut self) -> Result, String> { - let (rem, any) = parse_any(self.buf, self.mode)?; - let tag = header_to_single_byte_tag(&any.header)?; - if self.mode == CmsParseMode::DerStrict && tag != 0x04 { - return Err(format!( - "unexpected tag in DER strict mode: got 0x{tag:02X}, expected 0x04" - )); - } - if tag != 0x04 && tag != 0x24 { - return Err(format!("unexpected tag: got 0x{tag:02X}, expected 0x04")); - } - let octets = flatten_octet_string(any, self.mode)?; - self.buf = rem; - Ok(octets) - } - - fn take_uint_u64(&mut self) -> Result { - let value = self.take_tag(0x02)?; - der_uint_from_bytes(value) - } - - fn take_explicit(&mut self, expected_outer_tag: u8) -> Result<(u8, &'a [u8]), String> { - let inner_der = self.take_tag(expected_outer_tag)?; - let (tag, value, rem) = cms_take_tlv(inner_der, self.mode)?; - if !rem.is_empty() { - return Err("trailing bytes inside EXPLICIT value".into()); - } - Ok((tag, value)) - } - - fn take_explicit_der(&mut self, expected_outer_tag: u8) -> Result<&'a [u8], String> { - let inner_der = self.take_tag(expected_outer_tag)?; - let (_tag, _value, rem) = cms_take_tlv(inner_der, self.mode)?; - if !rem.is_empty() { - return Err("trailing bytes inside EXPLICIT value".into()); - } - Ok(inner_der) - } -} - -fn parse_signed_object_content_info( - raw_der: &[u8], - parse_der: &[u8], - mode: CmsParseMode, -) -> Result { - let mut r = CmsReader::new(parse_der, mode); - let mut content_info_seq = r - .take_sequence() - .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; - if !r.is_empty() { - return Err(SignedObjectParseError::TrailingBytes(r.remaining_len())); - } - - let content_type = take_oid_string(&mut content_info_seq)?; - let signed_data = parse_signed_data_from_contentinfo_cursor(&mut content_info_seq)?; - if !content_info_seq.is_empty() { - return Err(SignedObjectParseError::Parse( - "ContentInfo must be a SEQUENCE of 2 elements".into(), - )); - } - - Ok(RpkiSignedObjectParsed { - raw_der: raw_der.to_vec(), - content_info_content_type: content_type, - signed_data, - }) -} - -fn parse_any<'a>(input: &'a [u8], mode: CmsParseMode) -> Result<(&'a [u8], Any<'a>), String> { - match mode { - CmsParseMode::BerCompatible => { - Any::from_ber(input).map_err(|e| format!("BER parse error: {e}")) - } - CmsParseMode::DerStrict => { - Any::from_der(input).map_err(|e| format!("DER parse error: {e}")) - } - } -} - -fn header_to_single_byte_tag(header: &Header<'_>) -> Result { - let tag_no = header.tag().0; - if tag_no > 30 { - return Err(format!("high-tag-number form not supported: {tag_no}")); - } - Ok(((header.class() as u8) << 6) - | if header.constructed() { 0x20 } else { 0x00 } - | tag_no as u8) -} - -fn cms_take_tlv(input: &[u8], mode: CmsParseMode) -> Result<(u8, &[u8], &[u8]), String> { - let (rem, any) = parse_any(input, mode)?; - let tag = header_to_single_byte_tag(&any.header)?; - Ok((tag, any.data, rem)) -} - -fn flatten_octet_string(any: Any<'_>, mode: CmsParseMode) -> Result, String> { - if any.class() != Class::Universal || any.tag() != Tag::OctetString { - return Err("expected OCTET STRING".into()); - } - if !any.header.constructed() { - return Ok(any.data.to_vec()); - } - if mode == CmsParseMode::DerStrict { - return Err("constructed OCTET STRING is not allowed in DER strict mode".into()); - } - let mut out = Vec::new(); - let mut input = any.data; - while !input.is_empty() { - let (rem, child) = Any::from_ber(input).map_err(|e| format!("BER parse error: {e}"))?; - out.extend(flatten_octet_string(child, mode)?); - input = rem; - } - Ok(out) -} - -impl RpkiSignedObjectParsed { - pub fn validate_profile(self) -> Result { - self.validate_profile_with_strict_name(false) - } - - pub fn validate_profile_with_strict_name( - self, - strict_name: bool, - ) -> Result { - if self.content_info_content_type != OID_SIGNED_DATA { - return Err(SignedObjectValidateError::InvalidContentInfoContentType( - self.content_info_content_type, - )); - } - - let signed_data = validate_signed_data_profile(self.signed_data, strict_name)?; - - Ok(RpkiSignedObject { - raw_der: self.raw_der, - content_info_content_type: OID_SIGNED_DATA.to_string(), - signed_data, - }) - } -} - -fn parse_signed_data_from_contentinfo_cursor( - seq: &mut CmsReader<'_>, -) -> Result { - let inner_der = seq.take_explicit_der(0xA0).map_err(|_e| { - SignedObjectParseError::Parse("ContentInfo.content must be [0] EXPLICIT".into()) - })?; - let mut r = CmsReader::new(inner_der, seq.mode); - let signed_data_seq = r - .take_sequence() - .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; - if !r.is_empty() { - return Err(SignedObjectParseError::Parse( - "trailing bytes inside ContentInfo.content".into(), - )); - } - parse_signed_data_cursor(signed_data_seq) -} - -fn parse_signed_data_cursor( - mut seq: CmsReader<'_>, -) -> Result { - let version = seq - .take_uint_u64() - .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; - - let digest_set_bytes = seq - .take_tag(0x31) - .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; - let mut digest_set = CmsReader::new(digest_set_bytes, seq.mode); - let mut digest_algorithms: Vec = Vec::new(); - while !digest_set.is_empty() { - let alg = digest_set - .take_sequence() - .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; - let (oid, params_ok) = parse_algorithm_identifier_cursor(alg)?; - digest_algorithms.push(AlgorithmIdentifierParsed { oid, params_ok }); - } - - let encap_content_info = parse_encapsulated_content_info_cursor( - seq.take_sequence() - .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?, - )?; - - let mut certificates: Option>> = None; - let mut crls_present = false; - let mut signer_infos: Option> = None; - - while !seq.is_empty() { - let tag = seq - .peek_tag() - .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; - match tag { - 0xA0 => { - if certificates.is_some() { - return Err(SignedObjectParseError::Parse( - "SignedData.certificates appears more than once".into(), - )); - } - let content = seq - .take_tag(0xA0) - .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; - certificates = Some(split_der_objects(content, seq.mode)?); - } - 0xA1 => { - crls_present = true; - seq.skip_any() - .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; - } - 0x31 => { - if signer_infos.is_some() { - return Err(SignedObjectParseError::Parse( - "SignedData.signerInfos appears more than once".into(), - )); - } - let set_bytes = seq - .take_tag(0x31) - .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; - signer_infos = Some(parse_signer_infos_set_cursor(set_bytes, seq.mode)?); - } - _ => { - return Err(SignedObjectParseError::Parse( - "unexpected field in SignedData".into(), - )); - } - } - } - - let signer_infos = signer_infos - .ok_or_else(|| SignedObjectParseError::Parse("SignedData.signerInfos missing".into()))?; - - Ok(SignedDataParsed { - version, - digest_algorithms, - encap_content_info, - certificates, - crls_present, - signer_infos, - }) -} - -fn parse_encapsulated_content_info_cursor( - mut seq: CmsReader<'_>, -) -> Result { - if seq.is_empty() { - return Err(SignedObjectParseError::Parse( - "EncapsulatedContentInfo must be SEQUENCE of 1..2".into(), - )); - } - - let econtent_type = take_oid_string(&mut seq)?; - - let econtent = if seq.is_empty() { - None - } else { - let inner_der = seq.take_explicit_der(0xA0).map_err(|_e| { - SignedObjectParseError::Parse( - "EncapsulatedContentInfo.eContent must be [0] EXPLICIT".into(), - ) - })?; - let mut inner = CmsReader::new(inner_der, seq.mode); - let octets = inner - .take_octet_string() - .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; - if !inner.is_empty() { - return Err(SignedObjectParseError::Parse( - "trailing bytes inside EncapsulatedContentInfo.eContent".into(), - )); - } - Some(octets) - }; - if !seq.is_empty() { - return Err(SignedObjectParseError::Parse( - "EncapsulatedContentInfo must be SEQUENCE of 1..2".into(), - )); - } - - Ok(EncapsulatedContentInfoParsed { - econtent_type, - econtent, - }) -} - -fn split_der_objects( - mut input: &[u8], - mode: CmsParseMode, -) -> Result>, SignedObjectParseError> { - let mut out: Vec> = Vec::new(); - while !input.is_empty() { - let (_tag, _value, rem) = - cms_take_tlv(input, mode).map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; - let consumed = input.len() - rem.len(); - out.push(input[..consumed].to_vec()); - input = rem; - } - Ok(out) -} - -fn parse_signer_infos_set_cursor( - set_bytes: &[u8], - mode: CmsParseMode, -) -> Result, SignedObjectParseError> { - let mut set = CmsReader::new(set_bytes, mode); - let mut out: Vec = Vec::new(); - while !set.is_empty() { - let si = set - .take_sequence() - .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; - out.push(parse_signer_info_cursor(si)?); - } - Ok(out) -} - -fn validate_ee_certificate( - der: &[u8], - strict_name: bool, -) -> Result { - let (rem, cert) = X509Certificate::from_der(der) - .map_err(|e| SignedObjectValidateError::EeCertificateParse(e.to_string()))?; - if !rem.is_empty() { - return Err(SignedObjectValidateError::EeCertificateParse(format!( - "trailing bytes after EE certificate DER: {}", - rem.len() - ))); - } - - let rc = match ResourceCertificate::from_der(der) { - Ok(v) => v, - Err(e) => { - return match e { - crate::data_model::rc::ResourceCertificateDecodeError::Validate( - crate::data_model::rc::ResourceCertificateProfileError::SignedObjectSiaNotUri, - ) => Err(SignedObjectValidateError::EeCertificateSignedObjectSiaNotUri), - crate::data_model::rc::ResourceCertificateDecodeError::Validate( - crate::data_model::rc::ResourceCertificateProfileError::SignedObjectSiaNoRsync, - ) => Err(SignedObjectValidateError::EeCertificateSignedObjectSiaNoRsync), - _ => Err(SignedObjectValidateError::EeCertificateParse(e.to_string())), - }; - } - }; - rc.validate_rfc6487_profile(ResourceCertificateRole::SignedObjectEe) - .map_err(|e| SignedObjectValidateError::EeCertificateParse(e.to_string()))?; - if strict_name { - rc.validate_strict_name_profile() - .map_err(|e| SignedObjectValidateError::EeCertificateParse(e.to_string()))?; - } - - let ski = rc - .tbs - .extensions - .subject_key_identifier - .clone() - .ok_or(SignedObjectValidateError::EeCertificateMissingSki)?; - - let spki_der = rc.tbs.subject_public_key_info.clone(); - let (rem, spki) = SubjectPublicKeyInfo::from_der(&spki_der) - .map_err(|e| SignedObjectValidateError::EeCertificateParse(e.to_string()))?; - if !rem.is_empty() { - return Err(SignedObjectValidateError::EeCertificateParse( - "trailing bytes after EE SubjectPublicKeyInfo DER".to_string(), - )); - } - let parsed_pk = spki.parsed().map_err(|_e| { - SignedObjectValidateError::EeCertificateParse( - "unsupported EE public key algorithm".to_string(), - ) - })?; - let (rsa_public_modulus, rsa_public_exponent) = match parsed_pk { - PublicKey::RSA(rsa) => { - let modulus = strip_leading_zeros(rsa.modulus).to_vec(); - let exponent = strip_leading_zeros(rsa.exponent).to_vec(); - let _ = rsa.try_exponent().map_err(|_e| { - SignedObjectValidateError::EeCertificateParse("invalid EE RSA exponent".to_string()) - })?; - (modulus, exponent) - } - _ => { - return Err(SignedObjectValidateError::EeCertificateParse( - "unsupported EE public key algorithm".to_string(), - )); - } - }; - - let sia = rc - .tbs - .extensions - .subject_info_access - .as_ref() - .ok_or(SignedObjectValidateError::EeCertificateMissingSia)?; - let signed_object_uris: Vec = match sia { - SubjectInfoAccess::Ee(ee) => ee.signed_object_uris.clone(), - SubjectInfoAccess::Ca(_ca) => Vec::new(), - }; - if signed_object_uris.is_empty() { - return Err(SignedObjectValidateError::EeCertificateMissingSignedObjectSia); - } - if !signed_object_uris.iter().any(|u| u.starts_with("rsync://")) { - return Err(SignedObjectValidateError::EeCertificateSignedObjectSiaNoRsync); - } - - Ok(ResourceEeCertificate { - raw_der: der.to_vec(), - subject_key_identifier: ski, - spki_der, - rsa_public_modulus, - rsa_public_exponent, - tbs_certificate_der: cert.tbs_certificate.as_ref().to_vec(), - signature_bytes: cert.signature_value.data.to_vec(), - key_usage_summary: summarize_ee_key_usage(&cert), - sia_signed_object_uris: signed_object_uris, - resource_cert: rc, - }) -} - -fn summarize_ee_key_usage(cert: &X509Certificate<'_>) -> EeKeyUsageSummary { - for ext in cert.extensions() { - if ext.oid.as_bytes() == OID_KEY_USAGE_RAW { - match ext.parsed_extension() { - ParsedExtension::KeyUsage(ku) => { - if !ext.critical { - return EeKeyUsageSummary::NotCritical; - } - let ok = ku.digital_signature() - && !ku.key_cert_sign() - && !ku.crl_sign() - && !ku.non_repudiation() - && !ku.key_encipherment() - && !ku.data_encipherment() - && !ku.key_agreement() - && !ku.encipher_only() - && !ku.decipher_only(); - return if ok { - EeKeyUsageSummary::DigitalSignatureOnly - } else { - EeKeyUsageSummary::InvalidBits - }; - } - other => { - return EeKeyUsageSummary::ParseError(format!( - "unexpected parsed keyUsage extension: {other:?}" - )); - } - } - } - } - - EeKeyUsageSummary::Missing -} - -fn parse_signer_info_cursor( - mut seq: CmsReader<'_>, -) -> Result { - let version = seq - .take_uint_u64() - .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; - - let (sid_tag, sid_bytes) = seq - .take_any() - .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; - let sid = if (sid_tag & 0xC0) == 0x80 && (sid_tag & 0x1F) == 0 { - SignerIdentifierParsed::SubjectKeyIdentifier(sid_bytes.to_vec()) - } else { - SignerIdentifierParsed::Other - }; - - let digest_alg_seq = seq - .take_sequence() - .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; - let (digest_oid, digest_params_ok) = parse_algorithm_identifier_cursor(digest_alg_seq)?; - let digest_algorithm = AlgorithmIdentifierParsed { - oid: digest_oid, - params_ok: digest_params_ok, - }; - - let mut signed_attrs_content: Option> = None; - let mut signed_attrs_der_for_signature: Option> = None; - - if seq - .peek_tag() - .map_err(|e| SignedObjectParseError::Parse(e.to_string()))? - == 0xA0 - { - let (tag, full_tlv, value) = seq - .take_any_full() - .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; - if tag != 0xA0 { - return Err(SignedObjectParseError::Parse( - "SignerInfo.signedAttrs must be [0] IMPLICIT".into(), - )); - } - signed_attrs_content = Some(value.to_vec()); - signed_attrs_der_for_signature = Some(make_signed_attrs_der_for_signature(full_tlv)?); - } - - let sig_alg_seq = seq - .take_sequence() - .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; - let (signature_oid, signature_params_ok) = parse_algorithm_identifier_cursor(sig_alg_seq)?; - let signature_algorithm = AlgorithmIdentifierParsed { - oid: signature_oid, - params_ok: signature_params_ok, - }; - - let signature = seq - .take_octet_string() - .map_err(|e| SignedObjectParseError::Parse(e.to_string()))? - .to_vec(); - - let unsigned_attrs_present = !seq.is_empty(); - - Ok(SignerInfoParsed { - version, - sid, - digest_algorithm, - signature_algorithm, - signed_attrs_content, - signed_attrs_der_for_signature, - unsigned_attrs_present, - signature, - }) -} - -fn validate_signed_data_profile( - signed_data: SignedDataParsed, - strict_name: bool, -) -> Result { - if signed_data.version != 3 { - return Err(SignedObjectValidateError::InvalidSignedDataVersion( - signed_data.version, - )); - } - - if signed_data.digest_algorithms.len() != 1 { - return Err(SignedObjectValidateError::InvalidDigestAlgorithmsCount( - signed_data.digest_algorithms.len(), - )); - } - let digest_alg = &signed_data.digest_algorithms[0]; - if digest_alg.oid != OID_SHA256 { - return Err(SignedObjectValidateError::InvalidDigestAlgorithm( - digest_alg.oid.clone(), - )); - } - - if signed_data.crls_present { - return Err(SignedObjectValidateError::CrlsPresent); - } - - let econtent = signed_data - .encap_content_info - .econtent - .clone() - .ok_or(SignedObjectValidateError::EContentMissing)?; - if econtent.is_empty() { - return Err(SignedObjectValidateError::EContentMissing); - } - let encap_content_info = EncapsulatedContentInfo { - econtent_type: signed_data.encap_content_info.econtent_type.clone(), - econtent: econtent.clone(), - }; - - let certs = signed_data - .certificates - .as_ref() - .ok_or(SignedObjectValidateError::CertificatesMissing)?; - if certs.len() != 1 { - return Err(SignedObjectValidateError::InvalidCertificatesCount( - certs.len(), - )); - } - let ee = validate_ee_certificate(&certs[0], strict_name)?; - - if signed_data.signer_infos.len() != 1 { - return Err(SignedObjectValidateError::InvalidSignerInfosCount( - signed_data.signer_infos.len(), - )); - } - let signer = &signed_data.signer_infos[0]; - - if signer.version != 3 { - return Err(SignedObjectValidateError::InvalidSignerInfoVersion( - signer.version, - )); - } - let sid_ski = match &signer.sid { - SignerIdentifierParsed::SubjectKeyIdentifier(ski) => ski.clone(), - SignerIdentifierParsed::Other => { - return Err(SignedObjectValidateError::InvalidSignerIdentifier); - } - }; - - if signer.digest_algorithm.oid != OID_SHA256 { - return Err(SignedObjectValidateError::InvalidSignerInfoDigestAlgorithm( - signer.digest_algorithm.oid.clone(), - )); - } - - let signed_attrs_content = signer - .signed_attrs_content - .as_deref() - .ok_or(SignedObjectValidateError::SignedAttrsMissing)?; - let signed_attrs_der_for_signature = signer - .signed_attrs_der_for_signature - .clone() - .ok_or(SignedObjectValidateError::SignedAttrsMissing)?; - let signed_attrs = parse_signed_attrs_implicit(signed_attrs_content)?; - - if signer.unsigned_attrs_present { - return Err(SignedObjectValidateError::UnsignedAttrsPresent); - } - - if !signer.signature_algorithm.params_ok { - return Err(SignedObjectValidateError::InvalidSignatureAlgorithmParameters); - } - let signature_algorithm = signer.signature_algorithm.oid.clone(); - if signature_algorithm != OID_RSA_ENCRYPTION - && signature_algorithm != OID_SHA256_WITH_RSA_ENCRYPTION - { - return Err(SignedObjectValidateError::InvalidSignatureAlgorithm( - signature_algorithm, - )); - } - - if sid_ski != ee.subject_key_identifier { - return Err(SignedObjectValidateError::SidSkiMismatch); - } - if signed_attrs.content_type != encap_content_info.econtent_type { - return Err(SignedObjectValidateError::ContentTypeAttrMismatch { - econtent_type: encap_content_info.econtent_type.clone(), - attr_content_type: signed_attrs.content_type.clone(), - }); - } - - let computed = digest::digest(&digest::SHA256, &encap_content_info.econtent); - if computed.as_ref() != signed_attrs.message_digest.as_slice() { - return Err(SignedObjectValidateError::MessageDigestMismatch); - } - - Ok(SignedDataProfiled { - version: 3, - digest_algorithms: vec![OID_SHA256.to_string()], - encap_content_info, - certificates: vec![ee.clone()], - crls_present: false, - signer_infos: vec![SignerInfoProfiled { - version: 3, - sid_ski, - digest_algorithm: OID_SHA256.to_string(), - signature_algorithm: signer.signature_algorithm.oid.clone(), - signed_attrs, - unsigned_attrs_present: false, - signature: signer.signature.clone(), - signed_attrs_der_for_signature, - }], - }) -} - -fn parse_signed_attrs_implicit( - input: &[u8], -) -> Result { - let mut content_type: Option = None; - let mut message_digest: Option> = None; - let mut signing_time: Option = None; - - fn count_elements(mut r: DerReader<'_>) -> Result { - let mut n = 0usize; - while !r.is_empty() { - r.skip_any()?; - n += 1; - } - Ok(n) - } - - let mut remaining = DerReader::new(input); - while !remaining.is_empty() { - let mut attr = remaining - .take_sequence() - .map_err(|e| SignedObjectValidateError::SignedAttrsParse(e.to_string()))?; - - let oid_bytes = attr - .take_tag(0x06) - .map_err(|e| SignedObjectValidateError::SignedAttrsParse(e.to_string()))?; - let oid = oid_value_bytes_to_string(oid_bytes); - - let values_bytes = attr - .take_tag(0x31) - .map_err(|e| SignedObjectValidateError::SignedAttrsParse(e.to_string()))?; - if !attr.is_empty() { - return Err(SignedObjectValidateError::SignedAttrsParse( - "Attribute must be SEQUENCE of 2".into(), - )); - } - - let mut values = DerReader::new(values_bytes); - let count = if values.is_empty() { - 0 - } else { - values - .skip_any() - .map_err(|e| SignedObjectValidateError::SignedAttrsParse(e.to_string()))?; - if values.is_empty() { - 1 - } else { - 1 + count_elements(values) - .map_err(|e| SignedObjectValidateError::SignedAttrsParse(e.to_string()))? - } - }; - if count != 1 { - return Err( - SignedObjectValidateError::InvalidSignedAttributeValuesCount { oid, count }, - ); - } - - // Re-parse the sole value. - let mut values = DerReader::new(values_bytes); - let (val_tag, val_bytes) = values - .take_any() - .map_err(|e| SignedObjectValidateError::SignedAttrsParse(e.to_string()))?; - - match oid.as_str() { - OID_CMS_ATTR_CONTENT_TYPE => { - if content_type.is_some() { - return Err(SignedObjectValidateError::DuplicateSignedAttribute(oid)); - } - if val_tag != 0x06 { - return Err(SignedObjectValidateError::SignedAttrsParse( - "content-type attr value must be OBJECT IDENTIFIER".into(), - )); - } - content_type = Some(oid_value_bytes_to_string(val_bytes)); - } - OID_CMS_ATTR_MESSAGE_DIGEST => { - if message_digest.is_some() { - return Err(SignedObjectValidateError::DuplicateSignedAttribute(oid)); - } - if val_tag != 0x04 { - return Err(SignedObjectValidateError::SignedAttrsParse( - "message-digest attr value must be OCTET STRING".into(), - )); - } - message_digest = Some(val_bytes.to_vec()); - } - OID_CMS_ATTR_SIGNING_TIME => { - if signing_time.is_some() { - return Err(SignedObjectValidateError::DuplicateSignedAttribute(oid)); - } - signing_time = Some(parse_signing_time_value_tlv(val_tag, val_bytes)?); - } - _ => { - return Err(SignedObjectValidateError::UnsupportedSignedAttribute(oid)); - } - } - } - - Ok(SignedAttrsProfiled { - content_type: content_type - .ok_or(SignedObjectValidateError::SignedAttrsContentTypeMissing)?, - message_digest: message_digest - .ok_or(SignedObjectValidateError::SignedAttrsMessageDigestMissing)?, - signing_time: signing_time - .ok_or(SignedObjectValidateError::SignedAttrsSigningTimeMissing)?, - other_attrs_present: false, - }) -} - -fn parse_signing_time_value_tlv( - tag: u8, - value: &[u8], -) -> Result { - match tag { - 0x17 => Ok(Asn1TimeUtc { - utc: parse_utctime(value)?, - encoding: Asn1TimeEncoding::UtcTime, - }), - 0x18 => Ok(Asn1TimeUtc { - utc: parse_generalized_time(value)?, - encoding: Asn1TimeEncoding::GeneralizedTime, - }), - _ => Err(SignedObjectValidateError::InvalidSigningTimeValue), - } -} - -fn make_signed_attrs_der_for_signature(full_tlv: &[u8]) -> Result, SignedObjectParseError> { - // We need the DER encoding of SignedAttributes (SET OF Attribute) as signature input. - // The SignedAttributes field in SignerInfo is `[0] IMPLICIT`, so the on-wire bytes start with - // a context-specific constructed tag (0xA0 for tag 0). For signature verification, this tag - // is replaced with the universal SET tag (0x31), leaving length+content unchanged. - // - let mut cs_der = full_tlv.to_vec(); - if cs_der.is_empty() { - return Err(SignedObjectParseError::Parse( - "signedAttrs encoding is empty".into(), - )); - } - // The first byte should be the context-specific tag (0xA0) for [0] constructed. - // Replace it with universal SET (0x31) for signature input. - cs_der[0] = 0x31; - Ok(cs_der) -} - -fn take_oid_string(seq: &mut CmsReader<'_>) -> Result { - let oid = seq - .take_tag(0x06) - .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; - Ok(oid_value_bytes_to_string(oid)) -} - -fn oid_value_bytes_to_string(oid_value: &[u8]) -> String { - if oid_value == OID_SHA256_RAW { - return OID_SHA256.to_string(); - } - if oid_value == OID_SIGNED_DATA_RAW { - return OID_SIGNED_DATA.to_string(); - } - if oid_value == OID_CMS_ATTR_CONTENT_TYPE_RAW { - return OID_CMS_ATTR_CONTENT_TYPE.to_string(); - } - if oid_value == OID_CMS_ATTR_MESSAGE_DIGEST_RAW { - return OID_CMS_ATTR_MESSAGE_DIGEST.to_string(); - } - if oid_value == OID_CMS_ATTR_SIGNING_TIME_RAW { - return OID_CMS_ATTR_SIGNING_TIME.to_string(); - } - if oid_value == OID_RSA_ENCRYPTION_RAW { - return OID_RSA_ENCRYPTION.to_string(); - } - if oid_value == OID_SHA256_WITH_RSA_ENCRYPTION_RAW { - return OID_SHA256_WITH_RSA_ENCRYPTION.to_string(); - } - if oid_value == OID_CT_RPKI_MANIFEST_RAW { - return OID_CT_RPKI_MANIFEST.to_string(); - } - if oid_value == OID_CT_ROUTE_ORIGIN_AUTHZ_RAW { - return OID_CT_ROUTE_ORIGIN_AUTHZ.to_string(); - } - if oid_value == OID_CT_ASPA_RAW { - return OID_CT_ASPA.to_string(); - } - decode_oid_to_dotted_string(oid_value) -} - -fn decode_oid_to_dotted_string(value: &[u8]) -> String { - if value.is_empty() { - return "".into(); - } - let first = value[0]; - let a = (first / 40) as u32; - let b = (first % 40) as u32; - let mut out = String::new(); - out.push_str(&a.to_string()); - out.push('.'); - out.push_str(&b.to_string()); - - let mut idx = 1usize; - while idx < value.len() { - let mut v: u32 = 0; - loop { - if idx >= value.len() { - out.push_str("."); - return out; - } - let byte = value[idx]; - idx += 1; - v = (v << 7) | (byte as u32 & 0x7F); - if (byte & 0x80) == 0 { - break; - } - } - out.push('.'); - out.push_str(&v.to_string()); - } - out -} - -fn parse_algorithm_identifier_cursor( - mut seq: CmsReader<'_>, -) -> Result<(String, bool), SignedObjectParseError> { - if seq.is_empty() { - return Err(SignedObjectParseError::Parse( - "AlgorithmIdentifier must be SEQUENCE of 1..2".into(), - )); - } - let oid = take_oid_string(&mut seq)?; - let params_ok = if seq.is_empty() { - true - } else { - let (tag, value) = seq - .take_any() - .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; - tag == 0x05 && value.is_empty() - }; - if !seq.is_empty() { - return Err(SignedObjectParseError::Parse( - "AlgorithmIdentifier must be SEQUENCE of 1..2".into(), - )); - } - Ok((oid, params_ok)) -} - -fn parse_utctime(value: &[u8]) -> Result { - let s = std::str::from_utf8(value) - .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; - if !s.ends_with('Z') { - return Err(SignedObjectValidateError::InvalidSigningTimeValue); - } - let digits = &s[..s.len() - 1]; - if digits.len() != 10 && digits.len() != 12 { - return Err(SignedObjectValidateError::InvalidSigningTimeValue); - } - if !digits.as_bytes().iter().all(|b| b.is_ascii_digit()) { - return Err(SignedObjectValidateError::InvalidSigningTimeValue); - } - let yy: i32 = digits[0..2] - .parse() - .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; - let year = if yy <= 49 { 2000 + yy } else { 1900 + yy }; - let mon: u8 = digits[2..4] - .parse() - .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; - let day: u8 = digits[4..6] - .parse() - .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; - let hour: u8 = digits[6..8] - .parse() - .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; - let min: u8 = digits[8..10] - .parse() - .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; - let sec: u8 = if digits.len() == 12 { - digits[10..12] - .parse() - .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)? - } else { - 0 - }; - let month = time::Month::try_from(mon) - .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; - let date = time::Date::from_calendar_date(year, month, day) - .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; - let time = time::Time::from_hms(hour, min, sec) - .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; - Ok(time::OffsetDateTime::new_utc(date, time)) -} - -fn parse_generalized_time(value: &[u8]) -> Result { - let s = std::str::from_utf8(value) - .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; - if !s.ends_with('Z') { - return Err(SignedObjectValidateError::InvalidSigningTimeValue); - } - let digits = &s[..s.len() - 1]; - if digits.len() != 12 && digits.len() != 14 { - return Err(SignedObjectValidateError::InvalidSigningTimeValue); - } - if !digits.as_bytes().iter().all(|b| b.is_ascii_digit()) { - return Err(SignedObjectValidateError::InvalidSigningTimeValue); - } - let year: i32 = digits[0..4] - .parse() - .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; - let mon: u8 = digits[4..6] - .parse() - .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; - let day: u8 = digits[6..8] - .parse() - .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; - let hour: u8 = digits[8..10] - .parse() - .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; - let min: u8 = digits[10..12] - .parse() - .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; - let sec: u8 = if digits.len() == 14 { - digits[12..14] - .parse() - .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)? - } else { - 0 - }; - let month = time::Month::try_from(mon) - .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; - let date = time::Date::from_calendar_date(year, month, day) - .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; - let time = time::Time::from_hms(hour, min, sec) - .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; - Ok(time::OffsetDateTime::new_utc(date, time)) -} - -fn strip_leading_zeros(bytes: &[u8]) -> &[u8] { - let mut idx = 0; - while idx < bytes.len() && bytes[idx] == 0 { - idx += 1; - } - if idx == bytes.len() { - &bytes[bytes.len() - 1..] - } else { - &bytes[idx..] - } -} +include!("signed_object/types_errors.rs"); +include!("signed_object/signed_object_impl.rs"); +include!("signed_object/cms_reader.rs"); +include!("signed_object/parsed_profile.rs"); +include!("signed_object/signed_attrs.rs"); #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn strict_cms_der_rejects_constructed_octet_string_fixture() { - let der = std::fs::read(std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.mft", - )) - .expect("read manifest fixture"); - let parsed = RpkiSignedObject::parse_der(&der).expect("parse fixture"); - let econtent = parsed - .signed_data - .encap_content_info - .econtent - .clone() - .expect("fixture eContent"); - assert_eq!( - parsed.signed_data.encap_content_info.econtent.as_deref(), - Some(econtent.as_slice()) - ); - - let primitive_octets = der_tlv(0x04, &econtent); - let constructed_octets = same_size_constructed_octet_string(&primitive_octets, &econtent); - let mutated = replace_first_subslice(&der, &primitive_octets, &constructed_octets) - .expect("replace eContent OCTET STRING"); - let compatible = RpkiSignedObject::parse_der(&mutated).expect("BER-compatible parse"); - assert!( - econtent.starts_with( - compatible - .signed_data - .encap_content_info - .econtent - .as_deref() - .expect("compatible eContent") - ) - ); - let err = RpkiSignedObject::parse_der_strict_cms(&mutated) - .expect_err("DER strict rejects constructed OCTET STRING"); - assert!(err.to_string().contains("DER"), "{err}"); - let compatibility_error = RpkiSignedObject::strict_cms_der_error(&mutated) - .expect("BER-compatible fixture must report strict-DER incompatibility"); - assert_eq!(compatibility_error.to_string(), err.to_string()); - assert!(RpkiSignedObject::strict_cms_der_error(&der).is_none()); - } - - fn replace_first_subslice(input: &[u8], from: &[u8], to: &[u8]) -> Option> { - let pos = input - .windows(from.len()) - .position(|candidate| candidate == from)?; - let mut out = Vec::with_capacity(input.len() - from.len() + to.len()); - out.extend_from_slice(&input[..pos]); - out.extend_from_slice(to); - out.extend_from_slice(&input[pos + from.len()..]); - Some(out) - } - - fn same_size_constructed_octet_string(primitive: &[u8], content: &[u8]) -> Vec { - assert_eq!(primitive[0], 0x04); - let header_len = tlv_header_len(primitive); - let outer_value_len = primitive.len() - header_len; - let child_len = (0..=outer_value_len) - .rev() - .find(|candidate| 1 + len_len(*candidate) + *candidate == outer_value_len) - .expect("find child length"); - let mut out = primitive[..header_len].to_vec(); - out[0] = 0x24; - out.extend(der_tlv(0x04, &content[..child_len])); - assert_eq!(out.len(), primitive.len()); - out - } - - fn tlv_header_len(tlv: &[u8]) -> usize { - if tlv[1] & 0x80 == 0 { - 2 - } else { - 2 + (tlv[1] & 0x7F) as usize - } - } - - fn len_len(len: usize) -> usize { - if len < 0x80 { - return 1; - } - let mut value = len; - let mut n = 0usize; - while value > 0 { - n += 1; - value >>= 8; - } - 1 + n - } - - fn der_tlv(tag: u8, value: &[u8]) -> Vec { - let mut out = vec![tag]; - encode_len(value.len(), &mut out); - out.extend_from_slice(value); - out - } - - fn encode_len(len: usize, out: &mut Vec) { - if len < 0x80 { - out.push(len as u8); - return; - } - let mut bytes = Vec::new(); - let mut value = len; - while value > 0 { - bytes.push((value & 0xFF) as u8); - value >>= 8; - } - bytes.reverse(); - out.push(0x80 | bytes.len() as u8); - out.extend(bytes); - } -} +#[path = "signed_object/tests.rs"] +mod tests; diff --git a/crates/panda-rpki-validator/src/data_model/signed_object/cms_reader.rs b/crates/panda-rpki-validator/src/data_model/signed_object/cms_reader.rs new file mode 100644 index 0000000..a03ecd0 --- /dev/null +++ b/crates/panda-rpki-validator/src/data_model/signed_object/cms_reader.rs @@ -0,0 +1,181 @@ +// CMS reader and BER/DER content parsing primitives. + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CmsParseMode { + BerCompatible, + DerStrict, +} + +struct CmsReader<'a> { + buf: &'a [u8], + mode: CmsParseMode, +} + +impl<'a> CmsReader<'a> { + fn new(buf: &'a [u8], mode: CmsParseMode) -> Self { + Self { buf, mode } + } + + fn is_empty(&self) -> bool { + self.buf.is_empty() + } + + fn remaining_len(&self) -> usize { + self.buf.len() + } + + fn peek_tag(&self) -> Result { + let (_rem, any) = parse_any(self.buf, self.mode)?; + header_to_single_byte_tag(&any.header) + } + + fn take_any(&mut self) -> Result<(u8, &'a [u8]), String> { + let (rem, any) = parse_any(self.buf, self.mode)?; + let tag = header_to_single_byte_tag(&any.header)?; + self.buf = rem; + Ok((tag, any.data)) + } + + fn take_any_full(&mut self) -> Result<(u8, &'a [u8], &'a [u8]), String> { + let (rem, any) = parse_any(self.buf, self.mode)?; + let consumed = self.buf.len() - rem.len(); + let full = &self.buf[..consumed]; + let tag = header_to_single_byte_tag(&any.header)?; + self.buf = rem; + Ok((tag, full, any.data)) + } + + fn skip_any(&mut self) -> Result<(), String> { + let _ = self.take_any()?; + Ok(()) + } + + fn take_tag(&mut self, expected_tag: u8) -> Result<&'a [u8], String> { + let (tag, value) = self.take_any()?; + if tag != expected_tag { + return Err(format!( + "unexpected tag: got 0x{tag:02X}, expected 0x{expected_tag:02X}" + )); + } + Ok(value) + } + + fn take_sequence(&mut self) -> Result, String> { + let value = self.take_tag(0x30)?; + Ok(CmsReader::new(value, self.mode)) + } + + fn take_octet_string(&mut self) -> Result, String> { + let (rem, any) = parse_any(self.buf, self.mode)?; + let tag = header_to_single_byte_tag(&any.header)?; + if self.mode == CmsParseMode::DerStrict && tag != 0x04 { + return Err(format!( + "unexpected tag in DER strict mode: got 0x{tag:02X}, expected 0x04" + )); + } + if tag != 0x04 && tag != 0x24 { + return Err(format!("unexpected tag: got 0x{tag:02X}, expected 0x04")); + } + let octets = flatten_octet_string(any, self.mode)?; + self.buf = rem; + Ok(octets) + } + + fn take_uint_u64(&mut self) -> Result { + let value = self.take_tag(0x02)?; + der_uint_from_bytes(value) + } + + fn take_explicit(&mut self, expected_outer_tag: u8) -> Result<(u8, &'a [u8]), String> { + let inner_der = self.take_tag(expected_outer_tag)?; + let (tag, value, rem) = cms_take_tlv(inner_der, self.mode)?; + if !rem.is_empty() { + return Err("trailing bytes inside EXPLICIT value".into()); + } + Ok((tag, value)) + } + + fn take_explicit_der(&mut self, expected_outer_tag: u8) -> Result<&'a [u8], String> { + let inner_der = self.take_tag(expected_outer_tag)?; + let (_tag, _value, rem) = cms_take_tlv(inner_der, self.mode)?; + if !rem.is_empty() { + return Err("trailing bytes inside EXPLICIT value".into()); + } + Ok(inner_der) + } +} + +fn parse_signed_object_content_info( + raw_der: &[u8], + parse_der: &[u8], + mode: CmsParseMode, +) -> Result { + let mut r = CmsReader::new(parse_der, mode); + let mut content_info_seq = r + .take_sequence() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + if !r.is_empty() { + return Err(SignedObjectParseError::TrailingBytes(r.remaining_len())); + } + + let content_type = take_oid_string(&mut content_info_seq)?; + let signed_data = parse_signed_data_from_contentinfo_cursor(&mut content_info_seq)?; + if !content_info_seq.is_empty() { + return Err(SignedObjectParseError::Parse( + "ContentInfo must be a SEQUENCE of 2 elements".into(), + )); + } + + Ok(RpkiSignedObjectParsed { + raw_der: raw_der.to_vec(), + content_info_content_type: content_type, + signed_data, + }) +} + +fn parse_any<'a>(input: &'a [u8], mode: CmsParseMode) -> Result<(&'a [u8], Any<'a>), String> { + match mode { + CmsParseMode::BerCompatible => { + Any::from_ber(input).map_err(|e| format!("BER parse error: {e}")) + } + CmsParseMode::DerStrict => { + Any::from_der(input).map_err(|e| format!("DER parse error: {e}")) + } + } +} + +fn header_to_single_byte_tag(header: &Header<'_>) -> Result { + let tag_no = header.tag().0; + if tag_no > 30 { + return Err(format!("high-tag-number form not supported: {tag_no}")); + } + Ok(((header.class() as u8) << 6) + | if header.constructed() { 0x20 } else { 0x00 } + | tag_no as u8) +} + +fn cms_take_tlv(input: &[u8], mode: CmsParseMode) -> Result<(u8, &[u8], &[u8]), String> { + let (rem, any) = parse_any(input, mode)?; + let tag = header_to_single_byte_tag(&any.header)?; + Ok((tag, any.data, rem)) +} + +fn flatten_octet_string(any: Any<'_>, mode: CmsParseMode) -> Result, String> { + if any.class() != Class::Universal || any.tag() != Tag::OctetString { + return Err("expected OCTET STRING".into()); + } + if !any.header.constructed() { + return Ok(any.data.to_vec()); + } + if mode == CmsParseMode::DerStrict { + return Err("constructed OCTET STRING is not allowed in DER strict mode".into()); + } + let mut out = Vec::new(); + let mut input = any.data; + while !input.is_empty() { + let (rem, child) = Any::from_ber(input).map_err(|e| format!("BER parse error: {e}"))?; + out.extend(flatten_octet_string(child, mode)?); + input = rem; + } + Ok(out) +} diff --git a/crates/panda-rpki-validator/src/data_model/signed_object/parsed_profile.rs b/crates/panda-rpki-validator/src/data_model/signed_object/parsed_profile.rs new file mode 100644 index 0000000..1afb6f3 --- /dev/null +++ b/crates/panda-rpki-validator/src/data_model/signed_object/parsed_profile.rs @@ -0,0 +1,542 @@ +// SignedData profile validation and EE certificate checks. + +impl RpkiSignedObjectParsed { + pub fn validate_profile(self) -> Result { + self.validate_profile_with_strict_name(false) + } + + pub fn validate_profile_with_strict_name( + self, + strict_name: bool, + ) -> Result { + if self.content_info_content_type != OID_SIGNED_DATA { + return Err(SignedObjectValidateError::InvalidContentInfoContentType( + self.content_info_content_type, + )); + } + + let signed_data = validate_signed_data_profile(self.signed_data, strict_name)?; + + Ok(RpkiSignedObject { + raw_der: self.raw_der, + content_info_content_type: OID_SIGNED_DATA.to_string(), + signed_data, + }) + } +} + +fn parse_signed_data_from_contentinfo_cursor( + seq: &mut CmsReader<'_>, +) -> Result { + let inner_der = seq.take_explicit_der(0xA0).map_err(|_e| { + SignedObjectParseError::Parse("ContentInfo.content must be [0] EXPLICIT".into()) + })?; + let mut r = CmsReader::new(inner_der, seq.mode); + let signed_data_seq = r + .take_sequence() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + if !r.is_empty() { + return Err(SignedObjectParseError::Parse( + "trailing bytes inside ContentInfo.content".into(), + )); + } + parse_signed_data_cursor(signed_data_seq) +} + +fn parse_signed_data_cursor( + mut seq: CmsReader<'_>, +) -> Result { + let version = seq + .take_uint_u64() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + + let digest_set_bytes = seq + .take_tag(0x31) + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + let mut digest_set = CmsReader::new(digest_set_bytes, seq.mode); + let mut digest_algorithms: Vec = Vec::new(); + while !digest_set.is_empty() { + let alg = digest_set + .take_sequence() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + let (oid, params_ok) = parse_algorithm_identifier_cursor(alg)?; + digest_algorithms.push(AlgorithmIdentifierParsed { oid, params_ok }); + } + + let encap_content_info = parse_encapsulated_content_info_cursor( + seq.take_sequence() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?, + )?; + + let mut certificates: Option>> = None; + let mut crls_present = false; + let mut signer_infos: Option> = None; + + while !seq.is_empty() { + let tag = seq + .peek_tag() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + match tag { + 0xA0 => { + if certificates.is_some() { + return Err(SignedObjectParseError::Parse( + "SignedData.certificates appears more than once".into(), + )); + } + let content = seq + .take_tag(0xA0) + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + certificates = Some(split_der_objects(content, seq.mode)?); + } + 0xA1 => { + crls_present = true; + seq.skip_any() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + } + 0x31 => { + if signer_infos.is_some() { + return Err(SignedObjectParseError::Parse( + "SignedData.signerInfos appears more than once".into(), + )); + } + let set_bytes = seq + .take_tag(0x31) + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + signer_infos = Some(parse_signer_infos_set_cursor(set_bytes, seq.mode)?); + } + _ => { + return Err(SignedObjectParseError::Parse( + "unexpected field in SignedData".into(), + )); + } + } + } + + let signer_infos = signer_infos + .ok_or_else(|| SignedObjectParseError::Parse("SignedData.signerInfos missing".into()))?; + + Ok(SignedDataParsed { + version, + digest_algorithms, + encap_content_info, + certificates, + crls_present, + signer_infos, + }) +} + +fn parse_encapsulated_content_info_cursor( + mut seq: CmsReader<'_>, +) -> Result { + if seq.is_empty() { + return Err(SignedObjectParseError::Parse( + "EncapsulatedContentInfo must be SEQUENCE of 1..2".into(), + )); + } + + let econtent_type = take_oid_string(&mut seq)?; + + let econtent = if seq.is_empty() { + None + } else { + let inner_der = seq.take_explicit_der(0xA0).map_err(|_e| { + SignedObjectParseError::Parse( + "EncapsulatedContentInfo.eContent must be [0] EXPLICIT".into(), + ) + })?; + let mut inner = CmsReader::new(inner_der, seq.mode); + let octets = inner + .take_octet_string() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + if !inner.is_empty() { + return Err(SignedObjectParseError::Parse( + "trailing bytes inside EncapsulatedContentInfo.eContent".into(), + )); + } + Some(octets) + }; + if !seq.is_empty() { + return Err(SignedObjectParseError::Parse( + "EncapsulatedContentInfo must be SEQUENCE of 1..2".into(), + )); + } + + Ok(EncapsulatedContentInfoParsed { + econtent_type, + econtent, + }) +} + +fn split_der_objects( + mut input: &[u8], + mode: CmsParseMode, +) -> Result>, SignedObjectParseError> { + let mut out: Vec> = Vec::new(); + while !input.is_empty() { + let (_tag, _value, rem) = + cms_take_tlv(input, mode).map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + let consumed = input.len() - rem.len(); + out.push(input[..consumed].to_vec()); + input = rem; + } + Ok(out) +} + +fn parse_signer_infos_set_cursor( + set_bytes: &[u8], + mode: CmsParseMode, +) -> Result, SignedObjectParseError> { + let mut set = CmsReader::new(set_bytes, mode); + let mut out: Vec = Vec::new(); + while !set.is_empty() { + let si = set + .take_sequence() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + out.push(parse_signer_info_cursor(si)?); + } + Ok(out) +} + +fn validate_ee_certificate( + der: &[u8], + strict_name: bool, +) -> Result { + let (rem, cert) = X509Certificate::from_der(der) + .map_err(|e| SignedObjectValidateError::EeCertificateParse(e.to_string()))?; + if !rem.is_empty() { + return Err(SignedObjectValidateError::EeCertificateParse(format!( + "trailing bytes after EE certificate DER: {}", + rem.len() + ))); + } + + let rc = match ResourceCertificate::from_der(der) { + Ok(v) => v, + Err(e) => { + return match e { + crate::data_model::rc::ResourceCertificateDecodeError::Validate( + crate::data_model::rc::ResourceCertificateProfileError::SignedObjectSiaNotUri, + ) => Err(SignedObjectValidateError::EeCertificateSignedObjectSiaNotUri), + crate::data_model::rc::ResourceCertificateDecodeError::Validate( + crate::data_model::rc::ResourceCertificateProfileError::SignedObjectSiaNoRsync, + ) => Err(SignedObjectValidateError::EeCertificateSignedObjectSiaNoRsync), + _ => Err(SignedObjectValidateError::EeCertificateParse(e.to_string())), + }; + } + }; + rc.validate_rfc6487_profile(ResourceCertificateRole::SignedObjectEe) + .map_err(|e| SignedObjectValidateError::EeCertificateParse(e.to_string()))?; + if strict_name { + rc.validate_strict_name_profile() + .map_err(|e| SignedObjectValidateError::EeCertificateParse(e.to_string()))?; + } + + let ski = rc + .tbs + .extensions + .subject_key_identifier + .clone() + .ok_or(SignedObjectValidateError::EeCertificateMissingSki)?; + + let spki_der = rc.tbs.subject_public_key_info.clone(); + let (rem, spki) = SubjectPublicKeyInfo::from_der(&spki_der) + .map_err(|e| SignedObjectValidateError::EeCertificateParse(e.to_string()))?; + if !rem.is_empty() { + return Err(SignedObjectValidateError::EeCertificateParse( + "trailing bytes after EE SubjectPublicKeyInfo DER".to_string(), + )); + } + let parsed_pk = spki.parsed().map_err(|_e| { + SignedObjectValidateError::EeCertificateParse( + "unsupported EE public key algorithm".to_string(), + ) + })?; + let (rsa_public_modulus, rsa_public_exponent) = match parsed_pk { + PublicKey::RSA(rsa) => { + let modulus = strip_leading_zeros(rsa.modulus).to_vec(); + let exponent = strip_leading_zeros(rsa.exponent).to_vec(); + let _ = rsa.try_exponent().map_err(|_e| { + SignedObjectValidateError::EeCertificateParse("invalid EE RSA exponent".to_string()) + })?; + (modulus, exponent) + } + _ => { + return Err(SignedObjectValidateError::EeCertificateParse( + "unsupported EE public key algorithm".to_string(), + )); + } + }; + + let sia = rc + .tbs + .extensions + .subject_info_access + .as_ref() + .ok_or(SignedObjectValidateError::EeCertificateMissingSia)?; + let signed_object_uris: Vec = match sia { + SubjectInfoAccess::Ee(ee) => ee.signed_object_uris.clone(), + SubjectInfoAccess::Ca(_ca) => Vec::new(), + }; + if signed_object_uris.is_empty() { + return Err(SignedObjectValidateError::EeCertificateMissingSignedObjectSia); + } + if !signed_object_uris.iter().any(|u| u.starts_with("rsync://")) { + return Err(SignedObjectValidateError::EeCertificateSignedObjectSiaNoRsync); + } + + Ok(ResourceEeCertificate { + raw_der: der.to_vec(), + subject_key_identifier: ski, + spki_der, + rsa_public_modulus, + rsa_public_exponent, + tbs_certificate_der: cert.tbs_certificate.as_ref().to_vec(), + signature_bytes: cert.signature_value.data.to_vec(), + key_usage_summary: summarize_ee_key_usage(&cert), + sia_signed_object_uris: signed_object_uris, + resource_cert: rc, + }) +} + +fn summarize_ee_key_usage(cert: &X509Certificate<'_>) -> EeKeyUsageSummary { + for ext in cert.extensions() { + if ext.oid.as_bytes() == OID_KEY_USAGE_RAW { + match ext.parsed_extension() { + ParsedExtension::KeyUsage(ku) => { + if !ext.critical { + return EeKeyUsageSummary::NotCritical; + } + let ok = ku.digital_signature() + && !ku.key_cert_sign() + && !ku.crl_sign() + && !ku.non_repudiation() + && !ku.key_encipherment() + && !ku.data_encipherment() + && !ku.key_agreement() + && !ku.encipher_only() + && !ku.decipher_only(); + return if ok { + EeKeyUsageSummary::DigitalSignatureOnly + } else { + EeKeyUsageSummary::InvalidBits + }; + } + other => { + return EeKeyUsageSummary::ParseError(format!( + "unexpected parsed keyUsage extension: {other:?}" + )); + } + } + } + } + + EeKeyUsageSummary::Missing +} + +fn parse_signer_info_cursor( + mut seq: CmsReader<'_>, +) -> Result { + let version = seq + .take_uint_u64() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + + let (sid_tag, sid_bytes) = seq + .take_any() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + let sid = if (sid_tag & 0xC0) == 0x80 && (sid_tag & 0x1F) == 0 { + SignerIdentifierParsed::SubjectKeyIdentifier(sid_bytes.to_vec()) + } else { + SignerIdentifierParsed::Other + }; + + let digest_alg_seq = seq + .take_sequence() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + let (digest_oid, digest_params_ok) = parse_algorithm_identifier_cursor(digest_alg_seq)?; + let digest_algorithm = AlgorithmIdentifierParsed { + oid: digest_oid, + params_ok: digest_params_ok, + }; + + let mut signed_attrs_content: Option> = None; + let mut signed_attrs_der_for_signature: Option> = None; + + if seq + .peek_tag() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))? + == 0xA0 + { + let (tag, full_tlv, value) = seq + .take_any_full() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + if tag != 0xA0 { + return Err(SignedObjectParseError::Parse( + "SignerInfo.signedAttrs must be [0] IMPLICIT".into(), + )); + } + signed_attrs_content = Some(value.to_vec()); + signed_attrs_der_for_signature = Some(make_signed_attrs_der_for_signature(full_tlv)?); + } + + let sig_alg_seq = seq + .take_sequence() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + let (signature_oid, signature_params_ok) = parse_algorithm_identifier_cursor(sig_alg_seq)?; + let signature_algorithm = AlgorithmIdentifierParsed { + oid: signature_oid, + params_ok: signature_params_ok, + }; + + let signature = seq + .take_octet_string() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))? + .to_vec(); + + let unsigned_attrs_present = !seq.is_empty(); + + Ok(SignerInfoParsed { + version, + sid, + digest_algorithm, + signature_algorithm, + signed_attrs_content, + signed_attrs_der_for_signature, + unsigned_attrs_present, + signature, + }) +} + +fn validate_signed_data_profile( + signed_data: SignedDataParsed, + strict_name: bool, +) -> Result { + if signed_data.version != 3 { + return Err(SignedObjectValidateError::InvalidSignedDataVersion( + signed_data.version, + )); + } + + if signed_data.digest_algorithms.len() != 1 { + return Err(SignedObjectValidateError::InvalidDigestAlgorithmsCount( + signed_data.digest_algorithms.len(), + )); + } + let digest_alg = &signed_data.digest_algorithms[0]; + if digest_alg.oid != OID_SHA256 { + return Err(SignedObjectValidateError::InvalidDigestAlgorithm( + digest_alg.oid.clone(), + )); + } + + if signed_data.crls_present { + return Err(SignedObjectValidateError::CrlsPresent); + } + + let econtent = signed_data + .encap_content_info + .econtent + .clone() + .ok_or(SignedObjectValidateError::EContentMissing)?; + if econtent.is_empty() { + return Err(SignedObjectValidateError::EContentMissing); + } + let encap_content_info = EncapsulatedContentInfo { + econtent_type: signed_data.encap_content_info.econtent_type.clone(), + econtent: econtent.clone(), + }; + + let certs = signed_data + .certificates + .as_ref() + .ok_or(SignedObjectValidateError::CertificatesMissing)?; + if certs.len() != 1 { + return Err(SignedObjectValidateError::InvalidCertificatesCount( + certs.len(), + )); + } + let ee = validate_ee_certificate(&certs[0], strict_name)?; + + if signed_data.signer_infos.len() != 1 { + return Err(SignedObjectValidateError::InvalidSignerInfosCount( + signed_data.signer_infos.len(), + )); + } + let signer = &signed_data.signer_infos[0]; + + if signer.version != 3 { + return Err(SignedObjectValidateError::InvalidSignerInfoVersion( + signer.version, + )); + } + let sid_ski = match &signer.sid { + SignerIdentifierParsed::SubjectKeyIdentifier(ski) => ski.clone(), + SignerIdentifierParsed::Other => { + return Err(SignedObjectValidateError::InvalidSignerIdentifier); + } + }; + + if signer.digest_algorithm.oid != OID_SHA256 { + return Err(SignedObjectValidateError::InvalidSignerInfoDigestAlgorithm( + signer.digest_algorithm.oid.clone(), + )); + } + + let signed_attrs_content = signer + .signed_attrs_content + .as_deref() + .ok_or(SignedObjectValidateError::SignedAttrsMissing)?; + let signed_attrs_der_for_signature = signer + .signed_attrs_der_for_signature + .clone() + .ok_or(SignedObjectValidateError::SignedAttrsMissing)?; + let signed_attrs = parse_signed_attrs_implicit(signed_attrs_content)?; + + if signer.unsigned_attrs_present { + return Err(SignedObjectValidateError::UnsignedAttrsPresent); + } + + if !signer.signature_algorithm.params_ok { + return Err(SignedObjectValidateError::InvalidSignatureAlgorithmParameters); + } + let signature_algorithm = signer.signature_algorithm.oid.clone(); + if signature_algorithm != OID_RSA_ENCRYPTION + && signature_algorithm != OID_SHA256_WITH_RSA_ENCRYPTION + { + return Err(SignedObjectValidateError::InvalidSignatureAlgorithm( + signature_algorithm, + )); + } + + if sid_ski != ee.subject_key_identifier { + return Err(SignedObjectValidateError::SidSkiMismatch); + } + if signed_attrs.content_type != encap_content_info.econtent_type { + return Err(SignedObjectValidateError::ContentTypeAttrMismatch { + econtent_type: encap_content_info.econtent_type.clone(), + attr_content_type: signed_attrs.content_type.clone(), + }); + } + + let computed = digest::digest(&digest::SHA256, &encap_content_info.econtent); + if computed.as_ref() != signed_attrs.message_digest.as_slice() { + return Err(SignedObjectValidateError::MessageDigestMismatch); + } + + Ok(SignedDataProfiled { + version: 3, + digest_algorithms: vec![OID_SHA256.to_string()], + encap_content_info, + certificates: vec![ee.clone()], + crls_present: false, + signer_infos: vec![SignerInfoProfiled { + version: 3, + sid_ski, + digest_algorithm: OID_SHA256.to_string(), + signature_algorithm: signer.signature_algorithm.oid.clone(), + signed_attrs, + unsigned_attrs_present: false, + signature: signer.signature.clone(), + signed_attrs_der_for_signature, + }], + }) +} diff --git a/crates/panda-rpki-validator/src/data_model/signed_object/signed_attrs.rs b/crates/panda-rpki-validator/src/data_model/signed_object/signed_attrs.rs new file mode 100644 index 0000000..807aa41 --- /dev/null +++ b/crates/panda-rpki-validator/src/data_model/signed_object/signed_attrs.rs @@ -0,0 +1,344 @@ +// Signed attributes, signing-time, and algorithm parsing helpers. + +fn parse_signed_attrs_implicit( + input: &[u8], +) -> Result { + let mut content_type: Option = None; + let mut message_digest: Option> = None; + let mut signing_time: Option = None; + + fn count_elements(mut r: DerReader<'_>) -> Result { + let mut n = 0usize; + while !r.is_empty() { + r.skip_any()?; + n += 1; + } + Ok(n) + } + + let mut remaining = DerReader::new(input); + while !remaining.is_empty() { + let mut attr = remaining + .take_sequence() + .map_err(|e| SignedObjectValidateError::SignedAttrsParse(e.to_string()))?; + + let oid_bytes = attr + .take_tag(0x06) + .map_err(|e| SignedObjectValidateError::SignedAttrsParse(e.to_string()))?; + let oid = oid_value_bytes_to_string(oid_bytes); + + let values_bytes = attr + .take_tag(0x31) + .map_err(|e| SignedObjectValidateError::SignedAttrsParse(e.to_string()))?; + if !attr.is_empty() { + return Err(SignedObjectValidateError::SignedAttrsParse( + "Attribute must be SEQUENCE of 2".into(), + )); + } + + let mut values = DerReader::new(values_bytes); + let count = if values.is_empty() { + 0 + } else { + values + .skip_any() + .map_err(|e| SignedObjectValidateError::SignedAttrsParse(e.to_string()))?; + if values.is_empty() { + 1 + } else { + 1 + count_elements(values) + .map_err(|e| SignedObjectValidateError::SignedAttrsParse(e.to_string()))? + } + }; + if count != 1 { + return Err( + SignedObjectValidateError::InvalidSignedAttributeValuesCount { oid, count }, + ); + } + + // Re-parse the sole value. + let mut values = DerReader::new(values_bytes); + let (val_tag, val_bytes) = values + .take_any() + .map_err(|e| SignedObjectValidateError::SignedAttrsParse(e.to_string()))?; + + match oid.as_str() { + OID_CMS_ATTR_CONTENT_TYPE => { + if content_type.is_some() { + return Err(SignedObjectValidateError::DuplicateSignedAttribute(oid)); + } + if val_tag != 0x06 { + return Err(SignedObjectValidateError::SignedAttrsParse( + "content-type attr value must be OBJECT IDENTIFIER".into(), + )); + } + content_type = Some(oid_value_bytes_to_string(val_bytes)); + } + OID_CMS_ATTR_MESSAGE_DIGEST => { + if message_digest.is_some() { + return Err(SignedObjectValidateError::DuplicateSignedAttribute(oid)); + } + if val_tag != 0x04 { + return Err(SignedObjectValidateError::SignedAttrsParse( + "message-digest attr value must be OCTET STRING".into(), + )); + } + message_digest = Some(val_bytes.to_vec()); + } + OID_CMS_ATTR_SIGNING_TIME => { + if signing_time.is_some() { + return Err(SignedObjectValidateError::DuplicateSignedAttribute(oid)); + } + signing_time = Some(parse_signing_time_value_tlv(val_tag, val_bytes)?); + } + _ => { + return Err(SignedObjectValidateError::UnsupportedSignedAttribute(oid)); + } + } + } + + Ok(SignedAttrsProfiled { + content_type: content_type + .ok_or(SignedObjectValidateError::SignedAttrsContentTypeMissing)?, + message_digest: message_digest + .ok_or(SignedObjectValidateError::SignedAttrsMessageDigestMissing)?, + signing_time: signing_time + .ok_or(SignedObjectValidateError::SignedAttrsSigningTimeMissing)?, + other_attrs_present: false, + }) +} + +fn parse_signing_time_value_tlv( + tag: u8, + value: &[u8], +) -> Result { + match tag { + 0x17 => Ok(Asn1TimeUtc { + utc: parse_utctime(value)?, + encoding: Asn1TimeEncoding::UtcTime, + }), + 0x18 => Ok(Asn1TimeUtc { + utc: parse_generalized_time(value)?, + encoding: Asn1TimeEncoding::GeneralizedTime, + }), + _ => Err(SignedObjectValidateError::InvalidSigningTimeValue), + } +} + +fn make_signed_attrs_der_for_signature(full_tlv: &[u8]) -> Result, SignedObjectParseError> { + // We need the DER encoding of SignedAttributes (SET OF Attribute) as signature input. + // The SignedAttributes field in SignerInfo is `[0] IMPLICIT`, so the on-wire bytes start with + // a context-specific constructed tag (0xA0 for tag 0). For signature verification, this tag + // is replaced with the universal SET tag (0x31), leaving length+content unchanged. + // + let mut cs_der = full_tlv.to_vec(); + if cs_der.is_empty() { + return Err(SignedObjectParseError::Parse( + "signedAttrs encoding is empty".into(), + )); + } + // The first byte should be the context-specific tag (0xA0) for [0] constructed. + // Replace it with universal SET (0x31) for signature input. + cs_der[0] = 0x31; + Ok(cs_der) +} + +fn take_oid_string(seq: &mut CmsReader<'_>) -> Result { + let oid = seq + .take_tag(0x06) + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + Ok(oid_value_bytes_to_string(oid)) +} + +fn oid_value_bytes_to_string(oid_value: &[u8]) -> String { + if oid_value == OID_SHA256_RAW { + return OID_SHA256.to_string(); + } + if oid_value == OID_SIGNED_DATA_RAW { + return OID_SIGNED_DATA.to_string(); + } + if oid_value == OID_CMS_ATTR_CONTENT_TYPE_RAW { + return OID_CMS_ATTR_CONTENT_TYPE.to_string(); + } + if oid_value == OID_CMS_ATTR_MESSAGE_DIGEST_RAW { + return OID_CMS_ATTR_MESSAGE_DIGEST.to_string(); + } + if oid_value == OID_CMS_ATTR_SIGNING_TIME_RAW { + return OID_CMS_ATTR_SIGNING_TIME.to_string(); + } + if oid_value == OID_RSA_ENCRYPTION_RAW { + return OID_RSA_ENCRYPTION.to_string(); + } + if oid_value == OID_SHA256_WITH_RSA_ENCRYPTION_RAW { + return OID_SHA256_WITH_RSA_ENCRYPTION.to_string(); + } + if oid_value == OID_CT_RPKI_MANIFEST_RAW { + return OID_CT_RPKI_MANIFEST.to_string(); + } + if oid_value == OID_CT_ROUTE_ORIGIN_AUTHZ_RAW { + return OID_CT_ROUTE_ORIGIN_AUTHZ.to_string(); + } + if oid_value == OID_CT_ASPA_RAW { + return OID_CT_ASPA.to_string(); + } + decode_oid_to_dotted_string(oid_value) +} + +fn decode_oid_to_dotted_string(value: &[u8]) -> String { + if value.is_empty() { + return "".into(); + } + let first = value[0]; + let a = (first / 40) as u32; + let b = (first % 40) as u32; + let mut out = String::new(); + out.push_str(&a.to_string()); + out.push('.'); + out.push_str(&b.to_string()); + + let mut idx = 1usize; + while idx < value.len() { + let mut v: u32 = 0; + loop { + if idx >= value.len() { + out.push_str("."); + return out; + } + let byte = value[idx]; + idx += 1; + v = (v << 7) | (byte as u32 & 0x7F); + if (byte & 0x80) == 0 { + break; + } + } + out.push('.'); + out.push_str(&v.to_string()); + } + out +} + +fn parse_algorithm_identifier_cursor( + mut seq: CmsReader<'_>, +) -> Result<(String, bool), SignedObjectParseError> { + if seq.is_empty() { + return Err(SignedObjectParseError::Parse( + "AlgorithmIdentifier must be SEQUENCE of 1..2".into(), + )); + } + let oid = take_oid_string(&mut seq)?; + let params_ok = if seq.is_empty() { + true + } else { + let (tag, value) = seq + .take_any() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + tag == 0x05 && value.is_empty() + }; + if !seq.is_empty() { + return Err(SignedObjectParseError::Parse( + "AlgorithmIdentifier must be SEQUENCE of 1..2".into(), + )); + } + Ok((oid, params_ok)) +} + +fn parse_utctime(value: &[u8]) -> Result { + let s = std::str::from_utf8(value) + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + if !s.ends_with('Z') { + return Err(SignedObjectValidateError::InvalidSigningTimeValue); + } + let digits = &s[..s.len() - 1]; + if digits.len() != 10 && digits.len() != 12 { + return Err(SignedObjectValidateError::InvalidSigningTimeValue); + } + if !digits.as_bytes().iter().all(|b| b.is_ascii_digit()) { + return Err(SignedObjectValidateError::InvalidSigningTimeValue); + } + let yy: i32 = digits[0..2] + .parse() + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let year = if yy <= 49 { 2000 + yy } else { 1900 + yy }; + let mon: u8 = digits[2..4] + .parse() + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let day: u8 = digits[4..6] + .parse() + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let hour: u8 = digits[6..8] + .parse() + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let min: u8 = digits[8..10] + .parse() + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let sec: u8 = if digits.len() == 12 { + digits[10..12] + .parse() + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)? + } else { + 0 + }; + let month = time::Month::try_from(mon) + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let date = time::Date::from_calendar_date(year, month, day) + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let time = time::Time::from_hms(hour, min, sec) + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + Ok(time::OffsetDateTime::new_utc(date, time)) +} + +fn parse_generalized_time(value: &[u8]) -> Result { + let s = std::str::from_utf8(value) + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + if !s.ends_with('Z') { + return Err(SignedObjectValidateError::InvalidSigningTimeValue); + } + let digits = &s[..s.len() - 1]; + if digits.len() != 12 && digits.len() != 14 { + return Err(SignedObjectValidateError::InvalidSigningTimeValue); + } + if !digits.as_bytes().iter().all(|b| b.is_ascii_digit()) { + return Err(SignedObjectValidateError::InvalidSigningTimeValue); + } + let year: i32 = digits[0..4] + .parse() + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let mon: u8 = digits[4..6] + .parse() + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let day: u8 = digits[6..8] + .parse() + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let hour: u8 = digits[8..10] + .parse() + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let min: u8 = digits[10..12] + .parse() + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let sec: u8 = if digits.len() == 14 { + digits[12..14] + .parse() + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)? + } else { + 0 + }; + let month = time::Month::try_from(mon) + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let date = time::Date::from_calendar_date(year, month, day) + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let time = time::Time::from_hms(hour, min, sec) + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + Ok(time::OffsetDateTime::new_utc(date, time)) +} + +fn strip_leading_zeros(bytes: &[u8]) -> &[u8] { + let mut idx = 0; + while idx < bytes.len() && bytes[idx] == 0 { + idx += 1; + } + if idx == bytes.len() { + &bytes[bytes.len() - 1..] + } else { + &bytes[idx..] + } +} diff --git a/crates/panda-rpki-validator/src/data_model/signed_object/signed_object_impl.rs b/crates/panda-rpki-validator/src/data_model/signed_object/signed_object_impl.rs new file mode 100644 index 0000000..df666a7 --- /dev/null +++ b/crates/panda-rpki-validator/src/data_model/signed_object/signed_object_impl.rs @@ -0,0 +1,133 @@ +// Signed-object decoding and RSA signature verification API. + +impl RpkiSignedObject { + /// Parse a DER-encoded RPKI Signed Object (CMS ContentInfo wrapping SignedData). + /// + /// This performs encoding/structure parsing only. Profile constraints are enforced by + /// `RpkiSignedObjectParsed::validate_profile`. + pub fn parse_der(der: &[u8]) -> Result { + parse_signed_object_content_info(der, der, CmsParseMode::BerCompatible) + } + + pub fn parse_der_strict_cms( + der: &[u8], + ) -> Result { + parse_signed_object_content_info(der, der, CmsParseMode::DerStrict) + } + + /// Return the strict-DER CMS parse error for an object that was otherwise + /// accepted through the normal BER-compatible CMS parser. + /// + /// Callers must only surface this as a compatibility warning after normal + /// decoding and validation have succeeded; a strict parse failure alone + /// does not prove that an arbitrary byte string is an RPKI signed object. + pub fn strict_cms_der_error(der: &[u8]) -> Option { + Self::parse_der_strict_cms(der).err() + } + + /// Decode a DER-encoded RPKI Signed Object (CMS ContentInfo wrapping SignedData) and enforce + /// the profile constraints from RFC 6488 §2-§3 and RFC 9589 §4. + pub fn decode_der(der: &[u8]) -> Result { + let parsed = Self::parse_der(der)?; + Ok(parsed.validate_profile()?) + } + + pub fn decode_der_with_strict_options( + der: &[u8], + strict_cms_der: bool, + strict_name: bool, + ) -> Result { + let parsed = if strict_cms_der { + Self::parse_der_strict_cms(der)? + } else { + Self::parse_der(der)? + }; + Ok(parsed.validate_profile_with_strict_name(strict_name)?) + } + + /// Scheme-A naming for signature verification. + pub fn verify(&self) -> Result<(), SignedObjectVerifyError> { + self.verify_signature() + } + + /// Verify the CMS signature using the embedded EE certificate public key. + pub fn verify_signature(&self) -> Result<(), SignedObjectVerifyError> { + let ee = &self.signed_data.certificates[0]; + let signer = &self.signed_data.signer_infos[0]; + crate::crypto_sig_cache::verify_with_cache( + crate::crypto_sig_cache::CryptoSigVerifyPoint::CmsSignedObject, + &signer.signed_attrs_der_for_signature, + &signer.signature, + &ee.spki_der, + || { + self.verify_signature_with_rsa_components( + &ee.rsa_public_modulus, + &ee.rsa_public_exponent, + ) + }, + ) + } + + /// Verify the CMS signature using a DER-encoded SubjectPublicKeyInfo. + pub fn verify_signature_with_ee_spki_der( + &self, + ee_spki_der: &[u8], + ) -> Result<(), SignedObjectVerifyError> { + let (rem, spki) = SubjectPublicKeyInfo::from_der(ee_spki_der) + .map_err(|e| SignedObjectVerifyError::EeSpkiParse(e.to_string()))?; + if !rem.is_empty() { + return Err(SignedObjectVerifyError::EeSpkiTrailingBytes(rem.len())); + } + self.verify_signature_with_ee_spki(&spki) + } + + /// Verify the CMS signature using a parsed SubjectPublicKeyInfo. + pub fn verify_signature_with_ee_spki( + &self, + ee_spki: &SubjectPublicKeyInfo<'_>, + ) -> Result<(), SignedObjectVerifyError> { + let pk = ee_spki + .parsed() + .map_err(|_e| SignedObjectVerifyError::UnsupportedEePublicKeyAlgorithm)?; + let (n, e) = match pk { + PublicKey::RSA(rsa) => { + let n = strip_leading_zeros(rsa.modulus).to_vec(); + let e = strip_leading_zeros(rsa.exponent).to_vec(); + let _exp = rsa + .try_exponent() + .map_err(|_e| SignedObjectVerifyError::InvalidEeRsaExponent)?; + (n, e) + } + _ => return Err(SignedObjectVerifyError::UnsupportedEePublicKeyAlgorithm), + }; + + let signer = &self.signed_data.signer_infos[0]; + crate::crypto_sig_cache::verify_with_cache( + crate::crypto_sig_cache::CryptoSigVerifyPoint::CmsSignedObject, + &signer.signed_attrs_der_for_signature, + &signer.signature, + ee_spki.raw, + || self.verify_signature_with_rsa_components(n.as_slice(), e.as_slice()), + ) + } + + fn verify_signature_with_rsa_components( + &self, + modulus: &[u8], + exponent: &[u8], + ) -> Result<(), SignedObjectVerifyError> { + let signer = &self.signed_data.signer_infos[0]; + let msg = &signer.signed_attrs_der_for_signature; + + let pk = ring::signature::RsaPublicKeyComponents { + n: modulus, + e: exponent, + }; + pk.verify( + &ring::signature::RSA_PKCS1_2048_8192_SHA256, + msg, + &signer.signature, + ) + .map_err(|_e| SignedObjectVerifyError::InvalidSignature) + } +} diff --git a/crates/panda-rpki-validator/src/data_model/signed_object/tests.rs b/crates/panda-rpki-validator/src/data_model/signed_object/tests.rs new file mode 100644 index 0000000..bd2f7e3 --- /dev/null +++ b/crates/panda-rpki-validator/src/data_model/signed_object/tests.rs @@ -0,0 +1,116 @@ +// Signed-object CMS compatibility and profile tests. + +use super::*; + +#[test] +fn strict_cms_der_rejects_constructed_octet_string_fixture() { + let der = + std::fs::read(std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.mft", + )) + .expect("read manifest fixture"); + let parsed = RpkiSignedObject::parse_der(&der).expect("parse fixture"); + let econtent = parsed + .signed_data + .encap_content_info + .econtent + .clone() + .expect("fixture eContent"); + assert_eq!( + parsed.signed_data.encap_content_info.econtent.as_deref(), + Some(econtent.as_slice()) + ); + + let primitive_octets = der_tlv(0x04, &econtent); + let constructed_octets = same_size_constructed_octet_string(&primitive_octets, &econtent); + let mutated = replace_first_subslice(&der, &primitive_octets, &constructed_octets) + .expect("replace eContent OCTET STRING"); + let compatible = RpkiSignedObject::parse_der(&mutated).expect("BER-compatible parse"); + assert!( + econtent.starts_with( + compatible + .signed_data + .encap_content_info + .econtent + .as_deref() + .expect("compatible eContent") + ) + ); + let err = RpkiSignedObject::parse_der_strict_cms(&mutated) + .expect_err("DER strict rejects constructed OCTET STRING"); + assert!(err.to_string().contains("DER"), "{err}"); + let compatibility_error = RpkiSignedObject::strict_cms_der_error(&mutated) + .expect("BER-compatible fixture must report strict-DER incompatibility"); + assert_eq!(compatibility_error.to_string(), err.to_string()); + assert!(RpkiSignedObject::strict_cms_der_error(&der).is_none()); +} + +fn replace_first_subslice(input: &[u8], from: &[u8], to: &[u8]) -> Option> { + let pos = input + .windows(from.len()) + .position(|candidate| candidate == from)?; + let mut out = Vec::with_capacity(input.len() - from.len() + to.len()); + out.extend_from_slice(&input[..pos]); + out.extend_from_slice(to); + out.extend_from_slice(&input[pos + from.len()..]); + Some(out) +} + +fn same_size_constructed_octet_string(primitive: &[u8], content: &[u8]) -> Vec { + assert_eq!(primitive[0], 0x04); + let header_len = tlv_header_len(primitive); + let outer_value_len = primitive.len() - header_len; + let child_len = (0..=outer_value_len) + .rev() + .find(|candidate| 1 + len_len(*candidate) + *candidate == outer_value_len) + .expect("find child length"); + let mut out = primitive[..header_len].to_vec(); + out[0] = 0x24; + out.extend(der_tlv(0x04, &content[..child_len])); + assert_eq!(out.len(), primitive.len()); + out +} + +fn tlv_header_len(tlv: &[u8]) -> usize { + if tlv[1] & 0x80 == 0 { + 2 + } else { + 2 + (tlv[1] & 0x7F) as usize + } +} + +fn len_len(len: usize) -> usize { + if len < 0x80 { + return 1; + } + let mut value = len; + let mut n = 0usize; + while value > 0 { + n += 1; + value >>= 8; + } + 1 + n +} + +fn der_tlv(tag: u8, value: &[u8]) -> Vec { + let mut out = vec![tag]; + encode_len(value.len(), &mut out); + out.extend_from_slice(value); + out +} + +fn encode_len(len: usize, out: &mut Vec) { + if len < 0x80 { + out.push(len as u8); + return; + } + let mut bytes = Vec::new(); + let mut value = len; + while value > 0 { + bytes.push((value & 0xFF) as u8); + value >>= 8; + } + bytes.reverse(); + out.push(0x80 | bytes.len() as u8); + out.extend(bytes); +} diff --git a/crates/panda-rpki-validator/src/data_model/signed_object/types_errors.rs b/crates/panda-rpki-validator/src/data_model/signed_object/types_errors.rs new file mode 100644 index 0000000..00c9a3b --- /dev/null +++ b/crates/panda-rpki-validator/src/data_model/signed_object/types_errors.rs @@ -0,0 +1,300 @@ +// CMS signed-object model types and parse/validation errors. + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum EeKeyUsageSummary { + DigitalSignatureOnly, + Missing, + NotCritical, + InvalidBits, + ParseError(String), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ResourceEeCertificate { + pub raw_der: Vec, + pub subject_key_identifier: Vec, + pub spki_der: Vec, + pub rsa_public_modulus: Vec, + pub rsa_public_exponent: Vec, + pub tbs_certificate_der: Vec, + pub signature_bytes: Vec, + pub key_usage_summary: EeKeyUsageSummary, + pub sia_signed_object_uris: Vec, + pub resource_cert: ResourceCertificate, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RpkiSignedObject { + pub raw_der: Vec, + pub content_info_content_type: String, + pub signed_data: SignedDataProfiled, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SignedDataProfiled { + pub version: u32, + pub digest_algorithms: Vec, + pub encap_content_info: EncapsulatedContentInfo, + pub certificates: Vec, + pub crls_present: bool, + pub signer_infos: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EncapsulatedContentInfo { + pub econtent_type: String, + pub econtent: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SignerInfoProfiled { + pub version: u32, + pub sid_ski: Vec, + pub digest_algorithm: String, + pub signature_algorithm: String, + pub signed_attrs: SignedAttrsProfiled, + pub unsigned_attrs_present: bool, + pub signature: Vec, + pub signed_attrs_der_for_signature: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SignedAttrsProfiled { + pub content_type: String, + pub message_digest: Vec, + pub signing_time: Asn1TimeUtc, + pub other_attrs_present: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RpkiSignedObjectParsed { + pub raw_der: Vec, + pub content_info_content_type: String, + pub signed_data: SignedDataParsed, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SignedDataParsed { + pub version: u64, + pub digest_algorithms: Vec, + pub encap_content_info: EncapsulatedContentInfoParsed, + pub certificates: Option>>, + pub crls_present: bool, + pub signer_infos: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AlgorithmIdentifierParsed { + pub oid: String, + pub params_ok: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EncapsulatedContentInfoParsed { + pub econtent_type: String, + pub econtent: Option>, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SignerInfoParsed { + pub version: u64, + pub sid: SignerIdentifierParsed, + pub digest_algorithm: AlgorithmIdentifierParsed, + pub signature_algorithm: AlgorithmIdentifierParsed, + pub signed_attrs_content: Option>, + pub signed_attrs_der_for_signature: Option>, + pub unsigned_attrs_present: bool, + pub signature: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SignerIdentifierParsed { + SubjectKeyIdentifier(Vec), + Other, +} + +#[derive(Debug, thiserror::Error)] +pub enum SignedObjectParseError { + #[error("DER parse error: {0} (RFC 6488 §2; RFC 6488 §3(1l); RFC 5652 §3/§5)")] + Parse(String), + + #[error("trailing bytes after DER object: {0} bytes (DER; RFC 6488 §3(1l))")] + TrailingBytes(usize), +} + +#[derive(Debug, thiserror::Error)] +pub enum SignedObjectValidateError { + #[error( + "ContentInfo.contentType must be SignedData ({OID_SIGNED_DATA}), got {0} (RFC 6488 §3(1a); RFC 5652 §3)" + )] + InvalidContentInfoContentType(String), + + #[error( + "SignedData.version must be 3, got {0} (RFC 6488 §2.1.1; RFC 6488 §3(1b); RFC 5652 §5.1)" + )] + InvalidSignedDataVersion(u64), + + #[error( + "SignedData.digestAlgorithms must contain exactly one AlgorithmIdentifier, got {0} (RFC 6488 §2.1.2; RFC 6488 §3(1b); RFC 5652 §5.1)" + )] + InvalidDigestAlgorithmsCount(usize), + + #[error( + "digest algorithm must be id-sha256 ({OID_SHA256}), got {0} (RFC 6488 §2.1.2; RFC 6488 §3(1b); RFC 7935 §2)" + )] + InvalidDigestAlgorithm(String), + + #[error("SignedData.certificates MUST be present (RFC 6488 §3(1c); RFC 5652 §5.1)")] + CertificatesMissing, + + #[error( + "SignedData.certificates must contain exactly one EE certificate, got {0} (RFC 6488 §3(1c))" + )] + InvalidCertificatesCount(usize), + + #[error("SignedData.crls MUST be omitted (RFC 6488 §3(1d))")] + CrlsPresent, + + #[error( + "SignedData.signerInfos must contain exactly one SignerInfo, got {0} (RFC 6488 §2.1; RFC 6488 §3(1e); RFC 5652 §5.1)" + )] + InvalidSignerInfosCount(usize), + + #[error("SignerInfo.version must be 3, got {0} (RFC 6488 §3(1e); RFC 5652 §5.3)")] + InvalidSignerInfoVersion(u64), + + #[error("SignerInfo.sid must be subjectKeyIdentifier [0] (RFC 6488 §3(1c); RFC 5652 §5.3)")] + InvalidSignerIdentifier, + + #[error( + "SignerInfo.digestAlgorithm must be id-sha256 ({OID_SHA256}), got {0} (RFC 6488 §3(1j); RFC 7935 §2)" + )] + InvalidSignerInfoDigestAlgorithm(String), + + #[error("SignerInfo.signedAttrs MUST be present (RFC 9589 §4; RFC 6488 §3(1f))")] + SignedAttrsMissing, + + #[error("SignerInfo.unsignedAttrs MUST be omitted (RFC 6488 §3(1i))")] + UnsignedAttrsPresent, + + #[error( + "SignerInfo.signatureAlgorithm must be rsaEncryption ({OID_RSA_ENCRYPTION}) or \ +sha256WithRSAEncryption ({OID_SHA256_WITH_RSA_ENCRYPTION}), got {0} (RFC 6488 §3(1k); RFC 7935 §2)" + )] + InvalidSignatureAlgorithm(String), + + #[error( + "SignerInfo.signatureAlgorithm parameters must be absent or NULL (RFC 5280 §4.1.1.2; RFC 7935 §2)" + )] + InvalidSignatureAlgorithmParameters, + + #[error("signedAttrs contains unsupported attribute OID {0} (RFC 9589 §4; RFC 6488 §2.1.6.4)")] + UnsupportedSignedAttribute(String), + + #[error("signedAttrs contains duplicate attribute OID {0} (RFC 6488 §2.1.6.4; RFC 9589 §4)")] + DuplicateSignedAttribute(String), + + #[error("signedAttrs parse error: {0} (RFC 5652 §5.3; RFC 6488 §3(1f); RFC 9589 §4)")] + SignedAttrsParse(String), + + #[error( + "signedAttrs attribute {oid} attrValues must contain exactly one value, got {count} (RFC 6488 §2.1.6.4; RFC 5652 §5.3)" + )] + InvalidSignedAttributeValuesCount { oid: String, count: usize }, + + #[error( + "signedAttrs missing content-type attribute (RFC 9589 §4; RFC 5652 §11.1; RFC 6488 §2.1.6.4)" + )] + SignedAttrsContentTypeMissing, + + #[error( + "signedAttrs missing message-digest attribute (RFC 9589 §4; RFC 5652 §11.2; RFC 6488 §2.1.6.4)" + )] + SignedAttrsMessageDigestMissing, + + #[error( + "signedAttrs missing signing-time attribute (RFC 9589 §4; RFC 5652 §11.3; RFC 6488 §2.1.6.4)" + )] + SignedAttrsSigningTimeMissing, + + #[error( + "signedAttrs.content-type attrValues must equal eContentType ({econtent_type}), got {attr_content_type} (RFC 6488 §3(1h); RFC 9589 §4)" + )] + ContentTypeAttrMismatch { + econtent_type: String, + attr_content_type: String, + }, + + #[error("EncapsulatedContentInfo.eContent MUST be present (RFC 6488 §2.1.3; RFC 5652 §5.2)")] + EContentMissing, + + #[error( + "signedAttrs.message-digest does not match SHA-256(eContent) (RFC 6488 §3(1f); RFC 5652 §11.2)" + )] + MessageDigestMismatch, + + #[error("EE certificate parse error: {0} (RFC 6488 §3(1c); RFC 6487 §4)")] + EeCertificateParse(String), + + #[error( + "EE certificate missing SubjectKeyIdentifier extension (RFC 6488 §3(1c); RFC 6487 §4.8.2)" + )] + EeCertificateMissingSki, + + #[error( + "EE certificate missing SubjectInfoAccess extension ({OID_SUBJECT_INFO_ACCESS}) (RFC 6487 §4.8.8.2)" + )] + EeCertificateMissingSia, + + #[error( + "EE certificate SIA missing id-ad-signedObject access method ({OID_AD_SIGNED_OBJECT}) (RFC 6487 §4.8.8.2)" + )] + EeCertificateMissingSignedObjectSia, + + #[error( + "EE certificate SIA id-ad-signedObject accessLocation must be a URI (RFC 6487 §4.8.8.2; RFC 5280 §4.2.2.2)" + )] + EeCertificateSignedObjectSiaNotUri, + + #[error( + "EE certificate SIA id-ad-signedObject must include at least one rsync:// URI (RFC 6487 §4.8.8.2)" + )] + EeCertificateSignedObjectSiaNoRsync, + + #[error( + "SignerInfo.sid SKI does not match EE certificate SKI (RFC 6488 §3(1c); RFC 5652 §5.3)" + )] + SidSkiMismatch, + + #[error( + "invalid signing-time attribute value (expected UTCTime or GeneralizedTime) (RFC 5652 §11.3; RFC 9589 §4)" + )] + InvalidSigningTimeValue, +} + +#[derive(Debug, thiserror::Error)] +pub enum SignedObjectDecodeError { + #[error("SignedObject parse error: {0}")] + Parse(#[from] SignedObjectParseError), + + #[error("SignedObject validate error: {0}")] + Validate(#[from] SignedObjectValidateError), +} + +#[derive(Debug, thiserror::Error)] +pub enum SignedObjectVerifyError { + #[error("EE SubjectPublicKeyInfo parse error: {0} (RFC 5280 §4.1.2.7)")] + EeSpkiParse(String), + + #[error("trailing bytes after EE SubjectPublicKeyInfo DER: {0} bytes (DER; RFC 5280 §4.1.2.7)")] + EeSpkiTrailingBytes(usize), + + #[error("unsupported EE public key algorithm (only RSA is supported) (RFC 7935 §2)")] + UnsupportedEePublicKeyAlgorithm, + + #[error("EE RSA public exponent invalid (RFC 8017 §A.1.1; RFC 7935 §2)")] + InvalidEeRsaExponent, + + #[error("signature verification failed (RFC 6488 §3(2)-(3); RFC 5652 §5.3; RFC 7935 §2)")] + InvalidSignature, +} diff --git a/crates/panda-rpki-validator/src/fetch/http.rs b/crates/panda-rpki-validator/src/fetch/http.rs index 7286277..28aecd6 100644 --- a/crates/panda-rpki-validator/src/fetch/http.rs +++ b/crates/panda-rpki-validator/src/fetch/http.rs @@ -67,7 +67,7 @@ impl Default for HttpFetcherConfig { } } -/// Minimal blocking HTTP(S) fetcher for stage2. +/// Minimal blocking HTTP(S) fetcher for validation runs. /// /// This is used for: /// - downloading TAL / TA certificates (RFC 8630 §2) diff --git a/crates/panda-rpki-validator/src/fetch/rsync_system.rs b/crates/panda-rpki-validator/src/fetch/rsync_system.rs index 20e44cf..894d9e2 100644 --- a/crates/panda-rpki-validator/src/fetch/rsync_system.rs +++ b/crates/panda-rpki-validator/src/fetch/rsync_system.rs @@ -72,7 +72,7 @@ impl Default for SystemRsyncConfig { /// A `RsyncFetcher` implementation backed by the system `rsync` binary. /// -/// This is intended for live stage2 runs. For unit tests and offline fixtures, +/// This is intended for live synchronization runs. For unit tests and offline fixtures, /// prefer `LocalDirRsyncFetcher`. #[derive(Clone, Debug)] pub struct SystemRsyncFetcher { diff --git a/crates/panda-rpki-validator/src/lib.rs b/crates/panda-rpki-validator/src/lib.rs index fd9ed65..324ce1a 100644 --- a/crates/panda-rpki-validator/src/lib.rs +++ b/crates/panda-rpki-validator/src/lib.rs @@ -1,14 +1,12 @@ //! Public package boundary for the Panda RPKI synchronization validator. //! -//! The package intentionally keeps the audited implementation in one crate -//! while exposing only the normal synchronization/validation CLI. A separate -//! offline revalidation workflow remains in the original private repository -//! until a follow-up backlog item defines its public contract. +//! The package intentionally keeps the synchronization and validation runtime +//! in one crate. The command-line interface is the supported integration +//! surface; internal modules are progressively documented and narrowed as the +//! public API is defined. //! -//! The implementation is extracted from the private `rpki` tree. During the -//! parity window we intentionally keep the upstream implementation's lint -//! profile unchanged; functional and output-equivalence gates take precedence -//! over behavior-changing lint cleanup. +//! The crate preserves a stable run-artifact contract. Behavioural changes must +//! be validated against fixed offline inputs before they are released. #![allow(clippy::all)] #![allow(clippy::pedantic, clippy::nursery, clippy::restriction)] #![allow(dead_code, deprecated, unused_mut)] diff --git a/crates/panda-rpki-validator/src/parallel/config.rs b/crates/panda-rpki-validator/src/parallel/config.rs index a4d8ff1..984c6c3 100644 --- a/crates/panda-rpki-validator/src/parallel/config.rs +++ b/crates/panda-rpki-validator/src/parallel/config.rs @@ -5,7 +5,7 @@ pub struct ParallelPhase1Config { pub max_repo_sync_workers_global: usize, pub max_inflight_snapshot_bytes_global: usize, pub max_pending_repo_results: usize, - /// Dead-repo transport blacklist (#141). `None` disables the feature + /// Dead-repository transport blacklist. None disables the feature /// entirely (default, behavior unchanged). pub dead_repo_blacklist: Option, } diff --git a/crates/panda-rpki-validator/src/parallel/dead_repo_blacklist.rs b/crates/panda-rpki-validator/src/parallel/dead_repo_blacklist.rs index 3b8d641..3772512 100644 --- a/crates/panda-rpki-validator/src/parallel/dead_repo_blacklist.rs +++ b/crates/panda-rpki-validator/src/parallel/dead_repo_blacklist.rs @@ -1,4 +1,4 @@ -//! Persistent dead-repo transport blacklist (#141). +//! Persistent dead-repository transport blacklist. //! //! Tracks per-(repo, transport) consecutive transport-layer fetch failures //! across runs. Once an entry reaches the configured failure threshold it is @@ -340,7 +340,7 @@ pub fn classify_rsync_fetch_error(detail: &str) -> RepoTransportErrorClass { if crate::parallel::repo_scheduler::is_host_level_rsync_failure(detail) // Stall failures are transport death too: the fail-fast mechanism // gives up when wall-clock windows pass with no (additional) progress - // (#141 M9). Content-level errors fail immediately and never carry + // Content-level errors fail immediately and never carry // this marker, so they still do not count. || detail.contains("rsync fail-fast gave up") { diff --git a/crates/panda-rpki-validator/src/parallel/dead_repo_blacklist_tests.rs b/crates/panda-rpki-validator/src/parallel/dead_repo_blacklist_tests.rs index 3cb95d4..3fe0a19 100644 --- a/crates/panda-rpki-validator/src/parallel/dead_repo_blacklist_tests.rs +++ b/crates/panda-rpki-validator/src/parallel/dead_repo_blacklist_tests.rs @@ -371,7 +371,7 @@ fn rsync_fetch_error_classification() { classify_rsync_fetch_error("rsync file digest mismatch after download"), RepoTransportErrorClass::Unknown ); - // Stall failures (#141 M9): fail-fast give-up means the host delivered no + // Stall failures: fail-fast give-up means the host delivered no // bytes inside the wall-clock window — transport death, must count. assert_eq!( classify_rsync_fetch_error( diff --git a/crates/panda-rpki-validator/src/parallel/repo_runtime.rs b/crates/panda-rpki-validator/src/parallel/repo_runtime.rs index ac7a4a2..aa445f7 100644 --- a/crates/panda-rpki-validator/src/parallel/repo_runtime.rs +++ b/crates/panda-rpki-validator/src/parallel/repo_runtime.rs @@ -16,1499 +16,11 @@ use crate::policy::SyncPreference; use crate::report::Warning; use crate::validation::tree::{CaInstanceHandle, DiscoveredChildCaInstance}; -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RepoSyncRuntimeOutcome { - pub repo_sync_ok: bool, - pub repo_sync_err: Option, - pub repo_sync_source: Option, - pub repo_sync_phase: Option, - pub repo_sync_duration_ms: u64, - pub warnings: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum RepoSyncRequestStatus { - Ready { - identity: RepoIdentity, - outcome: RepoSyncRuntimeOutcome, - }, - Pending { - identity: RepoIdentity, - state: RepoRuntimeState, - }, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RepoSyncRuntimeCompletion { - pub identity: RepoIdentity, - pub state: RepoRuntimeState, - pub outcome: RepoSyncRuntimeOutcome, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RepoSyncRuntimeEvent { - pub transport_identity: RepoIdentity, - pub completions: Vec, -} - -pub trait RepoSyncRuntime: Send + Sync { - fn sync_publication_point_repo( - &self, - ca: &CaInstanceHandle, - ) -> Result; - - fn request_publication_point_repo( - &self, - ca: &CaInstanceHandle, - priority: u8, - ) -> Result; - - fn recv_repo_result_timeout( - &self, - timeout: Duration, - ) -> Result, String>; - - fn drain_repo_results_timeout( - &self, - timeout: Duration, - max_events: usize, - ) -> Result, String> { - let max_events = max_events.max(1); - let mut events = Vec::new(); - for index in 0..max_events { - let poll_timeout = if index == 0 { - timeout - } else { - Duration::from_millis(0) - }; - let Some(event) = self.recv_repo_result_timeout(poll_timeout)? else { - break; - }; - events.push(event); - } - Ok(events) - } - - fn reset_run_state(&self) -> Result<(), String>; - - fn prefetch_discovered_children( - &self, - children: &[DiscoveredChildCaInstance], - ) -> Result<(), String>; - - fn prefetch_transport_requests( - &self, - snapshot: &TransportPrefetchSnapshot, - validation_time: time::OffsetDateTime, - ) -> Result; - - fn transport_prefetch_snapshot(&self) -> TransportPrefetchSnapshot; - - /// Dead-repo blacklist working copy (#141) for run-end persistence. - /// `None` when the feature is disabled. - fn dead_repo_blacklist_state( - &self, - ) -> Option<( - crate::parallel::dead_repo_blacklist::DeadRepoBlacklistConfig, - crate::parallel::dead_repo_blacklist::DeadRepoBlacklist, - )> { - None - } -} - -pub struct Phase1RepoSyncRuntime { - coordinator: Mutex, - worker_pool: Mutex>, - transport_prefetch_recorder: Option>, - retry_short_rsync_scopes: Mutex>, - rsync_scope_resolver: Arc String + Send + Sync>, - rsync_failure_scope_resolver: Arc Option + Send + Sync>, - sync_preference: SyncPreference, -} - -impl Phase1RepoSyncRuntime { - pub fn new( - coordinator: GlobalRunCoordinator, - worker_pool: RepoTransportWorkerPool, - rsync_scope_resolver: Arc String + Send + Sync>, - sync_preference: SyncPreference, - ) -> Self { - Self::new_with_failure_scope( - coordinator, - worker_pool, - rsync_scope_resolver, - Arc::new(|_base: &str| None), - sync_preference, - ) - } - - pub fn new_with_failure_scope( - coordinator: GlobalRunCoordinator, - worker_pool: RepoTransportWorkerPool, - rsync_scope_resolver: Arc String + Send + Sync>, - rsync_failure_scope_resolver: Arc Option + Send + Sync>, - sync_preference: SyncPreference, - ) -> Self { - Self { - coordinator: Mutex::new(coordinator), - worker_pool: Mutex::new(worker_pool), - transport_prefetch_recorder: None, - retry_short_rsync_scopes: Mutex::new(HashSet::new()), - rsync_scope_resolver, - rsync_failure_scope_resolver, - sync_preference, - } - } - - pub fn new_with_failure_scope_and_prefetch_recording( - coordinator: GlobalRunCoordinator, - worker_pool: RepoTransportWorkerPool, - rsync_scope_resolver: Arc String + Send + Sync>, - rsync_failure_scope_resolver: Arc Option + Send + Sync>, - sync_preference: SyncPreference, - record_transport_prefetch_requests: bool, - ) -> Self { - Self { - coordinator: Mutex::new(coordinator), - worker_pool: Mutex::new(worker_pool), - transport_prefetch_recorder: record_transport_prefetch_requests - .then(|| Mutex::new(TransportPrefetchRecorder::default())), - retry_short_rsync_scopes: Mutex::new(HashSet::new()), - rsync_scope_resolver, - rsync_failure_scope_resolver, - sync_preference, - } - } - - fn build_requester(ca: &CaInstanceHandle) -> RepoRequester { - RepoRequester { - tal_id: ca.tal_id.clone(), - rir_id: ca.tal_id.clone(), - parent_node_id: None, - ca_instance_handle_id: format!("{}:{}", ca.tal_id, ca.manifest_rsync_uri), - publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(), - manifest_rsync_uri: ca.manifest_rsync_uri.clone(), - } - } - - fn build_identity(ca: &CaInstanceHandle) -> RepoIdentity { - RepoIdentity::new(ca.rrdp_notification_uri.clone(), ca.rsync_base_uri.clone()) - } - - fn request_transport_for_ca( - &self, - ca: &CaInstanceHandle, - priority: u8, - ) -> Result { - let identity = Self::build_identity(ca); - let requester = Self::build_requester(ca); - let rsync_scope_uri = (self.rsync_scope_resolver)(&identity.rsync_base_uri); - let rsync_failure_scope_uri = (self.rsync_failure_scope_resolver)(&identity.rsync_base_uri); - if let Some(recorder) = self.transport_prefetch_recorder.as_ref() { - let mut recorder = recorder - .lock() - .expect("transport prefetch recorder lock poisoned"); - recorder.record_registered_request( - &identity, - &requester, - priority, - rsync_scope_uri.clone(), - rsync_failure_scope_uri.clone(), - self.sync_preference, - ); - } - let action = { - let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); - coordinator.register_transport_request( - identity.clone(), - requester, - time::OffsetDateTime::now_utc(), - priority, - rsync_scope_uri, - rsync_failure_scope_uri, - self.sync_preference, - false, - ) - }; - - match action { - TransportRequestAction::Enqueue(task) => { - crate::progress_log::emit( - "phase1_repo_task_enqueued", - serde_json::json!({ - "manifest_rsync_uri": ca.manifest_rsync_uri, - "publication_point_rsync_uri": ca.publication_point_rsync_uri, - "repo_key_rsync_base_uri": task.repo_identity.rsync_base_uri, - "rsync_failure_scope_uri": task.rsync_failure_scope_uri, - "repo_key_notification_uri": task.repo_identity.notification_uri, - "priority": priority, - "transport_mode": match task.mode { - RepoTransportMode::Rrdp => "rrdp", - RepoTransportMode::Rsync => "rsync", - }, - }), - ); - self.drain_pending_transport_tasks()?; - Ok(RepoSyncRequestStatus::Pending { - identity, - state: self - .runtime_state_for_identity(&task.repo_identity) - .unwrap_or(RepoRuntimeState::WaitingRrdp), - }) - } - TransportRequestAction::Waiting { state } => { - crate::progress_log::emit( - "phase1_repo_task_waiting", - serde_json::json!({ - "manifest_rsync_uri": ca.manifest_rsync_uri, - "publication_point_rsync_uri": ca.publication_point_rsync_uri, - "repo_key_rsync_base_uri": identity.rsync_base_uri, - "rsync_failure_scope_uri": (self.rsync_failure_scope_resolver)(&identity.rsync_base_uri), - "repo_key_notification_uri": identity.notification_uri, - "priority": priority, - "runtime_state": format!("{state:?}"), - }), - ); - Ok(RepoSyncRequestStatus::Pending { identity, state }) - } - TransportRequestAction::ReusedSuccess(result) - | TransportRequestAction::ReusedTerminalFailure(result) => { - crate::progress_log::emit( - "phase1_repo_task_reused", - serde_json::json!({ - "manifest_rsync_uri": ca.manifest_rsync_uri, - "publication_point_rsync_uri": ca.publication_point_rsync_uri, - "repo_key_rsync_base_uri": identity.rsync_base_uri, - "rsync_failure_scope_uri": result.rsync_failure_scope_uri, - "repo_key_notification_uri": identity.notification_uri, - "priority": priority, - "transport_mode": match result.mode { - RepoTransportMode::Rrdp => "rrdp", - RepoTransportMode::Rsync => "rsync", - }, - }), - ); - Ok(RepoSyncRequestStatus::Ready { - outcome: outcome_from_transport_result( - &result, - self.runtime_state_for_identity(&identity) - .unwrap_or(RepoRuntimeState::Init), - ), - identity, - }) - } - } - } - - fn drain_pending_transport_tasks(&self) -> Result<(), String> { - loop { - let maybe_task = { - let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); - coordinator.pop_next_transport_task() - }; - let Some(task) = maybe_task else { - break; - }; - { - let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); - coordinator - .mark_transport_running(&task.dedup_key, time::OffsetDateTime::now_utc())?; - } - crate::progress_log::emit( - "phase1_repo_task_dispatched", - serde_json::json!({ - "repo_key_rsync_base_uri": task.repo_identity.rsync_base_uri, - "rsync_failure_scope_uri": task.rsync_failure_scope_uri, - "repo_key_notification_uri": task.repo_identity.notification_uri, - "requester_count": task.requesters.len(), - "priority": task.priority, - "transport_mode": match task.mode { - RepoTransportMode::Rrdp => "rrdp", - RepoTransportMode::Rsync => "rsync", - }, - }), - ); - let pool = self.worker_pool.lock().expect("worker pool lock poisoned"); - pool.submit(task)?; - } - Ok(()) - } - - fn pump_one_transport_result( - &self, - timeout: Duration, - ) -> Result, String> { - let envelope = { - let pool = self.worker_pool.lock().expect("worker pool lock poisoned"); - pool.recv_result_timeout(timeout)? - }; - let Some(envelope) = envelope else { - return Ok(None); - }; - let transport_identity = envelope.repo_identity.clone(); - let completed_envelope = envelope.clone(); - if let Some(recorder) = self.transport_prefetch_recorder.as_ref() { - recorder - .lock() - .expect("transport prefetch recorder lock poisoned") - .record_result(&envelope); - } - crate::progress_log::emit( - "phase1_repo_task_result", - serde_json::json!({ - "repo_key_rsync_base_uri": envelope.repo_identity.rsync_base_uri, - "rsync_failure_scope_uri": envelope.rsync_failure_scope_uri, - "repo_key_notification_uri": envelope.repo_identity.notification_uri, - "timing_ms": envelope.timing_ms, - "transport_mode": match envelope.mode { - RepoTransportMode::Rrdp => "rrdp", - RepoTransportMode::Rsync => "rsync", - }, - "result": match &envelope.result { - RepoTransportResultKind::Success { .. } => "success", - RepoTransportResultKind::Failed { .. } => "failed", - }, - }), - ); - let finished_at = time::OffsetDateTime::now_utc(); - let completion = { - let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); - coordinator.complete_transport_result(envelope, finished_at)? - }; - if !completion.follow_up_tasks.is_empty() { - let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); - for mut task in completion.follow_up_tasks { - if let crate::parallel::types::RepoDedupKey::RsyncScope { rsync_scope_uri } = - &task.dedup_key - { - if self - .retry_short_rsync_scopes - .lock() - .expect("retry short rsync scopes lock poisoned") - .contains(rsync_scope_uri) - { - task.retry_short_timeout = true; - } - } - crate::progress_log::emit( - "phase1_repo_task_enqueued", - serde_json::json!({ - "manifest_rsync_uri": serde_json::Value::Null, - "publication_point_rsync_uri": task.requesters.first().map(|r| r.publication_point_rsync_uri.clone()), - "repo_key_rsync_base_uri": task.repo_identity.rsync_base_uri, - "repo_key_notification_uri": task.repo_identity.notification_uri, - "priority": task.priority, - "transport_mode": "rsync", - }), - ); - coordinator.push_transport_task(task); - } - } - self.drain_pending_transport_tasks()?; - let completions = { - let coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); - coordinator - .finalized_runtime_records_for_transport_result(&completed_envelope) - .into_iter() - .filter_map(|record| { - let outcome = match record.state { - RepoRuntimeState::RrdpOk | RepoRuntimeState::RsyncOk => record - .last_success - .as_ref() - .map(|result| outcome_from_transport_result(result, record.state)), - RepoRuntimeState::FailedTerminal => record - .terminal_failure - .as_ref() - .map(|result| outcome_from_transport_result(result, record.state)), - _ => None, - }?; - Some(RepoSyncRuntimeCompletion { - identity: record.identity, - state: record.state, - outcome, - }) - }) - .collect::>() - }; - if completions.is_empty() { - return Ok(None); - } - Ok(Some(RepoSyncRuntimeEvent { - transport_identity, - completions, - })) - } - - fn pump_transport_results( - &self, - timeout: Duration, - max_events: usize, - ) -> Result, String> { - let max_events = max_events.max(1); - let mut events = Vec::new(); - for index in 0..max_events { - let poll_timeout = if index == 0 { - timeout - } else { - Duration::from_millis(0) - }; - let Some(event) = self.pump_one_transport_result(poll_timeout)? else { - break; - }; - events.push(event); - } - Ok(events) - } - - fn runtime_state_for_identity(&self, identity: &RepoIdentity) -> Option { - let coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); - coordinator - .runtime_record(identity) - .map(|record| record.state) - } - - fn resolved_outcome_for_identity( - &self, - identity: &RepoIdentity, - ) -> Option { - let coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); - let record = coordinator.runtime_record(identity)?; - match record.state { - RepoRuntimeState::RrdpOk | RepoRuntimeState::RsyncOk => record - .last_success - .as_ref() - .map(|result| outcome_from_transport_result(result, record.state)), - RepoRuntimeState::FailedTerminal => record - .terminal_failure - .as_ref() - .map(|result| outcome_from_transport_result(result, record.state)), - _ => None, - } - } -} - -impl RepoSyncRuntime for Phase1RepoSyncRuntime { - fn sync_publication_point_repo( - &self, - ca: &CaInstanceHandle, - ) -> Result { - if let RepoSyncRequestStatus::Ready { outcome, .. } = - self.request_publication_point_repo(ca, 0)? - { - return Ok(outcome); - } - let identity = Self::build_identity(ca); - loop { - if let Some(done) = self.resolved_outcome_for_identity(&identity) { - return Ok(done); - } - let _ = self.recv_repo_result_timeout(Duration::from_millis(50))?; - } - } - - fn request_publication_point_repo( - &self, - ca: &CaInstanceHandle, - priority: u8, - ) -> Result { - self.request_transport_for_ca(ca, priority) - } - - fn recv_repo_result_timeout( - &self, - timeout: Duration, - ) -> Result, String> { - self.pump_one_transport_result(timeout) - } - - fn drain_repo_results_timeout( - &self, - timeout: Duration, - max_events: usize, - ) -> Result, String> { - self.pump_transport_results(timeout, max_events) - } - - fn reset_run_state(&self) -> Result<(), String> { - { - let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); - if coordinator.stats.repo_tasks_running != 0 { - return Err(format!( - "cannot reset repo runtime with {} repo task(s) still running", - coordinator.stats.repo_tasks_running - )); - } - coordinator.reset_run_state(); - } - loop { - let maybe_result = { - let pool = self.worker_pool.lock().expect("worker pool lock poisoned"); - pool.recv_result_timeout(Duration::from_millis(0))? - }; - if maybe_result.is_none() { - break; - } - } - Ok(()) - } - - fn prefetch_discovered_children( - &self, - children: &[DiscoveredChildCaInstance], - ) -> Result<(), String> { - for child in children { - let _ = self.request_publication_point_repo(&child.handle, 1)?; - } - Ok(()) - } - - fn prefetch_transport_requests( - &self, - snapshot: &TransportPrefetchSnapshot, - validation_time: time::OffsetDateTime, - ) -> Result { - let mut stats = TransportPrefetchDispatchStats { - loaded_requests: snapshot.requests.len() as u64, - ..TransportPrefetchDispatchStats::default() - }; - - for request in &snapshot.requests { - let identity = request.to_identity(); - let current_rsync_scope_uri = (self.rsync_scope_resolver)(&identity.rsync_base_uri); - let current_rsync_failure_scope_uri = - (self.rsync_failure_scope_resolver)(&identity.rsync_base_uri); - if current_rsync_scope_uri != request.rsync_scope_uri - || current_rsync_failure_scope_uri != request.rsync_failure_scope_uri - { - stats.skipped_incompatible += 1; - continue; - } - if request.retry_short_rsync_timeout() { - self.retry_short_rsync_scopes - .lock() - .expect("retry short rsync scopes lock poisoned") - .insert(current_rsync_scope_uri.clone()); - } - let action = { - let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); - coordinator.register_transport_request( - identity, - request.to_requester(), - validation_time, - request.priority, - current_rsync_scope_uri, - current_rsync_failure_scope_uri, - self.sync_preference, - request.retry_short_timeout(), - ) - }; - - match action { - TransportRequestAction::Enqueue(task) => { - stats.enqueued_tasks += 1; - crate::progress_log::emit( - "phase1_repo_prefetch_enqueued", - serde_json::json!({ - "repo_key_rsync_base_uri": task.repo_identity.rsync_base_uri, - "rsync_failure_scope_uri": task.rsync_failure_scope_uri, - "repo_key_notification_uri": task.repo_identity.notification_uri, - "priority": task.priority, - "transport_mode": match task.mode { - RepoTransportMode::Rrdp => "rrdp", - RepoTransportMode::Rsync => "rsync", - }, - }), - ); - } - TransportRequestAction::Waiting { .. } => { - stats.waiting_requests += 1; - } - TransportRequestAction::ReusedSuccess(_) - | TransportRequestAction::ReusedTerminalFailure(_) => { - stats.reused_results += 1; - } - } - } - - self.drain_pending_transport_tasks()?; - Ok(stats) - } - - fn transport_prefetch_snapshot(&self) -> TransportPrefetchSnapshot { - self.transport_prefetch_recorder - .as_ref() - .map(|recorder| { - recorder - .lock() - .expect("transport prefetch recorder lock poisoned") - .snapshot(self.sync_preference) - }) - .unwrap_or_else(|| TransportPrefetchSnapshot::new(self.sync_preference, Vec::new())) - } - - fn dead_repo_blacklist_state( - &self, - ) -> Option<( - crate::parallel::dead_repo_blacklist::DeadRepoBlacklistConfig, - crate::parallel::dead_repo_blacklist::DeadRepoBlacklist, - )> { - let coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); - let config = coordinator.config.dead_repo_blacklist.clone()?; - let blacklist = coordinator.dead_repo_blacklist().cloned()?; - Some((config, blacklist)) - } -} - -fn outcome_from_transport_result( - envelope: &RepoTransportResultEnvelope, - state: RepoRuntimeState, -) -> RepoSyncRuntimeOutcome { - match (&envelope.result, state) { - (RepoTransportResultKind::Success { source, warnings }, RepoRuntimeState::RrdpOk) => { - RepoSyncRuntimeOutcome { - repo_sync_ok: true, - repo_sync_err: None, - repo_sync_source: Some(source.clone()), - repo_sync_phase: Some("rrdp_ok".to_string()), - repo_sync_duration_ms: envelope.timing_ms, - warnings: warnings.clone(), - } - } - (RepoTransportResultKind::Success { source, warnings }, RepoRuntimeState::RsyncOk) => { - RepoSyncRuntimeOutcome { - repo_sync_ok: true, - repo_sync_err: None, - repo_sync_source: Some(source.clone()), - repo_sync_phase: Some(if envelope.repo_identity.notification_uri.is_some() { - "rrdp_failed_rsync_ok".to_string() - } else { - "rsync_only_ok".to_string() - }), - repo_sync_duration_ms: envelope.timing_ms, - warnings: warnings.clone(), - } - } - ( - RepoTransportResultKind::Failed { - detail, warnings, .. - }, - RepoRuntimeState::FailedTerminal, - ) => RepoSyncRuntimeOutcome { - repo_sync_ok: false, - repo_sync_err: Some(detail.clone()), - repo_sync_source: None, - repo_sync_phase: Some(if envelope.repo_identity.notification_uri.is_some() { - "rrdp_failed_rsync_failed".to_string() - } else { - "rsync_failed".to_string() - }), - repo_sync_duration_ms: envelope.timing_ms, - warnings: warnings.clone(), - }, - _ => RepoSyncRuntimeOutcome { - repo_sync_ok: false, - repo_sync_err: Some("repo runtime state unresolved".to_string()), - repo_sync_source: None, - repo_sync_phase: Some("repo_runtime_unresolved".to_string()), - repo_sync_duration_ms: envelope.timing_ms, - warnings: Vec::new(), - }, - } -} +include!("repo_runtime/types_and_trait.rs"); +include!("repo_runtime/phase1_runtime.rs"); +include!("repo_runtime/runtime_trait_impl.rs"); +include!("repo_runtime/outcome.rs"); #[cfg(test)] -mod tests { - use std::sync::Arc; - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::time::{Duration, Instant}; - - use crate::parallel::config::ParallelPhase1Config; - use crate::parallel::repo_runtime::{Phase1RepoSyncRuntime, RepoSyncRuntime}; - use crate::parallel::repo_worker::{ - RepoTransportExecutor, RepoTransportWorkerPool, RepoWorkerPoolConfig, - }; - use crate::parallel::run_coordinator::GlobalRunCoordinator; - use crate::parallel::transport_prefetch::TransportPrefetchSnapshot; - use crate::parallel::types::{ - RepoRuntimeState, RepoTransportMode, RepoTransportResultEnvelope, RepoTransportResultKind, - RepoTransportTask, TalInputSpec, - }; - use crate::policy::SyncPreference; - use crate::report::Warning; - use crate::validation::tree::{CaCertificateRef, CaInstanceHandle, DiscoveredChildCaInstance}; - - fn sample_ca(manifest: &str) -> CaInstanceHandle { - CaInstanceHandle { - depth: 0, - tal_id: "arin".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(vec![1, 2, 3]), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/".to_string(), - manifest_rsync_uri: manifest.to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), - rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), - } - } - - struct SuccessTransportExecutor; - - impl RepoTransportExecutor for SuccessTransportExecutor { - fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { - RepoTransportResultEnvelope { - dedup_key: task.dedup_key, - rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(), - repo_identity: task.repo_identity, - mode: task.mode, - tal_id: task.tal_id, - rir_id: task.rir_id, - timing_ms: 7, - result: RepoTransportResultKind::Success { - source: match task.mode { - RepoTransportMode::Rrdp => "rrdp".to_string(), - RepoTransportMode::Rsync => "rsync".to_string(), - }, - warnings: vec![Warning::new("transport ok")], - }, - } - } - } - - struct CountingSuccessTransportExecutor { - count: Arc, - } - - impl RepoTransportExecutor for CountingSuccessTransportExecutor { - fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { - self.count.fetch_add(1, Ordering::SeqCst); - RepoTransportResultEnvelope { - dedup_key: task.dedup_key, - rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(), - repo_identity: task.repo_identity, - mode: task.mode, - tal_id: task.tal_id, - rir_id: task.rir_id, - timing_ms: 7, - result: RepoTransportResultKind::Success { - source: match task.mode { - RepoTransportMode::Rrdp => "rrdp".to_string(), - RepoTransportMode::Rsync => "rsync".to_string(), - }, - warnings: vec![Warning::new("transport ok")], - }, - } - } - } - - struct FailRrdpThenSucceedRsyncExecutor { - rrdp_count: Arc, - rsync_count: Arc, - } - - impl RepoTransportExecutor for FailRrdpThenSucceedRsyncExecutor { - fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { - match task.mode { - RepoTransportMode::Rrdp => { - self.rrdp_count.fetch_add(1, Ordering::SeqCst); - RepoTransportResultEnvelope { - dedup_key: task.dedup_key, - rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(), - repo_identity: task.repo_identity, - mode: RepoTransportMode::Rrdp, - tal_id: task.tal_id, - rir_id: task.rir_id, - timing_ms: 10, - result: RepoTransportResultKind::Failed { - detail: "rrdp failed".to_string(), - warnings: vec![Warning::new("rrdp failed")], - error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, - }, - } - } - RepoTransportMode::Rsync => { - self.rsync_count.fetch_add(1, Ordering::SeqCst); - RepoTransportResultEnvelope { - dedup_key: task.dedup_key, - rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(), - repo_identity: task.repo_identity, - mode: RepoTransportMode::Rsync, - tal_id: task.tal_id, - rir_id: task.rir_id, - timing_ms: 12, - result: RepoTransportResultKind::Success { - source: "rsync".to_string(), - warnings: vec![Warning::new("rsync ok")], - }, - } - } - } - } - } - - struct FailRrdpThenFailRsyncExecutor { - rrdp_count: Arc, - rsync_count: Arc, - } - - impl RepoTransportExecutor for FailRrdpThenFailRsyncExecutor { - fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { - match task.mode { - RepoTransportMode::Rrdp => { - self.rrdp_count.fetch_add(1, Ordering::SeqCst); - RepoTransportResultEnvelope { - dedup_key: task.dedup_key, - rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(), - repo_identity: task.repo_identity, - mode: RepoTransportMode::Rrdp, - tal_id: task.tal_id, - rir_id: task.rir_id, - timing_ms: 10, - result: RepoTransportResultKind::Failed { - detail: "rrdp failed".to_string(), - warnings: vec![Warning::new("rrdp failed")], - error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, - }, - } - } - RepoTransportMode::Rsync => { - self.rsync_count.fetch_add(1, Ordering::SeqCst); - RepoTransportResultEnvelope { - dedup_key: task.dedup_key, - rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(), - repo_identity: task.repo_identity, - mode: RepoTransportMode::Rsync, - tal_id: task.tal_id, - rir_id: task.rir_id, - timing_ms: 12, - result: RepoTransportResultKind::Failed { - detail: "rsync failed".to_string(), - warnings: vec![Warning::new("rsync failed")], - error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, - }, - } - } - } - } - } - - #[test] - fn phase1_runtime_waits_for_rrdp_transport_and_returns_rrdp_outcome() { - let coordinator = GlobalRunCoordinator::new( - ParallelPhase1Config::default(), - vec![TalInputSpec::from_url("https://example.test/arin.tal")], - ); - let pool = RepoTransportWorkerPool::new( - RepoWorkerPoolConfig { max_workers: 1 }, - SuccessTransportExecutor, - ) - .expect("pool"); - let runtime = Phase1RepoSyncRuntime::new( - coordinator, - pool, - Arc::new(|base: &str| base.to_string()), - SyncPreference::RrdpThenRsync, - ); - - let outcome = runtime - .sync_publication_point_repo(&sample_ca("rsync://example.test/repo/root.mft")) - .expect("sync repo"); - assert!(outcome.repo_sync_ok); - assert_eq!(outcome.repo_sync_source.as_deref(), Some("rrdp")); - assert_eq!(outcome.repo_sync_phase.as_deref(), Some("rrdp_ok")); - } - - #[test] - fn phase1_runtime_request_repo_returns_pending_then_repo_ready_event() { - let coordinator = GlobalRunCoordinator::new( - ParallelPhase1Config::default(), - vec![TalInputSpec::from_url("https://example.test/arin.tal")], - ); - let pool = RepoTransportWorkerPool::new( - RepoWorkerPoolConfig { max_workers: 1 }, - SuccessTransportExecutor, - ) - .expect("pool"); - let runtime = Phase1RepoSyncRuntime::new( - coordinator, - pool, - Arc::new(|base: &str| base.to_string()), - SyncPreference::RrdpThenRsync, - ); - let ca = sample_ca("rsync://example.test/repo/root.mft"); - - let status = runtime - .request_publication_point_repo(&ca, 0) - .expect("request repo"); - let identity = match status { - super::RepoSyncRequestStatus::Pending { identity, state } => { - assert_eq!(state, RepoRuntimeState::WaitingRrdp); - identity - } - other => panic!("expected pending, got {other:?}"), - }; - - let event = runtime - .recv_repo_result_timeout(Duration::from_secs(1)) - .expect("repo event") - .expect("event"); - assert_eq!(event.transport_identity, identity); - assert_eq!(event.completions.len(), 1); - assert_eq!(event.completions[0].identity, identity); - assert_eq!(event.completions[0].state, RepoRuntimeState::RrdpOk); - assert!(event.completions[0].outcome.repo_sync_ok); - assert_eq!( - event.completions[0].outcome.repo_sync_source.as_deref(), - Some("rrdp") - ); - } - - #[test] - fn phase1_runtime_request_repo_reuses_ready_event_result() { - let coordinator = GlobalRunCoordinator::new( - ParallelPhase1Config::default(), - vec![TalInputSpec::from_url("https://example.test/arin.tal")], - ); - let pool = RepoTransportWorkerPool::new( - RepoWorkerPoolConfig { max_workers: 1 }, - SuccessTransportExecutor, - ) - .expect("pool"); - let runtime = Phase1RepoSyncRuntime::new( - coordinator, - pool, - Arc::new(|base: &str| base.to_string()), - SyncPreference::RrdpThenRsync, - ); - let ca = sample_ca("rsync://example.test/repo/root.mft"); - let first = runtime - .request_publication_point_repo(&ca, 0) - .expect("request repo"); - assert!(matches!( - first, - super::RepoSyncRequestStatus::Pending { .. } - )); - let _ = runtime - .recv_repo_result_timeout(Duration::from_secs(1)) - .expect("repo event") - .expect("event"); - - let second = runtime - .request_publication_point_repo(&ca, 0) - .expect("request repo reused"); - match second { - super::RepoSyncRequestStatus::Ready { outcome, .. } => { - assert!(outcome.repo_sync_ok); - assert_eq!(outcome.repo_sync_phase.as_deref(), Some("rrdp_ok")); - } - other => panic!("expected ready reuse, got {other:?}"), - } - } - - #[test] - fn phase1_runtime_repo_event_reports_all_finalized_identities_for_shared_rrdp() { - let coordinator = GlobalRunCoordinator::new( - ParallelPhase1Config::default(), - vec![TalInputSpec::from_url("https://example.test/arin.tal")], - ); - let pool = RepoTransportWorkerPool::new( - RepoWorkerPoolConfig { max_workers: 1 }, - SuccessTransportExecutor, - ) - .expect("pool"); - let runtime = Phase1RepoSyncRuntime::new( - coordinator, - pool, - Arc::new(|base: &str| base.to_string()), - SyncPreference::RrdpThenRsync, - ); - let ca1 = sample_ca("rsync://example.test/repo/root.mft"); - let mut ca2 = sample_ca("rsync://example.test/other/root.mft"); - ca2.rsync_base_uri = "rsync://example.test/other/".to_string(); - ca2.publication_point_rsync_uri = "rsync://example.test/other/".to_string(); - - let id1 = match runtime - .request_publication_point_repo(&ca1, 0) - .expect("request first") - { - super::RepoSyncRequestStatus::Pending { identity, .. } => identity, - other => panic!("expected first pending, got {other:?}"), - }; - let id2 = match runtime - .request_publication_point_repo(&ca2, 0) - .expect("request second") - { - super::RepoSyncRequestStatus::Pending { identity, .. } => identity, - other => panic!("expected second pending, got {other:?}"), - }; - assert_ne!(id1, id2); - - let event = runtime - .recv_repo_result_timeout(Duration::from_secs(1)) - .expect("repo event") - .expect("event"); - let mut identities = event - .completions - .iter() - .map(|completion| completion.identity.clone()) - .collect::>(); - identities.sort_by(|a, b| a.rsync_base_uri.cmp(&b.rsync_base_uri)); - let mut expected = vec![id1, id2]; - expected.sort_by(|a, b| a.rsync_base_uri.cmp(&b.rsync_base_uri)); - assert_eq!(identities, expected); - assert!( - event - .completions - .iter() - .all(|completion| completion.state == RepoRuntimeState::RrdpOk) - ); - } - - #[test] - fn phase1_runtime_drains_multiple_ready_transport_events() { - let count = Arc::new(AtomicUsize::new(0)); - let coordinator = GlobalRunCoordinator::new( - ParallelPhase1Config::default(), - vec![TalInputSpec::from_url("https://example.test/arin.tal")], - ); - let pool = RepoTransportWorkerPool::new( - RepoWorkerPoolConfig { max_workers: 2 }, - CountingSuccessTransportExecutor { - count: Arc::clone(&count), - }, - ) - .expect("pool"); - let runtime = Phase1RepoSyncRuntime::new( - coordinator, - pool, - Arc::new(|base: &str| base.to_string()), - SyncPreference::RrdpThenRsync, - ); - let ca1 = sample_ca("rsync://example.test/repo/root.mft"); - let mut ca2 = sample_ca("rsync://example.net/repo/root.mft"); - ca2.rsync_base_uri = "rsync://example.net/repo/".to_string(); - ca2.publication_point_rsync_uri = "rsync://example.net/repo/".to_string(); - ca2.rrdp_notification_uri = Some("https://example.net/notify.xml".to_string()); - - assert!(matches!( - runtime - .request_publication_point_repo(&ca1, 0) - .expect("request ca1"), - super::RepoSyncRequestStatus::Pending { .. } - )); - assert!(matches!( - runtime - .request_publication_point_repo(&ca2, 0) - .expect("request ca2"), - super::RepoSyncRequestStatus::Pending { .. } - )); - - let started = Instant::now(); - while count.load(Ordering::SeqCst) < 2 && started.elapsed() < Duration::from_secs(1) { - std::thread::sleep(Duration::from_millis(5)); - } - assert_eq!(count.load(Ordering::SeqCst), 2); - - let events = runtime - .drain_repo_results_timeout(Duration::from_millis(0), 8) - .expect("drain events"); - assert_eq!(events.len(), 2); - assert_eq!( - events - .iter() - .map(|event| event.completions.len()) - .sum::(), - 2 - ); - } - - #[test] - fn phase1_runtime_reset_run_state_clears_completed_transport_reuse() { - let count = Arc::new(AtomicUsize::new(0)); - let coordinator = GlobalRunCoordinator::new( - ParallelPhase1Config::default(), - vec![TalInputSpec::from_url("https://example.test/arin.tal")], - ); - let pool = RepoTransportWorkerPool::new( - RepoWorkerPoolConfig { max_workers: 1 }, - CountingSuccessTransportExecutor { - count: Arc::clone(&count), - }, - ) - .expect("pool"); - let runtime = Phase1RepoSyncRuntime::new( - coordinator, - pool, - Arc::new(|base: &str| base.to_string()), - SyncPreference::RrdpThenRsync, - ); - let ca = sample_ca("rsync://example.test/repo/root.mft"); - - assert!(matches!( - runtime - .request_publication_point_repo(&ca, 0) - .expect("first request"), - super::RepoSyncRequestStatus::Pending { .. } - )); - let _ = runtime - .recv_repo_result_timeout(Duration::from_secs(1)) - .expect("first event") - .expect("event"); - assert_eq!(count.load(Ordering::SeqCst), 1); - assert!(matches!( - runtime - .request_publication_point_repo(&ca, 0) - .expect("ready reuse before reset"), - super::RepoSyncRequestStatus::Ready { .. } - )); - - runtime.reset_run_state().expect("reset"); - - assert!(matches!( - runtime - .request_publication_point_repo(&ca, 0) - .expect("second request after reset"), - super::RepoSyncRequestStatus::Pending { .. } - )); - let _ = runtime - .recv_repo_result_timeout(Duration::from_secs(1)) - .expect("second event") - .expect("event"); - assert_eq!(count.load(Ordering::SeqCst), 2); - } - - #[test] - fn phase1_runtime_records_prefetch_snapshot_only_when_enabled() { - let coordinator = GlobalRunCoordinator::new( - ParallelPhase1Config::default(), - vec![TalInputSpec::from_url("https://example.test/arin.tal")], - ); - let pool = RepoTransportWorkerPool::new( - RepoWorkerPoolConfig { max_workers: 1 }, - SuccessTransportExecutor, - ) - .expect("pool"); - let runtime = Phase1RepoSyncRuntime::new_with_failure_scope_and_prefetch_recording( - coordinator, - pool, - Arc::new(|base: &str| base.to_string()), - Arc::new(|_base: &str| Some("rsync://example.test/".to_string())), - SyncPreference::RrdpThenRsync, - true, - ); - let ca = sample_ca("rsync://example.test/repo/root.mft"); - - let _ = runtime - .request_publication_point_repo(&ca, 0) - .expect("request repo"); - let snapshot = runtime.transport_prefetch_snapshot(); - assert_eq!(snapshot.requests.len(), 1); - assert_eq!( - snapshot.requests[0].repo_identity.rsync_base_uri, - "rsync://example.test/repo/" - ); - assert_eq!( - snapshot.requests[0].rsync_failure_scope_uri.as_deref(), - Some("rsync://example.test/") - ); - - let coordinator = GlobalRunCoordinator::new( - ParallelPhase1Config::default(), - vec![TalInputSpec::from_url("https://example.test/arin.tal")], - ); - let pool = RepoTransportWorkerPool::new( - RepoWorkerPoolConfig { max_workers: 1 }, - SuccessTransportExecutor, - ) - .expect("pool"); - let runtime = Phase1RepoSyncRuntime::new( - coordinator, - pool, - Arc::new(|base: &str| base.to_string()), - SyncPreference::RrdpThenRsync, - ); - let _ = runtime - .request_publication_point_repo(&ca, 0) - .expect("request repo without recording"); - assert!(runtime.transport_prefetch_snapshot().requests.is_empty()); - } - - #[test] - fn phase1_runtime_prefetch_dispatches_and_later_request_waits_on_same_task() { - let count = Arc::new(AtomicUsize::new(0)); - let coordinator = GlobalRunCoordinator::new( - ParallelPhase1Config::default(), - vec![TalInputSpec::from_url("https://example.test/arin.tal")], - ); - let pool = RepoTransportWorkerPool::new( - RepoWorkerPoolConfig { max_workers: 1 }, - CountingSuccessTransportExecutor { - count: Arc::clone(&count), - }, - ) - .expect("pool"); - let runtime = Phase1RepoSyncRuntime::new_with_failure_scope_and_prefetch_recording( - coordinator, - pool, - Arc::new(|base: &str| base.to_string()), - Arc::new(|_base: &str| Some("rsync://example.test/".to_string())), - SyncPreference::RrdpThenRsync, - true, - ); - let ca = sample_ca("rsync://example.test/repo/root.mft"); - let mut recorder = - crate::parallel::transport_prefetch::TransportPrefetchRecorder::default(); - recorder.record_registered_request( - &super::Phase1RepoSyncRuntime::::build_identity(&ca), - &super::Phase1RepoSyncRuntime::::build_requester(&ca), - 0, - "rsync://example.test/repo/".to_string(), - Some("rsync://example.test/".to_string()), - SyncPreference::RrdpThenRsync, - ); - let snapshot = recorder.snapshot(SyncPreference::RrdpThenRsync); - - let stats = runtime - .prefetch_transport_requests(&snapshot, time::OffsetDateTime::UNIX_EPOCH) - .expect("prefetch transport requests"); - assert_eq!(stats.loaded_requests, 1); - assert_eq!(stats.enqueued_tasks, 1); - - let status = runtime - .request_publication_point_repo(&ca, 0) - .expect("request after prefetch"); - assert!(matches!( - status, - super::RepoSyncRequestStatus::Pending { - state: RepoRuntimeState::WaitingRrdp, - .. - } - )); - let _ = runtime - .recv_repo_result_timeout(Duration::from_secs(1)) - .expect("repo event") - .expect("event"); - assert_eq!(count.load(Ordering::SeqCst), 1); - } - - #[test] - fn phase1_runtime_does_not_persist_prefetch_only_requests() { - let coordinator = GlobalRunCoordinator::new( - ParallelPhase1Config::default(), - vec![TalInputSpec::from_url("https://example.test/arin.tal")], - ); - let pool = RepoTransportWorkerPool::new( - RepoWorkerPoolConfig { max_workers: 1 }, - SuccessTransportExecutor, - ) - .expect("pool"); - let runtime = Phase1RepoSyncRuntime::new_with_failure_scope_and_prefetch_recording( - coordinator, - pool, - Arc::new(|base: &str| base.to_string()), - Arc::new(|_base: &str| Some("rsync://example.test/".to_string())), - SyncPreference::RrdpThenRsync, - true, - ); - let snapshot = TransportPrefetchSnapshot::new( - SyncPreference::RrdpThenRsync, - vec![crate::parallel::transport_prefetch::TransportPrefetchRequest::from_registered_request( - &crate::parallel::types::RepoIdentity::new( - Some("https://example.test/notify.xml".to_string()), - "rsync://example.test/repo/", - ), - &crate::parallel::types::RepoRequester::with_tal_rir( - "arin", - "arin", - "rsync://example.test/repo/root.mft", - "rsync://example.test/repo/", - "arin:root", - ), - 0, - "rsync://example.test/repo/".to_string(), - Some("rsync://example.test/".to_string()), - SyncPreference::RrdpThenRsync, - )], - ); - - let stats = runtime - .prefetch_transport_requests(&snapshot, time::OffsetDateTime::UNIX_EPOCH) - .expect("prefetch transport requests"); - assert_eq!(stats.enqueued_tasks, 1); - assert!( - runtime.transport_prefetch_snapshot().requests.is_empty(), - "prefetch-only requests should not be carried forward forever" - ); - } - - #[test] - fn phase1_runtime_prefetch_skips_requests_when_scope_resolver_differs() { - let coordinator = GlobalRunCoordinator::new( - ParallelPhase1Config::default(), - vec![TalInputSpec::from_url("https://example.test/arin.tal")], - ); - let pool = RepoTransportWorkerPool::new( - RepoWorkerPoolConfig { max_workers: 1 }, - SuccessTransportExecutor, - ) - .expect("pool"); - let runtime = Phase1RepoSyncRuntime::new_with_failure_scope_and_prefetch_recording( - coordinator, - pool, - Arc::new(|_base: &str| "rsync://example.test/different/".to_string()), - Arc::new(|_base: &str| Some("rsync://example.test/".to_string())), - SyncPreference::RrdpThenRsync, - true, - ); - let snapshot = TransportPrefetchSnapshot::new( - SyncPreference::RrdpThenRsync, - vec![crate::parallel::transport_prefetch::TransportPrefetchRequest::from_registered_request( - &crate::parallel::types::RepoIdentity::new( - Some("https://example.test/notify.xml".to_string()), - "rsync://example.test/repo/", - ), - &crate::parallel::types::RepoRequester::with_tal_rir( - "arin", - "arin", - "rsync://example.test/repo/root.mft", - "rsync://example.test/repo/", - "arin:root", - ), - 0, - "rsync://example.test/repo/".to_string(), - Some("rsync://example.test/".to_string()), - SyncPreference::RrdpThenRsync, - )], - ); - - let stats = runtime - .prefetch_transport_requests(&snapshot, time::OffsetDateTime::UNIX_EPOCH) - .expect("prefetch transport requests"); - assert_eq!(stats.loaded_requests, 1); - assert_eq!(stats.enqueued_tasks, 0); - assert_eq!(stats.skipped_incompatible, 1); - } - - #[test] - fn phase1_runtime_transitions_rrdp_failure_to_rsync_success() { - let rrdp_count = Arc::new(AtomicUsize::new(0)); - let rsync_count = Arc::new(AtomicUsize::new(0)); - let coordinator = GlobalRunCoordinator::new( - ParallelPhase1Config::default(), - vec![TalInputSpec::from_url("https://example.test/arin.tal")], - ); - let pool = RepoTransportWorkerPool::new( - RepoWorkerPoolConfig { max_workers: 1 }, - FailRrdpThenSucceedRsyncExecutor { - rrdp_count: Arc::clone(&rrdp_count), - rsync_count: Arc::clone(&rsync_count), - }, - ) - .expect("pool"); - let runtime = Phase1RepoSyncRuntime::new( - coordinator, - pool, - Arc::new(|_base: &str| "rsync://example.test/module/".to_string()), - SyncPreference::RrdpThenRsync, - ); - - let outcome = runtime - .sync_publication_point_repo(&sample_ca("rsync://example.test/repo/root.mft")) - .expect("sync repo"); - assert!(outcome.repo_sync_ok); - assert_eq!(outcome.repo_sync_source.as_deref(), Some("rsync")); - assert_eq!( - outcome.repo_sync_phase.as_deref(), - Some("rrdp_failed_rsync_ok") - ); - assert_eq!(rrdp_count.load(Ordering::SeqCst), 1); - assert_eq!(rsync_count.load(Ordering::SeqCst), 1); - } - - #[test] - fn phase1_runtime_terminal_failure_keeps_rsync_failure_duration() { - let rrdp_count = Arc::new(AtomicUsize::new(0)); - let rsync_count = Arc::new(AtomicUsize::new(0)); - let coordinator = GlobalRunCoordinator::new( - ParallelPhase1Config::default(), - vec![TalInputSpec::from_url("https://example.test/arin.tal")], - ); - let pool = RepoTransportWorkerPool::new( - RepoWorkerPoolConfig { max_workers: 1 }, - FailRrdpThenFailRsyncExecutor { - rrdp_count: Arc::clone(&rrdp_count), - rsync_count: Arc::clone(&rsync_count), - }, - ) - .expect("pool"); - let runtime = Phase1RepoSyncRuntime::new( - coordinator, - pool, - Arc::new(|_base: &str| "rsync://example.test/module/".to_string()), - SyncPreference::RrdpThenRsync, - ); - - let outcome = runtime - .sync_publication_point_repo(&sample_ca("rsync://example.test/repo/root.mft")) - .expect("sync repo"); - assert!(!outcome.repo_sync_ok); - assert_eq!( - outcome.repo_sync_phase.as_deref(), - Some("rrdp_failed_rsync_failed") - ); - assert_eq!(outcome.repo_sync_duration_ms, 12); - assert_eq!(rrdp_count.load(Ordering::SeqCst), 1); - assert_eq!(rsync_count.load(Ordering::SeqCst), 1); - } - - #[test] - fn phase1_runtime_prefetch_submits_transport_task_before_consumption() { - let rrdp_count = Arc::new(AtomicUsize::new(0)); - let rsync_count = Arc::new(AtomicUsize::new(0)); - let coordinator = GlobalRunCoordinator::new( - ParallelPhase1Config::default(), - vec![TalInputSpec::from_url("https://example.test/arin.tal")], - ); - let pool = RepoTransportWorkerPool::new( - RepoWorkerPoolConfig { max_workers: 1 }, - FailRrdpThenSucceedRsyncExecutor { - rrdp_count: Arc::clone(&rrdp_count), - rsync_count: Arc::clone(&rsync_count), - }, - ) - .expect("pool"); - let runtime = Arc::new(Phase1RepoSyncRuntime::new( - coordinator, - pool, - Arc::new(|_base: &str| "rsync://example.test/module/".to_string()), - SyncPreference::RrdpThenRsync, - )); - - let child = DiscoveredChildCaInstance { - handle: sample_ca("rsync://example.test/repo/child.mft"), - discovered_from: crate::audit::DiscoveredFrom { - parent_manifest_rsync_uri: "rsync://example.test/repo/root.mft".to_string(), - child_ca_certificate_rsync_uri: "rsync://example.test/repo/child.cer".to_string(), - child_ca_certificate_sha256_hex: "00".repeat(32), - }, - child_entry_projection: None, - }; - - runtime - .prefetch_discovered_children(std::slice::from_ref(&child)) - .expect("prefetch"); - - let started = Instant::now(); - while rrdp_count.load(Ordering::SeqCst) == 0 && started.elapsed() < Duration::from_secs(1) { - std::thread::sleep(Duration::from_millis(10)); - } - assert_eq!(rrdp_count.load(Ordering::SeqCst), 1); - - let outcome = runtime - .sync_publication_point_repo(&child.handle) - .expect("sync child repo"); - assert!(outcome.repo_sync_ok); - assert_eq!(rsync_count.load(Ordering::SeqCst), 1); - } -} +#[path = "repo_runtime/tests.rs"] +mod tests; diff --git a/crates/panda-rpki-validator/src/parallel/repo_runtime/outcome.rs b/crates/panda-rpki-validator/src/parallel/repo_runtime/outcome.rs new file mode 100644 index 0000000..a34f6d0 --- /dev/null +++ b/crates/panda-rpki-validator/src/parallel/repo_runtime/outcome.rs @@ -0,0 +1,58 @@ +// Conversion from transport envelopes to runtime outcomes. + +fn outcome_from_transport_result( + envelope: &RepoTransportResultEnvelope, + state: RepoRuntimeState, +) -> RepoSyncRuntimeOutcome { + match (&envelope.result, state) { + (RepoTransportResultKind::Success { source, warnings }, RepoRuntimeState::RrdpOk) => { + RepoSyncRuntimeOutcome { + repo_sync_ok: true, + repo_sync_err: None, + repo_sync_source: Some(source.clone()), + repo_sync_phase: Some("rrdp_ok".to_string()), + repo_sync_duration_ms: envelope.timing_ms, + warnings: warnings.clone(), + } + } + (RepoTransportResultKind::Success { source, warnings }, RepoRuntimeState::RsyncOk) => { + RepoSyncRuntimeOutcome { + repo_sync_ok: true, + repo_sync_err: None, + repo_sync_source: Some(source.clone()), + repo_sync_phase: Some(if envelope.repo_identity.notification_uri.is_some() { + "rrdp_failed_rsync_ok".to_string() + } else { + "rsync_only_ok".to_string() + }), + repo_sync_duration_ms: envelope.timing_ms, + warnings: warnings.clone(), + } + } + ( + RepoTransportResultKind::Failed { + detail, warnings, .. + }, + RepoRuntimeState::FailedTerminal, + ) => RepoSyncRuntimeOutcome { + repo_sync_ok: false, + repo_sync_err: Some(detail.clone()), + repo_sync_source: None, + repo_sync_phase: Some(if envelope.repo_identity.notification_uri.is_some() { + "rrdp_failed_rsync_failed".to_string() + } else { + "rsync_failed".to_string() + }), + repo_sync_duration_ms: envelope.timing_ms, + warnings: warnings.clone(), + }, + _ => RepoSyncRuntimeOutcome { + repo_sync_ok: false, + repo_sync_err: Some("repo runtime state unresolved".to_string()), + repo_sync_source: None, + repo_sync_phase: Some("repo_runtime_unresolved".to_string()), + repo_sync_duration_ms: envelope.timing_ms, + warnings: Vec::new(), + }, + } +} diff --git a/crates/panda-rpki-validator/src/parallel/repo_runtime/phase1_runtime.rs b/crates/panda-rpki-validator/src/parallel/repo_runtime/phase1_runtime.rs new file mode 100644 index 0000000..5e2c039 --- /dev/null +++ b/crates/panda-rpki-validator/src/parallel/repo_runtime/phase1_runtime.rs @@ -0,0 +1,372 @@ +// Phase-one runtime request registration and result pumping. + +pub struct Phase1RepoSyncRuntime { + coordinator: Mutex, + worker_pool: Mutex>, + transport_prefetch_recorder: Option>, + retry_short_rsync_scopes: Mutex>, + rsync_scope_resolver: Arc String + Send + Sync>, + rsync_failure_scope_resolver: Arc Option + Send + Sync>, + sync_preference: SyncPreference, +} + +impl Phase1RepoSyncRuntime { + pub fn new( + coordinator: GlobalRunCoordinator, + worker_pool: RepoTransportWorkerPool, + rsync_scope_resolver: Arc String + Send + Sync>, + sync_preference: SyncPreference, + ) -> Self { + Self::new_with_failure_scope( + coordinator, + worker_pool, + rsync_scope_resolver, + Arc::new(|_base: &str| None), + sync_preference, + ) + } + + pub fn new_with_failure_scope( + coordinator: GlobalRunCoordinator, + worker_pool: RepoTransportWorkerPool, + rsync_scope_resolver: Arc String + Send + Sync>, + rsync_failure_scope_resolver: Arc Option + Send + Sync>, + sync_preference: SyncPreference, + ) -> Self { + Self { + coordinator: Mutex::new(coordinator), + worker_pool: Mutex::new(worker_pool), + transport_prefetch_recorder: None, + retry_short_rsync_scopes: Mutex::new(HashSet::new()), + rsync_scope_resolver, + rsync_failure_scope_resolver, + sync_preference, + } + } + + pub fn new_with_failure_scope_and_prefetch_recording( + coordinator: GlobalRunCoordinator, + worker_pool: RepoTransportWorkerPool, + rsync_scope_resolver: Arc String + Send + Sync>, + rsync_failure_scope_resolver: Arc Option + Send + Sync>, + sync_preference: SyncPreference, + record_transport_prefetch_requests: bool, + ) -> Self { + Self { + coordinator: Mutex::new(coordinator), + worker_pool: Mutex::new(worker_pool), + transport_prefetch_recorder: record_transport_prefetch_requests + .then(|| Mutex::new(TransportPrefetchRecorder::default())), + retry_short_rsync_scopes: Mutex::new(HashSet::new()), + rsync_scope_resolver, + rsync_failure_scope_resolver, + sync_preference, + } + } + + fn build_requester(ca: &CaInstanceHandle) -> RepoRequester { + RepoRequester { + tal_id: ca.tal_id.clone(), + rir_id: ca.tal_id.clone(), + parent_node_id: None, + ca_instance_handle_id: format!("{}:{}", ca.tal_id, ca.manifest_rsync_uri), + publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(), + manifest_rsync_uri: ca.manifest_rsync_uri.clone(), + } + } + + fn build_identity(ca: &CaInstanceHandle) -> RepoIdentity { + RepoIdentity::new(ca.rrdp_notification_uri.clone(), ca.rsync_base_uri.clone()) + } + + fn request_transport_for_ca( + &self, + ca: &CaInstanceHandle, + priority: u8, + ) -> Result { + let identity = Self::build_identity(ca); + let requester = Self::build_requester(ca); + let rsync_scope_uri = (self.rsync_scope_resolver)(&identity.rsync_base_uri); + let rsync_failure_scope_uri = (self.rsync_failure_scope_resolver)(&identity.rsync_base_uri); + if let Some(recorder) = self.transport_prefetch_recorder.as_ref() { + let mut recorder = recorder + .lock() + .expect("transport prefetch recorder lock poisoned"); + recorder.record_registered_request( + &identity, + &requester, + priority, + rsync_scope_uri.clone(), + rsync_failure_scope_uri.clone(), + self.sync_preference, + ); + } + let action = { + let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); + coordinator.register_transport_request( + identity.clone(), + requester, + time::OffsetDateTime::now_utc(), + priority, + rsync_scope_uri, + rsync_failure_scope_uri, + self.sync_preference, + false, + ) + }; + + match action { + TransportRequestAction::Enqueue(task) => { + crate::progress_log::emit( + "phase1_repo_task_enqueued", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "repo_key_rsync_base_uri": task.repo_identity.rsync_base_uri, + "rsync_failure_scope_uri": task.rsync_failure_scope_uri, + "repo_key_notification_uri": task.repo_identity.notification_uri, + "priority": priority, + "transport_mode": match task.mode { + RepoTransportMode::Rrdp => "rrdp", + RepoTransportMode::Rsync => "rsync", + }, + }), + ); + self.drain_pending_transport_tasks()?; + Ok(RepoSyncRequestStatus::Pending { + identity, + state: self + .runtime_state_for_identity(&task.repo_identity) + .unwrap_or(RepoRuntimeState::WaitingRrdp), + }) + } + TransportRequestAction::Waiting { state } => { + crate::progress_log::emit( + "phase1_repo_task_waiting", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "repo_key_rsync_base_uri": identity.rsync_base_uri, + "rsync_failure_scope_uri": (self.rsync_failure_scope_resolver)(&identity.rsync_base_uri), + "repo_key_notification_uri": identity.notification_uri, + "priority": priority, + "runtime_state": format!("{state:?}"), + }), + ); + Ok(RepoSyncRequestStatus::Pending { identity, state }) + } + TransportRequestAction::ReusedSuccess(result) + | TransportRequestAction::ReusedTerminalFailure(result) => { + crate::progress_log::emit( + "phase1_repo_task_reused", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "repo_key_rsync_base_uri": identity.rsync_base_uri, + "rsync_failure_scope_uri": result.rsync_failure_scope_uri, + "repo_key_notification_uri": identity.notification_uri, + "priority": priority, + "transport_mode": match result.mode { + RepoTransportMode::Rrdp => "rrdp", + RepoTransportMode::Rsync => "rsync", + }, + }), + ); + Ok(RepoSyncRequestStatus::Ready { + outcome: outcome_from_transport_result( + &result, + self.runtime_state_for_identity(&identity) + .unwrap_or(RepoRuntimeState::Init), + ), + identity, + }) + } + } + } + + fn drain_pending_transport_tasks(&self) -> Result<(), String> { + loop { + let maybe_task = { + let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); + coordinator.pop_next_transport_task() + }; + let Some(task) = maybe_task else { + break; + }; + { + let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); + coordinator + .mark_transport_running(&task.dedup_key, time::OffsetDateTime::now_utc())?; + } + crate::progress_log::emit( + "phase1_repo_task_dispatched", + serde_json::json!({ + "repo_key_rsync_base_uri": task.repo_identity.rsync_base_uri, + "rsync_failure_scope_uri": task.rsync_failure_scope_uri, + "repo_key_notification_uri": task.repo_identity.notification_uri, + "requester_count": task.requesters.len(), + "priority": task.priority, + "transport_mode": match task.mode { + RepoTransportMode::Rrdp => "rrdp", + RepoTransportMode::Rsync => "rsync", + }, + }), + ); + let pool = self.worker_pool.lock().expect("worker pool lock poisoned"); + pool.submit(task)?; + } + Ok(()) + } + + fn pump_one_transport_result( + &self, + timeout: Duration, + ) -> Result, String> { + let envelope = { + let pool = self.worker_pool.lock().expect("worker pool lock poisoned"); + pool.recv_result_timeout(timeout)? + }; + let Some(envelope) = envelope else { + return Ok(None); + }; + let transport_identity = envelope.repo_identity.clone(); + let completed_envelope = envelope.clone(); + if let Some(recorder) = self.transport_prefetch_recorder.as_ref() { + recorder + .lock() + .expect("transport prefetch recorder lock poisoned") + .record_result(&envelope); + } + crate::progress_log::emit( + "phase1_repo_task_result", + serde_json::json!({ + "repo_key_rsync_base_uri": envelope.repo_identity.rsync_base_uri, + "rsync_failure_scope_uri": envelope.rsync_failure_scope_uri, + "repo_key_notification_uri": envelope.repo_identity.notification_uri, + "timing_ms": envelope.timing_ms, + "transport_mode": match envelope.mode { + RepoTransportMode::Rrdp => "rrdp", + RepoTransportMode::Rsync => "rsync", + }, + "result": match &envelope.result { + RepoTransportResultKind::Success { .. } => "success", + RepoTransportResultKind::Failed { .. } => "failed", + }, + }), + ); + let finished_at = time::OffsetDateTime::now_utc(); + let completion = { + let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); + coordinator.complete_transport_result(envelope, finished_at)? + }; + if !completion.follow_up_tasks.is_empty() { + let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); + for mut task in completion.follow_up_tasks { + if let crate::parallel::types::RepoDedupKey::RsyncScope { rsync_scope_uri } = + &task.dedup_key + { + if self + .retry_short_rsync_scopes + .lock() + .expect("retry short rsync scopes lock poisoned") + .contains(rsync_scope_uri) + { + task.retry_short_timeout = true; + } + } + crate::progress_log::emit( + "phase1_repo_task_enqueued", + serde_json::json!({ + "manifest_rsync_uri": serde_json::Value::Null, + "publication_point_rsync_uri": task.requesters.first().map(|r| r.publication_point_rsync_uri.clone()), + "repo_key_rsync_base_uri": task.repo_identity.rsync_base_uri, + "repo_key_notification_uri": task.repo_identity.notification_uri, + "priority": task.priority, + "transport_mode": "rsync", + }), + ); + coordinator.push_transport_task(task); + } + } + self.drain_pending_transport_tasks()?; + let completions = { + let coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); + coordinator + .finalized_runtime_records_for_transport_result(&completed_envelope) + .into_iter() + .filter_map(|record| { + let outcome = match record.state { + RepoRuntimeState::RrdpOk | RepoRuntimeState::RsyncOk => record + .last_success + .as_ref() + .map(|result| outcome_from_transport_result(result, record.state)), + RepoRuntimeState::FailedTerminal => record + .terminal_failure + .as_ref() + .map(|result| outcome_from_transport_result(result, record.state)), + _ => None, + }?; + Some(RepoSyncRuntimeCompletion { + identity: record.identity, + state: record.state, + outcome, + }) + }) + .collect::>() + }; + if completions.is_empty() { + return Ok(None); + } + Ok(Some(RepoSyncRuntimeEvent { + transport_identity, + completions, + })) + } + + fn pump_transport_results( + &self, + timeout: Duration, + max_events: usize, + ) -> Result, String> { + let max_events = max_events.max(1); + let mut events = Vec::new(); + for index in 0..max_events { + let poll_timeout = if index == 0 { + timeout + } else { + Duration::from_millis(0) + }; + let Some(event) = self.pump_one_transport_result(poll_timeout)? else { + break; + }; + events.push(event); + } + Ok(events) + } + + fn runtime_state_for_identity(&self, identity: &RepoIdentity) -> Option { + let coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); + coordinator + .runtime_record(identity) + .map(|record| record.state) + } + + fn resolved_outcome_for_identity( + &self, + identity: &RepoIdentity, + ) -> Option { + let coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); + let record = coordinator.runtime_record(identity)?; + match record.state { + RepoRuntimeState::RrdpOk | RepoRuntimeState::RsyncOk => record + .last_success + .as_ref() + .map(|result| outcome_from_transport_result(result, record.state)), + RepoRuntimeState::FailedTerminal => record + .terminal_failure + .as_ref() + .map(|result| outcome_from_transport_result(result, record.state)), + _ => None, + } + } +} diff --git a/crates/panda-rpki-validator/src/parallel/repo_runtime/runtime_trait_impl.rs b/crates/panda-rpki-validator/src/parallel/repo_runtime/runtime_trait_impl.rs new file mode 100644 index 0000000..50962ff --- /dev/null +++ b/crates/panda-rpki-validator/src/parallel/repo_runtime/runtime_trait_impl.rs @@ -0,0 +1,173 @@ +// RepoSyncRuntime implementation for the phase-one runtime. + +impl RepoSyncRuntime for Phase1RepoSyncRuntime { + fn sync_publication_point_repo( + &self, + ca: &CaInstanceHandle, + ) -> Result { + if let RepoSyncRequestStatus::Ready { outcome, .. } = + self.request_publication_point_repo(ca, 0)? + { + return Ok(outcome); + } + let identity = Self::build_identity(ca); + loop { + if let Some(done) = self.resolved_outcome_for_identity(&identity) { + return Ok(done); + } + let _ = self.recv_repo_result_timeout(Duration::from_millis(50))?; + } + } + + fn request_publication_point_repo( + &self, + ca: &CaInstanceHandle, + priority: u8, + ) -> Result { + self.request_transport_for_ca(ca, priority) + } + + fn recv_repo_result_timeout( + &self, + timeout: Duration, + ) -> Result, String> { + self.pump_one_transport_result(timeout) + } + + fn drain_repo_results_timeout( + &self, + timeout: Duration, + max_events: usize, + ) -> Result, String> { + self.pump_transport_results(timeout, max_events) + } + + fn reset_run_state(&self) -> Result<(), String> { + { + let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); + if coordinator.stats.repo_tasks_running != 0 { + return Err(format!( + "cannot reset repo runtime with {} repo task(s) still running", + coordinator.stats.repo_tasks_running + )); + } + coordinator.reset_run_state(); + } + loop { + let maybe_result = { + let pool = self.worker_pool.lock().expect("worker pool lock poisoned"); + pool.recv_result_timeout(Duration::from_millis(0))? + }; + if maybe_result.is_none() { + break; + } + } + Ok(()) + } + + fn prefetch_discovered_children( + &self, + children: &[DiscoveredChildCaInstance], + ) -> Result<(), String> { + for child in children { + let _ = self.request_publication_point_repo(&child.handle, 1)?; + } + Ok(()) + } + + fn prefetch_transport_requests( + &self, + snapshot: &TransportPrefetchSnapshot, + validation_time: time::OffsetDateTime, + ) -> Result { + let mut stats = TransportPrefetchDispatchStats { + loaded_requests: snapshot.requests.len() as u64, + ..TransportPrefetchDispatchStats::default() + }; + + for request in &snapshot.requests { + let identity = request.to_identity(); + let current_rsync_scope_uri = (self.rsync_scope_resolver)(&identity.rsync_base_uri); + let current_rsync_failure_scope_uri = + (self.rsync_failure_scope_resolver)(&identity.rsync_base_uri); + if current_rsync_scope_uri != request.rsync_scope_uri + || current_rsync_failure_scope_uri != request.rsync_failure_scope_uri + { + stats.skipped_incompatible += 1; + continue; + } + if request.retry_short_rsync_timeout() { + self.retry_short_rsync_scopes + .lock() + .expect("retry short rsync scopes lock poisoned") + .insert(current_rsync_scope_uri.clone()); + } + let action = { + let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); + coordinator.register_transport_request( + identity, + request.to_requester(), + validation_time, + request.priority, + current_rsync_scope_uri, + current_rsync_failure_scope_uri, + self.sync_preference, + request.retry_short_timeout(), + ) + }; + + match action { + TransportRequestAction::Enqueue(task) => { + stats.enqueued_tasks += 1; + crate::progress_log::emit( + "phase1_repo_prefetch_enqueued", + serde_json::json!({ + "repo_key_rsync_base_uri": task.repo_identity.rsync_base_uri, + "rsync_failure_scope_uri": task.rsync_failure_scope_uri, + "repo_key_notification_uri": task.repo_identity.notification_uri, + "priority": task.priority, + "transport_mode": match task.mode { + RepoTransportMode::Rrdp => "rrdp", + RepoTransportMode::Rsync => "rsync", + }, + }), + ); + } + TransportRequestAction::Waiting { .. } => { + stats.waiting_requests += 1; + } + TransportRequestAction::ReusedSuccess(_) + | TransportRequestAction::ReusedTerminalFailure(_) => { + stats.reused_results += 1; + } + } + } + + self.drain_pending_transport_tasks()?; + Ok(stats) + } + + fn transport_prefetch_snapshot(&self) -> TransportPrefetchSnapshot { + self.transport_prefetch_recorder + .as_ref() + .map(|recorder| { + recorder + .lock() + .expect("transport prefetch recorder lock poisoned") + .snapshot(self.sync_preference) + }) + .unwrap_or_else(|| TransportPrefetchSnapshot::new(self.sync_preference, Vec::new())) + } + + fn dead_repo_blacklist_state( + &self, + ) -> Option<( + crate::parallel::dead_repo_blacklist::DeadRepoBlacklistConfig, + crate::parallel::dead_repo_blacklist::DeadRepoBlacklist, + )> { + let coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); + let config = coordinator.config.dead_repo_blacklist.clone()?; + let blacklist = coordinator.dead_repo_blacklist().cloned()?; + Some((config, blacklist)) + } +} diff --git a/crates/panda-rpki-validator/src/parallel/repo_runtime/tests.rs b/crates/panda-rpki-validator/src/parallel/repo_runtime/tests.rs new file mode 100644 index 0000000..1e54855 --- /dev/null +++ b/crates/panda-rpki-validator/src/parallel/repo_runtime/tests.rs @@ -0,0 +1,798 @@ +// Repository runtime behavior tests. + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +use crate::parallel::config::ParallelPhase1Config; +use crate::parallel::repo_runtime::{Phase1RepoSyncRuntime, RepoSyncRuntime}; +use crate::parallel::repo_worker::{ + RepoTransportExecutor, RepoTransportWorkerPool, RepoWorkerPoolConfig, +}; +use crate::parallel::run_coordinator::GlobalRunCoordinator; +use crate::parallel::transport_prefetch::TransportPrefetchSnapshot; +use crate::parallel::types::{ + RepoRuntimeState, RepoTransportMode, RepoTransportResultEnvelope, RepoTransportResultKind, + RepoTransportTask, TalInputSpec, +}; +use crate::policy::SyncPreference; +use crate::report::Warning; +use crate::validation::tree::{CaCertificateRef, CaInstanceHandle, DiscoveredChildCaInstance}; + +fn sample_ca(manifest: &str) -> CaInstanceHandle { + CaInstanceHandle { + depth: 0, + tal_id: "arin".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(vec![1, 2, 3]), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/".to_string(), + manifest_rsync_uri: manifest.to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), + rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), + } +} + +struct SuccessTransportExecutor; + +impl RepoTransportExecutor for SuccessTransportExecutor { + fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { + RepoTransportResultEnvelope { + dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(), + repo_identity: task.repo_identity, + mode: task.mode, + tal_id: task.tal_id, + rir_id: task.rir_id, + timing_ms: 7, + result: RepoTransportResultKind::Success { + source: match task.mode { + RepoTransportMode::Rrdp => "rrdp".to_string(), + RepoTransportMode::Rsync => "rsync".to_string(), + }, + warnings: vec![Warning::new("transport ok")], + }, + } + } +} + +struct CountingSuccessTransportExecutor { + count: Arc, +} + +impl RepoTransportExecutor for CountingSuccessTransportExecutor { + fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { + self.count.fetch_add(1, Ordering::SeqCst); + RepoTransportResultEnvelope { + dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(), + repo_identity: task.repo_identity, + mode: task.mode, + tal_id: task.tal_id, + rir_id: task.rir_id, + timing_ms: 7, + result: RepoTransportResultKind::Success { + source: match task.mode { + RepoTransportMode::Rrdp => "rrdp".to_string(), + RepoTransportMode::Rsync => "rsync".to_string(), + }, + warnings: vec![Warning::new("transport ok")], + }, + } + } +} + +struct FailRrdpThenSucceedRsyncExecutor { + rrdp_count: Arc, + rsync_count: Arc, +} + +impl RepoTransportExecutor for FailRrdpThenSucceedRsyncExecutor { + fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { + match task.mode { + RepoTransportMode::Rrdp => { + self.rrdp_count.fetch_add(1, Ordering::SeqCst); + RepoTransportResultEnvelope { + dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(), + repo_identity: task.repo_identity, + mode: RepoTransportMode::Rrdp, + tal_id: task.tal_id, + rir_id: task.rir_id, + timing_ms: 10, + result: RepoTransportResultKind::Failed { + detail: "rrdp failed".to_string(), + warnings: vec![Warning::new("rrdp failed")], + error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, + }, + } + } + RepoTransportMode::Rsync => { + self.rsync_count.fetch_add(1, Ordering::SeqCst); + RepoTransportResultEnvelope { + dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(), + repo_identity: task.repo_identity, + mode: RepoTransportMode::Rsync, + tal_id: task.tal_id, + rir_id: task.rir_id, + timing_ms: 12, + result: RepoTransportResultKind::Success { + source: "rsync".to_string(), + warnings: vec![Warning::new("rsync ok")], + }, + } + } + } + } +} + +struct FailRrdpThenFailRsyncExecutor { + rrdp_count: Arc, + rsync_count: Arc, +} + +impl RepoTransportExecutor for FailRrdpThenFailRsyncExecutor { + fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { + match task.mode { + RepoTransportMode::Rrdp => { + self.rrdp_count.fetch_add(1, Ordering::SeqCst); + RepoTransportResultEnvelope { + dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(), + repo_identity: task.repo_identity, + mode: RepoTransportMode::Rrdp, + tal_id: task.tal_id, + rir_id: task.rir_id, + timing_ms: 10, + result: RepoTransportResultKind::Failed { + detail: "rrdp failed".to_string(), + warnings: vec![Warning::new("rrdp failed")], + error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, + }, + } + } + RepoTransportMode::Rsync => { + self.rsync_count.fetch_add(1, Ordering::SeqCst); + RepoTransportResultEnvelope { + dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(), + repo_identity: task.repo_identity, + mode: RepoTransportMode::Rsync, + tal_id: task.tal_id, + rir_id: task.rir_id, + timing_ms: 12, + result: RepoTransportResultKind::Failed { + detail: "rsync failed".to_string(), + warnings: vec![Warning::new("rsync failed")], + error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, + }, + } + } + } + } +} + +#[test] +fn phase1_runtime_waits_for_rrdp_transport_and_returns_rrdp_outcome() { + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + SuccessTransportExecutor, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new( + coordinator, + pool, + Arc::new(|base: &str| base.to_string()), + SyncPreference::RrdpThenRsync, + ); + + let outcome = runtime + .sync_publication_point_repo(&sample_ca("rsync://example.test/repo/root.mft")) + .expect("sync repo"); + assert!(outcome.repo_sync_ok); + assert_eq!(outcome.repo_sync_source.as_deref(), Some("rrdp")); + assert_eq!(outcome.repo_sync_phase.as_deref(), Some("rrdp_ok")); +} + +#[test] +fn phase1_runtime_request_repo_returns_pending_then_repo_ready_event() { + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + SuccessTransportExecutor, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new( + coordinator, + pool, + Arc::new(|base: &str| base.to_string()), + SyncPreference::RrdpThenRsync, + ); + let ca = sample_ca("rsync://example.test/repo/root.mft"); + + let status = runtime + .request_publication_point_repo(&ca, 0) + .expect("request repo"); + let identity = match status { + super::RepoSyncRequestStatus::Pending { identity, state } => { + assert_eq!(state, RepoRuntimeState::WaitingRrdp); + identity + } + other => panic!("expected pending, got {other:?}"), + }; + + let event = runtime + .recv_repo_result_timeout(Duration::from_secs(1)) + .expect("repo event") + .expect("event"); + assert_eq!(event.transport_identity, identity); + assert_eq!(event.completions.len(), 1); + assert_eq!(event.completions[0].identity, identity); + assert_eq!(event.completions[0].state, RepoRuntimeState::RrdpOk); + assert!(event.completions[0].outcome.repo_sync_ok); + assert_eq!( + event.completions[0].outcome.repo_sync_source.as_deref(), + Some("rrdp") + ); +} + +#[test] +fn phase1_runtime_request_repo_reuses_ready_event_result() { + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + SuccessTransportExecutor, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new( + coordinator, + pool, + Arc::new(|base: &str| base.to_string()), + SyncPreference::RrdpThenRsync, + ); + let ca = sample_ca("rsync://example.test/repo/root.mft"); + let first = runtime + .request_publication_point_repo(&ca, 0) + .expect("request repo"); + assert!(matches!( + first, + super::RepoSyncRequestStatus::Pending { .. } + )); + let _ = runtime + .recv_repo_result_timeout(Duration::from_secs(1)) + .expect("repo event") + .expect("event"); + + let second = runtime + .request_publication_point_repo(&ca, 0) + .expect("request repo reused"); + match second { + super::RepoSyncRequestStatus::Ready { outcome, .. } => { + assert!(outcome.repo_sync_ok); + assert_eq!(outcome.repo_sync_phase.as_deref(), Some("rrdp_ok")); + } + other => panic!("expected ready reuse, got {other:?}"), + } +} + +#[test] +fn phase1_runtime_repo_event_reports_all_finalized_identities_for_shared_rrdp() { + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + SuccessTransportExecutor, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new( + coordinator, + pool, + Arc::new(|base: &str| base.to_string()), + SyncPreference::RrdpThenRsync, + ); + let ca1 = sample_ca("rsync://example.test/repo/root.mft"); + let mut ca2 = sample_ca("rsync://example.test/other/root.mft"); + ca2.rsync_base_uri = "rsync://example.test/other/".to_string(); + ca2.publication_point_rsync_uri = "rsync://example.test/other/".to_string(); + + let id1 = match runtime + .request_publication_point_repo(&ca1, 0) + .expect("request first") + { + super::RepoSyncRequestStatus::Pending { identity, .. } => identity, + other => panic!("expected first pending, got {other:?}"), + }; + let id2 = match runtime + .request_publication_point_repo(&ca2, 0) + .expect("request second") + { + super::RepoSyncRequestStatus::Pending { identity, .. } => identity, + other => panic!("expected second pending, got {other:?}"), + }; + assert_ne!(id1, id2); + + let event = runtime + .recv_repo_result_timeout(Duration::from_secs(1)) + .expect("repo event") + .expect("event"); + let mut identities = event + .completions + .iter() + .map(|completion| completion.identity.clone()) + .collect::>(); + identities.sort_by(|a, b| a.rsync_base_uri.cmp(&b.rsync_base_uri)); + let mut expected = vec![id1, id2]; + expected.sort_by(|a, b| a.rsync_base_uri.cmp(&b.rsync_base_uri)); + assert_eq!(identities, expected); + assert!( + event + .completions + .iter() + .all(|completion| completion.state == RepoRuntimeState::RrdpOk) + ); +} + +#[test] +fn phase1_runtime_drains_multiple_ready_transport_events() { + let count = Arc::new(AtomicUsize::new(0)); + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 2 }, + CountingSuccessTransportExecutor { + count: Arc::clone(&count), + }, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new( + coordinator, + pool, + Arc::new(|base: &str| base.to_string()), + SyncPreference::RrdpThenRsync, + ); + let ca1 = sample_ca("rsync://example.test/repo/root.mft"); + let mut ca2 = sample_ca("rsync://example.net/repo/root.mft"); + ca2.rsync_base_uri = "rsync://example.net/repo/".to_string(); + ca2.publication_point_rsync_uri = "rsync://example.net/repo/".to_string(); + ca2.rrdp_notification_uri = Some("https://example.net/notify.xml".to_string()); + + assert!(matches!( + runtime + .request_publication_point_repo(&ca1, 0) + .expect("request ca1"), + super::RepoSyncRequestStatus::Pending { .. } + )); + assert!(matches!( + runtime + .request_publication_point_repo(&ca2, 0) + .expect("request ca2"), + super::RepoSyncRequestStatus::Pending { .. } + )); + + let started = Instant::now(); + while count.load(Ordering::SeqCst) < 2 && started.elapsed() < Duration::from_secs(1) { + std::thread::sleep(Duration::from_millis(5)); + } + assert_eq!(count.load(Ordering::SeqCst), 2); + + let events = runtime + .drain_repo_results_timeout(Duration::from_millis(0), 8) + .expect("drain events"); + assert_eq!(events.len(), 2); + assert_eq!( + events + .iter() + .map(|event| event.completions.len()) + .sum::(), + 2 + ); +} + +#[test] +fn phase1_runtime_reset_run_state_clears_completed_transport_reuse() { + let count = Arc::new(AtomicUsize::new(0)); + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + CountingSuccessTransportExecutor { + count: Arc::clone(&count), + }, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new( + coordinator, + pool, + Arc::new(|base: &str| base.to_string()), + SyncPreference::RrdpThenRsync, + ); + let ca = sample_ca("rsync://example.test/repo/root.mft"); + + assert!(matches!( + runtime + .request_publication_point_repo(&ca, 0) + .expect("first request"), + super::RepoSyncRequestStatus::Pending { .. } + )); + let _ = runtime + .recv_repo_result_timeout(Duration::from_secs(1)) + .expect("first event") + .expect("event"); + assert_eq!(count.load(Ordering::SeqCst), 1); + assert!(matches!( + runtime + .request_publication_point_repo(&ca, 0) + .expect("ready reuse before reset"), + super::RepoSyncRequestStatus::Ready { .. } + )); + + runtime.reset_run_state().expect("reset"); + + assert!(matches!( + runtime + .request_publication_point_repo(&ca, 0) + .expect("second request after reset"), + super::RepoSyncRequestStatus::Pending { .. } + )); + let _ = runtime + .recv_repo_result_timeout(Duration::from_secs(1)) + .expect("second event") + .expect("event"); + assert_eq!(count.load(Ordering::SeqCst), 2); +} + +#[test] +fn phase1_runtime_records_prefetch_snapshot_only_when_enabled() { + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + SuccessTransportExecutor, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new_with_failure_scope_and_prefetch_recording( + coordinator, + pool, + Arc::new(|base: &str| base.to_string()), + Arc::new(|_base: &str| Some("rsync://example.test/".to_string())), + SyncPreference::RrdpThenRsync, + true, + ); + let ca = sample_ca("rsync://example.test/repo/root.mft"); + + let _ = runtime + .request_publication_point_repo(&ca, 0) + .expect("request repo"); + let snapshot = runtime.transport_prefetch_snapshot(); + assert_eq!(snapshot.requests.len(), 1); + assert_eq!( + snapshot.requests[0].repo_identity.rsync_base_uri, + "rsync://example.test/repo/" + ); + assert_eq!( + snapshot.requests[0].rsync_failure_scope_uri.as_deref(), + Some("rsync://example.test/") + ); + + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + SuccessTransportExecutor, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new( + coordinator, + pool, + Arc::new(|base: &str| base.to_string()), + SyncPreference::RrdpThenRsync, + ); + let _ = runtime + .request_publication_point_repo(&ca, 0) + .expect("request repo without recording"); + assert!(runtime.transport_prefetch_snapshot().requests.is_empty()); +} + +#[test] +fn phase1_runtime_prefetch_dispatches_and_later_request_waits_on_same_task() { + let count = Arc::new(AtomicUsize::new(0)); + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + CountingSuccessTransportExecutor { + count: Arc::clone(&count), + }, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new_with_failure_scope_and_prefetch_recording( + coordinator, + pool, + Arc::new(|base: &str| base.to_string()), + Arc::new(|_base: &str| Some("rsync://example.test/".to_string())), + SyncPreference::RrdpThenRsync, + true, + ); + let ca = sample_ca("rsync://example.test/repo/root.mft"); + let mut recorder = crate::parallel::transport_prefetch::TransportPrefetchRecorder::default(); + recorder.record_registered_request( + &super::Phase1RepoSyncRuntime::::build_identity(&ca), + &super::Phase1RepoSyncRuntime::::build_requester(&ca), + 0, + "rsync://example.test/repo/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RrdpThenRsync, + ); + let snapshot = recorder.snapshot(SyncPreference::RrdpThenRsync); + + let stats = runtime + .prefetch_transport_requests(&snapshot, time::OffsetDateTime::UNIX_EPOCH) + .expect("prefetch transport requests"); + assert_eq!(stats.loaded_requests, 1); + assert_eq!(stats.enqueued_tasks, 1); + + let status = runtime + .request_publication_point_repo(&ca, 0) + .expect("request after prefetch"); + assert!(matches!( + status, + super::RepoSyncRequestStatus::Pending { + state: RepoRuntimeState::WaitingRrdp, + .. + } + )); + let _ = runtime + .recv_repo_result_timeout(Duration::from_secs(1)) + .expect("repo event") + .expect("event"); + assert_eq!(count.load(Ordering::SeqCst), 1); +} + +#[test] +fn phase1_runtime_does_not_persist_prefetch_only_requests() { + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + SuccessTransportExecutor, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new_with_failure_scope_and_prefetch_recording( + coordinator, + pool, + Arc::new(|base: &str| base.to_string()), + Arc::new(|_base: &str| Some("rsync://example.test/".to_string())), + SyncPreference::RrdpThenRsync, + true, + ); + let snapshot = TransportPrefetchSnapshot::new( + SyncPreference::RrdpThenRsync, + vec![ + crate::parallel::transport_prefetch::TransportPrefetchRequest::from_registered_request( + &crate::parallel::types::RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/", + ), + &crate::parallel::types::RepoRequester::with_tal_rir( + "arin", + "arin", + "rsync://example.test/repo/root.mft", + "rsync://example.test/repo/", + "arin:root", + ), + 0, + "rsync://example.test/repo/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RrdpThenRsync, + ), + ], + ); + + let stats = runtime + .prefetch_transport_requests(&snapshot, time::OffsetDateTime::UNIX_EPOCH) + .expect("prefetch transport requests"); + assert_eq!(stats.enqueued_tasks, 1); + assert!( + runtime.transport_prefetch_snapshot().requests.is_empty(), + "prefetch-only requests should not be carried forward forever" + ); +} + +#[test] +fn phase1_runtime_prefetch_skips_requests_when_scope_resolver_differs() { + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + SuccessTransportExecutor, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new_with_failure_scope_and_prefetch_recording( + coordinator, + pool, + Arc::new(|_base: &str| "rsync://example.test/different/".to_string()), + Arc::new(|_base: &str| Some("rsync://example.test/".to_string())), + SyncPreference::RrdpThenRsync, + true, + ); + let snapshot = TransportPrefetchSnapshot::new( + SyncPreference::RrdpThenRsync, + vec![ + crate::parallel::transport_prefetch::TransportPrefetchRequest::from_registered_request( + &crate::parallel::types::RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/", + ), + &crate::parallel::types::RepoRequester::with_tal_rir( + "arin", + "arin", + "rsync://example.test/repo/root.mft", + "rsync://example.test/repo/", + "arin:root", + ), + 0, + "rsync://example.test/repo/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RrdpThenRsync, + ), + ], + ); + + let stats = runtime + .prefetch_transport_requests(&snapshot, time::OffsetDateTime::UNIX_EPOCH) + .expect("prefetch transport requests"); + assert_eq!(stats.loaded_requests, 1); + assert_eq!(stats.enqueued_tasks, 0); + assert_eq!(stats.skipped_incompatible, 1); +} + +#[test] +fn phase1_runtime_transitions_rrdp_failure_to_rsync_success() { + let rrdp_count = Arc::new(AtomicUsize::new(0)); + let rsync_count = Arc::new(AtomicUsize::new(0)); + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + FailRrdpThenSucceedRsyncExecutor { + rrdp_count: Arc::clone(&rrdp_count), + rsync_count: Arc::clone(&rsync_count), + }, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new( + coordinator, + pool, + Arc::new(|_base: &str| "rsync://example.test/module/".to_string()), + SyncPreference::RrdpThenRsync, + ); + + let outcome = runtime + .sync_publication_point_repo(&sample_ca("rsync://example.test/repo/root.mft")) + .expect("sync repo"); + assert!(outcome.repo_sync_ok); + assert_eq!(outcome.repo_sync_source.as_deref(), Some("rsync")); + assert_eq!( + outcome.repo_sync_phase.as_deref(), + Some("rrdp_failed_rsync_ok") + ); + assert_eq!(rrdp_count.load(Ordering::SeqCst), 1); + assert_eq!(rsync_count.load(Ordering::SeqCst), 1); +} + +#[test] +fn phase1_runtime_terminal_failure_keeps_rsync_failure_duration() { + let rrdp_count = Arc::new(AtomicUsize::new(0)); + let rsync_count = Arc::new(AtomicUsize::new(0)); + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + FailRrdpThenFailRsyncExecutor { + rrdp_count: Arc::clone(&rrdp_count), + rsync_count: Arc::clone(&rsync_count), + }, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new( + coordinator, + pool, + Arc::new(|_base: &str| "rsync://example.test/module/".to_string()), + SyncPreference::RrdpThenRsync, + ); + + let outcome = runtime + .sync_publication_point_repo(&sample_ca("rsync://example.test/repo/root.mft")) + .expect("sync repo"); + assert!(!outcome.repo_sync_ok); + assert_eq!( + outcome.repo_sync_phase.as_deref(), + Some("rrdp_failed_rsync_failed") + ); + assert_eq!(outcome.repo_sync_duration_ms, 12); + assert_eq!(rrdp_count.load(Ordering::SeqCst), 1); + assert_eq!(rsync_count.load(Ordering::SeqCst), 1); +} + +#[test] +fn phase1_runtime_prefetch_submits_transport_task_before_consumption() { + let rrdp_count = Arc::new(AtomicUsize::new(0)); + let rsync_count = Arc::new(AtomicUsize::new(0)); + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + FailRrdpThenSucceedRsyncExecutor { + rrdp_count: Arc::clone(&rrdp_count), + rsync_count: Arc::clone(&rsync_count), + }, + ) + .expect("pool"); + let runtime = Arc::new(Phase1RepoSyncRuntime::new( + coordinator, + pool, + Arc::new(|_base: &str| "rsync://example.test/module/".to_string()), + SyncPreference::RrdpThenRsync, + )); + + let child = DiscoveredChildCaInstance { + handle: sample_ca("rsync://example.test/repo/child.mft"), + discovered_from: crate::audit::DiscoveredFrom { + parent_manifest_rsync_uri: "rsync://example.test/repo/root.mft".to_string(), + child_ca_certificate_rsync_uri: "rsync://example.test/repo/child.cer".to_string(), + child_ca_certificate_sha256_hex: "00".repeat(32), + }, + child_entry_projection: None, + }; + + runtime + .prefetch_discovered_children(std::slice::from_ref(&child)) + .expect("prefetch"); + + let started = Instant::now(); + while rrdp_count.load(Ordering::SeqCst) == 0 && started.elapsed() < Duration::from_secs(1) { + std::thread::sleep(Duration::from_millis(10)); + } + assert_eq!(rrdp_count.load(Ordering::SeqCst), 1); + + let outcome = runtime + .sync_publication_point_repo(&child.handle) + .expect("sync child repo"); + assert!(outcome.repo_sync_ok); + assert_eq!(rsync_count.load(Ordering::SeqCst), 1); +} diff --git a/crates/panda-rpki-validator/src/parallel/repo_runtime/types_and_trait.rs b/crates/panda-rpki-validator/src/parallel/repo_runtime/types_and_trait.rs new file mode 100644 index 0000000..9b8bdc8 --- /dev/null +++ b/crates/panda-rpki-validator/src/parallel/repo_runtime/types_and_trait.rs @@ -0,0 +1,101 @@ +// Repository runtime outcomes, events, and runtime trait. + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RepoSyncRuntimeOutcome { + pub repo_sync_ok: bool, + pub repo_sync_err: Option, + pub repo_sync_source: Option, + pub repo_sync_phase: Option, + pub repo_sync_duration_ms: u64, + pub warnings: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RepoSyncRequestStatus { + Ready { + identity: RepoIdentity, + outcome: RepoSyncRuntimeOutcome, + }, + Pending { + identity: RepoIdentity, + state: RepoRuntimeState, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RepoSyncRuntimeCompletion { + pub identity: RepoIdentity, + pub state: RepoRuntimeState, + pub outcome: RepoSyncRuntimeOutcome, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RepoSyncRuntimeEvent { + pub transport_identity: RepoIdentity, + pub completions: Vec, +} + +pub trait RepoSyncRuntime: Send + Sync { + fn sync_publication_point_repo( + &self, + ca: &CaInstanceHandle, + ) -> Result; + + fn request_publication_point_repo( + &self, + ca: &CaInstanceHandle, + priority: u8, + ) -> Result; + + fn recv_repo_result_timeout( + &self, + timeout: Duration, + ) -> Result, String>; + + fn drain_repo_results_timeout( + &self, + timeout: Duration, + max_events: usize, + ) -> Result, String> { + let max_events = max_events.max(1); + let mut events = Vec::new(); + for index in 0..max_events { + let poll_timeout = if index == 0 { + timeout + } else { + Duration::from_millis(0) + }; + let Some(event) = self.recv_repo_result_timeout(poll_timeout)? else { + break; + }; + events.push(event); + } + Ok(events) + } + + fn reset_run_state(&self) -> Result<(), String>; + + fn prefetch_discovered_children( + &self, + children: &[DiscoveredChildCaInstance], + ) -> Result<(), String>; + + fn prefetch_transport_requests( + &self, + snapshot: &TransportPrefetchSnapshot, + validation_time: time::OffsetDateTime, + ) -> Result; + + fn transport_prefetch_snapshot(&self) -> TransportPrefetchSnapshot; + + /// Dead-repository blacklist working copy for run-end persistence. + /// `None` when the feature is disabled. + fn dead_repo_blacklist_state( + &self, + ) -> Option<( + crate::parallel::dead_repo_blacklist::DeadRepoBlacklistConfig, + crate::parallel::dead_repo_blacklist::DeadRepoBlacklist, + )> { + None + } +} diff --git a/crates/panda-rpki-validator/src/parallel/repo_scheduler.rs b/crates/panda-rpki-validator/src/parallel/repo_scheduler.rs index 08a6e29..2312dab 100644 --- a/crates/panda-rpki-validator/src/parallel/repo_scheduler.rs +++ b/crates/panda-rpki-validator/src/parallel/repo_scheduler.rs @@ -77,2168 +77,12 @@ pub struct TransportStateTables { rsync_failure_probe_inflight: HashMap, rsync_failure_scope_reachable: HashSet, runtime_records: HashMap, - /// Frozen run-start snapshot of the dead-repo blacklist (#141). Entries + /// Frozen run-start snapshot of the dead-repository blacklist. Entries /// admitted during this run only take effect from the next run. dead_repo_blacklist: Option, } -impl TransportStateTables { - pub fn new() -> Self { - Self::default() - } - - pub fn set_dead_repo_blacklist(&mut self, blacklist: DeadRepoBlacklist) { - self.dead_repo_blacklist = Some(blacklist); - } - - fn dead_repo_terminal_envelope( - identity: &RepoIdentity, - requesters: &[RepoRequester], - rsync_scope_uri: &str, - rsync_failure_scope_uri: Option, - ) -> RepoTransportResultEnvelope { - let first_requester = requesters - .first() - .expect("blacklist terminal record must keep at least one requester"); - RepoTransportResultEnvelope { - dedup_key: RepoDedupKey::RsyncScope { - rsync_scope_uri: rsync_scope_uri.to_string(), - }, - rsync_failure_scope_uri, - repo_identity: identity.clone(), - mode: RepoTransportMode::Rsync, - tal_id: first_requester.tal_id.clone(), - rir_id: first_requester.rir_id.clone(), - timing_ms: 0, - result: RepoTransportResultKind::Failed { - detail: format!( - "dead repo blacklist: rsync transport persistently unreachable: {}", - identity.rsync_base_uri - ), - warnings: vec![Warning::new(format!( - "dead_repo_blacklist_skip_all: repository {} skipped after repeated transport failures", - identity.rsync_base_uri - )) - .with_context(identity.rsync_base_uri.clone())], - error_class: RepoTransportErrorClass::Unknown, - }, - } - } - - pub fn runtime_record(&self, identity: &RepoIdentity) -> Option<&RepoRuntimeRecord> { - self.runtime_records.get(identity) - } - - pub fn finalized_runtime_records_for_transport( - &self, - dedup_key: &RepoDedupKey, - ) -> Vec { - self.runtime_records - .values() - .filter(|record| match dedup_key { - RepoDedupKey::RrdpNotify { notification_uri } => { - record.rrdp_notification_key.as_deref() == Some(notification_uri.as_str()) - } - RepoDedupKey::RsyncScope { rsync_scope_uri } => { - record.rsync_scope_key == *rsync_scope_uri - } - }) - .filter(|record| { - matches!( - record.state, - RepoRuntimeState::RrdpOk - | RepoRuntimeState::RsyncOk - | RepoRuntimeState::FailedTerminal - ) - }) - .cloned() - .collect() - } - - pub fn finalized_runtime_records_for_transport_result( - &self, - result: &RepoTransportResultEnvelope, - ) -> Vec { - let failure_scope = reusable_rsync_failure_scope(result); - self.runtime_records - .values() - .filter(|record| { - let exact_match = match &result.dedup_key { - RepoDedupKey::RrdpNotify { notification_uri } => { - record.rrdp_notification_key.as_deref() == Some(notification_uri.as_str()) - } - RepoDedupKey::RsyncScope { rsync_scope_uri } => { - record.rsync_scope_key == *rsync_scope_uri - } - }; - let failure_scope_match = failure_scope - .map(|scope| record.rsync_failure_scope_key.as_deref() == Some(scope)) - .unwrap_or(false); - exact_match || failure_scope_match - }) - .filter(|record| { - matches!( - record.state, - RepoRuntimeState::RrdpOk - | RepoRuntimeState::RsyncOk - | RepoRuntimeState::FailedTerminal - ) - }) - .cloned() - .collect() - } - - pub fn reset_run_state(&mut self) { - self.rrdp_inflight.clear(); - self.rsync_inflight.clear(); - self.rsync_failure_by_scope.clear(); - self.rsync_failure_probe_inflight.clear(); - self.rsync_failure_scope_reachable.clear(); - self.runtime_records.clear(); - } - - pub fn register_transport_request( - &mut self, - identity: RepoIdentity, - requester: RepoRequester, - validation_time: time::OffsetDateTime, - priority: u8, - rsync_scope_uri: String, - rsync_failure_scope_uri: Option, - sync_preference: SyncPreference, - ) -> TransportRequestAction { - if let Some(record) = self.runtime_records.get_mut(&identity) { - record.requesters.push(requester.clone()); - return match record.state { - RepoRuntimeState::WaitingRrdp => { - if let Some(key) = record.rrdp_notification_key.as_ref() { - if let Some(entry) = self.rrdp_inflight.get_mut(key) { - entry.waiting_requesters.push(requester); - } - } - TransportRequestAction::Waiting { - state: RepoRuntimeState::WaitingRrdp, - } - } - RepoRuntimeState::RrdpOk | RepoRuntimeState::RsyncOk => { - TransportRequestAction::ReusedSuccess( - record - .last_success - .clone() - .expect("success state must keep last_success"), - ) - } - RepoRuntimeState::RrdpFailedPendingRsync | RepoRuntimeState::WaitingRsync => { - if let Some(entry) = self.rsync_inflight.get_mut(&record.rsync_scope_key) { - entry.waiting_requesters.push(requester); - } - record.state = RepoRuntimeState::WaitingRsync; - TransportRequestAction::Waiting { - state: RepoRuntimeState::WaitingRsync, - } - } - RepoRuntimeState::FailedTerminal => TransportRequestAction::ReusedTerminalFailure( - record - .terminal_failure - .clone() - .expect("terminal failure must keep last result"), - ), - RepoRuntimeState::Init => TransportRequestAction::Waiting { - state: RepoRuntimeState::Init, - }, - }; - } - - // Dead-repo blacklist fast paths (#141), consulted only for fresh - // identities; existing records already encode their terminal state. - if let Some(blacklist) = self.dead_repo_blacklist.as_ref() { - let rrdp_wanted = sync_preference == SyncPreference::RrdpThenRsync - && identity.notification_uri.is_some(); - let rrdp_dead = rrdp_wanted - && identity - .notification_uri - .as_deref() - .map(|uri| blacklist.is_blacklisted(RepoTransportMode::Rrdp, uri)) - .unwrap_or(false); - let rsync_dead = blacklist - .is_blacklisted(RepoTransportMode::Rsync, identity.rsync_base_uri.as_str()); - if rsync_dead && (!rrdp_wanted || rrdp_dead) { - crate::progress_log::emit( - "dead_repo_blacklist_skip_all", - serde_json::json!({ - "repo_key_notification_uri": identity.notification_uri, - "repo_key_rsync_base_uri": identity.rsync_base_uri, - }), - ); - let envelope = Self::dead_repo_terminal_envelope( - &identity, - std::slice::from_ref(&requester), - &rsync_scope_uri, - rsync_failure_scope_uri.clone(), - ); - self.runtime_records.insert( - identity.clone(), - RepoRuntimeRecord { - identity, - state: RepoRuntimeState::FailedTerminal, - rrdp_notification_key: None, - rsync_scope_key: rsync_scope_uri, - rsync_failure_scope_key: rsync_failure_scope_uri, - requesters: vec![requester], - validation_time, - priority, - last_success: None, - terminal_failure: Some(envelope.clone()), - }, - ); - return TransportRequestAction::ReusedTerminalFailure(envelope); - } - if rrdp_dead { - crate::progress_log::emit( - "dead_repo_blacklist_skip_rrdp", - serde_json::json!({ - "repo_key_notification_uri": identity.notification_uri, - "repo_key_rsync_base_uri": identity.rsync_base_uri, - }), - ); - let mut action = self.register_rsync_request( - identity, - requester, - validation_time, - priority, - rsync_scope_uri, - rsync_failure_scope_uri, - ); - // A blacklisted rrdp transport is a known-persistent failure, - // so the rsync fallback must keep the short retry profile - // (#141): the prefetch layer records no real rrdp failure in - // skipped runs, which would otherwise silently drop the task - // back to the default 15s window from the second skipped run - // on. The coordinator wrapper only ever sets the flag to true, - // so setting it here survives registration. - if let TransportRequestAction::Enqueue(task) = &mut action { - task.retry_short_timeout = true; - } - return action; - } - } - - if sync_preference == SyncPreference::RrdpThenRsync { - if let Some(notification_uri) = identity.notification_uri.clone() { - if let Some(entry) = self.rrdp_inflight.get_mut(¬ification_uri) { - if let Some(result) = entry.last_result.clone() { - return match result.result { - RepoTransportResultKind::Success { .. } => { - self.runtime_records.insert( - identity.clone(), - RepoRuntimeRecord { - identity, - state: RepoRuntimeState::RrdpOk, - rrdp_notification_key: Some(notification_uri), - rsync_scope_key: rsync_scope_uri, - rsync_failure_scope_key: rsync_failure_scope_uri.clone(), - requesters: vec![requester], - validation_time, - priority, - last_success: Some(result.clone()), - terminal_failure: None, - }, - ); - TransportRequestAction::ReusedSuccess(result) - } - RepoTransportResultKind::Failed { .. } => { - self.runtime_records.insert( - identity.clone(), - RepoRuntimeRecord { - identity: identity.clone(), - state: RepoRuntimeState::RrdpFailedPendingRsync, - rrdp_notification_key: Some(notification_uri), - rsync_scope_key: rsync_scope_uri.clone(), - rsync_failure_scope_key: rsync_failure_scope_uri.clone(), - requesters: vec![requester.clone()], - validation_time, - priority, - last_success: None, - terminal_failure: None, - }, - ); - self.register_rsync_request( - identity, - requester, - validation_time, - priority, - rsync_scope_uri, - rsync_failure_scope_uri, - ) - } - }; - } - - entry.waiting_requesters.push(requester.clone()); - self.runtime_records.insert( - identity.clone(), - RepoRuntimeRecord { - identity, - state: RepoRuntimeState::WaitingRrdp, - rrdp_notification_key: Some(notification_uri), - rsync_scope_key: rsync_scope_uri, - rsync_failure_scope_key: rsync_failure_scope_uri.clone(), - requesters: vec![requester], - validation_time, - priority, - last_success: None, - terminal_failure: None, - }, - ); - return TransportRequestAction::Waiting { - state: RepoRuntimeState::WaitingRrdp, - }; - } - - let task = RepoTransportTask { - dedup_key: RepoDedupKey::RrdpNotify { - notification_uri: notification_uri.clone(), - }, - rsync_failure_scope_uri: None, - repo_identity: identity.clone(), - mode: RepoTransportMode::Rrdp, - retry_short_timeout: false, - tal_id: requester.tal_id.clone(), - rir_id: requester.rir_id.clone(), - validation_time, - priority, - requesters: vec![requester.clone()], - }; - self.rrdp_inflight.insert( - notification_uri.clone(), - TransportInFlightEntry { - state: TransportTaskState::Pending, - task: task.clone(), - waiting_requesters: Vec::new(), - last_result: None, - started_at: None, - finished_at: None, - }, - ); - self.runtime_records.insert( - identity.clone(), - RepoRuntimeRecord { - identity, - state: RepoRuntimeState::WaitingRrdp, - rrdp_notification_key: Some(notification_uri), - rsync_scope_key: rsync_scope_uri, - rsync_failure_scope_key: rsync_failure_scope_uri.clone(), - requesters: vec![requester], - validation_time, - priority, - last_success: None, - terminal_failure: None, - }, - ); - return TransportRequestAction::Enqueue(task); - } - } - - self.register_rsync_request( - identity, - requester, - validation_time, - priority, - rsync_scope_uri, - rsync_failure_scope_uri, - ) - } - - fn register_rsync_request( - &mut self, - identity: RepoIdentity, - requester: RepoRequester, - validation_time: time::OffsetDateTime, - priority: u8, - rsync_scope_uri: String, - rsync_failure_scope_uri: Option, - ) -> TransportRequestAction { - if let Some(entry) = self.rsync_inflight.get_mut(&rsync_scope_uri) { - if let Some(result) = entry.last_result.clone() { - return match result.result { - RepoTransportResultKind::Success { .. } => { - self.runtime_records.insert( - identity.clone(), - RepoRuntimeRecord { - identity, - state: RepoRuntimeState::RsyncOk, - rrdp_notification_key: None, - rsync_scope_key: rsync_scope_uri, - rsync_failure_scope_key: rsync_failure_scope_uri.clone(), - requesters: vec![requester], - validation_time, - priority, - last_success: Some(result.clone()), - terminal_failure: None, - }, - ); - TransportRequestAction::ReusedSuccess(result) - } - RepoTransportResultKind::Failed { .. } => { - self.runtime_records.insert( - identity.clone(), - RepoRuntimeRecord { - identity, - state: RepoRuntimeState::FailedTerminal, - rrdp_notification_key: None, - rsync_scope_key: rsync_scope_uri, - rsync_failure_scope_key: rsync_failure_scope_uri.clone(), - requesters: vec![requester], - validation_time, - priority, - last_success: None, - terminal_failure: Some(result.clone()), - }, - ); - TransportRequestAction::ReusedTerminalFailure(result) - } - }; - } - - entry.waiting_requesters.push(requester.clone()); - self.runtime_records.insert( - identity.clone(), - RepoRuntimeRecord { - identity, - state: RepoRuntimeState::WaitingRsync, - rrdp_notification_key: None, - rsync_scope_key: rsync_scope_uri, - rsync_failure_scope_key: rsync_failure_scope_uri.clone(), - requesters: vec![requester], - validation_time, - priority, - last_success: None, - terminal_failure: None, - }, - ); - return TransportRequestAction::Waiting { - state: RepoRuntimeState::WaitingRsync, - }; - } - - if let Some(failure_scope_uri) = rsync_failure_scope_uri.as_ref() { - if let Some(result) = self.rsync_failure_by_scope.get(failure_scope_uri).cloned() { - self.runtime_records.insert( - identity.clone(), - RepoRuntimeRecord { - identity, - state: RepoRuntimeState::FailedTerminal, - rrdp_notification_key: None, - rsync_scope_key: rsync_scope_uri, - rsync_failure_scope_key: rsync_failure_scope_uri.clone(), - requesters: vec![requester], - validation_time, - priority, - last_success: None, - terminal_failure: Some(result.clone()), - }, - ); - return TransportRequestAction::ReusedTerminalFailure(result); - } - - if !self - .rsync_failure_scope_reachable - .contains(failure_scope_uri) - { - if self - .rsync_failure_probe_inflight - .get(failure_scope_uri) - .is_some() - { - self.runtime_records.insert( - identity.clone(), - RepoRuntimeRecord { - identity, - state: RepoRuntimeState::WaitingRsync, - rrdp_notification_key: None, - rsync_scope_key: rsync_scope_uri, - rsync_failure_scope_key: rsync_failure_scope_uri.clone(), - requesters: vec![requester], - validation_time, - priority, - last_success: None, - terminal_failure: None, - }, - ); - return TransportRequestAction::Waiting { - state: RepoRuntimeState::WaitingRsync, - }; - } - } - } - - let task = RepoTransportTask { - dedup_key: RepoDedupKey::RsyncScope { - rsync_scope_uri: rsync_scope_uri.clone(), - }, - rsync_failure_scope_uri: rsync_failure_scope_uri.clone(), - repo_identity: identity.clone(), - mode: RepoTransportMode::Rsync, - retry_short_timeout: false, - tal_id: requester.tal_id.clone(), - rir_id: requester.rir_id.clone(), - validation_time, - priority, - requesters: vec![requester.clone()], - }; - self.rsync_inflight.insert( - rsync_scope_uri.clone(), - TransportInFlightEntry { - state: TransportTaskState::Pending, - task: task.clone(), - waiting_requesters: Vec::new(), - last_result: None, - started_at: None, - finished_at: None, - }, - ); - if let Some(failure_scope_uri) = rsync_failure_scope_uri.as_ref() { - if !self - .rsync_failure_scope_reachable - .contains(failure_scope_uri) - { - self.rsync_failure_probe_inflight - .insert(failure_scope_uri.clone(), rsync_scope_uri.clone()); - } - } - self.runtime_records.insert( - identity.clone(), - RepoRuntimeRecord { - identity, - state: RepoRuntimeState::WaitingRsync, - rrdp_notification_key: None, - rsync_scope_key: rsync_scope_uri, - rsync_failure_scope_key: rsync_failure_scope_uri.clone(), - requesters: vec![requester], - validation_time, - priority, - last_success: None, - terminal_failure: None, - }, - ); - TransportRequestAction::Enqueue(task) - } - - fn schedule_rsync_for_record( - record: &mut RepoRuntimeRecord, - rsync_inflight: &mut HashMap, - rsync_failure_by_scope: &HashMap, - rsync_failure_probe_inflight: &mut HashMap, - rsync_failure_scope_reachable: &HashSet, - dead_repo_blacklist: Option<&DeadRepoBlacklist>, - follow_up_tasks: &mut Vec, - ) { - let rsync_scope_uri = record.rsync_scope_key.clone(); - if let Some(entry) = rsync_inflight.get_mut(&rsync_scope_uri) { - if let Some(result) = entry.last_result.clone() { - match result.result { - RepoTransportResultKind::Success { .. } => { - record.state = RepoRuntimeState::RsyncOk; - record.last_success = Some(result); - } - RepoTransportResultKind::Failed { .. } => { - record.state = RepoRuntimeState::FailedTerminal; - record.terminal_failure = Some(result); - } - } - return; - } - entry.waiting_requesters.extend(record.requesters.clone()); - record.state = RepoRuntimeState::WaitingRsync; - return; - } - - if let Some(failure_scope_uri) = record.rsync_failure_scope_key.as_ref() { - if let Some(result) = rsync_failure_by_scope.get(failure_scope_uri).cloned() { - record.state = RepoRuntimeState::FailedTerminal; - record.terminal_failure = Some(result); - return; - } - if !rsync_failure_scope_reachable.contains(failure_scope_uri) - && rsync_failure_probe_inflight.contains_key(failure_scope_uri) - { - record.state = RepoRuntimeState::WaitingRsync; - return; - } - } - - // Dead-repo blacklist (#141): the rsync transport of this repo is - // persistently unreachable; terminate instantly instead of enqueueing - // another doomed fetch (rrdp was already attempted or skipped). - if let Some(blacklist) = dead_repo_blacklist { - if blacklist.is_blacklisted( - RepoTransportMode::Rsync, - record.identity.rsync_base_uri.as_str(), - ) { - crate::progress_log::emit( - "dead_repo_blacklist_skip_all", - serde_json::json!({ - "repo_key_notification_uri": record.identity.notification_uri, - "repo_key_rsync_base_uri": record.identity.rsync_base_uri, - "after_rrdp_failure": true, - }), - ); - let envelope = Self::dead_repo_terminal_envelope( - &record.identity, - &record.requesters, - &rsync_scope_uri, - record.rsync_failure_scope_key.clone(), - ); - record.state = RepoRuntimeState::FailedTerminal; - record.terminal_failure = Some(envelope); - return; - } - } - - let first_requester = record - .requesters - .first() - .expect("rsync record must keep at least one requester"); - let task = RepoTransportTask { - dedup_key: RepoDedupKey::RsyncScope { - rsync_scope_uri: rsync_scope_uri.clone(), - }, - rsync_failure_scope_uri: record.rsync_failure_scope_key.clone(), - repo_identity: record.identity.clone(), - mode: RepoTransportMode::Rsync, - retry_short_timeout: false, - tal_id: first_requester.tal_id.clone(), - rir_id: first_requester.rir_id.clone(), - validation_time: record.validation_time, - priority: record.priority, - requesters: record.requesters.clone(), - }; - rsync_inflight.insert( - rsync_scope_uri.clone(), - TransportInFlightEntry { - state: TransportTaskState::Pending, - task: task.clone(), - waiting_requesters: Vec::new(), - last_result: None, - started_at: None, - finished_at: None, - }, - ); - if let Some(failure_scope_uri) = record.rsync_failure_scope_key.as_ref() { - if !rsync_failure_scope_reachable.contains(failure_scope_uri) { - rsync_failure_probe_inflight.insert(failure_scope_uri.clone(), rsync_scope_uri); - } - } - record.state = RepoRuntimeState::WaitingRsync; - follow_up_tasks.push(task); - } - - pub fn mark_transport_running( - &mut self, - dedup_key: &RepoDedupKey, - started_at: time::OffsetDateTime, - ) -> Result<(), String> { - match dedup_key { - RepoDedupKey::RrdpNotify { notification_uri } => { - let entry = self - .rrdp_inflight - .get_mut(notification_uri) - .ok_or_else(|| format!("rrdp transport not found: {notification_uri}"))?; - entry.state = TransportTaskState::Running; - entry.started_at = Some(started_at); - } - RepoDedupKey::RsyncScope { rsync_scope_uri } => { - let entry = self - .rsync_inflight - .get_mut(rsync_scope_uri) - .ok_or_else(|| format!("rsync transport not found: {rsync_scope_uri}"))?; - entry.state = TransportTaskState::Running; - entry.started_at = Some(started_at); - } - } - Ok(()) - } - - pub fn complete_transport_result( - &mut self, - result: RepoTransportResultEnvelope, - finished_at: time::OffsetDateTime, - ) -> Result { - match (&result.dedup_key, &result.result) { - ( - RepoDedupKey::RrdpNotify { notification_uri }, - RepoTransportResultKind::Success { .. }, - ) => { - let entry = self - .rrdp_inflight - .get_mut(notification_uri) - .ok_or_else(|| format!("rrdp transport not found: {notification_uri}"))?; - entry.state = TransportTaskState::Finished; - entry.finished_at = Some(finished_at); - entry.last_result = Some(result.clone()); - let released_requesters = std::mem::take(&mut entry.waiting_requesters); - for record in self.runtime_records.values_mut() { - if record.rrdp_notification_key.as_deref() == Some(notification_uri) - && record.state == RepoRuntimeState::WaitingRrdp - { - record.state = RepoRuntimeState::RrdpOk; - record.last_success = Some(result.clone()); - } - } - Ok(TransportCompletion { - released_requesters, - follow_up_tasks: Vec::new(), - }) - } - ( - RepoDedupKey::RrdpNotify { notification_uri }, - RepoTransportResultKind::Failed { .. }, - ) => { - let entry = self - .rrdp_inflight - .get_mut(notification_uri) - .ok_or_else(|| format!("rrdp transport not found: {notification_uri}"))?; - entry.state = TransportTaskState::Finished; - entry.finished_at = Some(finished_at); - entry.last_result = Some(result.clone()); - let mut follow_up_tasks = Vec::new(); - for record in self.runtime_records.values_mut() { - if record.rrdp_notification_key.as_deref() == Some(notification_uri) - && record.state == RepoRuntimeState::WaitingRrdp - { - record.state = RepoRuntimeState::RrdpFailedPendingRsync; - Self::schedule_rsync_for_record( - record, - &mut self.rsync_inflight, - &self.rsync_failure_by_scope, - &mut self.rsync_failure_probe_inflight, - &self.rsync_failure_scope_reachable, - self.dead_repo_blacklist.as_ref(), - &mut follow_up_tasks, - ); - } - } - Ok(TransportCompletion { - released_requesters: Vec::new(), - follow_up_tasks, - }) - } - ( - RepoDedupKey::RsyncScope { rsync_scope_uri }, - RepoTransportResultKind::Success { .. }, - ) => { - let mut follow_up_tasks = Vec::new(); - let entry = self - .rsync_inflight - .get_mut(rsync_scope_uri) - .ok_or_else(|| format!("rsync transport not found: {rsync_scope_uri}"))?; - entry.state = TransportTaskState::Finished; - entry.finished_at = Some(finished_at); - entry.last_result = Some(result.clone()); - if let Some(failure_scope_uri) = result.rsync_failure_scope_uri.as_ref() { - self.rsync_failure_probe_inflight.remove(failure_scope_uri); - self.rsync_failure_scope_reachable - .insert(failure_scope_uri.clone()); - } - let released_requesters = std::mem::take(&mut entry.waiting_requesters); - for record in self.runtime_records.values_mut() { - if record.rsync_scope_key == *rsync_scope_uri - && matches!( - record.state, - RepoRuntimeState::WaitingRsync - | RepoRuntimeState::RrdpFailedPendingRsync - ) - { - record.state = RepoRuntimeState::RsyncOk; - record.last_success = Some(result.clone()); - } - } - if let Some(failure_scope_uri) = result.rsync_failure_scope_uri.as_ref() { - for record in self.runtime_records.values_mut() { - if record.rsync_scope_key != *rsync_scope_uri - && record.rsync_failure_scope_key.as_deref() - == Some(failure_scope_uri.as_str()) - && matches!(record.state, RepoRuntimeState::WaitingRsync) - { - Self::schedule_rsync_for_record( - record, - &mut self.rsync_inflight, - &self.rsync_failure_by_scope, - &mut self.rsync_failure_probe_inflight, - &self.rsync_failure_scope_reachable, - self.dead_repo_blacklist.as_ref(), - &mut follow_up_tasks, - ); - } - } - } - Ok(TransportCompletion { - released_requesters, - follow_up_tasks, - }) - } - ( - RepoDedupKey::RsyncScope { rsync_scope_uri }, - RepoTransportResultKind::Failed { .. }, - ) => { - let entry = self - .rsync_inflight - .get_mut(rsync_scope_uri) - .ok_or_else(|| format!("rsync transport not found: {rsync_scope_uri}"))?; - entry.state = TransportTaskState::Finished; - entry.finished_at = Some(finished_at); - entry.last_result = Some(result.clone()); - let reusable_failure_scope = - reusable_rsync_failure_scope(&result).map(str::to_string); - if let Some(failure_scope_uri) = result.rsync_failure_scope_uri.as_ref() { - self.rsync_failure_probe_inflight.remove(failure_scope_uri); - if reusable_failure_scope.as_deref() == Some(failure_scope_uri.as_str()) { - self.rsync_failure_by_scope - .insert(failure_scope_uri.clone(), result.clone()); - } else { - self.rsync_failure_scope_reachable - .insert(failure_scope_uri.clone()); - } - } - let released_requesters = std::mem::take(&mut entry.waiting_requesters); - for record in self.runtime_records.values_mut() { - if (record.rsync_scope_key == *rsync_scope_uri - || reusable_failure_scope - .as_deref() - .map(|scope| record.rsync_failure_scope_key.as_deref() == Some(scope)) - .unwrap_or(false)) - && matches!( - record.state, - RepoRuntimeState::WaitingRsync - | RepoRuntimeState::RrdpFailedPendingRsync - ) - { - record.state = RepoRuntimeState::FailedTerminal; - record.terminal_failure = Some(result.clone()); - } - } - if let Some(failure_scope_uri) = result.rsync_failure_scope_uri.as_ref() { - if reusable_failure_scope.as_deref() != Some(failure_scope_uri.as_str()) { - let mut follow_up_tasks = Vec::new(); - for record in self.runtime_records.values_mut() { - if record.rsync_scope_key != *rsync_scope_uri - && record.rsync_failure_scope_key.as_deref() - == Some(failure_scope_uri.as_str()) - && matches!(record.state, RepoRuntimeState::WaitingRsync) - { - Self::schedule_rsync_for_record( - record, - &mut self.rsync_inflight, - &self.rsync_failure_by_scope, - &mut self.rsync_failure_probe_inflight, - &self.rsync_failure_scope_reachable, - self.dead_repo_blacklist.as_ref(), - &mut follow_up_tasks, - ); - } - } - return Ok(TransportCompletion { - released_requesters, - follow_up_tasks, - }); - } - } - Ok(TransportCompletion { - released_requesters, - follow_up_tasks: Vec::new(), - }) - } - } - } -} - -fn reusable_rsync_failure_scope(result: &RepoTransportResultEnvelope) -> Option<&str> { - let scope = result.rsync_failure_scope_uri.as_deref()?; - match (&result.mode, &result.result) { - (RepoTransportMode::Rsync, RepoTransportResultKind::Failed { detail, .. }) - if is_host_level_rsync_failure(detail) => - { - Some(scope) - } - _ => None, - } -} - -pub(crate) fn is_host_level_rsync_failure(detail: &str) -> bool { - let lower = detail.to_ascii_lowercase(); - lower.contains("timeout waiting for daemon connection") - || lower.contains("failed to connect") - || lower.contains("no route to host") - || lower.contains("network is unreachable") - || lower.contains("connection refused") - || lower.contains("name or service not known") - || lower.contains("temporary failure in name resolution") -} - -#[derive(Default)] -pub struct InFlightRepoTable { - entries: HashMap, -} - -impl InFlightRepoTable { - pub fn new() -> Self { - Self::default() - } - - pub fn len(&self) -> usize { - self.entries.len() - } - - pub fn is_empty(&self) -> bool { - self.entries.is_empty() - } - - pub fn reset_run_state(&mut self) { - self.entries.clear(); - } - - pub fn get(&self, key: &RepoKey) -> Option<&InFlightRepoEntry> { - self.entries.get(key) - } - - pub fn last_result(&self, key: &RepoKey) -> Option<&RepoSyncResultEnvelope> { - self.entries - .get(key) - .and_then(|entry| entry.last_result.as_ref()) - } - - pub fn register_request( - &mut self, - repo_key: RepoKey, - requester: RepoRequester, - validation_time: time::OffsetDateTime, - sync_preference: SyncPreference, - priority: u8, - ) -> RepoRequestAction { - match self.entries.get_mut(&repo_key) { - Some(entry) => match entry.state { - RepoTaskState::Pending | RepoTaskState::Running => { - entry.waiting_requesters.push(requester); - RepoRequestAction::Waiting - } - RepoTaskState::Succeeded | RepoTaskState::Reused => { - RepoRequestAction::Reused(entry.result_ref.clone().expect("result_ref exists")) - } - RepoTaskState::Failed => RepoRequestAction::FailedReuse { - detail: entry - .last_error - .clone() - .unwrap_or_else(|| "repo sync failed".to_string()), - }, - }, - None => { - let task = RepoSyncTask { - repo_key: repo_key.clone(), - validation_time, - sync_preference, - tal_id: requester.tal_id.clone(), - rir_id: requester.rir_id.clone(), - priority, - requesters: vec![requester], - }; - self.entries.insert( - repo_key, - InFlightRepoEntry { - state: RepoTaskState::Pending, - task_ref: Some(task.clone()), - waiting_requesters: Vec::new(), - result_ref: None, - last_result: None, - last_error: None, - started_at: None, - finished_at: None, - }, - ); - RepoRequestAction::Enqueued(task) - } - } - } - - pub fn mark_running( - &mut self, - repo_key: &RepoKey, - started_at: time::OffsetDateTime, - ) -> Result<(), String> { - let entry = self - .entries - .get_mut(repo_key) - .ok_or_else(|| format!("repo not found: {}", repo_key.rsync_base_uri))?; - match entry.state { - RepoTaskState::Pending => { - entry.state = RepoTaskState::Running; - entry.started_at = Some(started_at); - Ok(()) - } - other => Err(format!("repo cannot transition to running from {other:?}")), - } - } - - pub fn complete_success( - &mut self, - repo_key: &RepoKey, - result: RepoSyncResultEnvelope, - finished_at: time::OffsetDateTime, - ) -> Result { - let entry = self - .entries - .get_mut(repo_key) - .ok_or_else(|| format!("repo not found: {}", repo_key.rsync_base_uri))?; - match entry.state { - RepoTaskState::Pending | RepoTaskState::Running => { - let result_ref = match &result.result { - RepoSyncResultKind::Success(result_ref) - | RepoSyncResultKind::Reused(result_ref) => result_ref.clone(), - RepoSyncResultKind::Failed { detail } => { - return Err(format!( - "success completion called with failure result: {detail}" - )); - } - }; - entry.state = RepoTaskState::Succeeded; - entry.result_ref = Some(result_ref); - entry.last_result = Some(result); - entry.last_error = None; - entry.finished_at = Some(finished_at); - entry.task_ref = None; - let released_requesters = std::mem::take(&mut entry.waiting_requesters); - Ok(RepoCompletion { - repo_key: repo_key.clone(), - released_requesters, - }) - } - other => Err(format!("repo cannot transition to success from {other:?}")), - } - } - - pub fn complete_failure( - &mut self, - repo_key: &RepoKey, - result: RepoSyncResultEnvelope, - finished_at: time::OffsetDateTime, - ) -> Result { - let entry = self - .entries - .get_mut(repo_key) - .ok_or_else(|| format!("repo not found: {}", repo_key.rsync_base_uri))?; - match entry.state { - RepoTaskState::Pending | RepoTaskState::Running => { - let detail = match &result.result { - RepoSyncResultKind::Failed { detail } => detail.clone(), - RepoSyncResultKind::Success(_) | RepoSyncResultKind::Reused(_) => { - return Err("failure completion called with success result".to_string()); - } - }; - entry.state = RepoTaskState::Failed; - entry.result_ref = None; - entry.last_result = Some(result); - entry.last_error = Some(detail); - entry.finished_at = Some(finished_at); - entry.task_ref = None; - let released_requesters = std::mem::take(&mut entry.waiting_requesters); - Ok(RepoCompletion { - repo_key: repo_key.clone(), - released_requesters, - }) - } - other => Err(format!("repo cannot transition to failure from {other:?}")), - } - } -} - -#[cfg(test)] -mod tests { - use crate::parallel::repo_scheduler::{InFlightRepoTable, RepoRequestAction}; - use crate::parallel::types::{ - RepoKey, RepoRequester, RepoSyncResultEnvelope, RepoSyncResultKind, RepoSyncResultRef, - RepoTaskState, - }; - use crate::policy::SyncPreference; - - fn requester(tal_id: &str, rir_id: &str, manifest: &str) -> RepoRequester { - RepoRequester { - tal_id: tal_id.to_string(), - rir_id: rir_id.to_string(), - parent_node_id: None, - ca_instance_handle_id: format!("{tal_id}:{manifest}"), - publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), - manifest_rsync_uri: manifest.to_string(), - } - } - - #[test] - fn inflight_repo_table_stores_entries_by_repo_key() { - let key = RepoKey::new("rsync://example.test/repo/", None); - let mut table = InFlightRepoTable::new(); - let action = table.register_request( - key.clone(), - requester("arin", "arin", "rsync://example.test/repo/root.mft"), - time::OffsetDateTime::UNIX_EPOCH, - SyncPreference::RrdpThenRsync, - 0, - ); - assert!(matches!(action, RepoRequestAction::Enqueued(_))); - assert_eq!(table.len(), 1); - assert_eq!( - table.get(&key).map(|entry| entry.state), - Some(RepoTaskState::Pending) - ); - } - - #[test] - fn running_repo_request_is_merged_into_waiting_list() { - let key = RepoKey::new("rsync://example.test/repo/", None); - let mut table = InFlightRepoTable::new(); - let _ = table.register_request( - key.clone(), - requester("arin", "arin", "rsync://example.test/repo/root.mft"), - time::OffsetDateTime::UNIX_EPOCH, - SyncPreference::RrdpThenRsync, - 0, - ); - table - .mark_running(&key, time::OffsetDateTime::UNIX_EPOCH) - .expect("mark running"); - - let action = table.register_request( - key.clone(), - requester("apnic", "apnic", "rsync://example.test/repo/other.mft"), - time::OffsetDateTime::UNIX_EPOCH, - SyncPreference::RrdpThenRsync, - 0, - ); - assert_eq!(action, RepoRequestAction::Waiting); - assert_eq!(table.get(&key).unwrap().waiting_requesters.len(), 1); - } - - #[test] - fn complete_success_releases_waiting_requesters_and_reuses_result() { - let key = RepoKey::new("rsync://example.test/repo/", None); - let mut table = InFlightRepoTable::new(); - let _ = table.register_request( - key.clone(), - requester("arin", "arin", "rsync://example.test/repo/root.mft"), - time::OffsetDateTime::UNIX_EPOCH, - SyncPreference::RrdpThenRsync, - 0, - ); - table - .mark_running(&key, time::OffsetDateTime::UNIX_EPOCH) - .expect("mark running"); - let waiting = requester("ripe", "ripe", "rsync://example.test/repo/child.mft"); - let _ = table.register_request( - key.clone(), - waiting.clone(), - time::OffsetDateTime::UNIX_EPOCH, - SyncPreference::RrdpThenRsync, - 0, - ); - - let result_ref = RepoSyncResultRef { - repo_key: key.clone(), - source: "rrdp".to_string(), - }; - let completion = table - .complete_success( - &key, - RepoSyncResultEnvelope { - repo_key: key.clone(), - tal_id: "arin".to_string(), - rir_id: "arin".to_string(), - result: RepoSyncResultKind::Success(result_ref.clone()), - phase: Some("rrdp_ok".to_string()), - timing_ms: 10, - warnings: Vec::new(), - }, - time::OffsetDateTime::UNIX_EPOCH, - ) - .expect("complete success"); - assert_eq!(completion.released_requesters, vec![waiting]); - assert_eq!(table.get(&key).unwrap().state, RepoTaskState::Succeeded); - - let action = table.register_request( - key, - requester("afrinic", "afrinic", "rsync://example.test/repo/again.mft"), - time::OffsetDateTime::UNIX_EPOCH, - SyncPreference::RrdpThenRsync, - 0, - ); - assert_eq!(action, RepoRequestAction::Reused(result_ref)); - } - - #[test] - fn complete_failure_releases_waiting_requesters_and_reuses_failure() { - let key = RepoKey::new("rsync://example.test/repo/", None); - let mut table = InFlightRepoTable::new(); - let _ = table.register_request( - key.clone(), - requester("arin", "arin", "rsync://example.test/repo/root.mft"), - time::OffsetDateTime::UNIX_EPOCH, - SyncPreference::RrdpThenRsync, - 0, - ); - table - .mark_running(&key, time::OffsetDateTime::UNIX_EPOCH) - .expect("mark running"); - let waiting = requester("ripe", "ripe", "rsync://example.test/repo/child.mft"); - let _ = table.register_request( - key.clone(), - waiting.clone(), - time::OffsetDateTime::UNIX_EPOCH, - SyncPreference::RrdpThenRsync, - 0, - ); - - let completion = table - .complete_failure( - &key, - RepoSyncResultEnvelope { - repo_key: key.clone(), - tal_id: "arin".to_string(), - rir_id: "arin".to_string(), - result: RepoSyncResultKind::Failed { - detail: "network timeout".to_string(), - }, - phase: Some("rrdp_failed_rsync_failed".to_string()), - timing_ms: 10, - warnings: Vec::new(), - }, - time::OffsetDateTime::UNIX_EPOCH, - ) - .expect("complete failure"); - assert_eq!(completion.released_requesters, vec![waiting]); - assert_eq!(table.get(&key).unwrap().state, RepoTaskState::Failed); - - let action = table.register_request( - key, - requester("afrinic", "afrinic", "rsync://example.test/repo/again.mft"), - time::OffsetDateTime::UNIX_EPOCH, - SyncPreference::RrdpThenRsync, - 0, - ); - assert_eq!( - action, - RepoRequestAction::FailedReuse { - detail: "network timeout".to_string() - } - ); - } - - #[test] - fn requesters_from_different_tals_do_not_lose_identity() { - let key = RepoKey::new("rsync://shared.example/repo/", None); - let mut table = InFlightRepoTable::new(); - let _ = table.register_request( - key.clone(), - requester("arin", "arin", "rsync://shared.example/repo/a.mft"), - time::OffsetDateTime::UNIX_EPOCH, - SyncPreference::RrdpThenRsync, - 0, - ); - table - .mark_running(&key, time::OffsetDateTime::UNIX_EPOCH) - .expect("mark running"); - let wait_apnic = requester("apnic", "apnic", "rsync://shared.example/repo/b.mft"); - let wait_ripe = requester("ripe", "ripe", "rsync://shared.example/repo/c.mft"); - let _ = table.register_request( - key.clone(), - wait_apnic.clone(), - time::OffsetDateTime::UNIX_EPOCH, - SyncPreference::RrdpThenRsync, - 0, - ); - let _ = table.register_request( - key.clone(), - wait_ripe.clone(), - time::OffsetDateTime::UNIX_EPOCH, - SyncPreference::RrdpThenRsync, - 0, - ); - let completion = table - .complete_success( - &key, - RepoSyncResultEnvelope { - repo_key: key.clone(), - tal_id: "arin".to_string(), - rir_id: "arin".to_string(), - result: RepoSyncResultKind::Success(RepoSyncResultRef { - repo_key: key.clone(), - source: "rrdp".to_string(), - }), - phase: Some("rrdp_ok".to_string()), - timing_ms: 10, - warnings: Vec::new(), - }, - time::OffsetDateTime::UNIX_EPOCH, - ) - .expect("complete success"); - assert_eq!(completion.released_requesters, vec![wait_apnic, wait_ripe]); - } -} - -#[cfg(test)] -mod transport_tests { - use crate::parallel::repo_scheduler::{ - TransportRequestAction, TransportStateTables, TransportTaskState, - }; - use crate::parallel::types::{ - RepoDedupKey, RepoIdentity, RepoRequester, RepoRuntimeState, RepoTransportMode, - RepoTransportResultEnvelope, RepoTransportResultKind, - }; - use crate::policy::SyncPreference; - - fn requester(id: &str) -> RepoRequester { - RepoRequester::with_tal_rir( - "apnic", - "apnic", - format!("rsync://example.test/repo/{id}.mft"), - "rsync://example.test/repo/".to_string(), - format!("node:{id}"), - ) - } - - #[test] - fn register_transport_request_enqueues_initial_rrdp_task() { - let mut tables = TransportStateTables::new(); - let identity = RepoIdentity::new( - Some("https://example.test/notify.xml".to_string()), - "rsync://example.test/repo/", - ); - let action = tables.register_transport_request( - identity.clone(), - requester("root"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/module/".to_string(), - None, - SyncPreference::RrdpThenRsync, - ); - let task = match action { - TransportRequestAction::Enqueue(task) => task, - other => panic!("expected enqueue, got {other:?}"), - }; - assert_eq!(task.mode, RepoTransportMode::Rrdp); - assert_eq!( - task.dedup_key, - RepoDedupKey::RrdpNotify { - notification_uri: "https://example.test/notify.xml".to_string() - } - ); - let key = "https://example.test/notify.xml".to_string(); - assert_eq!( - tables.rrdp_inflight.get(&key).unwrap().state, - TransportTaskState::Pending - ); - assert_eq!( - tables.runtime_records.get(&identity).unwrap().state, - RepoRuntimeState::WaitingRrdp - ); - } - - #[test] - fn register_transport_request_waits_on_existing_rrdp_task() { - let mut tables = TransportStateTables::new(); - let identity = RepoIdentity::new( - Some("https://example.test/notify.xml".to_string()), - "rsync://example.test/repo/a/", - ); - let _ = tables.register_transport_request( - identity.clone(), - requester("a"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/module/".to_string(), - None, - SyncPreference::RrdpThenRsync, - ); - let other_identity = RepoIdentity::new( - Some("https://example.test/notify.xml".to_string()), - "rsync://example.test/repo/b/", - ); - let action = tables.register_transport_request( - other_identity.clone(), - requester("b"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/module/".to_string(), - None, - SyncPreference::RrdpThenRsync, - ); - assert_eq!( - action, - TransportRequestAction::Waiting { - state: RepoRuntimeState::WaitingRrdp - } - ); - assert_eq!( - tables.runtime_records.get(&other_identity).unwrap().state, - RepoRuntimeState::WaitingRrdp - ); - assert_eq!( - tables - .rrdp_inflight - .get("https://example.test/notify.xml") - .unwrap() - .waiting_requesters - .len(), - 1 - ); - } - - #[test] - fn complete_rrdp_success_reuses_for_later_identity_requests() { - let mut tables = TransportStateTables::new(); - let identity = RepoIdentity::new( - Some("https://example.test/notify.xml".to_string()), - "rsync://example.test/repo/a/", - ); - let _ = tables.register_transport_request( - identity.clone(), - requester("a"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/module/".to_string(), - None, - SyncPreference::RrdpThenRsync, - ); - tables - .mark_transport_running( - &RepoDedupKey::RrdpNotify { - notification_uri: "https://example.test/notify.xml".to_string(), - }, - time::OffsetDateTime::UNIX_EPOCH, - ) - .expect("mark running"); - let completion = tables - .complete_transport_result( - RepoTransportResultEnvelope { - dedup_key: RepoDedupKey::RrdpNotify { - notification_uri: "https://example.test/notify.xml".to_string(), - }, - rsync_failure_scope_uri: None, - repo_identity: identity.clone(), - mode: RepoTransportMode::Rrdp, - tal_id: "apnic".to_string(), - rir_id: "apnic".to_string(), - timing_ms: 10, - result: RepoTransportResultKind::Success { - source: "rrdp".to_string(), - warnings: Vec::new(), - }, - }, - time::OffsetDateTime::UNIX_EPOCH, - ) - .expect("complete success"); - assert!(completion.follow_up_tasks.is_empty()); - let later_identity = RepoIdentity::new( - Some("https://example.test/notify.xml".to_string()), - "rsync://example.test/repo/b/", - ); - let action = tables.register_transport_request( - later_identity.clone(), - requester("b"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/module/".to_string(), - None, - SyncPreference::RrdpThenRsync, - ); - assert!(matches!(action, TransportRequestAction::ReusedSuccess(_))); - assert_eq!( - tables.runtime_records.get(&later_identity).unwrap().state, - RepoRuntimeState::RrdpOk - ); - - let same_identity_action = tables.register_transport_request( - later_identity.clone(), - requester("b-again"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/module/".to_string(), - None, - SyncPreference::RrdpThenRsync, - ); - assert!(matches!( - same_identity_action, - TransportRequestAction::ReusedSuccess(_) - )); - - let finalized = tables.finalized_runtime_records_for_transport(&RepoDedupKey::RrdpNotify { - notification_uri: "https://example.test/notify.xml".to_string(), - }); - assert_eq!(finalized.len(), 2); - } - - #[test] - fn complete_rrdp_failure_enqueues_rsync_follow_up() { - let mut tables = TransportStateTables::new(); - let identity = RepoIdentity::new( - Some("https://example.test/notify.xml".to_string()), - "rsync://example.test/repo/a/", - ); - let _ = tables.register_transport_request( - identity.clone(), - requester("a"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/module/".to_string(), - None, - SyncPreference::RrdpThenRsync, - ); - tables - .mark_transport_running( - &RepoDedupKey::RrdpNotify { - notification_uri: "https://example.test/notify.xml".to_string(), - }, - time::OffsetDateTime::UNIX_EPOCH, - ) - .expect("mark running"); - let completion = tables - .complete_transport_result( - RepoTransportResultEnvelope { - dedup_key: RepoDedupKey::RrdpNotify { - notification_uri: "https://example.test/notify.xml".to_string(), - }, - rsync_failure_scope_uri: None, - repo_identity: identity.clone(), - mode: RepoTransportMode::Rrdp, - tal_id: "apnic".to_string(), - rir_id: "apnic".to_string(), - timing_ms: 10, - result: RepoTransportResultKind::Failed { - detail: "rrdp timeout".to_string(), - warnings: Vec::new(), - error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, - }, - }, - time::OffsetDateTime::UNIX_EPOCH, - ) - .expect("complete failure"); - assert_eq!(completion.follow_up_tasks.len(), 1); - assert_eq!(completion.follow_up_tasks[0].mode, RepoTransportMode::Rsync); - assert_eq!( - tables.runtime_records.get(&identity).unwrap().state, - RepoRuntimeState::WaitingRsync - ); - - let same_identity_action = tables.register_transport_request( - identity.clone(), - requester("a-again"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/module/".to_string(), - None, - SyncPreference::RrdpThenRsync, - ); - assert_eq!( - same_identity_action, - TransportRequestAction::Waiting { - state: RepoRuntimeState::WaitingRsync - } - ); - - let later_identity = RepoIdentity::new( - Some("https://example.test/notify.xml".to_string()), - "rsync://example.test/repo/b/", - ); - let later_action = tables.register_transport_request( - later_identity.clone(), - requester("b"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/module/".to_string(), - None, - SyncPreference::RrdpThenRsync, - ); - assert_eq!( - later_action, - TransportRequestAction::Waiting { - state: RepoRuntimeState::WaitingRsync - } - ); - assert_eq!( - tables.runtime_records.get(&later_identity).unwrap().state, - RepoRuntimeState::WaitingRsync - ); - } - - #[test] - fn host_failure_scope_probes_once_then_reuses_terminal_failure() { - let mut tables = TransportStateTables::new(); - let identity_a = RepoIdentity::new( - Some("https://example.test/notify.xml".to_string()), - "rsync://example.test/repo/a/", - ); - let identity_b = RepoIdentity::new( - Some("https://example.test/notify.xml".to_string()), - "rsync://example.test/repo/b/", - ); - let _ = tables.register_transport_request( - identity_a.clone(), - requester("a"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/repo/a/".to_string(), - Some("rsync://example.test/".to_string()), - SyncPreference::RrdpThenRsync, - ); - let _ = tables.register_transport_request( - identity_b.clone(), - requester("b"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/repo/b/".to_string(), - Some("rsync://example.test/".to_string()), - SyncPreference::RrdpThenRsync, - ); - tables - .mark_transport_running( - &RepoDedupKey::RrdpNotify { - notification_uri: "https://example.test/notify.xml".to_string(), - }, - time::OffsetDateTime::UNIX_EPOCH, - ) - .expect("mark running"); - let completion = tables - .complete_transport_result( - RepoTransportResultEnvelope { - dedup_key: RepoDedupKey::RrdpNotify { - notification_uri: "https://example.test/notify.xml".to_string(), - }, - rsync_failure_scope_uri: None, - repo_identity: identity_a.clone(), - mode: RepoTransportMode::Rrdp, - tal_id: "apnic".to_string(), - rir_id: "apnic".to_string(), - timing_ms: 10, - result: RepoTransportResultKind::Failed { - detail: "rrdp timeout".to_string(), - warnings: Vec::new(), - error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, - }, - }, - time::OffsetDateTime::UNIX_EPOCH, - ) - .expect("complete rrdp failure"); - assert_eq!(completion.follow_up_tasks.len(), 1); - assert_eq!( - completion.follow_up_tasks[0] - .rsync_failure_scope_uri - .as_deref(), - Some("rsync://example.test/") - ); - let probe_task = completion.follow_up_tasks[0].clone(); - - tables - .mark_transport_running(&probe_task.dedup_key, time::OffsetDateTime::UNIX_EPOCH) - .expect("mark rsync running"); - let completion = tables - .complete_transport_result( - RepoTransportResultEnvelope { - dedup_key: probe_task.dedup_key, - rsync_failure_scope_uri: probe_task.rsync_failure_scope_uri, - repo_identity: probe_task.repo_identity, - mode: RepoTransportMode::Rsync, - tal_id: "apnic".to_string(), - rir_id: "apnic".to_string(), - timing_ms: 15_000, - result: RepoTransportResultKind::Failed { - detail: "rsync error: timeout waiting for daemon connection".to_string(), - warnings: Vec::new(), - error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, - }, - }, - time::OffsetDateTime::UNIX_EPOCH, - ) - .expect("complete rsync failure"); - assert!(completion.follow_up_tasks.is_empty()); - assert_eq!( - tables.runtime_records.get(&identity_a).unwrap().state, - RepoRuntimeState::FailedTerminal - ); - assert_eq!( - tables.runtime_records.get(&identity_b).unwrap().state, - RepoRuntimeState::FailedTerminal - ); - let finalized = tables.finalized_runtime_records_for_transport_result( - tables - .runtime_records - .get(&identity_a) - .unwrap() - .terminal_failure - .as_ref() - .unwrap(), - ); - assert_eq!(finalized.len(), 2); - } - - #[test] - fn complete_rsync_failure_reuses_terminal_failure() { - let mut tables = TransportStateTables::new(); - let identity = RepoIdentity::new(None, "rsync://example.test/repo/a/"); - let _ = tables.register_transport_request( - identity.clone(), - requester("a"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/module/".to_string(), - None, - SyncPreference::RsyncOnly, - ); - tables - .mark_transport_running( - &RepoDedupKey::RsyncScope { - rsync_scope_uri: "rsync://example.test/module/".to_string(), - }, - time::OffsetDateTime::UNIX_EPOCH, - ) - .expect("mark running"); - let _ = tables - .complete_transport_result( - RepoTransportResultEnvelope { - dedup_key: RepoDedupKey::RsyncScope { - rsync_scope_uri: "rsync://example.test/module/".to_string(), - }, - rsync_failure_scope_uri: None, - repo_identity: identity.clone(), - mode: RepoTransportMode::Rsync, - tal_id: "apnic".to_string(), - rir_id: "apnic".to_string(), - timing_ms: 20, - result: RepoTransportResultKind::Failed { - detail: "rsync timeout".to_string(), - warnings: Vec::new(), - error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, - }, - }, - time::OffsetDateTime::UNIX_EPOCH, - ) - .expect("complete rsync failure"); - let later_identity = RepoIdentity::new(None, "rsync://example.test/repo/b/"); - let action = tables.register_transport_request( - later_identity, - requester("b"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/module/".to_string(), - None, - SyncPreference::RsyncOnly, - ); - assert!(matches!( - action, - TransportRequestAction::ReusedTerminalFailure(_) - )); - - let finalized = tables.finalized_runtime_records_for_transport(&RepoDedupKey::RsyncScope { - rsync_scope_uri: "rsync://example.test/module/".to_string(), - }); - assert_eq!(finalized.len(), 2); - - let same_identity_action = tables.register_transport_request( - identity, - requester("a-again"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/module/".to_string(), - None, - SyncPreference::RsyncOnly, - ); - assert!(matches!( - same_identity_action, - TransportRequestAction::ReusedTerminalFailure(_) - )); - } - - #[test] - fn complete_rsync_success_reuses_for_later_identity_requests() { - let mut tables = TransportStateTables::new(); - let identity = RepoIdentity::new(None, "rsync://example.test/repo/a/"); - let action = tables.register_transport_request( - identity.clone(), - requester("a"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/module/".to_string(), - None, - SyncPreference::RsyncOnly, - ); - assert!(matches!(action, TransportRequestAction::Enqueue(_))); - tables - .mark_transport_running( - &RepoDedupKey::RsyncScope { - rsync_scope_uri: "rsync://example.test/module/".to_string(), - }, - time::OffsetDateTime::UNIX_EPOCH, - ) - .expect("mark running"); - tables - .complete_transport_result( - RepoTransportResultEnvelope { - dedup_key: RepoDedupKey::RsyncScope { - rsync_scope_uri: "rsync://example.test/module/".to_string(), - }, - rsync_failure_scope_uri: None, - repo_identity: identity.clone(), - mode: RepoTransportMode::Rsync, - tal_id: "apnic".to_string(), - rir_id: "apnic".to_string(), - timing_ms: 20, - result: RepoTransportResultKind::Success { - source: "rsync".to_string(), - warnings: Vec::new(), - }, - }, - time::OffsetDateTime::UNIX_EPOCH, - ) - .expect("complete rsync success"); - - let later_identity = RepoIdentity::new(None, "rsync://example.test/repo/b/"); - let action = tables.register_transport_request( - later_identity.clone(), - requester("b"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/module/".to_string(), - None, - SyncPreference::RsyncOnly, - ); - assert!(matches!(action, TransportRequestAction::ReusedSuccess(_))); - assert_eq!( - tables.runtime_records.get(&later_identity).unwrap().state, - RepoRuntimeState::RsyncOk - ); - } - - #[test] - fn register_rsync_request_waits_on_existing_rsync_task() { - let mut tables = TransportStateTables::new(); - let identity_a = RepoIdentity::new(None, "rsync://example.test/repo/a/"); - let identity_b = RepoIdentity::new(None, "rsync://example.test/repo/b/"); - let _ = tables.register_transport_request( - identity_a, - requester("a"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/module/".to_string(), - None, - SyncPreference::RsyncOnly, - ); - let action = tables.register_transport_request( - identity_b.clone(), - requester("b"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/module/".to_string(), - None, - SyncPreference::RsyncOnly, - ); - assert_eq!( - action, - TransportRequestAction::Waiting { - state: RepoRuntimeState::WaitingRsync - } - ); - assert_eq!( - tables.runtime_records.get(&identity_b).unwrap().state, - RepoRuntimeState::WaitingRsync - ); - assert_eq!( - tables - .rsync_inflight - .get("rsync://example.test/module/") - .unwrap() - .waiting_requesters - .len(), - 1 - ); - } - - #[test] - fn host_failure_scope_success_marks_host_reachable_and_schedules_waiters() { - let mut tables = TransportStateTables::new(); - let identity_a = RepoIdentity::new(None, "rsync://example.test/repo/a/"); - let identity_b = RepoIdentity::new(None, "rsync://example.test/repo/b/"); - let action_a = tables.register_transport_request( - identity_a.clone(), - requester("a"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/repo/a/".to_string(), - Some("rsync://example.test/".to_string()), - SyncPreference::RsyncOnly, - ); - let probe_task = match action_a { - TransportRequestAction::Enqueue(task) => task, - other => panic!("expected first host probe enqueue, got {other:?}"), - }; - let action_b = tables.register_transport_request( - identity_b.clone(), - requester("b"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/repo/b/".to_string(), - Some("rsync://example.test/".to_string()), - SyncPreference::RsyncOnly, - ); - assert_eq!( - action_b, - TransportRequestAction::Waiting { - state: RepoRuntimeState::WaitingRsync - } - ); - - tables - .mark_transport_running(&probe_task.dedup_key, time::OffsetDateTime::UNIX_EPOCH) - .expect("mark rsync running"); - let completion = tables - .complete_transport_result( - RepoTransportResultEnvelope { - dedup_key: probe_task.dedup_key, - rsync_failure_scope_uri: probe_task.rsync_failure_scope_uri, - repo_identity: probe_task.repo_identity, - mode: RepoTransportMode::Rsync, - tal_id: "apnic".to_string(), - rir_id: "apnic".to_string(), - timing_ms: 5, - result: RepoTransportResultKind::Success { - source: "rsync".to_string(), - warnings: Vec::new(), - }, - }, - time::OffsetDateTime::UNIX_EPOCH, - ) - .expect("complete rsync success"); - - assert_eq!( - tables.runtime_records.get(&identity_a).unwrap().state, - RepoRuntimeState::RsyncOk - ); - assert_eq!(completion.follow_up_tasks.len(), 1); - assert_eq!( - completion.follow_up_tasks[0].dedup_key, - RepoDedupKey::RsyncScope { - rsync_scope_uri: "rsync://example.test/repo/b/".to_string() - } - ); - assert_eq!( - tables.runtime_records.get(&identity_b).unwrap().state, - RepoRuntimeState::WaitingRsync - ); - } - - #[test] - fn non_host_level_rsync_failure_does_not_poison_host_scope() { - let mut tables = TransportStateTables::new(); - let identity_a = RepoIdentity::new(None, "rsync://example.test/repo/a/"); - let identity_b = RepoIdentity::new(None, "rsync://example.test/repo/b/"); - let action_a = tables.register_transport_request( - identity_a.clone(), - requester("a"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/repo/a/".to_string(), - Some("rsync://example.test/".to_string()), - SyncPreference::RsyncOnly, - ); - let probe_task = match action_a { - TransportRequestAction::Enqueue(task) => task, - other => panic!("expected first host probe enqueue, got {other:?}"), - }; - let _ = tables.register_transport_request( - identity_b.clone(), - requester("b"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/repo/b/".to_string(), - Some("rsync://example.test/".to_string()), - SyncPreference::RsyncOnly, - ); - - tables - .mark_transport_running(&probe_task.dedup_key, time::OffsetDateTime::UNIX_EPOCH) - .expect("mark rsync running"); - let completion = tables - .complete_transport_result( - RepoTransportResultEnvelope { - dedup_key: probe_task.dedup_key, - rsync_failure_scope_uri: probe_task.rsync_failure_scope_uri, - repo_identity: probe_task.repo_identity, - mode: RepoTransportMode::Rsync, - tal_id: "apnic".to_string(), - rir_id: "apnic".to_string(), - timing_ms: 5, - result: RepoTransportResultKind::Failed { - detail: "rsync file digest mismatch after download".to_string(), - warnings: Vec::new(), - error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, - }, - }, - time::OffsetDateTime::UNIX_EPOCH, - ) - .expect("complete rsync failure"); - - assert_eq!( - tables.runtime_records.get(&identity_a).unwrap().state, - RepoRuntimeState::FailedTerminal - ); - assert_eq!(completion.follow_up_tasks.len(), 1); - assert_eq!( - completion.follow_up_tasks[0].dedup_key, - RepoDedupKey::RsyncScope { - rsync_scope_uri: "rsync://example.test/repo/b/".to_string() - } - ); - } - - #[test] - fn cached_host_level_failure_reuses_for_later_rsync_only_requests() { - let mut tables = TransportStateTables::new(); - let identity_a = RepoIdentity::new(None, "rsync://example.test/repo/a/"); - let action_a = tables.register_transport_request( - identity_a, - requester("a"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/repo/a/".to_string(), - Some("rsync://example.test/".to_string()), - SyncPreference::RsyncOnly, - ); - let probe_task = match action_a { - TransportRequestAction::Enqueue(task) => task, - other => panic!("expected first host probe enqueue, got {other:?}"), - }; - tables - .mark_transport_running(&probe_task.dedup_key, time::OffsetDateTime::UNIX_EPOCH) - .expect("mark rsync running"); - let _ = tables - .complete_transport_result( - RepoTransportResultEnvelope { - dedup_key: probe_task.dedup_key, - rsync_failure_scope_uri: probe_task.rsync_failure_scope_uri, - repo_identity: probe_task.repo_identity, - mode: RepoTransportMode::Rsync, - tal_id: "apnic".to_string(), - rir_id: "apnic".to_string(), - timing_ms: 15_000, - result: RepoTransportResultKind::Failed { - detail: "rsync error: failed to connect to daemon".to_string(), - warnings: Vec::new(), - error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, - }, - }, - time::OffsetDateTime::UNIX_EPOCH, - ) - .expect("complete host failure"); - - let later_identity = RepoIdentity::new(None, "rsync://example.test/repo/c/"); - let action = tables.register_transport_request( - later_identity.clone(), - requester("c"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/repo/c/".to_string(), - Some("rsync://example.test/".to_string()), - SyncPreference::RsyncOnly, - ); - assert!(matches!( - action, - TransportRequestAction::ReusedTerminalFailure(_) - )); - assert_eq!( - tables.runtime_records.get(&later_identity).unwrap().state, - RepoRuntimeState::FailedTerminal - ); - } - - #[test] - fn reset_run_state_clears_host_failure_scope_cache() { - let mut tables = TransportStateTables::new(); - let identity = RepoIdentity::new(None, "rsync://example.test/repo/a/"); - let action = tables.register_transport_request( - identity, - requester("a"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/repo/a/".to_string(), - Some("rsync://example.test/".to_string()), - SyncPreference::RsyncOnly, - ); - let probe_task = match action { - TransportRequestAction::Enqueue(task) => task, - other => panic!("expected host probe enqueue, got {other:?}"), - }; - tables - .mark_transport_running(&probe_task.dedup_key, time::OffsetDateTime::UNIX_EPOCH) - .expect("mark rsync running"); - let _ = tables - .complete_transport_result( - RepoTransportResultEnvelope { - dedup_key: probe_task.dedup_key, - rsync_failure_scope_uri: probe_task.rsync_failure_scope_uri, - repo_identity: probe_task.repo_identity, - mode: RepoTransportMode::Rsync, - tal_id: "apnic".to_string(), - rir_id: "apnic".to_string(), - timing_ms: 15_000, - result: RepoTransportResultKind::Failed { - detail: "temporary failure in name resolution".to_string(), - warnings: Vec::new(), - error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, - }, - }, - time::OffsetDateTime::UNIX_EPOCH, - ) - .expect("complete host failure"); - - tables.reset_run_state(); - let later_identity = RepoIdentity::new(None, "rsync://example.test/repo/b/"); - let action = tables.register_transport_request( - later_identity, - requester("b"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/repo/b/".to_string(), - Some("rsync://example.test/".to_string()), - SyncPreference::RsyncOnly, - ); - assert!(matches!(action, TransportRequestAction::Enqueue(_))); - } - - #[test] - fn register_transport_request_skips_rrdp_when_sync_preference_is_rsync_only() { - let mut tables = TransportStateTables::new(); - let identity = RepoIdentity::new( - Some("https://example.test/notify.xml".to_string()), - "rsync://example.test/repo/", - ); - let action = tables.register_transport_request( - identity.clone(), - requester("root"), - time::OffsetDateTime::UNIX_EPOCH, - 0, - "rsync://example.test/module/".to_string(), - None, - SyncPreference::RsyncOnly, - ); - let task = match action { - TransportRequestAction::Enqueue(task) => task, - other => panic!("expected enqueue, got {other:?}"), - }; - assert_eq!(task.mode, RepoTransportMode::Rsync); - assert_eq!( - task.dedup_key, - RepoDedupKey::RsyncScope { - rsync_scope_uri: "rsync://example.test/module/".to_string() - } - ); - assert!(tables.rrdp_inflight.is_empty()); - assert_eq!( - tables.runtime_records.get(&identity).unwrap().state, - RepoRuntimeState::WaitingRsync - ); - } -} +include!("repo_scheduler/transport_state.rs"); +include!("repo_scheduler/repo_state.rs"); +include!("repo_scheduler/tests_repo.rs"); +include!("repo_scheduler/tests_transport.rs"); diff --git a/crates/panda-rpki-validator/src/parallel/repo_scheduler/repo_state.rs b/crates/panda-rpki-validator/src/parallel/repo_scheduler/repo_state.rs new file mode 100644 index 0000000..6e45f2c --- /dev/null +++ b/crates/panda-rpki-validator/src/parallel/repo_scheduler/repo_state.rs @@ -0,0 +1,199 @@ +// Reusable transport failures and repository in-flight state. + +fn reusable_rsync_failure_scope(result: &RepoTransportResultEnvelope) -> Option<&str> { + let scope = result.rsync_failure_scope_uri.as_deref()?; + match (&result.mode, &result.result) { + (RepoTransportMode::Rsync, RepoTransportResultKind::Failed { detail, .. }) + if is_host_level_rsync_failure(detail) => + { + Some(scope) + } + _ => None, + } +} + +pub(crate) fn is_host_level_rsync_failure(detail: &str) -> bool { + let lower = detail.to_ascii_lowercase(); + lower.contains("timeout waiting for daemon connection") + || lower.contains("failed to connect") + || lower.contains("no route to host") + || lower.contains("network is unreachable") + || lower.contains("connection refused") + || lower.contains("name or service not known") + || lower.contains("temporary failure in name resolution") +} + +#[derive(Default)] +pub struct InFlightRepoTable { + entries: HashMap, +} + +impl InFlightRepoTable { + pub fn new() -> Self { + Self::default() + } + + pub fn len(&self) -> usize { + self.entries.len() + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + pub fn reset_run_state(&mut self) { + self.entries.clear(); + } + + pub fn get(&self, key: &RepoKey) -> Option<&InFlightRepoEntry> { + self.entries.get(key) + } + + pub fn last_result(&self, key: &RepoKey) -> Option<&RepoSyncResultEnvelope> { + self.entries + .get(key) + .and_then(|entry| entry.last_result.as_ref()) + } + + pub fn register_request( + &mut self, + repo_key: RepoKey, + requester: RepoRequester, + validation_time: time::OffsetDateTime, + sync_preference: SyncPreference, + priority: u8, + ) -> RepoRequestAction { + match self.entries.get_mut(&repo_key) { + Some(entry) => match entry.state { + RepoTaskState::Pending | RepoTaskState::Running => { + entry.waiting_requesters.push(requester); + RepoRequestAction::Waiting + } + RepoTaskState::Succeeded | RepoTaskState::Reused => { + RepoRequestAction::Reused(entry.result_ref.clone().expect("result_ref exists")) + } + RepoTaskState::Failed => RepoRequestAction::FailedReuse { + detail: entry + .last_error + .clone() + .unwrap_or_else(|| "repo sync failed".to_string()), + }, + }, + None => { + let task = RepoSyncTask { + repo_key: repo_key.clone(), + validation_time, + sync_preference, + tal_id: requester.tal_id.clone(), + rir_id: requester.rir_id.clone(), + priority, + requesters: vec![requester], + }; + self.entries.insert( + repo_key, + InFlightRepoEntry { + state: RepoTaskState::Pending, + task_ref: Some(task.clone()), + waiting_requesters: Vec::new(), + result_ref: None, + last_result: None, + last_error: None, + started_at: None, + finished_at: None, + }, + ); + RepoRequestAction::Enqueued(task) + } + } + } + + pub fn mark_running( + &mut self, + repo_key: &RepoKey, + started_at: time::OffsetDateTime, + ) -> Result<(), String> { + let entry = self + .entries + .get_mut(repo_key) + .ok_or_else(|| format!("repo not found: {}", repo_key.rsync_base_uri))?; + match entry.state { + RepoTaskState::Pending => { + entry.state = RepoTaskState::Running; + entry.started_at = Some(started_at); + Ok(()) + } + other => Err(format!("repo cannot transition to running from {other:?}")), + } + } + + pub fn complete_success( + &mut self, + repo_key: &RepoKey, + result: RepoSyncResultEnvelope, + finished_at: time::OffsetDateTime, + ) -> Result { + let entry = self + .entries + .get_mut(repo_key) + .ok_or_else(|| format!("repo not found: {}", repo_key.rsync_base_uri))?; + match entry.state { + RepoTaskState::Pending | RepoTaskState::Running => { + let result_ref = match &result.result { + RepoSyncResultKind::Success(result_ref) + | RepoSyncResultKind::Reused(result_ref) => result_ref.clone(), + RepoSyncResultKind::Failed { detail } => { + return Err(format!( + "success completion called with failure result: {detail}" + )); + } + }; + entry.state = RepoTaskState::Succeeded; + entry.result_ref = Some(result_ref); + entry.last_result = Some(result); + entry.last_error = None; + entry.finished_at = Some(finished_at); + entry.task_ref = None; + let released_requesters = std::mem::take(&mut entry.waiting_requesters); + Ok(RepoCompletion { + repo_key: repo_key.clone(), + released_requesters, + }) + } + other => Err(format!("repo cannot transition to success from {other:?}")), + } + } + + pub fn complete_failure( + &mut self, + repo_key: &RepoKey, + result: RepoSyncResultEnvelope, + finished_at: time::OffsetDateTime, + ) -> Result { + let entry = self + .entries + .get_mut(repo_key) + .ok_or_else(|| format!("repo not found: {}", repo_key.rsync_base_uri))?; + match entry.state { + RepoTaskState::Pending | RepoTaskState::Running => { + let detail = match &result.result { + RepoSyncResultKind::Failed { detail } => detail.clone(), + RepoSyncResultKind::Success(_) | RepoSyncResultKind::Reused(_) => { + return Err("failure completion called with success result".to_string()); + } + }; + entry.state = RepoTaskState::Failed; + entry.result_ref = None; + entry.last_result = Some(result); + entry.last_error = Some(detail); + entry.finished_at = Some(finished_at); + entry.task_ref = None; + let released_requesters = std::mem::take(&mut entry.waiting_requesters); + Ok(RepoCompletion { + repo_key: repo_key.clone(), + released_requesters, + }) + } + other => Err(format!("repo cannot transition to failure from {other:?}")), + } + } +} diff --git a/crates/panda-rpki-validator/src/parallel/repo_scheduler/tests_repo.rs b/crates/panda-rpki-validator/src/parallel/repo_scheduler/tests_repo.rs new file mode 100644 index 0000000..7880b93 --- /dev/null +++ b/crates/panda-rpki-validator/src/parallel/repo_scheduler/tests_repo.rs @@ -0,0 +1,231 @@ +// Repository scheduler state transition tests. + +#[cfg(test)] +mod tests { + use crate::parallel::repo_scheduler::{InFlightRepoTable, RepoRequestAction}; + use crate::parallel::types::{ + RepoKey, RepoRequester, RepoSyncResultEnvelope, RepoSyncResultKind, RepoSyncResultRef, + RepoTaskState, + }; + use crate::policy::SyncPreference; + + fn requester(tal_id: &str, rir_id: &str, manifest: &str) -> RepoRequester { + RepoRequester { + tal_id: tal_id.to_string(), + rir_id: rir_id.to_string(), + parent_node_id: None, + ca_instance_handle_id: format!("{tal_id}:{manifest}"), + publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), + manifest_rsync_uri: manifest.to_string(), + } + } + + #[test] + fn inflight_repo_table_stores_entries_by_repo_key() { + let key = RepoKey::new("rsync://example.test/repo/", None); + let mut table = InFlightRepoTable::new(); + let action = table.register_request( + key.clone(), + requester("arin", "arin", "rsync://example.test/repo/root.mft"), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + assert!(matches!(action, RepoRequestAction::Enqueued(_))); + assert_eq!(table.len(), 1); + assert_eq!( + table.get(&key).map(|entry| entry.state), + Some(RepoTaskState::Pending) + ); + } + + #[test] + fn running_repo_request_is_merged_into_waiting_list() { + let key = RepoKey::new("rsync://example.test/repo/", None); + let mut table = InFlightRepoTable::new(); + let _ = table.register_request( + key.clone(), + requester("arin", "arin", "rsync://example.test/repo/root.mft"), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + table + .mark_running(&key, time::OffsetDateTime::UNIX_EPOCH) + .expect("mark running"); + + let action = table.register_request( + key.clone(), + requester("apnic", "apnic", "rsync://example.test/repo/other.mft"), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + assert_eq!(action, RepoRequestAction::Waiting); + assert_eq!(table.get(&key).unwrap().waiting_requesters.len(), 1); + } + + #[test] + fn complete_success_releases_waiting_requesters_and_reuses_result() { + let key = RepoKey::new("rsync://example.test/repo/", None); + let mut table = InFlightRepoTable::new(); + let _ = table.register_request( + key.clone(), + requester("arin", "arin", "rsync://example.test/repo/root.mft"), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + table + .mark_running(&key, time::OffsetDateTime::UNIX_EPOCH) + .expect("mark running"); + let waiting = requester("ripe", "ripe", "rsync://example.test/repo/child.mft"); + let _ = table.register_request( + key.clone(), + waiting.clone(), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + + let result_ref = RepoSyncResultRef { + repo_key: key.clone(), + source: "rrdp".to_string(), + }; + let completion = table + .complete_success( + &key, + RepoSyncResultEnvelope { + repo_key: key.clone(), + tal_id: "arin".to_string(), + rir_id: "arin".to_string(), + result: RepoSyncResultKind::Success(result_ref.clone()), + phase: Some("rrdp_ok".to_string()), + timing_ms: 10, + warnings: Vec::new(), + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete success"); + assert_eq!(completion.released_requesters, vec![waiting]); + assert_eq!(table.get(&key).unwrap().state, RepoTaskState::Succeeded); + + let action = table.register_request( + key, + requester("afrinic", "afrinic", "rsync://example.test/repo/again.mft"), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + assert_eq!(action, RepoRequestAction::Reused(result_ref)); + } + + #[test] + fn complete_failure_releases_waiting_requesters_and_reuses_failure() { + let key = RepoKey::new("rsync://example.test/repo/", None); + let mut table = InFlightRepoTable::new(); + let _ = table.register_request( + key.clone(), + requester("arin", "arin", "rsync://example.test/repo/root.mft"), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + table + .mark_running(&key, time::OffsetDateTime::UNIX_EPOCH) + .expect("mark running"); + let waiting = requester("ripe", "ripe", "rsync://example.test/repo/child.mft"); + let _ = table.register_request( + key.clone(), + waiting.clone(), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + + let completion = table + .complete_failure( + &key, + RepoSyncResultEnvelope { + repo_key: key.clone(), + tal_id: "arin".to_string(), + rir_id: "arin".to_string(), + result: RepoSyncResultKind::Failed { + detail: "network timeout".to_string(), + }, + phase: Some("rrdp_failed_rsync_failed".to_string()), + timing_ms: 10, + warnings: Vec::new(), + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete failure"); + assert_eq!(completion.released_requesters, vec![waiting]); + assert_eq!(table.get(&key).unwrap().state, RepoTaskState::Failed); + + let action = table.register_request( + key, + requester("afrinic", "afrinic", "rsync://example.test/repo/again.mft"), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + assert_eq!( + action, + RepoRequestAction::FailedReuse { + detail: "network timeout".to_string() + } + ); + } + + #[test] + fn requesters_from_different_tals_do_not_lose_identity() { + let key = RepoKey::new("rsync://shared.example/repo/", None); + let mut table = InFlightRepoTable::new(); + let _ = table.register_request( + key.clone(), + requester("arin", "arin", "rsync://shared.example/repo/a.mft"), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + table + .mark_running(&key, time::OffsetDateTime::UNIX_EPOCH) + .expect("mark running"); + let wait_apnic = requester("apnic", "apnic", "rsync://shared.example/repo/b.mft"); + let wait_ripe = requester("ripe", "ripe", "rsync://shared.example/repo/c.mft"); + let _ = table.register_request( + key.clone(), + wait_apnic.clone(), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + let _ = table.register_request( + key.clone(), + wait_ripe.clone(), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + let completion = table + .complete_success( + &key, + RepoSyncResultEnvelope { + repo_key: key.clone(), + tal_id: "arin".to_string(), + rir_id: "arin".to_string(), + result: RepoSyncResultKind::Success(RepoSyncResultRef { + repo_key: key.clone(), + source: "rrdp".to_string(), + }), + phase: Some("rrdp_ok".to_string()), + timing_ms: 10, + warnings: Vec::new(), + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete success"); + assert_eq!(completion.released_requesters, vec![wait_apnic, wait_ripe]); + } +} diff --git a/crates/panda-rpki-validator/src/parallel/repo_scheduler/tests_transport.rs b/crates/panda-rpki-validator/src/parallel/repo_scheduler/tests_transport.rs new file mode 100644 index 0000000..f15c6b8 --- /dev/null +++ b/crates/panda-rpki-validator/src/parallel/repo_scheduler/tests_transport.rs @@ -0,0 +1,864 @@ +// Transport scheduler deduplication and fallback tests. + +#[cfg(test)] +mod transport_tests { + use crate::parallel::repo_scheduler::{ + TransportRequestAction, TransportStateTables, TransportTaskState, + }; + use crate::parallel::types::{ + RepoDedupKey, RepoIdentity, RepoRequester, RepoRuntimeState, RepoTransportMode, + RepoTransportResultEnvelope, RepoTransportResultKind, + }; + use crate::policy::SyncPreference; + + fn requester(id: &str) -> RepoRequester { + RepoRequester::with_tal_rir( + "apnic", + "apnic", + format!("rsync://example.test/repo/{id}.mft"), + "rsync://example.test/repo/".to_string(), + format!("node:{id}"), + ) + } + + #[test] + fn register_transport_request_enqueues_initial_rrdp_task() { + let mut tables = TransportStateTables::new(); + let identity = RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/", + ); + let action = tables.register_transport_request( + identity.clone(), + requester("root"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RrdpThenRsync, + ); + let task = match action { + TransportRequestAction::Enqueue(task) => task, + other => panic!("expected enqueue, got {other:?}"), + }; + assert_eq!(task.mode, RepoTransportMode::Rrdp); + assert_eq!( + task.dedup_key, + RepoDedupKey::RrdpNotify { + notification_uri: "https://example.test/notify.xml".to_string() + } + ); + let key = "https://example.test/notify.xml".to_string(); + assert_eq!( + tables.rrdp_inflight.get(&key).unwrap().state, + TransportTaskState::Pending + ); + assert_eq!( + tables.runtime_records.get(&identity).unwrap().state, + RepoRuntimeState::WaitingRrdp + ); + } + + #[test] + fn register_transport_request_waits_on_existing_rrdp_task() { + let mut tables = TransportStateTables::new(); + let identity = RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/a/", + ); + let _ = tables.register_transport_request( + identity.clone(), + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RrdpThenRsync, + ); + let other_identity = RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/b/", + ); + let action = tables.register_transport_request( + other_identity.clone(), + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RrdpThenRsync, + ); + assert_eq!( + action, + TransportRequestAction::Waiting { + state: RepoRuntimeState::WaitingRrdp + } + ); + assert_eq!( + tables.runtime_records.get(&other_identity).unwrap().state, + RepoRuntimeState::WaitingRrdp + ); + assert_eq!( + tables + .rrdp_inflight + .get("https://example.test/notify.xml") + .unwrap() + .waiting_requesters + .len(), + 1 + ); + } + + #[test] + fn complete_rrdp_success_reuses_for_later_identity_requests() { + let mut tables = TransportStateTables::new(); + let identity = RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/a/", + ); + let _ = tables.register_transport_request( + identity.clone(), + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RrdpThenRsync, + ); + tables + .mark_transport_running( + &RepoDedupKey::RrdpNotify { + notification_uri: "https://example.test/notify.xml".to_string(), + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("mark running"); + let completion = tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: RepoDedupKey::RrdpNotify { + notification_uri: "https://example.test/notify.xml".to_string(), + }, + rsync_failure_scope_uri: None, + repo_identity: identity.clone(), + mode: RepoTransportMode::Rrdp, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 10, + result: RepoTransportResultKind::Success { + source: "rrdp".to_string(), + warnings: Vec::new(), + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete success"); + assert!(completion.follow_up_tasks.is_empty()); + let later_identity = RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/b/", + ); + let action = tables.register_transport_request( + later_identity.clone(), + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RrdpThenRsync, + ); + assert!(matches!(action, TransportRequestAction::ReusedSuccess(_))); + assert_eq!( + tables.runtime_records.get(&later_identity).unwrap().state, + RepoRuntimeState::RrdpOk + ); + + let same_identity_action = tables.register_transport_request( + later_identity.clone(), + requester("b-again"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RrdpThenRsync, + ); + assert!(matches!( + same_identity_action, + TransportRequestAction::ReusedSuccess(_) + )); + + let finalized = tables.finalized_runtime_records_for_transport(&RepoDedupKey::RrdpNotify { + notification_uri: "https://example.test/notify.xml".to_string(), + }); + assert_eq!(finalized.len(), 2); + } + + #[test] + fn complete_rrdp_failure_enqueues_rsync_follow_up() { + let mut tables = TransportStateTables::new(); + let identity = RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/a/", + ); + let _ = tables.register_transport_request( + identity.clone(), + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RrdpThenRsync, + ); + tables + .mark_transport_running( + &RepoDedupKey::RrdpNotify { + notification_uri: "https://example.test/notify.xml".to_string(), + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("mark running"); + let completion = tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: RepoDedupKey::RrdpNotify { + notification_uri: "https://example.test/notify.xml".to_string(), + }, + rsync_failure_scope_uri: None, + repo_identity: identity.clone(), + mode: RepoTransportMode::Rrdp, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 10, + result: RepoTransportResultKind::Failed { + detail: "rrdp timeout".to_string(), + warnings: Vec::new(), + error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete failure"); + assert_eq!(completion.follow_up_tasks.len(), 1); + assert_eq!(completion.follow_up_tasks[0].mode, RepoTransportMode::Rsync); + assert_eq!( + tables.runtime_records.get(&identity).unwrap().state, + RepoRuntimeState::WaitingRsync + ); + + let same_identity_action = tables.register_transport_request( + identity.clone(), + requester("a-again"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RrdpThenRsync, + ); + assert_eq!( + same_identity_action, + TransportRequestAction::Waiting { + state: RepoRuntimeState::WaitingRsync + } + ); + + let later_identity = RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/b/", + ); + let later_action = tables.register_transport_request( + later_identity.clone(), + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RrdpThenRsync, + ); + assert_eq!( + later_action, + TransportRequestAction::Waiting { + state: RepoRuntimeState::WaitingRsync + } + ); + assert_eq!( + tables.runtime_records.get(&later_identity).unwrap().state, + RepoRuntimeState::WaitingRsync + ); + } + + #[test] + fn host_failure_scope_probes_once_then_reuses_terminal_failure() { + let mut tables = TransportStateTables::new(); + let identity_a = RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/a/", + ); + let identity_b = RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/b/", + ); + let _ = tables.register_transport_request( + identity_a.clone(), + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/a/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RrdpThenRsync, + ); + let _ = tables.register_transport_request( + identity_b.clone(), + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/b/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RrdpThenRsync, + ); + tables + .mark_transport_running( + &RepoDedupKey::RrdpNotify { + notification_uri: "https://example.test/notify.xml".to_string(), + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("mark running"); + let completion = tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: RepoDedupKey::RrdpNotify { + notification_uri: "https://example.test/notify.xml".to_string(), + }, + rsync_failure_scope_uri: None, + repo_identity: identity_a.clone(), + mode: RepoTransportMode::Rrdp, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 10, + result: RepoTransportResultKind::Failed { + detail: "rrdp timeout".to_string(), + warnings: Vec::new(), + error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete rrdp failure"); + assert_eq!(completion.follow_up_tasks.len(), 1); + assert_eq!( + completion.follow_up_tasks[0] + .rsync_failure_scope_uri + .as_deref(), + Some("rsync://example.test/") + ); + let probe_task = completion.follow_up_tasks[0].clone(); + + tables + .mark_transport_running(&probe_task.dedup_key, time::OffsetDateTime::UNIX_EPOCH) + .expect("mark rsync running"); + let completion = tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: probe_task.dedup_key, + rsync_failure_scope_uri: probe_task.rsync_failure_scope_uri, + repo_identity: probe_task.repo_identity, + mode: RepoTransportMode::Rsync, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 15_000, + result: RepoTransportResultKind::Failed { + detail: "rsync error: timeout waiting for daemon connection".to_string(), + warnings: Vec::new(), + error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete rsync failure"); + assert!(completion.follow_up_tasks.is_empty()); + assert_eq!( + tables.runtime_records.get(&identity_a).unwrap().state, + RepoRuntimeState::FailedTerminal + ); + assert_eq!( + tables.runtime_records.get(&identity_b).unwrap().state, + RepoRuntimeState::FailedTerminal + ); + let finalized = tables.finalized_runtime_records_for_transport_result( + tables + .runtime_records + .get(&identity_a) + .unwrap() + .terminal_failure + .as_ref() + .unwrap(), + ); + assert_eq!(finalized.len(), 2); + } + + #[test] + fn complete_rsync_failure_reuses_terminal_failure() { + let mut tables = TransportStateTables::new(); + let identity = RepoIdentity::new(None, "rsync://example.test/repo/a/"); + let _ = tables.register_transport_request( + identity.clone(), + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RsyncOnly, + ); + tables + .mark_transport_running( + &RepoDedupKey::RsyncScope { + rsync_scope_uri: "rsync://example.test/module/".to_string(), + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("mark running"); + let _ = tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: RepoDedupKey::RsyncScope { + rsync_scope_uri: "rsync://example.test/module/".to_string(), + }, + rsync_failure_scope_uri: None, + repo_identity: identity.clone(), + mode: RepoTransportMode::Rsync, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 20, + result: RepoTransportResultKind::Failed { + detail: "rsync timeout".to_string(), + warnings: Vec::new(), + error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete rsync failure"); + let later_identity = RepoIdentity::new(None, "rsync://example.test/repo/b/"); + let action = tables.register_transport_request( + later_identity, + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RsyncOnly, + ); + assert!(matches!( + action, + TransportRequestAction::ReusedTerminalFailure(_) + )); + + let finalized = tables.finalized_runtime_records_for_transport(&RepoDedupKey::RsyncScope { + rsync_scope_uri: "rsync://example.test/module/".to_string(), + }); + assert_eq!(finalized.len(), 2); + + let same_identity_action = tables.register_transport_request( + identity, + requester("a-again"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RsyncOnly, + ); + assert!(matches!( + same_identity_action, + TransportRequestAction::ReusedTerminalFailure(_) + )); + } + + #[test] + fn complete_rsync_success_reuses_for_later_identity_requests() { + let mut tables = TransportStateTables::new(); + let identity = RepoIdentity::new(None, "rsync://example.test/repo/a/"); + let action = tables.register_transport_request( + identity.clone(), + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RsyncOnly, + ); + assert!(matches!(action, TransportRequestAction::Enqueue(_))); + tables + .mark_transport_running( + &RepoDedupKey::RsyncScope { + rsync_scope_uri: "rsync://example.test/module/".to_string(), + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("mark running"); + tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: RepoDedupKey::RsyncScope { + rsync_scope_uri: "rsync://example.test/module/".to_string(), + }, + rsync_failure_scope_uri: None, + repo_identity: identity.clone(), + mode: RepoTransportMode::Rsync, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 20, + result: RepoTransportResultKind::Success { + source: "rsync".to_string(), + warnings: Vec::new(), + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete rsync success"); + + let later_identity = RepoIdentity::new(None, "rsync://example.test/repo/b/"); + let action = tables.register_transport_request( + later_identity.clone(), + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RsyncOnly, + ); + assert!(matches!(action, TransportRequestAction::ReusedSuccess(_))); + assert_eq!( + tables.runtime_records.get(&later_identity).unwrap().state, + RepoRuntimeState::RsyncOk + ); + } + + #[test] + fn register_rsync_request_waits_on_existing_rsync_task() { + let mut tables = TransportStateTables::new(); + let identity_a = RepoIdentity::new(None, "rsync://example.test/repo/a/"); + let identity_b = RepoIdentity::new(None, "rsync://example.test/repo/b/"); + let _ = tables.register_transport_request( + identity_a, + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RsyncOnly, + ); + let action = tables.register_transport_request( + identity_b.clone(), + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RsyncOnly, + ); + assert_eq!( + action, + TransportRequestAction::Waiting { + state: RepoRuntimeState::WaitingRsync + } + ); + assert_eq!( + tables.runtime_records.get(&identity_b).unwrap().state, + RepoRuntimeState::WaitingRsync + ); + assert_eq!( + tables + .rsync_inflight + .get("rsync://example.test/module/") + .unwrap() + .waiting_requesters + .len(), + 1 + ); + } + + #[test] + fn host_failure_scope_success_marks_host_reachable_and_schedules_waiters() { + let mut tables = TransportStateTables::new(); + let identity_a = RepoIdentity::new(None, "rsync://example.test/repo/a/"); + let identity_b = RepoIdentity::new(None, "rsync://example.test/repo/b/"); + let action_a = tables.register_transport_request( + identity_a.clone(), + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/a/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RsyncOnly, + ); + let probe_task = match action_a { + TransportRequestAction::Enqueue(task) => task, + other => panic!("expected first host probe enqueue, got {other:?}"), + }; + let action_b = tables.register_transport_request( + identity_b.clone(), + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/b/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RsyncOnly, + ); + assert_eq!( + action_b, + TransportRequestAction::Waiting { + state: RepoRuntimeState::WaitingRsync + } + ); + + tables + .mark_transport_running(&probe_task.dedup_key, time::OffsetDateTime::UNIX_EPOCH) + .expect("mark rsync running"); + let completion = tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: probe_task.dedup_key, + rsync_failure_scope_uri: probe_task.rsync_failure_scope_uri, + repo_identity: probe_task.repo_identity, + mode: RepoTransportMode::Rsync, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 5, + result: RepoTransportResultKind::Success { + source: "rsync".to_string(), + warnings: Vec::new(), + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete rsync success"); + + assert_eq!( + tables.runtime_records.get(&identity_a).unwrap().state, + RepoRuntimeState::RsyncOk + ); + assert_eq!(completion.follow_up_tasks.len(), 1); + assert_eq!( + completion.follow_up_tasks[0].dedup_key, + RepoDedupKey::RsyncScope { + rsync_scope_uri: "rsync://example.test/repo/b/".to_string() + } + ); + assert_eq!( + tables.runtime_records.get(&identity_b).unwrap().state, + RepoRuntimeState::WaitingRsync + ); + } + + #[test] + fn non_host_level_rsync_failure_does_not_poison_host_scope() { + let mut tables = TransportStateTables::new(); + let identity_a = RepoIdentity::new(None, "rsync://example.test/repo/a/"); + let identity_b = RepoIdentity::new(None, "rsync://example.test/repo/b/"); + let action_a = tables.register_transport_request( + identity_a.clone(), + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/a/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RsyncOnly, + ); + let probe_task = match action_a { + TransportRequestAction::Enqueue(task) => task, + other => panic!("expected first host probe enqueue, got {other:?}"), + }; + let _ = tables.register_transport_request( + identity_b.clone(), + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/b/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RsyncOnly, + ); + + tables + .mark_transport_running(&probe_task.dedup_key, time::OffsetDateTime::UNIX_EPOCH) + .expect("mark rsync running"); + let completion = tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: probe_task.dedup_key, + rsync_failure_scope_uri: probe_task.rsync_failure_scope_uri, + repo_identity: probe_task.repo_identity, + mode: RepoTransportMode::Rsync, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 5, + result: RepoTransportResultKind::Failed { + detail: "rsync file digest mismatch after download".to_string(), + warnings: Vec::new(), + error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete rsync failure"); + + assert_eq!( + tables.runtime_records.get(&identity_a).unwrap().state, + RepoRuntimeState::FailedTerminal + ); + assert_eq!(completion.follow_up_tasks.len(), 1); + assert_eq!( + completion.follow_up_tasks[0].dedup_key, + RepoDedupKey::RsyncScope { + rsync_scope_uri: "rsync://example.test/repo/b/".to_string() + } + ); + } + + #[test] + fn cached_host_level_failure_reuses_for_later_rsync_only_requests() { + let mut tables = TransportStateTables::new(); + let identity_a = RepoIdentity::new(None, "rsync://example.test/repo/a/"); + let action_a = tables.register_transport_request( + identity_a, + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/a/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RsyncOnly, + ); + let probe_task = match action_a { + TransportRequestAction::Enqueue(task) => task, + other => panic!("expected first host probe enqueue, got {other:?}"), + }; + tables + .mark_transport_running(&probe_task.dedup_key, time::OffsetDateTime::UNIX_EPOCH) + .expect("mark rsync running"); + let _ = tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: probe_task.dedup_key, + rsync_failure_scope_uri: probe_task.rsync_failure_scope_uri, + repo_identity: probe_task.repo_identity, + mode: RepoTransportMode::Rsync, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 15_000, + result: RepoTransportResultKind::Failed { + detail: "rsync error: failed to connect to daemon".to_string(), + warnings: Vec::new(), + error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete host failure"); + + let later_identity = RepoIdentity::new(None, "rsync://example.test/repo/c/"); + let action = tables.register_transport_request( + later_identity.clone(), + requester("c"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/c/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RsyncOnly, + ); + assert!(matches!( + action, + TransportRequestAction::ReusedTerminalFailure(_) + )); + assert_eq!( + tables.runtime_records.get(&later_identity).unwrap().state, + RepoRuntimeState::FailedTerminal + ); + } + + #[test] + fn reset_run_state_clears_host_failure_scope_cache() { + let mut tables = TransportStateTables::new(); + let identity = RepoIdentity::new(None, "rsync://example.test/repo/a/"); + let action = tables.register_transport_request( + identity, + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/a/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RsyncOnly, + ); + let probe_task = match action { + TransportRequestAction::Enqueue(task) => task, + other => panic!("expected host probe enqueue, got {other:?}"), + }; + tables + .mark_transport_running(&probe_task.dedup_key, time::OffsetDateTime::UNIX_EPOCH) + .expect("mark rsync running"); + let _ = tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: probe_task.dedup_key, + rsync_failure_scope_uri: probe_task.rsync_failure_scope_uri, + repo_identity: probe_task.repo_identity, + mode: RepoTransportMode::Rsync, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 15_000, + result: RepoTransportResultKind::Failed { + detail: "temporary failure in name resolution".to_string(), + warnings: Vec::new(), + error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete host failure"); + + tables.reset_run_state(); + let later_identity = RepoIdentity::new(None, "rsync://example.test/repo/b/"); + let action = tables.register_transport_request( + later_identity, + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/b/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RsyncOnly, + ); + assert!(matches!(action, TransportRequestAction::Enqueue(_))); + } + + #[test] + fn register_transport_request_skips_rrdp_when_sync_preference_is_rsync_only() { + let mut tables = TransportStateTables::new(); + let identity = RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/", + ); + let action = tables.register_transport_request( + identity.clone(), + requester("root"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RsyncOnly, + ); + let task = match action { + TransportRequestAction::Enqueue(task) => task, + other => panic!("expected enqueue, got {other:?}"), + }; + assert_eq!(task.mode, RepoTransportMode::Rsync); + assert_eq!( + task.dedup_key, + RepoDedupKey::RsyncScope { + rsync_scope_uri: "rsync://example.test/module/".to_string() + } + ); + assert!(tables.rrdp_inflight.is_empty()); + assert_eq!( + tables.runtime_records.get(&identity).unwrap().state, + RepoRuntimeState::WaitingRsync + ); + } +} diff --git a/crates/panda-rpki-validator/src/parallel/repo_scheduler/transport_state.rs b/crates/panda-rpki-validator/src/parallel/repo_scheduler/transport_state.rs new file mode 100644 index 0000000..fbb5ef8 --- /dev/null +++ b/crates/panda-rpki-validator/src/parallel/repo_scheduler/transport_state.rs @@ -0,0 +1,871 @@ +// Transport request state tables and completion transitions. + +impl TransportStateTables { + pub fn new() -> Self { + Self::default() + } + + pub fn set_dead_repo_blacklist(&mut self, blacklist: DeadRepoBlacklist) { + self.dead_repo_blacklist = Some(blacklist); + } + + fn dead_repo_terminal_envelope( + identity: &RepoIdentity, + requesters: &[RepoRequester], + rsync_scope_uri: &str, + rsync_failure_scope_uri: Option, + ) -> RepoTransportResultEnvelope { + let first_requester = requesters + .first() + .expect("blacklist terminal record must keep at least one requester"); + RepoTransportResultEnvelope { + dedup_key: RepoDedupKey::RsyncScope { + rsync_scope_uri: rsync_scope_uri.to_string(), + }, + rsync_failure_scope_uri, + repo_identity: identity.clone(), + mode: RepoTransportMode::Rsync, + tal_id: first_requester.tal_id.clone(), + rir_id: first_requester.rir_id.clone(), + timing_ms: 0, + result: RepoTransportResultKind::Failed { + detail: format!( + "dead repo blacklist: rsync transport persistently unreachable: {}", + identity.rsync_base_uri + ), + warnings: vec![Warning::new(format!( + "dead_repo_blacklist_skip_all: repository {} skipped after repeated transport failures", + identity.rsync_base_uri + )) + .with_context(identity.rsync_base_uri.clone())], + error_class: RepoTransportErrorClass::Unknown, + }, + } + } + + pub fn runtime_record(&self, identity: &RepoIdentity) -> Option<&RepoRuntimeRecord> { + self.runtime_records.get(identity) + } + + pub fn finalized_runtime_records_for_transport( + &self, + dedup_key: &RepoDedupKey, + ) -> Vec { + self.runtime_records + .values() + .filter(|record| match dedup_key { + RepoDedupKey::RrdpNotify { notification_uri } => { + record.rrdp_notification_key.as_deref() == Some(notification_uri.as_str()) + } + RepoDedupKey::RsyncScope { rsync_scope_uri } => { + record.rsync_scope_key == *rsync_scope_uri + } + }) + .filter(|record| { + matches!( + record.state, + RepoRuntimeState::RrdpOk + | RepoRuntimeState::RsyncOk + | RepoRuntimeState::FailedTerminal + ) + }) + .cloned() + .collect() + } + + pub fn finalized_runtime_records_for_transport_result( + &self, + result: &RepoTransportResultEnvelope, + ) -> Vec { + let failure_scope = reusable_rsync_failure_scope(result); + self.runtime_records + .values() + .filter(|record| { + let exact_match = match &result.dedup_key { + RepoDedupKey::RrdpNotify { notification_uri } => { + record.rrdp_notification_key.as_deref() == Some(notification_uri.as_str()) + } + RepoDedupKey::RsyncScope { rsync_scope_uri } => { + record.rsync_scope_key == *rsync_scope_uri + } + }; + let failure_scope_match = failure_scope + .map(|scope| record.rsync_failure_scope_key.as_deref() == Some(scope)) + .unwrap_or(false); + exact_match || failure_scope_match + }) + .filter(|record| { + matches!( + record.state, + RepoRuntimeState::RrdpOk + | RepoRuntimeState::RsyncOk + | RepoRuntimeState::FailedTerminal + ) + }) + .cloned() + .collect() + } + + pub fn reset_run_state(&mut self) { + self.rrdp_inflight.clear(); + self.rsync_inflight.clear(); + self.rsync_failure_by_scope.clear(); + self.rsync_failure_probe_inflight.clear(); + self.rsync_failure_scope_reachable.clear(); + self.runtime_records.clear(); + } + + pub fn register_transport_request( + &mut self, + identity: RepoIdentity, + requester: RepoRequester, + validation_time: time::OffsetDateTime, + priority: u8, + rsync_scope_uri: String, + rsync_failure_scope_uri: Option, + sync_preference: SyncPreference, + ) -> TransportRequestAction { + if let Some(record) = self.runtime_records.get_mut(&identity) { + record.requesters.push(requester.clone()); + return match record.state { + RepoRuntimeState::WaitingRrdp => { + if let Some(key) = record.rrdp_notification_key.as_ref() { + if let Some(entry) = self.rrdp_inflight.get_mut(key) { + entry.waiting_requesters.push(requester); + } + } + TransportRequestAction::Waiting { + state: RepoRuntimeState::WaitingRrdp, + } + } + RepoRuntimeState::RrdpOk | RepoRuntimeState::RsyncOk => { + TransportRequestAction::ReusedSuccess( + record + .last_success + .clone() + .expect("success state must keep last_success"), + ) + } + RepoRuntimeState::RrdpFailedPendingRsync | RepoRuntimeState::WaitingRsync => { + if let Some(entry) = self.rsync_inflight.get_mut(&record.rsync_scope_key) { + entry.waiting_requesters.push(requester); + } + record.state = RepoRuntimeState::WaitingRsync; + TransportRequestAction::Waiting { + state: RepoRuntimeState::WaitingRsync, + } + } + RepoRuntimeState::FailedTerminal => TransportRequestAction::ReusedTerminalFailure( + record + .terminal_failure + .clone() + .expect("terminal failure must keep last result"), + ), + RepoRuntimeState::Init => TransportRequestAction::Waiting { + state: RepoRuntimeState::Init, + }, + }; + } + + // Dead-repository blacklist fast paths, consulted only for fresh + // identities; existing records already encode their terminal state. + if let Some(blacklist) = self.dead_repo_blacklist.as_ref() { + let rrdp_wanted = sync_preference == SyncPreference::RrdpThenRsync + && identity.notification_uri.is_some(); + let rrdp_dead = rrdp_wanted + && identity + .notification_uri + .as_deref() + .map(|uri| blacklist.is_blacklisted(RepoTransportMode::Rrdp, uri)) + .unwrap_or(false); + let rsync_dead = blacklist + .is_blacklisted(RepoTransportMode::Rsync, identity.rsync_base_uri.as_str()); + if rsync_dead && (!rrdp_wanted || rrdp_dead) { + crate::progress_log::emit( + "dead_repo_blacklist_skip_all", + serde_json::json!({ + "repo_key_notification_uri": identity.notification_uri, + "repo_key_rsync_base_uri": identity.rsync_base_uri, + }), + ); + let envelope = Self::dead_repo_terminal_envelope( + &identity, + std::slice::from_ref(&requester), + &rsync_scope_uri, + rsync_failure_scope_uri.clone(), + ); + self.runtime_records.insert( + identity.clone(), + RepoRuntimeRecord { + identity, + state: RepoRuntimeState::FailedTerminal, + rrdp_notification_key: None, + rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri, + requesters: vec![requester], + validation_time, + priority, + last_success: None, + terminal_failure: Some(envelope.clone()), + }, + ); + return TransportRequestAction::ReusedTerminalFailure(envelope); + } + if rrdp_dead { + crate::progress_log::emit( + "dead_repo_blacklist_skip_rrdp", + serde_json::json!({ + "repo_key_notification_uri": identity.notification_uri, + "repo_key_rsync_base_uri": identity.rsync_base_uri, + }), + ); + let mut action = self.register_rsync_request( + identity, + requester, + validation_time, + priority, + rsync_scope_uri, + rsync_failure_scope_uri, + ); + // A blacklisted rrdp transport is a known-persistent failure, + // so the rsync fallback must keep the short retry profile + // The prefetch layer records no real rrdp failure in + // skipped runs, which would otherwise silently drop the task + // back to the default 15s window from the second skipped run + // on. The coordinator wrapper only ever sets the flag to true, + // so setting it here survives registration. + if let TransportRequestAction::Enqueue(task) = &mut action { + task.retry_short_timeout = true; + } + return action; + } + } + + if sync_preference == SyncPreference::RrdpThenRsync { + if let Some(notification_uri) = identity.notification_uri.clone() { + if let Some(entry) = self.rrdp_inflight.get_mut(¬ification_uri) { + if let Some(result) = entry.last_result.clone() { + return match result.result { + RepoTransportResultKind::Success { .. } => { + self.runtime_records.insert( + identity.clone(), + RepoRuntimeRecord { + identity, + state: RepoRuntimeState::RrdpOk, + rrdp_notification_key: Some(notification_uri), + rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), + requesters: vec![requester], + validation_time, + priority, + last_success: Some(result.clone()), + terminal_failure: None, + }, + ); + TransportRequestAction::ReusedSuccess(result) + } + RepoTransportResultKind::Failed { .. } => { + self.runtime_records.insert( + identity.clone(), + RepoRuntimeRecord { + identity: identity.clone(), + state: RepoRuntimeState::RrdpFailedPendingRsync, + rrdp_notification_key: Some(notification_uri), + rsync_scope_key: rsync_scope_uri.clone(), + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), + requesters: vec![requester.clone()], + validation_time, + priority, + last_success: None, + terminal_failure: None, + }, + ); + self.register_rsync_request( + identity, + requester, + validation_time, + priority, + rsync_scope_uri, + rsync_failure_scope_uri, + ) + } + }; + } + + entry.waiting_requesters.push(requester.clone()); + self.runtime_records.insert( + identity.clone(), + RepoRuntimeRecord { + identity, + state: RepoRuntimeState::WaitingRrdp, + rrdp_notification_key: Some(notification_uri), + rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), + requesters: vec![requester], + validation_time, + priority, + last_success: None, + terminal_failure: None, + }, + ); + return TransportRequestAction::Waiting { + state: RepoRuntimeState::WaitingRrdp, + }; + } + + let task = RepoTransportTask { + dedup_key: RepoDedupKey::RrdpNotify { + notification_uri: notification_uri.clone(), + }, + rsync_failure_scope_uri: None, + repo_identity: identity.clone(), + mode: RepoTransportMode::Rrdp, + retry_short_timeout: false, + tal_id: requester.tal_id.clone(), + rir_id: requester.rir_id.clone(), + validation_time, + priority, + requesters: vec![requester.clone()], + }; + self.rrdp_inflight.insert( + notification_uri.clone(), + TransportInFlightEntry { + state: TransportTaskState::Pending, + task: task.clone(), + waiting_requesters: Vec::new(), + last_result: None, + started_at: None, + finished_at: None, + }, + ); + self.runtime_records.insert( + identity.clone(), + RepoRuntimeRecord { + identity, + state: RepoRuntimeState::WaitingRrdp, + rrdp_notification_key: Some(notification_uri), + rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), + requesters: vec![requester], + validation_time, + priority, + last_success: None, + terminal_failure: None, + }, + ); + return TransportRequestAction::Enqueue(task); + } + } + + self.register_rsync_request( + identity, + requester, + validation_time, + priority, + rsync_scope_uri, + rsync_failure_scope_uri, + ) + } + + fn register_rsync_request( + &mut self, + identity: RepoIdentity, + requester: RepoRequester, + validation_time: time::OffsetDateTime, + priority: u8, + rsync_scope_uri: String, + rsync_failure_scope_uri: Option, + ) -> TransportRequestAction { + if let Some(entry) = self.rsync_inflight.get_mut(&rsync_scope_uri) { + if let Some(result) = entry.last_result.clone() { + return match result.result { + RepoTransportResultKind::Success { .. } => { + self.runtime_records.insert( + identity.clone(), + RepoRuntimeRecord { + identity, + state: RepoRuntimeState::RsyncOk, + rrdp_notification_key: None, + rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), + requesters: vec![requester], + validation_time, + priority, + last_success: Some(result.clone()), + terminal_failure: None, + }, + ); + TransportRequestAction::ReusedSuccess(result) + } + RepoTransportResultKind::Failed { .. } => { + self.runtime_records.insert( + identity.clone(), + RepoRuntimeRecord { + identity, + state: RepoRuntimeState::FailedTerminal, + rrdp_notification_key: None, + rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), + requesters: vec![requester], + validation_time, + priority, + last_success: None, + terminal_failure: Some(result.clone()), + }, + ); + TransportRequestAction::ReusedTerminalFailure(result) + } + }; + } + + entry.waiting_requesters.push(requester.clone()); + self.runtime_records.insert( + identity.clone(), + RepoRuntimeRecord { + identity, + state: RepoRuntimeState::WaitingRsync, + rrdp_notification_key: None, + rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), + requesters: vec![requester], + validation_time, + priority, + last_success: None, + terminal_failure: None, + }, + ); + return TransportRequestAction::Waiting { + state: RepoRuntimeState::WaitingRsync, + }; + } + + if let Some(failure_scope_uri) = rsync_failure_scope_uri.as_ref() { + if let Some(result) = self.rsync_failure_by_scope.get(failure_scope_uri).cloned() { + self.runtime_records.insert( + identity.clone(), + RepoRuntimeRecord { + identity, + state: RepoRuntimeState::FailedTerminal, + rrdp_notification_key: None, + rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), + requesters: vec![requester], + validation_time, + priority, + last_success: None, + terminal_failure: Some(result.clone()), + }, + ); + return TransportRequestAction::ReusedTerminalFailure(result); + } + + if !self + .rsync_failure_scope_reachable + .contains(failure_scope_uri) + { + if self + .rsync_failure_probe_inflight + .get(failure_scope_uri) + .is_some() + { + self.runtime_records.insert( + identity.clone(), + RepoRuntimeRecord { + identity, + state: RepoRuntimeState::WaitingRsync, + rrdp_notification_key: None, + rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), + requesters: vec![requester], + validation_time, + priority, + last_success: None, + terminal_failure: None, + }, + ); + return TransportRequestAction::Waiting { + state: RepoRuntimeState::WaitingRsync, + }; + } + } + } + + let task = RepoTransportTask { + dedup_key: RepoDedupKey::RsyncScope { + rsync_scope_uri: rsync_scope_uri.clone(), + }, + rsync_failure_scope_uri: rsync_failure_scope_uri.clone(), + repo_identity: identity.clone(), + mode: RepoTransportMode::Rsync, + retry_short_timeout: false, + tal_id: requester.tal_id.clone(), + rir_id: requester.rir_id.clone(), + validation_time, + priority, + requesters: vec![requester.clone()], + }; + self.rsync_inflight.insert( + rsync_scope_uri.clone(), + TransportInFlightEntry { + state: TransportTaskState::Pending, + task: task.clone(), + waiting_requesters: Vec::new(), + last_result: None, + started_at: None, + finished_at: None, + }, + ); + if let Some(failure_scope_uri) = rsync_failure_scope_uri.as_ref() { + if !self + .rsync_failure_scope_reachable + .contains(failure_scope_uri) + { + self.rsync_failure_probe_inflight + .insert(failure_scope_uri.clone(), rsync_scope_uri.clone()); + } + } + self.runtime_records.insert( + identity.clone(), + RepoRuntimeRecord { + identity, + state: RepoRuntimeState::WaitingRsync, + rrdp_notification_key: None, + rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), + requesters: vec![requester], + validation_time, + priority, + last_success: None, + terminal_failure: None, + }, + ); + TransportRequestAction::Enqueue(task) + } + + fn schedule_rsync_for_record( + record: &mut RepoRuntimeRecord, + rsync_inflight: &mut HashMap, + rsync_failure_by_scope: &HashMap, + rsync_failure_probe_inflight: &mut HashMap, + rsync_failure_scope_reachable: &HashSet, + dead_repo_blacklist: Option<&DeadRepoBlacklist>, + follow_up_tasks: &mut Vec, + ) { + let rsync_scope_uri = record.rsync_scope_key.clone(); + if let Some(entry) = rsync_inflight.get_mut(&rsync_scope_uri) { + if let Some(result) = entry.last_result.clone() { + match result.result { + RepoTransportResultKind::Success { .. } => { + record.state = RepoRuntimeState::RsyncOk; + record.last_success = Some(result); + } + RepoTransportResultKind::Failed { .. } => { + record.state = RepoRuntimeState::FailedTerminal; + record.terminal_failure = Some(result); + } + } + return; + } + entry.waiting_requesters.extend(record.requesters.clone()); + record.state = RepoRuntimeState::WaitingRsync; + return; + } + + if let Some(failure_scope_uri) = record.rsync_failure_scope_key.as_ref() { + if let Some(result) = rsync_failure_by_scope.get(failure_scope_uri).cloned() { + record.state = RepoRuntimeState::FailedTerminal; + record.terminal_failure = Some(result); + return; + } + if !rsync_failure_scope_reachable.contains(failure_scope_uri) + && rsync_failure_probe_inflight.contains_key(failure_scope_uri) + { + record.state = RepoRuntimeState::WaitingRsync; + return; + } + } + + // The rsync transport of this repository is + // persistently unreachable; terminate instantly instead of enqueueing + // another doomed fetch (rrdp was already attempted or skipped). + if let Some(blacklist) = dead_repo_blacklist { + if blacklist.is_blacklisted( + RepoTransportMode::Rsync, + record.identity.rsync_base_uri.as_str(), + ) { + crate::progress_log::emit( + "dead_repo_blacklist_skip_all", + serde_json::json!({ + "repo_key_notification_uri": record.identity.notification_uri, + "repo_key_rsync_base_uri": record.identity.rsync_base_uri, + "after_rrdp_failure": true, + }), + ); + let envelope = Self::dead_repo_terminal_envelope( + &record.identity, + &record.requesters, + &rsync_scope_uri, + record.rsync_failure_scope_key.clone(), + ); + record.state = RepoRuntimeState::FailedTerminal; + record.terminal_failure = Some(envelope); + return; + } + } + + let first_requester = record + .requesters + .first() + .expect("rsync record must keep at least one requester"); + let task = RepoTransportTask { + dedup_key: RepoDedupKey::RsyncScope { + rsync_scope_uri: rsync_scope_uri.clone(), + }, + rsync_failure_scope_uri: record.rsync_failure_scope_key.clone(), + repo_identity: record.identity.clone(), + mode: RepoTransportMode::Rsync, + retry_short_timeout: false, + tal_id: first_requester.tal_id.clone(), + rir_id: first_requester.rir_id.clone(), + validation_time: record.validation_time, + priority: record.priority, + requesters: record.requesters.clone(), + }; + rsync_inflight.insert( + rsync_scope_uri.clone(), + TransportInFlightEntry { + state: TransportTaskState::Pending, + task: task.clone(), + waiting_requesters: Vec::new(), + last_result: None, + started_at: None, + finished_at: None, + }, + ); + if let Some(failure_scope_uri) = record.rsync_failure_scope_key.as_ref() { + if !rsync_failure_scope_reachable.contains(failure_scope_uri) { + rsync_failure_probe_inflight.insert(failure_scope_uri.clone(), rsync_scope_uri); + } + } + record.state = RepoRuntimeState::WaitingRsync; + follow_up_tasks.push(task); + } + + pub fn mark_transport_running( + &mut self, + dedup_key: &RepoDedupKey, + started_at: time::OffsetDateTime, + ) -> Result<(), String> { + match dedup_key { + RepoDedupKey::RrdpNotify { notification_uri } => { + let entry = self + .rrdp_inflight + .get_mut(notification_uri) + .ok_or_else(|| format!("rrdp transport not found: {notification_uri}"))?; + entry.state = TransportTaskState::Running; + entry.started_at = Some(started_at); + } + RepoDedupKey::RsyncScope { rsync_scope_uri } => { + let entry = self + .rsync_inflight + .get_mut(rsync_scope_uri) + .ok_or_else(|| format!("rsync transport not found: {rsync_scope_uri}"))?; + entry.state = TransportTaskState::Running; + entry.started_at = Some(started_at); + } + } + Ok(()) + } + + pub fn complete_transport_result( + &mut self, + result: RepoTransportResultEnvelope, + finished_at: time::OffsetDateTime, + ) -> Result { + match (&result.dedup_key, &result.result) { + ( + RepoDedupKey::RrdpNotify { notification_uri }, + RepoTransportResultKind::Success { .. }, + ) => { + let entry = self + .rrdp_inflight + .get_mut(notification_uri) + .ok_or_else(|| format!("rrdp transport not found: {notification_uri}"))?; + entry.state = TransportTaskState::Finished; + entry.finished_at = Some(finished_at); + entry.last_result = Some(result.clone()); + let released_requesters = std::mem::take(&mut entry.waiting_requesters); + for record in self.runtime_records.values_mut() { + if record.rrdp_notification_key.as_deref() == Some(notification_uri) + && record.state == RepoRuntimeState::WaitingRrdp + { + record.state = RepoRuntimeState::RrdpOk; + record.last_success = Some(result.clone()); + } + } + Ok(TransportCompletion { + released_requesters, + follow_up_tasks: Vec::new(), + }) + } + ( + RepoDedupKey::RrdpNotify { notification_uri }, + RepoTransportResultKind::Failed { .. }, + ) => { + let entry = self + .rrdp_inflight + .get_mut(notification_uri) + .ok_or_else(|| format!("rrdp transport not found: {notification_uri}"))?; + entry.state = TransportTaskState::Finished; + entry.finished_at = Some(finished_at); + entry.last_result = Some(result.clone()); + let mut follow_up_tasks = Vec::new(); + for record in self.runtime_records.values_mut() { + if record.rrdp_notification_key.as_deref() == Some(notification_uri) + && record.state == RepoRuntimeState::WaitingRrdp + { + record.state = RepoRuntimeState::RrdpFailedPendingRsync; + Self::schedule_rsync_for_record( + record, + &mut self.rsync_inflight, + &self.rsync_failure_by_scope, + &mut self.rsync_failure_probe_inflight, + &self.rsync_failure_scope_reachable, + self.dead_repo_blacklist.as_ref(), + &mut follow_up_tasks, + ); + } + } + Ok(TransportCompletion { + released_requesters: Vec::new(), + follow_up_tasks, + }) + } + ( + RepoDedupKey::RsyncScope { rsync_scope_uri }, + RepoTransportResultKind::Success { .. }, + ) => { + let mut follow_up_tasks = Vec::new(); + let entry = self + .rsync_inflight + .get_mut(rsync_scope_uri) + .ok_or_else(|| format!("rsync transport not found: {rsync_scope_uri}"))?; + entry.state = TransportTaskState::Finished; + entry.finished_at = Some(finished_at); + entry.last_result = Some(result.clone()); + if let Some(failure_scope_uri) = result.rsync_failure_scope_uri.as_ref() { + self.rsync_failure_probe_inflight.remove(failure_scope_uri); + self.rsync_failure_scope_reachable + .insert(failure_scope_uri.clone()); + } + let released_requesters = std::mem::take(&mut entry.waiting_requesters); + for record in self.runtime_records.values_mut() { + if record.rsync_scope_key == *rsync_scope_uri + && matches!( + record.state, + RepoRuntimeState::WaitingRsync + | RepoRuntimeState::RrdpFailedPendingRsync + ) + { + record.state = RepoRuntimeState::RsyncOk; + record.last_success = Some(result.clone()); + } + } + if let Some(failure_scope_uri) = result.rsync_failure_scope_uri.as_ref() { + for record in self.runtime_records.values_mut() { + if record.rsync_scope_key != *rsync_scope_uri + && record.rsync_failure_scope_key.as_deref() + == Some(failure_scope_uri.as_str()) + && matches!(record.state, RepoRuntimeState::WaitingRsync) + { + Self::schedule_rsync_for_record( + record, + &mut self.rsync_inflight, + &self.rsync_failure_by_scope, + &mut self.rsync_failure_probe_inflight, + &self.rsync_failure_scope_reachable, + self.dead_repo_blacklist.as_ref(), + &mut follow_up_tasks, + ); + } + } + } + Ok(TransportCompletion { + released_requesters, + follow_up_tasks, + }) + } + ( + RepoDedupKey::RsyncScope { rsync_scope_uri }, + RepoTransportResultKind::Failed { .. }, + ) => { + let entry = self + .rsync_inflight + .get_mut(rsync_scope_uri) + .ok_or_else(|| format!("rsync transport not found: {rsync_scope_uri}"))?; + entry.state = TransportTaskState::Finished; + entry.finished_at = Some(finished_at); + entry.last_result = Some(result.clone()); + let reusable_failure_scope = + reusable_rsync_failure_scope(&result).map(str::to_string); + if let Some(failure_scope_uri) = result.rsync_failure_scope_uri.as_ref() { + self.rsync_failure_probe_inflight.remove(failure_scope_uri); + if reusable_failure_scope.as_deref() == Some(failure_scope_uri.as_str()) { + self.rsync_failure_by_scope + .insert(failure_scope_uri.clone(), result.clone()); + } else { + self.rsync_failure_scope_reachable + .insert(failure_scope_uri.clone()); + } + } + let released_requesters = std::mem::take(&mut entry.waiting_requesters); + for record in self.runtime_records.values_mut() { + if (record.rsync_scope_key == *rsync_scope_uri + || reusable_failure_scope + .as_deref() + .map(|scope| record.rsync_failure_scope_key.as_deref() == Some(scope)) + .unwrap_or(false)) + && matches!( + record.state, + RepoRuntimeState::WaitingRsync + | RepoRuntimeState::RrdpFailedPendingRsync + ) + { + record.state = RepoRuntimeState::FailedTerminal; + record.terminal_failure = Some(result.clone()); + } + } + if let Some(failure_scope_uri) = result.rsync_failure_scope_uri.as_ref() { + if reusable_failure_scope.as_deref() != Some(failure_scope_uri.as_str()) { + let mut follow_up_tasks = Vec::new(); + for record in self.runtime_records.values_mut() { + if record.rsync_scope_key != *rsync_scope_uri + && record.rsync_failure_scope_key.as_deref() + == Some(failure_scope_uri.as_str()) + && matches!(record.state, RepoRuntimeState::WaitingRsync) + { + Self::schedule_rsync_for_record( + record, + &mut self.rsync_inflight, + &self.rsync_failure_by_scope, + &mut self.rsync_failure_probe_inflight, + &self.rsync_failure_scope_reachable, + self.dead_repo_blacklist.as_ref(), + &mut follow_up_tasks, + ); + } + } + return Ok(TransportCompletion { + released_requesters, + follow_up_tasks, + }); + } + } + Ok(TransportCompletion { + released_requesters, + follow_up_tasks: Vec::new(), + }) + } + } + } +} diff --git a/crates/panda-rpki-validator/src/parallel/repo_worker.rs b/crates/panda-rpki-validator/src/parallel/repo_worker.rs index ec226ae..36e59f1 100644 --- a/crates/panda-rpki-validator/src/parallel/repo_worker.rs +++ b/crates/panda-rpki-validator/src/parallel/repo_worker.rs @@ -22,1249 +22,9 @@ use crate::sync::repo::{ }; use crate::sync::rrdp::Fetcher; -const RETRY_SHORT_TIMEOUT: Duration = Duration::from_secs(1); - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RepoWorkerPoolConfig { - pub max_workers: usize, -} - -impl From<&ParallelPhase1Config> for RepoWorkerPoolConfig { - fn from(value: &ParallelPhase1Config) -> Self { - Self { - max_workers: value.max_repo_sync_workers_global, - } - } -} - -pub trait RepoSyncExecutor: Send + Sync + 'static { - fn execute(&self, task: RepoSyncTask) -> RepoSyncResultEnvelope; -} - -pub trait RepoTransportExecutor: Send + Sync + 'static { - fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope; -} - -pub struct LiveRrdpTransportExecutor { - store: Arc, - current_repo_index: CurrentRepoIndexHandle, - http_fetcher: Arc, - timing: Option, - download_log: Option, -} - -impl LiveRrdpTransportExecutor { - pub fn new( - store: Arc, - current_repo_index: CurrentRepoIndexHandle, - http_fetcher: Arc, - timing: Option, - download_log: Option, - ) -> Self { - Self { - store, - current_repo_index, - http_fetcher, - timing, - download_log, - } - } -} - -impl RepoTransportExecutor for LiveRrdpTransportExecutor { - fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { - let started = std::time::Instant::now(); - debug_assert_eq!(task.mode, RepoTransportMode::Rrdp); - let notification_uri = task - .repo_identity - .notification_uri - .as_deref() - .expect("rrdp transport requires notification uri"); - let sync_result = if task.retry_short_timeout { - crate::fetch::http::with_scoped_http_timeout_override(RETRY_SHORT_TIMEOUT, || { - run_rrdp_transport( - self.store.as_ref(), - notification_uri, - Some(&self.current_repo_index), - self.http_fetcher.as_ref(), - self.timing.as_ref(), - self.download_log.as_ref(), - ) - }) - } else { - run_rrdp_transport( - self.store.as_ref(), - notification_uri, - Some(&self.current_repo_index), - self.http_fetcher.as_ref(), - self.timing.as_ref(), - self.download_log.as_ref(), - ) - }; - match sync_result { - Ok(_) => RepoTransportResultEnvelope { - dedup_key: task.dedup_key, - rsync_failure_scope_uri: task.rsync_failure_scope_uri, - repo_identity: task.repo_identity, - mode: RepoTransportMode::Rrdp, - tal_id: task.tal_id, - rir_id: task.rir_id, - timing_ms: started.elapsed().as_millis() as u64, - result: RepoTransportResultKind::Success { - source: "rrdp".to_string(), - warnings: Vec::new(), - }, - }, - Err(err) => { - let error_class = - crate::parallel::dead_repo_blacklist::classify_rrdp_sync_error(&err); - RepoTransportResultEnvelope { - dedup_key: task.dedup_key, - rsync_failure_scope_uri: task.rsync_failure_scope_uri, - repo_identity: task.repo_identity, - mode: RepoTransportMode::Rrdp, - tal_id: task.tal_id, - rir_id: task.rir_id, - timing_ms: started.elapsed().as_millis() as u64, - result: RepoTransportResultKind::Failed { - detail: err.to_string(), - warnings: Vec::new(), - error_class, - }, - } - } - } - } -} - -pub struct LiveRsyncTransportExecutor { - store: Arc, - current_repo_index: CurrentRepoIndexHandle, - rsync_fetcher: Arc, - timing: Option, - download_log: Option, -} - -impl LiveRsyncTransportExecutor { - pub fn new( - store: Arc, - current_repo_index: CurrentRepoIndexHandle, - rsync_fetcher: Arc, - timing: Option, - download_log: Option, - ) -> Self { - Self { - store, - current_repo_index, - rsync_fetcher, - timing, - download_log, - } - } -} - -impl RepoTransportExecutor for LiveRsyncTransportExecutor { - fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { - let started = std::time::Instant::now(); - debug_assert_eq!(task.mode, RepoTransportMode::Rsync); - let sync_result = if task.retry_short_timeout { - with_scoped_rsync_timeout_override(RETRY_SHORT_TIMEOUT, || { - with_scoped_rsync_fail_fast_profile( - RsyncFailFastProfile { - initial_wall_clock_timeout: RETRY_SHORT_TIMEOUT, - max_wall_clock_timeout: RETRY_SHORT_TIMEOUT, - max_attempts: 1, - }, - || { - run_rsync_transport( - self.store.as_ref(), - &task.repo_identity.rsync_base_uri, - Some(&self.current_repo_index), - self.rsync_fetcher.as_ref(), - self.timing.as_ref(), - self.download_log.as_ref(), - ) - }, - ) - }) - } else { - run_rsync_transport( - self.store.as_ref(), - &task.repo_identity.rsync_base_uri, - Some(&self.current_repo_index), - self.rsync_fetcher.as_ref(), - self.timing.as_ref(), - self.download_log.as_ref(), - ) - }; - match sync_result { - Ok(_) => RepoTransportResultEnvelope { - dedup_key: task.dedup_key, - rsync_failure_scope_uri: task.rsync_failure_scope_uri, - repo_identity: task.repo_identity, - mode: RepoTransportMode::Rsync, - tal_id: task.tal_id, - rir_id: task.rir_id, - timing_ms: started.elapsed().as_millis() as u64, - result: RepoTransportResultKind::Success { - source: "rsync".to_string(), - warnings: Vec::new(), - }, - }, - Err(err) => { - let error_class = - crate::parallel::dead_repo_blacklist::classify_rsync_repo_sync_error(&err); - RepoTransportResultEnvelope { - dedup_key: task.dedup_key, - rsync_failure_scope_uri: task.rsync_failure_scope_uri, - repo_identity: task.repo_identity, - mode: RepoTransportMode::Rsync, - tal_id: task.tal_id, - rir_id: task.rir_id, - timing_ms: started.elapsed().as_millis() as u64, - result: RepoTransportResultKind::Failed { - detail: err.to_string(), - warnings: Vec::new(), - error_class, - }, - } - } - } - } -} - -pub struct LiveRepoTransportExecutor { - rrdp: LiveRrdpTransportExecutor, - rsync: LiveRsyncTransportExecutor, -} - -impl LiveRepoTransportExecutor { - pub fn new( - store: Arc, - current_repo_index: CurrentRepoIndexHandle, - http_fetcher: Arc, - rsync_fetcher: Arc, - timing: Option, - download_log: Option, - ) -> Self { - Self { - rrdp: LiveRrdpTransportExecutor::new( - Arc::clone(&store), - current_repo_index.clone(), - http_fetcher, - timing.clone(), - download_log.clone(), - ), - rsync: LiveRsyncTransportExecutor::new( - store, - current_repo_index, - rsync_fetcher, - timing, - download_log, - ), - } - } -} - -impl RepoTransportExecutor - for LiveRepoTransportExecutor -{ - fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { - match task.mode { - RepoTransportMode::Rrdp => self.rrdp.execute_transport(task), - RepoTransportMode::Rsync => self.rsync.execute_transport(task), - } - } -} - -pub struct LiveRepoSyncExecutor { - store: Arc, - policy: Policy, - http_fetcher: Arc, - rsync_fetcher: Arc, - timing: Option, - download_log: Option, -} - -impl LiveRepoSyncExecutor { - pub fn new( - store: Arc, - policy: Policy, - http_fetcher: Arc, - rsync_fetcher: Arc, - timing: Option, - download_log: Option, - ) -> Self { - Self { - store, - policy, - http_fetcher, - rsync_fetcher, - timing, - download_log, - } - } -} - -impl RepoSyncExecutor - for LiveRepoSyncExecutor -{ - fn execute(&self, task: RepoSyncTask) -> RepoSyncResultEnvelope { - let started = std::time::Instant::now(); - crate::progress_log::emit( - "phase1_repo_worker_execute_start", - serde_json::json!({ - "repo_key_rsync_base_uri": task.repo_key.rsync_base_uri, - "repo_key_notification_uri": task.repo_key.notification_uri, - "tal_id": task.tal_id, - "rir_id": task.rir_id, - }), - ); - match sync_publication_point( - self.store.as_ref(), - &self.policy, - task.repo_key.notification_uri.as_deref(), - &task.repo_key.rsync_base_uri, - self.http_fetcher.as_ref(), - self.rsync_fetcher.as_ref(), - self.timing.as_ref(), - self.download_log.as_ref(), - ) { - Ok(res) => { - let timing_ms = started.elapsed().as_millis() as u64; - crate::progress_log::emit( - "phase1_repo_worker_execute_finish", - serde_json::json!({ - "repo_key_rsync_base_uri": task.repo_key.rsync_base_uri, - "repo_key_notification_uri": task.repo_key.notification_uri, - "result": "success", - "phase": repo_sync_phase_label(res.phase), - "timing_ms": timing_ms, - }), - ); - RepoSyncResultEnvelope { - repo_key: task.repo_key.clone(), - tal_id: task.tal_id, - rir_id: task.rir_id, - result: super::types::RepoSyncResultKind::Success( - super::types::RepoSyncResultRef { - repo_key: task.repo_key, - source: repo_sync_source_label(res.source).to_string(), - }, - ), - phase: Some(repo_sync_phase_label(res.phase).to_string()), - timing_ms, - warnings: res.warnings, - } - } - Err(err) => { - let timing_ms = started.elapsed().as_millis() as u64; - crate::progress_log::emit( - "phase1_repo_worker_execute_finish", - serde_json::json!({ - "repo_key_rsync_base_uri": task.repo_key.rsync_base_uri, - "repo_key_notification_uri": task.repo_key.notification_uri, - "result": "failed", - "phase": "repo_sync_failed", - "timing_ms": timing_ms, - "error": err.to_string(), - }), - ); - RepoSyncResultEnvelope { - repo_key: task.repo_key, - tal_id: task.tal_id, - rir_id: task.rir_id, - result: super::types::RepoSyncResultKind::Failed { - detail: err.to_string(), - }, - phase: Some("repo_sync_failed".to_string()), - timing_ms, - warnings: Vec::new(), - } - } - } - } -} - -fn repo_sync_source_label(source: RepoSyncSource) -> &'static str { - match source { - RepoSyncSource::Rrdp => "rrdp", - RepoSyncSource::Rsync => "rsync", - } -} - -fn repo_sync_phase_label(phase: crate::sync::repo::RepoSyncPhase) -> &'static str { - match phase { - crate::sync::repo::RepoSyncPhase::RrdpOk => "rrdp_ok", - crate::sync::repo::RepoSyncPhase::RrdpFailedRsyncOk => "rrdp_failed_rsync_ok", - crate::sync::repo::RepoSyncPhase::RsyncOnlyOk => "rsync_only_ok", - crate::sync::repo::RepoSyncPhase::ReplayRrdpOk => "replay_rrdp_ok", - crate::sync::repo::RepoSyncPhase::ReplayRsyncOk => "replay_rsync_ok", - crate::sync::repo::RepoSyncPhase::ReplayNoopRrdp => "replay_noop_rrdp", - crate::sync::repo::RepoSyncPhase::ReplayNoopRsync => "replay_noop_rsync", - } -} - -enum RepoWorkerMessage { - Task(RepoSyncTask), - Shutdown, -} - -enum TransportWorkerMessage { - Task(RepoTransportTask), - Shutdown, -} - -pub struct RepoWorkerPool { - config: RepoWorkerPoolConfig, - task_tx: Sender, - result_rx: Receiver, - workers: Vec>, - _executor: Arc, -} - -pub struct RepoTransportWorkerPool { - config: RepoWorkerPoolConfig, - task_tx: Sender, - result_rx: Receiver, - workers: Vec>, - _executor: Arc, -} - -impl RepoWorkerPool { - pub fn new(config: RepoWorkerPoolConfig, executor: E) -> Result { - if config.max_workers == 0 { - return Err("RepoWorkerPool requires at least one worker".to_string()); - } - - let executor = Arc::new(executor); - let (task_tx, task_rx) = mpsc::channel::(); - let (result_tx, result_rx) = mpsc::channel::(); - let shared_task_rx = Arc::new(Mutex::new(task_rx)); - - let mut workers = Vec::with_capacity(config.max_workers); - for idx in 0..config.max_workers { - let task_rx = Arc::clone(&shared_task_rx); - let result_tx = result_tx.clone(); - let executor = Arc::clone(&executor); - workers.push( - thread::Builder::new() - .name(format!("repo-sync-worker-{idx}")) - .spawn(move || worker_loop(task_rx, result_tx, executor)) - .map_err(|e| format!("spawn repo worker failed: {e}"))?, - ); - } - - Ok(Self { - config, - task_tx, - result_rx, - workers, - _executor: executor, - }) - } - - pub fn worker_count(&self) -> usize { - self.config.max_workers - } - - pub fn submit(&self, task: RepoSyncTask) -> Result<(), String> { - self.task_tx - .send(RepoWorkerMessage::Task(task)) - .map_err(|e| format!("submit repo task failed: {e}")) - } - - pub fn recv_result_timeout( - &self, - timeout: Duration, - ) -> Result, String> { - match self.result_rx.recv_timeout(timeout) { - Ok(msg) => Ok(Some(msg)), - Err(RecvTimeoutError::Timeout) => Ok(None), - Err(RecvTimeoutError::Disconnected) => { - Err("repo worker result channel disconnected".to_string()) - } - } - } - - pub fn shutdown(mut self) -> Result<(), String> { - self.shutdown_inner() - } - - fn shutdown_inner(&mut self) -> Result<(), String> { - if self.workers.is_empty() { - return Ok(()); - } - - for _ in 0..self.workers.len() { - self.task_tx - .send(RepoWorkerMessage::Shutdown) - .map_err(|e| format!("send shutdown to repo worker failed: {e}"))?; - } - - let mut first_err: Option = None; - for handle in self.workers.drain(..) { - if let Err(e) = handle.join() { - if first_err.is_none() { - first_err = Some(format!("join repo worker failed: {e:?}")); - } - } - } - - if let Some(err) = first_err { - return Err(err); - } - Ok(()) - } -} - -impl Drop for RepoWorkerPool { - fn drop(&mut self) { - let _ = self.shutdown_inner(); - } -} - -impl RepoTransportWorkerPool { - pub fn new(config: RepoWorkerPoolConfig, executor: E) -> Result { - if config.max_workers == 0 { - return Err("RepoTransportWorkerPool requires at least one worker".to_string()); - } - let executor = Arc::new(executor); - let (task_tx, task_rx) = mpsc::channel::(); - let (result_tx, result_rx) = mpsc::channel::(); - let shared_task_rx = Arc::new(Mutex::new(task_rx)); - let mut workers = Vec::with_capacity(config.max_workers); - for idx in 0..config.max_workers { - let task_rx = Arc::clone(&shared_task_rx); - let result_tx = result_tx.clone(); - let executor = Arc::clone(&executor); - workers.push( - thread::Builder::new() - .name(format!("repo-transport-worker-{idx}")) - .spawn(move || transport_worker_loop(task_rx, result_tx, executor)) - .map_err(|e| format!("spawn repo transport worker failed: {e}"))?, - ); - } - Ok(Self { - config, - task_tx, - result_rx, - workers, - _executor: executor, - }) - } - - pub fn submit(&self, task: RepoTransportTask) -> Result<(), String> { - self.task_tx - .send(TransportWorkerMessage::Task(task)) - .map_err(|e| format!("submit repo transport task failed: {e}")) - } - - pub fn recv_result_timeout( - &self, - timeout: Duration, - ) -> Result, String> { - match self.result_rx.recv_timeout(timeout) { - Ok(msg) => Ok(Some(msg)), - Err(RecvTimeoutError::Timeout) => Ok(None), - Err(RecvTimeoutError::Disconnected) => { - Err("repo transport worker result channel disconnected".to_string()) - } - } - } - - pub fn worker_count(&self) -> usize { - self.config.max_workers - } - - pub fn shutdown(mut self) -> Result<(), String> { - self.shutdown_inner() - } - - fn shutdown_inner(&mut self) -> Result<(), String> { - if self.workers.is_empty() { - return Ok(()); - } - for _ in 0..self.workers.len() { - self.task_tx - .send(TransportWorkerMessage::Shutdown) - .map_err(|e| format!("send shutdown to repo transport worker failed: {e}"))?; - } - let mut first_err: Option = None; - for handle in self.workers.drain(..) { - if let Err(e) = handle.join() { - if first_err.is_none() { - first_err = Some(format!("join repo transport worker failed: {e:?}")); - } - } - } - if let Some(err) = first_err { - return Err(err); - } - Ok(()) - } -} - -impl Drop for RepoTransportWorkerPool { - fn drop(&mut self) { - let _ = self.shutdown_inner(); - } -} - -fn worker_loop( - task_rx: Arc>>, - result_tx: Sender, - executor: Arc, -) { - loop { - let message = { - let rx = task_rx.lock().expect("repo worker receiver lock poisoned"); - rx.recv() - }; - - match message { - Ok(RepoWorkerMessage::Task(task)) => { - let result = executor.execute(task); - if result_tx.send(result).is_err() { - break; - } - } - Ok(RepoWorkerMessage::Shutdown) | Err(_) => break, - } - } -} - -fn transport_worker_loop( - task_rx: Arc>>, - result_tx: Sender, - executor: Arc, -) { - loop { - let message = { - let rx = task_rx - .lock() - .expect("repo transport worker receiver lock poisoned"); - rx.recv() - }; - - match message { - Ok(TransportWorkerMessage::Task(task)) => { - let result = executor.execute_transport(task); - if result_tx.send(result).is_err() { - break; - } - } - Ok(TransportWorkerMessage::Shutdown) | Err(_) => break, - } - } -} +include!("repo_worker/executors.rs"); +include!("repo_worker/pools.rs"); #[cfg(test)] -mod tests { - use base64::Engine; - use sha2::Digest; - use std::collections::HashMap; - use std::fs; - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::{Arc, Barrier, Mutex}; - use std::time::Duration; - - use super::{ - LiveRepoSyncExecutor, LiveRrdpTransportExecutor, LiveRsyncTransportExecutor, - RepoSyncExecutor, RepoTransportExecutor, RepoTransportWorkerPool, RepoWorkerPool, - RepoWorkerPoolConfig, - }; - use crate::current_repo_index::CurrentRepoIndex; - use crate::fetch::rsync::{ - LocalDirRsyncFetcher, RsyncFetchError, RsyncFetchResult, RsyncFetcher, - }; - use crate::parallel::config::ParallelPhase1Config; - use crate::parallel::types::{ - RepoDedupKey, RepoIdentity, RepoKey, RepoRequester, RepoSyncResultEnvelope, - RepoSyncResultKind, RepoSyncResultRef, RepoSyncTask, RepoTransportMode, - RepoTransportResultEnvelope, RepoTransportResultKind, RepoTransportTask, - }; - use crate::policy::SyncPreference; - use crate::report::Warning; - use crate::storage::RocksStore; - use crate::sync::rrdp::Fetcher; - - fn sample_task(name: &str) -> RepoSyncTask { - RepoSyncTask { - repo_key: RepoKey::new(format!("rsync://example.test/{name}/"), None), - validation_time: time::OffsetDateTime::UNIX_EPOCH, - sync_preference: SyncPreference::RrdpThenRsync, - tal_id: "arin".to_string(), - rir_id: "arin".to_string(), - priority: 0, - requesters: vec![RepoRequester { - tal_id: "arin".to_string(), - rir_id: "arin".to_string(), - parent_node_id: None, - ca_instance_handle_id: format!("node:{name}"), - publication_point_rsync_uri: format!("rsync://example.test/{name}/"), - manifest_rsync_uri: format!("rsync://example.test/{name}/root.mft"), - }], - } - } - - fn sample_rrdp_transport_task( - notification_uri: &str, - rsync_base_uri: &str, - ) -> RepoTransportTask { - RepoTransportTask { - dedup_key: RepoDedupKey::RrdpNotify { - notification_uri: notification_uri.to_string(), - }, - rsync_failure_scope_uri: None, - repo_identity: RepoIdentity::new(Some(notification_uri.to_string()), rsync_base_uri), - mode: RepoTransportMode::Rrdp, - retry_short_timeout: false, - tal_id: "arin".to_string(), - rir_id: "arin".to_string(), - validation_time: time::OffsetDateTime::UNIX_EPOCH, - priority: 0, - requesters: vec![RepoRequester::with_tal_rir( - "arin", - "arin", - format!("{rsync_base_uri}root.mft"), - rsync_base_uri.to_string(), - "node:rrdp", - )], - } - } - - fn sample_rsync_transport_task( - rsync_scope_uri: &str, - rsync_base_uri: &str, - ) -> RepoTransportTask { - RepoTransportTask { - dedup_key: RepoDedupKey::RsyncScope { - rsync_scope_uri: rsync_scope_uri.to_string(), - }, - rsync_failure_scope_uri: None, - repo_identity: RepoIdentity::new(None, rsync_base_uri), - mode: RepoTransportMode::Rsync, - retry_short_timeout: false, - tal_id: "arin".to_string(), - rir_id: "arin".to_string(), - validation_time: time::OffsetDateTime::UNIX_EPOCH, - priority: 0, - requesters: vec![RepoRequester::with_tal_rir( - "arin", - "arin", - format!("{rsync_base_uri}root.mft"), - rsync_base_uri.to_string(), - "node:rsync", - )], - } - } - - struct SuccessExecutor; - - impl RepoSyncExecutor for SuccessExecutor { - fn execute(&self, task: RepoSyncTask) -> RepoSyncResultEnvelope { - RepoSyncResultEnvelope { - repo_key: task.repo_key.clone(), - tal_id: task.tal_id, - rir_id: task.rir_id, - result: RepoSyncResultKind::Success(RepoSyncResultRef { - repo_key: task.repo_key, - source: "rrdp".to_string(), - }), - phase: Some("rrdp_ok".to_string()), - timing_ms: 12, - warnings: Vec::new(), - } - } - } - - struct FailureExecutor; - - impl RepoSyncExecutor for FailureExecutor { - fn execute(&self, task: RepoSyncTask) -> RepoSyncResultEnvelope { - RepoSyncResultEnvelope { - repo_key: task.repo_key, - tal_id: task.tal_id, - rir_id: task.rir_id, - result: RepoSyncResultKind::Failed { - detail: "timeout".to_string(), - }, - phase: Some("repo_sync_failed".to_string()), - timing_ms: 33, - warnings: vec![Warning::new("timeout")], - } - } - } - - struct BarrierExecutor { - barrier: Arc, - active: Arc, - peak: Arc, - } - - #[derive(Clone)] - struct PanicHttpFetcher; - - impl Fetcher for PanicHttpFetcher { - fn fetch(&self, _uri: &str) -> Result, String> { - panic!("http fetch should not be used in this test") - } - } - - #[derive(Clone)] - struct ErrorHttpFetcher; - - impl Fetcher for ErrorHttpFetcher { - fn fetch(&self, uri: &str) -> Result, String> { - Err(format!("fetch blocked for {uri}")) - } - } - - #[derive(Clone)] - struct FailingRsyncFetcher; - - #[derive(Clone, Default)] - struct MockHttpFetcher { - map: Arc>>>, - } - - impl MockHttpFetcher { - fn new() -> Self { - Self::default() - } - - fn insert(&self, uri: &str, bytes: Vec) { - self.map - .lock() - .expect("http fixture lock") - .insert(uri.to_string(), bytes); - } - } - - impl RsyncFetcher for FailingRsyncFetcher { - fn fetch_objects(&self, _rsync_base_uri: &str) -> RsyncFetchResult)>> { - Err(RsyncFetchError::Fetch("boom".to_string())) - } - } - - impl Fetcher for MockHttpFetcher { - fn fetch(&self, uri: &str) -> Result, String> { - self.map - .lock() - .expect("http fixture lock") - .get(uri) - .cloned() - .ok_or_else(|| format!("missing fixture for {uri}")) - } - } - - fn sha256_hex(bytes: &[u8]) -> String { - hex::encode(sha2::Sha256::digest(bytes)) - } - - fn rrdp_notification_xml(session_id: &str, serial: u64, snapshot_uri: &str) -> String { - let snapshot_body = rrdp_snapshot_xml( - session_id, - &[("rsync://example.test/repo/a.roa", b"a".as_ref())], - ); - let snapshot_hash = sha256_hex(snapshot_body.as_bytes()); - format!( - r#" - - -"# - ) - } - - fn rrdp_snapshot_xml(session_id: &str, objects: &[(&str, &[u8])]) -> String { - let mut body = String::from(&format!( - r#" - -"# - )); - for (uri, bytes) in objects { - body.push_str(&format!( - " {}\n", - base64::engine::general_purpose::STANDARD.encode(bytes) - )); - } - body.push_str(""); - body - } - - impl RepoSyncExecutor for BarrierExecutor { - fn execute(&self, task: RepoSyncTask) -> RepoSyncResultEnvelope { - let now = self.active.fetch_add(1, Ordering::SeqCst) + 1; - loop { - let peak = self.peak.load(Ordering::SeqCst); - if now > peak { - if self - .peak - .compare_exchange(peak, now, Ordering::SeqCst, Ordering::SeqCst) - .is_ok() - { - break; - } - } else { - break; - } - } - self.barrier.wait(); - self.active.fetch_sub(1, Ordering::SeqCst); - RepoSyncResultEnvelope { - repo_key: task.repo_key.clone(), - tal_id: task.tal_id, - rir_id: task.rir_id, - result: RepoSyncResultKind::Success(RepoSyncResultRef { - repo_key: task.repo_key, - source: "rsync".to_string(), - }), - phase: Some("rsync_only_ok".to_string()), - timing_ms: 1, - warnings: Vec::new(), - } - } - } - - struct SuccessTransportExecutor; - - impl RepoTransportExecutor for SuccessTransportExecutor { - fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { - RepoTransportResultEnvelope { - dedup_key: task.dedup_key, - rsync_failure_scope_uri: task.rsync_failure_scope_uri, - repo_identity: task.repo_identity, - mode: task.mode, - tal_id: task.tal_id, - rir_id: task.rir_id, - timing_ms: 5, - result: RepoTransportResultKind::Success { - source: match task.mode { - RepoTransportMode::Rrdp => "rrdp".to_string(), - RepoTransportMode::Rsync => "rsync".to_string(), - }, - warnings: Vec::new(), - }, - } - } - } - - #[test] - fn repo_worker_pool_config_uses_parallel_phase1_budget() { - let cfg = ParallelPhase1Config { - max_repo_sync_workers_global: 9, - ..ParallelPhase1Config::default() - }; - let pool_cfg = RepoWorkerPoolConfig::from(&cfg); - assert_eq!(pool_cfg.max_workers, 9); - } - - #[test] - fn repo_worker_pool_rejects_zero_workers() { - let err = RepoWorkerPool::new(RepoWorkerPoolConfig { max_workers: 0 }, SuccessExecutor) - .err() - .expect("zero workers should fail"); - assert!(err.contains("at least one worker")); - } - - #[test] - fn repo_worker_pool_processes_tasks_and_returns_results() { - let pool = RepoWorkerPool::new(RepoWorkerPoolConfig { max_workers: 2 }, SuccessExecutor) - .expect("pool"); - pool.submit(sample_task("a")).expect("submit task a"); - pool.submit(sample_task("b")).expect("submit task b"); - - let mut results = Vec::new(); - results.push( - pool.recv_result_timeout(Duration::from_secs(1)) - .expect("recv result 1") - .expect("result 1"), - ); - results.push( - pool.recv_result_timeout(Duration::from_secs(1)) - .expect("recv result 2") - .expect("result 2"), - ); - - assert_eq!(results.len(), 2); - assert!( - results - .iter() - .all(|res| matches!(res.result, RepoSyncResultKind::Success(_))) - ); - } - - #[test] - fn repo_worker_pool_returns_failure_envelopes() { - let pool = RepoWorkerPool::new(RepoWorkerPoolConfig { max_workers: 1 }, FailureExecutor) - .expect("pool"); - pool.submit(sample_task("fail")).expect("submit"); - let result = pool - .recv_result_timeout(Duration::from_secs(1)) - .expect("recv") - .expect("result"); - assert!(matches!( - result.result, - RepoSyncResultKind::Failed { ref detail } if detail == "timeout" - )); - assert_eq!(result.warnings, vec![Warning::new("timeout")]); - } - - #[test] - fn repo_worker_pool_processes_work_in_parallel() { - let barrier = Arc::new(Barrier::new(2)); - let active = Arc::new(AtomicUsize::new(0)); - let peak = Arc::new(AtomicUsize::new(0)); - let executor = BarrierExecutor { - barrier: Arc::clone(&barrier), - active: Arc::clone(&active), - peak: Arc::clone(&peak), - }; - let pool = - RepoWorkerPool::new(RepoWorkerPoolConfig { max_workers: 2 }, executor).expect("pool"); - pool.submit(sample_task("a")).expect("submit a"); - pool.submit(sample_task("b")).expect("submit b"); - - let _ = pool - .recv_result_timeout(Duration::from_secs(1)) - .expect("recv 1") - .expect("result 1"); - let _ = pool - .recv_result_timeout(Duration::from_secs(1)) - .expect("recv 2") - .expect("result 2"); - - assert!(peak.load(Ordering::SeqCst) >= 2); - } - - #[test] - fn repo_worker_pool_exposes_configured_worker_count() { - let pool = RepoWorkerPool::new(RepoWorkerPoolConfig { max_workers: 3 }, SuccessExecutor) - .expect("pool"); - assert_eq!(pool.worker_count(), 3); - } - - #[test] - fn repo_transport_worker_pool_processes_transport_tasks() { - let pool = RepoTransportWorkerPool::new( - RepoWorkerPoolConfig { max_workers: 2 }, - SuccessTransportExecutor, - ) - .expect("pool"); - pool.submit(sample_rrdp_transport_task( - "https://example.test/notify.xml", - "rsync://example.test/repo/", - )) - .expect("submit rrdp"); - pool.submit(sample_rsync_transport_task( - "rsync://example.test/module/", - "rsync://example.test/repo/", - )) - .expect("submit rsync"); - - let first = pool - .recv_result_timeout(Duration::from_secs(1)) - .expect("recv first") - .expect("first result"); - let second = pool - .recv_result_timeout(Duration::from_secs(1)) - .expect("recv second") - .expect("second result"); - assert!(matches!( - first.result, - RepoTransportResultKind::Success { .. } - )); - assert!(matches!( - second.result, - RepoTransportResultKind::Success { .. } - )); - } - - #[test] - fn live_repo_sync_executor_runs_rsync_sync_and_updates_store() { - let td = tempfile::tempdir().expect("tempdir"); - fs::create_dir_all(td.path().join("nested")).expect("mkdir"); - fs::write(td.path().join("a.roa"), b"a").expect("write a"); - fs::write(td.path().join("nested").join("b.cer"), b"b").expect("write b"); - - let store_dir = tempfile::tempdir().expect("store tempdir"); - let store = Arc::new(RocksStore::open(store_dir.path()).expect("open store")); - let executor = LiveRepoSyncExecutor::new( - Arc::clone(&store), - crate::policy::Policy::default(), - Arc::new(PanicHttpFetcher), - Arc::new(LocalDirRsyncFetcher::new(td.path())), - None, - None, - ); - - let result = executor.execute(sample_task("repo")); - assert!(matches!(result.result, RepoSyncResultKind::Success(_))); - assert_eq!(result.tal_id, "arin"); - assert_eq!(result.rir_id, "arin"); - assert_eq!(result.warnings.len(), 0); - - let view = store - .get_repository_view_entry("rsync://example.test/repo/a.roa") - .expect("read view") - .expect("entry exists"); - assert_eq!(view.rsync_uri, "rsync://example.test/repo/a.roa"); - assert!(view.current_hash.is_some()); - } - - #[test] - fn live_repo_sync_executor_returns_failure_when_rsync_fails() { - let store_dir = tempfile::tempdir().expect("store tempdir"); - let store = Arc::new(RocksStore::open(store_dir.path()).expect("open store")); - let executor = LiveRepoSyncExecutor::new( - Arc::clone(&store), - crate::policy::Policy::default(), - Arc::new(PanicHttpFetcher), - Arc::new(FailingRsyncFetcher), - None, - None, - ); - - let result = executor.execute(sample_task("repo")); - assert!(matches!( - result.result, - RepoSyncResultKind::Failed { ref detail } if detail.contains("rsync fallback failed") - )); - assert!( - store - .get_repository_view_entry("rsync://example.test/repo/a.roa") - .expect("read view") - .is_none() - ); - } - - #[test] - fn live_rrdp_transport_executor_succeeds_on_valid_notification() { - let td = tempfile::tempdir().expect("tempdir"); - let store = Arc::new(RocksStore::open(td.path()).expect("open store")); - let http = MockHttpFetcher::new(); - let notify = "https://example.test/notification.xml"; - let snapshot = "https://example.test/snapshot.xml"; - let snapshot_bytes = rrdp_snapshot_xml( - "123e4567-e89b-12d3-a456-426614174000", - &[("rsync://example.test/repo/a.roa", b"a".as_ref())], - ); - let notification_bytes = - rrdp_notification_xml("123e4567-e89b-12d3-a456-426614174000", 1, snapshot); - http.insert(notify, notification_bytes.into_bytes()); - http.insert(snapshot, snapshot_bytes.into_bytes()); - let current_repo_index = CurrentRepoIndex::shared(); - let executor = LiveRrdpTransportExecutor::new( - Arc::clone(&store), - current_repo_index.clone(), - Arc::new(http), - None, - None, - ); - let result = executor.execute_transport(sample_rrdp_transport_task( - notify, - "rsync://example.test/repo/", - )); - assert!( - matches!(result.result, RepoTransportResultKind::Success { .. }), - "{result:?}" - ); - let index = current_repo_index.read().expect("index read lock"); - assert!( - index - .get_by_uri("rsync://example.test/repo/a.roa") - .is_some() - ); - assert_eq!( - index.list_scope_uris(notify), - vec!["rsync://example.test/repo/a.roa".to_string()] - ); - } - - #[test] - fn live_rrdp_transport_executor_reports_failure_without_rsync_fallback() { - let td = tempfile::tempdir().expect("tempdir"); - let store = Arc::new(RocksStore::open(td.path()).expect("open store")); - let executor = LiveRrdpTransportExecutor::new( - Arc::clone(&store), - CurrentRepoIndex::shared(), - Arc::new(ErrorHttpFetcher), - None, - None, - ); - let result = executor.execute_transport(sample_rrdp_transport_task( - "https://example.test/notification.xml", - "rsync://example.test/repo/", - )); - assert!(matches!( - result.result, - RepoTransportResultKind::Failed { .. } - )); - } - - #[test] - fn live_rsync_transport_executor_succeeds_on_local_repo() { - let td = tempfile::tempdir().expect("tempdir"); - fs::create_dir_all(td.path().join("nested")).expect("mkdir"); - fs::write(td.path().join("a.roa"), b"a").expect("write"); - fs::write(td.path().join("nested").join("b.cer"), b"b").expect("write"); - let store_dir = tempfile::tempdir().expect("store tempdir"); - let store = Arc::new(RocksStore::open(store_dir.path()).expect("open store")); - let current_repo_index = CurrentRepoIndex::shared(); - let executor = LiveRsyncTransportExecutor::new( - Arc::clone(&store), - current_repo_index.clone(), - Arc::new(LocalDirRsyncFetcher::new(td.path())), - None, - None, - ); - let result = executor.execute_transport(sample_rsync_transport_task( - "rsync://example.test/repo/", - "rsync://example.test/repo/", - )); - assert!(matches!( - result.result, - RepoTransportResultKind::Success { .. } - )); - let index = current_repo_index.read().expect("index read lock"); - assert!( - index - .get_by_uri("rsync://example.test/repo/a.roa") - .is_some() - ); - assert!( - index - .get_by_uri("rsync://example.test/repo/nested/b.cer") - .is_some() - ); - assert_eq!( - index.list_scope_uris("rsync://example.test/repo/"), - vec![ - "rsync://example.test/repo/a.roa".to_string(), - "rsync://example.test/repo/nested/b.cer".to_string() - ] - ); - } - - #[test] - fn live_rsync_transport_executor_reports_failure() { - let store_dir = tempfile::tempdir().expect("store tempdir"); - let store = Arc::new(RocksStore::open(store_dir.path()).expect("open store")); - let executor = LiveRsyncTransportExecutor::new( - Arc::clone(&store), - CurrentRepoIndex::shared(), - Arc::new(FailingRsyncFetcher), - None, - None, - ); - let result = executor.execute_transport(sample_rsync_transport_task( - "rsync://example.test/module/", - "rsync://example.test/repo/", - )); - assert!(matches!( - result.result, - RepoTransportResultKind::Failed { .. } - )); - } -} +#[path = "repo_worker/tests.rs"] +mod tests; diff --git a/crates/panda-rpki-validator/src/parallel/repo_worker/executors.rs b/crates/panda-rpki-validator/src/parallel/repo_worker/executors.rs new file mode 100644 index 0000000..b67f2a3 --- /dev/null +++ b/crates/panda-rpki-validator/src/parallel/repo_worker/executors.rs @@ -0,0 +1,384 @@ +// Repository sync and transport executor implementations. + +const RETRY_SHORT_TIMEOUT: Duration = Duration::from_secs(1); + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RepoWorkerPoolConfig { + pub max_workers: usize, +} + +impl From<&ParallelPhase1Config> for RepoWorkerPoolConfig { + fn from(value: &ParallelPhase1Config) -> Self { + Self { + max_workers: value.max_repo_sync_workers_global, + } + } +} + +pub trait RepoSyncExecutor: Send + Sync + 'static { + fn execute(&self, task: RepoSyncTask) -> RepoSyncResultEnvelope; +} + +pub trait RepoTransportExecutor: Send + Sync + 'static { + fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope; +} + +pub struct LiveRrdpTransportExecutor { + store: Arc, + current_repo_index: CurrentRepoIndexHandle, + http_fetcher: Arc, + timing: Option, + download_log: Option, +} + +impl LiveRrdpTransportExecutor { + pub fn new( + store: Arc, + current_repo_index: CurrentRepoIndexHandle, + http_fetcher: Arc, + timing: Option, + download_log: Option, + ) -> Self { + Self { + store, + current_repo_index, + http_fetcher, + timing, + download_log, + } + } +} + +impl RepoTransportExecutor for LiveRrdpTransportExecutor { + fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { + let started = std::time::Instant::now(); + debug_assert_eq!(task.mode, RepoTransportMode::Rrdp); + let notification_uri = task + .repo_identity + .notification_uri + .as_deref() + .expect("rrdp transport requires notification uri"); + let sync_result = if task.retry_short_timeout { + crate::fetch::http::with_scoped_http_timeout_override(RETRY_SHORT_TIMEOUT, || { + run_rrdp_transport( + self.store.as_ref(), + notification_uri, + Some(&self.current_repo_index), + self.http_fetcher.as_ref(), + self.timing.as_ref(), + self.download_log.as_ref(), + ) + }) + } else { + run_rrdp_transport( + self.store.as_ref(), + notification_uri, + Some(&self.current_repo_index), + self.http_fetcher.as_ref(), + self.timing.as_ref(), + self.download_log.as_ref(), + ) + }; + match sync_result { + Ok(_) => RepoTransportResultEnvelope { + dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri, + repo_identity: task.repo_identity, + mode: RepoTransportMode::Rrdp, + tal_id: task.tal_id, + rir_id: task.rir_id, + timing_ms: started.elapsed().as_millis() as u64, + result: RepoTransportResultKind::Success { + source: "rrdp".to_string(), + warnings: Vec::new(), + }, + }, + Err(err) => { + let error_class = + crate::parallel::dead_repo_blacklist::classify_rrdp_sync_error(&err); + RepoTransportResultEnvelope { + dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri, + repo_identity: task.repo_identity, + mode: RepoTransportMode::Rrdp, + tal_id: task.tal_id, + rir_id: task.rir_id, + timing_ms: started.elapsed().as_millis() as u64, + result: RepoTransportResultKind::Failed { + detail: err.to_string(), + warnings: Vec::new(), + error_class, + }, + } + } + } + } +} + +pub struct LiveRsyncTransportExecutor { + store: Arc, + current_repo_index: CurrentRepoIndexHandle, + rsync_fetcher: Arc, + timing: Option, + download_log: Option, +} + +impl LiveRsyncTransportExecutor { + pub fn new( + store: Arc, + current_repo_index: CurrentRepoIndexHandle, + rsync_fetcher: Arc, + timing: Option, + download_log: Option, + ) -> Self { + Self { + store, + current_repo_index, + rsync_fetcher, + timing, + download_log, + } + } +} + +impl RepoTransportExecutor for LiveRsyncTransportExecutor { + fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { + let started = std::time::Instant::now(); + debug_assert_eq!(task.mode, RepoTransportMode::Rsync); + let sync_result = if task.retry_short_timeout { + with_scoped_rsync_timeout_override(RETRY_SHORT_TIMEOUT, || { + with_scoped_rsync_fail_fast_profile( + RsyncFailFastProfile { + initial_wall_clock_timeout: RETRY_SHORT_TIMEOUT, + max_wall_clock_timeout: RETRY_SHORT_TIMEOUT, + max_attempts: 1, + }, + || { + run_rsync_transport( + self.store.as_ref(), + &task.repo_identity.rsync_base_uri, + Some(&self.current_repo_index), + self.rsync_fetcher.as_ref(), + self.timing.as_ref(), + self.download_log.as_ref(), + ) + }, + ) + }) + } else { + run_rsync_transport( + self.store.as_ref(), + &task.repo_identity.rsync_base_uri, + Some(&self.current_repo_index), + self.rsync_fetcher.as_ref(), + self.timing.as_ref(), + self.download_log.as_ref(), + ) + }; + match sync_result { + Ok(_) => RepoTransportResultEnvelope { + dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri, + repo_identity: task.repo_identity, + mode: RepoTransportMode::Rsync, + tal_id: task.tal_id, + rir_id: task.rir_id, + timing_ms: started.elapsed().as_millis() as u64, + result: RepoTransportResultKind::Success { + source: "rsync".to_string(), + warnings: Vec::new(), + }, + }, + Err(err) => { + let error_class = + crate::parallel::dead_repo_blacklist::classify_rsync_repo_sync_error(&err); + RepoTransportResultEnvelope { + dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri, + repo_identity: task.repo_identity, + mode: RepoTransportMode::Rsync, + tal_id: task.tal_id, + rir_id: task.rir_id, + timing_ms: started.elapsed().as_millis() as u64, + result: RepoTransportResultKind::Failed { + detail: err.to_string(), + warnings: Vec::new(), + error_class, + }, + } + } + } + } +} + +pub struct LiveRepoTransportExecutor { + rrdp: LiveRrdpTransportExecutor, + rsync: LiveRsyncTransportExecutor, +} + +impl LiveRepoTransportExecutor { + pub fn new( + store: Arc, + current_repo_index: CurrentRepoIndexHandle, + http_fetcher: Arc, + rsync_fetcher: Arc, + timing: Option, + download_log: Option, + ) -> Self { + Self { + rrdp: LiveRrdpTransportExecutor::new( + Arc::clone(&store), + current_repo_index.clone(), + http_fetcher, + timing.clone(), + download_log.clone(), + ), + rsync: LiveRsyncTransportExecutor::new( + store, + current_repo_index, + rsync_fetcher, + timing, + download_log, + ), + } + } +} + +impl RepoTransportExecutor + for LiveRepoTransportExecutor +{ + fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { + match task.mode { + RepoTransportMode::Rrdp => self.rrdp.execute_transport(task), + RepoTransportMode::Rsync => self.rsync.execute_transport(task), + } + } +} + +pub struct LiveRepoSyncExecutor { + store: Arc, + policy: Policy, + http_fetcher: Arc, + rsync_fetcher: Arc, + timing: Option, + download_log: Option, +} + +impl LiveRepoSyncExecutor { + pub fn new( + store: Arc, + policy: Policy, + http_fetcher: Arc, + rsync_fetcher: Arc, + timing: Option, + download_log: Option, + ) -> Self { + Self { + store, + policy, + http_fetcher, + rsync_fetcher, + timing, + download_log, + } + } +} + +impl RepoSyncExecutor + for LiveRepoSyncExecutor +{ + fn execute(&self, task: RepoSyncTask) -> RepoSyncResultEnvelope { + let started = std::time::Instant::now(); + crate::progress_log::emit( + "phase1_repo_worker_execute_start", + serde_json::json!({ + "repo_key_rsync_base_uri": task.repo_key.rsync_base_uri, + "repo_key_notification_uri": task.repo_key.notification_uri, + "tal_id": task.tal_id, + "rir_id": task.rir_id, + }), + ); + match sync_publication_point( + self.store.as_ref(), + &self.policy, + task.repo_key.notification_uri.as_deref(), + &task.repo_key.rsync_base_uri, + self.http_fetcher.as_ref(), + self.rsync_fetcher.as_ref(), + self.timing.as_ref(), + self.download_log.as_ref(), + ) { + Ok(res) => { + let timing_ms = started.elapsed().as_millis() as u64; + crate::progress_log::emit( + "phase1_repo_worker_execute_finish", + serde_json::json!({ + "repo_key_rsync_base_uri": task.repo_key.rsync_base_uri, + "repo_key_notification_uri": task.repo_key.notification_uri, + "result": "success", + "phase": repo_sync_phase_label(res.phase), + "timing_ms": timing_ms, + }), + ); + RepoSyncResultEnvelope { + repo_key: task.repo_key.clone(), + tal_id: task.tal_id, + rir_id: task.rir_id, + result: super::types::RepoSyncResultKind::Success( + super::types::RepoSyncResultRef { + repo_key: task.repo_key, + source: repo_sync_source_label(res.source).to_string(), + }, + ), + phase: Some(repo_sync_phase_label(res.phase).to_string()), + timing_ms, + warnings: res.warnings, + } + } + Err(err) => { + let timing_ms = started.elapsed().as_millis() as u64; + crate::progress_log::emit( + "phase1_repo_worker_execute_finish", + serde_json::json!({ + "repo_key_rsync_base_uri": task.repo_key.rsync_base_uri, + "repo_key_notification_uri": task.repo_key.notification_uri, + "result": "failed", + "phase": "repo_sync_failed", + "timing_ms": timing_ms, + "error": err.to_string(), + }), + ); + RepoSyncResultEnvelope { + repo_key: task.repo_key, + tal_id: task.tal_id, + rir_id: task.rir_id, + result: super::types::RepoSyncResultKind::Failed { + detail: err.to_string(), + }, + phase: Some("repo_sync_failed".to_string()), + timing_ms, + warnings: Vec::new(), + } + } + } + } +} + +fn repo_sync_source_label(source: RepoSyncSource) -> &'static str { + match source { + RepoSyncSource::Rrdp => "rrdp", + RepoSyncSource::Rsync => "rsync", + } +} + +fn repo_sync_phase_label(phase: crate::sync::repo::RepoSyncPhase) -> &'static str { + match phase { + crate::sync::repo::RepoSyncPhase::RrdpOk => "rrdp_ok", + crate::sync::repo::RepoSyncPhase::RrdpFailedRsyncOk => "rrdp_failed_rsync_ok", + crate::sync::repo::RepoSyncPhase::RsyncOnlyOk => "rsync_only_ok", + crate::sync::repo::RepoSyncPhase::ReplayRrdpOk => "replay_rrdp_ok", + crate::sync::repo::RepoSyncPhase::ReplayRsyncOk => "replay_rsync_ok", + crate::sync::repo::RepoSyncPhase::ReplayNoopRrdp => "replay_noop_rrdp", + crate::sync::repo::RepoSyncPhase::ReplayNoopRsync => "replay_noop_rsync", + } +} diff --git a/crates/panda-rpki-validator/src/parallel/repo_worker/pools.rs b/crates/panda-rpki-validator/src/parallel/repo_worker/pools.rs new file mode 100644 index 0000000..4e24287 --- /dev/null +++ b/crates/panda-rpki-validator/src/parallel/repo_worker/pools.rs @@ -0,0 +1,255 @@ +// Worker-pool channels, lifecycle, and worker loops. + +enum RepoWorkerMessage { + Task(RepoSyncTask), + Shutdown, +} + +enum TransportWorkerMessage { + Task(RepoTransportTask), + Shutdown, +} + +pub struct RepoWorkerPool { + config: RepoWorkerPoolConfig, + task_tx: Sender, + result_rx: Receiver, + workers: Vec>, + _executor: Arc, +} + +pub struct RepoTransportWorkerPool { + config: RepoWorkerPoolConfig, + task_tx: Sender, + result_rx: Receiver, + workers: Vec>, + _executor: Arc, +} + +impl RepoWorkerPool { + pub fn new(config: RepoWorkerPoolConfig, executor: E) -> Result { + if config.max_workers == 0 { + return Err("RepoWorkerPool requires at least one worker".to_string()); + } + + let executor = Arc::new(executor); + let (task_tx, task_rx) = mpsc::channel::(); + let (result_tx, result_rx) = mpsc::channel::(); + let shared_task_rx = Arc::new(Mutex::new(task_rx)); + + let mut workers = Vec::with_capacity(config.max_workers); + for idx in 0..config.max_workers { + let task_rx = Arc::clone(&shared_task_rx); + let result_tx = result_tx.clone(); + let executor = Arc::clone(&executor); + workers.push( + thread::Builder::new() + .name(format!("repo-sync-worker-{idx}")) + .spawn(move || worker_loop(task_rx, result_tx, executor)) + .map_err(|e| format!("spawn repo worker failed: {e}"))?, + ); + } + + Ok(Self { + config, + task_tx, + result_rx, + workers, + _executor: executor, + }) + } + + pub fn worker_count(&self) -> usize { + self.config.max_workers + } + + pub fn submit(&self, task: RepoSyncTask) -> Result<(), String> { + self.task_tx + .send(RepoWorkerMessage::Task(task)) + .map_err(|e| format!("submit repo task failed: {e}")) + } + + pub fn recv_result_timeout( + &self, + timeout: Duration, + ) -> Result, String> { + match self.result_rx.recv_timeout(timeout) { + Ok(msg) => Ok(Some(msg)), + Err(RecvTimeoutError::Timeout) => Ok(None), + Err(RecvTimeoutError::Disconnected) => { + Err("repo worker result channel disconnected".to_string()) + } + } + } + + pub fn shutdown(mut self) -> Result<(), String> { + self.shutdown_inner() + } + + fn shutdown_inner(&mut self) -> Result<(), String> { + if self.workers.is_empty() { + return Ok(()); + } + + for _ in 0..self.workers.len() { + self.task_tx + .send(RepoWorkerMessage::Shutdown) + .map_err(|e| format!("send shutdown to repo worker failed: {e}"))?; + } + + let mut first_err: Option = None; + for handle in self.workers.drain(..) { + if let Err(e) = handle.join() { + if first_err.is_none() { + first_err = Some(format!("join repo worker failed: {e:?}")); + } + } + } + + if let Some(err) = first_err { + return Err(err); + } + Ok(()) + } +} + +impl Drop for RepoWorkerPool { + fn drop(&mut self) { + let _ = self.shutdown_inner(); + } +} + +impl RepoTransportWorkerPool { + pub fn new(config: RepoWorkerPoolConfig, executor: E) -> Result { + if config.max_workers == 0 { + return Err("RepoTransportWorkerPool requires at least one worker".to_string()); + } + let executor = Arc::new(executor); + let (task_tx, task_rx) = mpsc::channel::(); + let (result_tx, result_rx) = mpsc::channel::(); + let shared_task_rx = Arc::new(Mutex::new(task_rx)); + let mut workers = Vec::with_capacity(config.max_workers); + for idx in 0..config.max_workers { + let task_rx = Arc::clone(&shared_task_rx); + let result_tx = result_tx.clone(); + let executor = Arc::clone(&executor); + workers.push( + thread::Builder::new() + .name(format!("repo-transport-worker-{idx}")) + .spawn(move || transport_worker_loop(task_rx, result_tx, executor)) + .map_err(|e| format!("spawn repo transport worker failed: {e}"))?, + ); + } + Ok(Self { + config, + task_tx, + result_rx, + workers, + _executor: executor, + }) + } + + pub fn submit(&self, task: RepoTransportTask) -> Result<(), String> { + self.task_tx + .send(TransportWorkerMessage::Task(task)) + .map_err(|e| format!("submit repo transport task failed: {e}")) + } + + pub fn recv_result_timeout( + &self, + timeout: Duration, + ) -> Result, String> { + match self.result_rx.recv_timeout(timeout) { + Ok(msg) => Ok(Some(msg)), + Err(RecvTimeoutError::Timeout) => Ok(None), + Err(RecvTimeoutError::Disconnected) => { + Err("repo transport worker result channel disconnected".to_string()) + } + } + } + + pub fn worker_count(&self) -> usize { + self.config.max_workers + } + + pub fn shutdown(mut self) -> Result<(), String> { + self.shutdown_inner() + } + + fn shutdown_inner(&mut self) -> Result<(), String> { + if self.workers.is_empty() { + return Ok(()); + } + for _ in 0..self.workers.len() { + self.task_tx + .send(TransportWorkerMessage::Shutdown) + .map_err(|e| format!("send shutdown to repo transport worker failed: {e}"))?; + } + let mut first_err: Option = None; + for handle in self.workers.drain(..) { + if let Err(e) = handle.join() { + if first_err.is_none() { + first_err = Some(format!("join repo transport worker failed: {e:?}")); + } + } + } + if let Some(err) = first_err { + return Err(err); + } + Ok(()) + } +} + +impl Drop for RepoTransportWorkerPool { + fn drop(&mut self) { + let _ = self.shutdown_inner(); + } +} + +fn worker_loop( + task_rx: Arc>>, + result_tx: Sender, + executor: Arc, +) { + loop { + let message = { + let rx = task_rx.lock().expect("repo worker receiver lock poisoned"); + rx.recv() + }; + + match message { + Ok(RepoWorkerMessage::Task(task)) => { + let result = executor.execute(task); + if result_tx.send(result).is_err() { + break; + } + } + Ok(RepoWorkerMessage::Shutdown) | Err(_) => break, + } + } +} + +fn transport_worker_loop( + task_rx: Arc>>, + result_tx: Sender, + executor: Arc, +) { + loop { + let message = { + let rx = task_rx + .lock() + .expect("repo transport worker receiver lock poisoned"); + rx.recv() + }; + + match message { + Ok(TransportWorkerMessage::Task(task)) => { + let result = executor.execute_transport(task); + if result_tx.send(result).is_err() { + break; + } + } + Ok(TransportWorkerMessage::Shutdown) | Err(_) => break, + } + } +} diff --git a/crates/panda-rpki-validator/src/parallel/repo_worker/tests.rs b/crates/panda-rpki-validator/src/parallel/repo_worker/tests.rs new file mode 100644 index 0000000..b46c825 --- /dev/null +++ b/crates/panda-rpki-validator/src/parallel/repo_worker/tests.rs @@ -0,0 +1,599 @@ +// Repository worker and transport executor tests. + +use base64::Engine; +use sha2::Digest; +use std::collections::HashMap; +use std::fs; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Barrier, Mutex}; +use std::time::Duration; + +use super::{ + LiveRepoSyncExecutor, LiveRrdpTransportExecutor, LiveRsyncTransportExecutor, RepoSyncExecutor, + RepoTransportExecutor, RepoTransportWorkerPool, RepoWorkerPool, RepoWorkerPoolConfig, +}; +use crate::current_repo_index::CurrentRepoIndex; +use crate::fetch::rsync::{LocalDirRsyncFetcher, RsyncFetchError, RsyncFetchResult, RsyncFetcher}; +use crate::parallel::config::ParallelPhase1Config; +use crate::parallel::types::{ + RepoDedupKey, RepoIdentity, RepoKey, RepoRequester, RepoSyncResultEnvelope, RepoSyncResultKind, + RepoSyncResultRef, RepoSyncTask, RepoTransportMode, RepoTransportResultEnvelope, + RepoTransportResultKind, RepoTransportTask, +}; +use crate::policy::SyncPreference; +use crate::report::Warning; +use crate::storage::RocksStore; +use crate::sync::rrdp::Fetcher; + +fn sample_task(name: &str) -> RepoSyncTask { + RepoSyncTask { + repo_key: RepoKey::new(format!("rsync://example.test/{name}/"), None), + validation_time: time::OffsetDateTime::UNIX_EPOCH, + sync_preference: SyncPreference::RrdpThenRsync, + tal_id: "arin".to_string(), + rir_id: "arin".to_string(), + priority: 0, + requesters: vec![RepoRequester { + tal_id: "arin".to_string(), + rir_id: "arin".to_string(), + parent_node_id: None, + ca_instance_handle_id: format!("node:{name}"), + publication_point_rsync_uri: format!("rsync://example.test/{name}/"), + manifest_rsync_uri: format!("rsync://example.test/{name}/root.mft"), + }], + } +} + +fn sample_rrdp_transport_task(notification_uri: &str, rsync_base_uri: &str) -> RepoTransportTask { + RepoTransportTask { + dedup_key: RepoDedupKey::RrdpNotify { + notification_uri: notification_uri.to_string(), + }, + rsync_failure_scope_uri: None, + repo_identity: RepoIdentity::new(Some(notification_uri.to_string()), rsync_base_uri), + mode: RepoTransportMode::Rrdp, + retry_short_timeout: false, + tal_id: "arin".to_string(), + rir_id: "arin".to_string(), + validation_time: time::OffsetDateTime::UNIX_EPOCH, + priority: 0, + requesters: vec![RepoRequester::with_tal_rir( + "arin", + "arin", + format!("{rsync_base_uri}root.mft"), + rsync_base_uri.to_string(), + "node:rrdp", + )], + } +} + +fn sample_rsync_transport_task(rsync_scope_uri: &str, rsync_base_uri: &str) -> RepoTransportTask { + RepoTransportTask { + dedup_key: RepoDedupKey::RsyncScope { + rsync_scope_uri: rsync_scope_uri.to_string(), + }, + rsync_failure_scope_uri: None, + repo_identity: RepoIdentity::new(None, rsync_base_uri), + mode: RepoTransportMode::Rsync, + retry_short_timeout: false, + tal_id: "arin".to_string(), + rir_id: "arin".to_string(), + validation_time: time::OffsetDateTime::UNIX_EPOCH, + priority: 0, + requesters: vec![RepoRequester::with_tal_rir( + "arin", + "arin", + format!("{rsync_base_uri}root.mft"), + rsync_base_uri.to_string(), + "node:rsync", + )], + } +} + +struct SuccessExecutor; + +impl RepoSyncExecutor for SuccessExecutor { + fn execute(&self, task: RepoSyncTask) -> RepoSyncResultEnvelope { + RepoSyncResultEnvelope { + repo_key: task.repo_key.clone(), + tal_id: task.tal_id, + rir_id: task.rir_id, + result: RepoSyncResultKind::Success(RepoSyncResultRef { + repo_key: task.repo_key, + source: "rrdp".to_string(), + }), + phase: Some("rrdp_ok".to_string()), + timing_ms: 12, + warnings: Vec::new(), + } + } +} + +struct FailureExecutor; + +impl RepoSyncExecutor for FailureExecutor { + fn execute(&self, task: RepoSyncTask) -> RepoSyncResultEnvelope { + RepoSyncResultEnvelope { + repo_key: task.repo_key, + tal_id: task.tal_id, + rir_id: task.rir_id, + result: RepoSyncResultKind::Failed { + detail: "timeout".to_string(), + }, + phase: Some("repo_sync_failed".to_string()), + timing_ms: 33, + warnings: vec![Warning::new("timeout")], + } + } +} + +struct BarrierExecutor { + barrier: Arc, + active: Arc, + peak: Arc, +} + +#[derive(Clone)] +struct PanicHttpFetcher; + +impl Fetcher for PanicHttpFetcher { + fn fetch(&self, _uri: &str) -> Result, String> { + panic!("http fetch should not be used in this test") + } +} + +#[derive(Clone)] +struct ErrorHttpFetcher; + +impl Fetcher for ErrorHttpFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + Err(format!("fetch blocked for {uri}")) + } +} + +#[derive(Clone)] +struct FailingRsyncFetcher; + +#[derive(Clone, Default)] +struct MockHttpFetcher { + map: Arc>>>, +} + +impl MockHttpFetcher { + fn new() -> Self { + Self::default() + } + + fn insert(&self, uri: &str, bytes: Vec) { + self.map + .lock() + .expect("http fixture lock") + .insert(uri.to_string(), bytes); + } +} + +impl RsyncFetcher for FailingRsyncFetcher { + fn fetch_objects(&self, _rsync_base_uri: &str) -> RsyncFetchResult)>> { + Err(RsyncFetchError::Fetch("boom".to_string())) + } +} + +impl Fetcher for MockHttpFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + self.map + .lock() + .expect("http fixture lock") + .get(uri) + .cloned() + .ok_or_else(|| format!("missing fixture for {uri}")) + } +} + +fn sha256_hex(bytes: &[u8]) -> String { + hex::encode(sha2::Sha256::digest(bytes)) +} + +fn rrdp_notification_xml(session_id: &str, serial: u64, snapshot_uri: &str) -> String { + let snapshot_body = rrdp_snapshot_xml( + session_id, + &[("rsync://example.test/repo/a.roa", b"a".as_ref())], + ); + let snapshot_hash = sha256_hex(snapshot_body.as_bytes()); + format!( + r#" + + +"# + ) +} + +fn rrdp_snapshot_xml(session_id: &str, objects: &[(&str, &[u8])]) -> String { + let mut body = String::from(&format!( + r#" + +"# + )); + for (uri, bytes) in objects { + body.push_str(&format!( + " {}\n", + base64::engine::general_purpose::STANDARD.encode(bytes) + )); + } + body.push_str(""); + body +} + +impl RepoSyncExecutor for BarrierExecutor { + fn execute(&self, task: RepoSyncTask) -> RepoSyncResultEnvelope { + let now = self.active.fetch_add(1, Ordering::SeqCst) + 1; + loop { + let peak = self.peak.load(Ordering::SeqCst); + if now > peak { + if self + .peak + .compare_exchange(peak, now, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + break; + } + } else { + break; + } + } + self.barrier.wait(); + self.active.fetch_sub(1, Ordering::SeqCst); + RepoSyncResultEnvelope { + repo_key: task.repo_key.clone(), + tal_id: task.tal_id, + rir_id: task.rir_id, + result: RepoSyncResultKind::Success(RepoSyncResultRef { + repo_key: task.repo_key, + source: "rsync".to_string(), + }), + phase: Some("rsync_only_ok".to_string()), + timing_ms: 1, + warnings: Vec::new(), + } + } +} + +struct SuccessTransportExecutor; + +impl RepoTransportExecutor for SuccessTransportExecutor { + fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { + RepoTransportResultEnvelope { + dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri, + repo_identity: task.repo_identity, + mode: task.mode, + tal_id: task.tal_id, + rir_id: task.rir_id, + timing_ms: 5, + result: RepoTransportResultKind::Success { + source: match task.mode { + RepoTransportMode::Rrdp => "rrdp".to_string(), + RepoTransportMode::Rsync => "rsync".to_string(), + }, + warnings: Vec::new(), + }, + } + } +} + +#[test] +fn repo_worker_pool_config_uses_parallel_phase1_budget() { + let cfg = ParallelPhase1Config { + max_repo_sync_workers_global: 9, + ..ParallelPhase1Config::default() + }; + let pool_cfg = RepoWorkerPoolConfig::from(&cfg); + assert_eq!(pool_cfg.max_workers, 9); +} + +#[test] +fn repo_worker_pool_rejects_zero_workers() { + let err = RepoWorkerPool::new(RepoWorkerPoolConfig { max_workers: 0 }, SuccessExecutor) + .err() + .expect("zero workers should fail"); + assert!(err.contains("at least one worker")); +} + +#[test] +fn repo_worker_pool_processes_tasks_and_returns_results() { + let pool = RepoWorkerPool::new(RepoWorkerPoolConfig { max_workers: 2 }, SuccessExecutor) + .expect("pool"); + pool.submit(sample_task("a")).expect("submit task a"); + pool.submit(sample_task("b")).expect("submit task b"); + + let mut results = Vec::new(); + results.push( + pool.recv_result_timeout(Duration::from_secs(1)) + .expect("recv result 1") + .expect("result 1"), + ); + results.push( + pool.recv_result_timeout(Duration::from_secs(1)) + .expect("recv result 2") + .expect("result 2"), + ); + + assert_eq!(results.len(), 2); + assert!( + results + .iter() + .all(|res| matches!(res.result, RepoSyncResultKind::Success(_))) + ); +} + +#[test] +fn repo_worker_pool_returns_failure_envelopes() { + let pool = RepoWorkerPool::new(RepoWorkerPoolConfig { max_workers: 1 }, FailureExecutor) + .expect("pool"); + pool.submit(sample_task("fail")).expect("submit"); + let result = pool + .recv_result_timeout(Duration::from_secs(1)) + .expect("recv") + .expect("result"); + assert!(matches!( + result.result, + RepoSyncResultKind::Failed { ref detail } if detail == "timeout" + )); + assert_eq!(result.warnings, vec![Warning::new("timeout")]); +} + +#[test] +fn repo_worker_pool_processes_work_in_parallel() { + let barrier = Arc::new(Barrier::new(2)); + let active = Arc::new(AtomicUsize::new(0)); + let peak = Arc::new(AtomicUsize::new(0)); + let executor = BarrierExecutor { + barrier: Arc::clone(&barrier), + active: Arc::clone(&active), + peak: Arc::clone(&peak), + }; + let pool = + RepoWorkerPool::new(RepoWorkerPoolConfig { max_workers: 2 }, executor).expect("pool"); + pool.submit(sample_task("a")).expect("submit a"); + pool.submit(sample_task("b")).expect("submit b"); + + let _ = pool + .recv_result_timeout(Duration::from_secs(1)) + .expect("recv 1") + .expect("result 1"); + let _ = pool + .recv_result_timeout(Duration::from_secs(1)) + .expect("recv 2") + .expect("result 2"); + + assert!(peak.load(Ordering::SeqCst) >= 2); +} + +#[test] +fn repo_worker_pool_exposes_configured_worker_count() { + let pool = RepoWorkerPool::new(RepoWorkerPoolConfig { max_workers: 3 }, SuccessExecutor) + .expect("pool"); + assert_eq!(pool.worker_count(), 3); +} + +#[test] +fn repo_transport_worker_pool_processes_transport_tasks() { + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 2 }, + SuccessTransportExecutor, + ) + .expect("pool"); + pool.submit(sample_rrdp_transport_task( + "https://example.test/notify.xml", + "rsync://example.test/repo/", + )) + .expect("submit rrdp"); + pool.submit(sample_rsync_transport_task( + "rsync://example.test/module/", + "rsync://example.test/repo/", + )) + .expect("submit rsync"); + + let first = pool + .recv_result_timeout(Duration::from_secs(1)) + .expect("recv first") + .expect("first result"); + let second = pool + .recv_result_timeout(Duration::from_secs(1)) + .expect("recv second") + .expect("second result"); + assert!(matches!( + first.result, + RepoTransportResultKind::Success { .. } + )); + assert!(matches!( + second.result, + RepoTransportResultKind::Success { .. } + )); +} + +#[test] +fn live_repo_sync_executor_runs_rsync_sync_and_updates_store() { + let td = tempfile::tempdir().expect("tempdir"); + fs::create_dir_all(td.path().join("nested")).expect("mkdir"); + fs::write(td.path().join("a.roa"), b"a").expect("write a"); + fs::write(td.path().join("nested").join("b.cer"), b"b").expect("write b"); + + let store_dir = tempfile::tempdir().expect("store tempdir"); + let store = Arc::new(RocksStore::open(store_dir.path()).expect("open store")); + let executor = LiveRepoSyncExecutor::new( + Arc::clone(&store), + crate::policy::Policy::default(), + Arc::new(PanicHttpFetcher), + Arc::new(LocalDirRsyncFetcher::new(td.path())), + None, + None, + ); + + let result = executor.execute(sample_task("repo")); + assert!(matches!(result.result, RepoSyncResultKind::Success(_))); + assert_eq!(result.tal_id, "arin"); + assert_eq!(result.rir_id, "arin"); + assert_eq!(result.warnings.len(), 0); + + let view = store + .get_repository_view_entry("rsync://example.test/repo/a.roa") + .expect("read view") + .expect("entry exists"); + assert_eq!(view.rsync_uri, "rsync://example.test/repo/a.roa"); + assert!(view.current_hash.is_some()); +} + +#[test] +fn live_repo_sync_executor_returns_failure_when_rsync_fails() { + let store_dir = tempfile::tempdir().expect("store tempdir"); + let store = Arc::new(RocksStore::open(store_dir.path()).expect("open store")); + let executor = LiveRepoSyncExecutor::new( + Arc::clone(&store), + crate::policy::Policy::default(), + Arc::new(PanicHttpFetcher), + Arc::new(FailingRsyncFetcher), + None, + None, + ); + + let result = executor.execute(sample_task("repo")); + assert!(matches!( + result.result, + RepoSyncResultKind::Failed { ref detail } if detail.contains("rsync fallback failed") + )); + assert!( + store + .get_repository_view_entry("rsync://example.test/repo/a.roa") + .expect("read view") + .is_none() + ); +} + +#[test] +fn live_rrdp_transport_executor_succeeds_on_valid_notification() { + let td = tempfile::tempdir().expect("tempdir"); + let store = Arc::new(RocksStore::open(td.path()).expect("open store")); + let http = MockHttpFetcher::new(); + let notify = "https://example.test/notification.xml"; + let snapshot = "https://example.test/snapshot.xml"; + let snapshot_bytes = rrdp_snapshot_xml( + "123e4567-e89b-12d3-a456-426614174000", + &[("rsync://example.test/repo/a.roa", b"a".as_ref())], + ); + let notification_bytes = + rrdp_notification_xml("123e4567-e89b-12d3-a456-426614174000", 1, snapshot); + http.insert(notify, notification_bytes.into_bytes()); + http.insert(snapshot, snapshot_bytes.into_bytes()); + let current_repo_index = CurrentRepoIndex::shared(); + let executor = LiveRrdpTransportExecutor::new( + Arc::clone(&store), + current_repo_index.clone(), + Arc::new(http), + None, + None, + ); + let result = executor.execute_transport(sample_rrdp_transport_task( + notify, + "rsync://example.test/repo/", + )); + assert!( + matches!(result.result, RepoTransportResultKind::Success { .. }), + "{result:?}" + ); + let index = current_repo_index.read().expect("index read lock"); + assert!( + index + .get_by_uri("rsync://example.test/repo/a.roa") + .is_some() + ); + assert_eq!( + index.list_scope_uris(notify), + vec!["rsync://example.test/repo/a.roa".to_string()] + ); +} + +#[test] +fn live_rrdp_transport_executor_reports_failure_without_rsync_fallback() { + let td = tempfile::tempdir().expect("tempdir"); + let store = Arc::new(RocksStore::open(td.path()).expect("open store")); + let executor = LiveRrdpTransportExecutor::new( + Arc::clone(&store), + CurrentRepoIndex::shared(), + Arc::new(ErrorHttpFetcher), + None, + None, + ); + let result = executor.execute_transport(sample_rrdp_transport_task( + "https://example.test/notification.xml", + "rsync://example.test/repo/", + )); + assert!(matches!( + result.result, + RepoTransportResultKind::Failed { .. } + )); +} + +#[test] +fn live_rsync_transport_executor_succeeds_on_local_repo() { + let td = tempfile::tempdir().expect("tempdir"); + fs::create_dir_all(td.path().join("nested")).expect("mkdir"); + fs::write(td.path().join("a.roa"), b"a").expect("write"); + fs::write(td.path().join("nested").join("b.cer"), b"b").expect("write"); + let store_dir = tempfile::tempdir().expect("store tempdir"); + let store = Arc::new(RocksStore::open(store_dir.path()).expect("open store")); + let current_repo_index = CurrentRepoIndex::shared(); + let executor = LiveRsyncTransportExecutor::new( + Arc::clone(&store), + current_repo_index.clone(), + Arc::new(LocalDirRsyncFetcher::new(td.path())), + None, + None, + ); + let result = executor.execute_transport(sample_rsync_transport_task( + "rsync://example.test/repo/", + "rsync://example.test/repo/", + )); + assert!(matches!( + result.result, + RepoTransportResultKind::Success { .. } + )); + let index = current_repo_index.read().expect("index read lock"); + assert!( + index + .get_by_uri("rsync://example.test/repo/a.roa") + .is_some() + ); + assert!( + index + .get_by_uri("rsync://example.test/repo/nested/b.cer") + .is_some() + ); + assert_eq!( + index.list_scope_uris("rsync://example.test/repo/"), + vec![ + "rsync://example.test/repo/a.roa".to_string(), + "rsync://example.test/repo/nested/b.cer".to_string() + ] + ); +} + +#[test] +fn live_rsync_transport_executor_reports_failure() { + let store_dir = tempfile::tempdir().expect("store tempdir"); + let store = Arc::new(RocksStore::open(store_dir.path()).expect("open store")); + let executor = LiveRsyncTransportExecutor::new( + Arc::clone(&store), + CurrentRepoIndex::shared(), + Arc::new(FailingRsyncFetcher), + None, + None, + ); + let result = executor.execute_transport(sample_rsync_transport_task( + "rsync://example.test/module/", + "rsync://example.test/repo/", + )); + assert!(matches!( + result.result, + RepoTransportResultKind::Failed { .. } + )); +} diff --git a/crates/panda-rpki-validator/src/parallel/run_coordinator.rs b/crates/panda-rpki-validator/src/parallel/run_coordinator.rs index 7d62627..55e3ebf 100644 --- a/crates/panda-rpki-validator/src/parallel/run_coordinator.rs +++ b/crates/panda-rpki-validator/src/parallel/run_coordinator.rs @@ -26,7 +26,7 @@ pub struct GlobalRunCoordinator { pub pending_repo_tasks: VecDeque, pub pending_transport_tasks: VecDeque, pub stats: ParallelRunStats, - /// Mutable working copy of the dead-repo blacklist (#141): failure/success + /// Mutable working copy of the dead-repository blacklist: failure/success /// counters update during the run and persist at run end. Skip decisions /// inside `transport_tables` use the frozen run-start snapshot. pub dead_repo_blacklist: Option, @@ -221,7 +221,7 @@ impl GlobalRunCoordinator { Ok(completion) } - /// Update dead-repo blacklist counters (#141) from a real worker result. + /// Update dead-repository blacklist counters from a real worker result. /// Runs on the single pump thread; synthesized terminal envelopes never /// pass through here, so no double counting is possible. fn update_dead_repo_blacklist( @@ -581,7 +581,7 @@ mod tests { assert_eq!(coordinator.stats.repo_tasks_reused, 1); } - // ---- Dead-repo blacklist (#141) integration tests ---- + // ---- Dead-repository blacklist integration tests ---- use crate::parallel::dead_repo_blacklist::{DeadRepoBlacklist, DeadRepoBlacklistConfig}; use crate::parallel::types::{ @@ -751,7 +751,7 @@ mod tests { assert_eq!(task.mode, RepoTransportMode::Rsync); // register_dead passes retry_short_timeout=false (mirroring a prefetch // snapshot that recorded no real rrdp failure for skipped runs); the - // skip path must still keep the short retry profile (#141). + // Skip decisions must retain the short retry profile. assert!(task.retry_short_timeout); } diff --git a/crates/panda-rpki-validator/src/replay/delta_archive.rs b/crates/panda-rpki-validator/src/replay/delta_archive.rs index cac5d4f..983004f 100644 --- a/crates/panda-rpki-validator/src/replay/delta_archive.rs +++ b/crates/panda-rpki-validator/src/replay/delta_archive.rs @@ -9,1153 +9,9 @@ use crate::replay::archive::{ canonical_rsync_module, sha256_hex, }; -#[derive(Debug, thiserror::Error)] -pub enum ReplayDeltaArchiveError { - #[error(transparent)] - Base(#[from] ReplayArchiveError), - - #[error("delta capture directory not found: {0}")] - MissingDeltaCaptureDirectory(String), - - #[error("delta capture.json captureId mismatch: locks={locks_capture}, capture={capture_json}")] - CaptureIdMismatch { - locks_capture: String, - capture_json: String, - }, - - #[error( - "delta base.json baseCapture mismatch: locks={locks_base_capture}, base_json={base_json_base_capture}" - )] - BaseCaptureMismatch { - locks_base_capture: String, - base_json_base_capture: String, - }, - - #[error( - "delta base.json baseLocksSha256 mismatch: locks={locks_sha256}, base_json={base_json_sha256}" - )] - BaseLocksShaMismatch { - locks_sha256: String, - base_json_sha256: String, - }, - - #[error("base locks sha256 mismatch: expected {expected}, actual {actual}")] - BaseLocksBytesShaMismatch { expected: String, actual: String }, - - #[error("delta repo bucket not found for {notify_uri}: {path}")] - MissingDeltaRepoBucket { notify_uri: String, path: String }, - - #[error("delta repo meta mismatch: expected {expected}, actual {actual}")] - RrdpMetaMismatch { expected: String, actual: String }, - - #[error( - "delta transition kind mismatch for {notify_uri}: locks={locks_kind}, transition={transition_kind}" - )] - TransitionKindMismatch { - notify_uri: String, - locks_kind: String, - transition_kind: String, - }, - - #[error("delta transition base mismatch for {notify_uri}")] - TransitionBaseMismatch { notify_uri: String }, - - #[error("delta transition target mismatch for {notify_uri}")] - TransitionTargetMismatch { notify_uri: String }, - - #[error("delta serial list mismatch for {notify_uri}")] - DeltaSerialListMismatch { notify_uri: String }, - - #[error("delta notification session directory not found for {notify_uri}: {path}")] - MissingDeltaSessionDir { notify_uri: String, path: String }, - - #[error("target notification file not found for {notify_uri}: {path}")] - MissingTargetNotification { notify_uri: String, path: String }, - - #[error("delta file not found for {notify_uri} serial={serial}: {path}")] - MissingDeltaFile { - notify_uri: String, - serial: u64, - path: String, - }, - - #[error("delta target archive missing for {notify_uri}: {path}")] - MissingTargetArchive { notify_uri: String, path: String }, - - #[error("delta rsync module bucket not found for {module_uri}: {path}")] - MissingRsyncModuleBucket { module_uri: String, path: String }, - - #[error("delta rsync module meta mismatch: expected {expected}, actual {actual}")] - RsyncMetaMismatch { expected: String, actual: String }, - - #[error("delta rsync files.json module mismatch: expected {expected}, actual {actual}")] - RsyncFilesModuleMismatch { expected: String, actual: String }, - - #[error( - "delta rsync file count mismatch for {module_uri}: declared={declared}, actual={actual}" - )] - RsyncFileCountMismatch { - module_uri: String, - declared: usize, - actual: usize, - }, - - #[error("delta rsync overlay file not found for {module_uri}: {path}")] - MissingRsyncOverlayFile { module_uri: String, path: String }, -} - -#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] -pub struct ReplayDeltaLocks { - pub version: u32, - pub capture: String, - #[serde(rename = "baseCapture")] - pub base_capture: String, - #[serde(rename = "baseLocksSha256")] - pub base_locks_sha256: String, - pub rrdp: BTreeMap, - pub rsync: BTreeMap, -} - -#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] -pub struct ReplayDeltaBaseMeta { - pub version: u32, - #[serde(rename = "baseCapture")] - pub base_capture: String, - #[serde(rename = "baseLocksSha256")] - pub base_locks_sha256: String, - #[serde(rename = "createdAt")] - pub created_at: String, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize)] -#[serde(rename_all = "kebab-case")] -pub enum ReplayDeltaRrdpKind { - Unchanged, - Delta, - FallbackRsync, - SessionReset, - Gap, -} - -impl ReplayDeltaRrdpKind { - pub fn as_str(self) -> &'static str { - match self { - Self::Unchanged => "unchanged", - Self::Delta => "delta", - Self::FallbackRsync => "fallback-rsync", - Self::SessionReset => "session-reset", - Self::Gap => "gap", - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] -pub struct ReplayDeltaRrdpState { - pub transport: ReplayTransport, - pub session: Option, - pub serial: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] -pub struct ReplayDeltaRrdpEntry { - pub kind: ReplayDeltaRrdpKind, - pub base: ReplayDeltaRrdpState, - pub target: ReplayDeltaRrdpState, - #[serde(rename = "delta_count")] - pub delta_count: usize, - pub deltas: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] -pub struct ReplayDeltaRsyncEntry { - #[serde(rename = "file_count")] - pub file_count: usize, - #[serde(rename = "overlay_only")] - pub overlay_only: bool, -} - -#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] -pub struct ReplayDeltaTransition { - pub kind: ReplayDeltaRrdpKind, - pub base: ReplayDeltaRrdpState, - pub target: ReplayDeltaRrdpState, - #[serde(rename = "delta_count")] - pub delta_count: usize, - pub deltas: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] -pub struct ReplayDeltaRsyncFiles { - pub version: u32, - pub module: String, - #[serde(rename = "fileCount")] - pub file_count: usize, - pub files: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ReplayDeltaRrdpRepo { - pub notify_uri: String, - pub bucket_hash: String, - pub bucket_dir: PathBuf, - pub meta: ReplayRrdpRepoMeta, - pub transition: ReplayDeltaTransition, - pub target_notification_path: Option, - pub delta_paths: Vec<(u64, PathBuf)>, - pub target_archive_path: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ReplayDeltaRsyncModule { - pub module_uri: String, - pub bucket_hash: String, - pub bucket_dir: PathBuf, - pub meta: ReplayRsyncModuleMeta, - pub overlay_only: bool, - pub files: ReplayDeltaRsyncFiles, - pub tree_dir: PathBuf, - pub overlay_files: Vec<(String, PathBuf)>, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ReplayDeltaArchiveIndex { - pub archive_root: PathBuf, - pub capture_root: PathBuf, - pub delta_locks_path: PathBuf, - pub delta_locks: ReplayDeltaLocks, - pub capture_meta: crate::replay::archive::ReplayCaptureMeta, - pub base_meta: ReplayDeltaBaseMeta, - pub rrdp_repos: BTreeMap, - pub rsync_modules: BTreeMap, -} - -impl ReplayDeltaArchiveIndex { - pub fn load( - delta_archive_root: impl AsRef, - delta_locks_path: impl AsRef, - ) -> Result { - let archive_root = delta_archive_root.as_ref().to_path_buf(); - let delta_locks_path = delta_locks_path.as_ref().to_path_buf(); - - let delta_locks: ReplayDeltaLocks = - read_delta_json_file(&delta_locks_path, "payload delta locks")?; - ensure_delta_version("payload delta locks", delta_locks.version)?; - - let capture_root = archive_root - .join("v1") - .join("captures") - .join(&delta_locks.capture); - if !capture_root.is_dir() { - return Err(ReplayDeltaArchiveError::MissingDeltaCaptureDirectory( - capture_root.display().to_string(), - )); - } - - let capture_meta: crate::replay::archive::ReplayCaptureMeta = - read_delta_json_file(&capture_root.join("capture.json"), "delta capture meta")?; - ensure_delta_version("delta capture meta", capture_meta.version)?; - if capture_meta.capture_id != delta_locks.capture { - return Err(ReplayDeltaArchiveError::CaptureIdMismatch { - locks_capture: delta_locks.capture.clone(), - capture_json: capture_meta.capture_id.clone(), - }); - } - - let base_meta: ReplayDeltaBaseMeta = - read_delta_json_file(&capture_root.join("base.json"), "delta base meta")?; - ensure_delta_version("delta base meta", base_meta.version)?; - if base_meta.base_capture != delta_locks.base_capture { - return Err(ReplayDeltaArchiveError::BaseCaptureMismatch { - locks_base_capture: delta_locks.base_capture.clone(), - base_json_base_capture: base_meta.base_capture.clone(), - }); - } - if base_meta.base_locks_sha256.to_ascii_lowercase() - != delta_locks.base_locks_sha256.to_ascii_lowercase() - { - return Err(ReplayDeltaArchiveError::BaseLocksShaMismatch { - locks_sha256: delta_locks.base_locks_sha256.clone(), - base_json_sha256: base_meta.base_locks_sha256.clone(), - }); - } - - let mut rrdp_repos = BTreeMap::new(); - for (notify_uri, entry) in &delta_locks.rrdp { - let repo = load_delta_rrdp_repo(&capture_root, notify_uri, entry)?; - rrdp_repos.insert(notify_uri.clone(), repo); - } - - let mut rsync_modules = BTreeMap::new(); - for (module_uri, entry) in &delta_locks.rsync { - let module = load_delta_rsync_module(&capture_root, module_uri, entry)?; - rsync_modules.insert(module.module_uri.clone(), module); - } - - Ok(Self { - archive_root, - capture_root, - delta_locks_path, - delta_locks, - capture_meta, - base_meta, - rrdp_repos, - rsync_modules, - }) - } - - pub fn rrdp_repo(&self, notify_uri: &str) -> Option<&ReplayDeltaRrdpRepo> { - self.rrdp_repos.get(notify_uri) - } - - pub fn rsync_module(&self, module_uri: &str) -> Option<&ReplayDeltaRsyncModule> { - self.rsync_modules.get(module_uri) - } - - pub fn resolve_rsync_module_for_base_uri( - &self, - rsync_base_uri: &str, - ) -> Result<&ReplayDeltaRsyncModule, ReplayDeltaArchiveError> { - let module_uri = - canonical_rsync_module(rsync_base_uri).map_err(ReplayDeltaArchiveError::Base)?; - self.rsync_modules.get(&module_uri).ok_or_else(|| { - ReplayDeltaArchiveError::MissingRsyncModuleBucket { - module_uri, - path: "".to_string(), - } - }) - } - - pub fn validate_base_locks_sha256_bytes( - &self, - base_locks_bytes: &[u8], - ) -> Result<(), ReplayDeltaArchiveError> { - let actual = sha256_hex(base_locks_bytes); - let expected = self.delta_locks.base_locks_sha256.to_ascii_lowercase(); - if actual != expected { - return Err(ReplayDeltaArchiveError::BaseLocksBytesShaMismatch { expected, actual }); - } - Ok(()) - } - - pub fn validate_base_locks_sha256_file( - &self, - path: &Path, - ) -> Result<(), ReplayDeltaArchiveError> { - let bytes = fs::read(path).map_err(|e| { - ReplayDeltaArchiveError::Base(ReplayArchiveError::ReadFile { - entity: "base locks file", - path: path.display().to_string(), - detail: e.to_string(), - }) - })?; - self.validate_base_locks_sha256_bytes(&bytes) - } -} - -fn load_delta_rrdp_repo( - capture_root: &Path, - notify_uri: &str, - entry: &ReplayDeltaRrdpEntry, -) -> Result { - let bucket_hash = sha256_hex(notify_uri.as_bytes()); - let bucket_dir = capture_root.join("rrdp").join("repos").join(&bucket_hash); - if !bucket_dir.is_dir() { - return Err(ReplayDeltaArchiveError::MissingDeltaRepoBucket { - notify_uri: notify_uri.to_string(), - path: bucket_dir.display().to_string(), - }); - } - - let meta: ReplayRrdpRepoMeta = - read_delta_json_file(&bucket_dir.join("meta.json"), "delta RRDP repo meta")?; - ensure_delta_version("delta RRDP repo meta", meta.version)?; - if meta.rpki_notify != notify_uri { - return Err(ReplayDeltaArchiveError::RrdpMetaMismatch { - expected: notify_uri.to_string(), - actual: meta.rpki_notify.clone(), - }); - } - - let transition: ReplayDeltaTransition = - read_delta_json_file(&bucket_dir.join("transition.json"), "delta transition")?; - if transition.kind != entry.kind { - return Err(ReplayDeltaArchiveError::TransitionKindMismatch { - notify_uri: notify_uri.to_string(), - locks_kind: entry.kind.as_str().to_string(), - transition_kind: transition.kind.as_str().to_string(), - }); - } - if transition.base != entry.base { - return Err(ReplayDeltaArchiveError::TransitionBaseMismatch { - notify_uri: notify_uri.to_string(), - }); - } - if transition.target != entry.target { - return Err(ReplayDeltaArchiveError::TransitionTargetMismatch { - notify_uri: notify_uri.to_string(), - }); - } - if transition.delta_count != entry.delta_count || transition.deltas != entry.deltas { - return Err(ReplayDeltaArchiveError::DeltaSerialListMismatch { - notify_uri: notify_uri.to_string(), - }); - } - - let (target_notification_path, delta_paths, target_archive_path) = match entry.kind { - ReplayDeltaRrdpKind::Delta => { - let session = entry.target.session.as_ref().ok_or_else(|| { - ReplayDeltaArchiveError::TransitionTargetMismatch { - notify_uri: notify_uri.to_string(), - } - })?; - let serial = entry.target.serial.ok_or_else(|| { - ReplayDeltaArchiveError::TransitionTargetMismatch { - notify_uri: notify_uri.to_string(), - } - })?; - let session_dir = bucket_dir.join(session); - if !session_dir.is_dir() { - return Err(ReplayDeltaArchiveError::MissingDeltaSessionDir { - notify_uri: notify_uri.to_string(), - path: session_dir.display().to_string(), - }); - } - let notification = session_dir.join(format!("notification-target-{serial}.xml")); - if !notification.is_file() { - return Err(ReplayDeltaArchiveError::MissingTargetNotification { - notify_uri: notify_uri.to_string(), - path: notification.display().to_string(), - }); - } - let mut delta_paths = Vec::new(); - for delta_serial in &entry.deltas { - let pattern = format!("delta-{delta_serial}-"); - let deltas_dir = session_dir.join("deltas"); - let mut matches = if deltas_dir.is_dir() { - fs::read_dir(&deltas_dir) - .map_err(|e| { - ReplayDeltaArchiveError::Base(ReplayArchiveError::ReadFile { - entity: "delta deltas dir", - path: deltas_dir.display().to_string(), - detail: e.to_string(), - }) - })? - .filter_map(|entry| entry.ok().map(|e| e.path())) - .filter(|path| path.is_file()) - .filter(|path| { - path.file_name() - .and_then(|n| n.to_str()) - .is_some_and(|n| n.starts_with(&pattern) && n.ends_with(".xml")) - }) - .collect::>() - } else { - Vec::new() - }; - matches.sort(); - let path = matches.into_iter().next().ok_or_else(|| { - ReplayDeltaArchiveError::MissingDeltaFile { - notify_uri: notify_uri.to_string(), - serial: *delta_serial, - path: deltas_dir - .join(format!("delta-{delta_serial}-.xml")) - .display() - .to_string(), - } - })?; - delta_paths.push((*delta_serial, path)); - } - let target_archive = bucket_dir.join(format!("target-archive-{serial}.bin")); - let target_archive_path = if target_archive.is_file() { - Some(target_archive) - } else { - None - }; - (Some(notification), delta_paths, target_archive_path) - } - ReplayDeltaRrdpKind::Unchanged | ReplayDeltaRrdpKind::FallbackRsync => { - (None, Vec::new(), None) - } - ReplayDeltaRrdpKind::SessionReset | ReplayDeltaRrdpKind::Gap => (None, Vec::new(), None), - }; - - Ok(ReplayDeltaRrdpRepo { - notify_uri: notify_uri.to_string(), - bucket_hash, - bucket_dir, - meta, - transition, - target_notification_path, - delta_paths, - target_archive_path, - }) -} - -fn load_delta_rsync_module( - capture_root: &Path, - module_uri: &str, - entry: &ReplayDeltaRsyncEntry, -) -> Result { - let canonical = canonical_rsync_module(module_uri).map_err(ReplayDeltaArchiveError::Base)?; - let bucket_hash = sha256_hex(canonical.as_bytes()); - let bucket_dir = capture_root - .join("rsync") - .join("modules") - .join(&bucket_hash); - if !bucket_dir.is_dir() { - return Err(ReplayDeltaArchiveError::MissingRsyncModuleBucket { - module_uri: canonical.clone(), - path: bucket_dir.display().to_string(), - }); - } - - let meta_path = bucket_dir.join("meta.json"); - let meta: ReplayRsyncModuleMeta = if meta_path.is_file() { - let meta: ReplayRsyncModuleMeta = - read_delta_json_file(&meta_path, "delta rsync module meta")?; - ensure_delta_version("delta rsync module meta", meta.version)?; - if meta.module != canonical { - return Err(ReplayDeltaArchiveError::RsyncMetaMismatch { - expected: canonical.clone(), - actual: meta.module.clone(), - }); - } - meta - } else { - ReplayRsyncModuleMeta { - version: 1, - module: canonical.clone(), - created_at: String::new(), - last_seen_at: String::new(), - } - }; - - let files: ReplayDeltaRsyncFiles = - read_delta_json_file(&bucket_dir.join("files.json"), "delta rsync files")?; - ensure_delta_version("delta rsync files", files.version)?; - if files.module != canonical { - return Err(ReplayDeltaArchiveError::RsyncFilesModuleMismatch { - expected: canonical.clone(), - actual: files.module.clone(), - }); - } - if files.file_count != entry.file_count || files.file_count != files.files.len() { - return Err(ReplayDeltaArchiveError::RsyncFileCountMismatch { - module_uri: canonical.clone(), - declared: entry.file_count, - actual: files.files.len(), - }); - } - - let tree_dir = bucket_dir.join("tree"); - let mut overlay_files = Vec::new(); - for uri in &files.files { - let rel = uri.strip_prefix(&canonical).ok_or_else(|| { - ReplayDeltaArchiveError::RsyncFilesModuleMismatch { - expected: canonical.clone(), - actual: uri.clone(), - } - })?; - let tree_root = module_tree_root(&canonical, &tree_dir)?; - let path = tree_root.join(rel); - if !path.is_file() { - return Err(ReplayDeltaArchiveError::MissingRsyncOverlayFile { - module_uri: canonical.clone(), - path: path.display().to_string(), - }); - } - overlay_files.push((uri.clone(), path)); - } - - Ok(ReplayDeltaRsyncModule { - module_uri: canonical, - bucket_hash, - bucket_dir, - meta, - overlay_only: entry.overlay_only, - files, - tree_dir, - overlay_files, - }) -} - -fn module_tree_root(module_uri: &str, tree_dir: &Path) -> Result { - let rest = module_uri.strip_prefix("rsync://").ok_or_else(|| { - ReplayDeltaArchiveError::Base(ReplayArchiveError::InvalidRsyncUri { - uri: module_uri.to_string(), - detail: "URI must start with rsync://".to_string(), - }) - })?; - let mut parts = rest.trim_end_matches('/').split('/'); - let authority = parts.next().unwrap_or_default(); - let module = parts.next().unwrap_or_default(); - Ok(tree_dir.join(authority).join(module)) -} - -fn ensure_delta_version(entity: &'static str, version: u32) -> Result<(), ReplayDeltaArchiveError> { - if version == 1 { - Ok(()) - } else { - Err(ReplayDeltaArchiveError::Base( - ReplayArchiveError::UnsupportedVersion { entity, version }, - )) - } -} - -fn read_delta_json_file Deserialize<'de>>( - path: &Path, - entity: &'static str, -) -> Result { - let bytes = fs::read(path).map_err(|e| { - ReplayDeltaArchiveError::Base(ReplayArchiveError::ReadFile { - entity, - path: path.display().to_string(), - detail: e.to_string(), - }) - })?; - serde_json::from_slice(&bytes).map_err(|e| { - ReplayDeltaArchiveError::Base(ReplayArchiveError::ParseJson { - entity, - path: path.display().to_string(), - detail: e.to_string(), - }) - }) -} +include!("delta_archive/models.rs"); +include!("delta_archive/loaders.rs"); #[cfg(test)] -mod tests { - use super::*; - - fn build_delta_fixture() -> (tempfile::TempDir, PathBuf, PathBuf, String, String) { - let temp = tempfile::tempdir().expect("tempdir"); - let archive_root = temp.path().join("payload-delta-archive"); - let capture = "delta-cap"; - let base_capture = "base-cap"; - let base_sha = "deadbeef"; - let capture_root = archive_root.join("v1").join("captures").join(capture); - std::fs::create_dir_all(&capture_root).expect("mkdir capture root"); - std::fs::write( - capture_root.join("capture.json"), - format!( - r#"{{"version":1,"captureId":"{capture}","createdAt":"2026-03-15T00:00:00Z","notes":""}}"# - ), - ) - .expect("write capture json"); - std::fs::write( - capture_root.join("base.json"), - format!( - r#"{{"version":1,"baseCapture":"{base_capture}","baseLocksSha256":"{base_sha}","createdAt":"2026-03-15T00:00:00Z"}}"# - ), - ) - .expect("write base json"); - - let notify_uri = "https://rrdp.example.test/notification.xml".to_string(); - let session = "11111111-1111-1111-1111-111111111111".to_string(); - let _target_serial = 12u64; - let repo_hash = sha256_hex(notify_uri.as_bytes()); - let session_dir = capture_root - .join("rrdp/repos") - .join(&repo_hash) - .join(&session); - let deltas_dir = session_dir.join("deltas"); - std::fs::create_dir_all(&deltas_dir).expect("mkdir deltas"); - std::fs::write( - session_dir.parent().unwrap().join("meta.json"), - format!( - r#"{{"version":1,"rpkiNotify":"{notify_uri}","createdAt":"2026-03-15T00:00:00Z","lastSeenAt":"2026-03-15T00:00:01Z"}}"# - ), - ) - .expect("write meta"); - std::fs::write( - session_dir.parent().unwrap().join("transition.json"), - format!( - r#"{{"kind":"delta","base":{{"transport":"rrdp","session":"{session}","serial":10}},"target":{{"transport":"rrdp","session":"{session}","serial":12}},"delta_count":2,"deltas":[11,12]}}"# - ), - ) - .expect("write transition"); - std::fs::write( - session_dir.join("notification-target-12.xml"), - b"", - ) - .expect("write notification"); - std::fs::write( - deltas_dir.join("delta-11-aaaa.xml"), - b"", - ) - .expect("write delta 11"); - std::fs::write( - deltas_dir.join("delta-12-bbbb.xml"), - b"", - ) - .expect("write delta 12"); - std::fs::write( - session_dir.parent().unwrap().join("target-archive-12.bin"), - b"bin", - ) - .expect("write target archive"); - - let module_uri = "rsync://rsync.example.test/repo/".to_string(); - let module_hash = sha256_hex(module_uri.as_bytes()); - let module_bucket = capture_root.join("rsync/modules").join(&module_hash); - let tree_root = module_bucket - .join("tree") - .join("rsync.example.test") - .join("repo"); - std::fs::create_dir_all(tree_root.join("sub")).expect("mkdir tree root"); - std::fs::write( - module_bucket.join("meta.json"), - format!( - r#"{{"version":1,"module":"{module_uri}","createdAt":"2026-03-15T00:00:00Z","lastSeenAt":"2026-03-15T00:00:01Z"}}"# - ), - ) - .expect("write rsync meta"); - std::fs::write( - module_bucket.join("files.json"), - format!( - r#"{{"version":1,"module":"{module_uri}","fileCount":2,"files":["{module_uri}a.roa","{module_uri}sub/b.cer"]}}"# - ), - ) - .expect("write files json"); - std::fs::write(tree_root.join("a.roa"), b"roa").expect("write a.roa"); - std::fs::write(tree_root.join("sub").join("b.cer"), b"cer").expect("write b.cer"); - - let locks_path = temp.path().join("locks-delta.json"); - std::fs::write( - &locks_path, - format!( - r#"{{"version":1,"capture":"{capture}","baseCapture":"{base_capture}","baseLocksSha256":"{base_sha}","rrdp":{{"{notify_uri}":{{"kind":"delta","base":{{"transport":"rrdp","session":"{session}","serial":10}},"target":{{"transport":"rrdp","session":"{session}","serial":12}},"delta_count":2,"deltas":[11,12]}}}},"rsync":{{"{module_uri}":{{"file_count":2,"overlay_only":true}}}}}}"# - ), - ) - .expect("write locks-delta"); - (temp, archive_root, locks_path, notify_uri, module_uri) - } - - #[test] - fn delta_archive_index_loads_unchanged_and_fallback_rsync_rrdp_entries() { - let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); - let capture_root = archive_root.join("v1/captures/delta-cap"); - let repo_hash = sha256_hex(notify_uri.as_bytes()); - let repo_dir = capture_root.join("rrdp/repos").join(&repo_hash); - std::fs::write( - repo_dir.join("transition.json"), - r#"{"kind":"unchanged","base":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10},"target":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10},"delta_count":0,"deltas":[]}"#, - ) - .expect("rewrite transition unchanged"); - std::fs::write( - &locks_path, - r#"{"version":1,"capture":"delta-cap","baseCapture":"base-cap","baseLocksSha256":"deadbeef","rrdp":{"https://rrdp.example.test/notification.xml":{"kind":"unchanged","base":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10},"target":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10},"delta_count":0,"deltas":[]}},"rsync":{"rsync://rsync.example.test/repo/":{"file_count":2,"overlay_only":true}}}"#, - ).expect("rewrite locks unchanged"); - let index = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path) - .expect("load unchanged index"); - let repo = index.rrdp_repo(¬ify_uri).expect("rrdp repo"); - assert_eq!(repo.transition.kind, ReplayDeltaRrdpKind::Unchanged); - assert!(repo.target_notification_path.is_none()); - assert!(repo.delta_paths.is_empty()); - assert!(repo.target_archive_path.is_none()); - - std::fs::write( - repo_dir.join("transition.json"), - r#"{"kind":"fallback-rsync","base":{"transport":"rsync","session":null,"serial":null},"target":{"transport":"rsync","session":null,"serial":null},"delta_count":0,"deltas":[]}"#, - ).expect("rewrite transition fallback-rsync"); - std::fs::write( - &locks_path, - r#"{"version":1,"capture":"delta-cap","baseCapture":"base-cap","baseLocksSha256":"deadbeef","rrdp":{"https://rrdp.example.test/notification.xml":{"kind":"fallback-rsync","base":{"transport":"rsync","session":null,"serial":null},"target":{"transport":"rsync","session":null,"serial":null},"delta_count":0,"deltas":[]}},"rsync":{"rsync://rsync.example.test/repo/":{"file_count":2,"overlay_only":true}}}"#, - ).expect("rewrite locks fallback-rsync"); - let index = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path) - .expect("load fallback-rsync index"); - let repo = index.rrdp_repo(¬ify_uri).expect("rrdp repo"); - assert_eq!(repo.transition.kind, ReplayDeltaRrdpKind::FallbackRsync); - assert!(repo.target_notification_path.is_none()); - assert!(repo.delta_paths.is_empty()); - } - - #[test] - fn delta_archive_index_resolves_rsync_module_from_base_uri() { - let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_delta_fixture(); - let index = - ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).expect("load delta index"); - let module = index - .resolve_rsync_module_for_base_uri("rsync://rsync.example.test/repo/sub/path") - .expect("resolve module"); - assert_eq!(module.module_uri, "rsync://rsync.example.test/repo/"); - } - - #[test] - fn delta_archive_index_rejects_unsupported_versions_and_meta_mismatches() { - let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_delta_fixture(); - std::fs::write( - &locks_path, - r#"{"version":2,"capture":"delta-cap","baseCapture":"base-cap","baseLocksSha256":"deadbeef","rrdp":{},"rsync":{}}"#, - ).expect("rewrite locks version"); - let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); - assert!( - matches!( - err, - ReplayDeltaArchiveError::Base(ReplayArchiveError::UnsupportedVersion { - entity: "payload delta locks", - version: 2 - }) - ), - "{err}" - ); - - let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); - let repo_hash = sha256_hex(notify_uri.as_bytes()); - std::fs::write( - archive_root.join("v1/captures/delta-cap/rrdp/repos").join(&repo_hash).join("meta.json"), - r#"{"version":1,"rpkiNotify":"https://other.example/notification.xml","createdAt":"2026-03-15T00:00:00Z","lastSeenAt":"2026-03-15T00:00:01Z"}"#, - ).expect("rewrite rrdp meta"); - let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); - assert!( - matches!(err, ReplayDeltaArchiveError::RrdpMetaMismatch { .. }), - "{err}" - ); - - let (_temp, archive_root, locks_path, _notify_uri, module_uri) = build_delta_fixture(); - let module_hash = sha256_hex(module_uri.as_bytes()); - std::fs::write( - archive_root.join("v1/captures/delta-cap/rsync/modules").join(&module_hash).join("meta.json"), - r#"{"version":1,"module":"rsync://other.example/repo/","createdAt":"2026-03-15T00:00:00Z","lastSeenAt":"2026-03-15T00:00:01Z"}"#, - ).expect("rewrite rsync meta"); - let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); - assert!( - matches!(err, ReplayDeltaArchiveError::RsyncMetaMismatch { .. }), - "{err}" - ); - } - - #[test] - fn delta_archive_index_rejects_transition_base_target_and_serial_mismatches() { - let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); - let repo_hash = sha256_hex(notify_uri.as_bytes()); - let repo_dir = archive_root - .join("v1/captures/delta-cap/rrdp/repos") - .join(&repo_hash); - - std::fs::write( - repo_dir.join("transition.json"), - r#"{"kind":"delta","base":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":9},"target":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":12},"delta_count":2,"deltas":[11,12]}"#, - ).expect("rewrite transition base"); - let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); - assert!( - matches!(err, ReplayDeltaArchiveError::TransitionBaseMismatch { .. }), - "{err}" - ); - - let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); - let repo_hash = sha256_hex(notify_uri.as_bytes()); - let repo_dir = archive_root - .join("v1/captures/delta-cap/rrdp/repos") - .join(&repo_hash); - std::fs::write( - repo_dir.join("transition.json"), - r#"{"kind":"delta","base":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10},"target":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":13},"delta_count":2,"deltas":[11,12]}"#, - ).expect("rewrite transition target"); - let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); - assert!( - matches!( - err, - ReplayDeltaArchiveError::TransitionTargetMismatch { .. } - ), - "{err}" - ); - - let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); - let repo_hash = sha256_hex(notify_uri.as_bytes()); - let repo_dir = archive_root - .join("v1/captures/delta-cap/rrdp/repos") - .join(&repo_hash); - std::fs::write( - repo_dir.join("transition.json"), - r#"{"kind":"delta","base":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10},"target":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":12},"delta_count":1,"deltas":[12]}"#, - ).expect("rewrite transition deltas"); - let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); - assert!( - matches!(err, ReplayDeltaArchiveError::DeltaSerialListMismatch { .. }), - "{err}" - ); - } - - #[test] - fn delta_archive_index_rejects_missing_session_dir_and_overlay_files() { - let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); - let repo_hash = sha256_hex(notify_uri.as_bytes()); - let session_dir = archive_root - .join("v1/captures/delta-cap/rrdp/repos") - .join(&repo_hash) - .join("11111111-1111-1111-1111-111111111111"); - std::fs::remove_dir_all(&session_dir).expect("remove session dir"); - let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); - assert!( - matches!(err, ReplayDeltaArchiveError::MissingDeltaSessionDir { .. }), - "{err}" - ); - - let (_temp, archive_root, locks_path, _notify_uri, module_uri) = build_delta_fixture(); - let module_hash = sha256_hex(module_uri.as_bytes()); - let overlay_path = archive_root - .join("v1/captures/delta-cap/rsync/modules") - .join(&module_hash) - .join("tree/rsync.example.test/repo/sub/b.cer"); - std::fs::remove_file(overlay_path).expect("remove overlay file"); - let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); - assert!( - matches!(err, ReplayDeltaArchiveError::MissingRsyncOverlayFile { .. }), - "{err}" - ); - } - - #[test] - fn delta_archive_index_accepts_missing_rsync_module_meta_when_files_and_tree_exist() { - let (_temp, archive_root, locks_path, _notify_uri, module_uri) = build_delta_fixture(); - let module_hash = sha256_hex(module_uri.as_bytes()); - let meta_path = archive_root - .join("v1/captures/delta-cap/rsync/modules") - .join(module_hash) - .join("meta.json"); - std::fs::remove_file(&meta_path).expect("remove delta rsync module meta"); - - let index = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path) - .expect("load delta replay index without rsync meta"); - let module = index - .rsync_modules - .get(&module_uri) - .expect("module present"); - assert_eq!(module.meta.module, module_uri); - assert_eq!(module.meta.version, 1); - } - - #[test] - fn delta_archive_index_accepts_correct_base_locks_sha_and_rejects_missing_module_resolution() { - let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_delta_fixture(); - let index = - ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).expect("load delta index"); - let locks_bytes = std::fs::read(&locks_path).expect("read locks bytes"); - assert!( - index - .validate_base_locks_sha256_bytes(&locks_bytes) - .is_err() - ); - let err = index - .resolve_rsync_module_for_base_uri("rsync://missing.example/repo/path") - .unwrap_err(); - assert!( - matches!( - err, - ReplayDeltaArchiveError::MissingRsyncModuleBucket { .. } - ), - "{err}" - ); - } - - #[test] - fn delta_archive_index_loads_session_reset_and_gap_entries_without_target_files() { - let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); - let repo_hash = sha256_hex(notify_uri.as_bytes()); - let repo_dir = archive_root - .join("v1/captures/delta-cap/rrdp/repos") - .join(&repo_hash); - - for kind in ["session-reset", "gap"] { - std::fs::write( - repo_dir.join("transition.json"), - format!( - r#"{{"kind":"{kind}","base":{{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10}},"target":{{"transport":"rrdp","session":"22222222-2222-2222-2222-222222222222","serial":12}},"delta_count":0,"deltas":[]}}"#, - ), - ) - .expect("rewrite transition kind"); - std::fs::write( - &locks_path, - format!( - r#"{{"version":1,"capture":"delta-cap","baseCapture":"base-cap","baseLocksSha256":"deadbeef","rrdp":{{"{notify_uri}":{{"kind":"{kind}","base":{{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10}},"target":{{"transport":"rrdp","session":"22222222-2222-2222-2222-222222222222","serial":12}},"delta_count":0,"deltas":[]}}}},"rsync":{{"rsync://rsync.example.test/repo/":{{"file_count":2,"overlay_only":true}}}}}}"#, - ), - ) - .expect("rewrite locks kind"); - let index = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path) - .expect("load delta index"); - let repo = index.rrdp_repo(¬ify_uri).expect("rrdp repo"); - assert!(repo.target_notification_path.is_none()); - assert!(repo.delta_paths.is_empty()); - } - } - #[test] - fn delta_archive_index_loads_rrdp_and_rsync_entries() { - let (_temp, archive_root, locks_path, notify_uri, module_uri) = build_delta_fixture(); - let index = - ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).expect("load delta index"); - assert_eq!(index.capture_meta.capture_id, "delta-cap"); - assert_eq!(index.base_meta.base_capture, "base-cap"); - assert_eq!(index.rrdp_repos.len(), 1); - assert_eq!(index.rsync_modules.len(), 1); - - let repo = index.rrdp_repo(¬ify_uri).expect("rrdp repo"); - assert_eq!(repo.transition.kind, ReplayDeltaRrdpKind::Delta); - assert_eq!(repo.transition.delta_count, 2); - assert_eq!(repo.delta_paths.len(), 2); - assert!(repo.target_notification_path.as_ref().unwrap().is_file()); - assert!(repo.target_archive_path.as_ref().unwrap().is_file()); - - let module = index.rsync_module(&module_uri).expect("rsync module"); - assert_eq!(module.files.file_count, 2); - assert_eq!(module.overlay_files.len(), 2); - assert!(module.overlay_files.iter().all(|(_, path)| path.is_file())); - } - - #[test] - fn delta_archive_index_rejects_capture_and_sha_mismatches() { - let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_delta_fixture(); - let capture_root = archive_root.join("v1/captures/delta-cap"); - std::fs::write( - capture_root.join("capture.json"), - r#"{"version":1,"captureId":"other-cap","createdAt":"2026-03-15T00:00:00Z","notes":""}"#, - ) - .expect("rewrite capture json"); - let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); - assert!( - matches!(err, ReplayDeltaArchiveError::CaptureIdMismatch { .. }), - "{err}" - ); - - let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_delta_fixture(); - let capture_root = archive_root.join("v1/captures/delta-cap"); - std::fs::write( - capture_root.join("base.json"), - r#"{"version":1,"baseCapture":"base-cap","baseLocksSha256":"beefdead","createdAt":"2026-03-15T00:00:00Z"}"#, - ) - .expect("rewrite base json sha"); - let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); - assert!( - matches!(err, ReplayDeltaArchiveError::BaseLocksShaMismatch { .. }), - "{err}" - ); - } - - #[test] - fn delta_archive_index_rejects_missing_target_notification_and_repo_bucket() { - let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); - let repo_hash = sha256_hex(notify_uri.as_bytes()); - let session_dir = archive_root - .join("v1/captures/delta-cap/rrdp/repos") - .join(&repo_hash) - .join("11111111-1111-1111-1111-111111111111"); - std::fs::remove_file(session_dir.join("notification-target-12.xml")) - .expect("remove target notification"); - let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); - assert!( - matches!( - err, - ReplayDeltaArchiveError::MissingTargetNotification { .. } - ), - "{err}" - ); - - let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); - let repo_hash = sha256_hex(notify_uri.as_bytes()); - let repo_dir = archive_root - .join("v1/captures/delta-cap/rrdp/repos") - .join(&repo_hash); - std::fs::remove_dir_all(repo_dir).expect("remove repo dir"); - let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); - assert!( - matches!(err, ReplayDeltaArchiveError::MissingDeltaRepoBucket { .. }), - "{err}" - ); - } - #[test] - fn delta_archive_index_rejects_base_meta_mismatch() { - let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_delta_fixture(); - let capture_root = archive_root.join("v1/captures/delta-cap"); - std::fs::write( - capture_root.join("base.json"), - r#"{"version":1,"baseCapture":"other","baseLocksSha256":"deadbeef","createdAt":"2026-03-15T00:00:00Z"}"#, - ) - .expect("rewrite base json"); - let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); - assert!( - matches!(err, ReplayDeltaArchiveError::BaseCaptureMismatch { .. }), - "{err}" - ); - } - - #[test] - fn delta_archive_index_rejects_transition_mismatch_and_missing_delta_file() { - let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); - let repo_hash = sha256_hex(notify_uri.as_bytes()); - let repo_dir = archive_root - .join("v1/captures/delta-cap/rrdp/repos") - .join(&repo_hash); - std::fs::write( - repo_dir.join("transition.json"), - r#"{"kind":"unchanged","base":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10},"target":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":12},"delta_count":2,"deltas":[11,12]}"#, - ) - .expect("rewrite transition"); - let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); - assert!( - matches!(err, ReplayDeltaArchiveError::TransitionKindMismatch { .. }), - "{err}" - ); - - let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); - let repo_hash = sha256_hex(notify_uri.as_bytes()); - let delta_path = archive_root - .join("v1/captures/delta-cap/rrdp/repos") - .join(&repo_hash) - .join("11111111-1111-1111-1111-111111111111/deltas/delta-12-bbbb.xml"); - std::fs::remove_file(delta_path).expect("remove delta"); - let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); - assert!( - matches!(err, ReplayDeltaArchiveError::MissingDeltaFile { .. }), - "{err}" - ); - } - - #[test] - fn delta_archive_index_validates_base_locks_sha256_bytes_and_file() { - let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_delta_fixture(); - let index = - ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).expect("load delta index"); - let err = index - .validate_base_locks_sha256_bytes(b"not-the-right-base-locks") - .unwrap_err(); - assert!( - matches!( - err, - ReplayDeltaArchiveError::BaseLocksBytesShaMismatch { .. } - ), - "{err}" - ); - - let temp_file = tempfile::NamedTempFile::new().expect("tempfile"); - std::fs::write(temp_file.path(), b"still-wrong").expect("write base locks file"); - let err = index - .validate_base_locks_sha256_file(temp_file.path()) - .unwrap_err(); - assert!( - matches!( - err, - ReplayDeltaArchiveError::BaseLocksBytesShaMismatch { .. } - ), - "{err}" - ); - } - - #[test] - fn delta_archive_index_rejects_rsync_files_mismatch() { - let (_temp, archive_root, locks_path, _notify_uri, module_uri) = build_delta_fixture(); - let module_hash = sha256_hex(module_uri.as_bytes()); - let module_dir = archive_root - .join("v1/captures/delta-cap/rsync/modules") - .join(&module_hash); - std::fs::write( - module_dir.join("files.json"), - format!( - r#"{{"version":1,"module":"{module_uri}","fileCount":3,"files":["{module_uri}a.roa"]}}"# - ), - ) - .expect("rewrite files json"); - let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); - assert!( - matches!(err, ReplayDeltaArchiveError::RsyncFileCountMismatch { .. }), - "{err}" - ); - } -} +#[path = "delta_archive/tests.rs"] +mod tests; diff --git a/crates/panda-rpki-validator/src/replay/delta_archive/loaders.rs b/crates/panda-rpki-validator/src/replay/delta_archive/loaders.rs new file mode 100644 index 0000000..d8fa54f --- /dev/null +++ b/crates/panda-rpki-validator/src/replay/delta_archive/loaders.rs @@ -0,0 +1,270 @@ +// Delta archive capture and filesystem loading helpers. + +fn load_delta_rrdp_repo( + capture_root: &Path, + notify_uri: &str, + entry: &ReplayDeltaRrdpEntry, +) -> Result { + let bucket_hash = sha256_hex(notify_uri.as_bytes()); + let bucket_dir = capture_root.join("rrdp").join("repos").join(&bucket_hash); + if !bucket_dir.is_dir() { + return Err(ReplayDeltaArchiveError::MissingDeltaRepoBucket { + notify_uri: notify_uri.to_string(), + path: bucket_dir.display().to_string(), + }); + } + + let meta: ReplayRrdpRepoMeta = + read_delta_json_file(&bucket_dir.join("meta.json"), "delta RRDP repo meta")?; + ensure_delta_version("delta RRDP repo meta", meta.version)?; + if meta.rpki_notify != notify_uri { + return Err(ReplayDeltaArchiveError::RrdpMetaMismatch { + expected: notify_uri.to_string(), + actual: meta.rpki_notify.clone(), + }); + } + + let transition: ReplayDeltaTransition = + read_delta_json_file(&bucket_dir.join("transition.json"), "delta transition")?; + if transition.kind != entry.kind { + return Err(ReplayDeltaArchiveError::TransitionKindMismatch { + notify_uri: notify_uri.to_string(), + locks_kind: entry.kind.as_str().to_string(), + transition_kind: transition.kind.as_str().to_string(), + }); + } + if transition.base != entry.base { + return Err(ReplayDeltaArchiveError::TransitionBaseMismatch { + notify_uri: notify_uri.to_string(), + }); + } + if transition.target != entry.target { + return Err(ReplayDeltaArchiveError::TransitionTargetMismatch { + notify_uri: notify_uri.to_string(), + }); + } + if transition.delta_count != entry.delta_count || transition.deltas != entry.deltas { + return Err(ReplayDeltaArchiveError::DeltaSerialListMismatch { + notify_uri: notify_uri.to_string(), + }); + } + + let (target_notification_path, delta_paths, target_archive_path) = match entry.kind { + ReplayDeltaRrdpKind::Delta => { + let session = entry.target.session.as_ref().ok_or_else(|| { + ReplayDeltaArchiveError::TransitionTargetMismatch { + notify_uri: notify_uri.to_string(), + } + })?; + let serial = entry.target.serial.ok_or_else(|| { + ReplayDeltaArchiveError::TransitionTargetMismatch { + notify_uri: notify_uri.to_string(), + } + })?; + let session_dir = bucket_dir.join(session); + if !session_dir.is_dir() { + return Err(ReplayDeltaArchiveError::MissingDeltaSessionDir { + notify_uri: notify_uri.to_string(), + path: session_dir.display().to_string(), + }); + } + let notification = session_dir.join(format!("notification-target-{serial}.xml")); + if !notification.is_file() { + return Err(ReplayDeltaArchiveError::MissingTargetNotification { + notify_uri: notify_uri.to_string(), + path: notification.display().to_string(), + }); + } + let mut delta_paths = Vec::new(); + for delta_serial in &entry.deltas { + let pattern = format!("delta-{delta_serial}-"); + let deltas_dir = session_dir.join("deltas"); + let mut matches = if deltas_dir.is_dir() { + fs::read_dir(&deltas_dir) + .map_err(|e| { + ReplayDeltaArchiveError::Base(ReplayArchiveError::ReadFile { + entity: "delta deltas dir", + path: deltas_dir.display().to_string(), + detail: e.to_string(), + }) + })? + .filter_map(|entry| entry.ok().map(|e| e.path())) + .filter(|path| path.is_file()) + .filter(|path| { + path.file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n.starts_with(&pattern) && n.ends_with(".xml")) + }) + .collect::>() + } else { + Vec::new() + }; + matches.sort(); + let path = matches.into_iter().next().ok_or_else(|| { + ReplayDeltaArchiveError::MissingDeltaFile { + notify_uri: notify_uri.to_string(), + serial: *delta_serial, + path: deltas_dir + .join(format!("delta-{delta_serial}-.xml")) + .display() + .to_string(), + } + })?; + delta_paths.push((*delta_serial, path)); + } + let target_archive = bucket_dir.join(format!("target-archive-{serial}.bin")); + let target_archive_path = if target_archive.is_file() { + Some(target_archive) + } else { + None + }; + (Some(notification), delta_paths, target_archive_path) + } + ReplayDeltaRrdpKind::Unchanged | ReplayDeltaRrdpKind::FallbackRsync => { + (None, Vec::new(), None) + } + ReplayDeltaRrdpKind::SessionReset | ReplayDeltaRrdpKind::Gap => (None, Vec::new(), None), + }; + + Ok(ReplayDeltaRrdpRepo { + notify_uri: notify_uri.to_string(), + bucket_hash, + bucket_dir, + meta, + transition, + target_notification_path, + delta_paths, + target_archive_path, + }) +} + +fn load_delta_rsync_module( + capture_root: &Path, + module_uri: &str, + entry: &ReplayDeltaRsyncEntry, +) -> Result { + let canonical = canonical_rsync_module(module_uri).map_err(ReplayDeltaArchiveError::Base)?; + let bucket_hash = sha256_hex(canonical.as_bytes()); + let bucket_dir = capture_root + .join("rsync") + .join("modules") + .join(&bucket_hash); + if !bucket_dir.is_dir() { + return Err(ReplayDeltaArchiveError::MissingRsyncModuleBucket { + module_uri: canonical.clone(), + path: bucket_dir.display().to_string(), + }); + } + + let meta_path = bucket_dir.join("meta.json"); + let meta: ReplayRsyncModuleMeta = if meta_path.is_file() { + let meta: ReplayRsyncModuleMeta = + read_delta_json_file(&meta_path, "delta rsync module meta")?; + ensure_delta_version("delta rsync module meta", meta.version)?; + if meta.module != canonical { + return Err(ReplayDeltaArchiveError::RsyncMetaMismatch { + expected: canonical.clone(), + actual: meta.module.clone(), + }); + } + meta + } else { + ReplayRsyncModuleMeta { + version: 1, + module: canonical.clone(), + created_at: String::new(), + last_seen_at: String::new(), + } + }; + + let files: ReplayDeltaRsyncFiles = + read_delta_json_file(&bucket_dir.join("files.json"), "delta rsync files")?; + ensure_delta_version("delta rsync files", files.version)?; + if files.module != canonical { + return Err(ReplayDeltaArchiveError::RsyncFilesModuleMismatch { + expected: canonical.clone(), + actual: files.module.clone(), + }); + } + if files.file_count != entry.file_count || files.file_count != files.files.len() { + return Err(ReplayDeltaArchiveError::RsyncFileCountMismatch { + module_uri: canonical.clone(), + declared: entry.file_count, + actual: files.files.len(), + }); + } + + let tree_dir = bucket_dir.join("tree"); + let mut overlay_files = Vec::new(); + for uri in &files.files { + let rel = uri.strip_prefix(&canonical).ok_or_else(|| { + ReplayDeltaArchiveError::RsyncFilesModuleMismatch { + expected: canonical.clone(), + actual: uri.clone(), + } + })?; + let tree_root = module_tree_root(&canonical, &tree_dir)?; + let path = tree_root.join(rel); + if !path.is_file() { + return Err(ReplayDeltaArchiveError::MissingRsyncOverlayFile { + module_uri: canonical.clone(), + path: path.display().to_string(), + }); + } + overlay_files.push((uri.clone(), path)); + } + + Ok(ReplayDeltaRsyncModule { + module_uri: canonical, + bucket_hash, + bucket_dir, + meta, + overlay_only: entry.overlay_only, + files, + tree_dir, + overlay_files, + }) +} + +fn module_tree_root(module_uri: &str, tree_dir: &Path) -> Result { + let rest = module_uri.strip_prefix("rsync://").ok_or_else(|| { + ReplayDeltaArchiveError::Base(ReplayArchiveError::InvalidRsyncUri { + uri: module_uri.to_string(), + detail: "URI must start with rsync://".to_string(), + }) + })?; + let mut parts = rest.trim_end_matches('/').split('/'); + let authority = parts.next().unwrap_or_default(); + let module = parts.next().unwrap_or_default(); + Ok(tree_dir.join(authority).join(module)) +} + +fn ensure_delta_version(entity: &'static str, version: u32) -> Result<(), ReplayDeltaArchiveError> { + if version == 1 { + Ok(()) + } else { + Err(ReplayDeltaArchiveError::Base( + ReplayArchiveError::UnsupportedVersion { entity, version }, + )) + } +} + +fn read_delta_json_file Deserialize<'de>>( + path: &Path, + entity: &'static str, +) -> Result { + let bytes = fs::read(path).map_err(|e| { + ReplayDeltaArchiveError::Base(ReplayArchiveError::ReadFile { + entity, + path: path.display().to_string(), + detail: e.to_string(), + }) + })?; + serde_json::from_slice(&bytes).map_err(|e| { + ReplayDeltaArchiveError::Base(ReplayArchiveError::ParseJson { + entity, + path: path.display().to_string(), + detail: e.to_string(), + }) + }) +} diff --git a/crates/panda-rpki-validator/src/replay/delta_archive/models.rs b/crates/panda-rpki-validator/src/replay/delta_archive/models.rs new file mode 100644 index 0000000..b6d466e --- /dev/null +++ b/crates/panda-rpki-validator/src/replay/delta_archive/models.rs @@ -0,0 +1,344 @@ +// Payload-delta lock, capture, and archive index models. + +#[derive(Debug, thiserror::Error)] +pub enum ReplayDeltaArchiveError { + #[error(transparent)] + Base(#[from] ReplayArchiveError), + + #[error("delta capture directory not found: {0}")] + MissingDeltaCaptureDirectory(String), + + #[error("delta capture.json captureId mismatch: locks={locks_capture}, capture={capture_json}")] + CaptureIdMismatch { + locks_capture: String, + capture_json: String, + }, + + #[error( + "delta base.json baseCapture mismatch: locks={locks_base_capture}, base_json={base_json_base_capture}" + )] + BaseCaptureMismatch { + locks_base_capture: String, + base_json_base_capture: String, + }, + + #[error( + "delta base.json baseLocksSha256 mismatch: locks={locks_sha256}, base_json={base_json_sha256}" + )] + BaseLocksShaMismatch { + locks_sha256: String, + base_json_sha256: String, + }, + + #[error("base locks sha256 mismatch: expected {expected}, actual {actual}")] + BaseLocksBytesShaMismatch { expected: String, actual: String }, + + #[error("delta repo bucket not found for {notify_uri}: {path}")] + MissingDeltaRepoBucket { notify_uri: String, path: String }, + + #[error("delta repo meta mismatch: expected {expected}, actual {actual}")] + RrdpMetaMismatch { expected: String, actual: String }, + + #[error( + "delta transition kind mismatch for {notify_uri}: locks={locks_kind}, transition={transition_kind}" + )] + TransitionKindMismatch { + notify_uri: String, + locks_kind: String, + transition_kind: String, + }, + + #[error("delta transition base mismatch for {notify_uri}")] + TransitionBaseMismatch { notify_uri: String }, + + #[error("delta transition target mismatch for {notify_uri}")] + TransitionTargetMismatch { notify_uri: String }, + + #[error("delta serial list mismatch for {notify_uri}")] + DeltaSerialListMismatch { notify_uri: String }, + + #[error("delta notification session directory not found for {notify_uri}: {path}")] + MissingDeltaSessionDir { notify_uri: String, path: String }, + + #[error("target notification file not found for {notify_uri}: {path}")] + MissingTargetNotification { notify_uri: String, path: String }, + + #[error("delta file not found for {notify_uri} serial={serial}: {path}")] + MissingDeltaFile { + notify_uri: String, + serial: u64, + path: String, + }, + + #[error("delta target archive missing for {notify_uri}: {path}")] + MissingTargetArchive { notify_uri: String, path: String }, + + #[error("delta rsync module bucket not found for {module_uri}: {path}")] + MissingRsyncModuleBucket { module_uri: String, path: String }, + + #[error("delta rsync module meta mismatch: expected {expected}, actual {actual}")] + RsyncMetaMismatch { expected: String, actual: String }, + + #[error("delta rsync files.json module mismatch: expected {expected}, actual {actual}")] + RsyncFilesModuleMismatch { expected: String, actual: String }, + + #[error( + "delta rsync file count mismatch for {module_uri}: declared={declared}, actual={actual}" + )] + RsyncFileCountMismatch { + module_uri: String, + declared: usize, + actual: usize, + }, + + #[error("delta rsync overlay file not found for {module_uri}: {path}")] + MissingRsyncOverlayFile { module_uri: String, path: String }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub struct ReplayDeltaLocks { + pub version: u32, + pub capture: String, + #[serde(rename = "baseCapture")] + pub base_capture: String, + #[serde(rename = "baseLocksSha256")] + pub base_locks_sha256: String, + pub rrdp: BTreeMap, + pub rsync: BTreeMap, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub struct ReplayDeltaBaseMeta { + pub version: u32, + #[serde(rename = "baseCapture")] + pub base_capture: String, + #[serde(rename = "baseLocksSha256")] + pub base_locks_sha256: String, + #[serde(rename = "createdAt")] + pub created_at: String, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ReplayDeltaRrdpKind { + Unchanged, + Delta, + FallbackRsync, + SessionReset, + Gap, +} + +impl ReplayDeltaRrdpKind { + pub fn as_str(self) -> &'static str { + match self { + Self::Unchanged => "unchanged", + Self::Delta => "delta", + Self::FallbackRsync => "fallback-rsync", + Self::SessionReset => "session-reset", + Self::Gap => "gap", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub struct ReplayDeltaRrdpState { + pub transport: ReplayTransport, + pub session: Option, + pub serial: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub struct ReplayDeltaRrdpEntry { + pub kind: ReplayDeltaRrdpKind, + pub base: ReplayDeltaRrdpState, + pub target: ReplayDeltaRrdpState, + #[serde(rename = "delta_count")] + pub delta_count: usize, + pub deltas: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub struct ReplayDeltaRsyncEntry { + #[serde(rename = "file_count")] + pub file_count: usize, + #[serde(rename = "overlay_only")] + pub overlay_only: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub struct ReplayDeltaTransition { + pub kind: ReplayDeltaRrdpKind, + pub base: ReplayDeltaRrdpState, + pub target: ReplayDeltaRrdpState, + #[serde(rename = "delta_count")] + pub delta_count: usize, + pub deltas: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub struct ReplayDeltaRsyncFiles { + pub version: u32, + pub module: String, + #[serde(rename = "fileCount")] + pub file_count: usize, + pub files: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ReplayDeltaRrdpRepo { + pub notify_uri: String, + pub bucket_hash: String, + pub bucket_dir: PathBuf, + pub meta: ReplayRrdpRepoMeta, + pub transition: ReplayDeltaTransition, + pub target_notification_path: Option, + pub delta_paths: Vec<(u64, PathBuf)>, + pub target_archive_path: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ReplayDeltaRsyncModule { + pub module_uri: String, + pub bucket_hash: String, + pub bucket_dir: PathBuf, + pub meta: ReplayRsyncModuleMeta, + pub overlay_only: bool, + pub files: ReplayDeltaRsyncFiles, + pub tree_dir: PathBuf, + pub overlay_files: Vec<(String, PathBuf)>, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ReplayDeltaArchiveIndex { + pub archive_root: PathBuf, + pub capture_root: PathBuf, + pub delta_locks_path: PathBuf, + pub delta_locks: ReplayDeltaLocks, + pub capture_meta: crate::replay::archive::ReplayCaptureMeta, + pub base_meta: ReplayDeltaBaseMeta, + pub rrdp_repos: BTreeMap, + pub rsync_modules: BTreeMap, +} + +impl ReplayDeltaArchiveIndex { + pub fn load( + delta_archive_root: impl AsRef, + delta_locks_path: impl AsRef, + ) -> Result { + let archive_root = delta_archive_root.as_ref().to_path_buf(); + let delta_locks_path = delta_locks_path.as_ref().to_path_buf(); + + let delta_locks: ReplayDeltaLocks = + read_delta_json_file(&delta_locks_path, "payload delta locks")?; + ensure_delta_version("payload delta locks", delta_locks.version)?; + + let capture_root = archive_root + .join("v1") + .join("captures") + .join(&delta_locks.capture); + if !capture_root.is_dir() { + return Err(ReplayDeltaArchiveError::MissingDeltaCaptureDirectory( + capture_root.display().to_string(), + )); + } + + let capture_meta: crate::replay::archive::ReplayCaptureMeta = + read_delta_json_file(&capture_root.join("capture.json"), "delta capture meta")?; + ensure_delta_version("delta capture meta", capture_meta.version)?; + if capture_meta.capture_id != delta_locks.capture { + return Err(ReplayDeltaArchiveError::CaptureIdMismatch { + locks_capture: delta_locks.capture.clone(), + capture_json: capture_meta.capture_id.clone(), + }); + } + + let base_meta: ReplayDeltaBaseMeta = + read_delta_json_file(&capture_root.join("base.json"), "delta base meta")?; + ensure_delta_version("delta base meta", base_meta.version)?; + if base_meta.base_capture != delta_locks.base_capture { + return Err(ReplayDeltaArchiveError::BaseCaptureMismatch { + locks_base_capture: delta_locks.base_capture.clone(), + base_json_base_capture: base_meta.base_capture.clone(), + }); + } + if base_meta.base_locks_sha256.to_ascii_lowercase() + != delta_locks.base_locks_sha256.to_ascii_lowercase() + { + return Err(ReplayDeltaArchiveError::BaseLocksShaMismatch { + locks_sha256: delta_locks.base_locks_sha256.clone(), + base_json_sha256: base_meta.base_locks_sha256.clone(), + }); + } + + let mut rrdp_repos = BTreeMap::new(); + for (notify_uri, entry) in &delta_locks.rrdp { + let repo = load_delta_rrdp_repo(&capture_root, notify_uri, entry)?; + rrdp_repos.insert(notify_uri.clone(), repo); + } + + let mut rsync_modules = BTreeMap::new(); + for (module_uri, entry) in &delta_locks.rsync { + let module = load_delta_rsync_module(&capture_root, module_uri, entry)?; + rsync_modules.insert(module.module_uri.clone(), module); + } + + Ok(Self { + archive_root, + capture_root, + delta_locks_path, + delta_locks, + capture_meta, + base_meta, + rrdp_repos, + rsync_modules, + }) + } + + pub fn rrdp_repo(&self, notify_uri: &str) -> Option<&ReplayDeltaRrdpRepo> { + self.rrdp_repos.get(notify_uri) + } + + pub fn rsync_module(&self, module_uri: &str) -> Option<&ReplayDeltaRsyncModule> { + self.rsync_modules.get(module_uri) + } + + pub fn resolve_rsync_module_for_base_uri( + &self, + rsync_base_uri: &str, + ) -> Result<&ReplayDeltaRsyncModule, ReplayDeltaArchiveError> { + let module_uri = + canonical_rsync_module(rsync_base_uri).map_err(ReplayDeltaArchiveError::Base)?; + self.rsync_modules.get(&module_uri).ok_or_else(|| { + ReplayDeltaArchiveError::MissingRsyncModuleBucket { + module_uri, + path: "".to_string(), + } + }) + } + + pub fn validate_base_locks_sha256_bytes( + &self, + base_locks_bytes: &[u8], + ) -> Result<(), ReplayDeltaArchiveError> { + let actual = sha256_hex(base_locks_bytes); + let expected = self.delta_locks.base_locks_sha256.to_ascii_lowercase(); + if actual != expected { + return Err(ReplayDeltaArchiveError::BaseLocksBytesShaMismatch { expected, actual }); + } + Ok(()) + } + + pub fn validate_base_locks_sha256_file( + &self, + path: &Path, + ) -> Result<(), ReplayDeltaArchiveError> { + let bytes = fs::read(path).map_err(|e| { + ReplayDeltaArchiveError::Base(ReplayArchiveError::ReadFile { + entity: "base locks file", + path: path.display().to_string(), + detail: e.to_string(), + }) + })?; + self.validate_base_locks_sha256_bytes(&bytes) + } +} diff --git a/crates/panda-rpki-validator/src/replay/delta_archive/tests.rs b/crates/panda-rpki-validator/src/replay/delta_archive/tests.rs new file mode 100644 index 0000000..c08c851 --- /dev/null +++ b/crates/panda-rpki-validator/src/replay/delta_archive/tests.rs @@ -0,0 +1,537 @@ +// Payload-delta archive loading and validation tests. + +use super::*; + +fn build_delta_fixture() -> (tempfile::TempDir, PathBuf, PathBuf, String, String) { + let temp = tempfile::tempdir().expect("tempdir"); + let archive_root = temp.path().join("payload-delta-archive"); + let capture = "delta-cap"; + let base_capture = "base-cap"; + let base_sha = "deadbeef"; + let capture_root = archive_root.join("v1").join("captures").join(capture); + std::fs::create_dir_all(&capture_root).expect("mkdir capture root"); + std::fs::write( + capture_root.join("capture.json"), + format!( + r#"{{"version":1,"captureId":"{capture}","createdAt":"2026-03-15T00:00:00Z","notes":""}}"# + ), + ) + .expect("write capture json"); + std::fs::write( + capture_root.join("base.json"), + format!( + r#"{{"version":1,"baseCapture":"{base_capture}","baseLocksSha256":"{base_sha}","createdAt":"2026-03-15T00:00:00Z"}}"# + ), + ) + .expect("write base json"); + + let notify_uri = "https://rrdp.example.test/notification.xml".to_string(); + let session = "11111111-1111-1111-1111-111111111111".to_string(); + let _target_serial = 12u64; + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let session_dir = capture_root + .join("rrdp/repos") + .join(&repo_hash) + .join(&session); + let deltas_dir = session_dir.join("deltas"); + std::fs::create_dir_all(&deltas_dir).expect("mkdir deltas"); + std::fs::write( + session_dir.parent().unwrap().join("meta.json"), + format!( + r#"{{"version":1,"rpkiNotify":"{notify_uri}","createdAt":"2026-03-15T00:00:00Z","lastSeenAt":"2026-03-15T00:00:01Z"}}"# + ), + ) + .expect("write meta"); + std::fs::write( + session_dir.parent().unwrap().join("transition.json"), + format!( + r#"{{"kind":"delta","base":{{"transport":"rrdp","session":"{session}","serial":10}},"target":{{"transport":"rrdp","session":"{session}","serial":12}},"delta_count":2,"deltas":[11,12]}}"# + ), + ) + .expect("write transition"); + std::fs::write( + session_dir.join("notification-target-12.xml"), + b"", + ) + .expect("write notification"); + std::fs::write( + deltas_dir.join("delta-11-aaaa.xml"), + b"", + ) + .expect("write delta 11"); + std::fs::write( + deltas_dir.join("delta-12-bbbb.xml"), + b"", + ) + .expect("write delta 12"); + std::fs::write( + session_dir.parent().unwrap().join("target-archive-12.bin"), + b"bin", + ) + .expect("write target archive"); + + let module_uri = "rsync://rsync.example.test/repo/".to_string(); + let module_hash = sha256_hex(module_uri.as_bytes()); + let module_bucket = capture_root.join("rsync/modules").join(&module_hash); + let tree_root = module_bucket + .join("tree") + .join("rsync.example.test") + .join("repo"); + std::fs::create_dir_all(tree_root.join("sub")).expect("mkdir tree root"); + std::fs::write( + module_bucket.join("meta.json"), + format!( + r#"{{"version":1,"module":"{module_uri}","createdAt":"2026-03-15T00:00:00Z","lastSeenAt":"2026-03-15T00:00:01Z"}}"# + ), + ) + .expect("write rsync meta"); + std::fs::write( + module_bucket.join("files.json"), + format!( + r#"{{"version":1,"module":"{module_uri}","fileCount":2,"files":["{module_uri}a.roa","{module_uri}sub/b.cer"]}}"# + ), + ) + .expect("write files json"); + std::fs::write(tree_root.join("a.roa"), b"roa").expect("write a.roa"); + std::fs::write(tree_root.join("sub").join("b.cer"), b"cer").expect("write b.cer"); + + let locks_path = temp.path().join("locks-delta.json"); + std::fs::write( + &locks_path, + format!( + r#"{{"version":1,"capture":"{capture}","baseCapture":"{base_capture}","baseLocksSha256":"{base_sha}","rrdp":{{"{notify_uri}":{{"kind":"delta","base":{{"transport":"rrdp","session":"{session}","serial":10}},"target":{{"transport":"rrdp","session":"{session}","serial":12}},"delta_count":2,"deltas":[11,12]}}}},"rsync":{{"{module_uri}":{{"file_count":2,"overlay_only":true}}}}}}"# + ), + ) + .expect("write locks-delta"); + (temp, archive_root, locks_path, notify_uri, module_uri) +} + +#[test] +fn delta_archive_index_loads_unchanged_and_fallback_rsync_rrdp_entries() { + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); + let capture_root = archive_root.join("v1/captures/delta-cap"); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let repo_dir = capture_root.join("rrdp/repos").join(&repo_hash); + std::fs::write( + repo_dir.join("transition.json"), + r#"{"kind":"unchanged","base":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10},"target":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10},"delta_count":0,"deltas":[]}"#, + ) + .expect("rewrite transition unchanged"); + std::fs::write( + &locks_path, + r#"{"version":1,"capture":"delta-cap","baseCapture":"base-cap","baseLocksSha256":"deadbeef","rrdp":{"https://rrdp.example.test/notification.xml":{"kind":"unchanged","base":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10},"target":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10},"delta_count":0,"deltas":[]}},"rsync":{"rsync://rsync.example.test/repo/":{"file_count":2,"overlay_only":true}}}"#, + ).expect("rewrite locks unchanged"); + let index = + ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).expect("load unchanged index"); + let repo = index.rrdp_repo(¬ify_uri).expect("rrdp repo"); + assert_eq!(repo.transition.kind, ReplayDeltaRrdpKind::Unchanged); + assert!(repo.target_notification_path.is_none()); + assert!(repo.delta_paths.is_empty()); + assert!(repo.target_archive_path.is_none()); + + std::fs::write( + repo_dir.join("transition.json"), + r#"{"kind":"fallback-rsync","base":{"transport":"rsync","session":null,"serial":null},"target":{"transport":"rsync","session":null,"serial":null},"delta_count":0,"deltas":[]}"#, + ).expect("rewrite transition fallback-rsync"); + std::fs::write( + &locks_path, + r#"{"version":1,"capture":"delta-cap","baseCapture":"base-cap","baseLocksSha256":"deadbeef","rrdp":{"https://rrdp.example.test/notification.xml":{"kind":"fallback-rsync","base":{"transport":"rsync","session":null,"serial":null},"target":{"transport":"rsync","session":null,"serial":null},"delta_count":0,"deltas":[]}},"rsync":{"rsync://rsync.example.test/repo/":{"file_count":2,"overlay_only":true}}}"#, + ).expect("rewrite locks fallback-rsync"); + let index = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path) + .expect("load fallback-rsync index"); + let repo = index.rrdp_repo(¬ify_uri).expect("rrdp repo"); + assert_eq!(repo.transition.kind, ReplayDeltaRrdpKind::FallbackRsync); + assert!(repo.target_notification_path.is_none()); + assert!(repo.delta_paths.is_empty()); +} + +#[test] +fn delta_archive_index_resolves_rsync_module_from_base_uri() { + let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_delta_fixture(); + let index = + ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).expect("load delta index"); + let module = index + .resolve_rsync_module_for_base_uri("rsync://rsync.example.test/repo/sub/path") + .expect("resolve module"); + assert_eq!(module.module_uri, "rsync://rsync.example.test/repo/"); +} + +#[test] +fn delta_archive_index_rejects_unsupported_versions_and_meta_mismatches() { + let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_delta_fixture(); + std::fs::write( + &locks_path, + r#"{"version":2,"capture":"delta-cap","baseCapture":"base-cap","baseLocksSha256":"deadbeef","rrdp":{},"rsync":{}}"#, + ).expect("rewrite locks version"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!( + err, + ReplayDeltaArchiveError::Base(ReplayArchiveError::UnsupportedVersion { + entity: "payload delta locks", + version: 2 + }) + ), + "{err}" + ); + + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + std::fs::write( + archive_root.join("v1/captures/delta-cap/rrdp/repos").join(&repo_hash).join("meta.json"), + r#"{"version":1,"rpkiNotify":"https://other.example/notification.xml","createdAt":"2026-03-15T00:00:00Z","lastSeenAt":"2026-03-15T00:00:01Z"}"#, + ).expect("rewrite rrdp meta"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::RrdpMetaMismatch { .. }), + "{err}" + ); + + let (_temp, archive_root, locks_path, _notify_uri, module_uri) = build_delta_fixture(); + let module_hash = sha256_hex(module_uri.as_bytes()); + std::fs::write( + archive_root.join("v1/captures/delta-cap/rsync/modules").join(&module_hash).join("meta.json"), + r#"{"version":1,"module":"rsync://other.example/repo/","createdAt":"2026-03-15T00:00:00Z","lastSeenAt":"2026-03-15T00:00:01Z"}"#, + ).expect("rewrite rsync meta"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::RsyncMetaMismatch { .. }), + "{err}" + ); +} + +#[test] +fn delta_archive_index_rejects_transition_base_target_and_serial_mismatches() { + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let repo_dir = archive_root + .join("v1/captures/delta-cap/rrdp/repos") + .join(&repo_hash); + + std::fs::write( + repo_dir.join("transition.json"), + r#"{"kind":"delta","base":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":9},"target":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":12},"delta_count":2,"deltas":[11,12]}"#, + ).expect("rewrite transition base"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::TransitionBaseMismatch { .. }), + "{err}" + ); + + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let repo_dir = archive_root + .join("v1/captures/delta-cap/rrdp/repos") + .join(&repo_hash); + std::fs::write( + repo_dir.join("transition.json"), + r#"{"kind":"delta","base":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10},"target":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":13},"delta_count":2,"deltas":[11,12]}"#, + ).expect("rewrite transition target"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!( + err, + ReplayDeltaArchiveError::TransitionTargetMismatch { .. } + ), + "{err}" + ); + + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let repo_dir = archive_root + .join("v1/captures/delta-cap/rrdp/repos") + .join(&repo_hash); + std::fs::write( + repo_dir.join("transition.json"), + r#"{"kind":"delta","base":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10},"target":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":12},"delta_count":1,"deltas":[12]}"#, + ).expect("rewrite transition deltas"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::DeltaSerialListMismatch { .. }), + "{err}" + ); +} + +#[test] +fn delta_archive_index_rejects_missing_session_dir_and_overlay_files() { + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let session_dir = archive_root + .join("v1/captures/delta-cap/rrdp/repos") + .join(&repo_hash) + .join("11111111-1111-1111-1111-111111111111"); + std::fs::remove_dir_all(&session_dir).expect("remove session dir"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::MissingDeltaSessionDir { .. }), + "{err}" + ); + + let (_temp, archive_root, locks_path, _notify_uri, module_uri) = build_delta_fixture(); + let module_hash = sha256_hex(module_uri.as_bytes()); + let overlay_path = archive_root + .join("v1/captures/delta-cap/rsync/modules") + .join(&module_hash) + .join("tree/rsync.example.test/repo/sub/b.cer"); + std::fs::remove_file(overlay_path).expect("remove overlay file"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::MissingRsyncOverlayFile { .. }), + "{err}" + ); +} + +#[test] +fn delta_archive_index_accepts_missing_rsync_module_meta_when_files_and_tree_exist() { + let (_temp, archive_root, locks_path, _notify_uri, module_uri) = build_delta_fixture(); + let module_hash = sha256_hex(module_uri.as_bytes()); + let meta_path = archive_root + .join("v1/captures/delta-cap/rsync/modules") + .join(module_hash) + .join("meta.json"); + std::fs::remove_file(&meta_path).expect("remove delta rsync module meta"); + + let index = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path) + .expect("load delta replay index without rsync meta"); + let module = index + .rsync_modules + .get(&module_uri) + .expect("module present"); + assert_eq!(module.meta.module, module_uri); + assert_eq!(module.meta.version, 1); +} + +#[test] +fn delta_archive_index_accepts_correct_base_locks_sha_and_rejects_missing_module_resolution() { + let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_delta_fixture(); + let index = + ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).expect("load delta index"); + let locks_bytes = std::fs::read(&locks_path).expect("read locks bytes"); + assert!( + index + .validate_base_locks_sha256_bytes(&locks_bytes) + .is_err() + ); + let err = index + .resolve_rsync_module_for_base_uri("rsync://missing.example/repo/path") + .unwrap_err(); + assert!( + matches!( + err, + ReplayDeltaArchiveError::MissingRsyncModuleBucket { .. } + ), + "{err}" + ); +} + +#[test] +fn delta_archive_index_loads_session_reset_and_gap_entries_without_target_files() { + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let repo_dir = archive_root + .join("v1/captures/delta-cap/rrdp/repos") + .join(&repo_hash); + + for kind in ["session-reset", "gap"] { + std::fs::write( + repo_dir.join("transition.json"), + format!( + r#"{{"kind":"{kind}","base":{{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10}},"target":{{"transport":"rrdp","session":"22222222-2222-2222-2222-222222222222","serial":12}},"delta_count":0,"deltas":[]}}"#, + ), + ) + .expect("rewrite transition kind"); + std::fs::write( + &locks_path, + format!( + r#"{{"version":1,"capture":"delta-cap","baseCapture":"base-cap","baseLocksSha256":"deadbeef","rrdp":{{"{notify_uri}":{{"kind":"{kind}","base":{{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10}},"target":{{"transport":"rrdp","session":"22222222-2222-2222-2222-222222222222","serial":12}},"delta_count":0,"deltas":[]}}}},"rsync":{{"rsync://rsync.example.test/repo/":{{"file_count":2,"overlay_only":true}}}}}}"#, + ), + ) + .expect("rewrite locks kind"); + let index = + ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).expect("load delta index"); + let repo = index.rrdp_repo(¬ify_uri).expect("rrdp repo"); + assert!(repo.target_notification_path.is_none()); + assert!(repo.delta_paths.is_empty()); + } +} +#[test] +fn delta_archive_index_loads_rrdp_and_rsync_entries() { + let (_temp, archive_root, locks_path, notify_uri, module_uri) = build_delta_fixture(); + let index = + ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).expect("load delta index"); + assert_eq!(index.capture_meta.capture_id, "delta-cap"); + assert_eq!(index.base_meta.base_capture, "base-cap"); + assert_eq!(index.rrdp_repos.len(), 1); + assert_eq!(index.rsync_modules.len(), 1); + + let repo = index.rrdp_repo(¬ify_uri).expect("rrdp repo"); + assert_eq!(repo.transition.kind, ReplayDeltaRrdpKind::Delta); + assert_eq!(repo.transition.delta_count, 2); + assert_eq!(repo.delta_paths.len(), 2); + assert!(repo.target_notification_path.as_ref().unwrap().is_file()); + assert!(repo.target_archive_path.as_ref().unwrap().is_file()); + + let module = index.rsync_module(&module_uri).expect("rsync module"); + assert_eq!(module.files.file_count, 2); + assert_eq!(module.overlay_files.len(), 2); + assert!(module.overlay_files.iter().all(|(_, path)| path.is_file())); +} + +#[test] +fn delta_archive_index_rejects_capture_and_sha_mismatches() { + let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_delta_fixture(); + let capture_root = archive_root.join("v1/captures/delta-cap"); + std::fs::write( + capture_root.join("capture.json"), + r#"{"version":1,"captureId":"other-cap","createdAt":"2026-03-15T00:00:00Z","notes":""}"#, + ) + .expect("rewrite capture json"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::CaptureIdMismatch { .. }), + "{err}" + ); + + let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_delta_fixture(); + let capture_root = archive_root.join("v1/captures/delta-cap"); + std::fs::write( + capture_root.join("base.json"), + r#"{"version":1,"baseCapture":"base-cap","baseLocksSha256":"beefdead","createdAt":"2026-03-15T00:00:00Z"}"#, + ) + .expect("rewrite base json sha"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::BaseLocksShaMismatch { .. }), + "{err}" + ); +} + +#[test] +fn delta_archive_index_rejects_missing_target_notification_and_repo_bucket() { + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let session_dir = archive_root + .join("v1/captures/delta-cap/rrdp/repos") + .join(&repo_hash) + .join("11111111-1111-1111-1111-111111111111"); + std::fs::remove_file(session_dir.join("notification-target-12.xml")) + .expect("remove target notification"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!( + err, + ReplayDeltaArchiveError::MissingTargetNotification { .. } + ), + "{err}" + ); + + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let repo_dir = archive_root + .join("v1/captures/delta-cap/rrdp/repos") + .join(&repo_hash); + std::fs::remove_dir_all(repo_dir).expect("remove repo dir"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::MissingDeltaRepoBucket { .. }), + "{err}" + ); +} +#[test] +fn delta_archive_index_rejects_base_meta_mismatch() { + let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_delta_fixture(); + let capture_root = archive_root.join("v1/captures/delta-cap"); + std::fs::write( + capture_root.join("base.json"), + r#"{"version":1,"baseCapture":"other","baseLocksSha256":"deadbeef","createdAt":"2026-03-15T00:00:00Z"}"#, + ) + .expect("rewrite base json"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::BaseCaptureMismatch { .. }), + "{err}" + ); +} + +#[test] +fn delta_archive_index_rejects_transition_mismatch_and_missing_delta_file() { + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let repo_dir = archive_root + .join("v1/captures/delta-cap/rrdp/repos") + .join(&repo_hash); + std::fs::write( + repo_dir.join("transition.json"), + r#"{"kind":"unchanged","base":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10},"target":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":12},"delta_count":2,"deltas":[11,12]}"#, + ) + .expect("rewrite transition"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::TransitionKindMismatch { .. }), + "{err}" + ); + + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let delta_path = archive_root + .join("v1/captures/delta-cap/rrdp/repos") + .join(&repo_hash) + .join("11111111-1111-1111-1111-111111111111/deltas/delta-12-bbbb.xml"); + std::fs::remove_file(delta_path).expect("remove delta"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::MissingDeltaFile { .. }), + "{err}" + ); +} + +#[test] +fn delta_archive_index_validates_base_locks_sha256_bytes_and_file() { + let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_delta_fixture(); + let index = + ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).expect("load delta index"); + let err = index + .validate_base_locks_sha256_bytes(b"not-the-right-base-locks") + .unwrap_err(); + assert!( + matches!( + err, + ReplayDeltaArchiveError::BaseLocksBytesShaMismatch { .. } + ), + "{err}" + ); + + let temp_file = tempfile::NamedTempFile::new().expect("tempfile"); + std::fs::write(temp_file.path(), b"still-wrong").expect("write base locks file"); + let err = index + .validate_base_locks_sha256_file(temp_file.path()) + .unwrap_err(); + assert!( + matches!( + err, + ReplayDeltaArchiveError::BaseLocksBytesShaMismatch { .. } + ), + "{err}" + ); +} + +#[test] +fn delta_archive_index_rejects_rsync_files_mismatch() { + let (_temp, archive_root, locks_path, _notify_uri, module_uri) = build_delta_fixture(); + let module_hash = sha256_hex(module_uri.as_bytes()); + let module_dir = archive_root + .join("v1/captures/delta-cap/rsync/modules") + .join(&module_hash); + std::fs::write( + module_dir.join("files.json"), + format!( + r#"{{"version":1,"module":"{module_uri}","fileCount":3,"files":["{module_uri}a.roa"]}}"# + ), + ) + .expect("rewrite files json"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::RsyncFileCountMismatch { .. }), + "{err}" + ); +} diff --git a/crates/panda-rpki-validator/src/storage.rs b/crates/panda-rpki-validator/src/storage.rs index a0561ee..d87c545 100644 --- a/crates/panda-rpki-validator/src/storage.rs +++ b/crates/panda-rpki-validator/src/storage.rs @@ -1,5 +1,6 @@ mod config; mod keys; +mod memory; mod pack; mod pp_cache_index; @@ -23,6 +24,12 @@ pub use config::{ CF_TRANSPORT_PREFETCH, CF_VCIR, CF_VCIR_FAILED_FETCH_REUSE_IDENTITY, column_family_descriptors, }; use keys::*; +pub(crate) use memory::memory_db_snapshot_for_column_families; +use memory::process_vm_rss_kb; +pub use memory::{ + RocksDbColumnFamilyMemoryProperties, RocksDbMemoryDbSnapshot, RocksDbMemoryProperties, + RocksDbMemorySnapshot, RocksDbMemoryTotals, +}; use pack::compute_sha256_32; pub use pack::{PackBytes, PackFile, PackTime}; pub use pp_cache_index::{PpCacheIndexLoadStats, PpCacheIndexRefreshStats}; @@ -31,4831 +38,18 @@ use pp_cache_index::{ load_pp_cache_mmap_index, load_pp_cache_mmap_index_set, pp_cache_index_directory_stats, write_pp_cache_index_atomic, write_pp_cache_index_segment, }; -#[derive(Debug, thiserror::Error)] -pub enum StorageError { - #[error("rocksdb error: {0}")] - RocksDb(String), - - #[error("missing column family: {0}")] - MissingColumnFamily(&'static str), - - #[error("cbor codec error for {entity}: {detail}")] - Codec { - entity: &'static str, - detail: String, - }, - - #[error("invalid {entity}: {detail}")] - InvalidData { - entity: &'static str, - detail: String, - }, -} - -pub type StorageResult = Result; - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] -pub struct RepositoryBlobVerificationSummary { - pub current_objects: u64, - pub bytes_verified: u64, - pub batches: u64, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] -pub struct ChildCertificateCacheMmapLookup { - pub projections: Vec>, - pub hits: usize, - pub misses: usize, - pub file_bytes: u64, -} - -pub struct RocksStore { - db: DB, - external_raw_store: Option, - external_repo_bytes: Option, - publication_point_cache_index_dir: PathBuf, - child_certificate_cache_index_dir: PathBuf, - publication_point_cache_projection_index: Mutex, -} - -enum PublicationPointCacheProjectionIndexState { - Uninitialized, - Disabled, - BuildingFromEmpty { - index: HashMap>, - bytes: usize, - limit: usize, - }, - Loaded { - index: HashMap>, - bytes: usize, - }, - LoadedMmap { - mmap: PpCacheMmapIndexSet, - dirty: HashMap>, - dirty_bytes: usize, - load_stats: PpCacheIndexLoadStats, - }, -} - -#[derive(Clone, Copy)] -pub(crate) enum PublicationPointCacheProjectionWriteAction<'a> { - Keep, - Write(&'a PublicationPointCacheProjection), - Delete { manifest_rsync_uri: &'a str }, -} - -fn process_vm_rss_kb() -> Option { - let status = std::fs::read_to_string("/proc/self/status").ok()?; - status.lines().find_map(|line| { - let rest = line.strip_prefix("VmRSS:")?; - rest.split_whitespace().next()?.parse::().ok() - }) -} - -fn default_child_certificate_cache_index_dir(db_path: &Path) -> PathBuf { - let file_name = db_path - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or("work-db"); - db_path.with_file_name(format!("{file_name}.child-cert-cache-index")) -} - -fn child_certificate_cache_segment_file_name(manifest_rsync_uri: &str) -> String { - format!( - "{}.idx", - hex::encode(compute_sha256_32(manifest_rsync_uri.as_bytes())) - ) -} - -const ROCKSDB_MEMORY_PROPERTY_NAMES: &[(&str, &str)] = &[ - ("cur_size_all_mem_tables", "rocksdb.cur-size-all-mem-tables"), - ("size_all_mem_tables", "rocksdb.size-all-mem-tables"), - ( - "estimate_table_readers_mem", - "rocksdb.estimate-table-readers-mem", - ), - ("block_cache_capacity", "rocksdb.block-cache-capacity"), - ("block_cache_usage", "rocksdb.block-cache-usage"), - ( - "block_cache_pinned_usage", - "rocksdb.block-cache-pinned-usage", - ), - ("num_snapshots", "rocksdb.num-snapshots"), - ("background_errors", "rocksdb.background-errors"), -]; - -const PP_CACHE_RAW_INDEX_ENV: &str = "RPKI_PP_CACHE_RAW_INDEX"; -const PP_CACHE_RAW_INDEX_EMPTY_BUILD_LIMIT_ENV: &str = - "RPKI_PP_CACHE_RAW_INDEX_EMPTY_BUILD_LIMIT_BYTES"; -const DEFAULT_PP_CACHE_RAW_INDEX_EMPTY_BUILD_LIMIT_BYTES: usize = 32 * 1024 * 1024; -const PP_CACHE_INDEX_COMPACTION_SEGMENT_THRESHOLD: usize = 16; -const PP_CACHE_INDEX_COMPACTION_BYTES_THRESHOLD: u64 = 1_610_612_736; - -fn pp_cache_raw_index_enabled() -> bool { - match std::env::var(PP_CACHE_RAW_INDEX_ENV) { - Ok(value) => !matches!( - value.trim().to_ascii_lowercase().as_str(), - "0" | "false" | "off" | "no" - ), - Err(_) => true, - } -} - -fn pp_cache_raw_index_empty_build_limit_bytes() -> usize { - std::env::var(PP_CACHE_RAW_INDEX_EMPTY_BUILD_LIMIT_ENV) - .ok() - .and_then(|value| value.trim().parse::().ok()) - .unwrap_or(DEFAULT_PP_CACHE_RAW_INDEX_EMPTY_BUILD_LIMIT_BYTES) -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] -pub struct RocksDbMemoryProperties { - pub cur_size_all_mem_tables: Option, - pub size_all_mem_tables: Option, - pub estimate_table_readers_mem: Option, - pub block_cache_capacity: Option, - pub block_cache_usage: Option, - pub block_cache_pinned_usage: Option, - pub num_snapshots: Option, - pub background_errors: Option, - pub errors: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize)] -pub struct RocksDbColumnFamilyMemoryProperties { - pub name: String, - pub properties: RocksDbMemoryProperties, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize)] -pub struct RocksDbMemoryDbSnapshot { - pub label: String, - pub properties: RocksDbMemoryProperties, - pub column_families: Vec, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] -pub struct RocksDbMemoryTotals { - pub cur_size_all_mem_tables: u64, - pub size_all_mem_tables: u64, - pub estimate_table_readers_mem: u64, - pub block_cache_capacity: u64, - pub block_cache_usage: u64, - pub block_cache_pinned_usage: u64, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize)] -pub struct RocksDbMemorySnapshot { - pub databases: Vec, - pub totals: RocksDbMemoryTotals, -} - -impl RocksDbMemoryTotals { - fn add_properties(&mut self, properties: &RocksDbMemoryProperties) { - self.cur_size_all_mem_tables += properties.cur_size_all_mem_tables.unwrap_or(0); - self.size_all_mem_tables += properties.size_all_mem_tables.unwrap_or(0); - self.estimate_table_readers_mem += properties.estimate_table_readers_mem.unwrap_or(0); - self.block_cache_capacity += properties.block_cache_capacity.unwrap_or(0); - self.block_cache_usage += properties.block_cache_usage.unwrap_or(0); - self.block_cache_pinned_usage += properties.block_cache_pinned_usage.unwrap_or(0); - } -} - -fn set_memory_property(properties: &mut RocksDbMemoryProperties, name: &str, value: u64) { - match name { - "cur_size_all_mem_tables" => properties.cur_size_all_mem_tables = Some(value), - "size_all_mem_tables" => properties.size_all_mem_tables = Some(value), - "estimate_table_readers_mem" => properties.estimate_table_readers_mem = Some(value), - "block_cache_capacity" => properties.block_cache_capacity = Some(value), - "block_cache_usage" => properties.block_cache_usage = Some(value), - "block_cache_pinned_usage" => properties.block_cache_pinned_usage = Some(value), - "num_snapshots" => properties.num_snapshots = Some(value), - "background_errors" => properties.background_errors = Some(value), - _ => {} - } -} - -fn parse_rocksdb_property_int(raw: Option) -> Option { - raw.and_then(|value| value.trim().parse::().ok()) -} - -fn memory_properties_for_db(db: &DB) -> RocksDbMemoryProperties { - let mut properties = RocksDbMemoryProperties::default(); - for (field_name, property_name) in ROCKSDB_MEMORY_PROPERTY_NAMES { - match db.property_value(*property_name) { - Ok(value) => { - if let Some(parsed) = parse_rocksdb_property_int(value) { - set_memory_property(&mut properties, field_name, parsed); - } - } - Err(err) => properties.errors.push(format!("{property_name}: {}", err)), - } - } - properties -} - -fn memory_properties_for_cf(db: &DB, cf_name: &'static str) -> RocksDbColumnFamilyMemoryProperties { - let mut properties = RocksDbMemoryProperties::default(); - let Some(cf) = db.cf_handle(cf_name) else { - properties - .errors - .push(format!("missing column family: {cf_name}")); - return RocksDbColumnFamilyMemoryProperties { - name: cf_name.to_string(), - properties, - }; - }; - for (field_name, property_name) in ROCKSDB_MEMORY_PROPERTY_NAMES { - match db.property_value_cf(cf, *property_name) { - Ok(value) => { - if let Some(parsed) = parse_rocksdb_property_int(value) { - set_memory_property(&mut properties, field_name, parsed); - } - } - Err(err) => properties.errors.push(format!("{property_name}: {}", err)), - } - } - RocksDbColumnFamilyMemoryProperties { - name: cf_name.to_string(), - properties, - } -} - -pub(crate) fn memory_db_snapshot_for_column_families( - label: impl Into, - db: &DB, - column_families: Option<&[&'static str]>, -) -> RocksDbMemoryDbSnapshot { - RocksDbMemoryDbSnapshot { - label: label.into(), - properties: memory_properties_for_db(db), - column_families: column_families - .map(|names| { - names - .iter() - .map(|name| memory_properties_for_cf(db, name)) - .collect() - }) - .unwrap_or_default(), - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum RrdpDeltaOp { - Upsert { rsync_uri: String, bytes: Vec }, - Delete { rsync_uri: String }, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum RepositoryViewState { - Present, - Withdrawn, - Replaced, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct RepositoryViewEntry { - pub rsync_uri: String, - pub current_hash: Option, - pub repository_source: Option, - pub object_type: Option, - pub state: RepositoryViewState, -} - -impl RepositoryViewEntry { - pub fn validate_internal(&self) -> StorageResult<()> { - validate_non_empty("repository_view.rsync_uri", &self.rsync_uri)?; - if let Some(source) = &self.repository_source { - validate_non_empty("repository_view.repository_source", source)?; - } - match self.state { - RepositoryViewState::Present | RepositoryViewState::Replaced => { - let hash = self - .current_hash - .as_deref() - .ok_or(StorageError::InvalidData { - entity: "repository_view", - detail: "current_hash is required when state is present or replaced" - .to_string(), - })?; - validate_sha256_hex("repository_view.current_hash", hash)?; - } - RepositoryViewState::Withdrawn => { - if let Some(hash) = &self.current_hash { - validate_sha256_hex("repository_view.current_hash", hash)?; - } - } - } - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct RawByHashEntry { - pub sha256_hex: String, - pub bytes: Vec, - pub origin_uris: Vec, - pub object_type: Option, - pub encoding: Option, -} - -impl RawByHashEntry { - pub fn from_bytes(sha256_hex: impl Into, bytes: Vec) -> Self { - Self { - sha256_hex: sha256_hex.into(), - bytes, - origin_uris: Vec::new(), - object_type: None, - encoding: None, - } - } - - pub fn validate_internal(&self) -> StorageResult<()> { - validate_sha256_hex("raw_by_hash.sha256_hex", &self.sha256_hex)?; - if self.bytes.is_empty() { - return Err(StorageError::InvalidData { - entity: "raw_by_hash", - detail: "bytes must not be empty".to_string(), - }); - } - let computed = hex::encode(compute_sha256_32(&self.bytes)); - if computed != self.sha256_hex.to_ascii_lowercase() { - return Err(StorageError::InvalidData { - entity: "raw_by_hash", - detail: "sha256_hex does not match bytes".to_string(), - }); - } - let mut seen = HashSet::with_capacity(self.origin_uris.len()); - for uri in &self.origin_uris { - validate_non_empty("raw_by_hash.origin_uris[]", uri)?; - if !seen.insert(uri.as_str()) { - return Err(StorageError::InvalidData { - entity: "raw_by_hash", - detail: format!("duplicate origin URI: {uri}"), - }); - } - } - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct CurrentObjectWithHash { - pub current_hash_hex: String, - pub current_hash: [u8; 32], - pub bytes: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ValidatedManifestMeta { - pub validated_manifest_number: Vec, - pub validated_manifest_this_update: PackTime, - pub validated_manifest_next_update: PackTime, -} - -impl ValidatedManifestMeta { - pub fn validate_internal(&self) -> StorageResult<()> { - validate_manifest_number_be( - "validated_manifest_meta.validated_manifest_number", - &self.validated_manifest_number, - )?; - let this_update = parse_time( - "validated_manifest_meta.validated_manifest_this_update", - &self.validated_manifest_this_update, - )?; - let next_update = parse_time( - "validated_manifest_meta.validated_manifest_next_update", - &self.validated_manifest_next_update, - )?; - if next_update < this_update { - return Err(StorageError::InvalidData { - entity: "validated_manifest_meta", - detail: "validated_manifest_next_update must be >= validated_manifest_this_update" - .to_string(), - }); - } - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ManifestReplayMeta { - pub manifest_rsync_uri: String, - pub manifest_number_be: Vec, - pub manifest_this_update: PackTime, - pub manifest_sha256: Vec, - pub updated_at_validation_time: PackTime, -} - -impl ManifestReplayMeta { - pub fn from_vcir(vcir: &ValidatedCaInstanceResult) -> Self { - Self { - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - manifest_number_be: vcir - .validated_manifest_meta - .validated_manifest_number - .clone(), - manifest_this_update: vcir - .validated_manifest_meta - .validated_manifest_this_update - .clone(), - manifest_sha256: vcir.ccr_manifest_projection.manifest_sha256.clone(), - updated_at_validation_time: vcir.last_successful_validation_time.clone(), - } - } - - pub fn validate_internal(&self) -> StorageResult<()> { - validate_non_empty( - "manifest_replay_meta.manifest_rsync_uri", - &self.manifest_rsync_uri, - )?; - validate_manifest_number_be( - "manifest_replay_meta.manifest_number_be", - &self.manifest_number_be, - )?; - parse_time( - "manifest_replay_meta.manifest_this_update", - &self.manifest_this_update, - )?; - validate_sha256_digest_bytes( - "manifest_replay_meta.manifest_sha256", - &self.manifest_sha256, - )?; - parse_time( - "manifest_replay_meta.updated_at_validation_time", - &self.updated_at_validation_time, - )?; - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct VcirCcrManifestProjection { - pub manifest_rsync_uri: String, - pub manifest_sha256: Vec, - pub manifest_size: u64, - pub manifest_ee_aki: Vec, - pub manifest_number_be: Vec, - pub manifest_this_update: PackTime, - pub manifest_sia_locations_der: Vec>, - pub subordinate_skis: Vec>, -} - -impl VcirCcrManifestProjection { - pub fn validate_internal(&self) -> StorageResult<()> { - validate_non_empty( - "vcir.ccr_manifest_projection.manifest_rsync_uri", - &self.manifest_rsync_uri, - )?; - validate_sha256_digest_bytes( - "vcir.ccr_manifest_projection.manifest_sha256", - &self.manifest_sha256, - )?; - if self.manifest_size < 1000 { - return Err(StorageError::InvalidData { - entity: "vcir.ccr_manifest_projection.manifest_size", - detail: format!("must be >= 1000, got {}", self.manifest_size), - }); - } - validate_fixed_len_bytes( - "vcir.ccr_manifest_projection.manifest_ee_aki", - &self.manifest_ee_aki, - 20, - )?; - validate_manifest_number_be( - "vcir.ccr_manifest_projection.manifest_number_be", - &self.manifest_number_be, - )?; - parse_time( - "vcir.ccr_manifest_projection.manifest_this_update", - &self.manifest_this_update, - )?; - if self.manifest_sia_locations_der.is_empty() { - return Err(StorageError::InvalidData { - entity: "vcir.ccr_manifest_projection.manifest_sia_locations_der", - detail: "must contain at least one AccessDescription".to_string(), - }); - } - for location in &self.manifest_sia_locations_der { - validate_full_der_with_tag( - "vcir.ccr_manifest_projection.manifest_sia_locations_der[]", - location, - Some(0x30), - )?; - } - validate_sorted_unique_fixed_len_bytes( - "vcir.ccr_manifest_projection.subordinate_skis", - &self.subordinate_skis, - 20, - )?; - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct VcirInstanceGate { - pub manifest_next_update: PackTime, - pub current_crl_next_update: PackTime, - pub self_ca_not_after: PackTime, - pub instance_effective_until: PackTime, -} - -impl VcirInstanceGate { - pub fn validate_internal(&self) -> StorageResult<()> { - let manifest_next_update = parse_time( - "vcir.instance_gate.manifest_next_update", - &self.manifest_next_update, - )?; - let current_crl_next_update = parse_time( - "vcir.instance_gate.current_crl_next_update", - &self.current_crl_next_update, - )?; - let self_ca_not_after = parse_time( - "vcir.instance_gate.self_ca_not_after", - &self.self_ca_not_after, - )?; - let instance_effective_until = parse_time( - "vcir.instance_gate.instance_effective_until", - &self.instance_effective_until, - )?; - let expected = manifest_next_update - .min(current_crl_next_update) - .min(self_ca_not_after); - if instance_effective_until != expected { - return Err(StorageError::InvalidData { - entity: "vcir.instance_gate", - detail: "instance_effective_until must equal min(manifest_next_update, current_crl_next_update, self_ca_not_after)".to_string(), - }); - } - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct VcirFailedFetchReuseIdentity { - #[serde(rename = "c")] - #[serde(with = "serde_bytes_32")] - pub current_ca_sha256: [u8; 32], - #[serde(rename = "t")] - #[serde(with = "serde_bytes_32")] - pub ta_context_digest: [u8; 32], - #[serde(rename = "p")] - #[serde(with = "serde_bytes_32")] - pub ca_validation_context_digest: [u8; 32], - #[serde(rename = "f")] - #[serde(with = "serde_bytes_32")] - pub policy_fingerprint: [u8; 32], - #[serde(rename = "nb")] - pub effective_not_before: PackTime, - #[serde(rename = "nu")] - pub effective_until: PackTime, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] -pub struct VcirReuseRecordsCleared { - pub vcir_records: u64, - pub failed_fetch_identity_records: u64, -} - -impl VcirFailedFetchReuseIdentity { - pub fn validate_internal(&self) -> StorageResult<()> { - let effective_not_before = parse_time( - "vcir_failed_fetch_reuse_identity.effective_not_before", - &self.effective_not_before, - )?; - let effective_until = parse_time( - "vcir_failed_fetch_reuse_identity.effective_until", - &self.effective_until, - )?; - if effective_not_before >= effective_until { - return Err(StorageError::InvalidData { - entity: "vcir_failed_fetch_reuse_identity.effective_window", - detail: "effective_not_before must be before effective_until".to_string(), - }); - } - Ok(()) - } - - pub fn contains_validation_time(&self, validation_time: time::OffsetDateTime) -> bool { - let Ok(effective_not_before) = self.effective_not_before.parse() else { - return false; - }; - let Ok(effective_until) = self.effective_until.parse() else { - return false; - }; - validation_time >= effective_not_before && validation_time < effective_until - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct VcirChildEntry { - pub child_manifest_rsync_uri: String, - pub child_cert_rsync_uri: String, - pub child_cert_hash: String, - pub child_ski: String, - pub child_rsync_base_uri: String, - pub child_publication_point_rsync_uri: String, - pub child_rrdp_notification_uri: Option, - pub child_effective_ip_resources: Option, - pub child_effective_as_resources: Option, - pub accepted_at_validation_time: PackTime, -} - -impl VcirChildEntry { - pub fn validate_internal(&self) -> StorageResult<()> { - validate_non_empty( - "vcir.child_entries[].child_manifest_rsync_uri", - &self.child_manifest_rsync_uri, - )?; - validate_non_empty( - "vcir.child_entries[].child_cert_rsync_uri", - &self.child_cert_rsync_uri, - )?; - validate_sha256_hex( - "vcir.child_entries[].child_cert_hash", - &self.child_cert_hash, - )?; - validate_non_empty("vcir.child_entries[].child_ski", &self.child_ski)?; - validate_non_empty( - "vcir.child_entries[].child_rsync_base_uri", - &self.child_rsync_base_uri, - )?; - validate_non_empty( - "vcir.child_entries[].child_publication_point_rsync_uri", - &self.child_publication_point_rsync_uri, - )?; - if let Some(uri) = &self.child_rrdp_notification_uri { - validate_non_empty("vcir.child_entries[].child_rrdp_notification_uri", uri)?; - } - parse_time( - "vcir.child_entries[].accepted_at_validation_time", - &self.accepted_at_validation_time, - )?; - Ok(()) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum VcirOutputType { - Vrp, - Aspa, - RouterKey, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum VcirSourceObjectType { - Roa, - Aspa, - RouterKey, - Other, -} - -impl VcirSourceObjectType { - pub fn as_str(self) -> &'static str { - match self { - Self::Roa => "roa", - Self::Aspa => "aspa", - Self::RouterKey => "router_key", - Self::Other => "other", - } - } -} - -struct FixedBytesVisitor; - -impl<'de, const N: usize> serde::de::Visitor<'de> for FixedBytesVisitor { - type Value = [u8; N]; - - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(formatter, "{N} bytes") - } - - fn visit_bytes(self, value: &[u8]) -> Result - where - E: serde::de::Error, - { - if value.len() != N { - return Err(E::invalid_length(value.len(), &self)); - } - let mut out = [0u8; N]; - out.copy_from_slice(value); - Ok(out) - } - - fn visit_byte_buf(self, value: Vec) -> Result - where - E: serde::de::Error, - { - self.visit_bytes(&value) - } - - fn visit_seq(self, mut seq: A) -> Result - where - A: serde::de::SeqAccess<'de>, - { - let mut out = [0u8; N]; - for (idx, slot) in out.iter_mut().enumerate() { - *slot = seq - .next_element()? - .ok_or_else(|| serde::de::Error::invalid_length(idx, &self))?; - } - Ok(out) - } -} - -fn deserialize_fixed_bytes<'de, D, const N: usize>(deserializer: D) -> Result<[u8; N], D::Error> -where - D: serde::Deserializer<'de>, -{ - deserializer.deserialize_bytes(FixedBytesVisitor::) -} - -mod serde_bytes_16 { - pub(super) fn serialize(value: &[u8; 16], serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_bytes(value) - } - - pub(super) fn deserialize<'de, D>(deserializer: D) -> Result<[u8; 16], D::Error> - where - D: serde::Deserializer<'de>, - { - super::deserialize_fixed_bytes::(deserializer) - } -} - -mod serde_bytes_32 { - pub(super) fn serialize(value: &[u8; 32], serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_bytes(value) - } - - pub(super) fn deserialize<'de, D>(deserializer: D) -> Result<[u8; 32], D::Error> - where - D: serde::Deserializer<'de>, - { - super::deserialize_fixed_bytes::(deserializer) - } -} - -struct ByteVecVisitor; - -impl<'de> serde::de::Visitor<'de> for ByteVecVisitor { - type Value = Vec; - - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("byte vector") - } - - fn visit_bytes(self, value: &[u8]) -> Result - where - E: serde::de::Error, - { - Ok(value.to_vec()) - } - - fn visit_byte_buf(self, value: Vec) -> Result - where - E: serde::de::Error, - { - Ok(value) - } - - fn visit_seq(self, mut seq: A) -> Result - where - A: serde::de::SeqAccess<'de>, - { - let mut out = Vec::with_capacity(seq.size_hint().unwrap_or(0)); - while let Some(byte) = seq.next_element()? { - out.push(byte); - } - Ok(out) - } -} - -mod serde_byte_vec { - pub(super) fn serialize(value: &[u8], serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_bytes(value) - } - - pub(super) fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> - where - D: serde::Deserializer<'de>, - { - deserializer.deserialize_bytes(super::ByteVecVisitor) - } -} - -mod serde_optional_byte_vec { - pub(super) fn serialize(value: &Option>, serializer: S) -> Result - where - S: serde::Serializer, - { - match value { - Some(bytes) => serializer.serialize_some(bytes), - None => serializer.serialize_none(), - } - } - - pub(super) fn deserialize<'de, D>(deserializer: D) -> Result>, D::Error> - where - D: serde::Deserializer<'de>, - { - struct OptionalByteVecVisitor; - - impl<'de> serde::de::Visitor<'de> for OptionalByteVecVisitor { - type Value = Option>; - - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("optional byte vector") - } - - fn visit_none(self) -> Result - where - E: serde::de::Error, - { - Ok(None) - } - - fn visit_some(self, deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - deserializer - .deserialize_bytes(super::ByteVecVisitor) - .map(Some) - } - } - - deserializer.deserialize_option(OptionalByteVecVisitor) - } -} - -mod serde_optional_bytes_32 { - pub(super) fn serialize(value: &Option<[u8; 32]>, serializer: S) -> Result - where - S: serde::Serializer, - { - match value { - Some(bytes) => serializer.serialize_some(bytes.as_slice()), - None => serializer.serialize_none(), - } - } - - pub(super) fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> - where - D: serde::Deserializer<'de>, - { - struct OptionalBytes32Visitor; - - impl<'de> serde::de::Visitor<'de> for OptionalBytes32Visitor { - type Value = Option<[u8; 32]>; - - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str("optional 32-byte array") - } - - fn visit_none(self) -> Result - where - E: serde::de::Error, - { - Ok(None) - } - - fn visit_some(self, deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - super::deserialize_fixed_bytes::(deserializer).map(Some) - } - } - - deserializer.deserialize_option(OptionalBytes32Visitor) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum VcirLocalOutputPayload { - Vrp { - asn: u32, - afi: crate::data_model::roa::RoaAfi, - prefix_len: u16, - #[serde(with = "serde_bytes_16")] - addr: [u8; 16], - max_length: u16, - }, - Aspa { - customer_as_id: u32, - provider_as_ids: Vec, - }, - RouterKey { - as_id: u32, - #[serde(with = "serde_byte_vec")] - ski: Vec, - #[serde(with = "serde_byte_vec")] - spki_der: Vec, - }, -} - -impl VcirLocalOutputPayload { - pub fn typed_body_bytes(&self) -> u64 { - match self { - Self::Vrp { .. } => 4 + 1 + 2 + 16 + 2, - Self::Aspa { - provider_as_ids, .. - } => 4 + (provider_as_ids.len() as u64 * 4), - Self::RouterKey { ski, spki_der, .. } => 4 + ski.len() as u64 + spki_der.len() as u64, - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct VcirLocalOutput { - #[serde(rename = "t")] - pub output_type: VcirOutputType, - #[serde(rename = "e")] - pub item_effective_until: PackTime, - #[serde(rename = "u")] - pub source_object_uri: String, - #[serde(rename = "k")] - pub source_object_type: VcirSourceObjectType, - #[serde(rename = "h")] - #[serde(with = "serde_bytes_32")] - pub source_object_hash: [u8; 32], - #[serde(rename = "c")] - #[serde(with = "serde_bytes_32")] - pub source_ee_cert_hash: [u8; 32], - #[serde(rename = "p")] - pub payload: VcirLocalOutputPayload, - #[serde(rename = "r")] - #[serde(with = "serde_bytes_32")] - pub rule_hash: [u8; 32], -} - -impl VcirLocalOutput { - pub fn output_id(&self) -> String { - self.rule_hash_hex() - } - - pub fn source_object_hash_hex(&self) -> String { - hex::encode(self.source_object_hash) - } - - pub fn source_ee_cert_hash_hex(&self) -> String { - hex::encode(self.source_ee_cert_hash) - } - - pub fn rule_hash_hex(&self) -> String { - hex::encode(self.rule_hash) - } - - pub fn source_object_type_name(&self) -> &'static str { - self.source_object_type.as_str() - } - - pub fn payload_json(&self) -> String { - match &self.payload { - VcirLocalOutputPayload::Vrp { - asn, - afi, - prefix_len, - addr, - max_length, - } => { - let prefix = match afi { - crate::data_model::roa::RoaAfi::Ipv4 => { - let ip = std::net::Ipv4Addr::new(addr[0], addr[1], addr[2], addr[3]); - format!("{ip}/{prefix_len}") - } - crate::data_model::roa::RoaAfi::Ipv6 => { - let ip = std::net::Ipv6Addr::from(*addr); - format!("{ip}/{prefix_len}") - } - }; - format!(r#"{{"asn":{asn},"max_length":{max_length},"prefix":"{prefix}"}}"#) - } - VcirLocalOutputPayload::Aspa { - customer_as_id, - provider_as_ids, - } => { - let providers = provider_as_ids - .iter() - .map(u32::to_string) - .collect::>() - .join(","); - format!(r#"{{"customer_as_id":{customer_as_id},"provider_as_ids":[{providers}]}}"#) - } - VcirLocalOutputPayload::RouterKey { - as_id, - ski, - spki_der, - } => { - let ski_hex = hex::encode(ski); - let spki_der_base64 = base64::engine::general_purpose::STANDARD.encode(spki_der); - format!( - r#"{{"as_id":{as_id},"ski_hex":"{ski_hex}","spki_der_base64":"{spki_der_base64}"}}"# - ) - } - } - } - - pub fn validate_internal(&self) -> StorageResult<()> { - parse_time( - "vcir.local_outputs[].item_effective_until", - &self.item_effective_until, - )?; - validate_non_empty( - "vcir.local_outputs[].source_object_uri", - &self.source_object_uri, - )?; - validate_local_output_type_matches_payload(self)?; - Ok(()) - } -} - -fn validate_local_output_type_matches_payload(output: &VcirLocalOutput) -> StorageResult<()> { - let matches_payload = matches!( - (&output.output_type, &output.payload), - (VcirOutputType::Vrp, VcirLocalOutputPayload::Vrp { .. }) - | (VcirOutputType::Aspa, VcirLocalOutputPayload::Aspa { .. }) - | ( - VcirOutputType::RouterKey, - VcirLocalOutputPayload::RouterKey { .. } - ) - ); - if !matches_payload { - return Err(StorageError::InvalidData { - entity: "vcir.local_outputs[]", - detail: "output_type must match payload variant".to_string(), - }); - } - Ok(()) -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct RoaCacheCrlProjection { - #[serde(rename = "u")] - pub uri: String, - #[serde(rename = "h")] - pub sha256: String, -} - -impl RoaCacheCrlProjection { - pub fn validate_internal(&self) -> StorageResult<()> { - validate_non_empty("roa_cache_projection.crls[].uri", &self.uri)?; - validate_sha256_hex("roa_cache_projection.crls[].sha256", &self.sha256)?; - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RoaCacheObjectMeta { - pub source_object_uri: String, - pub source_object_hash: [u8; 32], - pub ee_serial: Vec, - pub crl_uri: String, - pub earliest_safe_reuse_time: PackTime, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RoaCacheProjectionContext { - pub ca_validation_context_digest: [u8; 32], - pub policy_fingerprint: [u8; 32], - pub object_meta: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct RoaCacheLocalOutputProjection { - #[serde(rename = "e")] - pub item_effective_until: PackTime, - #[serde(rename = "c")] - #[serde(with = "serde_bytes_32")] - pub source_ee_cert_hash: [u8; 32], - #[serde(rename = "p")] - pub payload: VcirLocalOutputPayload, - #[serde(rename = "r")] - #[serde(with = "serde_bytes_32")] - pub rule_hash: [u8; 32], -} - -impl RoaCacheLocalOutputProjection { - fn from_local_output(output: &VcirLocalOutput) -> Option { - if output.output_type != VcirOutputType::Vrp - || output.source_object_type != VcirSourceObjectType::Roa - { - return None; - } - Some(Self { - item_effective_until: output.item_effective_until.clone(), - source_ee_cert_hash: output.source_ee_cert_hash, - payload: output.payload.clone(), - rule_hash: output.rule_hash, - }) - } - - pub fn validate_internal(&self) -> StorageResult<()> { - parse_time( - "roa_cache_projection.entries[].outputs[].item_effective_until", - &self.item_effective_until, - )?; - if !matches!(self.payload, VcirLocalOutputPayload::Vrp { .. }) { - return Err(StorageError::InvalidData { - entity: "roa_cache_projection.entries[].outputs[]", - detail: "payload must be VRP".to_string(), - }); - } - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct RoaCacheObjectProjection { - #[serde(rename = "u")] - pub source_object_uri: String, - #[serde(rename = "h")] - #[serde(with = "serde_bytes_32")] - pub source_object_hash: [u8; 32], - #[serde(rename = "s", default, skip_serializing_if = "Option::is_none")] - #[serde(with = "serde_optional_byte_vec")] - pub ee_serial: Option>, - #[serde(rename = "c", default, skip_serializing_if = "Option::is_none")] - pub crl_uri: Option, - #[serde(rename = "n", default, skip_serializing_if = "Option::is_none")] - pub earliest_safe_reuse_time_unix: Option, - #[serde(rename = "x")] - pub outputs_effective_until_unix: i64, - #[serde(rename = "o")] - pub outputs: Vec, -} - -impl RoaCacheObjectProjection { - pub fn validate_internal(&self) -> StorageResult<()> { - validate_non_empty( - "roa_cache_projection.entries[].source_object_uri", - &self.source_object_uri, - )?; - if let Some(serial) = &self.ee_serial { - if serial.is_empty() { - return Err(StorageError::InvalidData { - entity: "roa_cache_projection.entries[].ee_serial", - detail: "must not be empty when present".to_string(), - }); - } - } - if let Some(crl_uri) = &self.crl_uri { - validate_non_empty("roa_cache_projection.entries[].crl_uri", crl_uri)?; - } - if let Some(earliest_safe_reuse_time_unix) = self.earliest_safe_reuse_time_unix - && earliest_safe_reuse_time_unix >= self.outputs_effective_until_unix - { - return Err(StorageError::InvalidData { - entity: "roa_cache_projection.entries[].effective_window", - detail: "earliest_safe_reuse_time_unix must be before outputs_effective_until_unix" - .to_string(), - }); - } - if self.outputs.is_empty() { - return Err(StorageError::InvalidData { - entity: "roa_cache_projection.entries[]", - detail: "outputs must not be empty".to_string(), - }); - } - for output in &self.outputs { - output.validate_internal()?; - } - let expected_effective_until = self - .outputs - .iter() - .map(|output| { - output - .item_effective_until - .parse() - .map(|time| time.unix_timestamp()) - .map_err(|detail| StorageError::InvalidData { - entity: "roa_cache_projection.entries[].outputs_effective_until_unix", - detail, - }) - }) - .collect::>>()? - .into_iter() - .min() - .expect("outputs must not be empty"); - if self.outputs_effective_until_unix != expected_effective_until { - return Err(StorageError::InvalidData { - entity: "roa_cache_projection.entries[].outputs_effective_until_unix", - detail: "must equal the earliest output item_effective_until".to_string(), - }); - } - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct RoaCacheProjection { - #[serde(rename = "m")] - pub manifest_rsync_uri: String, - #[serde(rename = "e")] - pub instance_effective_until: PackTime, - #[serde(rename = "i")] - pub issuer_ca_sha256_hex: Option, - #[serde(rename = "p", default, skip_serializing_if = "Option::is_none")] - #[serde(with = "serde_optional_bytes_32")] - pub ca_validation_context_digest: Option<[u8; 32]>, - #[serde(rename = "f", default, skip_serializing_if = "Option::is_none")] - #[serde(with = "serde_optional_bytes_32")] - pub policy_fingerprint: Option<[u8; 32]>, - #[serde(rename = "c")] - pub crl_sha256_by_uri: Vec, - #[serde(rename = "r")] - pub entries: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ChildCertificateCacheRouterKeyProjection { - #[serde(rename = "a")] - pub as_id: u32, - #[serde(rename = "s")] - #[serde(with = "serde_byte_vec")] - pub ski: Vec, - #[serde(rename = "p")] - #[serde(with = "serde_byte_vec")] - pub spki_der: Vec, - #[serde(rename = "e")] - pub item_effective_until: PackTime, -} - -impl ChildCertificateCacheRouterKeyProjection { - pub fn validate_internal(&self) -> StorageResult<()> { - if self.ski.is_empty() { - return Err(StorageError::InvalidData { - entity: "child_certificate_cache_projection.router_keys[].ski", - detail: "must not be empty".to_string(), - }); - } - if self.spki_der.is_empty() { - return Err(StorageError::InvalidData { - entity: "child_certificate_cache_projection.router_keys[].spki_der", - detail: "must not be empty".to_string(), - }); - } - parse_time( - "child_certificate_cache_projection.router_keys[].item_effective_until", - &self.item_effective_until, - )?; - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ChildCertificateCachePayload { - ChildCa { - #[serde(rename = "cm")] - child_manifest_rsync_uri: String, - #[serde(rename = "ski")] - child_ski: String, - #[serde(rename = "rb")] - child_rsync_base_uri: String, - #[serde(rename = "pp")] - child_publication_point_rsync_uri: String, - #[serde(rename = "rn")] - child_rrdp_notification_uri: Option, - #[serde(rename = "ip")] - child_effective_ip_resources: Option, - #[serde(rename = "as")] - child_effective_as_resources: Option, - }, - Router { - #[serde(rename = "r")] - router_keys: Vec, - }, -} - -impl ChildCertificateCachePayload { - pub fn validate_internal(&self) -> StorageResult<()> { - match self { - Self::ChildCa { - child_manifest_rsync_uri, - child_ski, - child_rsync_base_uri, - child_publication_point_rsync_uri, - child_rrdp_notification_uri, - .. - } => { - validate_non_empty( - "child_certificate_cache_projection.child_manifest_rsync_uri", - child_manifest_rsync_uri, - )?; - validate_non_empty("child_certificate_cache_projection.child_ski", child_ski)?; - validate_non_empty( - "child_certificate_cache_projection.child_rsync_base_uri", - child_rsync_base_uri, - )?; - validate_non_empty( - "child_certificate_cache_projection.child_publication_point_rsync_uri", - child_publication_point_rsync_uri, - )?; - if let Some(uri) = child_rrdp_notification_uri { - validate_non_empty( - "child_certificate_cache_projection.child_rrdp_notification_uri", - uri, - )?; - } - Ok(()) - } - Self::Router { router_keys } => { - if router_keys.is_empty() { - return Err(StorageError::InvalidData { - entity: "child_certificate_cache_projection.router_keys", - detail: "must not be empty".to_string(), - }); - } - for router_key in router_keys { - router_key.validate_internal()?; - } - Ok(()) - } - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ChildCertificateCacheProjection { - #[serde(rename = "sv")] - pub schema_version: u32, - #[serde(rename = "av")] - pub algorithm_version: u32, - #[serde(rename = "k")] - pub cache_key_sha256_hex: String, - #[serde(rename = "cu")] - pub child_cert_uri: String, - #[serde(rename = "ch")] - pub child_cert_sha256_hex: String, - #[serde(rename = "cs")] - #[serde(with = "serde_byte_vec")] - pub child_cert_serial: Vec, - #[serde(rename = "ih")] - pub issuer_ca_sha256_hex: String, - #[serde(rename = "cru")] - pub issuer_crl_uri: String, - #[serde(rename = "crh")] - pub issuer_crl_sha256_hex: String, - #[serde(rename = "pc")] - #[serde(with = "serde_bytes_32")] - pub ca_validation_context_digest: [u8; 32], - #[serde(rename = "pf")] - #[serde(with = "serde_bytes_32")] - pub validation_policy_fingerprint: [u8; 32], - #[serde(rename = "nb")] - pub effective_not_before: PackTime, - #[serde(rename = "nu")] - pub effective_until: PackTime, - #[serde(rename = "p")] - pub payload: ChildCertificateCachePayload, -} - -pub const CHILD_CERTIFICATE_CACHE_SCHEMA_VERSION: u32 = 2; -pub const CHILD_CERTIFICATE_CACHE_ALGORITHM_VERSION: u32 = 3; - -impl ChildCertificateCacheProjection { - pub fn validate_internal(&self) -> StorageResult<()> { - if self.schema_version != CHILD_CERTIFICATE_CACHE_SCHEMA_VERSION { - return Err(StorageError::InvalidData { - entity: "child_certificate_cache_projection.schema_version", - detail: format!("unsupported schema_version {}", self.schema_version), - }); - } - if self.algorithm_version != CHILD_CERTIFICATE_CACHE_ALGORITHM_VERSION { - return Err(StorageError::InvalidData { - entity: "child_certificate_cache_projection.algorithm_version", - detail: format!("unsupported algorithm_version {}", self.algorithm_version), - }); - } - validate_sha256_hex( - "child_certificate_cache_projection.cache_key_sha256_hex", - &self.cache_key_sha256_hex, - )?; - validate_non_empty( - "child_certificate_cache_projection.child_cert_uri", - &self.child_cert_uri, - )?; - validate_sha256_hex( - "child_certificate_cache_projection.child_cert_sha256_hex", - &self.child_cert_sha256_hex, - )?; - if self.child_cert_serial.is_empty() { - return Err(StorageError::InvalidData { - entity: "child_certificate_cache_projection.child_cert_serial", - detail: "must not be empty".to_string(), - }); - } - validate_sha256_hex( - "child_certificate_cache_projection.issuer_ca_sha256_hex", - &self.issuer_ca_sha256_hex, - )?; - validate_non_empty( - "child_certificate_cache_projection.issuer_crl_uri", - &self.issuer_crl_uri, - )?; - validate_sha256_hex( - "child_certificate_cache_projection.issuer_crl_sha256_hex", - &self.issuer_crl_sha256_hex, - )?; - let effective_not_before = parse_time( - "child_certificate_cache_projection.effective_not_before", - &self.effective_not_before, - )?; - let effective_until = parse_time( - "child_certificate_cache_projection.effective_until", - &self.effective_until, - )?; - if effective_not_before >= effective_until { - return Err(StorageError::InvalidData { - entity: "child_certificate_cache_projection.effective_window", - detail: "effective_not_before must be before effective_until".to_string(), - }); - } - self.payload.validate_internal()?; - Ok(()) - } -} - -pub const PUBLICATION_POINT_CACHE_SCHEMA_VERSION: u32 = 1; -pub const PUBLICATION_POINT_CACHE_ALGORITHM_VERSION: u32 = 1; - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct PublicationPointCacheOutput { - #[serde(rename = "t")] - pub output_type: VcirOutputType, - #[serde(rename = "nb")] - pub item_effective_not_before: PackTime, - #[serde(rename = "nu")] - pub item_effective_until: PackTime, - #[serde(rename = "u")] - pub source_object_uri: String, - #[serde(rename = "k")] - pub source_object_type: VcirSourceObjectType, - #[serde(rename = "h")] - #[serde(with = "serde_bytes_32")] - pub source_object_hash: [u8; 32], - #[serde(rename = "c")] - #[serde(with = "serde_bytes_32")] - pub source_ee_cert_hash: [u8; 32], - #[serde(rename = "p")] - pub payload: VcirLocalOutputPayload, - #[serde(rename = "r")] - #[serde(with = "serde_bytes_32")] - pub rule_hash: [u8; 32], -} - -impl PublicationPointCacheOutput { - fn from_local_output(output: &VcirLocalOutput, default_not_before: &PackTime) -> Self { - Self { - output_type: output.output_type, - item_effective_not_before: default_not_before.clone(), - item_effective_until: output.item_effective_until.clone(), - source_object_uri: output.source_object_uri.clone(), - source_object_type: output.source_object_type, - source_object_hash: output.source_object_hash, - source_ee_cert_hash: output.source_ee_cert_hash, - payload: output.payload.clone(), - rule_hash: output.rule_hash, - } - } - - pub fn to_local_output(&self) -> VcirLocalOutput { - VcirLocalOutput { - output_type: self.output_type, - item_effective_until: self.item_effective_until.clone(), - source_object_uri: self.source_object_uri.clone(), - source_object_type: self.source_object_type, - source_object_hash: self.source_object_hash, - source_ee_cert_hash: self.source_ee_cert_hash, - payload: self.payload.clone(), - rule_hash: self.rule_hash, - } - } - - pub fn validate_internal(&self) -> StorageResult<()> { - parse_time( - "publication_point_cache_projection.outputs[].item_effective_not_before", - &self.item_effective_not_before, - )?; - parse_time( - "publication_point_cache_projection.outputs[].item_effective_until", - &self.item_effective_until, - )?; - validate_non_empty( - "publication_point_cache_projection.outputs[].source_object_uri", - &self.source_object_uri, - )?; - VcirLocalOutput { - output_type: self.output_type, - item_effective_until: self.item_effective_until.clone(), - source_object_uri: self.source_object_uri.clone(), - source_object_type: self.source_object_type, - source_object_hash: self.source_object_hash, - source_ee_cert_hash: self.source_ee_cert_hash, - payload: self.payload.clone(), - rule_hash: self.rule_hash, - } - .validate_internal() - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct PublicationPointCacheChild { - #[serde(rename = "cm")] - pub child_manifest_rsync_uri: String, - #[serde(rename = "cu")] - pub child_cert_rsync_uri: String, - #[serde(rename = "ch")] - pub child_cert_hash: String, - #[serde(rename = "ski")] - pub child_ski: String, - #[serde(rename = "rb")] - pub child_rsync_base_uri: String, - #[serde(rename = "pp")] - pub child_publication_point_rsync_uri: String, - #[serde(rename = "rn")] - pub child_rrdp_notification_uri: Option, - #[serde(rename = "ip")] - pub child_effective_ip_resources: Option, - #[serde(rename = "as")] - pub child_effective_as_resources: Option, - #[serde(rename = "nb")] - pub child_effective_not_before: PackTime, - #[serde(rename = "nu")] - pub child_effective_until: PackTime, -} - -impl PublicationPointCacheChild { - fn from_child_entry( - entry: &VcirChildEntry, - default_not_before: &PackTime, - default_until: &PackTime, - ) -> Self { - Self { - child_manifest_rsync_uri: entry.child_manifest_rsync_uri.clone(), - child_cert_rsync_uri: entry.child_cert_rsync_uri.clone(), - child_cert_hash: entry.child_cert_hash.clone(), - child_ski: entry.child_ski.clone(), - child_rsync_base_uri: entry.child_rsync_base_uri.clone(), - child_publication_point_rsync_uri: entry.child_publication_point_rsync_uri.clone(), - child_rrdp_notification_uri: entry.child_rrdp_notification_uri.clone(), - child_effective_ip_resources: entry.child_effective_ip_resources.clone(), - child_effective_as_resources: entry.child_effective_as_resources.clone(), - child_effective_not_before: default_not_before.clone(), - child_effective_until: default_until.clone(), - } - } - - pub fn to_child_entry(&self, accepted_at_validation_time: PackTime) -> VcirChildEntry { - VcirChildEntry { - child_manifest_rsync_uri: self.child_manifest_rsync_uri.clone(), - child_cert_rsync_uri: self.child_cert_rsync_uri.clone(), - child_cert_hash: self.child_cert_hash.clone(), - child_ski: self.child_ski.clone(), - child_rsync_base_uri: self.child_rsync_base_uri.clone(), - child_publication_point_rsync_uri: self.child_publication_point_rsync_uri.clone(), - child_rrdp_notification_uri: self.child_rrdp_notification_uri.clone(), - child_effective_ip_resources: self.child_effective_ip_resources.clone(), - child_effective_as_resources: self.child_effective_as_resources.clone(), - accepted_at_validation_time, - } - } - - pub fn validate_internal(&self) -> StorageResult<()> { - self.to_child_entry(self.child_effective_not_before.clone()) - .validate_internal()?; - parse_time( - "publication_point_cache_projection.children[].child_effective_not_before", - &self.child_effective_not_before, - )?; - parse_time( - "publication_point_cache_projection.children[].child_effective_until", - &self.child_effective_until, - )?; - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct PublicationPointCacheObject { - #[serde(rename = "r")] - pub artifact_role: VcirArtifactRole, - #[serde(rename = "k")] - pub artifact_kind: VcirArtifactKind, - #[serde(rename = "u")] - pub uri: Option, - #[serde(rename = "h")] - pub sha256: String, - #[serde(rename = "t")] - pub object_type: Option, - #[serde(rename = "s")] - pub validation_status: VcirArtifactValidationStatus, - /// See `VcirRelatedArtifact::reject_reason`. - #[serde(rename = "e", default, skip_serializing_if = "Option::is_none")] - pub reject_reason: Option, -} - -impl PublicationPointCacheObject { - fn from_related_artifact(artifact: &VcirRelatedArtifact) -> Self { - Self { - artifact_role: artifact.artifact_role, - artifact_kind: artifact.artifact_kind, - uri: artifact.uri.clone(), - sha256: artifact.sha256.clone(), - object_type: artifact.object_type.clone(), - validation_status: artifact.validation_status, - reject_reason: artifact.reject_reason.clone(), - } - } - - pub fn to_related_artifact(&self) -> VcirRelatedArtifact { - VcirRelatedArtifact { - artifact_role: self.artifact_role, - artifact_kind: self.artifact_kind, - uri: self.uri.clone(), - sha256: self.sha256.clone(), - object_type: self.object_type.clone(), - validation_status: self.validation_status, - reject_reason: self.reject_reason.clone(), - } - } - - pub fn validate_internal(&self) -> StorageResult<()> { - self.to_related_artifact().validate_internal() - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct PublicationPointCacheProjection { - #[serde(rename = "sv")] - pub schema_version: u32, - #[serde(rename = "av")] - pub algorithm_version: u32, - #[serde(rename = "m")] - pub manifest_rsync_uri: String, - #[serde(rename = "pp")] - pub publication_point_rsync_uri: String, - #[serde(rename = "cu")] - pub ca_cert_uri: Option, - #[serde(rename = "ch")] - #[serde(with = "serde_bytes_32")] - pub ca_cert_sha256: [u8; 32], - #[serde(rename = "mh")] - #[serde(with = "serde_bytes_32")] - pub manifest_sha256: [u8; 32], - #[serde(rename = "tal")] - pub tal_id: String, - #[serde(rename = "ta")] - #[serde(with = "serde_bytes_32")] - pub ta_context_digest: [u8; 32], - #[serde(rename = "pc")] - #[serde(with = "serde_bytes_32")] - pub ca_validation_context_digest: [u8; 32], - #[serde(rename = "pf")] - #[serde(with = "serde_bytes_32")] - pub validation_policy_fingerprint: [u8; 32], - #[serde(rename = "nb")] - pub instance_effective_not_before: PackTime, - #[serde(rename = "nu")] - pub instance_effective_until: PackTime, - #[serde(rename = "mt")] - pub manifest_this_update: PackTime, - #[serde(rename = "mn")] - pub manifest_next_update: PackTime, - #[serde(rename = "cn")] - pub current_crl_next_update: PackTime, - #[serde(rename = "ca")] - pub self_ca_not_after: PackTime, - #[serde(rename = "ccr")] - pub ccr_manifest_projection: VcirCcrManifestProjection, - #[serde(rename = "o")] - pub outputs: Vec, - #[serde(rename = "c")] - pub children: Vec, - #[serde(rename = "ra")] - pub related_objects: Vec, - #[serde(rename = "s")] - pub summary: VcirSummary, -} - -impl PublicationPointCacheProjection { - pub fn from_vcir_with_context( - vcir: &ValidatedCaInstanceResult, - publication_point_rsync_uri: String, - ca_cert_uri: Option, - ca_cert_sha256: [u8; 32], - manifest_sha256: [u8; 32], - ta_context_digest: [u8; 32], - ca_validation_context_digest: [u8; 32], - validation_policy_fingerprint: [u8; 32], - ) -> StorageResult { - let projection = Self { - schema_version: PUBLICATION_POINT_CACHE_SCHEMA_VERSION, - algorithm_version: PUBLICATION_POINT_CACHE_ALGORITHM_VERSION, - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - publication_point_rsync_uri, - ca_cert_uri, - ca_cert_sha256, - manifest_sha256, - tal_id: vcir.tal_id.clone(), - ta_context_digest, - ca_validation_context_digest, - validation_policy_fingerprint, - instance_effective_not_before: vcir.last_successful_validation_time.clone(), - instance_effective_until: vcir.instance_gate.instance_effective_until.clone(), - manifest_this_update: vcir - .validated_manifest_meta - .validated_manifest_this_update - .clone(), - manifest_next_update: vcir.instance_gate.manifest_next_update.clone(), - current_crl_next_update: vcir.instance_gate.current_crl_next_update.clone(), - self_ca_not_after: vcir.instance_gate.self_ca_not_after.clone(), - ccr_manifest_projection: vcir.ccr_manifest_projection.clone(), - outputs: vcir - .local_outputs - .iter() - .map(|output| { - PublicationPointCacheOutput::from_local_output( - output, - &vcir.last_successful_validation_time, - ) - }) - .collect(), - children: vcir - .child_entries - .iter() - .map(|child| { - PublicationPointCacheChild::from_child_entry( - child, - &vcir.last_successful_validation_time, - &vcir.instance_gate.instance_effective_until, - ) - }) - .collect(), - related_objects: vcir - .related_artifacts - .iter() - .map(PublicationPointCacheObject::from_related_artifact) - .collect(), - summary: vcir.summary.clone(), - }; - projection.validate_internal()?; - Ok(projection) - } - - pub fn to_vcir_for_reuse( - &self, - validation_time: time::OffsetDateTime, - ) -> ValidatedCaInstanceResult { - let validation_time = PackTime::from_utc_offset_datetime(validation_time); - ValidatedCaInstanceResult { - manifest_rsync_uri: self.manifest_rsync_uri.clone(), - parent_manifest_rsync_uri: None, - tal_id: self.tal_id.clone(), - ca_subject_name: String::from("publication-point-cache"), - ca_ski: hex::encode(self.ca_cert_sha256), - issuer_ski: hex::encode(self.ca_validation_context_digest), - last_successful_validation_time: validation_time.clone(), - current_manifest_rsync_uri: self.manifest_rsync_uri.clone(), - current_crl_rsync_uri: self - .related_objects - .iter() - .find(|object| object.artifact_role == VcirArtifactRole::CurrentCrl) - .and_then(|object| object.uri.clone()) - .unwrap_or_default(), - validated_manifest_meta: ValidatedManifestMeta { - validated_manifest_number: self.ccr_manifest_projection.manifest_number_be.clone(), - validated_manifest_this_update: self.manifest_this_update.clone(), - validated_manifest_next_update: self.manifest_next_update.clone(), - }, - ccr_manifest_projection: self.ccr_manifest_projection.clone(), - instance_gate: VcirInstanceGate { - manifest_next_update: self.manifest_next_update.clone(), - current_crl_next_update: self.current_crl_next_update.clone(), - self_ca_not_after: self.self_ca_not_after.clone(), - instance_effective_until: self.instance_effective_until.clone(), - }, - child_entries: self - .children - .iter() - .map(|child| child.to_child_entry(validation_time.clone())) - .collect(), - local_outputs: self - .outputs - .iter() - .map(PublicationPointCacheOutput::to_local_output) - .collect(), - related_artifacts: self - .related_objects - .iter() - .map(PublicationPointCacheObject::to_related_artifact) - .collect(), - summary: self.summary.clone(), - audit_summary: VcirAuditSummary { - failed_fetch_eligible: false, - last_failed_fetch_reason: None, - warning_count: 0, - audit_flags: vec!["publication_point_cache_projection".to_string()], - }, - } - } - - pub fn validate_internal(&self) -> StorageResult<()> { - if self.schema_version != PUBLICATION_POINT_CACHE_SCHEMA_VERSION { - return Err(StorageError::InvalidData { - entity: "publication_point_cache_projection.schema_version", - detail: format!("unsupported schema_version {}", self.schema_version), - }); - } - if self.algorithm_version != PUBLICATION_POINT_CACHE_ALGORITHM_VERSION { - return Err(StorageError::InvalidData { - entity: "publication_point_cache_projection.algorithm_version", - detail: format!("unsupported algorithm_version {}", self.algorithm_version), - }); - } - validate_non_empty( - "publication_point_cache_projection.manifest_rsync_uri", - &self.manifest_rsync_uri, - )?; - validate_non_empty( - "publication_point_cache_projection.publication_point_rsync_uri", - &self.publication_point_rsync_uri, - )?; - if let Some(uri) = &self.ca_cert_uri { - validate_non_empty("publication_point_cache_projection.ca_cert_uri", uri)?; - } - validate_non_empty("publication_point_cache_projection.tal_id", &self.tal_id)?; - parse_time( - "publication_point_cache_projection.instance_effective_not_before", - &self.instance_effective_not_before, - )?; - parse_time( - "publication_point_cache_projection.instance_effective_until", - &self.instance_effective_until, - )?; - parse_time( - "publication_point_cache_projection.manifest_this_update", - &self.manifest_this_update, - )?; - parse_time( - "publication_point_cache_projection.manifest_next_update", - &self.manifest_next_update, - )?; - parse_time( - "publication_point_cache_projection.current_crl_next_update", - &self.current_crl_next_update, - )?; - parse_time( - "publication_point_cache_projection.self_ca_not_after", - &self.self_ca_not_after, - )?; - self.ccr_manifest_projection.validate_internal()?; - for output in &self.outputs { - output.validate_internal()?; - } - for child in &self.children { - child.validate_internal()?; - } - for object in &self.related_objects { - object.validate_internal()?; - } - Ok(()) - } -} - -impl RoaCacheProjection { - pub fn from_vcir(vcir: &ValidatedCaInstanceResult) -> StorageResult> { - Self::from_vcir_with_context(vcir, None) - } - - pub fn from_vcir_with_context( - vcir: &ValidatedCaInstanceResult, - context: Option<&RoaCacheProjectionContext>, - ) -> StorageResult> { - let Some(context) = context else { - return Ok(None); - }; - let mut issuer_ca_sha256_hex = None; - let mut crl_sha256_by_uri = Vec::new(); - for artifact in &vcir.related_artifacts { - if artifact.validation_status != VcirArtifactValidationStatus::Accepted { - continue; - } - match (artifact.artifact_role, artifact.artifact_kind) { - ( - VcirArtifactRole::IssuerCert | VcirArtifactRole::TrustAnchorCert, - VcirArtifactKind::Cer, - ) => { - issuer_ca_sha256_hex = Some(artifact.sha256.clone()); - } - (_, VcirArtifactKind::Crl) => { - if let Some(uri) = artifact.uri.as_ref() { - crl_sha256_by_uri.push(RoaCacheCrlProjection { - uri: uri.clone(), - sha256: artifact.sha256.clone(), - }); - } - } - _ => {} - } - } - crl_sha256_by_uri.sort_by(|left, right| left.uri.cmp(&right.uri)); - - let meta_by_uri = context - .object_meta - .iter() - .map(|meta| (meta.source_object_uri.as_str(), meta)) - .collect::>(); - let vcir_validation_time = - vcir.last_successful_validation_time - .parse() - .map_err(|detail| StorageError::InvalidData { - entity: "roa_cache_projection.entries[].earliest_safe_reuse_time_unix", - detail, - })?; - let mut entries: Vec = Vec::new(); - let mut entry_index_by_uri: HashMap = HashMap::new(); - for output in &vcir.local_outputs { - let Some(projected_output) = RoaCacheLocalOutputProjection::from_local_output(output) - else { - continue; - }; - let projected_output_effective_until = projected_output - .item_effective_until - .parse() - .map(|time| time.unix_timestamp()) - .map_err(|detail| StorageError::InvalidData { - entity: "roa_cache_projection.entries[].outputs_effective_until_unix", - detail, - })?; - let Some(meta) = meta_by_uri.get(output.source_object_uri.as_str()).copied() else { - continue; - }; - if meta.source_object_hash != output.source_object_hash { - return Err(StorageError::InvalidData { - entity: "roa_cache_projection.entries[]", - detail: format!( - "metadata source object hash mismatch for {}", - output.source_object_uri - ), - }); - } - let earliest_safe_reuse_time_unix = meta - .earliest_safe_reuse_time - .parse() - .map(|time| time.max(vcir_validation_time).unix_timestamp()) - .map_err(|detail| StorageError::InvalidData { - entity: "roa_cache_projection.entries[].earliest_safe_reuse_time_unix", - detail, - })?; - if let Some(entry_index) = entry_index_by_uri.get(output.source_object_uri.as_str()) { - let entry = &mut entries[*entry_index]; - if entry.source_object_hash != output.source_object_hash { - return Err(StorageError::InvalidData { - entity: "roa_cache_projection.entries[]", - detail: format!( - "source object hash mismatch for {}", - output.source_object_uri - ), - }); - } - entry.outputs_effective_until_unix = entry - .outputs_effective_until_unix - .min(projected_output_effective_until); - entry.outputs.push(projected_output); - } else { - entry_index_by_uri.insert(output.source_object_uri.clone(), entries.len()); - entries.push(RoaCacheObjectProjection { - source_object_uri: output.source_object_uri.clone(), - source_object_hash: output.source_object_hash, - ee_serial: Some(meta.ee_serial.clone()), - crl_uri: Some(meta.crl_uri.clone()), - earliest_safe_reuse_time_unix: Some(earliest_safe_reuse_time_unix), - outputs_effective_until_unix: projected_output_effective_until, - outputs: vec![projected_output], - }); - } - } - if entries.is_empty() { - return Ok(None); - } - entries.sort_by(|left, right| left.source_object_uri.cmp(&right.source_object_uri)); - - let projection = Self { - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - instance_effective_until: vcir.instance_gate.instance_effective_until.clone(), - issuer_ca_sha256_hex, - ca_validation_context_digest: Some(context.ca_validation_context_digest), - policy_fingerprint: Some(context.policy_fingerprint), - crl_sha256_by_uri, - entries, - }; - projection.validate_internal()?; - Ok(Some(projection)) - } - - pub fn validate_internal(&self) -> StorageResult<()> { - validate_non_empty( - "roa_cache_projection.manifest_rsync_uri", - &self.manifest_rsync_uri, - )?; - parse_time( - "roa_cache_projection.instance_effective_until", - &self.instance_effective_until, - )?; - if let Some(hash) = &self.issuer_ca_sha256_hex { - validate_sha256_hex("roa_cache_projection.issuer_ca_sha256_hex", hash)?; - } - if self.ca_validation_context_digest.is_some() != self.policy_fingerprint.is_some() { - return Err(StorageError::InvalidData { - entity: "roa_cache_projection.context", - detail: "ca_validation_context_digest and policy_fingerprint must be both present or both absent" - .to_string(), - }); - } - let mut seen_crls = HashSet::with_capacity(self.crl_sha256_by_uri.len()); - for crl in &self.crl_sha256_by_uri { - crl.validate_internal()?; - if !seen_crls.insert(crl.uri.as_str()) { - return Err(StorageError::InvalidData { - entity: "roa_cache_projection.crls[]", - detail: format!("duplicate CRL URI: {}", crl.uri), - }); - } - } - if self.entries.is_empty() { - return Err(StorageError::InvalidData { - entity: "roa_cache_projection.entries", - detail: "must not be empty".to_string(), - }); - } - let mut seen_entries = HashSet::with_capacity(self.entries.len()); - for entry in &self.entries { - entry.validate_internal()?; - if !seen_entries.insert(entry.source_object_uri.as_str()) { - return Err(StorageError::InvalidData { - entity: "roa_cache_projection.entries[]", - detail: format!("duplicate ROA URI: {}", entry.source_object_uri), - }); - } - } - Ok(()) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum VcirArtifactRole { - Manifest, - CurrentCrl, - ChildCaCert, - SignedObject, - EeCert, - IssuerCert, - Tal, - TrustAnchorCert, - Other, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum VcirArtifactKind { - Cer, - Crl, - Mft, - Roa, - Aspa, - Gbr, - Tal, - Other, -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum VcirArtifactValidationStatus { - Accepted, - Rejected, - WarningOnly, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct VcirRelatedArtifact { - #[serde(rename = "r")] - pub artifact_role: VcirArtifactRole, - #[serde(rename = "k")] - pub artifact_kind: VcirArtifactKind, - #[serde(rename = "u")] - pub uri: Option, - #[serde(rename = "h")] - pub sha256: String, - #[serde(rename = "t")] - pub object_type: Option, - #[serde(rename = "s")] - pub validation_status: VcirArtifactValidationStatus, - /// Reject reason captured at fresh validation time for `Rejected` artifacts. - /// `None` for accepted/warning artifacts and for cache entries written - /// before this field existed. - #[serde(rename = "e", default, skip_serializing_if = "Option::is_none")] - pub reject_reason: Option, -} - -impl VcirRelatedArtifact { - pub fn validate_internal(&self) -> StorageResult<()> { - if let Some(uri) = &self.uri { - validate_non_empty("vcir.related_artifacts[].uri", uri)?; - } - validate_sha256_hex("vcir.related_artifacts[].sha256", &self.sha256)?; - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct VcirSummary { - #[serde(rename = "v")] - pub local_vrp_count: u32, - #[serde(rename = "a")] - pub local_aspa_count: u32, - #[serde(rename = "r")] - pub local_router_key_count: u32, - #[serde(rename = "c")] - pub child_count: u32, - #[serde(rename = "o")] - pub accepted_object_count: u32, - #[serde(rename = "x")] - pub rejected_object_count: u32, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct VcirAuditSummary { - #[serde(rename = "f")] - pub failed_fetch_eligible: bool, - #[serde(rename = "r")] - pub last_failed_fetch_reason: Option, - #[serde(rename = "w")] - pub warning_count: u32, - #[serde(rename = "a")] - pub audit_flags: Vec, -} - -impl VcirAuditSummary { - pub fn validate_internal(&self) -> StorageResult<()> { - if let Some(reason) = &self.last_failed_fetch_reason { - validate_non_empty("vcir.audit_summary.last_failed_fetch_reason", reason)?; - } - for flag in &self.audit_flags { - validate_non_empty("vcir.audit_summary.audit_flags[]", flag)?; - } - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct ValidatedCaInstanceResult { - #[serde(rename = "m")] - pub manifest_rsync_uri: String, - #[serde(rename = "pm")] - pub parent_manifest_rsync_uri: Option, - #[serde(rename = "tal")] - pub tal_id: String, - #[serde(rename = "subj")] - pub ca_subject_name: String, - #[serde(rename = "ski")] - pub ca_ski: String, - #[serde(rename = "aki")] - pub issuer_ski: String, - #[serde(rename = "vt")] - pub last_successful_validation_time: PackTime, - #[serde(rename = "cm")] - pub current_manifest_rsync_uri: String, - #[serde(rename = "crl")] - pub current_crl_rsync_uri: String, - #[serde(rename = "mm")] - pub validated_manifest_meta: ValidatedManifestMeta, - #[serde(rename = "ccr")] - pub ccr_manifest_projection: VcirCcrManifestProjection, - #[serde(rename = "g")] - pub instance_gate: VcirInstanceGate, - #[serde(rename = "ch")] - pub child_entries: Vec, - #[serde(rename = "lo")] - pub local_outputs: Vec, - #[serde(rename = "ra")] - pub related_artifacts: Vec, - #[serde(rename = "s")] - pub summary: VcirSummary, - #[serde(rename = "as")] - pub audit_summary: VcirAuditSummary, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] -pub struct VcirFieldSizeBreakdown { - pub local_output_count: u64, - pub local_output_source_uri_bytes: u64, - pub local_output_source_type_bytes: u64, - pub local_output_source_hash_hex_bytes: u64, - pub local_output_source_ee_hash_hex_bytes: u64, - pub local_output_payload_json_bytes: u64, - pub local_output_rule_hash_hex_bytes: u64, - pub local_output_source_hash_binary_bytes: u64, - pub local_output_source_ee_hash_binary_bytes: u64, - pub local_output_payload_typed_body_bytes: u64, - pub local_output_rule_hash_binary_bytes: u64, - pub related_artifact_count: u64, - pub related_artifact_uri_bytes: u64, - pub related_artifact_hash_hex_bytes: u64, - pub related_artifact_type_bytes: u64, - pub child_entry_count: u64, - pub child_entry_uri_bytes: u64, - pub child_entry_hash_hex_bytes: u64, -} - -fn serialized_cbor_len(value: &T) -> u64 { - serde_cbor::to_vec(value) - .map(|bytes| bytes.len() as u64) - .unwrap_or(0) -} - -impl VcirFieldSizeBreakdown { - pub fn from_vcir(vcir: &ValidatedCaInstanceResult) -> Self { - let mut out = Self::default(); - out.local_output_count = vcir.local_outputs.len() as u64; - for local in &vcir.local_outputs { - out.local_output_source_uri_bytes += local.source_object_uri.len() as u64; - out.local_output_source_type_bytes += local.source_object_type_name().len() as u64; - out.local_output_source_hash_hex_bytes += 64; - out.local_output_source_ee_hash_hex_bytes += 64; - out.local_output_payload_json_bytes += local.payload_json().len() as u64; - out.local_output_rule_hash_hex_bytes += 64; - out.local_output_source_hash_binary_bytes += 32; - out.local_output_source_ee_hash_binary_bytes += 32; - out.local_output_payload_typed_body_bytes += local.payload.typed_body_bytes(); - out.local_output_rule_hash_binary_bytes += 32; - } - out.related_artifact_count = vcir.related_artifacts.len() as u64; - for artifact in &vcir.related_artifacts { - if let Some(uri) = &artifact.uri { - out.related_artifact_uri_bytes += uri.len() as u64; - } - out.related_artifact_hash_hex_bytes += artifact.sha256.len() as u64; - if let Some(object_type) = &artifact.object_type { - out.related_artifact_type_bytes += object_type.len() as u64; - } - } - out.child_entry_count = vcir.child_entries.len() as u64; - for child in &vcir.child_entries { - out.child_entry_uri_bytes += child.child_manifest_rsync_uri.len() as u64 - + child.child_cert_rsync_uri.len() as u64 - + child.child_rsync_base_uri.len() as u64 - + child.child_publication_point_rsync_uri.len() as u64 - + child - .child_rrdp_notification_uri - .as_ref() - .map(|uri| uri.len() as u64) - .unwrap_or(0); - out.child_entry_hash_hex_bytes += - child.child_cert_hash.len() as u64 + child.child_ski.len() as u64; - } - out - } - - pub fn add_assign(&mut self, other: &Self) { - self.local_output_count += other.local_output_count; - self.local_output_source_uri_bytes += other.local_output_source_uri_bytes; - self.local_output_source_type_bytes += other.local_output_source_type_bytes; - self.local_output_source_hash_hex_bytes += other.local_output_source_hash_hex_bytes; - self.local_output_source_ee_hash_hex_bytes += other.local_output_source_ee_hash_hex_bytes; - self.local_output_payload_json_bytes += other.local_output_payload_json_bytes; - self.local_output_rule_hash_hex_bytes += other.local_output_rule_hash_hex_bytes; - self.local_output_source_hash_binary_bytes += other.local_output_source_hash_binary_bytes; - self.local_output_source_ee_hash_binary_bytes += - other.local_output_source_ee_hash_binary_bytes; - self.local_output_payload_typed_body_bytes += other.local_output_payload_typed_body_bytes; - self.local_output_rule_hash_binary_bytes += other.local_output_rule_hash_binary_bytes; - self.related_artifact_count += other.related_artifact_count; - self.related_artifact_uri_bytes += other.related_artifact_uri_bytes; - self.related_artifact_hash_hex_bytes += other.related_artifact_hash_hex_bytes; - self.related_artifact_type_bytes += other.related_artifact_type_bytes; - self.child_entry_count += other.child_entry_count; - self.child_entry_uri_bytes += other.child_entry_uri_bytes; - self.child_entry_hash_hex_bytes += other.child_entry_hash_hex_bytes; - } - - pub fn local_output_old_projection_bytes(&self) -> u64 { - self.local_output_source_type_bytes - + self.local_output_source_hash_hex_bytes - + self.local_output_source_ee_hash_hex_bytes - + self.local_output_payload_json_bytes - + self.local_output_rule_hash_hex_bytes - } - - pub fn local_output_typed_projection_bytes(&self) -> u64 { - self.local_output_count - + self.local_output_source_hash_binary_bytes - + self.local_output_source_ee_hash_binary_bytes - + self.local_output_payload_typed_body_bytes - + self.local_output_rule_hash_binary_bytes - } - - pub fn local_output_projection_saved_bytes(&self) -> u64 { - self.local_output_old_projection_bytes() - .saturating_sub(self.local_output_typed_projection_bytes()) - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] -pub struct VcirReplaceTimingBreakdown { - pub validate_ms: u64, - pub vcir_encode_ms: u64, - pub vcir_value_bytes: u64, - pub replay_meta_encode_ms: u64, - pub replay_meta_value_bytes: u64, - pub roa_cache_projection_encode_ms: u64, - pub roa_cache_projection_value_bytes: u64, - pub publication_point_cache_projection_encode_ms: u64, - pub publication_point_cache_projection_value_bytes: u64, - pub batch_build_ms: u64, - pub write_batch_ms: u64, - pub total_encoded_bytes: u64, - pub field_sizes: VcirFieldSizeBreakdown, - pub rss_before_kb: Option, - pub rss_after_validate_kb: Option, - pub rss_after_vcir_encode_kb: Option, - pub rss_after_replay_meta_encode_kb: Option, - pub rss_after_roa_cache_projection_encode_kb: Option, - pub rss_after_publication_point_cache_projection_encode_kb: Option, - pub rss_after_write_batch_kb: Option, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] -pub struct VcirStorageSummary { - pub entry_count: u64, - pub vcir_value_bytes: u64, - pub vcir_value_bytes_max: u64, - pub vcir_value_bytes_max_manifest_rsync_uri: Option, - pub core_fields: VcirCoreFieldSizeBreakdown, - pub ccr_projection: VcirCcrProjectionSizeBreakdown, - pub child_resources: VcirChildResourceSizeBreakdown, - pub field_sizes: VcirFieldSizeBreakdown, - pub local_output_old_projection_bytes: u64, - pub local_output_typed_projection_bytes: u64, - pub local_output_projection_saved_bytes: u64, - pub top_entries_by_vcir_value_bytes: Vec, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] -pub struct VcirStorageEntrySummary { - pub manifest_rsync_uri: String, - pub vcir_value_bytes: u64, - pub local_vrp_count: u32, - pub local_aspa_count: u32, - pub local_router_key_count: u32, - pub accepted_object_count: u32, - pub rejected_object_count: u32, - pub child_count: u32, - pub core_fields: VcirCoreFieldSizeBreakdown, - pub ccr_projection: VcirCcrProjectionSizeBreakdown, - pub child_resources: VcirChildResourceSizeBreakdown, - pub field_sizes: VcirFieldSizeBreakdown, - pub local_output_old_projection_bytes: u64, - pub local_output_typed_projection_bytes: u64, - pub local_output_projection_saved_bytes: u64, -} - -impl VcirStorageEntrySummary { - fn from_vcir(vcir: &ValidatedCaInstanceResult, vcir_value_bytes: u64) -> Self { - let field_sizes = VcirFieldSizeBreakdown::from_vcir(vcir); - Self { - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - vcir_value_bytes, - local_vrp_count: vcir.summary.local_vrp_count, - local_aspa_count: vcir.summary.local_aspa_count, - local_router_key_count: vcir.summary.local_router_key_count, - accepted_object_count: vcir.summary.accepted_object_count, - rejected_object_count: vcir.summary.rejected_object_count, - child_count: vcir.summary.child_count, - core_fields: VcirCoreFieldSizeBreakdown::from_vcir(vcir), - ccr_projection: VcirCcrProjectionSizeBreakdown::from_projection( - &vcir.ccr_manifest_projection, - ), - child_resources: VcirChildResourceSizeBreakdown::from_vcir(vcir), - local_output_old_projection_bytes: field_sizes.local_output_old_projection_bytes(), - local_output_typed_projection_bytes: field_sizes.local_output_typed_projection_bytes(), - local_output_projection_saved_bytes: field_sizes.local_output_projection_saved_bytes(), - field_sizes, - } - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] -pub struct VcirCoreFieldSizeBreakdown { - pub manifest_rsync_uri_bytes: u64, - pub parent_manifest_rsync_uri_bytes: u64, - pub tal_id_bytes: u64, - pub ca_subject_name_bytes: u64, - pub ca_ski_bytes: u64, - pub issuer_ski_bytes: u64, - pub current_manifest_rsync_uri_bytes: u64, - pub current_crl_rsync_uri_bytes: u64, - pub validated_manifest_number_bytes: u64, -} - -impl VcirCoreFieldSizeBreakdown { - fn from_vcir(vcir: &ValidatedCaInstanceResult) -> Self { - Self { - manifest_rsync_uri_bytes: vcir.manifest_rsync_uri.len() as u64, - parent_manifest_rsync_uri_bytes: vcir - .parent_manifest_rsync_uri - .as_ref() - .map(|uri| uri.len() as u64) - .unwrap_or(0), - tal_id_bytes: vcir.tal_id.len() as u64, - ca_subject_name_bytes: vcir.ca_subject_name.len() as u64, - ca_ski_bytes: vcir.ca_ski.len() as u64, - issuer_ski_bytes: vcir.issuer_ski.len() as u64, - current_manifest_rsync_uri_bytes: vcir.current_manifest_rsync_uri.len() as u64, - current_crl_rsync_uri_bytes: vcir.current_crl_rsync_uri.len() as u64, - validated_manifest_number_bytes: vcir - .validated_manifest_meta - .validated_manifest_number - .len() as u64, - } - } - - fn add_assign(&mut self, other: &Self) { - self.manifest_rsync_uri_bytes += other.manifest_rsync_uri_bytes; - self.parent_manifest_rsync_uri_bytes += other.parent_manifest_rsync_uri_bytes; - self.tal_id_bytes += other.tal_id_bytes; - self.ca_subject_name_bytes += other.ca_subject_name_bytes; - self.ca_ski_bytes += other.ca_ski_bytes; - self.issuer_ski_bytes += other.issuer_ski_bytes; - self.current_manifest_rsync_uri_bytes += other.current_manifest_rsync_uri_bytes; - self.current_crl_rsync_uri_bytes += other.current_crl_rsync_uri_bytes; - self.validated_manifest_number_bytes += other.validated_manifest_number_bytes; - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] -pub struct VcirChildResourceSizeBreakdown { - pub effective_ip_resource_cbor_bytes: u64, - pub effective_as_resource_cbor_bytes: u64, -} - -impl VcirChildResourceSizeBreakdown { - fn from_vcir(vcir: &ValidatedCaInstanceResult) -> Self { - let mut out = Self::default(); - for child in &vcir.child_entries { - out.effective_ip_resource_cbor_bytes += - serialized_cbor_len(&child.child_effective_ip_resources); - out.effective_as_resource_cbor_bytes += - serialized_cbor_len(&child.child_effective_as_resources); - } - out - } - - fn add_assign(&mut self, other: &Self) { - self.effective_ip_resource_cbor_bytes += other.effective_ip_resource_cbor_bytes; - self.effective_as_resource_cbor_bytes += other.effective_as_resource_cbor_bytes; - } -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] -pub struct VcirCcrProjectionSizeBreakdown { - pub manifest_rsync_uri_bytes: u64, - pub manifest_sha256_bytes: u64, - pub manifest_ee_aki_bytes: u64, - pub manifest_number_bytes: u64, - pub manifest_sia_locations_count: u64, - pub manifest_sia_locations_der_bytes: u64, - pub subordinate_ski_count: u64, - pub subordinate_ski_bytes: u64, -} - -impl VcirCcrProjectionSizeBreakdown { - fn from_projection(projection: &VcirCcrManifestProjection) -> Self { - Self { - manifest_rsync_uri_bytes: projection.manifest_rsync_uri.len() as u64, - manifest_sha256_bytes: projection.manifest_sha256.len() as u64, - manifest_ee_aki_bytes: projection.manifest_ee_aki.len() as u64, - manifest_number_bytes: projection.manifest_number_be.len() as u64, - manifest_sia_locations_count: projection.manifest_sia_locations_der.len() as u64, - manifest_sia_locations_der_bytes: projection - .manifest_sia_locations_der - .iter() - .map(|location| location.len() as u64) - .sum(), - subordinate_ski_count: projection.subordinate_skis.len() as u64, - subordinate_ski_bytes: projection - .subordinate_skis - .iter() - .map(|ski| ski.len() as u64) - .sum(), - } - } - - fn add_assign(&mut self, other: &Self) { - self.manifest_rsync_uri_bytes += other.manifest_rsync_uri_bytes; - self.manifest_sha256_bytes += other.manifest_sha256_bytes; - self.manifest_ee_aki_bytes += other.manifest_ee_aki_bytes; - self.manifest_number_bytes += other.manifest_number_bytes; - self.manifest_sia_locations_count += other.manifest_sia_locations_count; - self.manifest_sia_locations_der_bytes += other.manifest_sia_locations_der_bytes; - self.subordinate_ski_count += other.subordinate_ski_count; - self.subordinate_ski_bytes += other.subordinate_ski_bytes; - } -} - -fn push_top_vcir_storage_entry( - entries: &mut Vec, - entry: VcirStorageEntrySummary, -) { - const TOP_N: usize = 20; - entries.push(entry); - entries.sort_by(|left, right| { - right - .vcir_value_bytes - .cmp(&left.vcir_value_bytes) - .then_with(|| left.manifest_rsync_uri.cmp(&right.manifest_rsync_uri)) - }); - entries.truncate(TOP_N); -} - -impl ValidatedCaInstanceResult { - pub fn validate_internal(&self) -> StorageResult<()> { - validate_non_empty("vcir.manifest_rsync_uri", &self.manifest_rsync_uri)?; - if let Some(parent_manifest_rsync_uri) = &self.parent_manifest_rsync_uri { - validate_non_empty("vcir.parent_manifest_rsync_uri", parent_manifest_rsync_uri)?; - } - validate_non_empty("vcir.tal_id", &self.tal_id)?; - validate_non_empty("vcir.ca_subject_name", &self.ca_subject_name)?; - validate_non_empty("vcir.ca_ski", &self.ca_ski)?; - validate_non_empty("vcir.issuer_ski", &self.issuer_ski)?; - parse_time( - "vcir.last_successful_validation_time", - &self.last_successful_validation_time, - )?; - validate_non_empty( - "vcir.current_manifest_rsync_uri", - &self.current_manifest_rsync_uri, - )?; - validate_non_empty("vcir.current_crl_rsync_uri", &self.current_crl_rsync_uri)?; - self.validated_manifest_meta.validate_internal()?; - self.ccr_manifest_projection.validate_internal()?; - self.instance_gate.validate_internal()?; - - let expected_manifest_next = self - .validated_manifest_meta - .validated_manifest_next_update - .parse() - .map_err(|detail| StorageError::InvalidData { - entity: "vcir", - detail: format!( - "validated_manifest_meta.validated_manifest_next_update invalid: {detail}" - ), - })?; - let instance_manifest_next = - self.instance_gate - .manifest_next_update - .parse() - .map_err(|detail| StorageError::InvalidData { - entity: "vcir", - detail: format!("instance_gate.manifest_next_update invalid: {detail}"), - })?; - if expected_manifest_next != instance_manifest_next { - return Err(StorageError::InvalidData { - entity: "vcir", - detail: "instance_gate.manifest_next_update must equal validated_manifest_meta.validated_manifest_next_update".to_string(), - }); - } - - let mut child_manifests = HashSet::with_capacity(self.child_entries.len()); - for child in &self.child_entries { - child.validate_internal()?; - if !child_manifests.insert(child.child_manifest_rsync_uri.as_str()) { - return Err(StorageError::InvalidData { - entity: "vcir", - detail: format!( - "duplicate child_manifest_rsync_uri: {}", - child.child_manifest_rsync_uri - ), - }); - } - } - - let mut vrp_count = 0u32; - let mut aspa_count = 0u32; - let mut router_key_count = 0u32; - for output in &self.local_outputs { - output.validate_internal()?; - match output.output_type { - VcirOutputType::Vrp => vrp_count += 1, - VcirOutputType::Aspa => aspa_count += 1, - VcirOutputType::RouterKey => router_key_count += 1, - } - } - let mut output_ids = self - .local_outputs - .iter() - .map(VcirLocalOutput::output_id) - .collect::>(); - output_ids.sort_unstable(); - if let Some(duplicate) = output_ids - .windows(2) - .find_map(|pair| (pair[0] == pair[1]).then(|| pair[0].clone())) - { - return Err(StorageError::InvalidData { - entity: "vcir", - detail: format!("duplicate output_id: {duplicate}"), - }); - } - if self.summary.local_vrp_count != vrp_count { - return Err(StorageError::InvalidData { - entity: "vcir.summary", - detail: format!( - "local_vrp_count={} does not match local_outputs count {}", - self.summary.local_vrp_count, vrp_count - ), - }); - } - if self.summary.local_aspa_count != aspa_count { - return Err(StorageError::InvalidData { - entity: "vcir.summary", - detail: format!( - "local_aspa_count={} does not match local_outputs count {}", - self.summary.local_aspa_count, aspa_count - ), - }); - } - if self.summary.local_router_key_count != router_key_count { - return Err(StorageError::InvalidData { - entity: "vcir.summary", - detail: format!( - "local_router_key_count={} does not match local_outputs count {}", - self.summary.local_router_key_count, router_key_count - ), - }); - } - if self.summary.child_count != self.child_entries.len() as u32 { - return Err(StorageError::InvalidData { - entity: "vcir.summary", - detail: format!( - "child_count={} does not match child_entries length {}", - self.summary.child_count, - self.child_entries.len() - ), - }); - } - - for artifact in &self.related_artifacts { - artifact.validate_internal()?; - } - self.audit_summary.validate_internal()?; - Ok(()) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum RrdpSourceSyncState { - Empty, - SnapshotOnly, - DeltaReady, - Error, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct RrdpSourceRecord { - pub notify_uri: String, - pub last_session_id: Option, - pub last_serial: Option, - pub first_seen_at: PackTime, - pub last_seen_at: PackTime, - pub last_sync_at: Option, - pub sync_state: RrdpSourceSyncState, - pub last_snapshot_uri: Option, - pub last_snapshot_hash: Option, - pub last_error: Option, -} - -impl RrdpSourceRecord { - pub fn validate_internal(&self) -> StorageResult<()> { - validate_non_empty("rrdp_source.notify_uri", &self.notify_uri)?; - if let Some(session_id) = &self.last_session_id { - validate_non_empty("rrdp_source.last_session_id", session_id)?; - } - parse_time("rrdp_source.first_seen_at", &self.first_seen_at)?; - parse_time("rrdp_source.last_seen_at", &self.last_seen_at)?; - if let Some(last_sync_at) = &self.last_sync_at { - parse_time("rrdp_source.last_sync_at", last_sync_at)?; - } - if let Some(last_snapshot_uri) = &self.last_snapshot_uri { - validate_non_empty("rrdp_source.last_snapshot_uri", last_snapshot_uri)?; - } - if let Some(last_snapshot_hash) = &self.last_snapshot_hash { - validate_sha256_hex("rrdp_source.last_snapshot_hash", last_snapshot_hash)?; - } - if let Some(last_error) = &self.last_error { - validate_non_empty("rrdp_source.last_error", last_error)?; - } - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct RrdpSourceMemberRecord { - pub notify_uri: String, - pub rsync_uri: String, - pub current_hash: Option, - pub object_type: Option, - pub present: bool, - pub last_confirmed_session_id: String, - pub last_confirmed_serial: u64, - pub last_changed_at: PackTime, -} - -impl RrdpSourceMemberRecord { - pub fn validate_internal(&self) -> StorageResult<()> { - validate_non_empty("rrdp_source_member.notify_uri", &self.notify_uri)?; - validate_non_empty("rrdp_source_member.rsync_uri", &self.rsync_uri)?; - validate_non_empty( - "rrdp_source_member.last_confirmed_session_id", - &self.last_confirmed_session_id, - )?; - if self.present { - let hash = self - .current_hash - .as_deref() - .ok_or(StorageError::InvalidData { - entity: "rrdp_source_member", - detail: "current_hash is required when present=true".to_string(), - })?; - validate_sha256_hex("rrdp_source_member.current_hash", hash)?; - } else if let Some(hash) = &self.current_hash { - validate_sha256_hex("rrdp_source_member.current_hash", hash)?; - } - parse_time("rrdp_source_member.last_changed_at", &self.last_changed_at)?; - Ok(()) - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum RrdpUriOwnerState { - Active, - Conflict, - Withdrawn, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct RrdpUriOwnerRecord { - pub rsync_uri: String, - pub notify_uri: String, - pub current_hash: Option, - pub last_confirmed_session_id: String, - pub last_confirmed_serial: u64, - pub last_changed_at: PackTime, - pub owner_state: RrdpUriOwnerState, -} - -impl RrdpUriOwnerRecord { - pub fn validate_internal(&self) -> StorageResult<()> { - validate_non_empty("rrdp_uri_owner.rsync_uri", &self.rsync_uri)?; - validate_non_empty("rrdp_uri_owner.notify_uri", &self.notify_uri)?; - validate_non_empty( - "rrdp_uri_owner.last_confirmed_session_id", - &self.last_confirmed_session_id, - )?; - if let Some(hash) = &self.current_hash { - validate_sha256_hex("rrdp_uri_owner.current_hash", hash)?; - } - parse_time("rrdp_uri_owner.last_changed_at", &self.last_changed_at)?; - Ok(()) - } -} - -fn write_roa_cache_projection_to_batch( - projection_cf: &ColumnFamily, - batch: &mut WriteBatch, - vcir: &ValidatedCaInstanceResult, - context: Option<&RoaCacheProjectionContext>, - timing: Option<&mut VcirReplaceTimingBreakdown>, -) -> StorageResult<()> { - let projection_key = roa_cache_projection_key(&vcir.manifest_rsync_uri); - let projection = RoaCacheProjection::from_vcir_with_context(vcir, context)?; - match projection { - Some(projection) => { - let projection_value = encode_cbor(&projection, "roa_cache_projection")?; - if let Some(timing) = timing { - timing.roa_cache_projection_value_bytes = projection_value.len() as u64; - } - batch.put_cf(projection_cf, projection_key.as_bytes(), projection_value); - } - None => { - batch.delete_cf(projection_cf, projection_key.as_bytes()); - } - } - Ok(()) -} - -fn write_vcir_failed_fetch_reuse_identity_to_batch( - failed_fetch_reuse_identity_cf: &ColumnFamily, - batch: &mut WriteBatch, - manifest_rsync_uri: &str, - failed_fetch_reuse_identity: Option<&VcirFailedFetchReuseIdentity>, -) -> StorageResult<()> { - let key = vcir_failed_fetch_reuse_identity_key(manifest_rsync_uri); - match failed_fetch_reuse_identity { - Some(identity) => { - identity.validate_internal()?; - let value = encode_cbor(identity, "vcir_failed_fetch_reuse_identity")?; - batch.put_cf(failed_fetch_reuse_identity_cf, key.as_bytes(), value); - } - None => batch.delete_cf(failed_fetch_reuse_identity_cf, key.as_bytes()), - } - Ok(()) -} - -fn write_publication_point_cache_projection_to_batch( - projection_cf: &ColumnFamily, - batch: &mut WriteBatch, - action: PublicationPointCacheProjectionWriteAction<'_>, - timing: Option<&mut VcirReplaceTimingBreakdown>, -) -> StorageResult<()> { - match action { - PublicationPointCacheProjectionWriteAction::Keep => Ok(()), - PublicationPointCacheProjectionWriteAction::Write(projection) => { - projection.validate_internal()?; - let key = publication_point_cache_projection_key(&projection.manifest_rsync_uri); - let value = encode_cbor(projection, "publication_point_cache_projection")?; - if let Some(timing) = timing { - timing.publication_point_cache_projection_value_bytes = value.len() as u64; - } - batch.put_cf(projection_cf, key.as_bytes(), value); - Ok(()) - } - PublicationPointCacheProjectionWriteAction::Delete { manifest_rsync_uri } => { - let key = publication_point_cache_projection_key(manifest_rsync_uri); - batch.delete_cf(projection_cf, key.as_bytes()); - Ok(()) - } - } -} - -impl RocksStore { - pub fn create_read_only_checkpoint(source: &Path, destination: &Path) -> StorageResult<()> { - if destination.exists() { - return Err(StorageError::InvalidData { - entity: "work_db_checkpoint", - detail: format!( - "checkpoint destination already exists: {}", - destination.display() - ), - }); - } - if let Some(parent) = destination.parent() { - std::fs::create_dir_all(parent) - .map_err(|error| StorageError::RocksDb(error.to_string()))?; - } - let mut options = Options::default(); - let blob_mode = work_db_blob_mode_from_env(); - let memory_profile = work_db_memory_profile_from_env(); - configure_work_db_options(&mut options, blob_mode, memory_profile); - let db = DB::open_cf_descriptors_read_only( - &options, - source, - column_family_descriptors_for_blob_mode(blob_mode), - false, - ) - .map_err(|error| StorageError::RocksDb(error.to_string()))?; - let checkpoint = - Checkpoint::new(&db).map_err(|error| StorageError::RocksDb(error.to_string()))?; - checkpoint - .create_checkpoint(destination) - .map_err(|error| StorageError::RocksDb(error.to_string())) - } - - pub fn open(path: &Path) -> StorageResult { - let mut base_opts = Options::default(); - base_opts.create_if_missing(true); - base_opts.create_missing_column_families(true); - let blob_mode = work_db_blob_mode_from_env(); - let memory_profile = work_db_memory_profile_from_env(); - configure_work_db_options(&mut base_opts, blob_mode, memory_profile); - - let db = DB::open_cf_descriptors( - &base_opts, - path, - column_family_descriptors_for_blob_mode(blob_mode), - ) - .map_err(|e| StorageError::RocksDb(e.to_string()))?; - - Ok(Self { - db, - external_raw_store: None, - external_repo_bytes: None, - publication_point_cache_index_dir: default_pp_cache_index_dir(path), - child_certificate_cache_index_dir: default_child_certificate_cache_index_dir(path), - publication_point_cache_projection_index: Mutex::new( - PublicationPointCacheProjectionIndexState::Uninitialized, - ), - }) - } - - pub fn open_with_external_raw_store(path: &Path, raw_store_path: &Path) -> StorageResult { - Self::open_with_external_stores(path, Some(raw_store_path), None) - } - - pub fn open_with_external_repo_bytes( - path: &Path, - repo_bytes_path: &Path, - ) -> StorageResult { - Self::open_with_external_stores(path, None, Some(repo_bytes_path)) - } - - pub fn open_with_external_repo_bytes_read_only( - path: &Path, - repo_bytes_path: &Path, - ) -> StorageResult { - let mut store = Self::open(path)?; - store.external_repo_bytes = Some(ExternalRepoBytesDb::open_read_only(repo_bytes_path)?); - Ok(store) - } - - pub fn open_with_external_stores( - path: &Path, - raw_store_path: Option<&Path>, - repo_bytes_path: Option<&Path>, - ) -> StorageResult { - let mut store = Self::open(path)?; - if let Some(raw_store_path) = raw_store_path { - store.external_raw_store = Some(ExternalRawStoreDb::open(raw_store_path)?); - } - if let Some(repo_bytes_path) = repo_bytes_path { - store.external_repo_bytes = Some(ExternalRepoBytesDb::open(repo_bytes_path)?); - } - Ok(store) - } - - pub(crate) fn external_raw_store_ref(&self) -> Option<&ExternalRawStoreDb> { - self.external_raw_store.as_ref() - } - - pub(crate) fn external_repo_bytes_ref(&self) -> Option<&ExternalRepoBytesDb> { - self.external_repo_bytes.as_ref() - } - - pub fn memory_snapshot(&self) -> RocksDbMemorySnapshot { - let mut databases = Vec::new(); - databases.push(memory_db_snapshot_for_column_families( - "work-db", - &self.db, - Some(ALL_COLUMN_FAMILY_NAMES), - )); - if let Some(raw_store) = self.external_raw_store.as_ref() { - databases.push(raw_store.memory_snapshot("raw-store.db")); - } - if let Some(repo_bytes) = self.external_repo_bytes.as_ref() { - databases.push(repo_bytes.memory_snapshot("repo-bytes.db")); - } - - let mut totals = RocksDbMemoryTotals::default(); - for db in &databases { - totals.add_properties(&db.properties); - } - RocksDbMemorySnapshot { databases, totals } - } - - pub fn publication_point_cache_mmap_index_load_stats(&self) -> Option { - let guard = self.publication_point_cache_projection_index.lock().ok()?; - match &*guard { - PublicationPointCacheProjectionIndexState::LoadedMmap { load_stats, .. } => { - Some(load_stats.clone()) - } - PublicationPointCacheProjectionIndexState::Uninitialized - | PublicationPointCacheProjectionIndexState::Disabled - | PublicationPointCacheProjectionIndexState::BuildingFromEmpty { .. } - | PublicationPointCacheProjectionIndexState::Loaded { .. } => None, - } - } - - fn try_load_publication_point_cache_mmap_index_for_update( - &self, - reason: &'static str, - ) -> StorageResult<()> { - if !pp_cache_raw_index_enabled() { - return Ok(()); - } - let mut guard = self - .publication_point_cache_projection_index - .lock() - .map_err(|e| { - StorageError::RocksDb(format!("publication point cache index lock poisoned: {e}")) - })?; - if !matches!( - *guard, - PublicationPointCacheProjectionIndexState::Uninitialized - ) { - return Ok(()); - } - match load_pp_cache_mmap_index_set(&self.publication_point_cache_index_dir) { - Ok((mmap, stats)) => { - crate::progress_log::emit( - "publication_point_cache_mmap_index_load", - serde_json::json!({ - "state": "loaded", - "reason": reason, - "entries": stats.entries, - "bytes": stats.bytes, - "file_bytes": stats.file_bytes, - "load_ms": stats.load_ms, - }), - ); - *guard = PublicationPointCacheProjectionIndexState::LoadedMmap { - mmap, - dirty: HashMap::new(), - dirty_bytes: 0, - load_stats: stats, - }; - } - Err(e) => { - crate::progress_log::emit( - "publication_point_cache_mmap_index_load", - serde_json::json!({ - "state": "deferred_fallback_scan", - "reason": reason, - "error": e.to_string(), - }), - ); - } - } - Ok(()) - } - - fn cf(&self, name: &'static str) -> StorageResult<&ColumnFamily> { - self.db - .cf_handle(name) - .ok_or(StorageError::MissingColumnFamily(name)) - } - - pub fn put_repository_view_entry(&self, entry: &RepositoryViewEntry) -> StorageResult<()> { - entry.validate_internal()?; - let cf = self.cf(CF_REPOSITORY_VIEW)?; - let key = repository_view_key(&entry.rsync_uri); - let value = encode_cbor(entry, "repository_view")?; - self.db - .put_cf(cf, key.as_bytes(), value) - .map_err(|e| StorageError::RocksDb(e.to_string()))?; - Ok(()) - } - - pub fn get_repository_view_entry( - &self, - rsync_uri: &str, - ) -> StorageResult> { - let cf = self.cf(CF_REPOSITORY_VIEW)?; - let key = repository_view_key(rsync_uri); - let Some(bytes) = self - .db - .get_cf(cf, key.as_bytes()) - .map_err(|e| StorageError::RocksDb(e.to_string()))? - else { - return Ok(None); - }; - let entry = decode_cbor::(&bytes, "repository_view")?; - entry.validate_internal()?; - Ok(Some(entry)) - } - - pub fn delete_repository_view_entry(&self, rsync_uri: &str) -> StorageResult<()> { - let cf = self.cf(CF_REPOSITORY_VIEW)?; - let key = repository_view_key(rsync_uri); - self.db - .delete_cf(cf, key.as_bytes()) - .map_err(|e| StorageError::RocksDb(e.to_string()))?; - Ok(()) - } - - pub fn put_projection_batch( - &self, - repository_view_entries: &[RepositoryViewEntry], - member_records: &[RrdpSourceMemberRecord], - owner_records: &[RrdpUriOwnerRecord], - ) -> StorageResult<()> { - if repository_view_entries.is_empty() - && member_records.is_empty() - && owner_records.is_empty() - { - return Ok(()); - } - - let repo_cf = self.cf(CF_REPOSITORY_VIEW)?; - let member_cf = self.cf(CF_RRDP_SOURCE_MEMBER)?; - let owner_cf = self.cf(CF_RRDP_URI_OWNER)?; - let mut batch = WriteBatch::default(); - - for entry in repository_view_entries { - entry.validate_internal()?; - let key = repository_view_key(&entry.rsync_uri); - let value = encode_cbor(entry, "repository_view")?; - batch.put_cf(repo_cf, key.as_bytes(), value); - } - for record in member_records { - record.validate_internal()?; - let key = rrdp_source_member_key(&record.notify_uri, &record.rsync_uri); - let value = encode_cbor(record, "rrdp_source_member")?; - batch.put_cf(member_cf, key.as_bytes(), value); - } - for record in owner_records { - record.validate_internal()?; - let key = rrdp_uri_owner_key(&record.rsync_uri); - let value = encode_cbor(record, "rrdp_uri_owner")?; - batch.put_cf(owner_cf, key.as_bytes(), value); - } - - self.write_batch(batch) - } - - pub fn list_repository_view_entries_with_prefix( - &self, - rsync_uri_prefix: &str, - ) -> StorageResult> { - let cf = self.cf(CF_REPOSITORY_VIEW)?; - let prefix = repository_view_prefix(rsync_uri_prefix); - let mode = IteratorMode::From(prefix.as_bytes(), Direction::Forward); - self.db - .iterator_cf(cf, mode) - .take_while(|res| match res { - Ok((key, _)) => key.starts_with(prefix.as_bytes()), - Err(_) => false, - }) - .map(|res| { - let (_key, value) = res.map_err(|e| StorageError::RocksDb(e.to_string()))?; - let entry = decode_cbor::(&value, "repository_view")?; - entry.validate_internal()?; - Ok(entry) - }) - .collect() - } - - pub fn verify_current_repository_blobs( - &self, - batch_size: usize, - ) -> StorageResult { - if batch_size == 0 { - return Err(StorageError::InvalidData { - entity: "repository_blob_verification", - detail: "batch_size must be greater than zero".to_string(), - }); - } - let cf = self.cf(CF_REPOSITORY_VIEW)?; - let prefix = repository_view_prefix(""); - let mode = IteratorMode::From(prefix.as_bytes(), Direction::Forward); - let mut summary = RepositoryBlobVerificationSummary::default(); - let mut batch = Vec::with_capacity(batch_size); - for item in self.db.iterator_cf(cf, mode) { - let (key, value) = item.map_err(|error| StorageError::RocksDb(error.to_string()))?; - if !key.starts_with(prefix.as_bytes()) { - break; - } - let entry = decode_cbor::(&value, "repository_view")?; - entry.validate_internal()?; - if !matches!( - entry.state, - RepositoryViewState::Present | RepositoryViewState::Replaced - ) { - continue; - } - let hash = entry - .current_hash - .clone() - .ok_or(StorageError::InvalidData { - entity: "repository_blob_verification", - detail: format!("current hash missing for {}", entry.rsync_uri), - })?; - batch.push((entry.rsync_uri, hash)); - if batch.len() >= batch_size { - verify_repository_blob_batch(self, &batch, &mut summary)?; - batch.clear(); - } - } - if !batch.is_empty() { - verify_repository_blob_batch(self, &batch, &mut summary)?; - } - Ok(summary) - } - - pub fn put_raw_by_hash_entry(&self, entry: &RawByHashEntry) -> StorageResult<()> { - entry.validate_internal()?; - if let Some(raw_store) = self.external_raw_store.as_ref() { - return raw_store.put_raw_entry(entry); - } - let cf = self.cf(CF_RAW_BY_HASH)?; - let key = raw_by_hash_key(&entry.sha256_hex); - let value = encode_cbor(entry, "raw_by_hash")?; - self.db - .put_cf(cf, key.as_bytes(), value) - .map_err(|e| StorageError::RocksDb(e.to_string()))?; - Ok(()) - } - - pub fn put_raw_by_hash_entries_batch(&self, entries: &[RawByHashEntry]) -> StorageResult<()> { - if entries.is_empty() { - return Ok(()); - } - if let Some(raw_store) = self.external_raw_store.as_ref() { - return raw_store.put_raw_entries_batch(entries); - } - - let cf = self.cf(CF_RAW_BY_HASH)?; - let mut batch = WriteBatch::default(); - for entry in entries { - entry.validate_internal()?; - let key = raw_by_hash_key(&entry.sha256_hex); - let value = encode_cbor(entry, "raw_by_hash")?; - batch.put_cf(cf, key.as_bytes(), value); - } - self.write_batch(batch) - } - - pub fn put_raw_by_hash_entries_batch_unchecked( - &self, - entries: &[RawByHashEntry], - ) -> StorageResult<()> { - if entries.is_empty() { - return Ok(()); - } - if let Some(raw_store) = self.external_raw_store.as_ref() { - return raw_store.put_raw_entries_batch(entries); - } - - let cf = self.cf(CF_RAW_BY_HASH)?; - let mut batch = WriteBatch::default(); - for entry in entries { - let key = raw_by_hash_key(&entry.sha256_hex); - let value = encode_cbor(entry, "raw_by_hash")?; - batch.put_cf(cf, key.as_bytes(), value); - } - self.write_batch(batch) - } - - pub fn put_blob_bytes_batch(&self, blobs: &[(String, Vec)]) -> StorageResult<()> { - if blobs.is_empty() { - return Ok(()); - } - if let Some(repo_bytes) = self.external_repo_bytes.as_ref() { - if repo_bytes.is_read_only() { - return repo_bytes.require_existing_blob_bytes_batch(blobs); - } - return repo_bytes.put_blob_bytes_batch(blobs); - } - if let Some(raw_store) = self.external_raw_store.as_ref() { - return raw_store.put_blob_bytes_batch(blobs); - } - let cf = self.cf(CF_RAW_BLOB)?; - let mut batch = WriteBatch::default(); - for (sha256_hex, bytes) in blobs { - validate_sha256_hex("raw_blob.sha256_hex", sha256_hex)?; - if bytes.is_empty() { - return Err(StorageError::InvalidData { - entity: "raw_blob", - detail: "bytes must not be empty".to_string(), - }); - } - let key = raw_blob_key(sha256_hex); - batch.put_cf(cf, key.as_bytes(), bytes.as_slice()); - } - self.write_batch(batch) - } - - pub fn delete_raw_by_hash_entry(&self, sha256_hex: &str) -> StorageResult<()> { - validate_sha256_hex("raw_by_hash.sha256_hex", sha256_hex)?; - if let Some(raw_store) = self.external_raw_store.as_ref() { - return raw_store.delete_raw_entry(sha256_hex); - } - let cf = self.cf(CF_RAW_BY_HASH)?; - let key = raw_by_hash_key(sha256_hex); - self.db - .delete_cf(cf, key.as_bytes()) - .map_err(|e| StorageError::RocksDb(e.to_string()))?; - Ok(()) - } - - pub fn get_raw_by_hash_entry(&self, sha256_hex: &str) -> StorageResult> { - if let Some(raw_store) = self.external_raw_store.as_ref() { - return raw_store.get_raw_entry(sha256_hex); - } - let cf = self.cf(CF_RAW_BY_HASH)?; - let key = raw_by_hash_key(sha256_hex); - let Some(bytes) = self - .db - .get_cf(cf, key.as_bytes()) - .map_err(|e| StorageError::RocksDb(e.to_string()))? - else { - return Ok(None); - }; - let entry = decode_cbor::(&bytes, "raw_by_hash")?; - entry.validate_internal()?; - Ok(Some(entry)) - } - - pub fn get_raw_by_hash_entries_batch( - &self, - sha256_hexes: &[String], - ) -> StorageResult>> { - if sha256_hexes.is_empty() { - return Ok(Vec::new()); - } - if let Some(raw_store) = self.external_raw_store.as_ref() { - return raw_store.get_raw_entries_batch(sha256_hexes); - } - - let cf = self.cf(CF_RAW_BY_HASH)?; - let keys: Vec = sha256_hexes - .iter() - .map(|hash| raw_by_hash_key(hash)) - .collect(); - self.db - .multi_get_cf(keys.iter().map(|key| (cf, key.as_bytes()))) - .into_iter() - .map(|res| { - let maybe = res.map_err(|e| StorageError::RocksDb(e.to_string()))?; - match maybe { - Some(bytes) => { - let entry = decode_cbor::(&bytes, "raw_by_hash")?; - entry.validate_internal()?; - Ok(Some(entry)) - } - None => Ok(None), - } - }) - .collect() - } - - pub fn get_blob_bytes(&self, sha256_hex: &str) -> StorageResult>> { - if let Some(repo_bytes) = self.external_repo_bytes.as_ref() { - return repo_bytes.get_blob_bytes(sha256_hex); - } - if let Some(raw_store) = self.external_raw_store.as_ref() { - return raw_store.get_blob_bytes(sha256_hex); - } - validate_sha256_hex("raw_blob.sha256_hex", sha256_hex)?; - let cf = self.cf(CF_RAW_BLOB)?; - let key = raw_blob_key(sha256_hex); - if let Some(bytes) = self - .db - .get_cf(cf, key.as_bytes()) - .map_err(|e| StorageError::RocksDb(e.to_string()))? - { - return Ok(Some(bytes)); - } - self.get_raw_by_hash_entry(sha256_hex) - .map(|entry| entry.map(|entry| entry.bytes)) - } - - pub fn get_blob_bytes_batch( - &self, - sha256_hexes: &[String], - ) -> StorageResult>>> { - if sha256_hexes.is_empty() { - return Ok(Vec::new()); - } - if let Some(repo_bytes) = self.external_repo_bytes.as_ref() { - return repo_bytes.get_blob_bytes_batch(sha256_hexes); - } - if let Some(raw_store) = self.external_raw_store.as_ref() { - return raw_store.get_blob_bytes_batch(sha256_hexes); - } - - let cf = self.cf(CF_RAW_BLOB)?; - let keys: Vec = sha256_hexes - .iter() - .map(|hash| { - validate_sha256_hex("raw_blob.sha256_hex", hash)?; - Ok::(raw_blob_key(hash)) - }) - .collect::>()?; - let blob_results: Vec>> = self - .db - .multi_get_cf(keys.iter().map(|key| (cf, key.as_bytes()))) - .into_iter() - .map(|res| res.map_err(|e| StorageError::RocksDb(e.to_string()))) - .collect::>()?; - - let mut out = Vec::with_capacity(sha256_hexes.len()); - for (sha256_hex, maybe_blob) in sha256_hexes.iter().zip(blob_results.into_iter()) { - if maybe_blob.is_some() { - out.push(maybe_blob); - } else { - out.push( - self.get_raw_by_hash_entry(sha256_hex)? - .map(|entry| entry.bytes), - ); - } - } - Ok(out) - } - - pub fn put_vcir(&self, vcir: &ValidatedCaInstanceResult) -> StorageResult<()> { - self.put_vcir_with_publication_point_cache_projection(vcir, None) - } - - pub fn put_vcir_with_failed_fetch_reuse_identity( - &self, - vcir: &ValidatedCaInstanceResult, - failed_fetch_reuse_identity: &VcirFailedFetchReuseIdentity, - ) -> StorageResult<()> { - self.put_vcir_with_projection_action( - vcir, - None, - PublicationPointCacheProjectionWriteAction::Keep, - Some(failed_fetch_reuse_identity), - ) - } - - pub fn put_vcir_with_publication_point_cache_projection( - &self, - vcir: &ValidatedCaInstanceResult, - publication_point_projection: Option<&PublicationPointCacheProjection>, - ) -> StorageResult<()> { - self.put_vcir_with_projections(vcir, None, publication_point_projection) - } - - pub fn put_vcir_with_projections( - &self, - vcir: &ValidatedCaInstanceResult, - roa_cache_context: Option<&RoaCacheProjectionContext>, - publication_point_projection: Option<&PublicationPointCacheProjection>, - ) -> StorageResult<()> { - let publication_point_projection_action = publication_point_projection - .map(PublicationPointCacheProjectionWriteAction::Write) - .unwrap_or(PublicationPointCacheProjectionWriteAction::Keep); - self.put_vcir_with_projection_action( - vcir, - roa_cache_context, - publication_point_projection_action, - None, - ) - } - - fn put_vcir_with_projection_action( - &self, - vcir: &ValidatedCaInstanceResult, - roa_cache_context: Option<&RoaCacheProjectionContext>, - publication_point_projection_action: PublicationPointCacheProjectionWriteAction<'_>, - failed_fetch_reuse_identity: Option<&VcirFailedFetchReuseIdentity>, - ) -> StorageResult<()> { - vcir.validate_internal()?; - let vcir_cf = self.cf(CF_VCIR)?; - let failed_fetch_reuse_identity_cf = self.cf(CF_VCIR_FAILED_FETCH_REUSE_IDENTITY)?; - let replay_cf = self.cf(CF_MANIFEST_REPLAY_META)?; - let projection_cf = self.cf(CF_ROA_CACHE_PROJECTION)?; - let pp_projection_cf = self.cf(CF_PUBLICATION_POINT_CACHE_PROJECTION)?; - let replay_meta = ManifestReplayMeta::from_vcir(vcir); - replay_meta.validate_internal()?; - let mut batch = WriteBatch::default(); - let key = vcir_key(&vcir.manifest_rsync_uri); - let value = encode_cbor(vcir, "vcir")?; - batch.put_cf(vcir_cf, key.as_bytes(), value); - write_vcir_failed_fetch_reuse_identity_to_batch( - failed_fetch_reuse_identity_cf, - &mut batch, - &vcir.manifest_rsync_uri, - failed_fetch_reuse_identity, - )?; - let replay_key = manifest_replay_meta_key(&replay_meta.manifest_rsync_uri); - let replay_value = encode_cbor(&replay_meta, "manifest_replay_meta")?; - batch.put_cf(replay_cf, replay_key.as_bytes(), replay_value); - write_roa_cache_projection_to_batch( - projection_cf, - &mut batch, - vcir, - roa_cache_context, - None, - )?; - write_publication_point_cache_projection_to_batch( - pp_projection_cf, - &mut batch, - publication_point_projection_action, - None, - )?; - self.write_batch(batch)?; - self.apply_publication_point_cache_projection_index_action( - publication_point_projection_action, - )?; - Ok(()) - } - - pub fn replace_vcir_and_manifest_replay_meta( - &self, - vcir: &ValidatedCaInstanceResult, - ) -> StorageResult { - self.replace_vcir_manifest_replay_meta_and_publication_point_cache_projection(vcir, None) - } - - pub fn replace_vcir_manifest_replay_meta_and_publication_point_cache_projection( - &self, - vcir: &ValidatedCaInstanceResult, - publication_point_projection: Option<&PublicationPointCacheProjection>, - ) -> StorageResult { - self.replace_vcir_manifest_replay_meta_and_projections( - vcir, - None, - publication_point_projection, - ) - } - - pub fn replace_vcir_manifest_replay_meta_and_projections( - &self, - vcir: &ValidatedCaInstanceResult, - roa_cache_context: Option<&RoaCacheProjectionContext>, - publication_point_projection: Option<&PublicationPointCacheProjection>, - ) -> StorageResult { - let publication_point_projection_action = publication_point_projection - .map(PublicationPointCacheProjectionWriteAction::Write) - .unwrap_or(PublicationPointCacheProjectionWriteAction::Keep); - self.replace_vcir_manifest_replay_meta_and_projection_action( - vcir, - roa_cache_context, - publication_point_projection_action, - ) - } - - pub(crate) fn replace_vcir_manifest_replay_meta_and_projection_action( - &self, - vcir: &ValidatedCaInstanceResult, - roa_cache_context: Option<&RoaCacheProjectionContext>, - publication_point_projection_action: PublicationPointCacheProjectionWriteAction<'_>, - ) -> StorageResult { - self.replace_vcir_manifest_replay_meta_and_projection_action_with_failed_fetch_reuse_identity( - vcir, - roa_cache_context, - publication_point_projection_action, - None, - ) - } - - pub(crate) fn replace_vcir_manifest_replay_meta_and_projection_action_with_failed_fetch_reuse_identity( - &self, - vcir: &ValidatedCaInstanceResult, - roa_cache_context: Option<&RoaCacheProjectionContext>, - publication_point_projection_action: PublicationPointCacheProjectionWriteAction<'_>, - failed_fetch_reuse_identity: Option<&VcirFailedFetchReuseIdentity>, - ) -> StorageResult { - let mut timing = VcirReplaceTimingBreakdown { - rss_before_kb: process_vm_rss_kb(), - ..VcirReplaceTimingBreakdown::default() - }; - - let validate_started = std::time::Instant::now(); - vcir.validate_internal()?; - timing.validate_ms = validate_started.elapsed().as_millis() as u64; - timing.rss_after_validate_kb = process_vm_rss_kb(); - timing.field_sizes = VcirFieldSizeBreakdown::from_vcir(vcir); - - let batch_build_started = std::time::Instant::now(); - let vcir_cf = self.cf(CF_VCIR)?; - let failed_fetch_reuse_identity_cf = self.cf(CF_VCIR_FAILED_FETCH_REUSE_IDENTITY)?; - let replay_cf = self.cf(CF_MANIFEST_REPLAY_META)?; - let projection_cf = self.cf(CF_ROA_CACHE_PROJECTION)?; - let pp_projection_cf = self.cf(CF_PUBLICATION_POINT_CACHE_PROJECTION)?; - let mut batch = WriteBatch::default(); - - let vcir_key = vcir_key(&vcir.manifest_rsync_uri); - let vcir_encode_started = std::time::Instant::now(); - let vcir_value = encode_cbor(vcir, "vcir")?; - timing.vcir_encode_ms = vcir_encode_started.elapsed().as_millis() as u64; - timing.vcir_value_bytes = vcir_value.len() as u64; - batch.put_cf(vcir_cf, vcir_key.as_bytes(), vcir_value); - timing.rss_after_vcir_encode_kb = process_vm_rss_kb(); - write_vcir_failed_fetch_reuse_identity_to_batch( - failed_fetch_reuse_identity_cf, - &mut batch, - &vcir.manifest_rsync_uri, - failed_fetch_reuse_identity, - )?; - - let replay_meta_encode_started = std::time::Instant::now(); - let replay_meta = ManifestReplayMeta::from_vcir(vcir); - replay_meta.validate_internal()?; - let replay_key = manifest_replay_meta_key(&replay_meta.manifest_rsync_uri); - let replay_value = encode_cbor(&replay_meta, "manifest_replay_meta")?; - timing.replay_meta_encode_ms = replay_meta_encode_started.elapsed().as_millis() as u64; - timing.replay_meta_value_bytes = replay_value.len() as u64; - batch.put_cf(replay_cf, replay_key.as_bytes(), replay_value); - timing.rss_after_replay_meta_encode_kb = process_vm_rss_kb(); - - let projection_encode_started = std::time::Instant::now(); - write_roa_cache_projection_to_batch( - projection_cf, - &mut batch, - vcir, - roa_cache_context, - Some(&mut timing), - )?; - timing.roa_cache_projection_encode_ms = - projection_encode_started.elapsed().as_millis() as u64; - timing.rss_after_roa_cache_projection_encode_kb = process_vm_rss_kb(); - - let pp_projection_encode_started = std::time::Instant::now(); - write_publication_point_cache_projection_to_batch( - pp_projection_cf, - &mut batch, - publication_point_projection_action, - Some(&mut timing), - )?; - timing.publication_point_cache_projection_encode_ms = - pp_projection_encode_started.elapsed().as_millis() as u64; - timing.rss_after_publication_point_cache_projection_encode_kb = process_vm_rss_kb(); - - timing.total_encoded_bytes = timing.vcir_value_bytes - + timing.replay_meta_value_bytes - + timing.roa_cache_projection_value_bytes - + timing.publication_point_cache_projection_value_bytes; - timing.batch_build_ms = batch_build_started.elapsed().as_millis() as u64; - - let write_batch_started = std::time::Instant::now(); - self.write_batch(batch)?; - self.apply_publication_point_cache_projection_index_action( - publication_point_projection_action, - )?; - timing.write_batch_ms = write_batch_started.elapsed().as_millis() as u64; - timing.rss_after_write_batch_kb = process_vm_rss_kb(); - Ok(timing) - } - - pub fn get_vcir( - &self, - manifest_rsync_uri: &str, - ) -> StorageResult> { - let cf = self.cf(CF_VCIR)?; - let key = vcir_key(manifest_rsync_uri); - let Some(bytes) = self - .db - .get_cf(cf, key.as_bytes()) - .map_err(|e| StorageError::RocksDb(e.to_string()))? - else { - return Ok(None); - }; - let vcir = decode_cbor::(&bytes, "vcir")?; - vcir.validate_internal()?; - Ok(Some(vcir)) - } - - pub fn get_vcir_failed_fetch_reuse_identity( - &self, - manifest_rsync_uri: &str, - ) -> StorageResult> { - let cf = self.cf(CF_VCIR_FAILED_FETCH_REUSE_IDENTITY)?; - let key = vcir_failed_fetch_reuse_identity_key(manifest_rsync_uri); - let Some(bytes) = self - .db - .get_cf(cf, key.as_bytes()) - .map_err(|e| StorageError::RocksDb(e.to_string()))? - else { - return Ok(None); - }; - let identity = decode_cbor::( - &bytes, - "vcir_failed_fetch_reuse_identity", - )?; - identity.validate_internal()?; - Ok(Some(identity)) - } - - pub fn get_manifest_replay_meta( - &self, - manifest_rsync_uri: &str, - ) -> StorageResult> { - let cf = self.cf(CF_MANIFEST_REPLAY_META)?; - let key = manifest_replay_meta_key(manifest_rsync_uri); - let Some(bytes) = self - .db - .get_cf(cf, key.as_bytes()) - .map_err(|e| StorageError::RocksDb(e.to_string()))? - else { - return Ok(None); - }; - let meta = decode_cbor::(&bytes, "manifest_replay_meta")?; - meta.validate_internal()?; - Ok(Some(meta)) - } - - pub fn get_roa_cache_projection( - &self, - manifest_rsync_uri: &str, - ) -> StorageResult> { - let cf = self.cf(CF_ROA_CACHE_PROJECTION)?; - let key = roa_cache_projection_key(manifest_rsync_uri); - let Some(bytes) = self - .db - .get_cf(cf, key.as_bytes()) - .map_err(|e| StorageError::RocksDb(e.to_string()))? - else { - return Ok(None); - }; - let projection = decode_cbor::(&bytes, "roa_cache_projection")?; - projection.validate_internal()?; - Ok(Some(projection)) - } - - pub fn put_child_certificate_cache_projection( - &self, - projection: &ChildCertificateCacheProjection, - ) -> StorageResult<()> { - projection.validate_internal()?; - let cf = self.cf(CF_CHILD_CERTIFICATE_CACHE_PROJECTION)?; - let key = child_certificate_cache_projection_key(&projection.cache_key_sha256_hex); - let value = encode_cbor(projection, "child_certificate_cache_projection")?; - self.db - .put_cf(cf, key.as_bytes(), value) - .map_err(|e| StorageError::RocksDb(e.to_string()))?; - Ok(()) - } - - pub fn get_child_certificate_cache_projection( - &self, - cache_key_sha256_hex: &str, - ) -> StorageResult> { - validate_sha256_hex( - "child_certificate_cache_projection.cache_key_sha256_hex", - cache_key_sha256_hex, - )?; - let cf = self.cf(CF_CHILD_CERTIFICATE_CACHE_PROJECTION)?; - let key = child_certificate_cache_projection_key(cache_key_sha256_hex); - let Some(bytes) = self - .db - .get_cf(cf, key.as_bytes()) - .map_err(|e| StorageError::RocksDb(e.to_string()))? - else { - return Ok(None); - }; - let projection = decode_cbor::( - &bytes, - "child_certificate_cache_projection", - )?; - projection.validate_internal()?; - Ok(Some(projection)) - } - - pub fn get_child_certificate_cache_projections_batch( - &self, - cache_key_sha256_hexes: &[String], - ) -> StorageResult>> { - if cache_key_sha256_hexes.is_empty() { - return Ok(Vec::new()); - } - - for cache_key_sha256_hex in cache_key_sha256_hexes { - validate_sha256_hex( - "child_certificate_cache_projection.cache_key_sha256_hex", - cache_key_sha256_hex, - )?; - } - - let cf = self.cf(CF_CHILD_CERTIFICATE_CACHE_PROJECTION)?; - let keys: Vec = cache_key_sha256_hexes - .iter() - .map(|key| child_certificate_cache_projection_key(key)) - .collect(); - self.db - .multi_get_cf(keys.iter().map(|key| (cf, key.as_bytes()))) - .into_iter() - .map(|res| { - let Some(bytes) = res.map_err(|e| StorageError::RocksDb(e.to_string()))? else { - return Ok(None); - }; - let projection = decode_cbor::( - &bytes, - "child_certificate_cache_projection", - )?; - projection.validate_internal()?; - Ok(Some(projection)) - }) - .collect() - } - - fn child_certificate_cache_segment_path(&self, manifest_rsync_uri: &str) -> PathBuf { - self.child_certificate_cache_index_dir - .join(child_certificate_cache_segment_file_name( - manifest_rsync_uri, - )) - } - - pub fn get_child_certificate_cache_projections_mmap_segment( - &self, - manifest_rsync_uri: &str, - cache_key_sha256_hexes: &[String], - ) -> StorageResult> { - if cache_key_sha256_hexes.is_empty() { - return Ok(Some(ChildCertificateCacheMmapLookup::default())); - } - validate_non_empty( - "child_certificate_cache_mmap_segment.manifest_rsync_uri", - manifest_rsync_uri, - )?; - for cache_key_sha256_hex in cache_key_sha256_hexes { - validate_sha256_hex( - "child_certificate_cache_projection.cache_key_sha256_hex", - cache_key_sha256_hex, - )?; - } - - let path = self.child_certificate_cache_segment_path(manifest_rsync_uri); - if !path.exists() { - return Ok(None); - } - - let (index, _) = load_pp_cache_mmap_index(&path)?; - let file_bytes = index.file_bytes(); - let mut hits = 0usize; - let mut misses = 0usize; - let mut projections = Vec::with_capacity(cache_key_sha256_hexes.len()); - for cache_key_sha256_hex in cache_key_sha256_hexes { - match index.lookup(cache_key_sha256_hex) { - Some(PpCacheIndexLookup::Hit(bytes)) => { - let projection = decode_cbor::( - bytes, - "child_certificate_cache_projection_mmap_segment", - )?; - projection.validate_internal()?; - hits = hits.saturating_add(1); - projections.push(Some(projection)); - } - Some(PpCacheIndexLookup::Deleted) | None => { - misses = misses.saturating_add(1); - projections.push(None); - } - } - } - - Ok(Some(ChildCertificateCacheMmapLookup { - projections, - hits, - misses, - file_bytes, - })) - } - - pub fn write_child_certificate_cache_mmap_segment( - &self, - manifest_rsync_uri: &str, - projections: &[ChildCertificateCacheProjection], - ) -> StorageResult { - validate_non_empty( - "child_certificate_cache_mmap_segment.manifest_rsync_uri", - manifest_rsync_uri, - )?; - let mut entries = Vec::with_capacity(projections.len()); - for projection in projections { - projection.validate_internal()?; - entries.push(( - projection.cache_key_sha256_hex.clone(), - encode_cbor( - projection, - "child_certificate_cache_projection_mmap_segment", - )?, - )); - } - let path = self.child_certificate_cache_segment_path(manifest_rsync_uri); - write_pp_cache_index_atomic(&path, entries) - } - - pub fn write_child_certificate_cache_mmap_segment_overlay( - &self, - manifest_rsync_uri: &str, - cache_key_sha256_hexes: &[String], - projections: &[ChildCertificateCacheProjection], - ) -> StorageResult { - validate_non_empty( - "child_certificate_cache_mmap_segment.manifest_rsync_uri", - manifest_rsync_uri, - )?; - let mut dirty = HashMap::>::with_capacity(projections.len()); - for projection in projections { - projection.validate_internal()?; - dirty.insert( - projection.cache_key_sha256_hex.clone(), - encode_cbor( - projection, - "child_certificate_cache_projection_mmap_segment", - )?, - ); - } - - let path = self.child_certificate_cache_segment_path(manifest_rsync_uri); - let existing = if path.exists() { - Some(load_pp_cache_mmap_index(&path)?.0) - } else { - None - }; - let mut entries = Vec::with_capacity(cache_key_sha256_hexes.len()); - let mut emitted = HashSet::::new(); - for cache_key_sha256_hex in cache_key_sha256_hexes { - validate_sha256_hex( - "child_certificate_cache_projection.cache_key_sha256_hex", - cache_key_sha256_hex, - )?; - if !emitted.insert(cache_key_sha256_hex.clone()) { - continue; - } - if let Some(value) = dirty.remove(cache_key_sha256_hex) { - entries.push((cache_key_sha256_hex.clone(), value)); - continue; - } - if let Some(existing) = existing.as_ref() { - if let Some(PpCacheIndexLookup::Hit(bytes)) = existing.lookup(cache_key_sha256_hex) - { - entries.push((cache_key_sha256_hex.clone(), bytes.to_vec())); - } - } - } - for (key, value) in dirty { - if emitted.insert(key.clone()) { - entries.push((key, value)); - } - } - - write_pp_cache_index_atomic(&path, entries) - } - - pub fn get_publication_point_cache_projection( - &self, - manifest_rsync_uri: &str, - ) -> StorageResult> { - self.get_publication_point_cache_projection_from_db(manifest_rsync_uri) - } - - pub fn get_publication_point_cache_projection_cached( - &self, - manifest_rsync_uri: &str, - ) -> StorageResult> { - let mut owned_bytes: Option> = None; - { - let mut guard = self - .publication_point_cache_projection_index - .lock() - .map_err(|e| { - StorageError::RocksDb(format!( - "publication point cache index lock poisoned: {e}" - )) - })?; - if matches!( - *guard, - PublicationPointCacheProjectionIndexState::Uninitialized - ) { - let load_started = std::time::Instant::now(); - let raw_index_enabled = pp_cache_raw_index_enabled(); - *guard = if !raw_index_enabled { - crate::progress_log::emit( - "publication_point_cache_raw_index", - serde_json::json!({ - "state": "disabled_by_env", - "entries": 0, - "bytes": 0, - "load_ms": load_started.elapsed().as_millis() as u64, - }), - ); - PublicationPointCacheProjectionIndexState::Disabled - } else { - match load_pp_cache_mmap_index_set(&self.publication_point_cache_index_dir) { - Ok((mmap, stats)) => { - crate::progress_log::emit( - "publication_point_cache_mmap_index_load", - serde_json::json!({ - "state": "loaded", - "entries": stats.entries, - "bytes": stats.bytes, - "file_bytes": stats.file_bytes, - "load_ms": stats.load_ms, - }), - ); - PublicationPointCacheProjectionIndexState::LoadedMmap { - mmap, - dirty: HashMap::new(), - dirty_bytes: 0, - load_stats: stats, - } - } - Err(e) => { - crate::progress_log::emit( - "publication_point_cache_mmap_index_load", - serde_json::json!({ - "state": "fallback_scan", - "error": e.to_string(), - "load_ms": load_started.elapsed().as_millis() as u64, - }), - ); - let scan_started = std::time::Instant::now(); - let (index, bytes) = - self.load_publication_point_cache_projection_index()?; - if index.is_empty() { - let limit = pp_cache_raw_index_empty_build_limit_bytes(); - crate::progress_log::emit( - "publication_point_cache_raw_index", - serde_json::json!({ - "state": "empty_building_bounded", - "entries": 0, - "bytes": 0, - "empty_build_limit_bytes": limit, - "load_ms": scan_started.elapsed().as_millis() as u64, - }), - ); - PublicationPointCacheProjectionIndexState::BuildingFromEmpty { - index, - bytes: 0, - limit, - } - } else { - crate::progress_log::emit( - "publication_point_cache_raw_index", - serde_json::json!({ - "state": "loaded", - "entries": index.len(), - "bytes": bytes, - "load_ms": scan_started.elapsed().as_millis() as u64, - }), - ); - PublicationPointCacheProjectionIndexState::Loaded { index, bytes } - } - } - } - }; - } - match &*guard { - PublicationPointCacheProjectionIndexState::Loaded { index, .. } => { - owned_bytes = index.get(manifest_rsync_uri).cloned(); - } - PublicationPointCacheProjectionIndexState::BuildingFromEmpty { index, .. } => { - owned_bytes = index.get(manifest_rsync_uri).cloned(); - } - PublicationPointCacheProjectionIndexState::LoadedMmap { mmap, dirty, .. } => { - if let Some(bytes) = dirty.get(manifest_rsync_uri).cloned() { - if bytes.is_empty() { - return Ok(None); - } - owned_bytes = Some(bytes); - } else if let Some(lookup) = mmap.lookup(manifest_rsync_uri) { - match lookup { - PpCacheIndexLookup::Hit(bytes) => { - let projection = decode_cbor::( - bytes, - "publication_point_cache_projection", - )?; - projection.validate_internal()?; - return Ok(Some(projection)); - } - PpCacheIndexLookup::Deleted => return Ok(None), - } - } - } - PublicationPointCacheProjectionIndexState::Disabled - | PublicationPointCacheProjectionIndexState::Uninitialized => {} - } - } - let bytes = owned_bytes; - let Some(bytes) = bytes else { - return Ok(None); - }; - let projection = decode_cbor::( - bytes.as_ref(), - "publication_point_cache_projection", - )?; - projection.validate_internal()?; - Ok(Some(projection)) - } - - fn get_publication_point_cache_projection_from_db( - &self, - manifest_rsync_uri: &str, - ) -> StorageResult> { - let cf = self.cf(CF_PUBLICATION_POINT_CACHE_PROJECTION)?; - let key = publication_point_cache_projection_key(manifest_rsync_uri); - let Some(bytes) = self - .db - .get_cf(cf, key.as_bytes()) - .map_err(|e| StorageError::RocksDb(e.to_string()))? - else { - return Ok(None); - }; - let projection = decode_cbor::( - &bytes, - "publication_point_cache_projection", - )?; - projection.validate_internal()?; - Ok(Some(projection)) - } - - fn load_publication_point_cache_projection_index( - &self, - ) -> StorageResult<(HashMap>, usize)> { - let cf = self.cf(CF_PUBLICATION_POINT_CACHE_PROJECTION)?; - let mode = IteratorMode::Start; - let mut index = HashMap::new(); - let mut bytes_total = 0usize; - for res in self.db.iterator_cf(cf, mode) { - let (key, value) = res.map_err(|e| StorageError::RocksDb(e.to_string()))?; - let Some(manifest_rsync_uri) = - publication_point_cache_projection_key_manifest_uri(&key) - else { - continue; - }; - bytes_total = bytes_total.saturating_add(value.len()); - index.insert(manifest_rsync_uri, Arc::<[u8]>::from(value.to_vec())); - } - Ok((index, bytes_total)) - } - - fn load_publication_point_cache_projection_entries( - &self, - ) -> StorageResult)>> { - let cf = self.cf(CF_PUBLICATION_POINT_CACHE_PROJECTION)?; - let mode = IteratorMode::Start; - let mut entries = Vec::new(); - for res in self.db.iterator_cf(cf, mode) { - let (key, value) = res.map_err(|e| StorageError::RocksDb(e.to_string()))?; - let Some(manifest_rsync_uri) = - publication_point_cache_projection_key_manifest_uri(&key) - else { - continue; - }; - entries.push((manifest_rsync_uri, value.to_vec())); - } - Ok(entries) - } - - fn apply_publication_point_cache_projection_index_action( - &self, - action: PublicationPointCacheProjectionWriteAction<'_>, - ) -> StorageResult<()> { - match action { - PublicationPointCacheProjectionWriteAction::Keep => Ok(()), - PublicationPointCacheProjectionWriteAction::Write(projection) => { - self.update_publication_point_cache_projection_index(projection) - } - PublicationPointCacheProjectionWriteAction::Delete { manifest_rsync_uri } => { - self.delete_publication_point_cache_projection_index_entry(manifest_rsync_uri) - } - } - } - - fn update_publication_point_cache_projection_index( - &self, - projection: &PublicationPointCacheProjection, - ) -> StorageResult<()> { - self.try_load_publication_point_cache_mmap_index_for_update("write")?; - let mut guard = self - .publication_point_cache_projection_index - .lock() - .map_err(|e| { - StorageError::RocksDb(format!("publication point cache index lock poisoned: {e}")) - })?; - let bytes = encode_cbor(projection, "publication_point_cache_projection")?; - match &mut *guard { - PublicationPointCacheProjectionIndexState::Loaded { - index, - bytes: total_bytes, - } => { - if let Some(previous) = index.insert( - projection.manifest_rsync_uri.clone(), - Arc::<[u8]>::from(bytes.clone()), - ) { - *total_bytes = total_bytes.saturating_sub(previous.len()); - } - *total_bytes = total_bytes.saturating_add(bytes.len()); - } - PublicationPointCacheProjectionIndexState::BuildingFromEmpty { - index, - bytes: total_bytes, - limit, - } => { - if let Some(previous) = index.remove(&projection.manifest_rsync_uri) { - *total_bytes = total_bytes.saturating_sub(previous.len()); - } - if total_bytes.saturating_add(bytes.len()) <= *limit { - *total_bytes += bytes.len(); - index.insert( - projection.manifest_rsync_uri.clone(), - Arc::<[u8]>::from(bytes), - ); - } else { - *guard = PublicationPointCacheProjectionIndexState::Disabled; - } - } - PublicationPointCacheProjectionIndexState::LoadedMmap { - dirty, dirty_bytes, .. - } => { - if let Some(previous) = dirty.insert( - projection.manifest_rsync_uri.clone(), - Arc::<[u8]>::from(bytes.clone()), - ) { - *dirty_bytes = dirty_bytes.saturating_sub(previous.len()); - } - *dirty_bytes = dirty_bytes.saturating_add(bytes.len()); - } - PublicationPointCacheProjectionIndexState::Uninitialized - | PublicationPointCacheProjectionIndexState::Disabled => {} - } - Ok(()) - } - - fn delete_publication_point_cache_projection_index_entry( - &self, - manifest_rsync_uri: &str, - ) -> StorageResult<()> { - self.try_load_publication_point_cache_mmap_index_for_update("delete")?; - let mut guard = self - .publication_point_cache_projection_index - .lock() - .map_err(|e| { - StorageError::RocksDb(format!("publication point cache index lock poisoned: {e}")) - })?; - match &mut *guard { - PublicationPointCacheProjectionIndexState::Loaded { - index, - bytes: total_bytes, - } => { - if let Some(previous) = index.remove(manifest_rsync_uri) { - *total_bytes = total_bytes.saturating_sub(previous.len()); - } - } - PublicationPointCacheProjectionIndexState::BuildingFromEmpty { - index, - bytes: total_bytes, - .. - } => { - if let Some(previous) = index.remove(manifest_rsync_uri) { - *total_bytes = total_bytes.saturating_sub(previous.len()); - } - } - PublicationPointCacheProjectionIndexState::LoadedMmap { - dirty, dirty_bytes, .. - } => { - if let Some(previous) = - dirty.insert(manifest_rsync_uri.to_string(), Arc::<[u8]>::from([])) - { - *dirty_bytes = dirty_bytes.saturating_sub(previous.len()); - } - } - PublicationPointCacheProjectionIndexState::Uninitialized - | PublicationPointCacheProjectionIndexState::Disabled => {} - } - Ok(()) - } - - pub fn refresh_publication_point_cache_mmap_index( - &self, - ) -> StorageResult> { - if !pp_cache_raw_index_enabled() { - return Ok(None); - } - self.try_load_publication_point_cache_mmap_index_for_update("refresh")?; - enum RefreshAction { - Entries { - entries: Vec<(String, Vec)>, - write_segment: bool, - old_entries: usize, - dirty_entries: usize, - }, - ScanDb, - } - let action = { - let guard = self - .publication_point_cache_projection_index - .lock() - .map_err(|e| { - StorageError::RocksDb(format!( - "publication point cache index lock poisoned: {e}" - )) - })?; - match &*guard { - PublicationPointCacheProjectionIndexState::LoadedMmap { mmap, dirty, .. } => { - RefreshAction::Entries { - entries: dirty - .iter() - .map(|(key, value)| (key.clone(), value.as_ref().to_vec())) - .collect::>(), - write_segment: true, - old_entries: mmap.entries(), - dirty_entries: dirty.len(), - } - } - PublicationPointCacheProjectionIndexState::Loaded { index, .. } - | PublicationPointCacheProjectionIndexState::BuildingFromEmpty { index, .. } => { - RefreshAction::Entries { - entries: index - .iter() - .map(|(key, value)| (key.clone(), value.as_ref().to_vec())) - .collect::>(), - write_segment: false, - old_entries: 0, - dirty_entries: 0, - } - } - PublicationPointCacheProjectionIndexState::Disabled - | PublicationPointCacheProjectionIndexState::Uninitialized => RefreshAction::ScanDb, - } - }; - let (entries, write_segment, old_entries, dirty_entries) = match action { - RefreshAction::Entries { - entries, - write_segment, - old_entries, - dirty_entries, - } => (entries, write_segment, old_entries, dirty_entries), - RefreshAction::ScanDb => ( - self.load_publication_point_cache_projection_entries()?, - false, - 0, - 0, - ), - }; - if entries.is_empty() { - return Ok(None); - } - let current_path = self.publication_point_cache_index_dir.join("current.idx"); - let mut stats = if write_segment && current_path.exists() { - write_pp_cache_index_segment(&self.publication_point_cache_index_dir, entries)? - } else { - write_pp_cache_index_atomic(¤t_path, entries)? - }; - stats.old_entries = old_entries; - stats.dirty_entries = dirty_entries; - crate::progress_log::emit( - "publication_point_cache_mmap_index_refresh", - serde_json::json!({ - "state": stats.state, - "old_entries": stats.old_entries, - "dirty_entries": stats.dirty_entries, - "new_entries": stats.new_entries, - "file_bytes": stats.file_bytes, - "write_ms": stats.write_ms, - }), - ); - let directory_stats = - pp_cache_index_directory_stats(&self.publication_point_cache_index_dir)?; - let compaction_reason = if directory_stats.segment_count - >= PP_CACHE_INDEX_COMPACTION_SEGMENT_THRESHOLD - { - Some(format!( - "segment_count>={PP_CACHE_INDEX_COMPACTION_SEGMENT_THRESHOLD}" - )) - } else if directory_stats.total_file_bytes >= PP_CACHE_INDEX_COMPACTION_BYTES_THRESHOLD { - Some(format!( - "total_file_bytes>={PP_CACHE_INDEX_COMPACTION_BYTES_THRESHOLD}" - )) - } else { - None - }; - if let Some(reason) = compaction_reason { - crate::progress_log::emit( - "publication_point_cache_mmap_index_compaction", - serde_json::json!({ - "state": "started", - "reason": reason, - "segment_count": directory_stats.segment_count, - "total_file_bytes": directory_stats.total_file_bytes, - }), - ); - match compact_pp_cache_index(&self.publication_point_cache_index_dir) { - Ok(mut compact_stats) => { - compact_stats.compaction_reason = Some(reason); - compact_stats.old_entries = stats.old_entries; - compact_stats.dirty_entries = stats.dirty_entries; - crate::progress_log::emit( - "publication_point_cache_mmap_index_compaction", - serde_json::json!({ - "state": "completed", - "reason": compact_stats.compaction_reason, - "segments_before": compact_stats.compaction_segments_before, - "total_file_bytes_before": compact_stats.compaction_total_file_bytes_before, - "live_entries": compact_stats.compaction_live_entries, - "file_bytes": compact_stats.compaction_file_bytes, - "reclaimed_bytes": compact_stats.compaction_reclaimed_bytes, - "deleted_segments": compact_stats.compaction_deleted_segments, - "compaction_ms": compact_stats.compaction_ms, - }), - ); - stats.compaction_triggered = compact_stats.compaction_triggered; - stats.compaction_reason = compact_stats.compaction_reason; - stats.compaction_segments_before = compact_stats.compaction_segments_before; - stats.compaction_total_file_bytes_before = - compact_stats.compaction_total_file_bytes_before; - stats.compaction_live_entries = compact_stats.compaction_live_entries; - stats.compaction_file_bytes = compact_stats.compaction_file_bytes; - stats.compaction_reclaimed_bytes = compact_stats.compaction_reclaimed_bytes; - stats.compaction_ms = compact_stats.compaction_ms; - stats.compaction_deleted_segments = compact_stats.compaction_deleted_segments; - } - Err(e) => { - let error = e.to_string(); - crate::progress_log::emit( - "publication_point_cache_mmap_index_compaction", - serde_json::json!({ - "state": "failed", - "reason": reason, - "segment_count": directory_stats.segment_count, - "total_file_bytes": directory_stats.total_file_bytes, - "error": error, - }), - ); - stats.compaction_triggered = true; - stats.compaction_reason = Some(reason); - stats.compaction_segments_before = directory_stats.segment_count; - stats.compaction_total_file_bytes_before = directory_stats.total_file_bytes; - stats.compaction_error = Some(error); - } - } - } - let mut guard = self - .publication_point_cache_projection_index - .lock() - .map_err(|e| { - StorageError::RocksDb(format!("publication point cache index lock poisoned: {e}")) - })?; - if let PublicationPointCacheProjectionIndexState::LoadedMmap { - dirty, dirty_bytes, .. - } = &mut *guard - { - dirty.clear(); - *dirty_bytes = 0; - } - Ok(Some(stats)) - } - - pub fn put_transport_prefetch_snapshot( - &self, - snapshot: &crate::parallel::transport_prefetch::TransportPrefetchSnapshot, - ) -> StorageResult<()> { - let cf = self.cf(CF_TRANSPORT_PREFETCH)?; - let value = encode_cbor(snapshot, "transport_prefetch_snapshot")?; - self.db - .put_cf(cf, TRANSPORT_PREFETCH_LAST_SNAPSHOT_KEY.as_bytes(), value) - .map_err(|e| StorageError::RocksDb(e.to_string())) - } - - pub fn get_transport_prefetch_snapshot( - &self, - ) -> StorageResult> { - let cf = self.cf(CF_TRANSPORT_PREFETCH)?; - let Some(bytes) = self - .db - .get_cf(cf, TRANSPORT_PREFETCH_LAST_SNAPSHOT_KEY.as_bytes()) - .map_err(|e| StorageError::RocksDb(e.to_string()))? - else { - return Ok(None); - }; - decode_cbor::( - &bytes, - "transport_prefetch_snapshot", - ) - .map(Some) - } - - pub fn list_vcirs(&self) -> StorageResult> { - let cf = self.cf(CF_VCIR)?; - let mode = IteratorMode::Start; - let mut out = Vec::new(); - for res in self.db.iterator_cf(cf, mode) { - let (_key, bytes) = res.map_err(|e| StorageError::RocksDb(e.to_string()))?; - let vcir = decode_cbor::(&bytes, "vcir")?; - vcir.validate_internal()?; - out.push(vcir); - } - Ok(out) - } - - /// Remove every reusable VCIR and its failed-fetch identity while retaining - /// manifest replay metadata. The replay metadata is a fresh-validation - /// guard, not a reusable validation result; removing it changes the normal - /// traversal semantics for manifests that are not selected for replay. - /// - /// This is intentionally narrower than [`Self::delete_vcir`]. It preserves - /// manifest replay metadata while clearing reusable validation records. - pub fn clear_vcir_reuse_records(&self) -> StorageResult { - let vcir_cf = self.cf(CF_VCIR)?; - let identity_cf = self.cf(CF_VCIR_FAILED_FETCH_REUSE_IDENTITY)?; - let mut batch = WriteBatch::default(); - let mut summary = VcirReuseRecordsCleared::default(); - - for entry in self.db.iterator_cf(vcir_cf, IteratorMode::Start) { - let (key, _value) = entry.map_err(|error| StorageError::RocksDb(error.to_string()))?; - batch.delete_cf(vcir_cf, key); - summary.vcir_records += 1; - } - for entry in self.db.iterator_cf(identity_cf, IteratorMode::Start) { - let (key, _value) = entry.map_err(|error| StorageError::RocksDb(error.to_string()))?; - batch.delete_cf(identity_cf, key); - summary.failed_fetch_identity_records += 1; - } - if summary.vcir_records > 0 || summary.failed_fetch_identity_records > 0 { - self.write_batch(batch)?; - } - Ok(summary) - } - - pub fn summarize_vcir_storage(&self) -> StorageResult { - let cf = self.cf(CF_VCIR)?; - let mode = IteratorMode::Start; - let mut summary = VcirStorageSummary::default(); - for res in self.db.iterator_cf(cf, mode) { - let (_key, bytes) = res.map_err(|e| StorageError::RocksDb(e.to_string()))?; - let vcir = decode_cbor::(&bytes, "vcir")?; - vcir.validate_internal()?; - summary.entry_count += 1; - let value_bytes = bytes.len() as u64; - summary.vcir_value_bytes += value_bytes; - if value_bytes > summary.vcir_value_bytes_max { - summary.vcir_value_bytes_max = value_bytes; - summary.vcir_value_bytes_max_manifest_rsync_uri = - Some(vcir.manifest_rsync_uri.clone()); - } - let entry_summary = VcirStorageEntrySummary::from_vcir(&vcir, value_bytes); - summary.core_fields.add_assign(&entry_summary.core_fields); - summary - .ccr_projection - .add_assign(&entry_summary.ccr_projection); - summary - .child_resources - .add_assign(&entry_summary.child_resources); - summary.field_sizes.add_assign(&entry_summary.field_sizes); - push_top_vcir_storage_entry( - &mut summary.top_entries_by_vcir_value_bytes, - entry_summary, - ); - } - summary.local_output_old_projection_bytes = - summary.field_sizes.local_output_old_projection_bytes(); - summary.local_output_typed_projection_bytes = - summary.field_sizes.local_output_typed_projection_bytes(); - summary.local_output_projection_saved_bytes = - summary.field_sizes.local_output_projection_saved_bytes(); - Ok(summary) - } - - pub fn delete_vcir(&self, manifest_rsync_uri: &str) -> StorageResult<()> { - let vcir_cf = self.cf(CF_VCIR)?; - let failed_fetch_reuse_identity_cf = self.cf(CF_VCIR_FAILED_FETCH_REUSE_IDENTITY)?; - let replay_cf = self.cf(CF_MANIFEST_REPLAY_META)?; - let projection_cf = self.cf(CF_ROA_CACHE_PROJECTION)?; - let mut batch = WriteBatch::default(); - let key = vcir_key(manifest_rsync_uri); - batch.delete_cf(vcir_cf, key.as_bytes()); - let failed_fetch_reuse_identity_key = - vcir_failed_fetch_reuse_identity_key(manifest_rsync_uri); - batch.delete_cf( - failed_fetch_reuse_identity_cf, - failed_fetch_reuse_identity_key.as_bytes(), - ); - let replay_key = manifest_replay_meta_key(manifest_rsync_uri); - batch.delete_cf(replay_cf, replay_key.as_bytes()); - let projection_key = roa_cache_projection_key(manifest_rsync_uri); - batch.delete_cf(projection_cf, projection_key.as_bytes()); - self.write_batch(batch) - } - - pub fn put_rrdp_source_record(&self, record: &RrdpSourceRecord) -> StorageResult<()> { - record.validate_internal()?; - let cf = self.cf(CF_RRDP_SOURCE)?; - let key = rrdp_source_key(&record.notify_uri); - let value = encode_cbor(record, "rrdp_source")?; - self.db - .put_cf(cf, key.as_bytes(), value) - .map_err(|e| StorageError::RocksDb(e.to_string()))?; - Ok(()) - } - - pub fn get_rrdp_source_record( - &self, - notify_uri: &str, - ) -> StorageResult> { - let cf = self.cf(CF_RRDP_SOURCE)?; - let key = rrdp_source_key(notify_uri); - let Some(bytes) = self - .db - .get_cf(cf, key.as_bytes()) - .map_err(|e| StorageError::RocksDb(e.to_string()))? - else { - return Ok(None); - }; - let record = decode_cbor::(&bytes, "rrdp_source")?; - record.validate_internal()?; - Ok(Some(record)) - } - - pub fn put_rrdp_source_member_record( - &self, - record: &RrdpSourceMemberRecord, - ) -> StorageResult<()> { - record.validate_internal()?; - let cf = self.cf(CF_RRDP_SOURCE_MEMBER)?; - let key = rrdp_source_member_key(&record.notify_uri, &record.rsync_uri); - let value = encode_cbor(record, "rrdp_source_member")?; - self.db - .put_cf(cf, key.as_bytes(), value) - .map_err(|e| StorageError::RocksDb(e.to_string()))?; - Ok(()) - } - - pub fn get_rrdp_source_member_record( - &self, - notify_uri: &str, - rsync_uri: &str, - ) -> StorageResult> { - let cf = self.cf(CF_RRDP_SOURCE_MEMBER)?; - let key = rrdp_source_member_key(notify_uri, rsync_uri); - let Some(bytes) = self - .db - .get_cf(cf, key.as_bytes()) - .map_err(|e| StorageError::RocksDb(e.to_string()))? - else { - return Ok(None); - }; - let record = decode_cbor::(&bytes, "rrdp_source_member")?; - record.validate_internal()?; - Ok(Some(record)) - } - - pub fn list_rrdp_source_member_records( - &self, - notify_uri: &str, - ) -> StorageResult> { - let cf = self.cf(CF_RRDP_SOURCE_MEMBER)?; - let prefix = rrdp_source_member_prefix(notify_uri); - let mode = IteratorMode::From(prefix.as_bytes(), Direction::Forward); - self.db - .iterator_cf(cf, mode) - .take_while(|res| match res { - Ok((key, _)) => key.starts_with(prefix.as_bytes()), - Err(_) => false, - }) - .map(|res| { - let (_key, value) = res.map_err(|e| StorageError::RocksDb(e.to_string()))?; - let record = decode_cbor::(&value, "rrdp_source_member")?; - record.validate_internal()?; - Ok(record) - }) - .collect() - } - - pub fn list_current_rrdp_source_members( - &self, - notify_uri: &str, - ) -> StorageResult> { - let mut records = self.list_rrdp_source_member_records(notify_uri)?; - records.retain(|record| record.present); - records.sort_by(|a, b| a.rsync_uri.cmp(&b.rsync_uri)); - Ok(records) - } - - pub fn is_current_rrdp_source_member( - &self, - notify_uri: &str, - rsync_uri: &str, - ) -> StorageResult { - Ok(matches!( - self.get_rrdp_source_member_record(notify_uri, rsync_uri)?, - Some(record) if record.present - )) - } - - pub fn load_current_object_bytes_by_uri( - &self, - rsync_uri: &str, - ) -> StorageResult>> { - Ok(self - .load_current_object_with_hash_by_uri(rsync_uri)? - .map(|obj| obj.bytes)) - } - - pub fn load_current_object_with_hash_by_uri( - &self, - rsync_uri: &str, - ) -> StorageResult> { - let Some(view) = self.get_repository_view_entry(rsync_uri)? else { - return Ok(None); - }; - - match view.state { - RepositoryViewState::Withdrawn => Ok(None), - RepositoryViewState::Present | RepositoryViewState::Replaced => { - let hash = view - .current_hash - .as_deref() - .ok_or(StorageError::InvalidData { - entity: "repository_view", - detail: format!("current_hash missing for current object URI: {rsync_uri}"), - })?; - let bytes = self - .get_blob_bytes(hash)? - .ok_or(StorageError::InvalidData { - entity: "repository_view", - detail: format!( - "blob bytes missing for current object URI: {rsync_uri} (hash={hash})" - ), - })?; - let current_hash = decode_sha256_hex_32("repository_view.current_hash", hash)?; - Ok(Some(CurrentObjectWithHash { - current_hash_hex: hash.to_ascii_lowercase(), - current_hash, - bytes, - })) - } - } - } - - pub fn put_rrdp_uri_owner_record(&self, record: &RrdpUriOwnerRecord) -> StorageResult<()> { - record.validate_internal()?; - let cf = self.cf(CF_RRDP_URI_OWNER)?; - let key = rrdp_uri_owner_key(&record.rsync_uri); - let value = encode_cbor(record, "rrdp_uri_owner")?; - self.db - .put_cf(cf, key.as_bytes(), value) - .map_err(|e| StorageError::RocksDb(e.to_string()))?; - Ok(()) - } - - pub fn get_rrdp_uri_owner_record( - &self, - rsync_uri: &str, - ) -> StorageResult> { - let cf = self.cf(CF_RRDP_URI_OWNER)?; - let key = rrdp_uri_owner_key(rsync_uri); - let Some(bytes) = self - .db - .get_cf(cf, key.as_bytes()) - .map_err(|e| StorageError::RocksDb(e.to_string()))? - else { - return Ok(None); - }; - let record = decode_cbor::(&bytes, "rrdp_uri_owner")?; - record.validate_internal()?; - Ok(Some(record)) - } - - pub fn delete_rrdp_uri_owner_record(&self, rsync_uri: &str) -> StorageResult<()> { - let cf = self.cf(CF_RRDP_URI_OWNER)?; - let key = rrdp_uri_owner_key(rsync_uri); - self.db - .delete_cf(cf, key.as_bytes()) - .map_err(|e| StorageError::RocksDb(e.to_string()))?; - Ok(()) - } - - #[allow(dead_code)] - - pub fn write_batch(&self, batch: WriteBatch) -> StorageResult<()> { - self.db - .write(batch) - .map_err(|e| StorageError::RocksDb(e.to_string()))?; - Ok(()) - } -} - -fn verify_repository_blob_batch( - store: &RocksStore, - batch: &[(String, String)], - summary: &mut RepositoryBlobVerificationSummary, -) -> StorageResult<()> { - let hashes = batch - .iter() - .map(|(_, hash)| hash.clone()) - .collect::>(); - let blobs = store.get_blob_bytes_batch(&hashes)?; - for ((uri, expected_hash), blob) in batch.iter().zip(blobs.into_iter()) { - let bytes = blob.as_ref().ok_or(StorageError::InvalidData { - entity: "repository_blob_verification", - detail: format!("blob missing for URI {uri} (hash={expected_hash})"), - })?; - let actual_hash = hex::encode(compute_sha256_32(bytes)); - if !actual_hash.eq_ignore_ascii_case(expected_hash) { - return Err(StorageError::InvalidData { - entity: "repository_blob_verification", - detail: format!( - "blob hash mismatch for URI {uri}: expected={expected_hash}, actual={actual_hash}" - ), - }); - } - summary.current_objects += 1; - summary.bytes_verified += bytes.len() as u64; - } - summary.batches += 1; - Ok(()) -} +include!("storage/models_core.rs"); +include!("storage/models_vcir.rs"); +include!("storage/models_publication.rs"); +include!("storage/models_summary.rs"); +include!("storage/batch_helpers.rs"); +include!("storage/store_lifecycle.rs"); +include!("storage/store_repository.rs"); +include!("storage/store_vcir.rs"); +include!("storage/store_child_cache.rs"); +include!("storage/store_publication_cache.rs"); +include!("storage/store_transport_rrdp.rs"); +include!("storage/verification.rs"); #[cfg(test)] #[path = "storage/tests.rs"] diff --git a/crates/panda-rpki-validator/src/storage/batch_helpers.rs b/crates/panda-rpki-validator/src/storage/batch_helpers.rs new file mode 100644 index 0000000..fc6ff85 --- /dev/null +++ b/crates/panda-rpki-validator/src/storage/batch_helpers.rs @@ -0,0 +1,69 @@ +// Batch encoders shared by the RocksDB write paths. + +fn write_roa_cache_projection_to_batch( + projection_cf: &ColumnFamily, + batch: &mut WriteBatch, + vcir: &ValidatedCaInstanceResult, + context: Option<&RoaCacheProjectionContext>, + timing: Option<&mut VcirReplaceTimingBreakdown>, +) -> StorageResult<()> { + let projection_key = roa_cache_projection_key(&vcir.manifest_rsync_uri); + let projection = RoaCacheProjection::from_vcir_with_context(vcir, context)?; + match projection { + Some(projection) => { + let projection_value = encode_cbor(&projection, "roa_cache_projection")?; + if let Some(timing) = timing { + timing.roa_cache_projection_value_bytes = projection_value.len() as u64; + } + batch.put_cf(projection_cf, projection_key.as_bytes(), projection_value); + } + None => { + batch.delete_cf(projection_cf, projection_key.as_bytes()); + } + } + Ok(()) +} + +fn write_vcir_failed_fetch_reuse_identity_to_batch( + failed_fetch_reuse_identity_cf: &ColumnFamily, + batch: &mut WriteBatch, + manifest_rsync_uri: &str, + failed_fetch_reuse_identity: Option<&VcirFailedFetchReuseIdentity>, +) -> StorageResult<()> { + let key = vcir_failed_fetch_reuse_identity_key(manifest_rsync_uri); + match failed_fetch_reuse_identity { + Some(identity) => { + identity.validate_internal()?; + let value = encode_cbor(identity, "vcir_failed_fetch_reuse_identity")?; + batch.put_cf(failed_fetch_reuse_identity_cf, key.as_bytes(), value); + } + None => batch.delete_cf(failed_fetch_reuse_identity_cf, key.as_bytes()), + } + Ok(()) +} + +fn write_publication_point_cache_projection_to_batch( + projection_cf: &ColumnFamily, + batch: &mut WriteBatch, + action: PublicationPointCacheProjectionWriteAction<'_>, + timing: Option<&mut VcirReplaceTimingBreakdown>, +) -> StorageResult<()> { + match action { + PublicationPointCacheProjectionWriteAction::Keep => Ok(()), + PublicationPointCacheProjectionWriteAction::Write(projection) => { + projection.validate_internal()?; + let key = publication_point_cache_projection_key(&projection.manifest_rsync_uri); + let value = encode_cbor(projection, "publication_point_cache_projection")?; + if let Some(timing) = timing { + timing.publication_point_cache_projection_value_bytes = value.len() as u64; + } + batch.put_cf(projection_cf, key.as_bytes(), value); + Ok(()) + } + PublicationPointCacheProjectionWriteAction::Delete { manifest_rsync_uri } => { + let key = publication_point_cache_projection_key(manifest_rsync_uri); + batch.delete_cf(projection_cf, key.as_bytes()); + Ok(()) + } + } +} diff --git a/crates/panda-rpki-validator/src/storage/memory.rs b/crates/panda-rpki-validator/src/storage/memory.rs new file mode 100644 index 0000000..0165a54 --- /dev/null +++ b/crates/panda-rpki-validator/src/storage/memory.rs @@ -0,0 +1,199 @@ +//! RocksDB and process-memory observation. +//! +//! This module owns the serialization-friendly memory snapshot model and the +//! mapping from RocksDB property names to that model. Storage mutation stays +//! in the parent module. + +use rocksdb::DB; +use serde::Serialize; + +pub(super) fn process_vm_rss_kb() -> Option { + let status = std::fs::read_to_string("/proc/self/status").ok()?; + status.lines().find_map(|line| { + let rest = line.strip_prefix("VmRSS:")?; + rest.split_whitespace().next()?.parse::().ok() + }) +} + +const ROCKSDB_MEMORY_PROPERTY_NAMES: &[(&str, &str)] = &[ + ("cur_size_all_mem_tables", "rocksdb.cur-size-all-mem-tables"), + ("size_all_mem_tables", "rocksdb.size-all-mem-tables"), + ( + "estimate_table_readers_mem", + "rocksdb.estimate-table-readers-mem", + ), + ("block_cache_capacity", "rocksdb.block-cache-capacity"), + ("block_cache_usage", "rocksdb.block-cache-usage"), + ( + "block_cache_pinned_usage", + "rocksdb.block-cache-pinned-usage", + ), + ("num_snapshots", "rocksdb.num-snapshots"), + ("background_errors", "rocksdb.background-errors"), +]; + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct RocksDbMemoryProperties { + pub cur_size_all_mem_tables: Option, + pub size_all_mem_tables: Option, + pub estimate_table_readers_mem: Option, + pub block_cache_capacity: Option, + pub block_cache_usage: Option, + pub block_cache_pinned_usage: Option, + pub num_snapshots: Option, + pub background_errors: Option, + pub errors: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct RocksDbColumnFamilyMemoryProperties { + pub name: String, + pub properties: RocksDbMemoryProperties, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct RocksDbMemoryDbSnapshot { + pub label: String, + pub properties: RocksDbMemoryProperties, + pub column_families: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct RocksDbMemoryTotals { + pub cur_size_all_mem_tables: u64, + pub size_all_mem_tables: u64, + pub estimate_table_readers_mem: u64, + pub block_cache_capacity: u64, + pub block_cache_usage: u64, + pub block_cache_pinned_usage: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct RocksDbMemorySnapshot { + pub databases: Vec, + pub totals: RocksDbMemoryTotals, +} + +impl RocksDbMemoryTotals { + pub(super) fn add_properties(&mut self, properties: &RocksDbMemoryProperties) { + self.cur_size_all_mem_tables += properties.cur_size_all_mem_tables.unwrap_or(0); + self.size_all_mem_tables += properties.size_all_mem_tables.unwrap_or(0); + self.estimate_table_readers_mem += properties.estimate_table_readers_mem.unwrap_or(0); + self.block_cache_capacity += properties.block_cache_capacity.unwrap_or(0); + self.block_cache_usage += properties.block_cache_usage.unwrap_or(0); + self.block_cache_pinned_usage += properties.block_cache_pinned_usage.unwrap_or(0); + } +} + +fn set_memory_property(properties: &mut RocksDbMemoryProperties, name: &str, value: u64) { + match name { + "cur_size_all_mem_tables" => properties.cur_size_all_mem_tables = Some(value), + "size_all_mem_tables" => properties.size_all_mem_tables = Some(value), + "estimate_table_readers_mem" => properties.estimate_table_readers_mem = Some(value), + "block_cache_capacity" => properties.block_cache_capacity = Some(value), + "block_cache_usage" => properties.block_cache_usage = Some(value), + "block_cache_pinned_usage" => properties.block_cache_pinned_usage = Some(value), + "num_snapshots" => properties.num_snapshots = Some(value), + "background_errors" => properties.background_errors = Some(value), + _ => {} + } +} + +fn parse_rocksdb_property_int(raw: Option) -> Option { + raw.and_then(|value| value.trim().parse::().ok()) +} + +fn memory_properties_for_db(db: &DB) -> RocksDbMemoryProperties { + let mut properties = RocksDbMemoryProperties::default(); + for (field_name, property_name) in ROCKSDB_MEMORY_PROPERTY_NAMES { + match db.property_value(*property_name) { + Ok(value) => { + if let Some(parsed) = parse_rocksdb_property_int(value) { + set_memory_property(&mut properties, field_name, parsed); + } + } + Err(err) => properties.errors.push(format!("{property_name}: {}", err)), + } + } + properties +} + +fn memory_properties_for_cf(db: &DB, cf_name: &'static str) -> RocksDbColumnFamilyMemoryProperties { + let mut properties = RocksDbMemoryProperties::default(); + let Some(cf) = db.cf_handle(cf_name) else { + properties + .errors + .push(format!("missing column family: {cf_name}")); + return RocksDbColumnFamilyMemoryProperties { + name: cf_name.to_string(), + properties, + }; + }; + for (field_name, property_name) in ROCKSDB_MEMORY_PROPERTY_NAMES { + match db.property_value_cf(cf, *property_name) { + Ok(value) => { + if let Some(parsed) = parse_rocksdb_property_int(value) { + set_memory_property(&mut properties, field_name, parsed); + } + } + Err(err) => properties.errors.push(format!("{property_name}: {}", err)), + } + } + RocksDbColumnFamilyMemoryProperties { + name: cf_name.to_string(), + properties, + } +} + +pub(crate) fn memory_db_snapshot_for_column_families( + label: impl Into, + db: &DB, + column_families: Option<&[&'static str]>, +) -> RocksDbMemoryDbSnapshot { + RocksDbMemoryDbSnapshot { + label: label.into(), + properties: memory_properties_for_db(db), + column_families: column_families + .map(|names| { + names + .iter() + .map(|name| memory_properties_for_cf(db, name)) + .collect() + }) + .unwrap_or_default(), + } +} + +#[cfg(test)] +mod tests { + use super::{ + RocksDbMemoryProperties, RocksDbMemoryTotals, parse_rocksdb_property_int, + set_memory_property, + }; + + #[test] + fn parses_rocksdb_integer_properties_without_panicking() { + assert_eq!( + parse_rocksdb_property_int(Some(" 42 ".to_string())), + Some(42) + ); + assert_eq!( + parse_rocksdb_property_int(Some("not-a-number".to_string())), + None + ); + assert_eq!(parse_rocksdb_property_int(None), None); + } + + #[test] + fn aggregates_only_known_memory_properties() { + let mut properties = RocksDbMemoryProperties::default(); + set_memory_property(&mut properties, "block_cache_usage", 11); + set_memory_property(&mut properties, "unknown", 99); + + let mut totals = RocksDbMemoryTotals::default(); + totals.add_properties(&properties); + + assert_eq!(totals.block_cache_usage, 11); + assert_eq!(totals.block_cache_capacity, 0); + } +} diff --git a/crates/panda-rpki-validator/src/storage/models_core.rs b/crates/panda-rpki-validator/src/storage/models_core.rs new file mode 100644 index 0000000..e95a360 --- /dev/null +++ b/crates/panda-rpki-validator/src/storage/models_core.rs @@ -0,0 +1,758 @@ +// Core storage errors, repository records, and serde helpers. + +#[derive(Debug, thiserror::Error)] +pub enum StorageError { + #[error("rocksdb error: {0}")] + RocksDb(String), + + #[error("missing column family: {0}")] + MissingColumnFamily(&'static str), + + #[error("cbor codec error for {entity}: {detail}")] + Codec { + entity: &'static str, + detail: String, + }, + + #[error("invalid {entity}: {detail}")] + InvalidData { + entity: &'static str, + detail: String, + }, +} + +pub type StorageResult = Result; + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct RepositoryBlobVerificationSummary { + pub current_objects: u64, + pub bytes_verified: u64, + pub batches: u64, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct ChildCertificateCacheMmapLookup { + pub projections: Vec>, + pub hits: usize, + pub misses: usize, + pub file_bytes: u64, +} + +pub struct RocksStore { + db: DB, + external_raw_store: Option, + external_repo_bytes: Option, + publication_point_cache_index_dir: PathBuf, + child_certificate_cache_index_dir: PathBuf, + publication_point_cache_projection_index: Mutex, +} + +enum PublicationPointCacheProjectionIndexState { + Uninitialized, + Disabled, + BuildingFromEmpty { + index: HashMap>, + bytes: usize, + limit: usize, + }, + Loaded { + index: HashMap>, + bytes: usize, + }, + LoadedMmap { + mmap: PpCacheMmapIndexSet, + dirty: HashMap>, + dirty_bytes: usize, + load_stats: PpCacheIndexLoadStats, + }, +} + +#[derive(Clone, Copy)] +pub(crate) enum PublicationPointCacheProjectionWriteAction<'a> { + Keep, + Write(&'a PublicationPointCacheProjection), + Delete { manifest_rsync_uri: &'a str }, +} + +fn default_child_certificate_cache_index_dir(db_path: &Path) -> PathBuf { + let file_name = db_path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("work-db"); + db_path.with_file_name(format!("{file_name}.child-cert-cache-index")) +} + +fn child_certificate_cache_segment_file_name(manifest_rsync_uri: &str) -> String { + format!( + "{}.idx", + hex::encode(compute_sha256_32(manifest_rsync_uri.as_bytes())) + ) +} + +const PP_CACHE_RAW_INDEX_ENV: &str = "RPKI_PP_CACHE_RAW_INDEX"; +const PP_CACHE_RAW_INDEX_EMPTY_BUILD_LIMIT_ENV: &str = + "RPKI_PP_CACHE_RAW_INDEX_EMPTY_BUILD_LIMIT_BYTES"; +const DEFAULT_PP_CACHE_RAW_INDEX_EMPTY_BUILD_LIMIT_BYTES: usize = 32 * 1024 * 1024; +const PP_CACHE_INDEX_COMPACTION_SEGMENT_THRESHOLD: usize = 16; +const PP_CACHE_INDEX_COMPACTION_BYTES_THRESHOLD: u64 = 1_610_612_736; + +fn pp_cache_raw_index_enabled() -> bool { + match std::env::var(PP_CACHE_RAW_INDEX_ENV) { + Ok(value) => !matches!( + value.trim().to_ascii_lowercase().as_str(), + "0" | "false" | "off" | "no" + ), + Err(_) => true, + } +} + +fn pp_cache_raw_index_empty_build_limit_bytes() -> usize { + std::env::var(PP_CACHE_RAW_INDEX_EMPTY_BUILD_LIMIT_ENV) + .ok() + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or(DEFAULT_PP_CACHE_RAW_INDEX_EMPTY_BUILD_LIMIT_BYTES) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RrdpDeltaOp { + Upsert { rsync_uri: String, bytes: Vec }, + Delete { rsync_uri: String }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RepositoryViewState { + Present, + Withdrawn, + Replaced, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RepositoryViewEntry { + pub rsync_uri: String, + pub current_hash: Option, + pub repository_source: Option, + pub object_type: Option, + pub state: RepositoryViewState, +} + +impl RepositoryViewEntry { + pub fn validate_internal(&self) -> StorageResult<()> { + validate_non_empty("repository_view.rsync_uri", &self.rsync_uri)?; + if let Some(source) = &self.repository_source { + validate_non_empty("repository_view.repository_source", source)?; + } + match self.state { + RepositoryViewState::Present | RepositoryViewState::Replaced => { + let hash = self + .current_hash + .as_deref() + .ok_or(StorageError::InvalidData { + entity: "repository_view", + detail: "current_hash is required when state is present or replaced" + .to_string(), + })?; + validate_sha256_hex("repository_view.current_hash", hash)?; + } + RepositoryViewState::Withdrawn => { + if let Some(hash) = &self.current_hash { + validate_sha256_hex("repository_view.current_hash", hash)?; + } + } + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RawByHashEntry { + pub sha256_hex: String, + pub bytes: Vec, + pub origin_uris: Vec, + pub object_type: Option, + pub encoding: Option, +} + +impl RawByHashEntry { + pub fn from_bytes(sha256_hex: impl Into, bytes: Vec) -> Self { + Self { + sha256_hex: sha256_hex.into(), + bytes, + origin_uris: Vec::new(), + object_type: None, + encoding: None, + } + } + + pub fn validate_internal(&self) -> StorageResult<()> { + validate_sha256_hex("raw_by_hash.sha256_hex", &self.sha256_hex)?; + if self.bytes.is_empty() { + return Err(StorageError::InvalidData { + entity: "raw_by_hash", + detail: "bytes must not be empty".to_string(), + }); + } + let computed = hex::encode(compute_sha256_32(&self.bytes)); + if computed != self.sha256_hex.to_ascii_lowercase() { + return Err(StorageError::InvalidData { + entity: "raw_by_hash", + detail: "sha256_hex does not match bytes".to_string(), + }); + } + let mut seen = HashSet::with_capacity(self.origin_uris.len()); + for uri in &self.origin_uris { + validate_non_empty("raw_by_hash.origin_uris[]", uri)?; + if !seen.insert(uri.as_str()) { + return Err(StorageError::InvalidData { + entity: "raw_by_hash", + detail: format!("duplicate origin URI: {uri}"), + }); + } + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CurrentObjectWithHash { + pub current_hash_hex: String, + pub current_hash: [u8; 32], + pub bytes: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ValidatedManifestMeta { + pub validated_manifest_number: Vec, + pub validated_manifest_this_update: PackTime, + pub validated_manifest_next_update: PackTime, +} + +impl ValidatedManifestMeta { + pub fn validate_internal(&self) -> StorageResult<()> { + validate_manifest_number_be( + "validated_manifest_meta.validated_manifest_number", + &self.validated_manifest_number, + )?; + let this_update = parse_time( + "validated_manifest_meta.validated_manifest_this_update", + &self.validated_manifest_this_update, + )?; + let next_update = parse_time( + "validated_manifest_meta.validated_manifest_next_update", + &self.validated_manifest_next_update, + )?; + if next_update < this_update { + return Err(StorageError::InvalidData { + entity: "validated_manifest_meta", + detail: "validated_manifest_next_update must be >= validated_manifest_this_update" + .to_string(), + }); + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ManifestReplayMeta { + pub manifest_rsync_uri: String, + pub manifest_number_be: Vec, + pub manifest_this_update: PackTime, + pub manifest_sha256: Vec, + pub updated_at_validation_time: PackTime, +} + +impl ManifestReplayMeta { + pub fn from_vcir(vcir: &ValidatedCaInstanceResult) -> Self { + Self { + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + manifest_number_be: vcir + .validated_manifest_meta + .validated_manifest_number + .clone(), + manifest_this_update: vcir + .validated_manifest_meta + .validated_manifest_this_update + .clone(), + manifest_sha256: vcir.ccr_manifest_projection.manifest_sha256.clone(), + updated_at_validation_time: vcir.last_successful_validation_time.clone(), + } + } + + pub fn validate_internal(&self) -> StorageResult<()> { + validate_non_empty( + "manifest_replay_meta.manifest_rsync_uri", + &self.manifest_rsync_uri, + )?; + validate_manifest_number_be( + "manifest_replay_meta.manifest_number_be", + &self.manifest_number_be, + )?; + parse_time( + "manifest_replay_meta.manifest_this_update", + &self.manifest_this_update, + )?; + validate_sha256_digest_bytes( + "manifest_replay_meta.manifest_sha256", + &self.manifest_sha256, + )?; + parse_time( + "manifest_replay_meta.updated_at_validation_time", + &self.updated_at_validation_time, + )?; + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct VcirCcrManifestProjection { + pub manifest_rsync_uri: String, + pub manifest_sha256: Vec, + pub manifest_size: u64, + pub manifest_ee_aki: Vec, + pub manifest_number_be: Vec, + pub manifest_this_update: PackTime, + pub manifest_sia_locations_der: Vec>, + pub subordinate_skis: Vec>, +} + +impl VcirCcrManifestProjection { + pub fn validate_internal(&self) -> StorageResult<()> { + validate_non_empty( + "vcir.ccr_manifest_projection.manifest_rsync_uri", + &self.manifest_rsync_uri, + )?; + validate_sha256_digest_bytes( + "vcir.ccr_manifest_projection.manifest_sha256", + &self.manifest_sha256, + )?; + if self.manifest_size < 1000 { + return Err(StorageError::InvalidData { + entity: "vcir.ccr_manifest_projection.manifest_size", + detail: format!("must be >= 1000, got {}", self.manifest_size), + }); + } + validate_fixed_len_bytes( + "vcir.ccr_manifest_projection.manifest_ee_aki", + &self.manifest_ee_aki, + 20, + )?; + validate_manifest_number_be( + "vcir.ccr_manifest_projection.manifest_number_be", + &self.manifest_number_be, + )?; + parse_time( + "vcir.ccr_manifest_projection.manifest_this_update", + &self.manifest_this_update, + )?; + if self.manifest_sia_locations_der.is_empty() { + return Err(StorageError::InvalidData { + entity: "vcir.ccr_manifest_projection.manifest_sia_locations_der", + detail: "must contain at least one AccessDescription".to_string(), + }); + } + for location in &self.manifest_sia_locations_der { + validate_full_der_with_tag( + "vcir.ccr_manifest_projection.manifest_sia_locations_der[]", + location, + Some(0x30), + )?; + } + validate_sorted_unique_fixed_len_bytes( + "vcir.ccr_manifest_projection.subordinate_skis", + &self.subordinate_skis, + 20, + )?; + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct VcirInstanceGate { + pub manifest_next_update: PackTime, + pub current_crl_next_update: PackTime, + pub self_ca_not_after: PackTime, + pub instance_effective_until: PackTime, +} + +impl VcirInstanceGate { + pub fn validate_internal(&self) -> StorageResult<()> { + let manifest_next_update = parse_time( + "vcir.instance_gate.manifest_next_update", + &self.manifest_next_update, + )?; + let current_crl_next_update = parse_time( + "vcir.instance_gate.current_crl_next_update", + &self.current_crl_next_update, + )?; + let self_ca_not_after = parse_time( + "vcir.instance_gate.self_ca_not_after", + &self.self_ca_not_after, + )?; + let instance_effective_until = parse_time( + "vcir.instance_gate.instance_effective_until", + &self.instance_effective_until, + )?; + let expected = manifest_next_update + .min(current_crl_next_update) + .min(self_ca_not_after); + if instance_effective_until != expected { + return Err(StorageError::InvalidData { + entity: "vcir.instance_gate", + detail: "instance_effective_until must equal min(manifest_next_update, current_crl_next_update, self_ca_not_after)".to_string(), + }); + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct VcirFailedFetchReuseIdentity { + #[serde(rename = "c")] + #[serde(with = "serde_bytes_32")] + pub current_ca_sha256: [u8; 32], + #[serde(rename = "t")] + #[serde(with = "serde_bytes_32")] + pub ta_context_digest: [u8; 32], + #[serde(rename = "p")] + #[serde(with = "serde_bytes_32")] + pub ca_validation_context_digest: [u8; 32], + #[serde(rename = "f")] + #[serde(with = "serde_bytes_32")] + pub policy_fingerprint: [u8; 32], + #[serde(rename = "nb")] + pub effective_not_before: PackTime, + #[serde(rename = "nu")] + pub effective_until: PackTime, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct VcirReuseRecordsCleared { + pub vcir_records: u64, + pub failed_fetch_identity_records: u64, +} + +impl VcirFailedFetchReuseIdentity { + pub fn validate_internal(&self) -> StorageResult<()> { + let effective_not_before = parse_time( + "vcir_failed_fetch_reuse_identity.effective_not_before", + &self.effective_not_before, + )?; + let effective_until = parse_time( + "vcir_failed_fetch_reuse_identity.effective_until", + &self.effective_until, + )?; + if effective_not_before >= effective_until { + return Err(StorageError::InvalidData { + entity: "vcir_failed_fetch_reuse_identity.effective_window", + detail: "effective_not_before must be before effective_until".to_string(), + }); + } + Ok(()) + } + + pub fn contains_validation_time(&self, validation_time: time::OffsetDateTime) -> bool { + let Ok(effective_not_before) = self.effective_not_before.parse() else { + return false; + }; + let Ok(effective_until) = self.effective_until.parse() else { + return false; + }; + validation_time >= effective_not_before && validation_time < effective_until + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct VcirChildEntry { + pub child_manifest_rsync_uri: String, + pub child_cert_rsync_uri: String, + pub child_cert_hash: String, + pub child_ski: String, + pub child_rsync_base_uri: String, + pub child_publication_point_rsync_uri: String, + pub child_rrdp_notification_uri: Option, + pub child_effective_ip_resources: Option, + pub child_effective_as_resources: Option, + pub accepted_at_validation_time: PackTime, +} + +impl VcirChildEntry { + pub fn validate_internal(&self) -> StorageResult<()> { + validate_non_empty( + "vcir.child_entries[].child_manifest_rsync_uri", + &self.child_manifest_rsync_uri, + )?; + validate_non_empty( + "vcir.child_entries[].child_cert_rsync_uri", + &self.child_cert_rsync_uri, + )?; + validate_sha256_hex( + "vcir.child_entries[].child_cert_hash", + &self.child_cert_hash, + )?; + validate_non_empty("vcir.child_entries[].child_ski", &self.child_ski)?; + validate_non_empty( + "vcir.child_entries[].child_rsync_base_uri", + &self.child_rsync_base_uri, + )?; + validate_non_empty( + "vcir.child_entries[].child_publication_point_rsync_uri", + &self.child_publication_point_rsync_uri, + )?; + if let Some(uri) = &self.child_rrdp_notification_uri { + validate_non_empty("vcir.child_entries[].child_rrdp_notification_uri", uri)?; + } + parse_time( + "vcir.child_entries[].accepted_at_validation_time", + &self.accepted_at_validation_time, + )?; + Ok(()) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum VcirOutputType { + Vrp, + Aspa, + RouterKey, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum VcirSourceObjectType { + Roa, + Aspa, + RouterKey, + Other, +} + +impl VcirSourceObjectType { + pub fn as_str(self) -> &'static str { + match self { + Self::Roa => "roa", + Self::Aspa => "aspa", + Self::RouterKey => "router_key", + Self::Other => "other", + } + } +} + +struct FixedBytesVisitor; + +impl<'de, const N: usize> serde::de::Visitor<'de> for FixedBytesVisitor { + type Value = [u8; N]; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "{N} bytes") + } + + fn visit_bytes(self, value: &[u8]) -> Result + where + E: serde::de::Error, + { + if value.len() != N { + return Err(E::invalid_length(value.len(), &self)); + } + let mut out = [0u8; N]; + out.copy_from_slice(value); + Ok(out) + } + + fn visit_byte_buf(self, value: Vec) -> Result + where + E: serde::de::Error, + { + self.visit_bytes(&value) + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let mut out = [0u8; N]; + for (idx, slot) in out.iter_mut().enumerate() { + *slot = seq + .next_element()? + .ok_or_else(|| serde::de::Error::invalid_length(idx, &self))?; + } + Ok(out) + } +} + +fn deserialize_fixed_bytes<'de, D, const N: usize>(deserializer: D) -> Result<[u8; N], D::Error> +where + D: serde::Deserializer<'de>, +{ + deserializer.deserialize_bytes(FixedBytesVisitor::) +} + +mod serde_bytes_16 { + pub(super) fn serialize(value: &[u8; 16], serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_bytes(value) + } + + pub(super) fn deserialize<'de, D>(deserializer: D) -> Result<[u8; 16], D::Error> + where + D: serde::Deserializer<'de>, + { + super::deserialize_fixed_bytes::(deserializer) + } +} + +mod serde_bytes_32 { + pub(super) fn serialize(value: &[u8; 32], serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_bytes(value) + } + + pub(super) fn deserialize<'de, D>(deserializer: D) -> Result<[u8; 32], D::Error> + where + D: serde::Deserializer<'de>, + { + super::deserialize_fixed_bytes::(deserializer) + } +} + +struct ByteVecVisitor; + +impl<'de> serde::de::Visitor<'de> for ByteVecVisitor { + type Value = Vec; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("byte vector") + } + + fn visit_bytes(self, value: &[u8]) -> Result + where + E: serde::de::Error, + { + Ok(value.to_vec()) + } + + fn visit_byte_buf(self, value: Vec) -> Result + where + E: serde::de::Error, + { + Ok(value) + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let mut out = Vec::with_capacity(seq.size_hint().unwrap_or(0)); + while let Some(byte) = seq.next_element()? { + out.push(byte); + } + Ok(out) + } +} + +mod serde_byte_vec { + pub(super) fn serialize(value: &[u8], serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_bytes(value) + } + + pub(super) fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_bytes(super::ByteVecVisitor) + } +} + +mod serde_optional_byte_vec { + pub(super) fn serialize(value: &Option>, serializer: S) -> Result + where + S: serde::Serializer, + { + match value { + Some(bytes) => serializer.serialize_some(bytes), + None => serializer.serialize_none(), + } + } + + pub(super) fn deserialize<'de, D>(deserializer: D) -> Result>, D::Error> + where + D: serde::Deserializer<'de>, + { + struct OptionalByteVecVisitor; + + impl<'de> serde::de::Visitor<'de> for OptionalByteVecVisitor { + type Value = Option>; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("optional byte vector") + } + + fn visit_none(self) -> Result + where + E: serde::de::Error, + { + Ok(None) + } + + fn visit_some(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer + .deserialize_bytes(super::ByteVecVisitor) + .map(Some) + } + } + + deserializer.deserialize_option(OptionalByteVecVisitor) + } +} + +mod serde_optional_bytes_32 { + pub(super) fn serialize(value: &Option<[u8; 32]>, serializer: S) -> Result + where + S: serde::Serializer, + { + match value { + Some(bytes) => serializer.serialize_some(bytes.as_slice()), + None => serializer.serialize_none(), + } + } + + pub(super) fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: serde::Deserializer<'de>, + { + struct OptionalBytes32Visitor; + + impl<'de> serde::de::Visitor<'de> for OptionalBytes32Visitor { + type Value = Option<[u8; 32]>; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("optional 32-byte array") + } + + fn visit_none(self) -> Result + where + E: serde::de::Error, + { + Ok(None) + } + + fn visit_some(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + super::deserialize_fixed_bytes::(deserializer).map(Some) + } + } + + deserializer.deserialize_option(OptionalBytes32Visitor) + } +} diff --git a/crates/panda-rpki-validator/src/storage/models_publication.rs b/crates/panda-rpki-validator/src/storage/models_publication.rs new file mode 100644 index 0000000..f292fe3 --- /dev/null +++ b/crates/panda-rpki-validator/src/storage/models_publication.rs @@ -0,0 +1,628 @@ +// Publication-point and ROA cache projections. + +pub const PUBLICATION_POINT_CACHE_SCHEMA_VERSION: u32 = 1; +pub const PUBLICATION_POINT_CACHE_ALGORITHM_VERSION: u32 = 1; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PublicationPointCacheOutput { + #[serde(rename = "t")] + pub output_type: VcirOutputType, + #[serde(rename = "nb")] + pub item_effective_not_before: PackTime, + #[serde(rename = "nu")] + pub item_effective_until: PackTime, + #[serde(rename = "u")] + pub source_object_uri: String, + #[serde(rename = "k")] + pub source_object_type: VcirSourceObjectType, + #[serde(rename = "h")] + #[serde(with = "serde_bytes_32")] + pub source_object_hash: [u8; 32], + #[serde(rename = "c")] + #[serde(with = "serde_bytes_32")] + pub source_ee_cert_hash: [u8; 32], + #[serde(rename = "p")] + pub payload: VcirLocalOutputPayload, + #[serde(rename = "r")] + #[serde(with = "serde_bytes_32")] + pub rule_hash: [u8; 32], +} + +impl PublicationPointCacheOutput { + fn from_local_output(output: &VcirLocalOutput, default_not_before: &PackTime) -> Self { + Self { + output_type: output.output_type, + item_effective_not_before: default_not_before.clone(), + item_effective_until: output.item_effective_until.clone(), + source_object_uri: output.source_object_uri.clone(), + source_object_type: output.source_object_type, + source_object_hash: output.source_object_hash, + source_ee_cert_hash: output.source_ee_cert_hash, + payload: output.payload.clone(), + rule_hash: output.rule_hash, + } + } + + pub fn to_local_output(&self) -> VcirLocalOutput { + VcirLocalOutput { + output_type: self.output_type, + item_effective_until: self.item_effective_until.clone(), + source_object_uri: self.source_object_uri.clone(), + source_object_type: self.source_object_type, + source_object_hash: self.source_object_hash, + source_ee_cert_hash: self.source_ee_cert_hash, + payload: self.payload.clone(), + rule_hash: self.rule_hash, + } + } + + pub fn validate_internal(&self) -> StorageResult<()> { + parse_time( + "publication_point_cache_projection.outputs[].item_effective_not_before", + &self.item_effective_not_before, + )?; + parse_time( + "publication_point_cache_projection.outputs[].item_effective_until", + &self.item_effective_until, + )?; + validate_non_empty( + "publication_point_cache_projection.outputs[].source_object_uri", + &self.source_object_uri, + )?; + VcirLocalOutput { + output_type: self.output_type, + item_effective_until: self.item_effective_until.clone(), + source_object_uri: self.source_object_uri.clone(), + source_object_type: self.source_object_type, + source_object_hash: self.source_object_hash, + source_ee_cert_hash: self.source_ee_cert_hash, + payload: self.payload.clone(), + rule_hash: self.rule_hash, + } + .validate_internal() + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PublicationPointCacheChild { + #[serde(rename = "cm")] + pub child_manifest_rsync_uri: String, + #[serde(rename = "cu")] + pub child_cert_rsync_uri: String, + #[serde(rename = "ch")] + pub child_cert_hash: String, + #[serde(rename = "ski")] + pub child_ski: String, + #[serde(rename = "rb")] + pub child_rsync_base_uri: String, + #[serde(rename = "pp")] + pub child_publication_point_rsync_uri: String, + #[serde(rename = "rn")] + pub child_rrdp_notification_uri: Option, + #[serde(rename = "ip")] + pub child_effective_ip_resources: Option, + #[serde(rename = "as")] + pub child_effective_as_resources: Option, + #[serde(rename = "nb")] + pub child_effective_not_before: PackTime, + #[serde(rename = "nu")] + pub child_effective_until: PackTime, +} + +impl PublicationPointCacheChild { + fn from_child_entry( + entry: &VcirChildEntry, + default_not_before: &PackTime, + default_until: &PackTime, + ) -> Self { + Self { + child_manifest_rsync_uri: entry.child_manifest_rsync_uri.clone(), + child_cert_rsync_uri: entry.child_cert_rsync_uri.clone(), + child_cert_hash: entry.child_cert_hash.clone(), + child_ski: entry.child_ski.clone(), + child_rsync_base_uri: entry.child_rsync_base_uri.clone(), + child_publication_point_rsync_uri: entry.child_publication_point_rsync_uri.clone(), + child_rrdp_notification_uri: entry.child_rrdp_notification_uri.clone(), + child_effective_ip_resources: entry.child_effective_ip_resources.clone(), + child_effective_as_resources: entry.child_effective_as_resources.clone(), + child_effective_not_before: default_not_before.clone(), + child_effective_until: default_until.clone(), + } + } + + pub fn to_child_entry(&self, accepted_at_validation_time: PackTime) -> VcirChildEntry { + VcirChildEntry { + child_manifest_rsync_uri: self.child_manifest_rsync_uri.clone(), + child_cert_rsync_uri: self.child_cert_rsync_uri.clone(), + child_cert_hash: self.child_cert_hash.clone(), + child_ski: self.child_ski.clone(), + child_rsync_base_uri: self.child_rsync_base_uri.clone(), + child_publication_point_rsync_uri: self.child_publication_point_rsync_uri.clone(), + child_rrdp_notification_uri: self.child_rrdp_notification_uri.clone(), + child_effective_ip_resources: self.child_effective_ip_resources.clone(), + child_effective_as_resources: self.child_effective_as_resources.clone(), + accepted_at_validation_time, + } + } + + pub fn validate_internal(&self) -> StorageResult<()> { + self.to_child_entry(self.child_effective_not_before.clone()) + .validate_internal()?; + parse_time( + "publication_point_cache_projection.children[].child_effective_not_before", + &self.child_effective_not_before, + )?; + parse_time( + "publication_point_cache_projection.children[].child_effective_until", + &self.child_effective_until, + )?; + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PublicationPointCacheObject { + #[serde(rename = "r")] + pub artifact_role: VcirArtifactRole, + #[serde(rename = "k")] + pub artifact_kind: VcirArtifactKind, + #[serde(rename = "u")] + pub uri: Option, + #[serde(rename = "h")] + pub sha256: String, + #[serde(rename = "t")] + pub object_type: Option, + #[serde(rename = "s")] + pub validation_status: VcirArtifactValidationStatus, + /// See `VcirRelatedArtifact::reject_reason`. + #[serde(rename = "e", default, skip_serializing_if = "Option::is_none")] + pub reject_reason: Option, +} + +impl PublicationPointCacheObject { + fn from_related_artifact(artifact: &VcirRelatedArtifact) -> Self { + Self { + artifact_role: artifact.artifact_role, + artifact_kind: artifact.artifact_kind, + uri: artifact.uri.clone(), + sha256: artifact.sha256.clone(), + object_type: artifact.object_type.clone(), + validation_status: artifact.validation_status, + reject_reason: artifact.reject_reason.clone(), + } + } + + pub fn to_related_artifact(&self) -> VcirRelatedArtifact { + VcirRelatedArtifact { + artifact_role: self.artifact_role, + artifact_kind: self.artifact_kind, + uri: self.uri.clone(), + sha256: self.sha256.clone(), + object_type: self.object_type.clone(), + validation_status: self.validation_status, + reject_reason: self.reject_reason.clone(), + } + } + + pub fn validate_internal(&self) -> StorageResult<()> { + self.to_related_artifact().validate_internal() + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PublicationPointCacheProjection { + #[serde(rename = "sv")] + pub schema_version: u32, + #[serde(rename = "av")] + pub algorithm_version: u32, + #[serde(rename = "m")] + pub manifest_rsync_uri: String, + #[serde(rename = "pp")] + pub publication_point_rsync_uri: String, + #[serde(rename = "cu")] + pub ca_cert_uri: Option, + #[serde(rename = "ch")] + #[serde(with = "serde_bytes_32")] + pub ca_cert_sha256: [u8; 32], + #[serde(rename = "mh")] + #[serde(with = "serde_bytes_32")] + pub manifest_sha256: [u8; 32], + #[serde(rename = "tal")] + pub tal_id: String, + #[serde(rename = "ta")] + #[serde(with = "serde_bytes_32")] + pub ta_context_digest: [u8; 32], + #[serde(rename = "pc")] + #[serde(with = "serde_bytes_32")] + pub ca_validation_context_digest: [u8; 32], + #[serde(rename = "pf")] + #[serde(with = "serde_bytes_32")] + pub validation_policy_fingerprint: [u8; 32], + #[serde(rename = "nb")] + pub instance_effective_not_before: PackTime, + #[serde(rename = "nu")] + pub instance_effective_until: PackTime, + #[serde(rename = "mt")] + pub manifest_this_update: PackTime, + #[serde(rename = "mn")] + pub manifest_next_update: PackTime, + #[serde(rename = "cn")] + pub current_crl_next_update: PackTime, + #[serde(rename = "ca")] + pub self_ca_not_after: PackTime, + #[serde(rename = "ccr")] + pub ccr_manifest_projection: VcirCcrManifestProjection, + #[serde(rename = "o")] + pub outputs: Vec, + #[serde(rename = "c")] + pub children: Vec, + #[serde(rename = "ra")] + pub related_objects: Vec, + #[serde(rename = "s")] + pub summary: VcirSummary, +} + +impl PublicationPointCacheProjection { + pub fn from_vcir_with_context( + vcir: &ValidatedCaInstanceResult, + publication_point_rsync_uri: String, + ca_cert_uri: Option, + ca_cert_sha256: [u8; 32], + manifest_sha256: [u8; 32], + ta_context_digest: [u8; 32], + ca_validation_context_digest: [u8; 32], + validation_policy_fingerprint: [u8; 32], + ) -> StorageResult { + let projection = Self { + schema_version: PUBLICATION_POINT_CACHE_SCHEMA_VERSION, + algorithm_version: PUBLICATION_POINT_CACHE_ALGORITHM_VERSION, + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri, + ca_cert_uri, + ca_cert_sha256, + manifest_sha256, + tal_id: vcir.tal_id.clone(), + ta_context_digest, + ca_validation_context_digest, + validation_policy_fingerprint, + instance_effective_not_before: vcir.last_successful_validation_time.clone(), + instance_effective_until: vcir.instance_gate.instance_effective_until.clone(), + manifest_this_update: vcir + .validated_manifest_meta + .validated_manifest_this_update + .clone(), + manifest_next_update: vcir.instance_gate.manifest_next_update.clone(), + current_crl_next_update: vcir.instance_gate.current_crl_next_update.clone(), + self_ca_not_after: vcir.instance_gate.self_ca_not_after.clone(), + ccr_manifest_projection: vcir.ccr_manifest_projection.clone(), + outputs: vcir + .local_outputs + .iter() + .map(|output| { + PublicationPointCacheOutput::from_local_output( + output, + &vcir.last_successful_validation_time, + ) + }) + .collect(), + children: vcir + .child_entries + .iter() + .map(|child| { + PublicationPointCacheChild::from_child_entry( + child, + &vcir.last_successful_validation_time, + &vcir.instance_gate.instance_effective_until, + ) + }) + .collect(), + related_objects: vcir + .related_artifacts + .iter() + .map(PublicationPointCacheObject::from_related_artifact) + .collect(), + summary: vcir.summary.clone(), + }; + projection.validate_internal()?; + Ok(projection) + } + + pub fn to_vcir_for_reuse( + &self, + validation_time: time::OffsetDateTime, + ) -> ValidatedCaInstanceResult { + let validation_time = PackTime::from_utc_offset_datetime(validation_time); + ValidatedCaInstanceResult { + manifest_rsync_uri: self.manifest_rsync_uri.clone(), + parent_manifest_rsync_uri: None, + tal_id: self.tal_id.clone(), + ca_subject_name: String::from("publication-point-cache"), + ca_ski: hex::encode(self.ca_cert_sha256), + issuer_ski: hex::encode(self.ca_validation_context_digest), + last_successful_validation_time: validation_time.clone(), + current_manifest_rsync_uri: self.manifest_rsync_uri.clone(), + current_crl_rsync_uri: self + .related_objects + .iter() + .find(|object| object.artifact_role == VcirArtifactRole::CurrentCrl) + .and_then(|object| object.uri.clone()) + .unwrap_or_default(), + validated_manifest_meta: ValidatedManifestMeta { + validated_manifest_number: self.ccr_manifest_projection.manifest_number_be.clone(), + validated_manifest_this_update: self.manifest_this_update.clone(), + validated_manifest_next_update: self.manifest_next_update.clone(), + }, + ccr_manifest_projection: self.ccr_manifest_projection.clone(), + instance_gate: VcirInstanceGate { + manifest_next_update: self.manifest_next_update.clone(), + current_crl_next_update: self.current_crl_next_update.clone(), + self_ca_not_after: self.self_ca_not_after.clone(), + instance_effective_until: self.instance_effective_until.clone(), + }, + child_entries: self + .children + .iter() + .map(|child| child.to_child_entry(validation_time.clone())) + .collect(), + local_outputs: self + .outputs + .iter() + .map(PublicationPointCacheOutput::to_local_output) + .collect(), + related_artifacts: self + .related_objects + .iter() + .map(PublicationPointCacheObject::to_related_artifact) + .collect(), + summary: self.summary.clone(), + audit_summary: VcirAuditSummary { + failed_fetch_eligible: false, + last_failed_fetch_reason: None, + warning_count: 0, + audit_flags: vec!["publication_point_cache_projection".to_string()], + }, + } + } + + pub fn validate_internal(&self) -> StorageResult<()> { + if self.schema_version != PUBLICATION_POINT_CACHE_SCHEMA_VERSION { + return Err(StorageError::InvalidData { + entity: "publication_point_cache_projection.schema_version", + detail: format!("unsupported schema_version {}", self.schema_version), + }); + } + if self.algorithm_version != PUBLICATION_POINT_CACHE_ALGORITHM_VERSION { + return Err(StorageError::InvalidData { + entity: "publication_point_cache_projection.algorithm_version", + detail: format!("unsupported algorithm_version {}", self.algorithm_version), + }); + } + validate_non_empty( + "publication_point_cache_projection.manifest_rsync_uri", + &self.manifest_rsync_uri, + )?; + validate_non_empty( + "publication_point_cache_projection.publication_point_rsync_uri", + &self.publication_point_rsync_uri, + )?; + if let Some(uri) = &self.ca_cert_uri { + validate_non_empty("publication_point_cache_projection.ca_cert_uri", uri)?; + } + validate_non_empty("publication_point_cache_projection.tal_id", &self.tal_id)?; + parse_time( + "publication_point_cache_projection.instance_effective_not_before", + &self.instance_effective_not_before, + )?; + parse_time( + "publication_point_cache_projection.instance_effective_until", + &self.instance_effective_until, + )?; + parse_time( + "publication_point_cache_projection.manifest_this_update", + &self.manifest_this_update, + )?; + parse_time( + "publication_point_cache_projection.manifest_next_update", + &self.manifest_next_update, + )?; + parse_time( + "publication_point_cache_projection.current_crl_next_update", + &self.current_crl_next_update, + )?; + parse_time( + "publication_point_cache_projection.self_ca_not_after", + &self.self_ca_not_after, + )?; + self.ccr_manifest_projection.validate_internal()?; + for output in &self.outputs { + output.validate_internal()?; + } + for child in &self.children { + child.validate_internal()?; + } + for object in &self.related_objects { + object.validate_internal()?; + } + Ok(()) + } +} + +impl RoaCacheProjection { + pub fn from_vcir(vcir: &ValidatedCaInstanceResult) -> StorageResult> { + Self::from_vcir_with_context(vcir, None) + } + + pub fn from_vcir_with_context( + vcir: &ValidatedCaInstanceResult, + context: Option<&RoaCacheProjectionContext>, + ) -> StorageResult> { + let Some(context) = context else { + return Ok(None); + }; + let mut issuer_ca_sha256_hex = None; + let mut crl_sha256_by_uri = Vec::new(); + for artifact in &vcir.related_artifacts { + if artifact.validation_status != VcirArtifactValidationStatus::Accepted { + continue; + } + match (artifact.artifact_role, artifact.artifact_kind) { + ( + VcirArtifactRole::IssuerCert | VcirArtifactRole::TrustAnchorCert, + VcirArtifactKind::Cer, + ) => { + issuer_ca_sha256_hex = Some(artifact.sha256.clone()); + } + (_, VcirArtifactKind::Crl) => { + if let Some(uri) = artifact.uri.as_ref() { + crl_sha256_by_uri.push(RoaCacheCrlProjection { + uri: uri.clone(), + sha256: artifact.sha256.clone(), + }); + } + } + _ => {} + } + } + crl_sha256_by_uri.sort_by(|left, right| left.uri.cmp(&right.uri)); + + let meta_by_uri = context + .object_meta + .iter() + .map(|meta| (meta.source_object_uri.as_str(), meta)) + .collect::>(); + let vcir_validation_time = + vcir.last_successful_validation_time + .parse() + .map_err(|detail| StorageError::InvalidData { + entity: "roa_cache_projection.entries[].earliest_safe_reuse_time_unix", + detail, + })?; + let mut entries: Vec = Vec::new(); + let mut entry_index_by_uri: HashMap = HashMap::new(); + for output in &vcir.local_outputs { + let Some(projected_output) = RoaCacheLocalOutputProjection::from_local_output(output) + else { + continue; + }; + let projected_output_effective_until = projected_output + .item_effective_until + .parse() + .map(|time| time.unix_timestamp()) + .map_err(|detail| StorageError::InvalidData { + entity: "roa_cache_projection.entries[].outputs_effective_until_unix", + detail, + })?; + let Some(meta) = meta_by_uri.get(output.source_object_uri.as_str()).copied() else { + continue; + }; + if meta.source_object_hash != output.source_object_hash { + return Err(StorageError::InvalidData { + entity: "roa_cache_projection.entries[]", + detail: format!( + "metadata source object hash mismatch for {}", + output.source_object_uri + ), + }); + } + let earliest_safe_reuse_time_unix = meta + .earliest_safe_reuse_time + .parse() + .map(|time| time.max(vcir_validation_time).unix_timestamp()) + .map_err(|detail| StorageError::InvalidData { + entity: "roa_cache_projection.entries[].earliest_safe_reuse_time_unix", + detail, + })?; + if let Some(entry_index) = entry_index_by_uri.get(output.source_object_uri.as_str()) { + let entry = &mut entries[*entry_index]; + if entry.source_object_hash != output.source_object_hash { + return Err(StorageError::InvalidData { + entity: "roa_cache_projection.entries[]", + detail: format!( + "source object hash mismatch for {}", + output.source_object_uri + ), + }); + } + entry.outputs_effective_until_unix = entry + .outputs_effective_until_unix + .min(projected_output_effective_until); + entry.outputs.push(projected_output); + } else { + entry_index_by_uri.insert(output.source_object_uri.clone(), entries.len()); + entries.push(RoaCacheObjectProjection { + source_object_uri: output.source_object_uri.clone(), + source_object_hash: output.source_object_hash, + ee_serial: Some(meta.ee_serial.clone()), + crl_uri: Some(meta.crl_uri.clone()), + earliest_safe_reuse_time_unix: Some(earliest_safe_reuse_time_unix), + outputs_effective_until_unix: projected_output_effective_until, + outputs: vec![projected_output], + }); + } + } + if entries.is_empty() { + return Ok(None); + } + entries.sort_by(|left, right| left.source_object_uri.cmp(&right.source_object_uri)); + + let projection = Self { + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + instance_effective_until: vcir.instance_gate.instance_effective_until.clone(), + issuer_ca_sha256_hex, + ca_validation_context_digest: Some(context.ca_validation_context_digest), + policy_fingerprint: Some(context.policy_fingerprint), + crl_sha256_by_uri, + entries, + }; + projection.validate_internal()?; + Ok(Some(projection)) + } + + pub fn validate_internal(&self) -> StorageResult<()> { + validate_non_empty( + "roa_cache_projection.manifest_rsync_uri", + &self.manifest_rsync_uri, + )?; + parse_time( + "roa_cache_projection.instance_effective_until", + &self.instance_effective_until, + )?; + if let Some(hash) = &self.issuer_ca_sha256_hex { + validate_sha256_hex("roa_cache_projection.issuer_ca_sha256_hex", hash)?; + } + if self.ca_validation_context_digest.is_some() != self.policy_fingerprint.is_some() { + return Err(StorageError::InvalidData { + entity: "roa_cache_projection.context", + detail: "ca_validation_context_digest and policy_fingerprint must be both present or both absent" + .to_string(), + }); + } + let mut seen_crls = HashSet::with_capacity(self.crl_sha256_by_uri.len()); + for crl in &self.crl_sha256_by_uri { + crl.validate_internal()?; + if !seen_crls.insert(crl.uri.as_str()) { + return Err(StorageError::InvalidData { + entity: "roa_cache_projection.crls[]", + detail: format!("duplicate CRL URI: {}", crl.uri), + }); + } + } + if self.entries.is_empty() { + return Err(StorageError::InvalidData { + entity: "roa_cache_projection.entries", + detail: "must not be empty".to_string(), + }); + } + let mut seen_entries = HashSet::with_capacity(self.entries.len()); + for entry in &self.entries { + entry.validate_internal()?; + if !seen_entries.insert(entry.source_object_uri.as_str()) { + return Err(StorageError::InvalidData { + entity: "roa_cache_projection.entries[]", + detail: format!("duplicate ROA URI: {}", entry.source_object_uri), + }); + } + } + Ok(()) + } +} diff --git a/crates/panda-rpki-validator/src/storage/models_summary.rs b/crates/panda-rpki-validator/src/storage/models_summary.rs new file mode 100644 index 0000000..ce95f44 --- /dev/null +++ b/crates/panda-rpki-validator/src/storage/models_summary.rs @@ -0,0 +1,730 @@ +// VCIR summaries, RRDP records, and validation metadata. + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum VcirArtifactRole { + Manifest, + CurrentCrl, + ChildCaCert, + SignedObject, + EeCert, + IssuerCert, + Tal, + TrustAnchorCert, + Other, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum VcirArtifactKind { + Cer, + Crl, + Mft, + Roa, + Aspa, + Gbr, + Tal, + Other, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum VcirArtifactValidationStatus { + Accepted, + Rejected, + WarningOnly, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct VcirRelatedArtifact { + #[serde(rename = "r")] + pub artifact_role: VcirArtifactRole, + #[serde(rename = "k")] + pub artifact_kind: VcirArtifactKind, + #[serde(rename = "u")] + pub uri: Option, + #[serde(rename = "h")] + pub sha256: String, + #[serde(rename = "t")] + pub object_type: Option, + #[serde(rename = "s")] + pub validation_status: VcirArtifactValidationStatus, + /// Reject reason captured at fresh validation time for `Rejected` artifacts. + /// `None` for accepted/warning artifacts and for cache entries written + /// before this field existed. + #[serde(rename = "e", default, skip_serializing_if = "Option::is_none")] + pub reject_reason: Option, +} + +impl VcirRelatedArtifact { + pub fn validate_internal(&self) -> StorageResult<()> { + if let Some(uri) = &self.uri { + validate_non_empty("vcir.related_artifacts[].uri", uri)?; + } + validate_sha256_hex("vcir.related_artifacts[].sha256", &self.sha256)?; + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct VcirSummary { + #[serde(rename = "v")] + pub local_vrp_count: u32, + #[serde(rename = "a")] + pub local_aspa_count: u32, + #[serde(rename = "r")] + pub local_router_key_count: u32, + #[serde(rename = "c")] + pub child_count: u32, + #[serde(rename = "o")] + pub accepted_object_count: u32, + #[serde(rename = "x")] + pub rejected_object_count: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct VcirAuditSummary { + #[serde(rename = "f")] + pub failed_fetch_eligible: bool, + #[serde(rename = "r")] + pub last_failed_fetch_reason: Option, + #[serde(rename = "w")] + pub warning_count: u32, + #[serde(rename = "a")] + pub audit_flags: Vec, +} + +impl VcirAuditSummary { + pub fn validate_internal(&self) -> StorageResult<()> { + if let Some(reason) = &self.last_failed_fetch_reason { + validate_non_empty("vcir.audit_summary.last_failed_fetch_reason", reason)?; + } + for flag in &self.audit_flags { + validate_non_empty("vcir.audit_summary.audit_flags[]", flag)?; + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ValidatedCaInstanceResult { + #[serde(rename = "m")] + pub manifest_rsync_uri: String, + #[serde(rename = "pm")] + pub parent_manifest_rsync_uri: Option, + #[serde(rename = "tal")] + pub tal_id: String, + #[serde(rename = "subj")] + pub ca_subject_name: String, + #[serde(rename = "ski")] + pub ca_ski: String, + #[serde(rename = "aki")] + pub issuer_ski: String, + #[serde(rename = "vt")] + pub last_successful_validation_time: PackTime, + #[serde(rename = "cm")] + pub current_manifest_rsync_uri: String, + #[serde(rename = "crl")] + pub current_crl_rsync_uri: String, + #[serde(rename = "mm")] + pub validated_manifest_meta: ValidatedManifestMeta, + #[serde(rename = "ccr")] + pub ccr_manifest_projection: VcirCcrManifestProjection, + #[serde(rename = "g")] + pub instance_gate: VcirInstanceGate, + #[serde(rename = "ch")] + pub child_entries: Vec, + #[serde(rename = "lo")] + pub local_outputs: Vec, + #[serde(rename = "ra")] + pub related_artifacts: Vec, + #[serde(rename = "s")] + pub summary: VcirSummary, + #[serde(rename = "as")] + pub audit_summary: VcirAuditSummary, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct VcirFieldSizeBreakdown { + pub local_output_count: u64, + pub local_output_source_uri_bytes: u64, + pub local_output_source_type_bytes: u64, + pub local_output_source_hash_hex_bytes: u64, + pub local_output_source_ee_hash_hex_bytes: u64, + pub local_output_payload_json_bytes: u64, + pub local_output_rule_hash_hex_bytes: u64, + pub local_output_source_hash_binary_bytes: u64, + pub local_output_source_ee_hash_binary_bytes: u64, + pub local_output_payload_typed_body_bytes: u64, + pub local_output_rule_hash_binary_bytes: u64, + pub related_artifact_count: u64, + pub related_artifact_uri_bytes: u64, + pub related_artifact_hash_hex_bytes: u64, + pub related_artifact_type_bytes: u64, + pub child_entry_count: u64, + pub child_entry_uri_bytes: u64, + pub child_entry_hash_hex_bytes: u64, +} + +fn serialized_cbor_len(value: &T) -> u64 { + serde_cbor::to_vec(value) + .map(|bytes| bytes.len() as u64) + .unwrap_or(0) +} + +impl VcirFieldSizeBreakdown { + pub fn from_vcir(vcir: &ValidatedCaInstanceResult) -> Self { + let mut out = Self::default(); + out.local_output_count = vcir.local_outputs.len() as u64; + for local in &vcir.local_outputs { + out.local_output_source_uri_bytes += local.source_object_uri.len() as u64; + out.local_output_source_type_bytes += local.source_object_type_name().len() as u64; + out.local_output_source_hash_hex_bytes += 64; + out.local_output_source_ee_hash_hex_bytes += 64; + out.local_output_payload_json_bytes += local.payload_json().len() as u64; + out.local_output_rule_hash_hex_bytes += 64; + out.local_output_source_hash_binary_bytes += 32; + out.local_output_source_ee_hash_binary_bytes += 32; + out.local_output_payload_typed_body_bytes += local.payload.typed_body_bytes(); + out.local_output_rule_hash_binary_bytes += 32; + } + out.related_artifact_count = vcir.related_artifacts.len() as u64; + for artifact in &vcir.related_artifacts { + if let Some(uri) = &artifact.uri { + out.related_artifact_uri_bytes += uri.len() as u64; + } + out.related_artifact_hash_hex_bytes += artifact.sha256.len() as u64; + if let Some(object_type) = &artifact.object_type { + out.related_artifact_type_bytes += object_type.len() as u64; + } + } + out.child_entry_count = vcir.child_entries.len() as u64; + for child in &vcir.child_entries { + out.child_entry_uri_bytes += child.child_manifest_rsync_uri.len() as u64 + + child.child_cert_rsync_uri.len() as u64 + + child.child_rsync_base_uri.len() as u64 + + child.child_publication_point_rsync_uri.len() as u64 + + child + .child_rrdp_notification_uri + .as_ref() + .map(|uri| uri.len() as u64) + .unwrap_or(0); + out.child_entry_hash_hex_bytes += + child.child_cert_hash.len() as u64 + child.child_ski.len() as u64; + } + out + } + + pub fn add_assign(&mut self, other: &Self) { + self.local_output_count += other.local_output_count; + self.local_output_source_uri_bytes += other.local_output_source_uri_bytes; + self.local_output_source_type_bytes += other.local_output_source_type_bytes; + self.local_output_source_hash_hex_bytes += other.local_output_source_hash_hex_bytes; + self.local_output_source_ee_hash_hex_bytes += other.local_output_source_ee_hash_hex_bytes; + self.local_output_payload_json_bytes += other.local_output_payload_json_bytes; + self.local_output_rule_hash_hex_bytes += other.local_output_rule_hash_hex_bytes; + self.local_output_source_hash_binary_bytes += other.local_output_source_hash_binary_bytes; + self.local_output_source_ee_hash_binary_bytes += + other.local_output_source_ee_hash_binary_bytes; + self.local_output_payload_typed_body_bytes += other.local_output_payload_typed_body_bytes; + self.local_output_rule_hash_binary_bytes += other.local_output_rule_hash_binary_bytes; + self.related_artifact_count += other.related_artifact_count; + self.related_artifact_uri_bytes += other.related_artifact_uri_bytes; + self.related_artifact_hash_hex_bytes += other.related_artifact_hash_hex_bytes; + self.related_artifact_type_bytes += other.related_artifact_type_bytes; + self.child_entry_count += other.child_entry_count; + self.child_entry_uri_bytes += other.child_entry_uri_bytes; + self.child_entry_hash_hex_bytes += other.child_entry_hash_hex_bytes; + } + + pub fn local_output_old_projection_bytes(&self) -> u64 { + self.local_output_source_type_bytes + + self.local_output_source_hash_hex_bytes + + self.local_output_source_ee_hash_hex_bytes + + self.local_output_payload_json_bytes + + self.local_output_rule_hash_hex_bytes + } + + pub fn local_output_typed_projection_bytes(&self) -> u64 { + self.local_output_count + + self.local_output_source_hash_binary_bytes + + self.local_output_source_ee_hash_binary_bytes + + self.local_output_payload_typed_body_bytes + + self.local_output_rule_hash_binary_bytes + } + + pub fn local_output_projection_saved_bytes(&self) -> u64 { + self.local_output_old_projection_bytes() + .saturating_sub(self.local_output_typed_projection_bytes()) + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct VcirReplaceTimingBreakdown { + pub validate_ms: u64, + pub vcir_encode_ms: u64, + pub vcir_value_bytes: u64, + pub replay_meta_encode_ms: u64, + pub replay_meta_value_bytes: u64, + pub roa_cache_projection_encode_ms: u64, + pub roa_cache_projection_value_bytes: u64, + pub publication_point_cache_projection_encode_ms: u64, + pub publication_point_cache_projection_value_bytes: u64, + pub batch_build_ms: u64, + pub write_batch_ms: u64, + pub total_encoded_bytes: u64, + pub field_sizes: VcirFieldSizeBreakdown, + pub rss_before_kb: Option, + pub rss_after_validate_kb: Option, + pub rss_after_vcir_encode_kb: Option, + pub rss_after_replay_meta_encode_kb: Option, + pub rss_after_roa_cache_projection_encode_kb: Option, + pub rss_after_publication_point_cache_projection_encode_kb: Option, + pub rss_after_write_batch_kb: Option, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct VcirStorageSummary { + pub entry_count: u64, + pub vcir_value_bytes: u64, + pub vcir_value_bytes_max: u64, + pub vcir_value_bytes_max_manifest_rsync_uri: Option, + pub core_fields: VcirCoreFieldSizeBreakdown, + pub ccr_projection: VcirCcrProjectionSizeBreakdown, + pub child_resources: VcirChildResourceSizeBreakdown, + pub field_sizes: VcirFieldSizeBreakdown, + pub local_output_old_projection_bytes: u64, + pub local_output_typed_projection_bytes: u64, + pub local_output_projection_saved_bytes: u64, + pub top_entries_by_vcir_value_bytes: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct VcirStorageEntrySummary { + pub manifest_rsync_uri: String, + pub vcir_value_bytes: u64, + pub local_vrp_count: u32, + pub local_aspa_count: u32, + pub local_router_key_count: u32, + pub accepted_object_count: u32, + pub rejected_object_count: u32, + pub child_count: u32, + pub core_fields: VcirCoreFieldSizeBreakdown, + pub ccr_projection: VcirCcrProjectionSizeBreakdown, + pub child_resources: VcirChildResourceSizeBreakdown, + pub field_sizes: VcirFieldSizeBreakdown, + pub local_output_old_projection_bytes: u64, + pub local_output_typed_projection_bytes: u64, + pub local_output_projection_saved_bytes: u64, +} + +impl VcirStorageEntrySummary { + fn from_vcir(vcir: &ValidatedCaInstanceResult, vcir_value_bytes: u64) -> Self { + let field_sizes = VcirFieldSizeBreakdown::from_vcir(vcir); + Self { + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + vcir_value_bytes, + local_vrp_count: vcir.summary.local_vrp_count, + local_aspa_count: vcir.summary.local_aspa_count, + local_router_key_count: vcir.summary.local_router_key_count, + accepted_object_count: vcir.summary.accepted_object_count, + rejected_object_count: vcir.summary.rejected_object_count, + child_count: vcir.summary.child_count, + core_fields: VcirCoreFieldSizeBreakdown::from_vcir(vcir), + ccr_projection: VcirCcrProjectionSizeBreakdown::from_projection( + &vcir.ccr_manifest_projection, + ), + child_resources: VcirChildResourceSizeBreakdown::from_vcir(vcir), + local_output_old_projection_bytes: field_sizes.local_output_old_projection_bytes(), + local_output_typed_projection_bytes: field_sizes.local_output_typed_projection_bytes(), + local_output_projection_saved_bytes: field_sizes.local_output_projection_saved_bytes(), + field_sizes, + } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct VcirCoreFieldSizeBreakdown { + pub manifest_rsync_uri_bytes: u64, + pub parent_manifest_rsync_uri_bytes: u64, + pub tal_id_bytes: u64, + pub ca_subject_name_bytes: u64, + pub ca_ski_bytes: u64, + pub issuer_ski_bytes: u64, + pub current_manifest_rsync_uri_bytes: u64, + pub current_crl_rsync_uri_bytes: u64, + pub validated_manifest_number_bytes: u64, +} + +impl VcirCoreFieldSizeBreakdown { + fn from_vcir(vcir: &ValidatedCaInstanceResult) -> Self { + Self { + manifest_rsync_uri_bytes: vcir.manifest_rsync_uri.len() as u64, + parent_manifest_rsync_uri_bytes: vcir + .parent_manifest_rsync_uri + .as_ref() + .map(|uri| uri.len() as u64) + .unwrap_or(0), + tal_id_bytes: vcir.tal_id.len() as u64, + ca_subject_name_bytes: vcir.ca_subject_name.len() as u64, + ca_ski_bytes: vcir.ca_ski.len() as u64, + issuer_ski_bytes: vcir.issuer_ski.len() as u64, + current_manifest_rsync_uri_bytes: vcir.current_manifest_rsync_uri.len() as u64, + current_crl_rsync_uri_bytes: vcir.current_crl_rsync_uri.len() as u64, + validated_manifest_number_bytes: vcir + .validated_manifest_meta + .validated_manifest_number + .len() as u64, + } + } + + fn add_assign(&mut self, other: &Self) { + self.manifest_rsync_uri_bytes += other.manifest_rsync_uri_bytes; + self.parent_manifest_rsync_uri_bytes += other.parent_manifest_rsync_uri_bytes; + self.tal_id_bytes += other.tal_id_bytes; + self.ca_subject_name_bytes += other.ca_subject_name_bytes; + self.ca_ski_bytes += other.ca_ski_bytes; + self.issuer_ski_bytes += other.issuer_ski_bytes; + self.current_manifest_rsync_uri_bytes += other.current_manifest_rsync_uri_bytes; + self.current_crl_rsync_uri_bytes += other.current_crl_rsync_uri_bytes; + self.validated_manifest_number_bytes += other.validated_manifest_number_bytes; + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct VcirChildResourceSizeBreakdown { + pub effective_ip_resource_cbor_bytes: u64, + pub effective_as_resource_cbor_bytes: u64, +} + +impl VcirChildResourceSizeBreakdown { + fn from_vcir(vcir: &ValidatedCaInstanceResult) -> Self { + let mut out = Self::default(); + for child in &vcir.child_entries { + out.effective_ip_resource_cbor_bytes += + serialized_cbor_len(&child.child_effective_ip_resources); + out.effective_as_resource_cbor_bytes += + serialized_cbor_len(&child.child_effective_as_resources); + } + out + } + + fn add_assign(&mut self, other: &Self) { + self.effective_ip_resource_cbor_bytes += other.effective_ip_resource_cbor_bytes; + self.effective_as_resource_cbor_bytes += other.effective_as_resource_cbor_bytes; + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct VcirCcrProjectionSizeBreakdown { + pub manifest_rsync_uri_bytes: u64, + pub manifest_sha256_bytes: u64, + pub manifest_ee_aki_bytes: u64, + pub manifest_number_bytes: u64, + pub manifest_sia_locations_count: u64, + pub manifest_sia_locations_der_bytes: u64, + pub subordinate_ski_count: u64, + pub subordinate_ski_bytes: u64, +} + +impl VcirCcrProjectionSizeBreakdown { + fn from_projection(projection: &VcirCcrManifestProjection) -> Self { + Self { + manifest_rsync_uri_bytes: projection.manifest_rsync_uri.len() as u64, + manifest_sha256_bytes: projection.manifest_sha256.len() as u64, + manifest_ee_aki_bytes: projection.manifest_ee_aki.len() as u64, + manifest_number_bytes: projection.manifest_number_be.len() as u64, + manifest_sia_locations_count: projection.manifest_sia_locations_der.len() as u64, + manifest_sia_locations_der_bytes: projection + .manifest_sia_locations_der + .iter() + .map(|location| location.len() as u64) + .sum(), + subordinate_ski_count: projection.subordinate_skis.len() as u64, + subordinate_ski_bytes: projection + .subordinate_skis + .iter() + .map(|ski| ski.len() as u64) + .sum(), + } + } + + fn add_assign(&mut self, other: &Self) { + self.manifest_rsync_uri_bytes += other.manifest_rsync_uri_bytes; + self.manifest_sha256_bytes += other.manifest_sha256_bytes; + self.manifest_ee_aki_bytes += other.manifest_ee_aki_bytes; + self.manifest_number_bytes += other.manifest_number_bytes; + self.manifest_sia_locations_count += other.manifest_sia_locations_count; + self.manifest_sia_locations_der_bytes += other.manifest_sia_locations_der_bytes; + self.subordinate_ski_count += other.subordinate_ski_count; + self.subordinate_ski_bytes += other.subordinate_ski_bytes; + } +} + +fn push_top_vcir_storage_entry( + entries: &mut Vec, + entry: VcirStorageEntrySummary, +) { + const TOP_N: usize = 20; + entries.push(entry); + entries.sort_by(|left, right| { + right + .vcir_value_bytes + .cmp(&left.vcir_value_bytes) + .then_with(|| left.manifest_rsync_uri.cmp(&right.manifest_rsync_uri)) + }); + entries.truncate(TOP_N); +} + +impl ValidatedCaInstanceResult { + pub fn validate_internal(&self) -> StorageResult<()> { + validate_non_empty("vcir.manifest_rsync_uri", &self.manifest_rsync_uri)?; + if let Some(parent_manifest_rsync_uri) = &self.parent_manifest_rsync_uri { + validate_non_empty("vcir.parent_manifest_rsync_uri", parent_manifest_rsync_uri)?; + } + validate_non_empty("vcir.tal_id", &self.tal_id)?; + validate_non_empty("vcir.ca_subject_name", &self.ca_subject_name)?; + validate_non_empty("vcir.ca_ski", &self.ca_ski)?; + validate_non_empty("vcir.issuer_ski", &self.issuer_ski)?; + parse_time( + "vcir.last_successful_validation_time", + &self.last_successful_validation_time, + )?; + validate_non_empty( + "vcir.current_manifest_rsync_uri", + &self.current_manifest_rsync_uri, + )?; + validate_non_empty("vcir.current_crl_rsync_uri", &self.current_crl_rsync_uri)?; + self.validated_manifest_meta.validate_internal()?; + self.ccr_manifest_projection.validate_internal()?; + self.instance_gate.validate_internal()?; + + let expected_manifest_next = self + .validated_manifest_meta + .validated_manifest_next_update + .parse() + .map_err(|detail| StorageError::InvalidData { + entity: "vcir", + detail: format!( + "validated_manifest_meta.validated_manifest_next_update invalid: {detail}" + ), + })?; + let instance_manifest_next = + self.instance_gate + .manifest_next_update + .parse() + .map_err(|detail| StorageError::InvalidData { + entity: "vcir", + detail: format!("instance_gate.manifest_next_update invalid: {detail}"), + })?; + if expected_manifest_next != instance_manifest_next { + return Err(StorageError::InvalidData { + entity: "vcir", + detail: "instance_gate.manifest_next_update must equal validated_manifest_meta.validated_manifest_next_update".to_string(), + }); + } + + let mut child_manifests = HashSet::with_capacity(self.child_entries.len()); + for child in &self.child_entries { + child.validate_internal()?; + if !child_manifests.insert(child.child_manifest_rsync_uri.as_str()) { + return Err(StorageError::InvalidData { + entity: "vcir", + detail: format!( + "duplicate child_manifest_rsync_uri: {}", + child.child_manifest_rsync_uri + ), + }); + } + } + + let mut vrp_count = 0u32; + let mut aspa_count = 0u32; + let mut router_key_count = 0u32; + for output in &self.local_outputs { + output.validate_internal()?; + match output.output_type { + VcirOutputType::Vrp => vrp_count += 1, + VcirOutputType::Aspa => aspa_count += 1, + VcirOutputType::RouterKey => router_key_count += 1, + } + } + let mut output_ids = self + .local_outputs + .iter() + .map(VcirLocalOutput::output_id) + .collect::>(); + output_ids.sort_unstable(); + if let Some(duplicate) = output_ids + .windows(2) + .find_map(|pair| (pair[0] == pair[1]).then(|| pair[0].clone())) + { + return Err(StorageError::InvalidData { + entity: "vcir", + detail: format!("duplicate output_id: {duplicate}"), + }); + } + if self.summary.local_vrp_count != vrp_count { + return Err(StorageError::InvalidData { + entity: "vcir.summary", + detail: format!( + "local_vrp_count={} does not match local_outputs count {}", + self.summary.local_vrp_count, vrp_count + ), + }); + } + if self.summary.local_aspa_count != aspa_count { + return Err(StorageError::InvalidData { + entity: "vcir.summary", + detail: format!( + "local_aspa_count={} does not match local_outputs count {}", + self.summary.local_aspa_count, aspa_count + ), + }); + } + if self.summary.local_router_key_count != router_key_count { + return Err(StorageError::InvalidData { + entity: "vcir.summary", + detail: format!( + "local_router_key_count={} does not match local_outputs count {}", + self.summary.local_router_key_count, router_key_count + ), + }); + } + if self.summary.child_count != self.child_entries.len() as u32 { + return Err(StorageError::InvalidData { + entity: "vcir.summary", + detail: format!( + "child_count={} does not match child_entries length {}", + self.summary.child_count, + self.child_entries.len() + ), + }); + } + + for artifact in &self.related_artifacts { + artifact.validate_internal()?; + } + self.audit_summary.validate_internal()?; + Ok(()) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RrdpSourceSyncState { + Empty, + SnapshotOnly, + DeltaReady, + Error, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RrdpSourceRecord { + pub notify_uri: String, + pub last_session_id: Option, + pub last_serial: Option, + pub first_seen_at: PackTime, + pub last_seen_at: PackTime, + pub last_sync_at: Option, + pub sync_state: RrdpSourceSyncState, + pub last_snapshot_uri: Option, + pub last_snapshot_hash: Option, + pub last_error: Option, +} + +impl RrdpSourceRecord { + pub fn validate_internal(&self) -> StorageResult<()> { + validate_non_empty("rrdp_source.notify_uri", &self.notify_uri)?; + if let Some(session_id) = &self.last_session_id { + validate_non_empty("rrdp_source.last_session_id", session_id)?; + } + parse_time("rrdp_source.first_seen_at", &self.first_seen_at)?; + parse_time("rrdp_source.last_seen_at", &self.last_seen_at)?; + if let Some(last_sync_at) = &self.last_sync_at { + parse_time("rrdp_source.last_sync_at", last_sync_at)?; + } + if let Some(last_snapshot_uri) = &self.last_snapshot_uri { + validate_non_empty("rrdp_source.last_snapshot_uri", last_snapshot_uri)?; + } + if let Some(last_snapshot_hash) = &self.last_snapshot_hash { + validate_sha256_hex("rrdp_source.last_snapshot_hash", last_snapshot_hash)?; + } + if let Some(last_error) = &self.last_error { + validate_non_empty("rrdp_source.last_error", last_error)?; + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RrdpSourceMemberRecord { + pub notify_uri: String, + pub rsync_uri: String, + pub current_hash: Option, + pub object_type: Option, + pub present: bool, + pub last_confirmed_session_id: String, + pub last_confirmed_serial: u64, + pub last_changed_at: PackTime, +} + +impl RrdpSourceMemberRecord { + pub fn validate_internal(&self) -> StorageResult<()> { + validate_non_empty("rrdp_source_member.notify_uri", &self.notify_uri)?; + validate_non_empty("rrdp_source_member.rsync_uri", &self.rsync_uri)?; + validate_non_empty( + "rrdp_source_member.last_confirmed_session_id", + &self.last_confirmed_session_id, + )?; + if self.present { + let hash = self + .current_hash + .as_deref() + .ok_or(StorageError::InvalidData { + entity: "rrdp_source_member", + detail: "current_hash is required when present=true".to_string(), + })?; + validate_sha256_hex("rrdp_source_member.current_hash", hash)?; + } else if let Some(hash) = &self.current_hash { + validate_sha256_hex("rrdp_source_member.current_hash", hash)?; + } + parse_time("rrdp_source_member.last_changed_at", &self.last_changed_at)?; + Ok(()) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RrdpUriOwnerState { + Active, + Conflict, + Withdrawn, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RrdpUriOwnerRecord { + pub rsync_uri: String, + pub notify_uri: String, + pub current_hash: Option, + pub last_confirmed_session_id: String, + pub last_confirmed_serial: u64, + pub last_changed_at: PackTime, + pub owner_state: RrdpUriOwnerState, +} + +impl RrdpUriOwnerRecord { + pub fn validate_internal(&self) -> StorageResult<()> { + validate_non_empty("rrdp_uri_owner.rsync_uri", &self.rsync_uri)?; + validate_non_empty("rrdp_uri_owner.notify_uri", &self.notify_uri)?; + validate_non_empty( + "rrdp_uri_owner.last_confirmed_session_id", + &self.last_confirmed_session_id, + )?; + if let Some(hash) = &self.current_hash { + validate_sha256_hex("rrdp_uri_owner.current_hash", hash)?; + } + parse_time("rrdp_uri_owner.last_changed_at", &self.last_changed_at)?; + Ok(()) + } +} diff --git a/crates/panda-rpki-validator/src/storage/models_vcir.rs b/crates/panda-rpki-validator/src/storage/models_vcir.rs new file mode 100644 index 0000000..6fb5d2a --- /dev/null +++ b/crates/panda-rpki-validator/src/storage/models_vcir.rs @@ -0,0 +1,548 @@ +// VCIR local-output and child-certificate cache projections. + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum VcirLocalOutputPayload { + Vrp { + asn: u32, + afi: crate::data_model::roa::RoaAfi, + prefix_len: u16, + #[serde(with = "serde_bytes_16")] + addr: [u8; 16], + max_length: u16, + }, + Aspa { + customer_as_id: u32, + provider_as_ids: Vec, + }, + RouterKey { + as_id: u32, + #[serde(with = "serde_byte_vec")] + ski: Vec, + #[serde(with = "serde_byte_vec")] + spki_der: Vec, + }, +} + +impl VcirLocalOutputPayload { + pub fn typed_body_bytes(&self) -> u64 { + match self { + Self::Vrp { .. } => 4 + 1 + 2 + 16 + 2, + Self::Aspa { + provider_as_ids, .. + } => 4 + (provider_as_ids.len() as u64 * 4), + Self::RouterKey { ski, spki_der, .. } => 4 + ski.len() as u64 + spki_der.len() as u64, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct VcirLocalOutput { + #[serde(rename = "t")] + pub output_type: VcirOutputType, + #[serde(rename = "e")] + pub item_effective_until: PackTime, + #[serde(rename = "u")] + pub source_object_uri: String, + #[serde(rename = "k")] + pub source_object_type: VcirSourceObjectType, + #[serde(rename = "h")] + #[serde(with = "serde_bytes_32")] + pub source_object_hash: [u8; 32], + #[serde(rename = "c")] + #[serde(with = "serde_bytes_32")] + pub source_ee_cert_hash: [u8; 32], + #[serde(rename = "p")] + pub payload: VcirLocalOutputPayload, + #[serde(rename = "r")] + #[serde(with = "serde_bytes_32")] + pub rule_hash: [u8; 32], +} + +impl VcirLocalOutput { + pub fn output_id(&self) -> String { + self.rule_hash_hex() + } + + pub fn source_object_hash_hex(&self) -> String { + hex::encode(self.source_object_hash) + } + + pub fn source_ee_cert_hash_hex(&self) -> String { + hex::encode(self.source_ee_cert_hash) + } + + pub fn rule_hash_hex(&self) -> String { + hex::encode(self.rule_hash) + } + + pub fn source_object_type_name(&self) -> &'static str { + self.source_object_type.as_str() + } + + pub fn payload_json(&self) -> String { + match &self.payload { + VcirLocalOutputPayload::Vrp { + asn, + afi, + prefix_len, + addr, + max_length, + } => { + let prefix = match afi { + crate::data_model::roa::RoaAfi::Ipv4 => { + let ip = std::net::Ipv4Addr::new(addr[0], addr[1], addr[2], addr[3]); + format!("{ip}/{prefix_len}") + } + crate::data_model::roa::RoaAfi::Ipv6 => { + let ip = std::net::Ipv6Addr::from(*addr); + format!("{ip}/{prefix_len}") + } + }; + format!(r#"{{"asn":{asn},"max_length":{max_length},"prefix":"{prefix}"}}"#) + } + VcirLocalOutputPayload::Aspa { + customer_as_id, + provider_as_ids, + } => { + let providers = provider_as_ids + .iter() + .map(u32::to_string) + .collect::>() + .join(","); + format!(r#"{{"customer_as_id":{customer_as_id},"provider_as_ids":[{providers}]}}"#) + } + VcirLocalOutputPayload::RouterKey { + as_id, + ski, + spki_der, + } => { + let ski_hex = hex::encode(ski); + let spki_der_base64 = base64::engine::general_purpose::STANDARD.encode(spki_der); + format!( + r#"{{"as_id":{as_id},"ski_hex":"{ski_hex}","spki_der_base64":"{spki_der_base64}"}}"# + ) + } + } + } + + pub fn validate_internal(&self) -> StorageResult<()> { + parse_time( + "vcir.local_outputs[].item_effective_until", + &self.item_effective_until, + )?; + validate_non_empty( + "vcir.local_outputs[].source_object_uri", + &self.source_object_uri, + )?; + validate_local_output_type_matches_payload(self)?; + Ok(()) + } +} + +fn validate_local_output_type_matches_payload(output: &VcirLocalOutput) -> StorageResult<()> { + let matches_payload = matches!( + (&output.output_type, &output.payload), + (VcirOutputType::Vrp, VcirLocalOutputPayload::Vrp { .. }) + | (VcirOutputType::Aspa, VcirLocalOutputPayload::Aspa { .. }) + | ( + VcirOutputType::RouterKey, + VcirLocalOutputPayload::RouterKey { .. } + ) + ); + if !matches_payload { + return Err(StorageError::InvalidData { + entity: "vcir.local_outputs[]", + detail: "output_type must match payload variant".to_string(), + }); + } + Ok(()) +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RoaCacheCrlProjection { + #[serde(rename = "u")] + pub uri: String, + #[serde(rename = "h")] + pub sha256: String, +} + +impl RoaCacheCrlProjection { + pub fn validate_internal(&self) -> StorageResult<()> { + validate_non_empty("roa_cache_projection.crls[].uri", &self.uri)?; + validate_sha256_hex("roa_cache_projection.crls[].sha256", &self.sha256)?; + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RoaCacheObjectMeta { + pub source_object_uri: String, + pub source_object_hash: [u8; 32], + pub ee_serial: Vec, + pub crl_uri: String, + pub earliest_safe_reuse_time: PackTime, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RoaCacheProjectionContext { + pub ca_validation_context_digest: [u8; 32], + pub policy_fingerprint: [u8; 32], + pub object_meta: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RoaCacheLocalOutputProjection { + #[serde(rename = "e")] + pub item_effective_until: PackTime, + #[serde(rename = "c")] + #[serde(with = "serde_bytes_32")] + pub source_ee_cert_hash: [u8; 32], + #[serde(rename = "p")] + pub payload: VcirLocalOutputPayload, + #[serde(rename = "r")] + #[serde(with = "serde_bytes_32")] + pub rule_hash: [u8; 32], +} + +impl RoaCacheLocalOutputProjection { + fn from_local_output(output: &VcirLocalOutput) -> Option { + if output.output_type != VcirOutputType::Vrp + || output.source_object_type != VcirSourceObjectType::Roa + { + return None; + } + Some(Self { + item_effective_until: output.item_effective_until.clone(), + source_ee_cert_hash: output.source_ee_cert_hash, + payload: output.payload.clone(), + rule_hash: output.rule_hash, + }) + } + + pub fn validate_internal(&self) -> StorageResult<()> { + parse_time( + "roa_cache_projection.entries[].outputs[].item_effective_until", + &self.item_effective_until, + )?; + if !matches!(self.payload, VcirLocalOutputPayload::Vrp { .. }) { + return Err(StorageError::InvalidData { + entity: "roa_cache_projection.entries[].outputs[]", + detail: "payload must be VRP".to_string(), + }); + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RoaCacheObjectProjection { + #[serde(rename = "u")] + pub source_object_uri: String, + #[serde(rename = "h")] + #[serde(with = "serde_bytes_32")] + pub source_object_hash: [u8; 32], + #[serde(rename = "s", default, skip_serializing_if = "Option::is_none")] + #[serde(with = "serde_optional_byte_vec")] + pub ee_serial: Option>, + #[serde(rename = "c", default, skip_serializing_if = "Option::is_none")] + pub crl_uri: Option, + #[serde(rename = "n", default, skip_serializing_if = "Option::is_none")] + pub earliest_safe_reuse_time_unix: Option, + #[serde(rename = "x")] + pub outputs_effective_until_unix: i64, + #[serde(rename = "o")] + pub outputs: Vec, +} + +impl RoaCacheObjectProjection { + pub fn validate_internal(&self) -> StorageResult<()> { + validate_non_empty( + "roa_cache_projection.entries[].source_object_uri", + &self.source_object_uri, + )?; + if let Some(serial) = &self.ee_serial { + if serial.is_empty() { + return Err(StorageError::InvalidData { + entity: "roa_cache_projection.entries[].ee_serial", + detail: "must not be empty when present".to_string(), + }); + } + } + if let Some(crl_uri) = &self.crl_uri { + validate_non_empty("roa_cache_projection.entries[].crl_uri", crl_uri)?; + } + if let Some(earliest_safe_reuse_time_unix) = self.earliest_safe_reuse_time_unix + && earliest_safe_reuse_time_unix >= self.outputs_effective_until_unix + { + return Err(StorageError::InvalidData { + entity: "roa_cache_projection.entries[].effective_window", + detail: "earliest_safe_reuse_time_unix must be before outputs_effective_until_unix" + .to_string(), + }); + } + if self.outputs.is_empty() { + return Err(StorageError::InvalidData { + entity: "roa_cache_projection.entries[]", + detail: "outputs must not be empty".to_string(), + }); + } + for output in &self.outputs { + output.validate_internal()?; + } + let expected_effective_until = self + .outputs + .iter() + .map(|output| { + output + .item_effective_until + .parse() + .map(|time| time.unix_timestamp()) + .map_err(|detail| StorageError::InvalidData { + entity: "roa_cache_projection.entries[].outputs_effective_until_unix", + detail, + }) + }) + .collect::>>()? + .into_iter() + .min() + .expect("outputs must not be empty"); + if self.outputs_effective_until_unix != expected_effective_until { + return Err(StorageError::InvalidData { + entity: "roa_cache_projection.entries[].outputs_effective_until_unix", + detail: "must equal the earliest output item_effective_until".to_string(), + }); + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RoaCacheProjection { + #[serde(rename = "m")] + pub manifest_rsync_uri: String, + #[serde(rename = "e")] + pub instance_effective_until: PackTime, + #[serde(rename = "i")] + pub issuer_ca_sha256_hex: Option, + #[serde(rename = "p", default, skip_serializing_if = "Option::is_none")] + #[serde(with = "serde_optional_bytes_32")] + pub ca_validation_context_digest: Option<[u8; 32]>, + #[serde(rename = "f", default, skip_serializing_if = "Option::is_none")] + #[serde(with = "serde_optional_bytes_32")] + pub policy_fingerprint: Option<[u8; 32]>, + #[serde(rename = "c")] + pub crl_sha256_by_uri: Vec, + #[serde(rename = "r")] + pub entries: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ChildCertificateCacheRouterKeyProjection { + #[serde(rename = "a")] + pub as_id: u32, + #[serde(rename = "s")] + #[serde(with = "serde_byte_vec")] + pub ski: Vec, + #[serde(rename = "p")] + #[serde(with = "serde_byte_vec")] + pub spki_der: Vec, + #[serde(rename = "e")] + pub item_effective_until: PackTime, +} + +impl ChildCertificateCacheRouterKeyProjection { + pub fn validate_internal(&self) -> StorageResult<()> { + if self.ski.is_empty() { + return Err(StorageError::InvalidData { + entity: "child_certificate_cache_projection.router_keys[].ski", + detail: "must not be empty".to_string(), + }); + } + if self.spki_der.is_empty() { + return Err(StorageError::InvalidData { + entity: "child_certificate_cache_projection.router_keys[].spki_der", + detail: "must not be empty".to_string(), + }); + } + parse_time( + "child_certificate_cache_projection.router_keys[].item_effective_until", + &self.item_effective_until, + )?; + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ChildCertificateCachePayload { + ChildCa { + #[serde(rename = "cm")] + child_manifest_rsync_uri: String, + #[serde(rename = "ski")] + child_ski: String, + #[serde(rename = "rb")] + child_rsync_base_uri: String, + #[serde(rename = "pp")] + child_publication_point_rsync_uri: String, + #[serde(rename = "rn")] + child_rrdp_notification_uri: Option, + #[serde(rename = "ip")] + child_effective_ip_resources: Option, + #[serde(rename = "as")] + child_effective_as_resources: Option, + }, + Router { + #[serde(rename = "r")] + router_keys: Vec, + }, +} + +impl ChildCertificateCachePayload { + pub fn validate_internal(&self) -> StorageResult<()> { + match self { + Self::ChildCa { + child_manifest_rsync_uri, + child_ski, + child_rsync_base_uri, + child_publication_point_rsync_uri, + child_rrdp_notification_uri, + .. + } => { + validate_non_empty( + "child_certificate_cache_projection.child_manifest_rsync_uri", + child_manifest_rsync_uri, + )?; + validate_non_empty("child_certificate_cache_projection.child_ski", child_ski)?; + validate_non_empty( + "child_certificate_cache_projection.child_rsync_base_uri", + child_rsync_base_uri, + )?; + validate_non_empty( + "child_certificate_cache_projection.child_publication_point_rsync_uri", + child_publication_point_rsync_uri, + )?; + if let Some(uri) = child_rrdp_notification_uri { + validate_non_empty( + "child_certificate_cache_projection.child_rrdp_notification_uri", + uri, + )?; + } + Ok(()) + } + Self::Router { router_keys } => { + if router_keys.is_empty() { + return Err(StorageError::InvalidData { + entity: "child_certificate_cache_projection.router_keys", + detail: "must not be empty".to_string(), + }); + } + for router_key in router_keys { + router_key.validate_internal()?; + } + Ok(()) + } + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ChildCertificateCacheProjection { + #[serde(rename = "sv")] + pub schema_version: u32, + #[serde(rename = "av")] + pub algorithm_version: u32, + #[serde(rename = "k")] + pub cache_key_sha256_hex: String, + #[serde(rename = "cu")] + pub child_cert_uri: String, + #[serde(rename = "ch")] + pub child_cert_sha256_hex: String, + #[serde(rename = "cs")] + #[serde(with = "serde_byte_vec")] + pub child_cert_serial: Vec, + #[serde(rename = "ih")] + pub issuer_ca_sha256_hex: String, + #[serde(rename = "cru")] + pub issuer_crl_uri: String, + #[serde(rename = "crh")] + pub issuer_crl_sha256_hex: String, + #[serde(rename = "pc")] + #[serde(with = "serde_bytes_32")] + pub ca_validation_context_digest: [u8; 32], + #[serde(rename = "pf")] + #[serde(with = "serde_bytes_32")] + pub validation_policy_fingerprint: [u8; 32], + #[serde(rename = "nb")] + pub effective_not_before: PackTime, + #[serde(rename = "nu")] + pub effective_until: PackTime, + #[serde(rename = "p")] + pub payload: ChildCertificateCachePayload, +} + +pub const CHILD_CERTIFICATE_CACHE_SCHEMA_VERSION: u32 = 2; +pub const CHILD_CERTIFICATE_CACHE_ALGORITHM_VERSION: u32 = 3; + +impl ChildCertificateCacheProjection { + pub fn validate_internal(&self) -> StorageResult<()> { + if self.schema_version != CHILD_CERTIFICATE_CACHE_SCHEMA_VERSION { + return Err(StorageError::InvalidData { + entity: "child_certificate_cache_projection.schema_version", + detail: format!("unsupported schema_version {}", self.schema_version), + }); + } + if self.algorithm_version != CHILD_CERTIFICATE_CACHE_ALGORITHM_VERSION { + return Err(StorageError::InvalidData { + entity: "child_certificate_cache_projection.algorithm_version", + detail: format!("unsupported algorithm_version {}", self.algorithm_version), + }); + } + validate_sha256_hex( + "child_certificate_cache_projection.cache_key_sha256_hex", + &self.cache_key_sha256_hex, + )?; + validate_non_empty( + "child_certificate_cache_projection.child_cert_uri", + &self.child_cert_uri, + )?; + validate_sha256_hex( + "child_certificate_cache_projection.child_cert_sha256_hex", + &self.child_cert_sha256_hex, + )?; + if self.child_cert_serial.is_empty() { + return Err(StorageError::InvalidData { + entity: "child_certificate_cache_projection.child_cert_serial", + detail: "must not be empty".to_string(), + }); + } + validate_sha256_hex( + "child_certificate_cache_projection.issuer_ca_sha256_hex", + &self.issuer_ca_sha256_hex, + )?; + validate_non_empty( + "child_certificate_cache_projection.issuer_crl_uri", + &self.issuer_crl_uri, + )?; + validate_sha256_hex( + "child_certificate_cache_projection.issuer_crl_sha256_hex", + &self.issuer_crl_sha256_hex, + )?; + let effective_not_before = parse_time( + "child_certificate_cache_projection.effective_not_before", + &self.effective_not_before, + )?; + let effective_until = parse_time( + "child_certificate_cache_projection.effective_until", + &self.effective_until, + )?; + if effective_not_before >= effective_until { + return Err(StorageError::InvalidData { + entity: "child_certificate_cache_projection.effective_window", + detail: "effective_not_before must be before effective_until".to_string(), + }); + } + self.payload.validate_internal()?; + Ok(()) + } +} diff --git a/crates/panda-rpki-validator/src/storage/store_child_cache.rs b/crates/panda-rpki-validator/src/storage/store_child_cache.rs new file mode 100644 index 0000000..4cf55aa --- /dev/null +++ b/crates/panda-rpki-validator/src/storage/store_child_cache.rs @@ -0,0 +1,300 @@ +// VCIR reads and child-certificate cache operations. + +impl RocksStore { + + pub fn get_vcir( + &self, + manifest_rsync_uri: &str, + ) -> StorageResult> { + let cf = self.cf(CF_VCIR)?; + let key = vcir_key(manifest_rsync_uri); + let Some(bytes) = self + .db + .get_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + else { + return Ok(None); + }; + let vcir = decode_cbor::(&bytes, "vcir")?; + vcir.validate_internal()?; + Ok(Some(vcir)) + } + + pub fn get_vcir_failed_fetch_reuse_identity( + &self, + manifest_rsync_uri: &str, + ) -> StorageResult> { + let cf = self.cf(CF_VCIR_FAILED_FETCH_REUSE_IDENTITY)?; + let key = vcir_failed_fetch_reuse_identity_key(manifest_rsync_uri); + let Some(bytes) = self + .db + .get_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + else { + return Ok(None); + }; + let identity = decode_cbor::( + &bytes, + "vcir_failed_fetch_reuse_identity", + )?; + identity.validate_internal()?; + Ok(Some(identity)) + } + + pub fn get_manifest_replay_meta( + &self, + manifest_rsync_uri: &str, + ) -> StorageResult> { + let cf = self.cf(CF_MANIFEST_REPLAY_META)?; + let key = manifest_replay_meta_key(manifest_rsync_uri); + let Some(bytes) = self + .db + .get_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + else { + return Ok(None); + }; + let meta = decode_cbor::(&bytes, "manifest_replay_meta")?; + meta.validate_internal()?; + Ok(Some(meta)) + } + + pub fn get_roa_cache_projection( + &self, + manifest_rsync_uri: &str, + ) -> StorageResult> { + let cf = self.cf(CF_ROA_CACHE_PROJECTION)?; + let key = roa_cache_projection_key(manifest_rsync_uri); + let Some(bytes) = self + .db + .get_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + else { + return Ok(None); + }; + let projection = decode_cbor::(&bytes, "roa_cache_projection")?; + projection.validate_internal()?; + Ok(Some(projection)) + } + + pub fn put_child_certificate_cache_projection( + &self, + projection: &ChildCertificateCacheProjection, + ) -> StorageResult<()> { + projection.validate_internal()?; + let cf = self.cf(CF_CHILD_CERTIFICATE_CACHE_PROJECTION)?; + let key = child_certificate_cache_projection_key(&projection.cache_key_sha256_hex); + let value = encode_cbor(projection, "child_certificate_cache_projection")?; + self.db + .put_cf(cf, key.as_bytes(), value) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(()) + } + + pub fn get_child_certificate_cache_projection( + &self, + cache_key_sha256_hex: &str, + ) -> StorageResult> { + validate_sha256_hex( + "child_certificate_cache_projection.cache_key_sha256_hex", + cache_key_sha256_hex, + )?; + let cf = self.cf(CF_CHILD_CERTIFICATE_CACHE_PROJECTION)?; + let key = child_certificate_cache_projection_key(cache_key_sha256_hex); + let Some(bytes) = self + .db + .get_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + else { + return Ok(None); + }; + let projection = decode_cbor::( + &bytes, + "child_certificate_cache_projection", + )?; + projection.validate_internal()?; + Ok(Some(projection)) + } + + pub fn get_child_certificate_cache_projections_batch( + &self, + cache_key_sha256_hexes: &[String], + ) -> StorageResult>> { + if cache_key_sha256_hexes.is_empty() { + return Ok(Vec::new()); + } + + for cache_key_sha256_hex in cache_key_sha256_hexes { + validate_sha256_hex( + "child_certificate_cache_projection.cache_key_sha256_hex", + cache_key_sha256_hex, + )?; + } + + let cf = self.cf(CF_CHILD_CERTIFICATE_CACHE_PROJECTION)?; + let keys: Vec = cache_key_sha256_hexes + .iter() + .map(|key| child_certificate_cache_projection_key(key)) + .collect(); + self.db + .multi_get_cf(keys.iter().map(|key| (cf, key.as_bytes()))) + .into_iter() + .map(|res| { + let Some(bytes) = res.map_err(|e| StorageError::RocksDb(e.to_string()))? else { + return Ok(None); + }; + let projection = decode_cbor::( + &bytes, + "child_certificate_cache_projection", + )?; + projection.validate_internal()?; + Ok(Some(projection)) + }) + .collect() + } + + fn child_certificate_cache_segment_path(&self, manifest_rsync_uri: &str) -> PathBuf { + self.child_certificate_cache_index_dir + .join(child_certificate_cache_segment_file_name( + manifest_rsync_uri, + )) + } + + pub fn get_child_certificate_cache_projections_mmap_segment( + &self, + manifest_rsync_uri: &str, + cache_key_sha256_hexes: &[String], + ) -> StorageResult> { + if cache_key_sha256_hexes.is_empty() { + return Ok(Some(ChildCertificateCacheMmapLookup::default())); + } + validate_non_empty( + "child_certificate_cache_mmap_segment.manifest_rsync_uri", + manifest_rsync_uri, + )?; + for cache_key_sha256_hex in cache_key_sha256_hexes { + validate_sha256_hex( + "child_certificate_cache_projection.cache_key_sha256_hex", + cache_key_sha256_hex, + )?; + } + + let path = self.child_certificate_cache_segment_path(manifest_rsync_uri); + if !path.exists() { + return Ok(None); + } + + let (index, _) = load_pp_cache_mmap_index(&path)?; + let file_bytes = index.file_bytes(); + let mut hits = 0usize; + let mut misses = 0usize; + let mut projections = Vec::with_capacity(cache_key_sha256_hexes.len()); + for cache_key_sha256_hex in cache_key_sha256_hexes { + match index.lookup(cache_key_sha256_hex) { + Some(PpCacheIndexLookup::Hit(bytes)) => { + let projection = decode_cbor::( + bytes, + "child_certificate_cache_projection_mmap_segment", + )?; + projection.validate_internal()?; + hits = hits.saturating_add(1); + projections.push(Some(projection)); + } + Some(PpCacheIndexLookup::Deleted) | None => { + misses = misses.saturating_add(1); + projections.push(None); + } + } + } + + Ok(Some(ChildCertificateCacheMmapLookup { + projections, + hits, + misses, + file_bytes, + })) + } + + pub fn write_child_certificate_cache_mmap_segment( + &self, + manifest_rsync_uri: &str, + projections: &[ChildCertificateCacheProjection], + ) -> StorageResult { + validate_non_empty( + "child_certificate_cache_mmap_segment.manifest_rsync_uri", + manifest_rsync_uri, + )?; + let mut entries = Vec::with_capacity(projections.len()); + for projection in projections { + projection.validate_internal()?; + entries.push(( + projection.cache_key_sha256_hex.clone(), + encode_cbor( + projection, + "child_certificate_cache_projection_mmap_segment", + )?, + )); + } + let path = self.child_certificate_cache_segment_path(manifest_rsync_uri); + write_pp_cache_index_atomic(&path, entries) + } + + pub fn write_child_certificate_cache_mmap_segment_overlay( + &self, + manifest_rsync_uri: &str, + cache_key_sha256_hexes: &[String], + projections: &[ChildCertificateCacheProjection], + ) -> StorageResult { + validate_non_empty( + "child_certificate_cache_mmap_segment.manifest_rsync_uri", + manifest_rsync_uri, + )?; + let mut dirty = HashMap::>::with_capacity(projections.len()); + for projection in projections { + projection.validate_internal()?; + dirty.insert( + projection.cache_key_sha256_hex.clone(), + encode_cbor( + projection, + "child_certificate_cache_projection_mmap_segment", + )?, + ); + } + + let path = self.child_certificate_cache_segment_path(manifest_rsync_uri); + let existing = if path.exists() { + Some(load_pp_cache_mmap_index(&path)?.0) + } else { + None + }; + let mut entries = Vec::with_capacity(cache_key_sha256_hexes.len()); + let mut emitted = HashSet::::new(); + for cache_key_sha256_hex in cache_key_sha256_hexes { + validate_sha256_hex( + "child_certificate_cache_projection.cache_key_sha256_hex", + cache_key_sha256_hex, + )?; + if !emitted.insert(cache_key_sha256_hex.clone()) { + continue; + } + if let Some(value) = dirty.remove(cache_key_sha256_hex) { + entries.push((cache_key_sha256_hex.clone(), value)); + continue; + } + if let Some(existing) = existing.as_ref() { + if let Some(PpCacheIndexLookup::Hit(bytes)) = existing.lookup(cache_key_sha256_hex) + { + entries.push((cache_key_sha256_hex.clone(), bytes.to_vec())); + } + } + } + for (key, value) in dirty { + if emitted.insert(key.clone()) { + entries.push((key, value)); + } + } + + write_pp_cache_index_atomic(&path, entries) + } + +} diff --git a/crates/panda-rpki-validator/src/storage/store_lifecycle.rs b/crates/panda-rpki-validator/src/storage/store_lifecycle.rs new file mode 100644 index 0000000..c1cea46 --- /dev/null +++ b/crates/panda-rpki-validator/src/storage/store_lifecycle.rs @@ -0,0 +1,199 @@ +// RocksDB lifecycle, external stores, and cache-index setup. + +impl RocksStore { + pub fn create_read_only_checkpoint(source: &Path, destination: &Path) -> StorageResult<()> { + if destination.exists() { + return Err(StorageError::InvalidData { + entity: "work_db_checkpoint", + detail: format!( + "checkpoint destination already exists: {}", + destination.display() + ), + }); + } + if let Some(parent) = destination.parent() { + std::fs::create_dir_all(parent) + .map_err(|error| StorageError::RocksDb(error.to_string()))?; + } + let mut options = Options::default(); + let blob_mode = work_db_blob_mode_from_env(); + let memory_profile = work_db_memory_profile_from_env(); + configure_work_db_options(&mut options, blob_mode, memory_profile); + let db = DB::open_cf_descriptors_read_only( + &options, + source, + column_family_descriptors_for_blob_mode(blob_mode), + false, + ) + .map_err(|error| StorageError::RocksDb(error.to_string()))?; + let checkpoint = + Checkpoint::new(&db).map_err(|error| StorageError::RocksDb(error.to_string()))?; + checkpoint + .create_checkpoint(destination) + .map_err(|error| StorageError::RocksDb(error.to_string())) + } + + pub fn open(path: &Path) -> StorageResult { + let mut base_opts = Options::default(); + base_opts.create_if_missing(true); + base_opts.create_missing_column_families(true); + let blob_mode = work_db_blob_mode_from_env(); + let memory_profile = work_db_memory_profile_from_env(); + configure_work_db_options(&mut base_opts, blob_mode, memory_profile); + + let db = DB::open_cf_descriptors( + &base_opts, + path, + column_family_descriptors_for_blob_mode(blob_mode), + ) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + + Ok(Self { + db, + external_raw_store: None, + external_repo_bytes: None, + publication_point_cache_index_dir: default_pp_cache_index_dir(path), + child_certificate_cache_index_dir: default_child_certificate_cache_index_dir(path), + publication_point_cache_projection_index: Mutex::new( + PublicationPointCacheProjectionIndexState::Uninitialized, + ), + }) + } + + pub fn open_with_external_raw_store(path: &Path, raw_store_path: &Path) -> StorageResult { + Self::open_with_external_stores(path, Some(raw_store_path), None) + } + + pub fn open_with_external_repo_bytes( + path: &Path, + repo_bytes_path: &Path, + ) -> StorageResult { + Self::open_with_external_stores(path, None, Some(repo_bytes_path)) + } + + pub fn open_with_external_repo_bytes_read_only( + path: &Path, + repo_bytes_path: &Path, + ) -> StorageResult { + let mut store = Self::open(path)?; + store.external_repo_bytes = Some(ExternalRepoBytesDb::open_read_only(repo_bytes_path)?); + Ok(store) + } + + pub fn open_with_external_stores( + path: &Path, + raw_store_path: Option<&Path>, + repo_bytes_path: Option<&Path>, + ) -> StorageResult { + let mut store = Self::open(path)?; + if let Some(raw_store_path) = raw_store_path { + store.external_raw_store = Some(ExternalRawStoreDb::open(raw_store_path)?); + } + if let Some(repo_bytes_path) = repo_bytes_path { + store.external_repo_bytes = Some(ExternalRepoBytesDb::open(repo_bytes_path)?); + } + Ok(store) + } + + pub(crate) fn external_raw_store_ref(&self) -> Option<&ExternalRawStoreDb> { + self.external_raw_store.as_ref() + } + + pub(crate) fn external_repo_bytes_ref(&self) -> Option<&ExternalRepoBytesDb> { + self.external_repo_bytes.as_ref() + } + + pub fn memory_snapshot(&self) -> RocksDbMemorySnapshot { + let mut databases = Vec::new(); + databases.push(memory_db_snapshot_for_column_families( + "work-db", + &self.db, + Some(ALL_COLUMN_FAMILY_NAMES), + )); + if let Some(raw_store) = self.external_raw_store.as_ref() { + databases.push(raw_store.memory_snapshot("raw-store.db")); + } + if let Some(repo_bytes) = self.external_repo_bytes.as_ref() { + databases.push(repo_bytes.memory_snapshot("repo-bytes.db")); + } + + let mut totals = RocksDbMemoryTotals::default(); + for db in &databases { + totals.add_properties(&db.properties); + } + RocksDbMemorySnapshot { databases, totals } + } + + pub fn publication_point_cache_mmap_index_load_stats(&self) -> Option { + let guard = self.publication_point_cache_projection_index.lock().ok()?; + match &*guard { + PublicationPointCacheProjectionIndexState::LoadedMmap { load_stats, .. } => { + Some(load_stats.clone()) + } + PublicationPointCacheProjectionIndexState::Uninitialized + | PublicationPointCacheProjectionIndexState::Disabled + | PublicationPointCacheProjectionIndexState::BuildingFromEmpty { .. } + | PublicationPointCacheProjectionIndexState::Loaded { .. } => None, + } + } + + fn try_load_publication_point_cache_mmap_index_for_update( + &self, + reason: &'static str, + ) -> StorageResult<()> { + if !pp_cache_raw_index_enabled() { + return Ok(()); + } + let mut guard = self + .publication_point_cache_projection_index + .lock() + .map_err(|e| { + StorageError::RocksDb(format!("publication point cache index lock poisoned: {e}")) + })?; + if !matches!( + *guard, + PublicationPointCacheProjectionIndexState::Uninitialized + ) { + return Ok(()); + } + match load_pp_cache_mmap_index_set(&self.publication_point_cache_index_dir) { + Ok((mmap, stats)) => { + crate::progress_log::emit( + "publication_point_cache_mmap_index_load", + serde_json::json!({ + "state": "loaded", + "reason": reason, + "entries": stats.entries, + "bytes": stats.bytes, + "file_bytes": stats.file_bytes, + "load_ms": stats.load_ms, + }), + ); + *guard = PublicationPointCacheProjectionIndexState::LoadedMmap { + mmap, + dirty: HashMap::new(), + dirty_bytes: 0, + load_stats: stats, + }; + } + Err(e) => { + crate::progress_log::emit( + "publication_point_cache_mmap_index_load", + serde_json::json!({ + "state": "deferred_fallback_scan", + "reason": reason, + "error": e.to_string(), + }), + ); + } + } + Ok(()) + } + + fn cf(&self, name: &'static str) -> StorageResult<&ColumnFamily> { + self.db + .cf_handle(name) + .ok_or(StorageError::MissingColumnFamily(name)) + } + +} diff --git a/crates/panda-rpki-validator/src/storage/store_publication_cache.rs b/crates/panda-rpki-validator/src/storage/store_publication_cache.rs new file mode 100644 index 0000000..c1a580a --- /dev/null +++ b/crates/panda-rpki-validator/src/storage/store_publication_cache.rs @@ -0,0 +1,509 @@ +// Publication-point cache projection and mmap index operations. + +impl RocksStore { + + pub fn get_publication_point_cache_projection( + &self, + manifest_rsync_uri: &str, + ) -> StorageResult> { + self.get_publication_point_cache_projection_from_db(manifest_rsync_uri) + } + + pub fn get_publication_point_cache_projection_cached( + &self, + manifest_rsync_uri: &str, + ) -> StorageResult> { + let mut owned_bytes: Option> = None; + { + let mut guard = self + .publication_point_cache_projection_index + .lock() + .map_err(|e| { + StorageError::RocksDb(format!( + "publication point cache index lock poisoned: {e}" + )) + })?; + if matches!( + *guard, + PublicationPointCacheProjectionIndexState::Uninitialized + ) { + let load_started = std::time::Instant::now(); + let raw_index_enabled = pp_cache_raw_index_enabled(); + *guard = if !raw_index_enabled { + crate::progress_log::emit( + "publication_point_cache_raw_index", + serde_json::json!({ + "state": "disabled_by_env", + "entries": 0, + "bytes": 0, + "load_ms": load_started.elapsed().as_millis() as u64, + }), + ); + PublicationPointCacheProjectionIndexState::Disabled + } else { + match load_pp_cache_mmap_index_set(&self.publication_point_cache_index_dir) { + Ok((mmap, stats)) => { + crate::progress_log::emit( + "publication_point_cache_mmap_index_load", + serde_json::json!({ + "state": "loaded", + "entries": stats.entries, + "bytes": stats.bytes, + "file_bytes": stats.file_bytes, + "load_ms": stats.load_ms, + }), + ); + PublicationPointCacheProjectionIndexState::LoadedMmap { + mmap, + dirty: HashMap::new(), + dirty_bytes: 0, + load_stats: stats, + } + } + Err(e) => { + crate::progress_log::emit( + "publication_point_cache_mmap_index_load", + serde_json::json!({ + "state": "fallback_scan", + "error": e.to_string(), + "load_ms": load_started.elapsed().as_millis() as u64, + }), + ); + let scan_started = std::time::Instant::now(); + let (index, bytes) = + self.load_publication_point_cache_projection_index()?; + if index.is_empty() { + let limit = pp_cache_raw_index_empty_build_limit_bytes(); + crate::progress_log::emit( + "publication_point_cache_raw_index", + serde_json::json!({ + "state": "empty_building_bounded", + "entries": 0, + "bytes": 0, + "empty_build_limit_bytes": limit, + "load_ms": scan_started.elapsed().as_millis() as u64, + }), + ); + PublicationPointCacheProjectionIndexState::BuildingFromEmpty { + index, + bytes: 0, + limit, + } + } else { + crate::progress_log::emit( + "publication_point_cache_raw_index", + serde_json::json!({ + "state": "loaded", + "entries": index.len(), + "bytes": bytes, + "load_ms": scan_started.elapsed().as_millis() as u64, + }), + ); + PublicationPointCacheProjectionIndexState::Loaded { index, bytes } + } + } + } + }; + } + match &*guard { + PublicationPointCacheProjectionIndexState::Loaded { index, .. } => { + owned_bytes = index.get(manifest_rsync_uri).cloned(); + } + PublicationPointCacheProjectionIndexState::BuildingFromEmpty { index, .. } => { + owned_bytes = index.get(manifest_rsync_uri).cloned(); + } + PublicationPointCacheProjectionIndexState::LoadedMmap { mmap, dirty, .. } => { + if let Some(bytes) = dirty.get(manifest_rsync_uri).cloned() { + if bytes.is_empty() { + return Ok(None); + } + owned_bytes = Some(bytes); + } else if let Some(lookup) = mmap.lookup(manifest_rsync_uri) { + match lookup { + PpCacheIndexLookup::Hit(bytes) => { + let projection = decode_cbor::( + bytes, + "publication_point_cache_projection", + )?; + projection.validate_internal()?; + return Ok(Some(projection)); + } + PpCacheIndexLookup::Deleted => return Ok(None), + } + } + } + PublicationPointCacheProjectionIndexState::Disabled + | PublicationPointCacheProjectionIndexState::Uninitialized => {} + } + } + let bytes = owned_bytes; + let Some(bytes) = bytes else { + return Ok(None); + }; + let projection = decode_cbor::( + bytes.as_ref(), + "publication_point_cache_projection", + )?; + projection.validate_internal()?; + Ok(Some(projection)) + } + + fn get_publication_point_cache_projection_from_db( + &self, + manifest_rsync_uri: &str, + ) -> StorageResult> { + let cf = self.cf(CF_PUBLICATION_POINT_CACHE_PROJECTION)?; + let key = publication_point_cache_projection_key(manifest_rsync_uri); + let Some(bytes) = self + .db + .get_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + else { + return Ok(None); + }; + let projection = decode_cbor::( + &bytes, + "publication_point_cache_projection", + )?; + projection.validate_internal()?; + Ok(Some(projection)) + } + + fn load_publication_point_cache_projection_index( + &self, + ) -> StorageResult<(HashMap>, usize)> { + let cf = self.cf(CF_PUBLICATION_POINT_CACHE_PROJECTION)?; + let mode = IteratorMode::Start; + let mut index = HashMap::new(); + let mut bytes_total = 0usize; + for res in self.db.iterator_cf(cf, mode) { + let (key, value) = res.map_err(|e| StorageError::RocksDb(e.to_string()))?; + let Some(manifest_rsync_uri) = + publication_point_cache_projection_key_manifest_uri(&key) + else { + continue; + }; + bytes_total = bytes_total.saturating_add(value.len()); + index.insert(manifest_rsync_uri, Arc::<[u8]>::from(value.to_vec())); + } + Ok((index, bytes_total)) + } + + fn load_publication_point_cache_projection_entries( + &self, + ) -> StorageResult)>> { + let cf = self.cf(CF_PUBLICATION_POINT_CACHE_PROJECTION)?; + let mode = IteratorMode::Start; + let mut entries = Vec::new(); + for res in self.db.iterator_cf(cf, mode) { + let (key, value) = res.map_err(|e| StorageError::RocksDb(e.to_string()))?; + let Some(manifest_rsync_uri) = + publication_point_cache_projection_key_manifest_uri(&key) + else { + continue; + }; + entries.push((manifest_rsync_uri, value.to_vec())); + } + Ok(entries) + } + + fn apply_publication_point_cache_projection_index_action( + &self, + action: PublicationPointCacheProjectionWriteAction<'_>, + ) -> StorageResult<()> { + match action { + PublicationPointCacheProjectionWriteAction::Keep => Ok(()), + PublicationPointCacheProjectionWriteAction::Write(projection) => { + self.update_publication_point_cache_projection_index(projection) + } + PublicationPointCacheProjectionWriteAction::Delete { manifest_rsync_uri } => { + self.delete_publication_point_cache_projection_index_entry(manifest_rsync_uri) + } + } + } + + fn update_publication_point_cache_projection_index( + &self, + projection: &PublicationPointCacheProjection, + ) -> StorageResult<()> { + self.try_load_publication_point_cache_mmap_index_for_update("write")?; + let mut guard = self + .publication_point_cache_projection_index + .lock() + .map_err(|e| { + StorageError::RocksDb(format!("publication point cache index lock poisoned: {e}")) + })?; + let bytes = encode_cbor(projection, "publication_point_cache_projection")?; + match &mut *guard { + PublicationPointCacheProjectionIndexState::Loaded { + index, + bytes: total_bytes, + } => { + if let Some(previous) = index.insert( + projection.manifest_rsync_uri.clone(), + Arc::<[u8]>::from(bytes.clone()), + ) { + *total_bytes = total_bytes.saturating_sub(previous.len()); + } + *total_bytes = total_bytes.saturating_add(bytes.len()); + } + PublicationPointCacheProjectionIndexState::BuildingFromEmpty { + index, + bytes: total_bytes, + limit, + } => { + if let Some(previous) = index.remove(&projection.manifest_rsync_uri) { + *total_bytes = total_bytes.saturating_sub(previous.len()); + } + if total_bytes.saturating_add(bytes.len()) <= *limit { + *total_bytes += bytes.len(); + index.insert( + projection.manifest_rsync_uri.clone(), + Arc::<[u8]>::from(bytes), + ); + } else { + *guard = PublicationPointCacheProjectionIndexState::Disabled; + } + } + PublicationPointCacheProjectionIndexState::LoadedMmap { + dirty, dirty_bytes, .. + } => { + if let Some(previous) = dirty.insert( + projection.manifest_rsync_uri.clone(), + Arc::<[u8]>::from(bytes.clone()), + ) { + *dirty_bytes = dirty_bytes.saturating_sub(previous.len()); + } + *dirty_bytes = dirty_bytes.saturating_add(bytes.len()); + } + PublicationPointCacheProjectionIndexState::Uninitialized + | PublicationPointCacheProjectionIndexState::Disabled => {} + } + Ok(()) + } + + fn delete_publication_point_cache_projection_index_entry( + &self, + manifest_rsync_uri: &str, + ) -> StorageResult<()> { + self.try_load_publication_point_cache_mmap_index_for_update("delete")?; + let mut guard = self + .publication_point_cache_projection_index + .lock() + .map_err(|e| { + StorageError::RocksDb(format!("publication point cache index lock poisoned: {e}")) + })?; + match &mut *guard { + PublicationPointCacheProjectionIndexState::Loaded { + index, + bytes: total_bytes, + } => { + if let Some(previous) = index.remove(manifest_rsync_uri) { + *total_bytes = total_bytes.saturating_sub(previous.len()); + } + } + PublicationPointCacheProjectionIndexState::BuildingFromEmpty { + index, + bytes: total_bytes, + .. + } => { + if let Some(previous) = index.remove(manifest_rsync_uri) { + *total_bytes = total_bytes.saturating_sub(previous.len()); + } + } + PublicationPointCacheProjectionIndexState::LoadedMmap { + dirty, dirty_bytes, .. + } => { + if let Some(previous) = + dirty.insert(manifest_rsync_uri.to_string(), Arc::<[u8]>::from([])) + { + *dirty_bytes = dirty_bytes.saturating_sub(previous.len()); + } + } + PublicationPointCacheProjectionIndexState::Uninitialized + | PublicationPointCacheProjectionIndexState::Disabled => {} + } + Ok(()) + } + + pub fn refresh_publication_point_cache_mmap_index( + &self, + ) -> StorageResult> { + if !pp_cache_raw_index_enabled() { + return Ok(None); + } + self.try_load_publication_point_cache_mmap_index_for_update("refresh")?; + enum RefreshAction { + Entries { + entries: Vec<(String, Vec)>, + write_segment: bool, + old_entries: usize, + dirty_entries: usize, + }, + ScanDb, + } + let action = { + let guard = self + .publication_point_cache_projection_index + .lock() + .map_err(|e| { + StorageError::RocksDb(format!( + "publication point cache index lock poisoned: {e}" + )) + })?; + match &*guard { + PublicationPointCacheProjectionIndexState::LoadedMmap { mmap, dirty, .. } => { + RefreshAction::Entries { + entries: dirty + .iter() + .map(|(key, value)| (key.clone(), value.as_ref().to_vec())) + .collect::>(), + write_segment: true, + old_entries: mmap.entries(), + dirty_entries: dirty.len(), + } + } + PublicationPointCacheProjectionIndexState::Loaded { index, .. } + | PublicationPointCacheProjectionIndexState::BuildingFromEmpty { index, .. } => { + RefreshAction::Entries { + entries: index + .iter() + .map(|(key, value)| (key.clone(), value.as_ref().to_vec())) + .collect::>(), + write_segment: false, + old_entries: 0, + dirty_entries: 0, + } + } + PublicationPointCacheProjectionIndexState::Disabled + | PublicationPointCacheProjectionIndexState::Uninitialized => RefreshAction::ScanDb, + } + }; + let (entries, write_segment, old_entries, dirty_entries) = match action { + RefreshAction::Entries { + entries, + write_segment, + old_entries, + dirty_entries, + } => (entries, write_segment, old_entries, dirty_entries), + RefreshAction::ScanDb => ( + self.load_publication_point_cache_projection_entries()?, + false, + 0, + 0, + ), + }; + if entries.is_empty() { + return Ok(None); + } + let current_path = self.publication_point_cache_index_dir.join("current.idx"); + let mut stats = if write_segment && current_path.exists() { + write_pp_cache_index_segment(&self.publication_point_cache_index_dir, entries)? + } else { + write_pp_cache_index_atomic(¤t_path, entries)? + }; + stats.old_entries = old_entries; + stats.dirty_entries = dirty_entries; + crate::progress_log::emit( + "publication_point_cache_mmap_index_refresh", + serde_json::json!({ + "state": stats.state, + "old_entries": stats.old_entries, + "dirty_entries": stats.dirty_entries, + "new_entries": stats.new_entries, + "file_bytes": stats.file_bytes, + "write_ms": stats.write_ms, + }), + ); + let directory_stats = + pp_cache_index_directory_stats(&self.publication_point_cache_index_dir)?; + let compaction_reason = if directory_stats.segment_count + >= PP_CACHE_INDEX_COMPACTION_SEGMENT_THRESHOLD + { + Some(format!( + "segment_count>={PP_CACHE_INDEX_COMPACTION_SEGMENT_THRESHOLD}" + )) + } else if directory_stats.total_file_bytes >= PP_CACHE_INDEX_COMPACTION_BYTES_THRESHOLD { + Some(format!( + "total_file_bytes>={PP_CACHE_INDEX_COMPACTION_BYTES_THRESHOLD}" + )) + } else { + None + }; + if let Some(reason) = compaction_reason { + crate::progress_log::emit( + "publication_point_cache_mmap_index_compaction", + serde_json::json!({ + "state": "started", + "reason": reason, + "segment_count": directory_stats.segment_count, + "total_file_bytes": directory_stats.total_file_bytes, + }), + ); + match compact_pp_cache_index(&self.publication_point_cache_index_dir) { + Ok(mut compact_stats) => { + compact_stats.compaction_reason = Some(reason); + compact_stats.old_entries = stats.old_entries; + compact_stats.dirty_entries = stats.dirty_entries; + crate::progress_log::emit( + "publication_point_cache_mmap_index_compaction", + serde_json::json!({ + "state": "completed", + "reason": compact_stats.compaction_reason, + "segments_before": compact_stats.compaction_segments_before, + "total_file_bytes_before": compact_stats.compaction_total_file_bytes_before, + "live_entries": compact_stats.compaction_live_entries, + "file_bytes": compact_stats.compaction_file_bytes, + "reclaimed_bytes": compact_stats.compaction_reclaimed_bytes, + "deleted_segments": compact_stats.compaction_deleted_segments, + "compaction_ms": compact_stats.compaction_ms, + }), + ); + stats.compaction_triggered = compact_stats.compaction_triggered; + stats.compaction_reason = compact_stats.compaction_reason; + stats.compaction_segments_before = compact_stats.compaction_segments_before; + stats.compaction_total_file_bytes_before = + compact_stats.compaction_total_file_bytes_before; + stats.compaction_live_entries = compact_stats.compaction_live_entries; + stats.compaction_file_bytes = compact_stats.compaction_file_bytes; + stats.compaction_reclaimed_bytes = compact_stats.compaction_reclaimed_bytes; + stats.compaction_ms = compact_stats.compaction_ms; + stats.compaction_deleted_segments = compact_stats.compaction_deleted_segments; + } + Err(e) => { + let error = e.to_string(); + crate::progress_log::emit( + "publication_point_cache_mmap_index_compaction", + serde_json::json!({ + "state": "failed", + "reason": reason, + "segment_count": directory_stats.segment_count, + "total_file_bytes": directory_stats.total_file_bytes, + "error": error, + }), + ); + stats.compaction_triggered = true; + stats.compaction_reason = Some(reason); + stats.compaction_segments_before = directory_stats.segment_count; + stats.compaction_total_file_bytes_before = directory_stats.total_file_bytes; + stats.compaction_error = Some(error); + } + } + } + let mut guard = self + .publication_point_cache_projection_index + .lock() + .map_err(|e| { + StorageError::RocksDb(format!("publication point cache index lock poisoned: {e}")) + })?; + if let PublicationPointCacheProjectionIndexState::LoadedMmap { + dirty, dirty_bytes, .. + } = &mut *guard + { + dirty.clear(); + *dirty_bytes = 0; + } + Ok(Some(stats)) + } + +} diff --git a/crates/panda-rpki-validator/src/storage/store_repository.rs b/crates/panda-rpki-validator/src/storage/store_repository.rs new file mode 100644 index 0000000..3a48a67 --- /dev/null +++ b/crates/panda-rpki-validator/src/storage/store_repository.rs @@ -0,0 +1,363 @@ +// Repository-view and raw-object/blob storage operations. + +impl RocksStore { + + pub fn put_repository_view_entry(&self, entry: &RepositoryViewEntry) -> StorageResult<()> { + entry.validate_internal()?; + let cf = self.cf(CF_REPOSITORY_VIEW)?; + let key = repository_view_key(&entry.rsync_uri); + let value = encode_cbor(entry, "repository_view")?; + self.db + .put_cf(cf, key.as_bytes(), value) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(()) + } + + pub fn get_repository_view_entry( + &self, + rsync_uri: &str, + ) -> StorageResult> { + let cf = self.cf(CF_REPOSITORY_VIEW)?; + let key = repository_view_key(rsync_uri); + let Some(bytes) = self + .db + .get_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + else { + return Ok(None); + }; + let entry = decode_cbor::(&bytes, "repository_view")?; + entry.validate_internal()?; + Ok(Some(entry)) + } + + pub fn delete_repository_view_entry(&self, rsync_uri: &str) -> StorageResult<()> { + let cf = self.cf(CF_REPOSITORY_VIEW)?; + let key = repository_view_key(rsync_uri); + self.db + .delete_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(()) + } + + pub fn put_projection_batch( + &self, + repository_view_entries: &[RepositoryViewEntry], + member_records: &[RrdpSourceMemberRecord], + owner_records: &[RrdpUriOwnerRecord], + ) -> StorageResult<()> { + if repository_view_entries.is_empty() + && member_records.is_empty() + && owner_records.is_empty() + { + return Ok(()); + } + + let repo_cf = self.cf(CF_REPOSITORY_VIEW)?; + let member_cf = self.cf(CF_RRDP_SOURCE_MEMBER)?; + let owner_cf = self.cf(CF_RRDP_URI_OWNER)?; + let mut batch = WriteBatch::default(); + + for entry in repository_view_entries { + entry.validate_internal()?; + let key = repository_view_key(&entry.rsync_uri); + let value = encode_cbor(entry, "repository_view")?; + batch.put_cf(repo_cf, key.as_bytes(), value); + } + for record in member_records { + record.validate_internal()?; + let key = rrdp_source_member_key(&record.notify_uri, &record.rsync_uri); + let value = encode_cbor(record, "rrdp_source_member")?; + batch.put_cf(member_cf, key.as_bytes(), value); + } + for record in owner_records { + record.validate_internal()?; + let key = rrdp_uri_owner_key(&record.rsync_uri); + let value = encode_cbor(record, "rrdp_uri_owner")?; + batch.put_cf(owner_cf, key.as_bytes(), value); + } + + self.write_batch(batch) + } + + pub fn list_repository_view_entries_with_prefix( + &self, + rsync_uri_prefix: &str, + ) -> StorageResult> { + let cf = self.cf(CF_REPOSITORY_VIEW)?; + let prefix = repository_view_prefix(rsync_uri_prefix); + let mode = IteratorMode::From(prefix.as_bytes(), Direction::Forward); + self.db + .iterator_cf(cf, mode) + .take_while(|res| match res { + Ok((key, _)) => key.starts_with(prefix.as_bytes()), + Err(_) => false, + }) + .map(|res| { + let (_key, value) = res.map_err(|e| StorageError::RocksDb(e.to_string()))?; + let entry = decode_cbor::(&value, "repository_view")?; + entry.validate_internal()?; + Ok(entry) + }) + .collect() + } + + pub fn verify_current_repository_blobs( + &self, + batch_size: usize, + ) -> StorageResult { + if batch_size == 0 { + return Err(StorageError::InvalidData { + entity: "repository_blob_verification", + detail: "batch_size must be greater than zero".to_string(), + }); + } + let cf = self.cf(CF_REPOSITORY_VIEW)?; + let prefix = repository_view_prefix(""); + let mode = IteratorMode::From(prefix.as_bytes(), Direction::Forward); + let mut summary = RepositoryBlobVerificationSummary::default(); + let mut batch = Vec::with_capacity(batch_size); + for item in self.db.iterator_cf(cf, mode) { + let (key, value) = item.map_err(|error| StorageError::RocksDb(error.to_string()))?; + if !key.starts_with(prefix.as_bytes()) { + break; + } + let entry = decode_cbor::(&value, "repository_view")?; + entry.validate_internal()?; + if !matches!( + entry.state, + RepositoryViewState::Present | RepositoryViewState::Replaced + ) { + continue; + } + let hash = entry + .current_hash + .clone() + .ok_or(StorageError::InvalidData { + entity: "repository_blob_verification", + detail: format!("current hash missing for {}", entry.rsync_uri), + })?; + batch.push((entry.rsync_uri, hash)); + if batch.len() >= batch_size { + verify_repository_blob_batch(self, &batch, &mut summary)?; + batch.clear(); + } + } + if !batch.is_empty() { + verify_repository_blob_batch(self, &batch, &mut summary)?; + } + Ok(summary) + } + + pub fn put_raw_by_hash_entry(&self, entry: &RawByHashEntry) -> StorageResult<()> { + entry.validate_internal()?; + if let Some(raw_store) = self.external_raw_store.as_ref() { + return raw_store.put_raw_entry(entry); + } + let cf = self.cf(CF_RAW_BY_HASH)?; + let key = raw_by_hash_key(&entry.sha256_hex); + let value = encode_cbor(entry, "raw_by_hash")?; + self.db + .put_cf(cf, key.as_bytes(), value) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(()) + } + + pub fn put_raw_by_hash_entries_batch(&self, entries: &[RawByHashEntry]) -> StorageResult<()> { + if entries.is_empty() { + return Ok(()); + } + if let Some(raw_store) = self.external_raw_store.as_ref() { + return raw_store.put_raw_entries_batch(entries); + } + + let cf = self.cf(CF_RAW_BY_HASH)?; + let mut batch = WriteBatch::default(); + for entry in entries { + entry.validate_internal()?; + let key = raw_by_hash_key(&entry.sha256_hex); + let value = encode_cbor(entry, "raw_by_hash")?; + batch.put_cf(cf, key.as_bytes(), value); + } + self.write_batch(batch) + } + + pub fn put_raw_by_hash_entries_batch_unchecked( + &self, + entries: &[RawByHashEntry], + ) -> StorageResult<()> { + if entries.is_empty() { + return Ok(()); + } + if let Some(raw_store) = self.external_raw_store.as_ref() { + return raw_store.put_raw_entries_batch(entries); + } + + let cf = self.cf(CF_RAW_BY_HASH)?; + let mut batch = WriteBatch::default(); + for entry in entries { + let key = raw_by_hash_key(&entry.sha256_hex); + let value = encode_cbor(entry, "raw_by_hash")?; + batch.put_cf(cf, key.as_bytes(), value); + } + self.write_batch(batch) + } + + pub fn put_blob_bytes_batch(&self, blobs: &[(String, Vec)]) -> StorageResult<()> { + if blobs.is_empty() { + return Ok(()); + } + if let Some(repo_bytes) = self.external_repo_bytes.as_ref() { + if repo_bytes.is_read_only() { + return repo_bytes.require_existing_blob_bytes_batch(blobs); + } + return repo_bytes.put_blob_bytes_batch(blobs); + } + if let Some(raw_store) = self.external_raw_store.as_ref() { + return raw_store.put_blob_bytes_batch(blobs); + } + let cf = self.cf(CF_RAW_BLOB)?; + let mut batch = WriteBatch::default(); + for (sha256_hex, bytes) in blobs { + validate_sha256_hex("raw_blob.sha256_hex", sha256_hex)?; + if bytes.is_empty() { + return Err(StorageError::InvalidData { + entity: "raw_blob", + detail: "bytes must not be empty".to_string(), + }); + } + let key = raw_blob_key(sha256_hex); + batch.put_cf(cf, key.as_bytes(), bytes.as_slice()); + } + self.write_batch(batch) + } + + pub fn delete_raw_by_hash_entry(&self, sha256_hex: &str) -> StorageResult<()> { + validate_sha256_hex("raw_by_hash.sha256_hex", sha256_hex)?; + if let Some(raw_store) = self.external_raw_store.as_ref() { + return raw_store.delete_raw_entry(sha256_hex); + } + let cf = self.cf(CF_RAW_BY_HASH)?; + let key = raw_by_hash_key(sha256_hex); + self.db + .delete_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(()) + } + + pub fn get_raw_by_hash_entry(&self, sha256_hex: &str) -> StorageResult> { + if let Some(raw_store) = self.external_raw_store.as_ref() { + return raw_store.get_raw_entry(sha256_hex); + } + let cf = self.cf(CF_RAW_BY_HASH)?; + let key = raw_by_hash_key(sha256_hex); + let Some(bytes) = self + .db + .get_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + else { + return Ok(None); + }; + let entry = decode_cbor::(&bytes, "raw_by_hash")?; + entry.validate_internal()?; + Ok(Some(entry)) + } + + pub fn get_raw_by_hash_entries_batch( + &self, + sha256_hexes: &[String], + ) -> StorageResult>> { + if sha256_hexes.is_empty() { + return Ok(Vec::new()); + } + if let Some(raw_store) = self.external_raw_store.as_ref() { + return raw_store.get_raw_entries_batch(sha256_hexes); + } + + let cf = self.cf(CF_RAW_BY_HASH)?; + let keys: Vec = sha256_hexes + .iter() + .map(|hash| raw_by_hash_key(hash)) + .collect(); + self.db + .multi_get_cf(keys.iter().map(|key| (cf, key.as_bytes()))) + .into_iter() + .map(|res| { + let maybe = res.map_err(|e| StorageError::RocksDb(e.to_string()))?; + match maybe { + Some(bytes) => { + let entry = decode_cbor::(&bytes, "raw_by_hash")?; + entry.validate_internal()?; + Ok(Some(entry)) + } + None => Ok(None), + } + }) + .collect() + } + + pub fn get_blob_bytes(&self, sha256_hex: &str) -> StorageResult>> { + if let Some(repo_bytes) = self.external_repo_bytes.as_ref() { + return repo_bytes.get_blob_bytes(sha256_hex); + } + if let Some(raw_store) = self.external_raw_store.as_ref() { + return raw_store.get_blob_bytes(sha256_hex); + } + validate_sha256_hex("raw_blob.sha256_hex", sha256_hex)?; + let cf = self.cf(CF_RAW_BLOB)?; + let key = raw_blob_key(sha256_hex); + if let Some(bytes) = self + .db + .get_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + { + return Ok(Some(bytes)); + } + self.get_raw_by_hash_entry(sha256_hex) + .map(|entry| entry.map(|entry| entry.bytes)) + } + + pub fn get_blob_bytes_batch( + &self, + sha256_hexes: &[String], + ) -> StorageResult>>> { + if sha256_hexes.is_empty() { + return Ok(Vec::new()); + } + if let Some(repo_bytes) = self.external_repo_bytes.as_ref() { + return repo_bytes.get_blob_bytes_batch(sha256_hexes); + } + if let Some(raw_store) = self.external_raw_store.as_ref() { + return raw_store.get_blob_bytes_batch(sha256_hexes); + } + + let cf = self.cf(CF_RAW_BLOB)?; + let keys: Vec = sha256_hexes + .iter() + .map(|hash| { + validate_sha256_hex("raw_blob.sha256_hex", hash)?; + Ok::(raw_blob_key(hash)) + }) + .collect::>()?; + let blob_results: Vec>> = self + .db + .multi_get_cf(keys.iter().map(|key| (cf, key.as_bytes()))) + .into_iter() + .map(|res| res.map_err(|e| StorageError::RocksDb(e.to_string()))) + .collect::>()?; + + let mut out = Vec::with_capacity(sha256_hexes.len()); + for (sha256_hex, maybe_blob) in sha256_hexes.iter().zip(blob_results.into_iter()) { + if maybe_blob.is_some() { + out.push(maybe_blob); + } else { + out.push( + self.get_raw_by_hash_entry(sha256_hex)? + .map(|entry| entry.bytes), + ); + } + } + Ok(out) + } + +} diff --git a/crates/panda-rpki-validator/src/storage/store_transport_rrdp.rs b/crates/panda-rpki-validator/src/storage/store_transport_rrdp.rs new file mode 100644 index 0000000..8402cef --- /dev/null +++ b/crates/panda-rpki-validator/src/storage/store_transport_rrdp.rs @@ -0,0 +1,332 @@ +// Transport prefetch, VCIR listing, and RRDP metadata operations. + +impl RocksStore { + + pub fn put_transport_prefetch_snapshot( + &self, + snapshot: &crate::parallel::transport_prefetch::TransportPrefetchSnapshot, + ) -> StorageResult<()> { + let cf = self.cf(CF_TRANSPORT_PREFETCH)?; + let value = encode_cbor(snapshot, "transport_prefetch_snapshot")?; + self.db + .put_cf(cf, TRANSPORT_PREFETCH_LAST_SNAPSHOT_KEY.as_bytes(), value) + .map_err(|e| StorageError::RocksDb(e.to_string())) + } + + pub fn get_transport_prefetch_snapshot( + &self, + ) -> StorageResult> { + let cf = self.cf(CF_TRANSPORT_PREFETCH)?; + let Some(bytes) = self + .db + .get_cf(cf, TRANSPORT_PREFETCH_LAST_SNAPSHOT_KEY.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + else { + return Ok(None); + }; + decode_cbor::( + &bytes, + "transport_prefetch_snapshot", + ) + .map(Some) + } + + pub fn list_vcirs(&self) -> StorageResult> { + let cf = self.cf(CF_VCIR)?; + let mode = IteratorMode::Start; + let mut out = Vec::new(); + for res in self.db.iterator_cf(cf, mode) { + let (_key, bytes) = res.map_err(|e| StorageError::RocksDb(e.to_string()))?; + let vcir = decode_cbor::(&bytes, "vcir")?; + vcir.validate_internal()?; + out.push(vcir); + } + Ok(out) + } + + /// Remove every reusable VCIR and its failed-fetch identity while retaining + /// manifest replay metadata. The replay metadata is a fresh-validation + /// guard, not a reusable validation result; removing it changes the normal + /// traversal semantics for manifests that are not selected for replay. + /// + /// This is intentionally narrower than [`Self::delete_vcir`]. It preserves + /// manifest replay metadata while clearing reusable validation records. + pub fn clear_vcir_reuse_records(&self) -> StorageResult { + let vcir_cf = self.cf(CF_VCIR)?; + let identity_cf = self.cf(CF_VCIR_FAILED_FETCH_REUSE_IDENTITY)?; + let mut batch = WriteBatch::default(); + let mut summary = VcirReuseRecordsCleared::default(); + + for entry in self.db.iterator_cf(vcir_cf, IteratorMode::Start) { + let (key, _value) = entry.map_err(|error| StorageError::RocksDb(error.to_string()))?; + batch.delete_cf(vcir_cf, key); + summary.vcir_records += 1; + } + for entry in self.db.iterator_cf(identity_cf, IteratorMode::Start) { + let (key, _value) = entry.map_err(|error| StorageError::RocksDb(error.to_string()))?; + batch.delete_cf(identity_cf, key); + summary.failed_fetch_identity_records += 1; + } + if summary.vcir_records > 0 || summary.failed_fetch_identity_records > 0 { + self.write_batch(batch)?; + } + Ok(summary) + } + + pub fn summarize_vcir_storage(&self) -> StorageResult { + let cf = self.cf(CF_VCIR)?; + let mode = IteratorMode::Start; + let mut summary = VcirStorageSummary::default(); + for res in self.db.iterator_cf(cf, mode) { + let (_key, bytes) = res.map_err(|e| StorageError::RocksDb(e.to_string()))?; + let vcir = decode_cbor::(&bytes, "vcir")?; + vcir.validate_internal()?; + summary.entry_count += 1; + let value_bytes = bytes.len() as u64; + summary.vcir_value_bytes += value_bytes; + if value_bytes > summary.vcir_value_bytes_max { + summary.vcir_value_bytes_max = value_bytes; + summary.vcir_value_bytes_max_manifest_rsync_uri = + Some(vcir.manifest_rsync_uri.clone()); + } + let entry_summary = VcirStorageEntrySummary::from_vcir(&vcir, value_bytes); + summary.core_fields.add_assign(&entry_summary.core_fields); + summary + .ccr_projection + .add_assign(&entry_summary.ccr_projection); + summary + .child_resources + .add_assign(&entry_summary.child_resources); + summary.field_sizes.add_assign(&entry_summary.field_sizes); + push_top_vcir_storage_entry( + &mut summary.top_entries_by_vcir_value_bytes, + entry_summary, + ); + } + summary.local_output_old_projection_bytes = + summary.field_sizes.local_output_old_projection_bytes(); + summary.local_output_typed_projection_bytes = + summary.field_sizes.local_output_typed_projection_bytes(); + summary.local_output_projection_saved_bytes = + summary.field_sizes.local_output_projection_saved_bytes(); + Ok(summary) + } + + pub fn delete_vcir(&self, manifest_rsync_uri: &str) -> StorageResult<()> { + let vcir_cf = self.cf(CF_VCIR)?; + let failed_fetch_reuse_identity_cf = self.cf(CF_VCIR_FAILED_FETCH_REUSE_IDENTITY)?; + let replay_cf = self.cf(CF_MANIFEST_REPLAY_META)?; + let projection_cf = self.cf(CF_ROA_CACHE_PROJECTION)?; + let mut batch = WriteBatch::default(); + let key = vcir_key(manifest_rsync_uri); + batch.delete_cf(vcir_cf, key.as_bytes()); + let failed_fetch_reuse_identity_key = + vcir_failed_fetch_reuse_identity_key(manifest_rsync_uri); + batch.delete_cf( + failed_fetch_reuse_identity_cf, + failed_fetch_reuse_identity_key.as_bytes(), + ); + let replay_key = manifest_replay_meta_key(manifest_rsync_uri); + batch.delete_cf(replay_cf, replay_key.as_bytes()); + let projection_key = roa_cache_projection_key(manifest_rsync_uri); + batch.delete_cf(projection_cf, projection_key.as_bytes()); + self.write_batch(batch) + } + + pub fn put_rrdp_source_record(&self, record: &RrdpSourceRecord) -> StorageResult<()> { + record.validate_internal()?; + let cf = self.cf(CF_RRDP_SOURCE)?; + let key = rrdp_source_key(&record.notify_uri); + let value = encode_cbor(record, "rrdp_source")?; + self.db + .put_cf(cf, key.as_bytes(), value) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(()) + } + + pub fn get_rrdp_source_record( + &self, + notify_uri: &str, + ) -> StorageResult> { + let cf = self.cf(CF_RRDP_SOURCE)?; + let key = rrdp_source_key(notify_uri); + let Some(bytes) = self + .db + .get_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + else { + return Ok(None); + }; + let record = decode_cbor::(&bytes, "rrdp_source")?; + record.validate_internal()?; + Ok(Some(record)) + } + + pub fn put_rrdp_source_member_record( + &self, + record: &RrdpSourceMemberRecord, + ) -> StorageResult<()> { + record.validate_internal()?; + let cf = self.cf(CF_RRDP_SOURCE_MEMBER)?; + let key = rrdp_source_member_key(&record.notify_uri, &record.rsync_uri); + let value = encode_cbor(record, "rrdp_source_member")?; + self.db + .put_cf(cf, key.as_bytes(), value) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(()) + } + + pub fn get_rrdp_source_member_record( + &self, + notify_uri: &str, + rsync_uri: &str, + ) -> StorageResult> { + let cf = self.cf(CF_RRDP_SOURCE_MEMBER)?; + let key = rrdp_source_member_key(notify_uri, rsync_uri); + let Some(bytes) = self + .db + .get_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + else { + return Ok(None); + }; + let record = decode_cbor::(&bytes, "rrdp_source_member")?; + record.validate_internal()?; + Ok(Some(record)) + } + + pub fn list_rrdp_source_member_records( + &self, + notify_uri: &str, + ) -> StorageResult> { + let cf = self.cf(CF_RRDP_SOURCE_MEMBER)?; + let prefix = rrdp_source_member_prefix(notify_uri); + let mode = IteratorMode::From(prefix.as_bytes(), Direction::Forward); + self.db + .iterator_cf(cf, mode) + .take_while(|res| match res { + Ok((key, _)) => key.starts_with(prefix.as_bytes()), + Err(_) => false, + }) + .map(|res| { + let (_key, value) = res.map_err(|e| StorageError::RocksDb(e.to_string()))?; + let record = decode_cbor::(&value, "rrdp_source_member")?; + record.validate_internal()?; + Ok(record) + }) + .collect() + } + + pub fn list_current_rrdp_source_members( + &self, + notify_uri: &str, + ) -> StorageResult> { + let mut records = self.list_rrdp_source_member_records(notify_uri)?; + records.retain(|record| record.present); + records.sort_by(|a, b| a.rsync_uri.cmp(&b.rsync_uri)); + Ok(records) + } + + pub fn is_current_rrdp_source_member( + &self, + notify_uri: &str, + rsync_uri: &str, + ) -> StorageResult { + Ok(matches!( + self.get_rrdp_source_member_record(notify_uri, rsync_uri)?, + Some(record) if record.present + )) + } + + pub fn load_current_object_bytes_by_uri( + &self, + rsync_uri: &str, + ) -> StorageResult>> { + Ok(self + .load_current_object_with_hash_by_uri(rsync_uri)? + .map(|obj| obj.bytes)) + } + + pub fn load_current_object_with_hash_by_uri( + &self, + rsync_uri: &str, + ) -> StorageResult> { + let Some(view) = self.get_repository_view_entry(rsync_uri)? else { + return Ok(None); + }; + + match view.state { + RepositoryViewState::Withdrawn => Ok(None), + RepositoryViewState::Present | RepositoryViewState::Replaced => { + let hash = view + .current_hash + .as_deref() + .ok_or(StorageError::InvalidData { + entity: "repository_view", + detail: format!("current_hash missing for current object URI: {rsync_uri}"), + })?; + let bytes = self + .get_blob_bytes(hash)? + .ok_or(StorageError::InvalidData { + entity: "repository_view", + detail: format!( + "blob bytes missing for current object URI: {rsync_uri} (hash={hash})" + ), + })?; + let current_hash = decode_sha256_hex_32("repository_view.current_hash", hash)?; + Ok(Some(CurrentObjectWithHash { + current_hash_hex: hash.to_ascii_lowercase(), + current_hash, + bytes, + })) + } + } + } + + pub fn put_rrdp_uri_owner_record(&self, record: &RrdpUriOwnerRecord) -> StorageResult<()> { + record.validate_internal()?; + let cf = self.cf(CF_RRDP_URI_OWNER)?; + let key = rrdp_uri_owner_key(&record.rsync_uri); + let value = encode_cbor(record, "rrdp_uri_owner")?; + self.db + .put_cf(cf, key.as_bytes(), value) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(()) + } + + pub fn get_rrdp_uri_owner_record( + &self, + rsync_uri: &str, + ) -> StorageResult> { + let cf = self.cf(CF_RRDP_URI_OWNER)?; + let key = rrdp_uri_owner_key(rsync_uri); + let Some(bytes) = self + .db + .get_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + else { + return Ok(None); + }; + let record = decode_cbor::(&bytes, "rrdp_uri_owner")?; + record.validate_internal()?; + Ok(Some(record)) + } + + pub fn delete_rrdp_uri_owner_record(&self, rsync_uri: &str) -> StorageResult<()> { + let cf = self.cf(CF_RRDP_URI_OWNER)?; + let key = rrdp_uri_owner_key(rsync_uri); + self.db + .delete_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(()) + } + + #[allow(dead_code)] + + pub fn write_batch(&self, batch: WriteBatch) -> StorageResult<()> { + self.db + .write(batch) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(()) + } +} diff --git a/crates/panda-rpki-validator/src/storage/store_vcir.rs b/crates/panda-rpki-validator/src/storage/store_vcir.rs new file mode 100644 index 0000000..cea5e82 --- /dev/null +++ b/crates/panda-rpki-validator/src/storage/store_vcir.rs @@ -0,0 +1,233 @@ +// VCIR replacement and failed-fetch reuse operations. + +impl RocksStore { + + pub fn put_vcir(&self, vcir: &ValidatedCaInstanceResult) -> StorageResult<()> { + self.put_vcir_with_publication_point_cache_projection(vcir, None) + } + + pub fn put_vcir_with_failed_fetch_reuse_identity( + &self, + vcir: &ValidatedCaInstanceResult, + failed_fetch_reuse_identity: &VcirFailedFetchReuseIdentity, + ) -> StorageResult<()> { + self.put_vcir_with_projection_action( + vcir, + None, + PublicationPointCacheProjectionWriteAction::Keep, + Some(failed_fetch_reuse_identity), + ) + } + + pub fn put_vcir_with_publication_point_cache_projection( + &self, + vcir: &ValidatedCaInstanceResult, + publication_point_projection: Option<&PublicationPointCacheProjection>, + ) -> StorageResult<()> { + self.put_vcir_with_projections(vcir, None, publication_point_projection) + } + + pub fn put_vcir_with_projections( + &self, + vcir: &ValidatedCaInstanceResult, + roa_cache_context: Option<&RoaCacheProjectionContext>, + publication_point_projection: Option<&PublicationPointCacheProjection>, + ) -> StorageResult<()> { + let publication_point_projection_action = publication_point_projection + .map(PublicationPointCacheProjectionWriteAction::Write) + .unwrap_or(PublicationPointCacheProjectionWriteAction::Keep); + self.put_vcir_with_projection_action( + vcir, + roa_cache_context, + publication_point_projection_action, + None, + ) + } + + fn put_vcir_with_projection_action( + &self, + vcir: &ValidatedCaInstanceResult, + roa_cache_context: Option<&RoaCacheProjectionContext>, + publication_point_projection_action: PublicationPointCacheProjectionWriteAction<'_>, + failed_fetch_reuse_identity: Option<&VcirFailedFetchReuseIdentity>, + ) -> StorageResult<()> { + vcir.validate_internal()?; + let vcir_cf = self.cf(CF_VCIR)?; + let failed_fetch_reuse_identity_cf = self.cf(CF_VCIR_FAILED_FETCH_REUSE_IDENTITY)?; + let replay_cf = self.cf(CF_MANIFEST_REPLAY_META)?; + let projection_cf = self.cf(CF_ROA_CACHE_PROJECTION)?; + let pp_projection_cf = self.cf(CF_PUBLICATION_POINT_CACHE_PROJECTION)?; + let replay_meta = ManifestReplayMeta::from_vcir(vcir); + replay_meta.validate_internal()?; + let mut batch = WriteBatch::default(); + let key = vcir_key(&vcir.manifest_rsync_uri); + let value = encode_cbor(vcir, "vcir")?; + batch.put_cf(vcir_cf, key.as_bytes(), value); + write_vcir_failed_fetch_reuse_identity_to_batch( + failed_fetch_reuse_identity_cf, + &mut batch, + &vcir.manifest_rsync_uri, + failed_fetch_reuse_identity, + )?; + let replay_key = manifest_replay_meta_key(&replay_meta.manifest_rsync_uri); + let replay_value = encode_cbor(&replay_meta, "manifest_replay_meta")?; + batch.put_cf(replay_cf, replay_key.as_bytes(), replay_value); + write_roa_cache_projection_to_batch( + projection_cf, + &mut batch, + vcir, + roa_cache_context, + None, + )?; + write_publication_point_cache_projection_to_batch( + pp_projection_cf, + &mut batch, + publication_point_projection_action, + None, + )?; + self.write_batch(batch)?; + self.apply_publication_point_cache_projection_index_action( + publication_point_projection_action, + )?; + Ok(()) + } + + pub fn replace_vcir_and_manifest_replay_meta( + &self, + vcir: &ValidatedCaInstanceResult, + ) -> StorageResult { + self.replace_vcir_manifest_replay_meta_and_publication_point_cache_projection(vcir, None) + } + + pub fn replace_vcir_manifest_replay_meta_and_publication_point_cache_projection( + &self, + vcir: &ValidatedCaInstanceResult, + publication_point_projection: Option<&PublicationPointCacheProjection>, + ) -> StorageResult { + self.replace_vcir_manifest_replay_meta_and_projections( + vcir, + None, + publication_point_projection, + ) + } + + pub fn replace_vcir_manifest_replay_meta_and_projections( + &self, + vcir: &ValidatedCaInstanceResult, + roa_cache_context: Option<&RoaCacheProjectionContext>, + publication_point_projection: Option<&PublicationPointCacheProjection>, + ) -> StorageResult { + let publication_point_projection_action = publication_point_projection + .map(PublicationPointCacheProjectionWriteAction::Write) + .unwrap_or(PublicationPointCacheProjectionWriteAction::Keep); + self.replace_vcir_manifest_replay_meta_and_projection_action( + vcir, + roa_cache_context, + publication_point_projection_action, + ) + } + + pub(crate) fn replace_vcir_manifest_replay_meta_and_projection_action( + &self, + vcir: &ValidatedCaInstanceResult, + roa_cache_context: Option<&RoaCacheProjectionContext>, + publication_point_projection_action: PublicationPointCacheProjectionWriteAction<'_>, + ) -> StorageResult { + self.replace_vcir_manifest_replay_meta_and_projection_action_with_failed_fetch_reuse_identity( + vcir, + roa_cache_context, + publication_point_projection_action, + None, + ) + } + + pub(crate) fn replace_vcir_manifest_replay_meta_and_projection_action_with_failed_fetch_reuse_identity( + &self, + vcir: &ValidatedCaInstanceResult, + roa_cache_context: Option<&RoaCacheProjectionContext>, + publication_point_projection_action: PublicationPointCacheProjectionWriteAction<'_>, + failed_fetch_reuse_identity: Option<&VcirFailedFetchReuseIdentity>, + ) -> StorageResult { + let mut timing = VcirReplaceTimingBreakdown { + rss_before_kb: process_vm_rss_kb(), + ..VcirReplaceTimingBreakdown::default() + }; + + let validate_started = std::time::Instant::now(); + vcir.validate_internal()?; + timing.validate_ms = validate_started.elapsed().as_millis() as u64; + timing.rss_after_validate_kb = process_vm_rss_kb(); + timing.field_sizes = VcirFieldSizeBreakdown::from_vcir(vcir); + + let batch_build_started = std::time::Instant::now(); + let vcir_cf = self.cf(CF_VCIR)?; + let failed_fetch_reuse_identity_cf = self.cf(CF_VCIR_FAILED_FETCH_REUSE_IDENTITY)?; + let replay_cf = self.cf(CF_MANIFEST_REPLAY_META)?; + let projection_cf = self.cf(CF_ROA_CACHE_PROJECTION)?; + let pp_projection_cf = self.cf(CF_PUBLICATION_POINT_CACHE_PROJECTION)?; + let mut batch = WriteBatch::default(); + + let vcir_key = vcir_key(&vcir.manifest_rsync_uri); + let vcir_encode_started = std::time::Instant::now(); + let vcir_value = encode_cbor(vcir, "vcir")?; + timing.vcir_encode_ms = vcir_encode_started.elapsed().as_millis() as u64; + timing.vcir_value_bytes = vcir_value.len() as u64; + batch.put_cf(vcir_cf, vcir_key.as_bytes(), vcir_value); + timing.rss_after_vcir_encode_kb = process_vm_rss_kb(); + write_vcir_failed_fetch_reuse_identity_to_batch( + failed_fetch_reuse_identity_cf, + &mut batch, + &vcir.manifest_rsync_uri, + failed_fetch_reuse_identity, + )?; + + let replay_meta_encode_started = std::time::Instant::now(); + let replay_meta = ManifestReplayMeta::from_vcir(vcir); + replay_meta.validate_internal()?; + let replay_key = manifest_replay_meta_key(&replay_meta.manifest_rsync_uri); + let replay_value = encode_cbor(&replay_meta, "manifest_replay_meta")?; + timing.replay_meta_encode_ms = replay_meta_encode_started.elapsed().as_millis() as u64; + timing.replay_meta_value_bytes = replay_value.len() as u64; + batch.put_cf(replay_cf, replay_key.as_bytes(), replay_value); + timing.rss_after_replay_meta_encode_kb = process_vm_rss_kb(); + + let projection_encode_started = std::time::Instant::now(); + write_roa_cache_projection_to_batch( + projection_cf, + &mut batch, + vcir, + roa_cache_context, + Some(&mut timing), + )?; + timing.roa_cache_projection_encode_ms = + projection_encode_started.elapsed().as_millis() as u64; + timing.rss_after_roa_cache_projection_encode_kb = process_vm_rss_kb(); + + let pp_projection_encode_started = std::time::Instant::now(); + write_publication_point_cache_projection_to_batch( + pp_projection_cf, + &mut batch, + publication_point_projection_action, + Some(&mut timing), + )?; + timing.publication_point_cache_projection_encode_ms = + pp_projection_encode_started.elapsed().as_millis() as u64; + timing.rss_after_publication_point_cache_projection_encode_kb = process_vm_rss_kb(); + + timing.total_encoded_bytes = timing.vcir_value_bytes + + timing.replay_meta_value_bytes + + timing.roa_cache_projection_value_bytes + + timing.publication_point_cache_projection_value_bytes; + timing.batch_build_ms = batch_build_started.elapsed().as_millis() as u64; + + let write_batch_started = std::time::Instant::now(); + self.write_batch(batch)?; + self.apply_publication_point_cache_projection_index_action( + publication_point_projection_action, + )?; + timing.write_batch_ms = write_batch_started.elapsed().as_millis() as u64; + timing.rss_after_write_batch_kb = process_vm_rss_kb(); + Ok(timing) + } + +} diff --git a/crates/panda-rpki-validator/src/storage/tests.rs b/crates/panda-rpki-validator/src/storage/tests.rs index 34d8f3b..1c80d3f 100644 --- a/crates/panda-rpki-validator/src/storage/tests.rs +++ b/crates/panda-rpki-validator/src/storage/tests.rs @@ -1,2460 +1,10 @@ +// Storage tests are grouped by the public storage surface they protect. use super::*; -fn pack_time(hour: i64) -> PackTime { - PackTime::from_utc_offset_datetime( - time::OffsetDateTime::UNIX_EPOCH + time::Duration::hours(hour), - ) -} - -fn sha256_hex(input: &[u8]) -> String { - hex::encode(compute_sha256_32(input)) -} - -fn sha256_32(input: &[u8]) -> [u8; 32] { - compute_sha256_32(input) -} - -fn sample_child_certificate_cache_projection( - cache_key_sha256_hex: String, - child_cert_uri: &str, - child_cert_sha256_hex: &str, -) -> ChildCertificateCacheProjection { - ChildCertificateCacheProjection { - schema_version: CHILD_CERTIFICATE_CACHE_SCHEMA_VERSION, - algorithm_version: CHILD_CERTIFICATE_CACHE_ALGORITHM_VERSION, - cache_key_sha256_hex, - child_cert_uri: child_cert_uri.to_string(), - child_cert_sha256_hex: child_cert_sha256_hex.to_string(), - child_cert_serial: vec![1], - issuer_ca_sha256_hex: sha256_hex(b"issuer-ca"), - issuer_crl_uri: "rsync://example.test/repo/issuer.crl".to_string(), - issuer_crl_sha256_hex: sha256_hex(b"issuer-crl"), - ca_validation_context_digest: sha256_32(b"parent-context"), - validation_policy_fingerprint: sha256_32(b"policy"), - effective_not_before: pack_time(0), - effective_until: pack_time(24), - payload: ChildCertificateCachePayload::ChildCa { - child_manifest_rsync_uri: format!("{child_cert_uri}.mft"), - child_ski: "11".repeat(20), - child_rsync_base_uri: "rsync://example.test/repo/child/".to_string(), - child_publication_point_rsync_uri: "rsync://example.test/repo/child/".to_string(), - child_rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), - child_effective_ip_resources: None, - child_effective_as_resources: None, - }, - } -} - -#[test] -fn parse_work_db_blob_mode_accepts_supported_values() { - assert_eq!(default_work_db_blob_mode(), WorkDbBlobMode::Disabled); - assert_eq!( - parse_work_db_blob_mode("default"), - Some(WorkDbBlobMode::Disabled) - ); - assert_eq!( - parse_work_db_blob_mode("current"), - Some(WorkDbBlobMode::Current) - ); - assert_eq!( - parse_work_db_blob_mode("legacy"), - Some(WorkDbBlobMode::Current) - ); - assert_eq!( - parse_work_db_blob_mode("disabled"), - Some(WorkDbBlobMode::Disabled) - ); - assert_eq!( - parse_work_db_blob_mode("no-blob"), - Some(WorkDbBlobMode::Disabled) - ); - assert_eq!(parse_work_db_blob_mode("lz4"), Some(WorkDbBlobMode::Lz4)); - assert_eq!( - parse_work_db_blob_mode("blob-lz4"), - Some(WorkDbBlobMode::Lz4) - ); - assert_eq!(parse_work_db_blob_mode("unexpected"), None); -} - -#[test] -fn parse_work_db_memory_profile_accepts_supported_values() { - assert_eq!( - parse_work_db_memory_profile("default"), - Some(WorkDbMemoryProfile::Default) - ); - assert_eq!( - parse_work_db_memory_profile("none"), - Some(WorkDbMemoryProfile::Default) - ); - assert_eq!( - parse_work_db_memory_profile("compact"), - Some(WorkDbMemoryProfile::Compact) - ); - assert_eq!( - parse_work_db_memory_profile("low-memory"), - Some(WorkDbMemoryProfile::Compact) - ); - assert_eq!(parse_work_db_memory_profile("unexpected"), None); -} - -#[test] -fn vcir_field_size_breakdown_counts_local_outputs_and_artifacts() { - let vcir = sample_vcir("rsync://example.test/repo/current.mft"); - let breakdown = VcirFieldSizeBreakdown::from_vcir(&vcir); - assert_eq!(breakdown.local_output_count, 2); - assert_eq!( - breakdown.local_output_payload_json_bytes, - vcir.local_outputs - .iter() - .map(|output| output.payload_json().len() as u64) - .sum::() - ); - assert_eq!( - breakdown.local_output_rule_hash_hex_bytes, - vcir.local_outputs.len() as u64 * 64 - ); - assert_eq!(breakdown.related_artifact_count, 2); - assert!(breakdown.related_artifact_uri_bytes > 0); - assert_eq!(breakdown.child_entry_count, 1); - assert!(breakdown.child_entry_uri_bytes > 0); - assert_eq!( - breakdown.local_output_old_projection_bytes(), - breakdown.local_output_source_type_bytes - + breakdown.local_output_source_hash_hex_bytes - + breakdown.local_output_source_ee_hash_hex_bytes - + breakdown.local_output_payload_json_bytes - + breakdown.local_output_rule_hash_hex_bytes - ); - assert!( - breakdown.local_output_old_projection_bytes() - > breakdown.local_output_typed_projection_bytes() - ); -} - -fn sample_repository_view_entry(rsync_uri: &str, bytes: &[u8]) -> RepositoryViewEntry { - RepositoryViewEntry { - rsync_uri: rsync_uri.to_string(), - current_hash: Some(sha256_hex(bytes)), - repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), - object_type: Some("cer".to_string()), - state: RepositoryViewState::Present, - } -} - -fn sample_raw_by_hash_entry(bytes: Vec) -> RawByHashEntry { - RawByHashEntry { - sha256_hex: sha256_hex(&bytes), - bytes, - origin_uris: vec!["rsync://example.test/repo/object.cer".to_string()], - object_type: Some("cer".to_string()), - encoding: Some("der".to_string()), - } -} - -fn sample_ccr_manifest_projection( - manifest_rsync_uri: &str, - manifest_this_update: PackTime, - subordinate_skis: Vec>, -) -> VcirCcrManifestProjection { - VcirCcrManifestProjection { - manifest_rsync_uri: manifest_rsync_uri.to_string(), - manifest_sha256: vec![0x11; 32], - manifest_size: 4096, - manifest_ee_aki: vec![0x22; 20], - manifest_number_be: vec![3], - manifest_this_update, - manifest_sia_locations_der: vec![vec![ - 0x30, 0x11, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x05, 0x86, 0x05, - b'r', b's', b'y', b'n', b'c', - ]], - subordinate_skis, - } -} - -fn sample_vcir(manifest_rsync_uri: &str) -> ValidatedCaInstanceResult { - let roa_bytes = b"roa-object".to_vec(); - let ee_bytes = b"ee-cert".to_vec(); - let child_bytes = b"child-cert".to_vec(); - let child_ski = "1234567890abcdef1234567890abcdef12345678".to_string(); - ValidatedCaInstanceResult { - manifest_rsync_uri: manifest_rsync_uri.to_string(), - parent_manifest_rsync_uri: Some("rsync://example.test/repo/parent/parent.mft".to_string()), - tal_id: "apnic".to_string(), - ca_subject_name: "CN=Example CA".to_string(), - ca_ski: "00112233445566778899aabbccddeeff00112233".to_string(), - issuer_ski: "ffeeddccbbaa99887766554433221100ffeeddcc".to_string(), - last_successful_validation_time: pack_time(0), - current_manifest_rsync_uri: manifest_rsync_uri.to_string(), - current_crl_rsync_uri: "rsync://example.test/repo/current.crl".to_string(), - validated_manifest_meta: ValidatedManifestMeta { - validated_manifest_number: vec![3], - validated_manifest_this_update: pack_time(0), - validated_manifest_next_update: pack_time(24), - }, - ccr_manifest_projection: sample_ccr_manifest_projection( - manifest_rsync_uri, - pack_time(0), - vec![hex::decode(&child_ski).expect("decode child ski")], - ), - instance_gate: VcirInstanceGate { - manifest_next_update: pack_time(24), - current_crl_next_update: pack_time(12), - self_ca_not_after: pack_time(48), - instance_effective_until: pack_time(12), - }, - child_entries: vec![VcirChildEntry { - child_manifest_rsync_uri: "rsync://example.test/repo/child/child.mft".to_string(), - child_cert_rsync_uri: "rsync://example.test/repo/child/child.cer".to_string(), - child_cert_hash: sha256_hex(&child_bytes), - child_ski, - child_rsync_base_uri: "rsync://example.test/repo/child/".to_string(), - child_publication_point_rsync_uri: "rsync://example.test/repo/child/".to_string(), - child_rrdp_notification_uri: Some("https://example.test/child-notify.xml".to_string()), - child_effective_ip_resources: None, - child_effective_as_resources: None, - accepted_at_validation_time: pack_time(0), - }], - local_outputs: vec![ - VcirLocalOutput { - output_type: VcirOutputType::Vrp, - item_effective_until: pack_time(12), - source_object_uri: "rsync://example.test/repo/object.roa".to_string(), - source_object_type: VcirSourceObjectType::Roa, - source_object_hash: sha256_32(&roa_bytes), - source_ee_cert_hash: sha256_32(&ee_bytes), - payload: VcirLocalOutputPayload::Vrp { - asn: 64496, - afi: crate::data_model::roa::RoaAfi::Ipv4, - prefix_len: 24, - addr: { - let mut addr = [0u8; 16]; - addr[..4].copy_from_slice(&[203, 0, 113, 0]); - addr - }, - max_length: 24, - }, - rule_hash: sha256_32(b"vrp-rule-1"), - }, - VcirLocalOutput { - output_type: VcirOutputType::Aspa, - item_effective_until: pack_time(10), - source_object_uri: "rsync://example.test/repo/object.asa".to_string(), - source_object_type: VcirSourceObjectType::Aspa, - source_object_hash: sha256_32(b"aspa-object"), - source_ee_cert_hash: sha256_32(b"aspa-ee-cert"), - payload: VcirLocalOutputPayload::Aspa { - customer_as_id: 64496, - provider_as_ids: vec![64497], - }, - rule_hash: sha256_32(b"aspa-rule-1"), - }, - ], - related_artifacts: vec![ - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::Manifest, - artifact_kind: VcirArtifactKind::Mft, - uri: Some(manifest_rsync_uri.to_string()), - sha256: sha256_hex(b"manifest-object"), - object_type: Some("mft".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - reject_reason: None, - }, - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::CurrentCrl, - artifact_kind: VcirArtifactKind::Crl, - uri: Some("rsync://example.test/repo/current.crl".to_string()), - sha256: sha256_hex(b"current-crl"), - object_type: Some("crl".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - reject_reason: None, - }, - ], - summary: VcirSummary { - local_vrp_count: 1, - local_aspa_count: 1, - local_router_key_count: 0, - child_count: 1, - accepted_object_count: 4, - rejected_object_count: 0, - }, - audit_summary: VcirAuditSummary { - failed_fetch_eligible: true, - last_failed_fetch_reason: None, - warning_count: 0, - audit_flags: vec!["validated-fresh".to_string()], - }, - } -} - -fn roa_cache_projection_context(vcir: &ValidatedCaInstanceResult) -> RoaCacheProjectionContext { - let roa_output = vcir - .local_outputs - .iter() - .find(|output| { - output.output_type == VcirOutputType::Vrp - && output.source_object_type == VcirSourceObjectType::Roa - }) - .expect("sample VCIR has a ROA output"); - RoaCacheProjectionContext { - ca_validation_context_digest: [0x31; 32], - policy_fingerprint: [0x32; 32], - object_meta: vec![RoaCacheObjectMeta { - source_object_uri: roa_output.source_object_uri.clone(), - source_object_hash: roa_output.source_object_hash, - ee_serial: vec![0x01], - crl_uri: vcir.current_crl_rsync_uri.clone(), - earliest_safe_reuse_time: vcir.last_successful_validation_time.clone(), - }], - } -} - -#[test] -fn vcir_ccr_manifest_projection_validate_accepts_valid_projection() { - let projection = sample_ccr_manifest_projection( - "rsync://example.test/repo/current.mft", - pack_time(0), - vec![vec![0x33; 20], vec![0x44; 20]], - ); - projection.validate_internal().expect("valid projection"); -} - -#[test] -fn vcir_ccr_manifest_projection_validate_rejects_invalid_fields() { - let mut bad_hash = sample_ccr_manifest_projection( - "rsync://example.test/repo/current.mft", - pack_time(0), - vec![vec![0x33; 20]], - ); - bad_hash.manifest_sha256 = vec![0x11; 31]; - assert!(matches!( - bad_hash.validate_internal(), - Err(StorageError::InvalidData { .. }) - )); - - let mut bad_locations = sample_ccr_manifest_projection( - "rsync://example.test/repo/current.mft", - pack_time(0), - vec![vec![0x33; 20]], - ); - bad_locations.manifest_sia_locations_der = vec![vec![0x04, 0x00]]; - assert!(matches!( - bad_locations.validate_internal(), - Err(StorageError::InvalidData { .. }) - )); - - let bad_subordinates = sample_ccr_manifest_projection( - "rsync://example.test/repo/current.mft", - pack_time(0), - vec![vec![0x44; 20], vec![0x33; 20]], - ); - assert!(matches!( - bad_subordinates.validate_internal(), - Err(StorageError::InvalidData { .. }) - )); -} - -#[test] -fn roa_cache_projection_from_vcir_keeps_only_roa_vrp_outputs() { - let vcir = sample_vcir("rsync://example.test/repo/current.mft"); - let context = roa_cache_projection_context(&vcir); - let projection = RoaCacheProjection::from_vcir_with_context(&vcir, Some(&context)) - .expect("projection build") - .expect("projection exists"); - - assert_eq!(projection.manifest_rsync_uri, vcir.manifest_rsync_uri); - assert_eq!(projection.instance_effective_until, pack_time(12)); - assert_eq!(projection.crl_sha256_by_uri.len(), 1); - assert_eq!(projection.entries.len(), 1); - assert_eq!( - projection.entries[0].source_object_uri, - "rsync://example.test/repo/object.roa" - ); - assert_eq!( - projection.entries[0].outputs_effective_until_unix, - 12 * 3600 - ); - assert_eq!(projection.entries[0].outputs.len(), 1); - assert!(matches!( - projection.entries[0].outputs[0].payload, - VcirLocalOutputPayload::Vrp { .. } - )); -} - -#[test] -fn roa_cache_projection_groups_multiple_outputs_by_roa_uri() { - let mut vcir = sample_vcir("rsync://example.test/repo/current.mft"); - let mut second = vcir.local_outputs[0].clone(); - second.rule_hash = sha256_32(b"vrp-rule-2"); - if let VcirLocalOutputPayload::Vrp { max_length, .. } = &mut second.payload { - *max_length = 25; - } - vcir.local_outputs.push(second); - vcir.summary.local_vrp_count = 2; - - let context = roa_cache_projection_context(&vcir); - let projection = RoaCacheProjection::from_vcir_with_context(&vcir, Some(&context)) - .expect("projection build") - .expect("projection exists"); - - assert_eq!(projection.entries.len(), 1); - assert_eq!(projection.entries[0].outputs.len(), 2); - assert_eq!( - projection.entries[0].outputs_effective_until_unix, - 12 * 3600 - ); -} - -#[test] -fn publication_point_cache_projection_roundtrips_with_vcir() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - let vcir = sample_vcir("rsync://example.test/repo/current.mft"); - let projection = PublicationPointCacheProjection::from_vcir_with_context( - &vcir, - "rsync://example.test/repo/".to_string(), - Some("rsync://example.test/repo/ca.cer".to_string()), - sha256_32(b"ca-cert"), - sha256_32(b"manifest"), - sha256_32(b"ta-context"), - sha256_32(b"parent-context"), - sha256_32(b"policy"), - ) - .expect("build publication point projection"); - - store - .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) - .expect("put vcir with publication point projection"); - - let got_vcir = store - .get_vcir(&vcir.manifest_rsync_uri) - .expect("get vcir") - .expect("vcir exists"); - assert_eq!(got_vcir, vcir); - let got_projection = store - .get_publication_point_cache_projection(&vcir.manifest_rsync_uri) - .expect("get publication point projection") - .expect("projection exists"); - assert_eq!(got_projection, projection); - assert_eq!(got_projection.outputs.len(), 2); - assert_eq!(got_projection.children.len(), 1); - assert_eq!(got_projection.related_objects.len(), 2); -} - -#[test] -fn publication_point_cache_projection_index_updates_after_first_read() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - let vcir = sample_vcir("rsync://example.test/repo/current.mft"); - let mut projection = PublicationPointCacheProjection::from_vcir_with_context( - &vcir, - "rsync://example.test/repo/".to_string(), - Some("rsync://example.test/repo/ca.cer".to_string()), - sha256_32(b"ca-cert"), - sha256_32(b"manifest-old"), - sha256_32(b"ta-context"), - sha256_32(b"parent-context"), - sha256_32(b"policy"), - ) - .expect("build old publication point projection"); - - store - .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) - .expect("put old projection"); - let got_old = store - .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) - .expect("get old projection") - .expect("old projection exists"); - assert_eq!(got_old.manifest_sha256, sha256_32(b"manifest-old")); - - projection.manifest_sha256 = sha256_32(b"manifest-new"); - store - .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) - .expect("put new projection"); - let got_new = store - .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) - .expect("get new projection") - .expect("new projection exists"); - assert_eq!(got_new.manifest_sha256, sha256_32(b"manifest-new")); -} - -#[test] -fn publication_point_cache_projection_cached_empty_db_accepts_bounded_new_entries() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - let vcir = sample_vcir("rsync://example.test/repo/current.mft"); - let projection = PublicationPointCacheProjection::from_vcir_with_context( - &vcir, - "rsync://example.test/repo/".to_string(), - Some("rsync://example.test/repo/ca.cer".to_string()), - sha256_32(b"ca-cert"), - sha256_32(b"manifest"), - sha256_32(b"ta-context"), - sha256_32(b"parent-context"), - sha256_32(b"policy"), - ) - .expect("build publication point projection"); - - assert!( - store - .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) - .expect("cached empty lookup") - .is_none() - ); - store - .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) - .expect("put projection after cached empty lookup"); - assert_eq!( - store - .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) - .expect("cached lookup sees bounded new entry") - .expect("cached projection exists"), - projection - ); - assert_eq!( - store - .get_publication_point_cache_projection(&vcir.manifest_rsync_uri) - .expect("direct db lookup") - .expect("projection exists"), - projection - ); -} - -#[test] -fn publication_point_cache_mmap_index_refresh_roundtrips_after_reopen() { - let td = tempfile::tempdir().expect("tempdir"); - let db_path = td.path().join("work-db"); - let vcir = sample_vcir("rsync://example.test/repo/current.mft"); - let projection = PublicationPointCacheProjection::from_vcir_with_context( - &vcir, - "rsync://example.test/repo/".to_string(), - Some("rsync://example.test/repo/ca.cer".to_string()), - sha256_32(b"ca-cert"), - sha256_32(b"manifest"), - sha256_32(b"ta-context"), - sha256_32(b"parent-context"), - sha256_32(b"policy"), - ) - .expect("build publication point projection"); - - { - let store = RocksStore::open(&db_path).expect("open rocksdb"); - store - .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) - .expect("put projection"); - let stats = store - .refresh_publication_point_cache_mmap_index() - .expect("refresh mmap index") - .expect("refresh stats"); - assert_eq!(stats.new_entries, 1); - assert_eq!(stats.state, "written"); - } - - let store = RocksStore::open(&db_path).expect("reopen rocksdb"); - let got = store - .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) - .expect("get cached projection from mmap") - .expect("projection exists"); - assert_eq!(got, projection); -} - -#[test] -fn publication_point_cache_mmap_index_dirty_overlay_wins_and_refreshes_segment() { - let td = tempfile::tempdir().expect("tempdir"); - let db_path = td.path().join("work-db"); - let vcir = sample_vcir("rsync://example.test/repo/current.mft"); - let mut projection = PublicationPointCacheProjection::from_vcir_with_context( - &vcir, - "rsync://example.test/repo/".to_string(), - Some("rsync://example.test/repo/ca.cer".to_string()), - sha256_32(b"ca-cert"), - sha256_32(b"manifest-old"), - sha256_32(b"ta-context"), - sha256_32(b"parent-context"), - sha256_32(b"policy"), - ) - .expect("build publication point projection"); - - { - let store = RocksStore::open(&db_path).expect("open rocksdb"); - store - .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) - .expect("put old projection"); - store - .refresh_publication_point_cache_mmap_index() - .expect("refresh base mmap index"); - } - - { - let store = RocksStore::open(&db_path).expect("reopen rocksdb"); - assert_eq!( - store - .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) - .expect("get old projection") - .expect("old projection exists") - .manifest_sha256, - sha256_32(b"manifest-old") - ); - projection.manifest_sha256 = sha256_32(b"manifest-new"); - store - .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) - .expect("put new projection"); - assert_eq!( - store - .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) - .expect("get dirty projection") - .expect("dirty projection exists") - .manifest_sha256, - sha256_32(b"manifest-new") - ); - let stats = store - .refresh_publication_point_cache_mmap_index() - .expect("refresh dirty mmap segment") - .expect("refresh stats"); - assert_eq!(stats.state, "segment_written"); - assert_eq!(stats.dirty_entries, 1); - } - - let store = RocksStore::open(&db_path).expect("reopen rocksdb after segment"); - let got = store - .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) - .expect("get refreshed projection") - .expect("projection exists"); - assert_eq!(got.manifest_sha256, sha256_32(b"manifest-new")); -} - -#[test] -fn publication_point_cache_mmap_index_write_before_read_refreshes_segment() { - let td = tempfile::tempdir().expect("tempdir"); - let db_path = td.path().join("work-db"); - let vcir = sample_vcir("rsync://example.test/repo/current.mft"); - let mut projection = PublicationPointCacheProjection::from_vcir_with_context( - &vcir, - "rsync://example.test/repo/".to_string(), - Some("rsync://example.test/repo/ca.cer".to_string()), - sha256_32(b"ca-cert"), - sha256_32(b"manifest-old"), - sha256_32(b"ta-context"), - sha256_32(b"parent-context"), - sha256_32(b"policy"), - ) - .expect("build publication point projection"); - - { - let store = RocksStore::open(&db_path).expect("open rocksdb"); - store - .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) - .expect("put old projection"); - let stats = store - .refresh_publication_point_cache_mmap_index() - .expect("refresh base mmap index") - .expect("refresh stats"); - assert_eq!(stats.state, "written"); - } - - { - let store = RocksStore::open(&db_path).expect("reopen rocksdb"); - projection.manifest_sha256 = sha256_32(b"manifest-new"); - store - .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) - .expect("put new projection before any cached read"); - let stats = store - .refresh_publication_point_cache_mmap_index() - .expect("refresh dirty mmap segment") - .expect("refresh stats"); - assert_eq!(stats.state, "segment_written"); - assert_eq!(stats.dirty_entries, 1); - assert_eq!(stats.new_entries, 1); - } - - let store = RocksStore::open(&db_path).expect("reopen rocksdb after segment"); - let got = store - .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) - .expect("get refreshed projection") - .expect("projection exists"); - assert_eq!(got.manifest_sha256, sha256_32(b"manifest-new")); -} - -#[test] -fn publication_point_cache_mmap_index_delete_before_read_refreshes_tombstone_segment() { - let td = tempfile::tempdir().expect("tempdir"); - let db_path = td.path().join("work-db"); - let vcir = sample_vcir("rsync://example.test/repo/current.mft"); - let projection = PublicationPointCacheProjection::from_vcir_with_context( - &vcir, - "rsync://example.test/repo/".to_string(), - Some("rsync://example.test/repo/ca.cer".to_string()), - sha256_32(b"ca-cert"), - sha256_32(b"manifest"), - sha256_32(b"ta-context"), - sha256_32(b"parent-context"), - sha256_32(b"policy"), - ) - .expect("build publication point projection"); - - { - let store = RocksStore::open(&db_path).expect("open rocksdb"); - store - .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) - .expect("put projection"); - let stats = store - .refresh_publication_point_cache_mmap_index() - .expect("refresh base mmap index") - .expect("refresh stats"); - assert_eq!(stats.state, "written"); - } - - { - let store = RocksStore::open(&db_path).expect("reopen rocksdb"); - store - .replace_vcir_manifest_replay_meta_and_projection_action( - &vcir, - None, - PublicationPointCacheProjectionWriteAction::Delete { - manifest_rsync_uri: &vcir.manifest_rsync_uri, - }, - ) - .expect("delete projection before any cached read"); - assert!( - store - .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) - .expect("dirty tombstone lookup") - .is_none() - ); - let stats = store - .refresh_publication_point_cache_mmap_index() - .expect("refresh tombstone mmap segment") - .expect("refresh stats"); - assert_eq!(stats.state, "segment_written"); - assert_eq!(stats.dirty_entries, 1); - assert_eq!(stats.new_entries, 1); - } - - let store = RocksStore::open(&db_path).expect("reopen rocksdb after tombstone segment"); - assert!( - store - .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) - .expect("tombstone shadows old current index") - .is_none() - ); - assert!( - store - .get_publication_point_cache_projection(&vcir.manifest_rsync_uri) - .expect("direct db lookup") - .is_none() - ); -} - -#[test] -fn publication_point_cache_projection_rejects_version_mismatch() { - let vcir = sample_vcir("rsync://example.test/repo/current.mft"); - let mut projection = PublicationPointCacheProjection::from_vcir_with_context( - &vcir, - "rsync://example.test/repo/".to_string(), - Some("rsync://example.test/repo/ca.cer".to_string()), - sha256_32(b"ca-cert"), - sha256_32(b"manifest"), - sha256_32(b"ta-context"), - sha256_32(b"parent-context"), - sha256_32(b"policy"), - ) - .expect("build publication point projection"); - projection.schema_version = PUBLICATION_POINT_CACHE_SCHEMA_VERSION + 1; - assert!(matches!( - projection.validate_internal(), - Err(StorageError::InvalidData { .. }) - )); -} - -#[test] -fn roa_cache_projection_rejects_duplicate_uri_with_different_hash() { - let mut vcir = sample_vcir("rsync://example.test/repo/current.mft"); - let mut duplicate = vcir.local_outputs[0].clone(); - duplicate.source_object_hash = sha256_32(b"different-roa"); - duplicate.rule_hash = sha256_32(b"vrp-rule-2"); - vcir.local_outputs.push(duplicate); - vcir.summary.local_vrp_count = 2; - - let context = roa_cache_projection_context(&vcir); - let err = RoaCacheProjection::from_vcir_with_context(&vcir, Some(&context)) - .expect_err("same ROA URI with different hash must fail"); - assert!(err.to_string().contains("source object hash mismatch")); -} - -fn sample_rrdp_source_record(notify_uri: &str) -> RrdpSourceRecord { - RrdpSourceRecord { - notify_uri: notify_uri.to_string(), - last_session_id: Some("session-1".to_string()), - last_serial: Some(42), - first_seen_at: pack_time(0), - last_seen_at: pack_time(1), - last_sync_at: Some(pack_time(1)), - sync_state: RrdpSourceSyncState::DeltaReady, - last_snapshot_uri: Some("https://rrdp.example.test/snapshot.xml".to_string()), - last_snapshot_hash: Some(sha256_hex(b"snapshot-bytes")), - last_error: None, - } -} - -fn sample_rrdp_source_member_record( - notify_uri: &str, - rsync_uri: &str, - serial: u64, -) -> RrdpSourceMemberRecord { - RrdpSourceMemberRecord { - notify_uri: notify_uri.to_string(), - rsync_uri: rsync_uri.to_string(), - current_hash: Some(sha256_hex(rsync_uri.as_bytes())), - object_type: Some("cer".to_string()), - present: true, - last_confirmed_session_id: "session-1".to_string(), - last_confirmed_serial: serial, - last_changed_at: pack_time(serial as i64), - } -} - -fn sample_rrdp_uri_owner_record(notify_uri: &str, rsync_uri: &str) -> RrdpUriOwnerRecord { - RrdpUriOwnerRecord { - rsync_uri: rsync_uri.to_string(), - notify_uri: notify_uri.to_string(), - current_hash: Some(sha256_hex(rsync_uri.as_bytes())), - last_confirmed_session_id: "session-1".to_string(), - last_confirmed_serial: 7, - last_changed_at: pack_time(7), - owner_state: RrdpUriOwnerState::Active, - } -} - -#[test] -fn repository_view_and_raw_by_hash_roundtrip() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let entry1 = sample_repository_view_entry("rsync://example.test/repo/a.cer", b"object-a"); - let entry2 = sample_repository_view_entry("rsync://example.test/repo/sub/b.roa", b"object-b"); - store - .put_repository_view_entry(&entry1) - .expect("put repository view entry1"); - store - .put_repository_view_entry(&entry2) - .expect("put repository view entry2"); - - let got1 = store - .get_repository_view_entry(&entry1.rsync_uri) - .expect("get repository view entry1") - .expect("entry1 exists"); - assert_eq!(got1, entry1); - - let got_prefix = store - .list_repository_view_entries_with_prefix("rsync://example.test/repo/sub/") - .expect("list repository view prefix"); - assert_eq!(got_prefix, vec![entry2.clone()]); - - store - .delete_repository_view_entry(&entry1.rsync_uri) - .expect("delete repository view entry1"); - assert!( - store - .get_repository_view_entry(&entry1.rsync_uri) - .expect("get deleted repository view entry1") - .is_none() - ); - - let raw = sample_raw_by_hash_entry(b"raw-der-object".to_vec()); - store - .put_raw_by_hash_entry(&raw) - .expect("put raw_by_hash entry"); - let got_raw = store - .get_raw_by_hash_entry(&raw.sha256_hex) - .expect("get raw_by_hash entry") - .expect("raw entry exists"); - assert_eq!(got_raw, raw); -} - -#[test] -fn raw_by_hash_routes_to_external_raw_store_when_configured() { - let td = tempfile::tempdir().expect("tempdir"); - let main_db = td.path().join("main-db"); - let raw_db = td.path().join("raw-store.db"); - - let raw = sample_raw_by_hash_entry(b"external-raw".to_vec()); - { - let store = - RocksStore::open_with_external_raw_store(&main_db, &raw_db).expect("open store"); - store.put_raw_by_hash_entry(&raw).expect("put external raw"); - - let got = store - .get_raw_by_hash_entry(&raw.sha256_hex) - .expect("get external raw") - .expect("raw exists"); - assert_eq!(got, raw); - } - - let main_store = RocksStore::open(&main_db).expect("open main only"); - assert!( - main_store - .get_raw_by_hash_entry(&raw.sha256_hex) - .expect("read main store") - .is_none(), - "main db should not contain raw entry when external raw store is configured" - ); -} - -#[test] -fn put_blob_bytes_batch_uses_internal_blob_cf_without_raw_entry() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - let bytes = b"internal-blob-only".to_vec(); - let hash = sha256_hex(&bytes); - - store - .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) - .expect("put blob bytes"); - - assert_eq!( - store.get_blob_bytes(&hash).expect("get blob bytes"), - Some(bytes.clone()) - ); - assert!( - store - .get_raw_by_hash_entry(&hash) - .expect("get raw entry") - .is_none() - ); -} - -#[test] -fn put_blob_bytes_batch_routes_to_external_raw_store_without_raw_entry() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open_with_external_raw_store( - &td.path().join("main-db"), - &td.path().join("raw-store.db"), - ) - .expect("open store"); - let bytes = b"external-blob-only".to_vec(); - let hash = sha256_hex(&bytes); - - store - .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) - .expect("put external blob bytes"); - - assert_eq!(store.get_blob_bytes(&hash).unwrap(), Some(bytes)); - assert!(store.get_raw_by_hash_entry(&hash).unwrap().is_none()); -} - -#[test] -fn repo_bytes_db_is_physically_separate_from_external_raw_store() { - let td = tempfile::tempdir().expect("tempdir"); - let main_db = td.path().join("main-db"); - let raw_db = td.path().join("raw-store.db"); - let repo_bytes_db = td.path().join("repo-bytes.db"); - let store = - RocksStore::open_with_external_stores(&main_db, Some(&raw_db), Some(&repo_bytes_db)) - .expect("open store"); - let repo_bytes = b"repo-object".to_vec(); - let repo_hash = sha256_hex(&repo_bytes); - let raw = sample_raw_by_hash_entry(b"raw-evidence".to_vec()); - - store - .put_blob_bytes_batch(&[(repo_hash.clone(), repo_bytes.clone())]) - .expect("put repo bytes"); - store.put_raw_by_hash_entry(&raw).expect("put raw evidence"); - - assert_eq!(store.get_blob_bytes(&repo_hash).unwrap(), Some(repo_bytes)); - assert_eq!( - store.get_raw_by_hash_entry(&raw.sha256_hex).unwrap(), - Some(raw.clone()) - ); - drop(store); - - let raw_only = RocksStore::open_with_external_raw_store(&td.path().join("raw-reader"), &raw_db) - .expect("open raw only"); - assert!( - raw_only.get_blob_bytes(&repo_hash).unwrap().is_none(), - "repo object bytes must not be written into raw-store.db" - ); - - let repo_only = - RocksStore::open_with_external_repo_bytes(&td.path().join("repo-reader"), &repo_bytes_db) - .expect("open repo bytes only"); - assert_eq!( - repo_only.get_blob_bytes(&repo_hash).unwrap(), - Some(b"repo-object".to_vec()) - ); - assert!( - repo_only.get_blob_bytes(&raw.sha256_hex).unwrap().is_none(), - "raw evidence bytes must not be written into repo-bytes.db" - ); -} - -#[test] -fn memory_snapshot_includes_work_db_and_external_stores() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open_with_external_stores( - &td.path().join("main-db"), - Some(&td.path().join("raw-store.db")), - Some(&td.path().join("repo-bytes.db")), - ) - .expect("open store"); - - let snapshot = store.memory_snapshot(); - let labels: Vec<&str> = snapshot - .databases - .iter() - .map(|db| db.label.as_str()) - .collect(); - assert_eq!(labels, vec!["work-db", "raw-store.db", "repo-bytes.db"]); - assert!( - snapshot.databases[0] - .column_families - .iter() - .any(|cf| cf.name == CF_REPOSITORY_VIEW) - ); - serde_json::to_value(&snapshot).expect("serialize memory snapshot"); -} - -#[test] -fn put_blob_bytes_batch_accepts_empty_batch_with_external_raw_store() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open_with_external_raw_store( - &td.path().join("main-db"), - &td.path().join("raw-store.db"), - ) - .expect("open store"); - - store - .put_blob_bytes_batch(&[]) - .expect("empty external blob batch should be a no-op"); -} - -#[test] -fn get_blob_bytes_internal_falls_back_to_raw_entry_when_blob_missing() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - let raw = sample_raw_by_hash_entry(b"raw-fallback".to_vec()); - - store.put_raw_by_hash_entry(&raw).expect("put raw entry"); - - assert_eq!( - store - .get_blob_bytes(&raw.sha256_hex) - .expect("get blob bytes via raw fallback"), - Some(raw.bytes.clone()) - ); -} - -#[test] -fn get_blob_bytes_batch_internal_prefers_blob_cf_and_falls_back_to_raw_entry() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let blob_bytes = b"blob-cf-object".to_vec(); - let blob_hash = sha256_hex(&blob_bytes); - store - .put_blob_bytes_batch(&[(blob_hash.clone(), blob_bytes.clone())]) - .expect("put blob bytes"); - - let raw = sample_raw_by_hash_entry(b"raw-fallback-batch".to_vec()); - store.put_raw_by_hash_entry(&raw).expect("put raw fallback"); - - let batch = store - .get_blob_bytes_batch(&[blob_hash.clone(), raw.sha256_hex.clone(), "00".repeat(32)]) - .expect("get blob bytes batch"); - assert_eq!(batch, vec![Some(blob_bytes), Some(raw.bytes.clone()), None]); -} - -#[test] -fn get_blob_bytes_batch_routes_to_external_raw_store_without_raw_entry() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open_with_external_raw_store( - &td.path().join("main-db"), - &td.path().join("raw-store.db"), - ) - .expect("open store"); - let bytes = b"external-batch-blob".to_vec(); - let hash = sha256_hex(&bytes); - - store - .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) - .expect("put external blob bytes"); - - assert_eq!( - store - .get_blob_bytes_batch(&[hash, "00".repeat(32)]) - .expect("get external blob batch"), - vec![Some(bytes), None] - ); -} - -#[test] -fn get_blob_bytes_rejects_invalid_hash_for_internal_store() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let err = store - .get_blob_bytes("not-a-valid-hash") - .expect_err("invalid hash must fail"); - assert!(matches!(err, StorageError::InvalidData { .. })); -} - -#[test] -fn get_blob_bytes_batch_rejects_invalid_hash_for_internal_store() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let err = store - .get_blob_bytes_batch(&["not-a-valid-hash".to_string()]) - .expect_err("invalid hash must fail"); - assert!(matches!(err, StorageError::InvalidData { .. })); -} - -#[test] -fn get_blob_bytes_batch_returns_empty_for_empty_request_internal() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - assert!( - store - .get_blob_bytes_batch(&[]) - .expect("empty blob batch request") - .is_empty() - ); -} - -#[test] -fn put_blob_bytes_batch_accepts_empty_batch() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - store - .put_blob_bytes_batch(&[]) - .expect("empty blob batch should be a no-op"); -} - -#[test] -fn put_blob_bytes_batch_rejects_empty_bytes() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let err = store - .put_blob_bytes_batch(&[(sha256_hex(b"valid"), Vec::new())]) - .expect_err("empty bytes must fail"); - assert!(matches!(err, StorageError::InvalidData { .. })); -} - -#[test] -fn delete_raw_by_hash_entry_internal_preserves_blob_bytes() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - let bytes = b"blob-persists-after-raw-delete".to_vec(); - let hash = sha256_hex(&bytes); - let raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone()); - - store - .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) - .expect("put blob bytes"); - store.put_raw_by_hash_entry(&raw).expect("put raw entry"); - - store - .delete_raw_by_hash_entry(&hash) - .expect("delete raw entry only"); - - assert!(store.get_raw_by_hash_entry(&hash).unwrap().is_none()); - assert_eq!(store.get_blob_bytes(&hash).unwrap(), Some(bytes)); -} - -#[test] -fn delete_raw_by_hash_entry_rejects_invalid_hash() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let err = store - .delete_raw_by_hash_entry("not-a-valid-hash") - .expect_err("invalid hash must fail"); - assert!(matches!(err, StorageError::InvalidData { .. })); -} - -#[test] -fn delete_raw_by_hash_entry_routes_to_external_raw_store() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open_with_external_raw_store( - &td.path().join("main-db"), - &td.path().join("raw-store.db"), - ) - .expect("open store"); - let raw = sample_raw_by_hash_entry(b"external-delete".to_vec()); - - store.put_raw_by_hash_entry(&raw).expect("put raw entry"); - store - .delete_raw_by_hash_entry(&raw.sha256_hex) - .expect("delete external raw entry"); - - assert!( - store - .get_raw_by_hash_entry(&raw.sha256_hex) - .unwrap() - .is_none() - ); - assert!(store.get_blob_bytes(&raw.sha256_hex).unwrap().is_none()); -} - -#[test] -fn repository_view_and_raw_by_hash_validation_errors_are_reported() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let invalid_view = RepositoryViewEntry { - rsync_uri: "rsync://example.test/repo/withdrawn.cer".to_string(), - current_hash: None, - repository_source: None, - object_type: None, - state: RepositoryViewState::Present, - }; - let err = store - .put_repository_view_entry(&invalid_view) - .expect_err("missing current_hash must fail"); - assert!(err.to_string().contains("current_hash is required")); - - let invalid_raw = RawByHashEntry { - sha256_hex: sha256_hex(b"expected"), - bytes: b"actual".to_vec(), - origin_uris: vec!["rsync://example.test/repo/object.cer".to_string()], - object_type: None, - encoding: None, - }; - let err = store - .put_raw_by_hash_entry(&invalid_raw) - .expect_err("mismatched raw_by_hash entry must fail"); - assert!(err.to_string().contains("does not match bytes")); -} - -#[test] -fn vcir_roundtrip_and_validation_failures_are_reported() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let vcir = sample_vcir("rsync://example.test/repo/current.mft"); - let roa_context = roa_cache_projection_context(&vcir); - store - .put_vcir_with_projections(&vcir, Some(&roa_context), None) - .expect("put vcir"); - let got = store - .get_vcir(&vcir.manifest_rsync_uri) - .expect("get vcir") - .expect("vcir exists"); - assert_eq!(got, vcir); - let replay_meta = store - .get_manifest_replay_meta(&vcir.manifest_rsync_uri) - .expect("get manifest replay meta") - .expect("manifest replay meta exists"); - assert_eq!( - replay_meta, - ManifestReplayMeta { - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - manifest_number_be: vcir - .validated_manifest_meta - .validated_manifest_number - .clone(), - manifest_this_update: vcir - .validated_manifest_meta - .validated_manifest_this_update - .clone(), - manifest_sha256: vcir.ccr_manifest_projection.manifest_sha256.clone(), - updated_at_validation_time: vcir.last_successful_validation_time.clone(), - } - ); - let projection = store - .get_roa_cache_projection(&vcir.manifest_rsync_uri) - .expect("get roa cache projection") - .expect("roa cache projection exists"); - assert_eq!(projection.manifest_rsync_uri, vcir.manifest_rsync_uri); - assert_eq!(projection.entries.len(), 1); - assert_eq!( - projection.entries[0].source_object_uri, - "rsync://example.test/repo/object.roa" - ); - - let mut invalid = sample_vcir("rsync://example.test/repo/invalid.mft"); - invalid.summary.local_vrp_count = 9; - let err = store - .put_vcir(&invalid) - .expect_err("invalid vcir must fail"); - assert!(err.to_string().contains("local_vrp_count=9")); - - let mut invalid = sample_vcir("rsync://example.test/repo/invalid-2.mft"); - invalid.instance_gate.instance_effective_until = pack_time(11); - let err = store - .put_vcir(&invalid) - .expect_err("invalid instance gate must fail"); - assert!(err.to_string().contains("instance_effective_until")); - - store - .delete_vcir(&vcir.manifest_rsync_uri) - .expect("delete vcir"); - assert!( - store - .get_vcir(&vcir.manifest_rsync_uri) - .expect("get deleted vcir") - .is_none() - ); - assert!( - store - .get_manifest_replay_meta(&vcir.manifest_rsync_uri) - .expect("get deleted manifest replay meta") - .is_none() - ); - assert!( - store - .get_roa_cache_projection(&vcir.manifest_rsync_uri) - .expect("get deleted roa cache projection") - .is_none() - ); -} - -#[test] -fn clear_vcir_reuse_records_preserves_manifest_replay_meta() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - let vcir = sample_vcir("rsync://example.test/repo/current.mft"); - store.put_vcir(&vcir).expect("put vcir"); - let replay_meta = store - .get_manifest_replay_meta(&vcir.manifest_rsync_uri) - .expect("get replay meta") - .expect("replay meta exists"); - - let cleared = store - .clear_vcir_reuse_records() - .expect("clear reusable VCIR records"); - assert_eq!(cleared.vcir_records, 1); - assert_eq!(cleared.failed_fetch_identity_records, 0); - assert!( - store - .get_vcir(&vcir.manifest_rsync_uri) - .expect("get cleared vcir") - .is_none() - ); - assert_eq!( - store - .get_manifest_replay_meta(&vcir.manifest_rsync_uri) - .expect("get preserved replay meta"), - Some(replay_meta) - ); -} - -#[test] -fn transport_prefetch_snapshot_roundtrips() { - use crate::parallel::transport_prefetch::{ - TransportPrefetchDedupKey, TransportPrefetchMode, TransportPrefetchRepoIdentity, - TransportPrefetchRequest, TransportPrefetchRequester, TransportPrefetchSnapshot, - }; - use crate::policy::SyncPreference; - - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - assert!( - store - .get_transport_prefetch_snapshot() - .expect("get empty prefetch snapshot") - .is_none() - ); - - let snapshot = TransportPrefetchSnapshot::new( - SyncPreference::RrdpThenRsync, - vec![TransportPrefetchRequest { - dedup_key: TransportPrefetchDedupKey::RrdpNotify { - notification_uri: "https://example.test/notification.xml".to_string(), - }, - rsync_scope_uri: "rsync://example.test/repo/".to_string(), - rsync_failure_scope_uri: Some("rsync://example.test/".to_string()), - repo_identity: TransportPrefetchRepoIdentity { - notification_uri: Some("https://example.test/notification.xml".to_string()), - rsync_base_uri: "rsync://example.test/repo/".to_string(), - }, - mode: TransportPrefetchMode::Rrdp, - last_result: None, - last_rsync_result: None, - tal_id: "apnic".to_string(), - rir_id: "apnic".to_string(), - priority: 0, - requesters: vec![TransportPrefetchRequester { - tal_id: "apnic".to_string(), - rir_id: "apnic".to_string(), - parent_node_id: None, - ca_instance_handle_id: "apnic:rsync://example.test/repo/root.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), - manifest_rsync_uri: "rsync://example.test/repo/root.mft".to_string(), - }], - }], - ); - store - .put_transport_prefetch_snapshot(&snapshot) - .expect("put prefetch snapshot"); - let got = store - .get_transport_prefetch_snapshot() - .expect("get prefetch snapshot") - .expect("snapshot exists"); - assert_eq!(got, snapshot); -} - -#[test] -fn manifest_replay_meta_validation_reports_invalid_fields() { - let mut meta = ManifestReplayMeta { - manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(), - manifest_number_be: vec![3], - manifest_this_update: pack_time(0), - manifest_sha256: vec![0x11; 32], - updated_at_validation_time: pack_time(1), - }; - meta.validate_internal().expect("valid replay meta"); - - meta.manifest_sha256 = vec![0x11; 31]; - let err = meta - .validate_internal() - .expect_err("short manifest sha must fail"); - assert!(err.to_string().contains("must be 32 bytes")); - - meta.manifest_sha256 = vec![0x11; 32]; - meta.manifest_number_be = vec![0, 3]; - let err = meta - .validate_internal() - .expect_err("non-minimal manifest number must fail"); - assert!(err.to_string().contains("minimal big-endian")); -} - -#[test] -fn list_vcirs_returns_all_entries() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let vcir1 = sample_vcir("rsync://example.test/repo/a.mft"); - let vcir2 = sample_vcir("rsync://example.test/repo/b.mft"); - store.put_vcir(&vcir1).expect("put vcir1"); - store.put_vcir(&vcir2).expect("put vcir2"); - - let mut got = store.list_vcirs().expect("list vcirs"); - got.sort_by(|a, b| a.manifest_rsync_uri.cmp(&b.manifest_rsync_uri)); - assert_eq!(got, vec![vcir1, vcir2]); -} - -#[test] -fn summarize_vcir_storage_aggregates_values_and_field_sizes() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let vcir1 = sample_vcir("rsync://example.test/repo/a.mft"); - let vcir2 = sample_vcir("rsync://example.test/repo/b.mft"); - store.put_vcir(&vcir1).expect("put vcir1"); - store.put_vcir(&vcir2).expect("put vcir2"); - - let summary = store.summarize_vcir_storage().expect("summarize vcirs"); - let mut expected_fields = VcirFieldSizeBreakdown::default(); - expected_fields.add_assign(&VcirFieldSizeBreakdown::from_vcir(&vcir1)); - expected_fields.add_assign(&VcirFieldSizeBreakdown::from_vcir(&vcir2)); - - assert_eq!(summary.entry_count, 2); - assert!(summary.vcir_value_bytes > 0); - assert!(summary.vcir_value_bytes_max > 0); - assert!(summary.vcir_value_bytes_max_manifest_rsync_uri.is_some()); - assert_eq!(summary.top_entries_by_vcir_value_bytes.len(), 2); - assert!( - summary.top_entries_by_vcir_value_bytes[0].vcir_value_bytes - >= summary.top_entries_by_vcir_value_bytes[1].vcir_value_bytes - ); - assert_eq!(summary.field_sizes, expected_fields); - assert!(summary.core_fields.manifest_rsync_uri_bytes > 0); - assert!(summary.ccr_projection.manifest_sha256_bytes > 0); - assert!(summary.child_resources.effective_ip_resource_cbor_bytes > 0); - assert_eq!( - summary.local_output_old_projection_bytes, - expected_fields.local_output_old_projection_bytes() - ); - assert_eq!( - summary.local_output_typed_projection_bytes, - expected_fields.local_output_typed_projection_bytes() - ); - assert_eq!( - summary.local_output_projection_saved_bytes, - expected_fields.local_output_projection_saved_bytes() - ); -} - -#[test] -fn replace_vcir_and_manifest_replay_meta_replaces_current_entry() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let mut previous = sample_vcir("rsync://example.test/repo/current.mft"); - previous.local_outputs = vec![VcirLocalOutput { - output_type: VcirOutputType::Vrp, - item_effective_until: pack_time(10), - source_object_uri: "rsync://example.test/repo/old.roa".to_string(), - source_object_type: VcirSourceObjectType::Roa, - source_object_hash: sha256_32(b"old-roa"), - source_ee_cert_hash: sha256_32(b"old-ee"), - payload: VcirLocalOutputPayload::Vrp { - asn: 64496, - afi: crate::data_model::roa::RoaAfi::Ipv4, - prefix_len: 24, - addr: { - let mut addr = [0u8; 16]; - addr[..4].copy_from_slice(&[203, 0, 113, 0]); - addr - }, - max_length: 24, - }, - rule_hash: sha256_32(b"old-rule"), - }]; - previous.summary.local_vrp_count = 1; - previous.summary.local_aspa_count = 0; - previous.summary.local_router_key_count = 0; - let previous_roa_context = roa_cache_projection_context(&previous); - let previous_timing = store - .replace_vcir_manifest_replay_meta_and_projections( - &previous, - Some(&previous_roa_context), - None, - ) - .expect("store previous vcir"); - assert!(previous_timing.vcir_value_bytes > 0); - assert!(previous_timing.replay_meta_value_bytes > 0); - assert!(previous_timing.roa_cache_projection_value_bytes > 0); - assert_eq!( - previous_timing.total_encoded_bytes, - previous_timing.vcir_value_bytes - + previous_timing.replay_meta_value_bytes - + previous_timing.roa_cache_projection_value_bytes - ); - assert!( - store - .get_roa_cache_projection(&previous.manifest_rsync_uri) - .expect("get previous projection") - .is_some() - ); - - let mut current = sample_vcir("rsync://example.test/repo/current.mft"); - current.local_outputs = vec![VcirLocalOutput { - output_type: VcirOutputType::Aspa, - item_effective_until: pack_time(11), - source_object_uri: "rsync://example.test/repo/new.asa".to_string(), - source_object_type: VcirSourceObjectType::Aspa, - source_object_hash: sha256_32(b"new-aspa"), - source_ee_cert_hash: sha256_32(b"new-ee"), - payload: VcirLocalOutputPayload::Aspa { - customer_as_id: 64496, - provider_as_ids: vec![64497], - }, - rule_hash: sha256_32(b"new-rule"), - }]; - current.summary.local_vrp_count = 0; - current.summary.local_aspa_count = 1; - let current_timing = store - .replace_vcir_and_manifest_replay_meta(¤t) - .expect("replace vcir and replay meta"); - assert!(current_timing.vcir_value_bytes > 0); - assert!(current_timing.replay_meta_value_bytes > 0); - assert_eq!(current_timing.roa_cache_projection_value_bytes, 0); - assert_eq!( - current_timing.total_encoded_bytes, - current_timing.vcir_value_bytes + current_timing.replay_meta_value_bytes - ); - - let got = store - .get_vcir(¤t.manifest_rsync_uri) - .expect("get replaced vcir") - .expect("vcir exists"); - assert_eq!(got, current); - let replay_meta = store - .get_manifest_replay_meta(¤t.manifest_rsync_uri) - .expect("get replaced replay meta") - .expect("replay meta exists"); - assert_eq!( - replay_meta.manifest_number_be, - current.validated_manifest_meta.validated_manifest_number - ); - assert_eq!( - replay_meta.manifest_sha256, - current.ccr_manifest_projection.manifest_sha256 - ); - assert!( - store - .get_roa_cache_projection(¤t.manifest_rsync_uri) - .expect("get current projection") - .is_none() - ); -} - -#[test] -fn get_child_certificate_cache_projections_batch_preserves_order_and_misses() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let first_key = sha256_hex(b"child-cache-key-first"); - let missing_key = sha256_hex(b"child-cache-key-missing"); - let second_key = sha256_hex(b"child-cache-key-second"); - let first_hash = sha256_hex(b"first-child-cert"); - let second_hash = sha256_hex(b"second-child-cert"); - let first = sample_child_certificate_cache_projection( - first_key.clone(), - "rsync://example.test/repo/first.cer", - &first_hash, - ); - let second = sample_child_certificate_cache_projection( - second_key.clone(), - "rsync://example.test/repo/second.cer", - &second_hash, - ); - - store - .put_child_certificate_cache_projection(&first) - .expect("put first projection"); - store - .put_child_certificate_cache_projection(&second) - .expect("put second projection"); - - let got = store - .get_child_certificate_cache_projections_batch(&[ - second_key.clone(), - missing_key, - first_key.clone(), - ]) - .expect("batch get child projections"); - - assert_eq!(got.len(), 3); - assert_eq!( - got[0] - .as_ref() - .expect("second projection") - .child_cert_sha256_hex, - second_hash - ); - assert!(got[1].is_none()); - assert_eq!( - got[2] - .as_ref() - .expect("first projection") - .child_cert_sha256_hex, - first_hash - ); -} - -#[test] -fn child_certificate_cache_mmap_segment_preserves_order_and_misses() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let manifest_uri = "rsync://example.test/repo/parent.mft"; - let first_key = sha256_hex(b"child-cache-mmap-key-first"); - let missing_key = sha256_hex(b"child-cache-mmap-key-missing"); - let second_key = sha256_hex(b"child-cache-mmap-key-second"); - let first_hash = sha256_hex(b"first-child-cert-mmap"); - let second_hash = sha256_hex(b"second-child-cert-mmap"); - let first = sample_child_certificate_cache_projection( - first_key.clone(), - "rsync://example.test/repo/first.cer", - &first_hash, - ); - let second = sample_child_certificate_cache_projection( - second_key.clone(), - "rsync://example.test/repo/second.cer", - &second_hash, - ); - - assert!( - store - .get_child_certificate_cache_projections_mmap_segment( - manifest_uri, - &[first_key.clone()] - ) - .expect("missing segment lookup") - .is_none() - ); - - let write_stats = store - .write_child_certificate_cache_mmap_segment(manifest_uri, &[first.clone(), second.clone()]) - .expect("write child projection segment"); - assert_eq!(write_stats.new_entries, 2); - assert!(write_stats.file_bytes > 0); - - let got = store - .get_child_certificate_cache_projections_mmap_segment( - manifest_uri, - &[second_key.clone(), missing_key, first_key.clone()], - ) - .expect("lookup child projection segment") - .expect("segment exists"); - - assert_eq!(got.hits, 2); - assert_eq!(got.misses, 1); - assert!(got.file_bytes > 0); - assert_eq!(got.projections.len(), 3); - assert_eq!( - got.projections[0] - .as_ref() - .expect("second projection") - .child_cert_sha256_hex, - second_hash - ); - assert!(got.projections[1].is_none()); - assert_eq!( - got.projections[2] - .as_ref() - .expect("first projection") - .child_cert_sha256_hex, - first_hash - ); -} - -#[test] -fn child_certificate_cache_mmap_segment_overlay_preserves_existing_values() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let manifest_uri = "rsync://example.test/repo/parent-overlay.mft"; - let first_key = sha256_hex(b"child-cache-overlay-key-first"); - let second_key = sha256_hex(b"child-cache-overlay-key-second"); - let missing_key = sha256_hex(b"child-cache-overlay-key-missing"); - let first_hash = sha256_hex(b"first-child-cert-overlay"); - let old_second_hash = sha256_hex(b"old-second-child-cert-overlay"); - let new_second_hash = sha256_hex(b"new-second-child-cert-overlay"); - let first = sample_child_certificate_cache_projection( - first_key.clone(), - "rsync://example.test/repo/first-overlay.cer", - &first_hash, - ); - let old_second = sample_child_certificate_cache_projection( - second_key.clone(), - "rsync://example.test/repo/second-overlay.cer", - &old_second_hash, - ); - let new_second = sample_child_certificate_cache_projection( - second_key.clone(), - "rsync://example.test/repo/second-overlay.cer", - &new_second_hash, - ); - - store - .write_child_certificate_cache_mmap_segment( - manifest_uri, - &[first.clone(), old_second.clone()], - ) - .expect("write initial segment"); - let stats = store - .write_child_certificate_cache_mmap_segment_overlay( - manifest_uri, - &[first_key.clone(), second_key.clone(), missing_key.clone()], - std::slice::from_ref(&new_second), - ) - .expect("write segment overlay"); - assert_eq!(stats.new_entries, 2); - - let got = store - .get_child_certificate_cache_projections_mmap_segment( - manifest_uri, - &[first_key.clone(), second_key.clone(), missing_key], - ) - .expect("lookup segment") - .expect("segment exists"); - assert_eq!(got.hits, 2); - assert_eq!(got.misses, 1); - assert_eq!( - got.projections[0] - .as_ref() - .expect("first projection") - .child_cert_sha256_hex, - first_hash - ); - assert_eq!( - got.projections[1] - .as_ref() - .expect("updated second projection") - .child_cert_sha256_hex, - new_second_hash - ); - assert!(got.projections[2].is_none()); -} - -#[test] -fn storage_helpers_cover_optional_validation_paths() { - let withdrawn = RepositoryViewEntry { - rsync_uri: "rsync://example.test/repo/withdrawn.cer".to_string(), - current_hash: Some(sha256_hex(b"withdrawn")), - repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), - object_type: Some("cer".to_string()), - state: RepositoryViewState::Withdrawn, - }; - withdrawn - .validate_internal() - .expect("withdrawn repository view validates"); - - let raw = RawByHashEntry::from_bytes(sha256_hex(b"helper-bytes"), b"helper-bytes".to_vec()); - raw.validate_internal() - .expect("raw_by_hash helper validates"); - - let empty_raw = RawByHashEntry { - sha256_hex: sha256_hex(b"x"), - bytes: Vec::new(), - origin_uris: Vec::new(), - object_type: None, - encoding: None, - }; - let err = empty_raw - .validate_internal() - .expect_err("empty raw bytes must fail"); - assert!(err.to_string().contains("bytes must not be empty")); - - let duplicate_origin_raw = RawByHashEntry { - sha256_hex: sha256_hex(b"dup-origin"), - bytes: b"dup-origin".to_vec(), - origin_uris: vec![ - "rsync://example.test/repo/object.cer".to_string(), - "rsync://example.test/repo/object.cer".to_string(), - ], - object_type: Some("cer".to_string()), - encoding: Some("der".to_string()), - }; - let err = duplicate_origin_raw - .validate_internal() - .expect_err("duplicate origin URI must fail"); - assert!(err.to_string().contains("duplicate origin URI")); -} - -#[test] -fn rrdp_source_optional_fields_and_owner_without_hash_validate() { - let source = RrdpSourceRecord { - notify_uri: "https://rrdp.example.test/notification.xml".to_string(), - last_session_id: None, - last_serial: None, - first_seen_at: pack_time(0), - last_seen_at: pack_time(1), - last_sync_at: None, - sync_state: RrdpSourceSyncState::Empty, - last_snapshot_uri: None, - last_snapshot_hash: None, - last_error: Some("network timeout".to_string()), - }; - source - .validate_internal() - .expect("source with optional fields validates"); - - let owner = RrdpUriOwnerRecord { - rsync_uri: "rsync://example.test/repo/object.cer".to_string(), - notify_uri: "https://rrdp.example.test/notification.xml".to_string(), - current_hash: None, - last_confirmed_session_id: "session-1".to_string(), - last_confirmed_serial: 5, - last_changed_at: pack_time(5), - owner_state: RrdpUriOwnerState::Withdrawn, - }; - owner - .validate_internal() - .expect("owner without hash validates when withdrawn"); -} - -#[test] -fn rrdp_source_binding_records_roundtrip_and_prefix_iteration() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let notify_uri = "https://rrdp.example.test/notification.xml"; - let source = sample_rrdp_source_record(notify_uri); - store - .put_rrdp_source_record(&source) - .expect("put rrdp source record"); - let got_source = store - .get_rrdp_source_record(notify_uri) - .expect("get rrdp source record") - .expect("rrdp source exists"); - assert_eq!(got_source, source); - - let member1 = - sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/a.cer", 1); - let member2 = - sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/b.roa", 2); - let other_member = sample_rrdp_source_member_record( - "https://other.example.test/notification.xml", - "rsync://other.example.test/repo/c.cer", - 3, - ); - store - .put_rrdp_source_member_record(&member1) - .expect("put member1"); - store - .put_rrdp_source_member_record(&member2) - .expect("put member2"); - store - .put_rrdp_source_member_record(&other_member) - .expect("put other member"); - - let mut members = store - .list_rrdp_source_member_records(notify_uri) - .expect("list rrdp source members"); - members.sort_by(|a, b| a.rsync_uri.cmp(&b.rsync_uri)); - assert_eq!(members, vec![member1.clone(), member2.clone()]); - - let got_member = store - .get_rrdp_source_member_record(notify_uri, &member1.rsync_uri) - .expect("get member1") - .expect("member1 exists"); - assert_eq!(got_member, member1); - - let owner = sample_rrdp_uri_owner_record(notify_uri, &member1.rsync_uri); - store - .put_rrdp_uri_owner_record(&owner) - .expect("put uri owner record"); - let got_owner = store - .get_rrdp_uri_owner_record(&member1.rsync_uri) - .expect("get uri owner record") - .expect("uri owner exists"); - assert_eq!(got_owner, owner); - store - .delete_rrdp_uri_owner_record(&member1.rsync_uri) - .expect("delete uri owner record"); - assert!( - store - .get_rrdp_uri_owner_record(&member1.rsync_uri) - .expect("get deleted uri owner") - .is_none() - ); - - let mut invalid_source = sample_rrdp_source_record("https://invalid.example/notification.xml"); - invalid_source.last_snapshot_hash = Some("bad".to_string()); - let err = store - .put_rrdp_source_record(&invalid_source) - .expect_err("invalid source hash must fail"); - assert!(err.to_string().contains("last_snapshot_hash")); - - let invalid_member = RrdpSourceMemberRecord { - notify_uri: notify_uri.to_string(), - rsync_uri: "rsync://example.test/repo/deleted.cer".to_string(), - current_hash: None, - object_type: None, - present: true, - last_confirmed_session_id: "session-1".to_string(), - last_confirmed_serial: 10, - last_changed_at: pack_time(10), - }; - let err = store - .put_rrdp_source_member_record(&invalid_member) - .expect_err("present member without hash must fail"); - assert!(err.to_string().contains("current_hash is required")); -} -#[test] -fn projection_batch_roundtrip_writes_repository_view_member_and_owner_records() { - let dir = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(dir.path()).expect("open store"); - - let view = RepositoryViewEntry { - rsync_uri: "rsync://example.test/repo/a.roa".to_string(), - current_hash: Some(hex::encode([1u8; 32])), - repository_source: Some("https://example.test/notify.xml".to_string()), - object_type: Some("roa".to_string()), - state: RepositoryViewState::Present, - }; - let member = RrdpSourceMemberRecord { - notify_uri: "https://example.test/notify.xml".to_string(), - rsync_uri: "rsync://example.test/repo/a.roa".to_string(), - current_hash: Some(hex::encode([1u8; 32])), - object_type: Some("roa".to_string()), - present: true, - last_confirmed_session_id: "session-1".to_string(), - last_confirmed_serial: 7, - last_changed_at: pack_time(1), - }; - let owner = RrdpUriOwnerRecord { - rsync_uri: "rsync://example.test/repo/a.roa".to_string(), - notify_uri: "https://example.test/notify.xml".to_string(), - current_hash: Some(hex::encode([1u8; 32])), - last_confirmed_session_id: "session-1".to_string(), - last_confirmed_serial: 7, - last_changed_at: pack_time(1), - owner_state: RrdpUriOwnerState::Active, - }; - - store - .put_projection_batch(&[view.clone()], &[member.clone()], &[owner.clone()]) - .expect("write projection batch"); - - assert_eq!( - store - .get_repository_view_entry(&view.rsync_uri) - .expect("get view") - .expect("present view"), - view - ); - assert_eq!( - store - .get_rrdp_source_member_record(&member.notify_uri, &member.rsync_uri) - .expect("get member") - .expect("present member"), - member - ); - assert_eq!( - store - .get_rrdp_uri_owner_record(&owner.rsync_uri) - .expect("get owner") - .expect("present owner"), - owner - ); -} - -#[test] -fn current_rrdp_source_member_helpers_filter_present_records() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let notify_uri = "https://rrdp.example.test/notification.xml"; - let mut present_a = - sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/a.cer", 1); - let mut withdrawn_b = - sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/b.roa", 2); - withdrawn_b.present = false; - let present_c = - sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/c.crl", 3); - let other_source = sample_rrdp_source_member_record( - "https://other.example.test/notification.xml", - "rsync://other.example.test/repo/x.cer", - 4, - ); - present_a.last_confirmed_serial = 10; - - store - .put_rrdp_source_member_record(&present_a) - .expect("put present a"); - store - .put_rrdp_source_member_record(&withdrawn_b) - .expect("put withdrawn b"); - store - .put_rrdp_source_member_record(&present_c) - .expect("put present c"); - store - .put_rrdp_source_member_record(&other_source) - .expect("put other source"); - - let members = store - .list_current_rrdp_source_members(notify_uri) - .expect("list current members"); - assert_eq!( - members - .iter() - .map(|record| record.rsync_uri.as_str()) - .collect::>(), - vec![ - "rsync://example.test/repo/a.cer", - "rsync://example.test/repo/c.crl", - ] - ); - - assert!( - store - .is_current_rrdp_source_member(notify_uri, &present_a.rsync_uri) - .expect("current a") - ); - assert!( - !store - .is_current_rrdp_source_member(notify_uri, &withdrawn_b.rsync_uri) - .expect("withdrawn b") - ); - assert!( - !store - .is_current_rrdp_source_member(notify_uri, &other_source.rsync_uri) - .expect("other source") - ); -} - -#[test] -fn load_current_object_bytes_by_uri_uses_repository_view_and_raw_by_hash() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - - let present_bytes = b"present-object".to_vec(); - let present_hash = sha256_hex(&present_bytes); - let mut present_raw = RawByHashEntry::from_bytes(present_hash.clone(), present_bytes.clone()); - present_raw - .origin_uris - .push("rsync://example.test/repo/present.roa".to_string()); - present_raw.object_type = Some("roa".to_string()); - store - .put_raw_by_hash_entry(&present_raw) - .expect("put present raw"); - store - .put_repository_view_entry(&RepositoryViewEntry { - rsync_uri: "rsync://example.test/repo/present.roa".to_string(), - current_hash: Some(present_hash), - repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), - object_type: Some("roa".to_string()), - state: RepositoryViewState::Present, - }) - .expect("put present view"); - - let replaced_bytes = b"replaced-object".to_vec(); - let replaced_hash = sha256_hex(&replaced_bytes); - let mut replaced_raw = - RawByHashEntry::from_bytes(replaced_hash.clone(), replaced_bytes.clone()); - replaced_raw - .origin_uris - .push("rsync://example.test/repo/replaced.cer".to_string()); - replaced_raw.object_type = Some("cer".to_string()); - store - .put_raw_by_hash_entry(&replaced_raw) - .expect("put replaced raw"); - store - .put_repository_view_entry(&RepositoryViewEntry { - rsync_uri: "rsync://example.test/repo/replaced.cer".to_string(), - current_hash: Some(replaced_hash), - repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), - object_type: Some("cer".to_string()), - state: RepositoryViewState::Replaced, - }) - .expect("put replaced view"); - - store - .put_repository_view_entry(&RepositoryViewEntry { - rsync_uri: "rsync://example.test/repo/withdrawn.crl".to_string(), - current_hash: Some(sha256_hex(b"withdrawn")), - repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), - object_type: Some("crl".to_string()), - state: RepositoryViewState::Withdrawn, - }) - .expect("put withdrawn view"); - - assert_eq!( - store - .load_current_object_bytes_by_uri("rsync://example.test/repo/present.roa") - .expect("load present"), - Some(present_bytes) - ); - assert_eq!( - store - .load_current_object_bytes_by_uri("rsync://example.test/repo/replaced.cer") - .expect("load replaced"), - Some(replaced_bytes) - ); - assert_eq!( - store - .load_current_object_bytes_by_uri("rsync://example.test/repo/withdrawn.crl") - .expect("load withdrawn"), - None - ); - assert_eq!( - store - .load_current_object_bytes_by_uri("rsync://example.test/repo/missing.roa") - .expect("load missing"), - None - ); -} - -#[test] -fn load_current_object_bytes_by_uri_errors_when_raw_by_hash_is_missing() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - let rsync_uri = "rsync://example.test/repo/missing.cer"; - - store - .put_repository_view_entry(&RepositoryViewEntry { - rsync_uri: rsync_uri.to_string(), - current_hash: Some(hex::encode([0x11; 32])), - repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), - object_type: Some("cer".to_string()), - state: RepositoryViewState::Present, - }) - .expect("put view"); - let err = store - .load_current_object_bytes_by_uri(rsync_uri) - .expect_err("missing raw_by_hash should error"); - assert!(matches!(err, StorageError::InvalidData { .. })); -} - -#[test] -fn load_current_object_with_hash_by_uri_returns_hash_and_bytes() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - let rsync_uri = "rsync://example.test/repo/present.roa"; - let bytes = b"present-object".to_vec(); - let hash = sha256_hex(&bytes); - - let mut raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone()); - raw.origin_uris.push(rsync_uri.to_string()); - raw.object_type = Some("roa".to_string()); - store.put_raw_by_hash_entry(&raw).expect("put raw"); - store - .put_repository_view_entry(&RepositoryViewEntry { - rsync_uri: rsync_uri.to_string(), - current_hash: Some(hash.clone()), - repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), - object_type: Some("roa".to_string()), - state: RepositoryViewState::Present, - }) - .expect("put view"); - - let got = store - .load_current_object_with_hash_by_uri(rsync_uri) - .expect("load current object") - .expect("current object exists"); - assert_eq!(got.current_hash_hex, hash); - assert_eq!(got.current_hash, compute_sha256_32(&bytes)); - assert_eq!(got.bytes, bytes); -} - -#[test] -fn load_current_object_with_hash_by_uri_uses_internal_blob_cf_without_raw_entry() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - let rsync_uri = "rsync://example.test/repo/blob-only.roa"; - let bytes = b"blob-only-current-object".to_vec(); - let hash = sha256_hex(&bytes); - - store - .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) - .expect("put blob bytes"); - store - .put_repository_view_entry(&RepositoryViewEntry { - rsync_uri: rsync_uri.to_string(), - current_hash: Some(hash.clone()), - repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), - object_type: Some("roa".to_string()), - state: RepositoryViewState::Present, - }) - .expect("put view"); - - let got = store - .load_current_object_with_hash_by_uri(rsync_uri) - .expect("load current object") - .expect("current object exists"); - assert_eq!(got.current_hash_hex, hash); - assert_eq!(got.current_hash, compute_sha256_32(&bytes)); - assert_eq!(got.bytes, bytes); - assert!( - store - .get_raw_by_hash_entry(&got.current_hash_hex) - .expect("get raw entry") - .is_none() - ); -} - -#[test] -fn load_current_object_bytes_by_uri_uses_internal_blob_cf_without_raw_entry() { - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - let rsync_uri = "rsync://example.test/repo/blob-only-bytes.roa"; - let bytes = b"blob-only-current-object-bytes".to_vec(); - let hash = sha256_hex(&bytes); - - store - .put_blob_bytes_batch(&[(hash, bytes.clone())]) - .expect("put blob bytes"); - store - .put_repository_view_entry(&RepositoryViewEntry { - rsync_uri: rsync_uri.to_string(), - current_hash: Some(sha256_hex(&bytes)), - repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), - object_type: Some("roa".to_string()), - state: RepositoryViewState::Present, - }) - .expect("put view"); - - assert_eq!( - store - .load_current_object_bytes_by_uri(rsync_uri) - .expect("load current object bytes"), - Some(bytes) - ); -} - -#[test] -fn pack_file_can_lazy_load_bytes_from_external_raw_store() { - let td = tempfile::tempdir().expect("tempdir"); - let raw_store = std::sync::Arc::new( - ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store"), - ); - let bytes = b"lazy-pack-file".to_vec(); - let sha256_hex = sha256_hex(&bytes); - raw_store - .put_raw_entry(&RawByHashEntry::from_bytes( - sha256_hex.clone(), - bytes.clone(), - )) - .expect("put raw entry"); - - let file = PackFile::from_lazy_external_raw_store( - "rsync://example.test/repo/a.roa", - sha256_hex, - compute_sha256_32(&bytes), - raw_store, - ); - - assert_eq!(file.bytes().expect("lazy bytes"), bytes.as_slice()); - assert_eq!(file.bytes_cloned().expect("cloned bytes"), bytes); -} - -#[test] -fn pack_file_can_lazy_load_bytes_from_external_repo_bytes_store() { - let td = tempfile::tempdir().expect("tempdir"); - let repo_bytes_store = std::sync::Arc::new( - ExternalRepoBytesDb::open(td.path().join("repo-bytes.db")).expect("open repo bytes"), - ); - let bytes = b"repo-object-pack-file".to_vec(); - let sha256_hex = sha256_hex(&bytes); - repo_bytes_store - .put_blob_bytes_batch(&[(sha256_hex.clone(), bytes.clone())]) - .expect("put repo bytes"); - - let file = PackFile::from_lazy_repo_bytes( - "rsync://example.test/repo/a.roa", - sha256_hex, - compute_sha256_32(&bytes), - repo_bytes_store, - ); - - assert_eq!(file.bytes().expect("lazy repo bytes"), bytes.as_slice()); - assert_eq!(file.bytes_cloned().expect("cloned repo bytes"), bytes); - assert_eq!(file.compute_sha256().expect("compute sha256"), file.sha256); -} - -#[test] -fn read_only_checkpoint_isolated_from_source_work_db() { - let td = tempfile::tempdir().expect("tempdir"); - let source_path = td.path().join("source-work-db"); - let checkpoint_path = td.path().join("checkpoint-work-db"); - let uri = "rsync://example.test/repo/a.roa"; - let bytes = b"checkpoint-object"; - let hash = sha256_hex(bytes); - { - let source = RocksStore::open(&source_path).expect("open source"); - source - .put_blob_bytes_batch(&[(hash.clone(), bytes.to_vec())]) - .expect("put blob"); - source - .put_repository_view_entry(&RepositoryViewEntry { - rsync_uri: uri.to_string(), - current_hash: Some(hash), - repository_source: Some("fixture".to_string()), - object_type: Some("roa".to_string()), - state: RepositoryViewState::Present, - }) - .expect("put view"); - } - - RocksStore::create_read_only_checkpoint(&source_path, &checkpoint_path) - .expect("create checkpoint"); - let checkpoint = RocksStore::open(&checkpoint_path).expect("open checkpoint"); - checkpoint - .delete_repository_view_entry(uri) - .expect("delete checkpoint entry"); - drop(checkpoint); - - let source = RocksStore::open(&source_path).expect("reopen source"); - assert!( - source - .get_repository_view_entry(uri) - .expect("get source entry") - .is_some() - ); -} - -#[test] -fn read_only_external_repo_bytes_rejects_writes() { - let td = tempfile::tempdir().expect("tempdir"); - let path = td.path().join("repo-bytes.db"); - let hash = sha256_hex(b"repo-object"); - { - let writable = ExternalRepoBytesDb::open(&path).expect("open writable repo bytes"); - writable - .put_blob_bytes_batch(&[(hash.clone(), b"repo-object".to_vec())]) - .expect("seed repo bytes"); - } - let read_only = ExternalRepoBytesDb::open_read_only(&path).expect("open read-only repo bytes"); - assert_eq!( - read_only - .get_blob_bytes(&hash) - .expect("read repo bytes") - .expect("blob"), - b"repo-object" - ); - assert!( - read_only - .put_blob_bytes_batch(&[(hash, b"repo-object".to_vec())]) - .is_err() - ); -} - -#[test] -fn read_only_store_accepts_only_idempotent_existing_blob_apply() { - let td = tempfile::tempdir().expect("tempdir"); - let work_db_path = td.path().join("work-db"); - let repo_bytes_path = td.path().join("repo-bytes.db"); - let bytes = b"repo-object".to_vec(); - let hash = sha256_hex(&bytes); - { - let repo_bytes = ExternalRepoBytesDb::open(&repo_bytes_path).expect("open repo bytes"); - repo_bytes - .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) - .expect("seed repo bytes"); - } - let store = - RocksStore::open_with_external_repo_bytes_read_only(&work_db_path, &repo_bytes_path) - .expect("open read-only store"); - store - .put_blob_bytes_batch(&[(hash.clone(), bytes)]) - .expect("idempotent apply"); - assert!( - store - .put_blob_bytes_batch(&[(hash, b"different".to_vec())]) - .is_err() - ); -} - -#[test] -fn repository_blob_verification_detects_missing_external_blob() { - let td = tempfile::tempdir().expect("tempdir"); - let repo_bytes_path = td.path().join("repo-bytes.db"); - ExternalRepoBytesDb::open(&repo_bytes_path).expect("create repo bytes"); - let store = RocksStore::open_with_external_repo_bytes_read_only( - &td.path().join("work-db"), - &repo_bytes_path, - ) - .expect("open work db"); - store - .put_repository_view_entry(&RepositoryViewEntry { - rsync_uri: "rsync://example.test/repo/missing.roa".to_string(), - current_hash: Some(sha256_hex(b"missing")), - repository_source: Some("fixture".to_string()), - object_type: Some("roa".to_string()), - state: RepositoryViewState::Present, - }) - .expect("put view"); - let error = store - .verify_current_repository_blobs(16) - .expect_err("missing blob must fail"); - assert!(error.to_string().contains("missing.roa"), "{error}"); -} - -#[test] -fn vcir_related_artifact_reject_reason_serde_backward_compatible() { - // Legacy cache JSON written before the reject_reason ("e") field existed. - let legacy = r#"{"r":"manifest","k":"mft","u":"rsync://example.test/a.mft","h":"0000000000000000000000000000000000000000000000000000000000000000","t":"mft","s":"accepted"}"#; - let artifact: VcirRelatedArtifact = serde_json::from_str(legacy).expect("legacy artifact"); - assert_eq!(artifact.reject_reason, None); - - // None reason is skipped on serialization, keeping cache entries compact. - let json_none = serde_json::to_string(&artifact).expect("serialize none"); - assert!(!json_none.contains("\"e\"")); - - // A recorded reason round-trips through the "e" field. - let mut with_reason = artifact.clone(); - with_reason.validation_status = VcirArtifactValidationStatus::Rejected; - with_reason.reject_reason = Some("bad object".to_string()); - let json = serde_json::to_string(&with_reason).expect("serialize"); - assert!(json.contains("\"e\":\"bad object\"")); - let decoded: VcirRelatedArtifact = serde_json::from_str(&json).expect("deserialize"); - assert_eq!(decoded.reject_reason.as_deref(), Some("bad object")); - - // PublicationPointCacheObject conversion carries the reason both ways. - let cache_object = PublicationPointCacheObject::from_related_artifact(&with_reason); - assert_eq!(cache_object.reject_reason.as_deref(), Some("bad object")); - let back = cache_object.to_related_artifact(); - assert_eq!(back.reject_reason.as_deref(), Some("bad object")); - - // Legacy PublicationPointCacheObject JSON without "e" still deserializes. - let legacy_object: PublicationPointCacheObject = - serde_json::from_str(legacy).expect("legacy cache object"); - assert_eq!(legacy_object.reject_reason, None); -} +include!("tests_parts/helpers_and_models.rs"); +include!("tests_parts/projection.rs"); +include!("tests_parts/repository.rs"); +include!("tests_parts/vcir.rs"); +include!("tests_parts/child_cache.rs"); +include!("tests_parts/rrdp.rs"); +include!("tests_parts/object_loading.rs"); diff --git a/crates/panda-rpki-validator/src/storage/tests_parts/child_cache.rs b/crates/panda-rpki-validator/src/storage/tests_parts/child_cache.rs new file mode 100644 index 0000000..d7ae311 --- /dev/null +++ b/crates/panda-rpki-validator/src/storage/tests_parts/child_cache.rs @@ -0,0 +1,357 @@ +// Storage test group: child cache. + +#[test] +fn get_child_certificate_cache_projections_batch_preserves_order_and_misses() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let first_key = sha256_hex(b"child-cache-key-first"); + let missing_key = sha256_hex(b"child-cache-key-missing"); + let second_key = sha256_hex(b"child-cache-key-second"); + let first_hash = sha256_hex(b"first-child-cert"); + let second_hash = sha256_hex(b"second-child-cert"); + let first = sample_child_certificate_cache_projection( + first_key.clone(), + "rsync://example.test/repo/first.cer", + &first_hash, + ); + let second = sample_child_certificate_cache_projection( + second_key.clone(), + "rsync://example.test/repo/second.cer", + &second_hash, + ); + + store + .put_child_certificate_cache_projection(&first) + .expect("put first projection"); + store + .put_child_certificate_cache_projection(&second) + .expect("put second projection"); + + let got = store + .get_child_certificate_cache_projections_batch(&[ + second_key.clone(), + missing_key, + first_key.clone(), + ]) + .expect("batch get child projections"); + + assert_eq!(got.len(), 3); + assert_eq!( + got[0] + .as_ref() + .expect("second projection") + .child_cert_sha256_hex, + second_hash + ); + assert!(got[1].is_none()); + assert_eq!( + got[2] + .as_ref() + .expect("first projection") + .child_cert_sha256_hex, + first_hash + ); +} + +#[test] +fn child_certificate_cache_mmap_segment_preserves_order_and_misses() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let manifest_uri = "rsync://example.test/repo/parent.mft"; + let first_key = sha256_hex(b"child-cache-mmap-key-first"); + let missing_key = sha256_hex(b"child-cache-mmap-key-missing"); + let second_key = sha256_hex(b"child-cache-mmap-key-second"); + let first_hash = sha256_hex(b"first-child-cert-mmap"); + let second_hash = sha256_hex(b"second-child-cert-mmap"); + let first = sample_child_certificate_cache_projection( + first_key.clone(), + "rsync://example.test/repo/first.cer", + &first_hash, + ); + let second = sample_child_certificate_cache_projection( + second_key.clone(), + "rsync://example.test/repo/second.cer", + &second_hash, + ); + + assert!( + store + .get_child_certificate_cache_projections_mmap_segment( + manifest_uri, + &[first_key.clone()] + ) + .expect("missing segment lookup") + .is_none() + ); + + let write_stats = store + .write_child_certificate_cache_mmap_segment(manifest_uri, &[first.clone(), second.clone()]) + .expect("write child projection segment"); + assert_eq!(write_stats.new_entries, 2); + assert!(write_stats.file_bytes > 0); + + let got = store + .get_child_certificate_cache_projections_mmap_segment( + manifest_uri, + &[second_key.clone(), missing_key, first_key.clone()], + ) + .expect("lookup child projection segment") + .expect("segment exists"); + + assert_eq!(got.hits, 2); + assert_eq!(got.misses, 1); + assert!(got.file_bytes > 0); + assert_eq!(got.projections.len(), 3); + assert_eq!( + got.projections[0] + .as_ref() + .expect("second projection") + .child_cert_sha256_hex, + second_hash + ); + assert!(got.projections[1].is_none()); + assert_eq!( + got.projections[2] + .as_ref() + .expect("first projection") + .child_cert_sha256_hex, + first_hash + ); +} + +#[test] +fn child_certificate_cache_mmap_segment_overlay_preserves_existing_values() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let manifest_uri = "rsync://example.test/repo/parent-overlay.mft"; + let first_key = sha256_hex(b"child-cache-overlay-key-first"); + let second_key = sha256_hex(b"child-cache-overlay-key-second"); + let missing_key = sha256_hex(b"child-cache-overlay-key-missing"); + let first_hash = sha256_hex(b"first-child-cert-overlay"); + let old_second_hash = sha256_hex(b"old-second-child-cert-overlay"); + let new_second_hash = sha256_hex(b"new-second-child-cert-overlay"); + let first = sample_child_certificate_cache_projection( + first_key.clone(), + "rsync://example.test/repo/first-overlay.cer", + &first_hash, + ); + let old_second = sample_child_certificate_cache_projection( + second_key.clone(), + "rsync://example.test/repo/second-overlay.cer", + &old_second_hash, + ); + let new_second = sample_child_certificate_cache_projection( + second_key.clone(), + "rsync://example.test/repo/second-overlay.cer", + &new_second_hash, + ); + + store + .write_child_certificate_cache_mmap_segment( + manifest_uri, + &[first.clone(), old_second.clone()], + ) + .expect("write initial segment"); + let stats = store + .write_child_certificate_cache_mmap_segment_overlay( + manifest_uri, + &[first_key.clone(), second_key.clone(), missing_key.clone()], + std::slice::from_ref(&new_second), + ) + .expect("write segment overlay"); + assert_eq!(stats.new_entries, 2); + + let got = store + .get_child_certificate_cache_projections_mmap_segment( + manifest_uri, + &[first_key.clone(), second_key.clone(), missing_key], + ) + .expect("lookup segment") + .expect("segment exists"); + assert_eq!(got.hits, 2); + assert_eq!(got.misses, 1); + assert_eq!( + got.projections[0] + .as_ref() + .expect("first projection") + .child_cert_sha256_hex, + first_hash + ); + assert_eq!( + got.projections[1] + .as_ref() + .expect("updated second projection") + .child_cert_sha256_hex, + new_second_hash + ); + assert!(got.projections[2].is_none()); +} + +#[test] +fn storage_helpers_cover_optional_validation_paths() { + let withdrawn = RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/withdrawn.cer".to_string(), + current_hash: Some(sha256_hex(b"withdrawn")), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("cer".to_string()), + state: RepositoryViewState::Withdrawn, + }; + withdrawn + .validate_internal() + .expect("withdrawn repository view validates"); + + let raw = RawByHashEntry::from_bytes(sha256_hex(b"helper-bytes"), b"helper-bytes".to_vec()); + raw.validate_internal() + .expect("raw_by_hash helper validates"); + + let empty_raw = RawByHashEntry { + sha256_hex: sha256_hex(b"x"), + bytes: Vec::new(), + origin_uris: Vec::new(), + object_type: None, + encoding: None, + }; + let err = empty_raw + .validate_internal() + .expect_err("empty raw bytes must fail"); + assert!(err.to_string().contains("bytes must not be empty")); + + let duplicate_origin_raw = RawByHashEntry { + sha256_hex: sha256_hex(b"dup-origin"), + bytes: b"dup-origin".to_vec(), + origin_uris: vec![ + "rsync://example.test/repo/object.cer".to_string(), + "rsync://example.test/repo/object.cer".to_string(), + ], + object_type: Some("cer".to_string()), + encoding: Some("der".to_string()), + }; + let err = duplicate_origin_raw + .validate_internal() + .expect_err("duplicate origin URI must fail"); + assert!(err.to_string().contains("duplicate origin URI")); +} + +#[test] +fn rrdp_source_optional_fields_and_owner_without_hash_validate() { + let source = RrdpSourceRecord { + notify_uri: "https://rrdp.example.test/notification.xml".to_string(), + last_session_id: None, + last_serial: None, + first_seen_at: pack_time(0), + last_seen_at: pack_time(1), + last_sync_at: None, + sync_state: RrdpSourceSyncState::Empty, + last_snapshot_uri: None, + last_snapshot_hash: None, + last_error: Some("network timeout".to_string()), + }; + source + .validate_internal() + .expect("source with optional fields validates"); + + let owner = RrdpUriOwnerRecord { + rsync_uri: "rsync://example.test/repo/object.cer".to_string(), + notify_uri: "https://rrdp.example.test/notification.xml".to_string(), + current_hash: None, + last_confirmed_session_id: "session-1".to_string(), + last_confirmed_serial: 5, + last_changed_at: pack_time(5), + owner_state: RrdpUriOwnerState::Withdrawn, + }; + owner + .validate_internal() + .expect("owner without hash validates when withdrawn"); +} + +#[test] +fn rrdp_source_binding_records_roundtrip_and_prefix_iteration() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let notify_uri = "https://rrdp.example.test/notification.xml"; + let source = sample_rrdp_source_record(notify_uri); + store + .put_rrdp_source_record(&source) + .expect("put rrdp source record"); + let got_source = store + .get_rrdp_source_record(notify_uri) + .expect("get rrdp source record") + .expect("rrdp source exists"); + assert_eq!(got_source, source); + + let member1 = + sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/a.cer", 1); + let member2 = + sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/b.roa", 2); + let other_member = sample_rrdp_source_member_record( + "https://other.example.test/notification.xml", + "rsync://other.example.test/repo/c.cer", + 3, + ); + store + .put_rrdp_source_member_record(&member1) + .expect("put member1"); + store + .put_rrdp_source_member_record(&member2) + .expect("put member2"); + store + .put_rrdp_source_member_record(&other_member) + .expect("put other member"); + + let mut members = store + .list_rrdp_source_member_records(notify_uri) + .expect("list rrdp source members"); + members.sort_by(|a, b| a.rsync_uri.cmp(&b.rsync_uri)); + assert_eq!(members, vec![member1.clone(), member2.clone()]); + + let got_member = store + .get_rrdp_source_member_record(notify_uri, &member1.rsync_uri) + .expect("get member1") + .expect("member1 exists"); + assert_eq!(got_member, member1); + + let owner = sample_rrdp_uri_owner_record(notify_uri, &member1.rsync_uri); + store + .put_rrdp_uri_owner_record(&owner) + .expect("put uri owner record"); + let got_owner = store + .get_rrdp_uri_owner_record(&member1.rsync_uri) + .expect("get uri owner record") + .expect("uri owner exists"); + assert_eq!(got_owner, owner); + store + .delete_rrdp_uri_owner_record(&member1.rsync_uri) + .expect("delete uri owner record"); + assert!( + store + .get_rrdp_uri_owner_record(&member1.rsync_uri) + .expect("get deleted uri owner") + .is_none() + ); + + let mut invalid_source = sample_rrdp_source_record("https://invalid.example/notification.xml"); + invalid_source.last_snapshot_hash = Some("bad".to_string()); + let err = store + .put_rrdp_source_record(&invalid_source) + .expect_err("invalid source hash must fail"); + assert!(err.to_string().contains("last_snapshot_hash")); + + let invalid_member = RrdpSourceMemberRecord { + notify_uri: notify_uri.to_string(), + rsync_uri: "rsync://example.test/repo/deleted.cer".to_string(), + current_hash: None, + object_type: None, + present: true, + last_confirmed_session_id: "session-1".to_string(), + last_confirmed_serial: 10, + last_changed_at: pack_time(10), + }; + let err = store + .put_rrdp_source_member_record(&invalid_member) + .expect_err("present member without hash must fail"); + assert!(err.to_string().contains("current_hash is required")); +} diff --git a/crates/panda-rpki-validator/src/storage/tests_parts/helpers_and_models.rs b/crates/panda-rpki-validator/src/storage/tests_parts/helpers_and_models.rs new file mode 100644 index 0000000..c5b8ece --- /dev/null +++ b/crates/panda-rpki-validator/src/storage/tests_parts/helpers_and_models.rs @@ -0,0 +1,310 @@ +// Storage test group: helpers and models. + + +fn pack_time(hour: i64) -> PackTime { + PackTime::from_utc_offset_datetime( + time::OffsetDateTime::UNIX_EPOCH + time::Duration::hours(hour), + ) +} + +fn sha256_hex(input: &[u8]) -> String { + hex::encode(compute_sha256_32(input)) +} + +fn sha256_32(input: &[u8]) -> [u8; 32] { + compute_sha256_32(input) +} + +fn sample_child_certificate_cache_projection( + cache_key_sha256_hex: String, + child_cert_uri: &str, + child_cert_sha256_hex: &str, +) -> ChildCertificateCacheProjection { + ChildCertificateCacheProjection { + schema_version: CHILD_CERTIFICATE_CACHE_SCHEMA_VERSION, + algorithm_version: CHILD_CERTIFICATE_CACHE_ALGORITHM_VERSION, + cache_key_sha256_hex, + child_cert_uri: child_cert_uri.to_string(), + child_cert_sha256_hex: child_cert_sha256_hex.to_string(), + child_cert_serial: vec![1], + issuer_ca_sha256_hex: sha256_hex(b"issuer-ca"), + issuer_crl_uri: "rsync://example.test/repo/issuer.crl".to_string(), + issuer_crl_sha256_hex: sha256_hex(b"issuer-crl"), + ca_validation_context_digest: sha256_32(b"parent-context"), + validation_policy_fingerprint: sha256_32(b"policy"), + effective_not_before: pack_time(0), + effective_until: pack_time(24), + payload: ChildCertificateCachePayload::ChildCa { + child_manifest_rsync_uri: format!("{child_cert_uri}.mft"), + child_ski: "11".repeat(20), + child_rsync_base_uri: "rsync://example.test/repo/child/".to_string(), + child_publication_point_rsync_uri: "rsync://example.test/repo/child/".to_string(), + child_rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), + child_effective_ip_resources: None, + child_effective_as_resources: None, + }, + } +} + +#[test] +fn parse_work_db_blob_mode_accepts_supported_values() { + assert_eq!(default_work_db_blob_mode(), WorkDbBlobMode::Disabled); + assert_eq!( + parse_work_db_blob_mode("default"), + Some(WorkDbBlobMode::Disabled) + ); + assert_eq!( + parse_work_db_blob_mode("current"), + Some(WorkDbBlobMode::Current) + ); + assert_eq!( + parse_work_db_blob_mode("legacy"), + Some(WorkDbBlobMode::Current) + ); + assert_eq!( + parse_work_db_blob_mode("disabled"), + Some(WorkDbBlobMode::Disabled) + ); + assert_eq!( + parse_work_db_blob_mode("no-blob"), + Some(WorkDbBlobMode::Disabled) + ); + assert_eq!(parse_work_db_blob_mode("lz4"), Some(WorkDbBlobMode::Lz4)); + assert_eq!( + parse_work_db_blob_mode("blob-lz4"), + Some(WorkDbBlobMode::Lz4) + ); + assert_eq!(parse_work_db_blob_mode("unexpected"), None); +} + +#[test] +fn parse_work_db_memory_profile_accepts_supported_values() { + assert_eq!( + parse_work_db_memory_profile("default"), + Some(WorkDbMemoryProfile::Default) + ); + assert_eq!( + parse_work_db_memory_profile("none"), + Some(WorkDbMemoryProfile::Default) + ); + assert_eq!( + parse_work_db_memory_profile("compact"), + Some(WorkDbMemoryProfile::Compact) + ); + assert_eq!( + parse_work_db_memory_profile("low-memory"), + Some(WorkDbMemoryProfile::Compact) + ); + assert_eq!(parse_work_db_memory_profile("unexpected"), None); +} + +#[test] +fn vcir_field_size_breakdown_counts_local_outputs_and_artifacts() { + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let breakdown = VcirFieldSizeBreakdown::from_vcir(&vcir); + assert_eq!(breakdown.local_output_count, 2); + assert_eq!( + breakdown.local_output_payload_json_bytes, + vcir.local_outputs + .iter() + .map(|output| output.payload_json().len() as u64) + .sum::() + ); + assert_eq!( + breakdown.local_output_rule_hash_hex_bytes, + vcir.local_outputs.len() as u64 * 64 + ); + assert_eq!(breakdown.related_artifact_count, 2); + assert!(breakdown.related_artifact_uri_bytes > 0); + assert_eq!(breakdown.child_entry_count, 1); + assert!(breakdown.child_entry_uri_bytes > 0); + assert_eq!( + breakdown.local_output_old_projection_bytes(), + breakdown.local_output_source_type_bytes + + breakdown.local_output_source_hash_hex_bytes + + breakdown.local_output_source_ee_hash_hex_bytes + + breakdown.local_output_payload_json_bytes + + breakdown.local_output_rule_hash_hex_bytes + ); + assert!( + breakdown.local_output_old_projection_bytes() + > breakdown.local_output_typed_projection_bytes() + ); +} + +fn sample_repository_view_entry(rsync_uri: &str, bytes: &[u8]) -> RepositoryViewEntry { + RepositoryViewEntry { + rsync_uri: rsync_uri.to_string(), + current_hash: Some(sha256_hex(bytes)), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("cer".to_string()), + state: RepositoryViewState::Present, + } +} + +fn sample_raw_by_hash_entry(bytes: Vec) -> RawByHashEntry { + RawByHashEntry { + sha256_hex: sha256_hex(&bytes), + bytes, + origin_uris: vec!["rsync://example.test/repo/object.cer".to_string()], + object_type: Some("cer".to_string()), + encoding: Some("der".to_string()), + } +} + +fn sample_ccr_manifest_projection( + manifest_rsync_uri: &str, + manifest_this_update: PackTime, + subordinate_skis: Vec>, +) -> VcirCcrManifestProjection { + VcirCcrManifestProjection { + manifest_rsync_uri: manifest_rsync_uri.to_string(), + manifest_sha256: vec![0x11; 32], + manifest_size: 4096, + manifest_ee_aki: vec![0x22; 20], + manifest_number_be: vec![3], + manifest_this_update, + manifest_sia_locations_der: vec![vec![ + 0x30, 0x11, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x05, 0x86, 0x05, + b'r', b's', b'y', b'n', b'c', + ]], + subordinate_skis, + } +} + +fn sample_vcir(manifest_rsync_uri: &str) -> ValidatedCaInstanceResult { + let roa_bytes = b"roa-object".to_vec(); + let ee_bytes = b"ee-cert".to_vec(); + let child_bytes = b"child-cert".to_vec(); + let child_ski = "1234567890abcdef1234567890abcdef12345678".to_string(); + ValidatedCaInstanceResult { + manifest_rsync_uri: manifest_rsync_uri.to_string(), + parent_manifest_rsync_uri: Some("rsync://example.test/repo/parent/parent.mft".to_string()), + tal_id: "apnic".to_string(), + ca_subject_name: "CN=Example CA".to_string(), + ca_ski: "00112233445566778899aabbccddeeff00112233".to_string(), + issuer_ski: "ffeeddccbbaa99887766554433221100ffeeddcc".to_string(), + last_successful_validation_time: pack_time(0), + current_manifest_rsync_uri: manifest_rsync_uri.to_string(), + current_crl_rsync_uri: "rsync://example.test/repo/current.crl".to_string(), + validated_manifest_meta: ValidatedManifestMeta { + validated_manifest_number: vec![3], + validated_manifest_this_update: pack_time(0), + validated_manifest_next_update: pack_time(24), + }, + ccr_manifest_projection: sample_ccr_manifest_projection( + manifest_rsync_uri, + pack_time(0), + vec![hex::decode(&child_ski).expect("decode child ski")], + ), + instance_gate: VcirInstanceGate { + manifest_next_update: pack_time(24), + current_crl_next_update: pack_time(12), + self_ca_not_after: pack_time(48), + instance_effective_until: pack_time(12), + }, + child_entries: vec![VcirChildEntry { + child_manifest_rsync_uri: "rsync://example.test/repo/child/child.mft".to_string(), + child_cert_rsync_uri: "rsync://example.test/repo/child/child.cer".to_string(), + child_cert_hash: sha256_hex(&child_bytes), + child_ski, + child_rsync_base_uri: "rsync://example.test/repo/child/".to_string(), + child_publication_point_rsync_uri: "rsync://example.test/repo/child/".to_string(), + child_rrdp_notification_uri: Some("https://example.test/child-notify.xml".to_string()), + child_effective_ip_resources: None, + child_effective_as_resources: None, + accepted_at_validation_time: pack_time(0), + }], + local_outputs: vec![ + VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: pack_time(12), + source_object_uri: "rsync://example.test/repo/object.roa".to_string(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: sha256_32(&roa_bytes), + source_ee_cert_hash: sha256_32(&ee_bytes), + payload: VcirLocalOutputPayload::Vrp { + asn: 64496, + afi: crate::data_model::roa::RoaAfi::Ipv4, + prefix_len: 24, + addr: { + let mut addr = [0u8; 16]; + addr[..4].copy_from_slice(&[203, 0, 113, 0]); + addr + }, + max_length: 24, + }, + rule_hash: sha256_32(b"vrp-rule-1"), + }, + VcirLocalOutput { + output_type: VcirOutputType::Aspa, + item_effective_until: pack_time(10), + source_object_uri: "rsync://example.test/repo/object.asa".to_string(), + source_object_type: VcirSourceObjectType::Aspa, + source_object_hash: sha256_32(b"aspa-object"), + source_ee_cert_hash: sha256_32(b"aspa-ee-cert"), + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: 64496, + provider_as_ids: vec![64497], + }, + rule_hash: sha256_32(b"aspa-rule-1"), + }, + ], + related_artifacts: vec![ + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::Manifest, + artifact_kind: VcirArtifactKind::Mft, + uri: Some(manifest_rsync_uri.to_string()), + sha256: sha256_hex(b"manifest-object"), + object_type: Some("mft".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::CurrentCrl, + artifact_kind: VcirArtifactKind::Crl, + uri: Some("rsync://example.test/repo/current.crl".to_string()), + sha256: sha256_hex(b"current-crl"), + object_type: Some("crl".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }, + ], + summary: VcirSummary { + local_vrp_count: 1, + local_aspa_count: 1, + local_router_key_count: 0, + child_count: 1, + accepted_object_count: 4, + rejected_object_count: 0, + }, + audit_summary: VcirAuditSummary { + failed_fetch_eligible: true, + last_failed_fetch_reason: None, + warning_count: 0, + audit_flags: vec!["validated-fresh".to_string()], + }, + } +} + +fn roa_cache_projection_context(vcir: &ValidatedCaInstanceResult) -> RoaCacheProjectionContext { + let roa_output = vcir + .local_outputs + .iter() + .find(|output| { + output.output_type == VcirOutputType::Vrp + && output.source_object_type == VcirSourceObjectType::Roa + }) + .expect("sample VCIR has a ROA output"); + RoaCacheProjectionContext { + ca_validation_context_digest: [0x31; 32], + policy_fingerprint: [0x32; 32], + object_meta: vec![RoaCacheObjectMeta { + source_object_uri: roa_output.source_object_uri.clone(), + source_object_hash: roa_output.source_object_hash, + ee_serial: vec![0x01], + crl_uri: vcir.current_crl_rsync_uri.clone(), + earliest_safe_reuse_time: vcir.last_successful_validation_time.clone(), + }], + } +} diff --git a/crates/panda-rpki-validator/src/storage/tests_parts/object_loading.rs b/crates/panda-rpki-validator/src/storage/tests_parts/object_loading.rs new file mode 100644 index 0000000..823a6f8 --- /dev/null +++ b/crates/panda-rpki-validator/src/storage/tests_parts/object_loading.rs @@ -0,0 +1,266 @@ +// Storage test group: object loading. + +#[test] +fn load_current_object_with_hash_by_uri_uses_internal_blob_cf_without_raw_entry() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let rsync_uri = "rsync://example.test/repo/blob-only.roa"; + let bytes = b"blob-only-current-object".to_vec(); + let hash = sha256_hex(&bytes); + + store + .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) + .expect("put blob bytes"); + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: rsync_uri.to_string(), + current_hash: Some(hash.clone()), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }) + .expect("put view"); + + let got = store + .load_current_object_with_hash_by_uri(rsync_uri) + .expect("load current object") + .expect("current object exists"); + assert_eq!(got.current_hash_hex, hash); + assert_eq!(got.current_hash, compute_sha256_32(&bytes)); + assert_eq!(got.bytes, bytes); + assert!( + store + .get_raw_by_hash_entry(&got.current_hash_hex) + .expect("get raw entry") + .is_none() + ); +} + +#[test] +fn load_current_object_bytes_by_uri_uses_internal_blob_cf_without_raw_entry() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let rsync_uri = "rsync://example.test/repo/blob-only-bytes.roa"; + let bytes = b"blob-only-current-object-bytes".to_vec(); + let hash = sha256_hex(&bytes); + + store + .put_blob_bytes_batch(&[(hash, bytes.clone())]) + .expect("put blob bytes"); + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: rsync_uri.to_string(), + current_hash: Some(sha256_hex(&bytes)), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }) + .expect("put view"); + + assert_eq!( + store + .load_current_object_bytes_by_uri(rsync_uri) + .expect("load current object bytes"), + Some(bytes) + ); +} + +#[test] +fn pack_file_can_lazy_load_bytes_from_external_raw_store() { + let td = tempfile::tempdir().expect("tempdir"); + let raw_store = std::sync::Arc::new( + ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store"), + ); + let bytes = b"lazy-pack-file".to_vec(); + let sha256_hex = sha256_hex(&bytes); + raw_store + .put_raw_entry(&RawByHashEntry::from_bytes( + sha256_hex.clone(), + bytes.clone(), + )) + .expect("put raw entry"); + + let file = PackFile::from_lazy_external_raw_store( + "rsync://example.test/repo/a.roa", + sha256_hex, + compute_sha256_32(&bytes), + raw_store, + ); + + assert_eq!(file.bytes().expect("lazy bytes"), bytes.as_slice()); + assert_eq!(file.bytes_cloned().expect("cloned bytes"), bytes); +} + +#[test] +fn pack_file_can_lazy_load_bytes_from_external_repo_bytes_store() { + let td = tempfile::tempdir().expect("tempdir"); + let repo_bytes_store = std::sync::Arc::new( + ExternalRepoBytesDb::open(td.path().join("repo-bytes.db")).expect("open repo bytes"), + ); + let bytes = b"repo-object-pack-file".to_vec(); + let sha256_hex = sha256_hex(&bytes); + repo_bytes_store + .put_blob_bytes_batch(&[(sha256_hex.clone(), bytes.clone())]) + .expect("put repo bytes"); + + let file = PackFile::from_lazy_repo_bytes( + "rsync://example.test/repo/a.roa", + sha256_hex, + compute_sha256_32(&bytes), + repo_bytes_store, + ); + + assert_eq!(file.bytes().expect("lazy repo bytes"), bytes.as_slice()); + assert_eq!(file.bytes_cloned().expect("cloned repo bytes"), bytes); + assert_eq!(file.compute_sha256().expect("compute sha256"), file.sha256); +} + +#[test] +fn read_only_checkpoint_isolated_from_source_work_db() { + let td = tempfile::tempdir().expect("tempdir"); + let source_path = td.path().join("source-work-db"); + let checkpoint_path = td.path().join("checkpoint-work-db"); + let uri = "rsync://example.test/repo/a.roa"; + let bytes = b"checkpoint-object"; + let hash = sha256_hex(bytes); + { + let source = RocksStore::open(&source_path).expect("open source"); + source + .put_blob_bytes_batch(&[(hash.clone(), bytes.to_vec())]) + .expect("put blob"); + source + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: uri.to_string(), + current_hash: Some(hash), + repository_source: Some("fixture".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }) + .expect("put view"); + } + + RocksStore::create_read_only_checkpoint(&source_path, &checkpoint_path) + .expect("create checkpoint"); + let checkpoint = RocksStore::open(&checkpoint_path).expect("open checkpoint"); + checkpoint + .delete_repository_view_entry(uri) + .expect("delete checkpoint entry"); + drop(checkpoint); + + let source = RocksStore::open(&source_path).expect("reopen source"); + assert!( + source + .get_repository_view_entry(uri) + .expect("get source entry") + .is_some() + ); +} + +#[test] +fn read_only_external_repo_bytes_rejects_writes() { + let td = tempfile::tempdir().expect("tempdir"); + let path = td.path().join("repo-bytes.db"); + let hash = sha256_hex(b"repo-object"); + { + let writable = ExternalRepoBytesDb::open(&path).expect("open writable repo bytes"); + writable + .put_blob_bytes_batch(&[(hash.clone(), b"repo-object".to_vec())]) + .expect("seed repo bytes"); + } + let read_only = ExternalRepoBytesDb::open_read_only(&path).expect("open read-only repo bytes"); + assert_eq!( + read_only + .get_blob_bytes(&hash) + .expect("read repo bytes") + .expect("blob"), + b"repo-object" + ); + assert!( + read_only + .put_blob_bytes_batch(&[(hash, b"repo-object".to_vec())]) + .is_err() + ); +} + +#[test] +fn read_only_store_accepts_only_idempotent_existing_blob_apply() { + let td = tempfile::tempdir().expect("tempdir"); + let work_db_path = td.path().join("work-db"); + let repo_bytes_path = td.path().join("repo-bytes.db"); + let bytes = b"repo-object".to_vec(); + let hash = sha256_hex(&bytes); + { + let repo_bytes = ExternalRepoBytesDb::open(&repo_bytes_path).expect("open repo bytes"); + repo_bytes + .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) + .expect("seed repo bytes"); + } + let store = + RocksStore::open_with_external_repo_bytes_read_only(&work_db_path, &repo_bytes_path) + .expect("open read-only store"); + store + .put_blob_bytes_batch(&[(hash.clone(), bytes)]) + .expect("idempotent apply"); + assert!( + store + .put_blob_bytes_batch(&[(hash, b"different".to_vec())]) + .is_err() + ); +} + +#[test] +fn repository_blob_verification_detects_missing_external_blob() { + let td = tempfile::tempdir().expect("tempdir"); + let repo_bytes_path = td.path().join("repo-bytes.db"); + ExternalRepoBytesDb::open(&repo_bytes_path).expect("create repo bytes"); + let store = RocksStore::open_with_external_repo_bytes_read_only( + &td.path().join("work-db"), + &repo_bytes_path, + ) + .expect("open work db"); + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/missing.roa".to_string(), + current_hash: Some(sha256_hex(b"missing")), + repository_source: Some("fixture".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }) + .expect("put view"); + let error = store + .verify_current_repository_blobs(16) + .expect_err("missing blob must fail"); + assert!(error.to_string().contains("missing.roa"), "{error}"); +} + +#[test] +fn vcir_related_artifact_reject_reason_serde_backward_compatible() { + // Legacy cache JSON written before the reject_reason ("e") field existed. + let legacy = r#"{"r":"manifest","k":"mft","u":"rsync://example.test/a.mft","h":"0000000000000000000000000000000000000000000000000000000000000000","t":"mft","s":"accepted"}"#; + let artifact: VcirRelatedArtifact = serde_json::from_str(legacy).expect("legacy artifact"); + assert_eq!(artifact.reject_reason, None); + + // None reason is skipped on serialization, keeping cache entries compact. + let json_none = serde_json::to_string(&artifact).expect("serialize none"); + assert!(!json_none.contains("\"e\"")); + + // A recorded reason round-trips through the "e" field. + let mut with_reason = artifact.clone(); + with_reason.validation_status = VcirArtifactValidationStatus::Rejected; + with_reason.reject_reason = Some("bad object".to_string()); + let json = serde_json::to_string(&with_reason).expect("serialize"); + assert!(json.contains("\"e\":\"bad object\"")); + let decoded: VcirRelatedArtifact = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(decoded.reject_reason.as_deref(), Some("bad object")); + + // PublicationPointCacheObject conversion carries the reason both ways. + let cache_object = PublicationPointCacheObject::from_related_artifact(&with_reason); + assert_eq!(cache_object.reject_reason.as_deref(), Some("bad object")); + let back = cache_object.to_related_artifact(); + assert_eq!(back.reject_reason.as_deref(), Some("bad object")); + + // Legacy PublicationPointCacheObject JSON without "e" still deserializes. + let legacy_object: PublicationPointCacheObject = + serde_json::from_str(legacy).expect("legacy cache object"); + assert_eq!(legacy_object.reject_reason, None); +} diff --git a/crates/panda-rpki-validator/src/storage/tests_parts/projection.rs b/crates/panda-rpki-validator/src/storage/tests_parts/projection.rs new file mode 100644 index 0000000..3136c70 --- /dev/null +++ b/crates/panda-rpki-validator/src/storage/tests_parts/projection.rs @@ -0,0 +1,473 @@ +// Storage test group: projection. + +#[test] +fn vcir_ccr_manifest_projection_validate_accepts_valid_projection() { + let projection = sample_ccr_manifest_projection( + "rsync://example.test/repo/current.mft", + pack_time(0), + vec![vec![0x33; 20], vec![0x44; 20]], + ); + projection.validate_internal().expect("valid projection"); +} + +#[test] +fn vcir_ccr_manifest_projection_validate_rejects_invalid_fields() { + let mut bad_hash = sample_ccr_manifest_projection( + "rsync://example.test/repo/current.mft", + pack_time(0), + vec![vec![0x33; 20]], + ); + bad_hash.manifest_sha256 = vec![0x11; 31]; + assert!(matches!( + bad_hash.validate_internal(), + Err(StorageError::InvalidData { .. }) + )); + + let mut bad_locations = sample_ccr_manifest_projection( + "rsync://example.test/repo/current.mft", + pack_time(0), + vec![vec![0x33; 20]], + ); + bad_locations.manifest_sia_locations_der = vec![vec![0x04, 0x00]]; + assert!(matches!( + bad_locations.validate_internal(), + Err(StorageError::InvalidData { .. }) + )); + + let bad_subordinates = sample_ccr_manifest_projection( + "rsync://example.test/repo/current.mft", + pack_time(0), + vec![vec![0x44; 20], vec![0x33; 20]], + ); + assert!(matches!( + bad_subordinates.validate_internal(), + Err(StorageError::InvalidData { .. }) + )); +} + +#[test] +fn roa_cache_projection_from_vcir_keeps_only_roa_vrp_outputs() { + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let context = roa_cache_projection_context(&vcir); + let projection = RoaCacheProjection::from_vcir_with_context(&vcir, Some(&context)) + .expect("projection build") + .expect("projection exists"); + + assert_eq!(projection.manifest_rsync_uri, vcir.manifest_rsync_uri); + assert_eq!(projection.instance_effective_until, pack_time(12)); + assert_eq!(projection.crl_sha256_by_uri.len(), 1); + assert_eq!(projection.entries.len(), 1); + assert_eq!( + projection.entries[0].source_object_uri, + "rsync://example.test/repo/object.roa" + ); + assert_eq!( + projection.entries[0].outputs_effective_until_unix, + 12 * 3600 + ); + assert_eq!(projection.entries[0].outputs.len(), 1); + assert!(matches!( + projection.entries[0].outputs[0].payload, + VcirLocalOutputPayload::Vrp { .. } + )); +} + +#[test] +fn roa_cache_projection_groups_multiple_outputs_by_roa_uri() { + let mut vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let mut second = vcir.local_outputs[0].clone(); + second.rule_hash = sha256_32(b"vrp-rule-2"); + if let VcirLocalOutputPayload::Vrp { max_length, .. } = &mut second.payload { + *max_length = 25; + } + vcir.local_outputs.push(second); + vcir.summary.local_vrp_count = 2; + + let context = roa_cache_projection_context(&vcir); + let projection = RoaCacheProjection::from_vcir_with_context(&vcir, Some(&context)) + .expect("projection build") + .expect("projection exists"); + + assert_eq!(projection.entries.len(), 1); + assert_eq!(projection.entries[0].outputs.len(), 2); + assert_eq!( + projection.entries[0].outputs_effective_until_unix, + 12 * 3600 + ); +} + +#[test] +fn publication_point_cache_projection_roundtrips_with_vcir() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let projection = PublicationPointCacheProjection::from_vcir_with_context( + &vcir, + "rsync://example.test/repo/".to_string(), + Some("rsync://example.test/repo/ca.cer".to_string()), + sha256_32(b"ca-cert"), + sha256_32(b"manifest"), + sha256_32(b"ta-context"), + sha256_32(b"parent-context"), + sha256_32(b"policy"), + ) + .expect("build publication point projection"); + + store + .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) + .expect("put vcir with publication point projection"); + + let got_vcir = store + .get_vcir(&vcir.manifest_rsync_uri) + .expect("get vcir") + .expect("vcir exists"); + assert_eq!(got_vcir, vcir); + let got_projection = store + .get_publication_point_cache_projection(&vcir.manifest_rsync_uri) + .expect("get publication point projection") + .expect("projection exists"); + assert_eq!(got_projection, projection); + assert_eq!(got_projection.outputs.len(), 2); + assert_eq!(got_projection.children.len(), 1); + assert_eq!(got_projection.related_objects.len(), 2); +} + +#[test] +fn publication_point_cache_projection_index_updates_after_first_read() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let mut projection = PublicationPointCacheProjection::from_vcir_with_context( + &vcir, + "rsync://example.test/repo/".to_string(), + Some("rsync://example.test/repo/ca.cer".to_string()), + sha256_32(b"ca-cert"), + sha256_32(b"manifest-old"), + sha256_32(b"ta-context"), + sha256_32(b"parent-context"), + sha256_32(b"policy"), + ) + .expect("build old publication point projection"); + + store + .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) + .expect("put old projection"); + let got_old = store + .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) + .expect("get old projection") + .expect("old projection exists"); + assert_eq!(got_old.manifest_sha256, sha256_32(b"manifest-old")); + + projection.manifest_sha256 = sha256_32(b"manifest-new"); + store + .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) + .expect("put new projection"); + let got_new = store + .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) + .expect("get new projection") + .expect("new projection exists"); + assert_eq!(got_new.manifest_sha256, sha256_32(b"manifest-new")); +} + +#[test] +fn publication_point_cache_projection_cached_empty_db_accepts_bounded_new_entries() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let projection = PublicationPointCacheProjection::from_vcir_with_context( + &vcir, + "rsync://example.test/repo/".to_string(), + Some("rsync://example.test/repo/ca.cer".to_string()), + sha256_32(b"ca-cert"), + sha256_32(b"manifest"), + sha256_32(b"ta-context"), + sha256_32(b"parent-context"), + sha256_32(b"policy"), + ) + .expect("build publication point projection"); + + assert!( + store + .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) + .expect("cached empty lookup") + .is_none() + ); + store + .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) + .expect("put projection after cached empty lookup"); + assert_eq!( + store + .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) + .expect("cached lookup sees bounded new entry") + .expect("cached projection exists"), + projection + ); + assert_eq!( + store + .get_publication_point_cache_projection(&vcir.manifest_rsync_uri) + .expect("direct db lookup") + .expect("projection exists"), + projection + ); +} + +#[test] +fn publication_point_cache_mmap_index_refresh_roundtrips_after_reopen() { + let td = tempfile::tempdir().expect("tempdir"); + let db_path = td.path().join("work-db"); + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let projection = PublicationPointCacheProjection::from_vcir_with_context( + &vcir, + "rsync://example.test/repo/".to_string(), + Some("rsync://example.test/repo/ca.cer".to_string()), + sha256_32(b"ca-cert"), + sha256_32(b"manifest"), + sha256_32(b"ta-context"), + sha256_32(b"parent-context"), + sha256_32(b"policy"), + ) + .expect("build publication point projection"); + + { + let store = RocksStore::open(&db_path).expect("open rocksdb"); + store + .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) + .expect("put projection"); + let stats = store + .refresh_publication_point_cache_mmap_index() + .expect("refresh mmap index") + .expect("refresh stats"); + assert_eq!(stats.new_entries, 1); + assert_eq!(stats.state, "written"); + } + + let store = RocksStore::open(&db_path).expect("reopen rocksdb"); + let got = store + .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) + .expect("get cached projection from mmap") + .expect("projection exists"); + assert_eq!(got, projection); +} + +#[test] +fn publication_point_cache_mmap_index_dirty_overlay_wins_and_refreshes_segment() { + let td = tempfile::tempdir().expect("tempdir"); + let db_path = td.path().join("work-db"); + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let mut projection = PublicationPointCacheProjection::from_vcir_with_context( + &vcir, + "rsync://example.test/repo/".to_string(), + Some("rsync://example.test/repo/ca.cer".to_string()), + sha256_32(b"ca-cert"), + sha256_32(b"manifest-old"), + sha256_32(b"ta-context"), + sha256_32(b"parent-context"), + sha256_32(b"policy"), + ) + .expect("build publication point projection"); + + { + let store = RocksStore::open(&db_path).expect("open rocksdb"); + store + .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) + .expect("put old projection"); + store + .refresh_publication_point_cache_mmap_index() + .expect("refresh base mmap index"); + } + + { + let store = RocksStore::open(&db_path).expect("reopen rocksdb"); + assert_eq!( + store + .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) + .expect("get old projection") + .expect("old projection exists") + .manifest_sha256, + sha256_32(b"manifest-old") + ); + projection.manifest_sha256 = sha256_32(b"manifest-new"); + store + .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) + .expect("put new projection"); + assert_eq!( + store + .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) + .expect("get dirty projection") + .expect("dirty projection exists") + .manifest_sha256, + sha256_32(b"manifest-new") + ); + let stats = store + .refresh_publication_point_cache_mmap_index() + .expect("refresh dirty mmap segment") + .expect("refresh stats"); + assert_eq!(stats.state, "segment_written"); + assert_eq!(stats.dirty_entries, 1); + } + + let store = RocksStore::open(&db_path).expect("reopen rocksdb after segment"); + let got = store + .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) + .expect("get refreshed projection") + .expect("projection exists"); + assert_eq!(got.manifest_sha256, sha256_32(b"manifest-new")); +} + +#[test] +fn publication_point_cache_mmap_index_write_before_read_refreshes_segment() { + let td = tempfile::tempdir().expect("tempdir"); + let db_path = td.path().join("work-db"); + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let mut projection = PublicationPointCacheProjection::from_vcir_with_context( + &vcir, + "rsync://example.test/repo/".to_string(), + Some("rsync://example.test/repo/ca.cer".to_string()), + sha256_32(b"ca-cert"), + sha256_32(b"manifest-old"), + sha256_32(b"ta-context"), + sha256_32(b"parent-context"), + sha256_32(b"policy"), + ) + .expect("build publication point projection"); + + { + let store = RocksStore::open(&db_path).expect("open rocksdb"); + store + .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) + .expect("put old projection"); + let stats = store + .refresh_publication_point_cache_mmap_index() + .expect("refresh base mmap index") + .expect("refresh stats"); + assert_eq!(stats.state, "written"); + } + + { + let store = RocksStore::open(&db_path).expect("reopen rocksdb"); + projection.manifest_sha256 = sha256_32(b"manifest-new"); + store + .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) + .expect("put new projection before any cached read"); + let stats = store + .refresh_publication_point_cache_mmap_index() + .expect("refresh dirty mmap segment") + .expect("refresh stats"); + assert_eq!(stats.state, "segment_written"); + assert_eq!(stats.dirty_entries, 1); + assert_eq!(stats.new_entries, 1); + } + + let store = RocksStore::open(&db_path).expect("reopen rocksdb after segment"); + let got = store + .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) + .expect("get refreshed projection") + .expect("projection exists"); + assert_eq!(got.manifest_sha256, sha256_32(b"manifest-new")); +} + +#[test] +fn publication_point_cache_mmap_index_delete_before_read_refreshes_tombstone_segment() { + let td = tempfile::tempdir().expect("tempdir"); + let db_path = td.path().join("work-db"); + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let projection = PublicationPointCacheProjection::from_vcir_with_context( + &vcir, + "rsync://example.test/repo/".to_string(), + Some("rsync://example.test/repo/ca.cer".to_string()), + sha256_32(b"ca-cert"), + sha256_32(b"manifest"), + sha256_32(b"ta-context"), + sha256_32(b"parent-context"), + sha256_32(b"policy"), + ) + .expect("build publication point projection"); + + { + let store = RocksStore::open(&db_path).expect("open rocksdb"); + store + .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) + .expect("put projection"); + let stats = store + .refresh_publication_point_cache_mmap_index() + .expect("refresh base mmap index") + .expect("refresh stats"); + assert_eq!(stats.state, "written"); + } + + { + let store = RocksStore::open(&db_path).expect("reopen rocksdb"); + store + .replace_vcir_manifest_replay_meta_and_projection_action( + &vcir, + None, + PublicationPointCacheProjectionWriteAction::Delete { + manifest_rsync_uri: &vcir.manifest_rsync_uri, + }, + ) + .expect("delete projection before any cached read"); + assert!( + store + .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) + .expect("dirty tombstone lookup") + .is_none() + ); + let stats = store + .refresh_publication_point_cache_mmap_index() + .expect("refresh tombstone mmap segment") + .expect("refresh stats"); + assert_eq!(stats.state, "segment_written"); + assert_eq!(stats.dirty_entries, 1); + assert_eq!(stats.new_entries, 1); + } + + let store = RocksStore::open(&db_path).expect("reopen rocksdb after tombstone segment"); + assert!( + store + .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) + .expect("tombstone shadows old current index") + .is_none() + ); + assert!( + store + .get_publication_point_cache_projection(&vcir.manifest_rsync_uri) + .expect("direct db lookup") + .is_none() + ); +} + +#[test] +fn publication_point_cache_projection_rejects_version_mismatch() { + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let mut projection = PublicationPointCacheProjection::from_vcir_with_context( + &vcir, + "rsync://example.test/repo/".to_string(), + Some("rsync://example.test/repo/ca.cer".to_string()), + sha256_32(b"ca-cert"), + sha256_32(b"manifest"), + sha256_32(b"ta-context"), + sha256_32(b"parent-context"), + sha256_32(b"policy"), + ) + .expect("build publication point projection"); + projection.schema_version = PUBLICATION_POINT_CACHE_SCHEMA_VERSION + 1; + assert!(matches!( + projection.validate_internal(), + Err(StorageError::InvalidData { .. }) + )); +} + +#[test] +fn roa_cache_projection_rejects_duplicate_uri_with_different_hash() { + let mut vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let mut duplicate = vcir.local_outputs[0].clone(); + duplicate.source_object_hash = sha256_32(b"different-roa"); + duplicate.rule_hash = sha256_32(b"vrp-rule-2"); + vcir.local_outputs.push(duplicate); + vcir.summary.local_vrp_count = 2; + + let context = roa_cache_projection_context(&vcir); + let err = RoaCacheProjection::from_vcir_with_context(&vcir, Some(&context)) + .expect_err("same ROA URI with different hash must fail"); + assert!(err.to_string().contains("source object hash mismatch")); +} diff --git a/crates/panda-rpki-validator/src/storage/tests_parts/repository.rs b/crates/panda-rpki-validator/src/storage/tests_parts/repository.rs new file mode 100644 index 0000000..ffb6d85 --- /dev/null +++ b/crates/panda-rpki-validator/src/storage/tests_parts/repository.rs @@ -0,0 +1,418 @@ +// Storage test group: repository. + +fn sample_rrdp_source_record(notify_uri: &str) -> RrdpSourceRecord { + RrdpSourceRecord { + notify_uri: notify_uri.to_string(), + last_session_id: Some("session-1".to_string()), + last_serial: Some(42), + first_seen_at: pack_time(0), + last_seen_at: pack_time(1), + last_sync_at: Some(pack_time(1)), + sync_state: RrdpSourceSyncState::DeltaReady, + last_snapshot_uri: Some("https://rrdp.example.test/snapshot.xml".to_string()), + last_snapshot_hash: Some(sha256_hex(b"snapshot-bytes")), + last_error: None, + } +} + +fn sample_rrdp_source_member_record( + notify_uri: &str, + rsync_uri: &str, + serial: u64, +) -> RrdpSourceMemberRecord { + RrdpSourceMemberRecord { + notify_uri: notify_uri.to_string(), + rsync_uri: rsync_uri.to_string(), + current_hash: Some(sha256_hex(rsync_uri.as_bytes())), + object_type: Some("cer".to_string()), + present: true, + last_confirmed_session_id: "session-1".to_string(), + last_confirmed_serial: serial, + last_changed_at: pack_time(serial as i64), + } +} + +fn sample_rrdp_uri_owner_record(notify_uri: &str, rsync_uri: &str) -> RrdpUriOwnerRecord { + RrdpUriOwnerRecord { + rsync_uri: rsync_uri.to_string(), + notify_uri: notify_uri.to_string(), + current_hash: Some(sha256_hex(rsync_uri.as_bytes())), + last_confirmed_session_id: "session-1".to_string(), + last_confirmed_serial: 7, + last_changed_at: pack_time(7), + owner_state: RrdpUriOwnerState::Active, + } +} + +#[test] +fn repository_view_and_raw_by_hash_roundtrip() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let entry1 = sample_repository_view_entry("rsync://example.test/repo/a.cer", b"object-a"); + let entry2 = sample_repository_view_entry("rsync://example.test/repo/sub/b.roa", b"object-b"); + store + .put_repository_view_entry(&entry1) + .expect("put repository view entry1"); + store + .put_repository_view_entry(&entry2) + .expect("put repository view entry2"); + + let got1 = store + .get_repository_view_entry(&entry1.rsync_uri) + .expect("get repository view entry1") + .expect("entry1 exists"); + assert_eq!(got1, entry1); + + let got_prefix = store + .list_repository_view_entries_with_prefix("rsync://example.test/repo/sub/") + .expect("list repository view prefix"); + assert_eq!(got_prefix, vec![entry2.clone()]); + + store + .delete_repository_view_entry(&entry1.rsync_uri) + .expect("delete repository view entry1"); + assert!( + store + .get_repository_view_entry(&entry1.rsync_uri) + .expect("get deleted repository view entry1") + .is_none() + ); + + let raw = sample_raw_by_hash_entry(b"raw-der-object".to_vec()); + store + .put_raw_by_hash_entry(&raw) + .expect("put raw_by_hash entry"); + let got_raw = store + .get_raw_by_hash_entry(&raw.sha256_hex) + .expect("get raw_by_hash entry") + .expect("raw entry exists"); + assert_eq!(got_raw, raw); +} + +#[test] +fn raw_by_hash_routes_to_external_raw_store_when_configured() { + let td = tempfile::tempdir().expect("tempdir"); + let main_db = td.path().join("main-db"); + let raw_db = td.path().join("raw-store.db"); + + let raw = sample_raw_by_hash_entry(b"external-raw".to_vec()); + { + let store = + RocksStore::open_with_external_raw_store(&main_db, &raw_db).expect("open store"); + store.put_raw_by_hash_entry(&raw).expect("put external raw"); + + let got = store + .get_raw_by_hash_entry(&raw.sha256_hex) + .expect("get external raw") + .expect("raw exists"); + assert_eq!(got, raw); + } + + let main_store = RocksStore::open(&main_db).expect("open main only"); + assert!( + main_store + .get_raw_by_hash_entry(&raw.sha256_hex) + .expect("read main store") + .is_none(), + "main db should not contain raw entry when external raw store is configured" + ); +} + +#[test] +fn put_blob_bytes_batch_uses_internal_blob_cf_without_raw_entry() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let bytes = b"internal-blob-only".to_vec(); + let hash = sha256_hex(&bytes); + + store + .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) + .expect("put blob bytes"); + + assert_eq!( + store.get_blob_bytes(&hash).expect("get blob bytes"), + Some(bytes.clone()) + ); + assert!( + store + .get_raw_by_hash_entry(&hash) + .expect("get raw entry") + .is_none() + ); +} + +#[test] +fn put_blob_bytes_batch_routes_to_external_raw_store_without_raw_entry() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open_with_external_raw_store( + &td.path().join("main-db"), + &td.path().join("raw-store.db"), + ) + .expect("open store"); + let bytes = b"external-blob-only".to_vec(); + let hash = sha256_hex(&bytes); + + store + .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) + .expect("put external blob bytes"); + + assert_eq!(store.get_blob_bytes(&hash).unwrap(), Some(bytes)); + assert!(store.get_raw_by_hash_entry(&hash).unwrap().is_none()); +} + +#[test] +fn repo_bytes_db_is_physically_separate_from_external_raw_store() { + let td = tempfile::tempdir().expect("tempdir"); + let main_db = td.path().join("main-db"); + let raw_db = td.path().join("raw-store.db"); + let repo_bytes_db = td.path().join("repo-bytes.db"); + let store = + RocksStore::open_with_external_stores(&main_db, Some(&raw_db), Some(&repo_bytes_db)) + .expect("open store"); + let repo_bytes = b"repo-object".to_vec(); + let repo_hash = sha256_hex(&repo_bytes); + let raw = sample_raw_by_hash_entry(b"raw-evidence".to_vec()); + + store + .put_blob_bytes_batch(&[(repo_hash.clone(), repo_bytes.clone())]) + .expect("put repo bytes"); + store.put_raw_by_hash_entry(&raw).expect("put raw evidence"); + + assert_eq!(store.get_blob_bytes(&repo_hash).unwrap(), Some(repo_bytes)); + assert_eq!( + store.get_raw_by_hash_entry(&raw.sha256_hex).unwrap(), + Some(raw.clone()) + ); + drop(store); + + let raw_only = RocksStore::open_with_external_raw_store(&td.path().join("raw-reader"), &raw_db) + .expect("open raw only"); + assert!( + raw_only.get_blob_bytes(&repo_hash).unwrap().is_none(), + "repo object bytes must not be written into raw-store.db" + ); + + let repo_only = + RocksStore::open_with_external_repo_bytes(&td.path().join("repo-reader"), &repo_bytes_db) + .expect("open repo bytes only"); + assert_eq!( + repo_only.get_blob_bytes(&repo_hash).unwrap(), + Some(b"repo-object".to_vec()) + ); + assert!( + repo_only.get_blob_bytes(&raw.sha256_hex).unwrap().is_none(), + "raw evidence bytes must not be written into repo-bytes.db" + ); +} + +#[test] +fn memory_snapshot_includes_work_db_and_external_stores() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open_with_external_stores( + &td.path().join("main-db"), + Some(&td.path().join("raw-store.db")), + Some(&td.path().join("repo-bytes.db")), + ) + .expect("open store"); + + let snapshot = store.memory_snapshot(); + let labels: Vec<&str> = snapshot + .databases + .iter() + .map(|db| db.label.as_str()) + .collect(); + assert_eq!(labels, vec!["work-db", "raw-store.db", "repo-bytes.db"]); + assert!( + snapshot.databases[0] + .column_families + .iter() + .any(|cf| cf.name == CF_REPOSITORY_VIEW) + ); + serde_json::to_value(&snapshot).expect("serialize memory snapshot"); +} + +#[test] +fn put_blob_bytes_batch_accepts_empty_batch_with_external_raw_store() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open_with_external_raw_store( + &td.path().join("main-db"), + &td.path().join("raw-store.db"), + ) + .expect("open store"); + + store + .put_blob_bytes_batch(&[]) + .expect("empty external blob batch should be a no-op"); +} + +#[test] +fn get_blob_bytes_internal_falls_back_to_raw_entry_when_blob_missing() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let raw = sample_raw_by_hash_entry(b"raw-fallback".to_vec()); + + store.put_raw_by_hash_entry(&raw).expect("put raw entry"); + + assert_eq!( + store + .get_blob_bytes(&raw.sha256_hex) + .expect("get blob bytes via raw fallback"), + Some(raw.bytes.clone()) + ); +} + +#[test] +fn get_blob_bytes_batch_internal_prefers_blob_cf_and_falls_back_to_raw_entry() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let blob_bytes = b"blob-cf-object".to_vec(); + let blob_hash = sha256_hex(&blob_bytes); + store + .put_blob_bytes_batch(&[(blob_hash.clone(), blob_bytes.clone())]) + .expect("put blob bytes"); + + let raw = sample_raw_by_hash_entry(b"raw-fallback-batch".to_vec()); + store.put_raw_by_hash_entry(&raw).expect("put raw fallback"); + + let batch = store + .get_blob_bytes_batch(&[blob_hash.clone(), raw.sha256_hex.clone(), "00".repeat(32)]) + .expect("get blob bytes batch"); + assert_eq!(batch, vec![Some(blob_bytes), Some(raw.bytes.clone()), None]); +} + +#[test] +fn get_blob_bytes_batch_routes_to_external_raw_store_without_raw_entry() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open_with_external_raw_store( + &td.path().join("main-db"), + &td.path().join("raw-store.db"), + ) + .expect("open store"); + let bytes = b"external-batch-blob".to_vec(); + let hash = sha256_hex(&bytes); + + store + .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) + .expect("put external blob bytes"); + + assert_eq!( + store + .get_blob_bytes_batch(&[hash, "00".repeat(32)]) + .expect("get external blob batch"), + vec![Some(bytes), None] + ); +} + +#[test] +fn get_blob_bytes_rejects_invalid_hash_for_internal_store() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let err = store + .get_blob_bytes("not-a-valid-hash") + .expect_err("invalid hash must fail"); + assert!(matches!(err, StorageError::InvalidData { .. })); +} + +#[test] +fn get_blob_bytes_batch_rejects_invalid_hash_for_internal_store() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let err = store + .get_blob_bytes_batch(&["not-a-valid-hash".to_string()]) + .expect_err("invalid hash must fail"); + assert!(matches!(err, StorageError::InvalidData { .. })); +} + +#[test] +fn get_blob_bytes_batch_returns_empty_for_empty_request_internal() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + assert!( + store + .get_blob_bytes_batch(&[]) + .expect("empty blob batch request") + .is_empty() + ); +} + +#[test] +fn put_blob_bytes_batch_accepts_empty_batch() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + store + .put_blob_bytes_batch(&[]) + .expect("empty blob batch should be a no-op"); +} + +#[test] +fn put_blob_bytes_batch_rejects_empty_bytes() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let err = store + .put_blob_bytes_batch(&[(sha256_hex(b"valid"), Vec::new())]) + .expect_err("empty bytes must fail"); + assert!(matches!(err, StorageError::InvalidData { .. })); +} + +#[test] +fn delete_raw_by_hash_entry_internal_preserves_blob_bytes() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let bytes = b"blob-persists-after-raw-delete".to_vec(); + let hash = sha256_hex(&bytes); + let raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone()); + + store + .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) + .expect("put blob bytes"); + store.put_raw_by_hash_entry(&raw).expect("put raw entry"); + + store + .delete_raw_by_hash_entry(&hash) + .expect("delete raw entry only"); + + assert!(store.get_raw_by_hash_entry(&hash).unwrap().is_none()); + assert_eq!(store.get_blob_bytes(&hash).unwrap(), Some(bytes)); +} + +#[test] +fn delete_raw_by_hash_entry_rejects_invalid_hash() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let err = store + .delete_raw_by_hash_entry("not-a-valid-hash") + .expect_err("invalid hash must fail"); + assert!(matches!(err, StorageError::InvalidData { .. })); +} + +#[test] +fn delete_raw_by_hash_entry_routes_to_external_raw_store() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open_with_external_raw_store( + &td.path().join("main-db"), + &td.path().join("raw-store.db"), + ) + .expect("open store"); + let raw = sample_raw_by_hash_entry(b"external-delete".to_vec()); + + store.put_raw_by_hash_entry(&raw).expect("put raw entry"); + store + .delete_raw_by_hash_entry(&raw.sha256_hex) + .expect("delete external raw entry"); + + assert!( + store + .get_raw_by_hash_entry(&raw.sha256_hex) + .unwrap() + .is_none() + ); + assert!(store.get_blob_bytes(&raw.sha256_hex).unwrap().is_none()); +} diff --git a/crates/panda-rpki-validator/src/storage/tests_parts/rrdp.rs b/crates/panda-rpki-validator/src/storage/tests_parts/rrdp.rs new file mode 100644 index 0000000..3906843 --- /dev/null +++ b/crates/panda-rpki-validator/src/storage/tests_parts/rrdp.rs @@ -0,0 +1,258 @@ +// Storage test group: rrdp. + +#[test] +fn projection_batch_roundtrip_writes_repository_view_member_and_owner_records() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(dir.path()).expect("open store"); + + let view = RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/a.roa".to_string(), + current_hash: Some(hex::encode([1u8; 32])), + repository_source: Some("https://example.test/notify.xml".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }; + let member = RrdpSourceMemberRecord { + notify_uri: "https://example.test/notify.xml".to_string(), + rsync_uri: "rsync://example.test/repo/a.roa".to_string(), + current_hash: Some(hex::encode([1u8; 32])), + object_type: Some("roa".to_string()), + present: true, + last_confirmed_session_id: "session-1".to_string(), + last_confirmed_serial: 7, + last_changed_at: pack_time(1), + }; + let owner = RrdpUriOwnerRecord { + rsync_uri: "rsync://example.test/repo/a.roa".to_string(), + notify_uri: "https://example.test/notify.xml".to_string(), + current_hash: Some(hex::encode([1u8; 32])), + last_confirmed_session_id: "session-1".to_string(), + last_confirmed_serial: 7, + last_changed_at: pack_time(1), + owner_state: RrdpUriOwnerState::Active, + }; + + store + .put_projection_batch(&[view.clone()], &[member.clone()], &[owner.clone()]) + .expect("write projection batch"); + + assert_eq!( + store + .get_repository_view_entry(&view.rsync_uri) + .expect("get view") + .expect("present view"), + view + ); + assert_eq!( + store + .get_rrdp_source_member_record(&member.notify_uri, &member.rsync_uri) + .expect("get member") + .expect("present member"), + member + ); + assert_eq!( + store + .get_rrdp_uri_owner_record(&owner.rsync_uri) + .expect("get owner") + .expect("present owner"), + owner + ); +} + +#[test] +fn current_rrdp_source_member_helpers_filter_present_records() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let notify_uri = "https://rrdp.example.test/notification.xml"; + let mut present_a = + sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/a.cer", 1); + let mut withdrawn_b = + sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/b.roa", 2); + withdrawn_b.present = false; + let present_c = + sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/c.crl", 3); + let other_source = sample_rrdp_source_member_record( + "https://other.example.test/notification.xml", + "rsync://other.example.test/repo/x.cer", + 4, + ); + present_a.last_confirmed_serial = 10; + + store + .put_rrdp_source_member_record(&present_a) + .expect("put present a"); + store + .put_rrdp_source_member_record(&withdrawn_b) + .expect("put withdrawn b"); + store + .put_rrdp_source_member_record(&present_c) + .expect("put present c"); + store + .put_rrdp_source_member_record(&other_source) + .expect("put other source"); + + let members = store + .list_current_rrdp_source_members(notify_uri) + .expect("list current members"); + assert_eq!( + members + .iter() + .map(|record| record.rsync_uri.as_str()) + .collect::>(), + vec![ + "rsync://example.test/repo/a.cer", + "rsync://example.test/repo/c.crl", + ] + ); + + assert!( + store + .is_current_rrdp_source_member(notify_uri, &present_a.rsync_uri) + .expect("current a") + ); + assert!( + !store + .is_current_rrdp_source_member(notify_uri, &withdrawn_b.rsync_uri) + .expect("withdrawn b") + ); + assert!( + !store + .is_current_rrdp_source_member(notify_uri, &other_source.rsync_uri) + .expect("other source") + ); +} + +#[test] +fn load_current_object_bytes_by_uri_uses_repository_view_and_raw_by_hash() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let present_bytes = b"present-object".to_vec(); + let present_hash = sha256_hex(&present_bytes); + let mut present_raw = RawByHashEntry::from_bytes(present_hash.clone(), present_bytes.clone()); + present_raw + .origin_uris + .push("rsync://example.test/repo/present.roa".to_string()); + present_raw.object_type = Some("roa".to_string()); + store + .put_raw_by_hash_entry(&present_raw) + .expect("put present raw"); + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/present.roa".to_string(), + current_hash: Some(present_hash), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }) + .expect("put present view"); + + let replaced_bytes = b"replaced-object".to_vec(); + let replaced_hash = sha256_hex(&replaced_bytes); + let mut replaced_raw = + RawByHashEntry::from_bytes(replaced_hash.clone(), replaced_bytes.clone()); + replaced_raw + .origin_uris + .push("rsync://example.test/repo/replaced.cer".to_string()); + replaced_raw.object_type = Some("cer".to_string()); + store + .put_raw_by_hash_entry(&replaced_raw) + .expect("put replaced raw"); + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/replaced.cer".to_string(), + current_hash: Some(replaced_hash), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("cer".to_string()), + state: RepositoryViewState::Replaced, + }) + .expect("put replaced view"); + + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/withdrawn.crl".to_string(), + current_hash: Some(sha256_hex(b"withdrawn")), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("crl".to_string()), + state: RepositoryViewState::Withdrawn, + }) + .expect("put withdrawn view"); + + assert_eq!( + store + .load_current_object_bytes_by_uri("rsync://example.test/repo/present.roa") + .expect("load present"), + Some(present_bytes) + ); + assert_eq!( + store + .load_current_object_bytes_by_uri("rsync://example.test/repo/replaced.cer") + .expect("load replaced"), + Some(replaced_bytes) + ); + assert_eq!( + store + .load_current_object_bytes_by_uri("rsync://example.test/repo/withdrawn.crl") + .expect("load withdrawn"), + None + ); + assert_eq!( + store + .load_current_object_bytes_by_uri("rsync://example.test/repo/missing.roa") + .expect("load missing"), + None + ); +} + +#[test] +fn load_current_object_bytes_by_uri_errors_when_raw_by_hash_is_missing() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let rsync_uri = "rsync://example.test/repo/missing.cer"; + + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: rsync_uri.to_string(), + current_hash: Some(hex::encode([0x11; 32])), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("cer".to_string()), + state: RepositoryViewState::Present, + }) + .expect("put view"); + let err = store + .load_current_object_bytes_by_uri(rsync_uri) + .expect_err("missing raw_by_hash should error"); + assert!(matches!(err, StorageError::InvalidData { .. })); +} + +#[test] +fn load_current_object_with_hash_by_uri_returns_hash_and_bytes() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let rsync_uri = "rsync://example.test/repo/present.roa"; + let bytes = b"present-object".to_vec(); + let hash = sha256_hex(&bytes); + + let mut raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone()); + raw.origin_uris.push(rsync_uri.to_string()); + raw.object_type = Some("roa".to_string()); + store.put_raw_by_hash_entry(&raw).expect("put raw"); + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: rsync_uri.to_string(), + current_hash: Some(hash.clone()), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }) + .expect("put view"); + + let got = store + .load_current_object_with_hash_by_uri(rsync_uri) + .expect("load current object") + .expect("current object exists"); + assert_eq!(got.current_hash_hex, hash); + assert_eq!(got.current_hash, compute_sha256_32(&bytes)); + assert_eq!(got.bytes, bytes); +} diff --git a/crates/panda-rpki-validator/src/storage/tests_parts/vcir.rs b/crates/panda-rpki-validator/src/storage/tests_parts/vcir.rs new file mode 100644 index 0000000..15386df --- /dev/null +++ b/crates/panda-rpki-validator/src/storage/tests_parts/vcir.rs @@ -0,0 +1,386 @@ +// Storage test group: vcir. + +#[test] +fn repository_view_and_raw_by_hash_validation_errors_are_reported() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let invalid_view = RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/withdrawn.cer".to_string(), + current_hash: None, + repository_source: None, + object_type: None, + state: RepositoryViewState::Present, + }; + let err = store + .put_repository_view_entry(&invalid_view) + .expect_err("missing current_hash must fail"); + assert!(err.to_string().contains("current_hash is required")); + + let invalid_raw = RawByHashEntry { + sha256_hex: sha256_hex(b"expected"), + bytes: b"actual".to_vec(), + origin_uris: vec!["rsync://example.test/repo/object.cer".to_string()], + object_type: None, + encoding: None, + }; + let err = store + .put_raw_by_hash_entry(&invalid_raw) + .expect_err("mismatched raw_by_hash entry must fail"); + assert!(err.to_string().contains("does not match bytes")); +} + +#[test] +fn vcir_roundtrip_and_validation_failures_are_reported() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let roa_context = roa_cache_projection_context(&vcir); + store + .put_vcir_with_projections(&vcir, Some(&roa_context), None) + .expect("put vcir"); + let got = store + .get_vcir(&vcir.manifest_rsync_uri) + .expect("get vcir") + .expect("vcir exists"); + assert_eq!(got, vcir); + let replay_meta = store + .get_manifest_replay_meta(&vcir.manifest_rsync_uri) + .expect("get manifest replay meta") + .expect("manifest replay meta exists"); + assert_eq!( + replay_meta, + ManifestReplayMeta { + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + manifest_number_be: vcir + .validated_manifest_meta + .validated_manifest_number + .clone(), + manifest_this_update: vcir + .validated_manifest_meta + .validated_manifest_this_update + .clone(), + manifest_sha256: vcir.ccr_manifest_projection.manifest_sha256.clone(), + updated_at_validation_time: vcir.last_successful_validation_time.clone(), + } + ); + let projection = store + .get_roa_cache_projection(&vcir.manifest_rsync_uri) + .expect("get roa cache projection") + .expect("roa cache projection exists"); + assert_eq!(projection.manifest_rsync_uri, vcir.manifest_rsync_uri); + assert_eq!(projection.entries.len(), 1); + assert_eq!( + projection.entries[0].source_object_uri, + "rsync://example.test/repo/object.roa" + ); + + let mut invalid = sample_vcir("rsync://example.test/repo/invalid.mft"); + invalid.summary.local_vrp_count = 9; + let err = store + .put_vcir(&invalid) + .expect_err("invalid vcir must fail"); + assert!(err.to_string().contains("local_vrp_count=9")); + + let mut invalid = sample_vcir("rsync://example.test/repo/invalid-2.mft"); + invalid.instance_gate.instance_effective_until = pack_time(11); + let err = store + .put_vcir(&invalid) + .expect_err("invalid instance gate must fail"); + assert!(err.to_string().contains("instance_effective_until")); + + store + .delete_vcir(&vcir.manifest_rsync_uri) + .expect("delete vcir"); + assert!( + store + .get_vcir(&vcir.manifest_rsync_uri) + .expect("get deleted vcir") + .is_none() + ); + assert!( + store + .get_manifest_replay_meta(&vcir.manifest_rsync_uri) + .expect("get deleted manifest replay meta") + .is_none() + ); + assert!( + store + .get_roa_cache_projection(&vcir.manifest_rsync_uri) + .expect("get deleted roa cache projection") + .is_none() + ); +} + +#[test] +fn clear_vcir_reuse_records_preserves_manifest_replay_meta() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + store.put_vcir(&vcir).expect("put vcir"); + let replay_meta = store + .get_manifest_replay_meta(&vcir.manifest_rsync_uri) + .expect("get replay meta") + .expect("replay meta exists"); + + let cleared = store + .clear_vcir_reuse_records() + .expect("clear reusable VCIR records"); + assert_eq!(cleared.vcir_records, 1); + assert_eq!(cleared.failed_fetch_identity_records, 0); + assert!( + store + .get_vcir(&vcir.manifest_rsync_uri) + .expect("get cleared vcir") + .is_none() + ); + assert_eq!( + store + .get_manifest_replay_meta(&vcir.manifest_rsync_uri) + .expect("get preserved replay meta"), + Some(replay_meta) + ); +} + +#[test] +fn transport_prefetch_snapshot_roundtrips() { + use crate::parallel::transport_prefetch::{ + TransportPrefetchDedupKey, TransportPrefetchMode, TransportPrefetchRepoIdentity, + TransportPrefetchRequest, TransportPrefetchRequester, TransportPrefetchSnapshot, + }; + use crate::policy::SyncPreference; + + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + assert!( + store + .get_transport_prefetch_snapshot() + .expect("get empty prefetch snapshot") + .is_none() + ); + + let snapshot = TransportPrefetchSnapshot::new( + SyncPreference::RrdpThenRsync, + vec![TransportPrefetchRequest { + dedup_key: TransportPrefetchDedupKey::RrdpNotify { + notification_uri: "https://example.test/notification.xml".to_string(), + }, + rsync_scope_uri: "rsync://example.test/repo/".to_string(), + rsync_failure_scope_uri: Some("rsync://example.test/".to_string()), + repo_identity: TransportPrefetchRepoIdentity { + notification_uri: Some("https://example.test/notification.xml".to_string()), + rsync_base_uri: "rsync://example.test/repo/".to_string(), + }, + mode: TransportPrefetchMode::Rrdp, + last_result: None, + last_rsync_result: None, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + priority: 0, + requesters: vec![TransportPrefetchRequester { + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + parent_node_id: None, + ca_instance_handle_id: "apnic:rsync://example.test/repo/root.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/root.mft".to_string(), + }], + }], + ); + store + .put_transport_prefetch_snapshot(&snapshot) + .expect("put prefetch snapshot"); + let got = store + .get_transport_prefetch_snapshot() + .expect("get prefetch snapshot") + .expect("snapshot exists"); + assert_eq!(got, snapshot); +} + +#[test] +fn manifest_replay_meta_validation_reports_invalid_fields() { + let mut meta = ManifestReplayMeta { + manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(), + manifest_number_be: vec![3], + manifest_this_update: pack_time(0), + manifest_sha256: vec![0x11; 32], + updated_at_validation_time: pack_time(1), + }; + meta.validate_internal().expect("valid replay meta"); + + meta.manifest_sha256 = vec![0x11; 31]; + let err = meta + .validate_internal() + .expect_err("short manifest sha must fail"); + assert!(err.to_string().contains("must be 32 bytes")); + + meta.manifest_sha256 = vec![0x11; 32]; + meta.manifest_number_be = vec![0, 3]; + let err = meta + .validate_internal() + .expect_err("non-minimal manifest number must fail"); + assert!(err.to_string().contains("minimal big-endian")); +} + +#[test] +fn list_vcirs_returns_all_entries() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let vcir1 = sample_vcir("rsync://example.test/repo/a.mft"); + let vcir2 = sample_vcir("rsync://example.test/repo/b.mft"); + store.put_vcir(&vcir1).expect("put vcir1"); + store.put_vcir(&vcir2).expect("put vcir2"); + + let mut got = store.list_vcirs().expect("list vcirs"); + got.sort_by(|a, b| a.manifest_rsync_uri.cmp(&b.manifest_rsync_uri)); + assert_eq!(got, vec![vcir1, vcir2]); +} + +#[test] +fn summarize_vcir_storage_aggregates_values_and_field_sizes() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let vcir1 = sample_vcir("rsync://example.test/repo/a.mft"); + let vcir2 = sample_vcir("rsync://example.test/repo/b.mft"); + store.put_vcir(&vcir1).expect("put vcir1"); + store.put_vcir(&vcir2).expect("put vcir2"); + + let summary = store.summarize_vcir_storage().expect("summarize vcirs"); + let mut expected_fields = VcirFieldSizeBreakdown::default(); + expected_fields.add_assign(&VcirFieldSizeBreakdown::from_vcir(&vcir1)); + expected_fields.add_assign(&VcirFieldSizeBreakdown::from_vcir(&vcir2)); + + assert_eq!(summary.entry_count, 2); + assert!(summary.vcir_value_bytes > 0); + assert!(summary.vcir_value_bytes_max > 0); + assert!(summary.vcir_value_bytes_max_manifest_rsync_uri.is_some()); + assert_eq!(summary.top_entries_by_vcir_value_bytes.len(), 2); + assert!( + summary.top_entries_by_vcir_value_bytes[0].vcir_value_bytes + >= summary.top_entries_by_vcir_value_bytes[1].vcir_value_bytes + ); + assert_eq!(summary.field_sizes, expected_fields); + assert!(summary.core_fields.manifest_rsync_uri_bytes > 0); + assert!(summary.ccr_projection.manifest_sha256_bytes > 0); + assert!(summary.child_resources.effective_ip_resource_cbor_bytes > 0); + assert_eq!( + summary.local_output_old_projection_bytes, + expected_fields.local_output_old_projection_bytes() + ); + assert_eq!( + summary.local_output_typed_projection_bytes, + expected_fields.local_output_typed_projection_bytes() + ); + assert_eq!( + summary.local_output_projection_saved_bytes, + expected_fields.local_output_projection_saved_bytes() + ); +} + +#[test] +fn replace_vcir_and_manifest_replay_meta_replaces_current_entry() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let mut previous = sample_vcir("rsync://example.test/repo/current.mft"); + previous.local_outputs = vec![VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: pack_time(10), + source_object_uri: "rsync://example.test/repo/old.roa".to_string(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: sha256_32(b"old-roa"), + source_ee_cert_hash: sha256_32(b"old-ee"), + payload: VcirLocalOutputPayload::Vrp { + asn: 64496, + afi: crate::data_model::roa::RoaAfi::Ipv4, + prefix_len: 24, + addr: { + let mut addr = [0u8; 16]; + addr[..4].copy_from_slice(&[203, 0, 113, 0]); + addr + }, + max_length: 24, + }, + rule_hash: sha256_32(b"old-rule"), + }]; + previous.summary.local_vrp_count = 1; + previous.summary.local_aspa_count = 0; + previous.summary.local_router_key_count = 0; + let previous_roa_context = roa_cache_projection_context(&previous); + let previous_timing = store + .replace_vcir_manifest_replay_meta_and_projections( + &previous, + Some(&previous_roa_context), + None, + ) + .expect("store previous vcir"); + assert!(previous_timing.vcir_value_bytes > 0); + assert!(previous_timing.replay_meta_value_bytes > 0); + assert!(previous_timing.roa_cache_projection_value_bytes > 0); + assert_eq!( + previous_timing.total_encoded_bytes, + previous_timing.vcir_value_bytes + + previous_timing.replay_meta_value_bytes + + previous_timing.roa_cache_projection_value_bytes + ); + assert!( + store + .get_roa_cache_projection(&previous.manifest_rsync_uri) + .expect("get previous projection") + .is_some() + ); + + let mut current = sample_vcir("rsync://example.test/repo/current.mft"); + current.local_outputs = vec![VcirLocalOutput { + output_type: VcirOutputType::Aspa, + item_effective_until: pack_time(11), + source_object_uri: "rsync://example.test/repo/new.asa".to_string(), + source_object_type: VcirSourceObjectType::Aspa, + source_object_hash: sha256_32(b"new-aspa"), + source_ee_cert_hash: sha256_32(b"new-ee"), + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: 64496, + provider_as_ids: vec![64497], + }, + rule_hash: sha256_32(b"new-rule"), + }]; + current.summary.local_vrp_count = 0; + current.summary.local_aspa_count = 1; + let current_timing = store + .replace_vcir_and_manifest_replay_meta(¤t) + .expect("replace vcir and replay meta"); + assert!(current_timing.vcir_value_bytes > 0); + assert!(current_timing.replay_meta_value_bytes > 0); + assert_eq!(current_timing.roa_cache_projection_value_bytes, 0); + assert_eq!( + current_timing.total_encoded_bytes, + current_timing.vcir_value_bytes + current_timing.replay_meta_value_bytes + ); + + let got = store + .get_vcir(¤t.manifest_rsync_uri) + .expect("get replaced vcir") + .expect("vcir exists"); + assert_eq!(got, current); + let replay_meta = store + .get_manifest_replay_meta(¤t.manifest_rsync_uri) + .expect("get replaced replay meta") + .expect("replay meta exists"); + assert_eq!( + replay_meta.manifest_number_be, + current.validated_manifest_meta.validated_manifest_number + ); + assert_eq!( + replay_meta.manifest_sha256, + current.ccr_manifest_projection.manifest_sha256 + ); + assert!( + store + .get_roa_cache_projection(¤t.manifest_rsync_uri) + .expect("get current projection") + .is_none() + ); +} diff --git a/crates/panda-rpki-validator/src/storage/verification.rs b/crates/panda-rpki-validator/src/storage/verification.rs new file mode 100644 index 0000000..c897d45 --- /dev/null +++ b/crates/panda-rpki-validator/src/storage/verification.rs @@ -0,0 +1,32 @@ +// Repository blob integrity verification helper. + +fn verify_repository_blob_batch( + store: &RocksStore, + batch: &[(String, String)], + summary: &mut RepositoryBlobVerificationSummary, +) -> StorageResult<()> { + let hashes = batch + .iter() + .map(|(_, hash)| hash.clone()) + .collect::>(); + let blobs = store.get_blob_bytes_batch(&hashes)?; + for ((uri, expected_hash), blob) in batch.iter().zip(blobs.into_iter()) { + let bytes = blob.as_ref().ok_or(StorageError::InvalidData { + entity: "repository_blob_verification", + detail: format!("blob missing for URI {uri} (hash={expected_hash})"), + })?; + let actual_hash = hex::encode(compute_sha256_32(bytes)); + if !actual_hash.eq_ignore_ascii_case(expected_hash) { + return Err(StorageError::InvalidData { + entity: "repository_blob_verification", + detail: format!( + "blob hash mismatch for URI {uri}: expected={expected_hash}, actual={actual_hash}" + ), + }); + } + summary.current_objects += 1; + summary.bytes_verified += bytes.len() as u64; + } + summary.batches += 1; + Ok(()) +} diff --git a/crates/panda-rpki-validator/src/sync/repo/tests.rs b/crates/panda-rpki-validator/src/sync/repo/tests.rs index d35cc6f..f85599a 100644 --- a/crates/panda-rpki-validator/src/sync/repo/tests.rs +++ b/crates/panda-rpki-validator/src/sync/repo/tests.rs @@ -1,1332 +1,4 @@ -use super::*; -use crate::analysis::timing::{TimingHandle, TimingMeta}; -use crate::fetch::rsync::LocalDirRsyncFetcher; -use crate::replay::archive::{ReplayArchiveIndex, sha256_hex}; -use crate::replay::delta_archive::ReplayDeltaArchiveIndex; -use crate::replay::delta_fetch_http::PayloadDeltaReplayHttpFetcher; -use crate::replay::delta_fetch_rsync::PayloadDeltaReplayRsyncFetcher; -use crate::replay::fetch_http::PayloadReplayHttpFetcher; -use crate::replay::fetch_rsync::PayloadReplayRsyncFetcher; -use crate::storage::RepositoryViewState; -use crate::sync::rrdp::Fetcher as HttpFetcher; -use crate::sync::rrdp::RrdpState; -use crate::sync::store_projection::{build_repository_view_present_entry, compute_sha256_hex}; -use base64::Engine; -use sha2::Digest; -use std::collections::HashMap; -use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; - -struct DummyHttpFetcher; - -impl HttpFetcher for DummyHttpFetcher { - fn fetch(&self, _url: &str) -> Result, String> { - panic!("http fetcher must not be used in rsync-only mode") - } -} - -struct PanicRsyncFetcher; -impl RsyncFetcher for PanicRsyncFetcher { - fn fetch_objects( - &self, - _rsync_base_uri: &str, - ) -> Result)>, RsyncFetchError> { - panic!("rsync must not be used in this test") - } -} - -struct MapFetcher { - map: HashMap>, -} - -impl HttpFetcher for MapFetcher { - fn fetch(&self, uri: &str) -> Result, String> { - self.map - .get(uri) - .cloned() - .ok_or_else(|| format!("not found: {uri}")) - } -} - -fn assert_current_object(store: &RocksStore, uri: &str, expected: &[u8]) { - assert_eq!( - store - .load_current_object_bytes_by_uri(uri) - .expect("load current object"), - Some(expected.to_vec()) - ); -} - -#[test] -fn rsync_sync_uses_fetcher_dedup_scope_for_repository_view_projection() { - struct ScopeFetcher; - impl RsyncFetcher for ScopeFetcher { - fn fetch_objects( - &self, - _rsync_base_uri: &str, - ) -> Result)>, RsyncFetchError> { - Ok(vec![( - "rsync://example.net/repo/child/a.mft".to_string(), - b"manifest".to_vec(), - )]) - } - - fn dedup_key(&self, _rsync_base_uri: &str) -> String { - "rsync://example.net/repo/".to_string() - } - } - - let td = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(td.path()).expect("open rocksdb"); - let seeded = build_repository_view_present_entry( - "rsync://example.net/repo/", - "rsync://example.net/repo/sibling/old.roa", - &compute_sha256_hex(b"old"), - ); - store - .put_projection_batch(&[seeded], &[], &[]) - .expect("seed repository view"); - - let fetcher = ScopeFetcher; - let written = rsync_sync_into_current_store( - &store, - "rsync://example.net/repo/child/", - None, - &fetcher, - None, - None, - ) - .expect("sync ok"); - assert_eq!(written, 1); - - let entries = store - .list_repository_view_entries_with_prefix("rsync://example.net/repo/") - .expect("list repository view"); - let sibling = entries - .iter() - .find(|entry| entry.rsync_uri == "rsync://example.net/repo/sibling/old.roa") - .expect("sibling entry exists"); - assert_eq!(sibling.state, RepositoryViewState::Withdrawn); - let child = entries - .iter() - .find(|entry| entry.rsync_uri == "rsync://example.net/repo/child/a.mft") - .expect("child entry exists"); - assert_eq!(child.state, RepositoryViewState::Present); -} - -fn notification_xml( - session_id: &str, - serial: u64, - snapshot_uri: &str, - snapshot_hash: &str, -) -> Vec { - format!( - r#""# - ) - .into_bytes() -} - -fn snapshot_xml(session_id: &str, serial: u64, published: &[(&str, &[u8])]) -> Vec { - let mut out = format!( - r#""# - ); - for (uri, bytes) in published { - let b64 = base64::engine::general_purpose::STANDARD.encode(bytes); - out.push_str(&format!(r#"{b64}"#)); - } - out.push_str(""); - out.into_bytes() -} - -fn build_replay_archive_fixture() -> ( - tempfile::TempDir, - std::path::PathBuf, - std::path::PathBuf, - String, - String, - String, - String, -) { - let temp = tempfile::tempdir().expect("tempdir"); - let archive_root = temp.path().join("payload-archive"); - let capture = "repo-replay"; - let capture_root = archive_root.join("v1").join("captures").join(capture); - std::fs::create_dir_all(&capture_root).expect("mkdir capture root"); - std::fs::write( - capture_root.join("capture.json"), - format!( - r#"{{"version":1,"captureId":"{capture}","createdAt":"2026-03-13T00:00:00Z","notes":""}}"# - ), - ) - .expect("write capture json"); - - let notify_uri = "https://rrdp.example.test/notification.xml".to_string(); - let snapshot_uri = "https://rrdp.example.test/snapshot.xml".to_string(); - let session = "00000000-0000-0000-0000-000000000001".to_string(); - let serial = 7u64; - let published_uri = "rsync://example.test/repo/a.mft".to_string(); - let published_bytes = b"mft"; - let snapshot = snapshot_xml(&session, serial, &[(&published_uri, published_bytes)]); - let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); - let notification = notification_xml(&session, serial, &snapshot_uri, &snapshot_hash); - let repo_hash = sha256_hex(notify_uri.as_bytes()); - let session_dir = capture_root - .join("rrdp/repos") - .join(&repo_hash) - .join(&session); - std::fs::create_dir_all(&session_dir).expect("mkdir session dir"); - std::fs::write( - session_dir.parent().unwrap().join("meta.json"), - format!( - r#"{{"version":1,"rpkiNotify":"{notify_uri}","createdAt":"2026-03-13T00:00:00Z","lastSeenAt":"2026-03-13T00:00:01Z"}}"# - ), - ) - .expect("write repo meta"); - std::fs::write(session_dir.join("notification-7.xml"), notification) - .expect("write notification"); - std::fs::write( - session_dir.join(format!("snapshot-7-{snapshot_hash}.xml")), - &snapshot, - ) - .expect("write snapshot"); - - let rsync_base_uri = "rsync://rsync.example.test/repo/".to_string(); - let rsync_locked_notify = "https://rrdp-fallback.example.test/notification.xml".to_string(); - let mod_hash = sha256_hex(rsync_base_uri.as_bytes()); - let module_bucket_dir = capture_root.join("rsync/modules").join(&mod_hash); - let module_root = module_bucket_dir - .join("tree") - .join("rsync.example.test") - .join("repo"); - std::fs::create_dir_all(module_root.join("sub")).expect("mkdir module tree"); - std::fs::write( - module_bucket_dir.join("meta.json"), - format!( - r#"{{"version":1,"module":"{rsync_base_uri}","createdAt":"2026-03-13T00:00:00Z","lastSeenAt":"2026-03-13T00:00:01Z"}}"# - ), - ) - .expect("write rsync meta"); - std::fs::write(module_root.join("sub").join("fallback.cer"), b"cer") - .expect("write rsync object"); - - let locks_path = temp.path().join("locks.json"); - std::fs::write( - &locks_path, - format!( - r#"{{ - "version":1, - "capture":"{capture}", - "rrdp":{{ - "{notify_uri}":{{"transport":"rrdp","session":"{session}","serial":{serial}}}, - "{rsync_locked_notify}":{{"transport":"rsync","session":null,"serial":null}} - }}, - "rsync":{{ - "{rsync_base_uri}":{{"transport":"rsync"}} - }} -}}"# - ), - ) - .expect("write locks"); - - ( - temp, - archive_root, - locks_path, - notify_uri, - rsync_locked_notify, - rsync_base_uri, - published_uri, - ) -} - -fn build_delta_replay_fixture() -> ( - tempfile::TempDir, - std::path::PathBuf, - std::path::PathBuf, - std::path::PathBuf, - std::path::PathBuf, - String, - String, - String, -) { - let temp = tempfile::tempdir().expect("tempdir"); - - let base_archive = temp.path().join("payload-archive"); - let base_capture_root = base_archive.join("v1/captures/base-cap"); - std::fs::create_dir_all(&base_capture_root).expect("mkdir base capture"); - std::fs::write( - base_capture_root.join("capture.json"), - r#"{"version":1,"captureId":"base-cap","createdAt":"2026-03-16T00:00:00Z","notes":""}"#, - ) - .expect("write base capture meta"); - - let notify_uri = "https://rrdp.example.test/notification.xml".to_string(); - let snapshot_uri = "https://rrdp.example.test/snapshot.xml".to_string(); - let session = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".to_string(); - let base_serial = 10u64; - let delta1_uri = "https://rrdp.example.test/d1.xml".to_string(); - let delta2_uri = "https://rrdp.example.test/d2.xml".to_string(); - let repo_hash = sha256_hex(notify_uri.as_bytes()); - let base_session_dir = base_capture_root - .join("rrdp/repos") - .join(&repo_hash) - .join(&session); - std::fs::create_dir_all(&base_session_dir).expect("mkdir base session dir"); - std::fs::write( - base_session_dir.parent().unwrap().join("meta.json"), - format!(r#"{{"version":1,"rpkiNotify":"{notify_uri}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), - ) - .expect("write base rrdp meta"); - let base_snapshot = snapshot_xml( - &session, - base_serial, - &[("rsync://example.test/repo/a.mft", b"base")], - ); - let base_snapshot_hash = hex::encode(sha2::Sha256::digest(&base_snapshot)); - let base_notification = - notification_xml(&session, base_serial, &snapshot_uri, &base_snapshot_hash); - std::fs::write( - base_session_dir.join("notification-10.xml"), - base_notification, - ) - .expect("write base notif"); - std::fs::write( - base_session_dir.join(format!("snapshot-10-{base_snapshot_hash}.xml")), - base_snapshot, - ) - .expect("write base snapshot"); - - let module_uri = "rsync://rsync.example.test/repo/".to_string(); - let module_hash = sha256_hex(module_uri.as_bytes()); - let base_module_bucket = base_capture_root.join("rsync/modules").join(&module_hash); - let base_module_tree = base_module_bucket.join("tree/rsync.example.test/repo"); - std::fs::create_dir_all(base_module_tree.join("sub")).expect("mkdir base rsync tree"); - std::fs::write( - base_module_bucket.join("meta.json"), - format!(r#"{{"version":1,"module":"{module_uri}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), - ) - .expect("write base module meta"); - std::fs::write(base_module_tree.join("a.mft"), b"base").expect("write base a.mft"); - std::fs::write(base_module_tree.join("sub").join("x.cer"), b"base-cer") - .expect("write base x.cer"); - - let base_locks = temp.path().join("base-locks.json"); - let fallback_notify = "https://rrdp-fallback.example.test/notification.xml".to_string(); - let base_locks_body = format!( - r#"{{"version":1,"capture":"base-cap","rrdp":{{"{notify_uri}":{{"transport":"rrdp","session":"{session}","serial":10}},"{fallback_notify}":{{"transport":"rsync","session":null,"serial":null}}}},"rsync":{{"{module_uri}":{{"transport":"rsync"}}}}}}"# - ); - std::fs::write(&base_locks, &base_locks_body).expect("write base locks"); - let base_locks_sha = sha256_hex(base_locks_body.as_bytes()); - - let delta_archive = temp.path().join("payload-delta-archive"); - let delta_capture_root = delta_archive.join("v1/captures/delta-cap"); - std::fs::create_dir_all(&delta_capture_root).expect("mkdir delta capture"); - std::fs::write( - delta_capture_root.join("capture.json"), - r#"{"version":1,"captureId":"delta-cap","createdAt":"2026-03-16T00:00:00Z","notes":""}"#, - ) - .expect("write delta capture meta"); - std::fs::write( - delta_capture_root.join("base.json"), - format!(r#"{{"version":1,"baseCapture":"base-cap","baseLocksSha256":"{base_locks_sha}","createdAt":"2026-03-16T00:00:00Z"}}"#), - ) - .expect("write delta base meta"); - - let delta_session_dir = delta_capture_root - .join("rrdp/repos") - .join(&repo_hash) - .join(&session); - let delta_deltas_dir = delta_session_dir.join("deltas"); - std::fs::create_dir_all(&delta_deltas_dir).expect("mkdir delta deltas"); - std::fs::write( - delta_session_dir.parent().unwrap().join("meta.json"), - format!(r#"{{"version":1,"rpkiNotify":"{notify_uri}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), - ) - .expect("write delta meta"); - std::fs::write( - delta_session_dir.parent().unwrap().join("transition.json"), - format!(r#"{{"kind":"delta","base":{{"transport":"rrdp","session":"{session}","serial":10}},"target":{{"transport":"rrdp","session":"{session}","serial":12}},"delta_count":2,"deltas":[11,12]}}"#), - ) - .expect("write delta transition"); - let delta1 = format!( - r#"{}"#, - base64::engine::general_purpose::STANDARD.encode(b"delta-a") - ); - let delta2 = format!( - r#"{}"#, - base64::engine::general_purpose::STANDARD.encode(b"delta-b") - ); - let delta1_hash = hex::encode(sha2::Sha256::digest(delta1.as_bytes())); - let delta2_hash = hex::encode(sha2::Sha256::digest(delta2.as_bytes())); - let target_notification = format!( - r#" - - - - -"# - ); - std::fs::write( - delta_session_dir.join("notification-target-12.xml"), - target_notification, - ) - .expect("write target notification"); - std::fs::write(delta_deltas_dir.join("delta-11-aaaa.xml"), delta1).expect("write delta11"); - std::fs::write(delta_deltas_dir.join("delta-12-bbbb.xml"), delta2).expect("write delta12"); - - let delta_module_bucket = delta_capture_root.join("rsync/modules").join(&module_hash); - let delta_module_tree = delta_module_bucket.join("tree/rsync.example.test/repo"); - std::fs::create_dir_all(delta_module_tree.join("sub")).expect("mkdir delta rsync tree"); - std::fs::write( - delta_module_bucket.join("meta.json"), - format!(r#"{{"version":1,"module":"{module_uri}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), - ) - .expect("write delta rsync meta"); - std::fs::write( - delta_module_bucket.join("files.json"), - format!(r#"{{"version":1,"module":"{module_uri}","fileCount":1,"files":["{module_uri}sub/x.cer"]}}"#), - ) - .expect("write delta files"); - std::fs::write(delta_module_tree.join("sub").join("x.cer"), b"overlay-cer") - .expect("write overlay file"); - - let fallback_hash = sha256_hex(fallback_notify.as_bytes()); - let fallback_repo_dir = delta_capture_root.join("rrdp/repos").join(&fallback_hash); - std::fs::create_dir_all(&fallback_repo_dir).expect("mkdir fallback repo dir"); - std::fs::write( - fallback_repo_dir.join("meta.json"), - format!(r#"{{"version":1,"rpkiNotify":"{fallback_notify}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), - ) - .expect("write fallback meta"); - std::fs::write( - fallback_repo_dir.join("transition.json"), - r#"{"kind":"fallback-rsync","base":{"transport":"rsync","session":null,"serial":null},"target":{"transport":"rsync","session":null,"serial":null},"delta_count":0,"deltas":[]}"#, - ) - .expect("write fallback transition"); - - let delta_locks = temp.path().join("locks-delta.json"); - std::fs::write( - &delta_locks, - format!(r#"{{"version":1,"capture":"delta-cap","baseCapture":"base-cap","baseLocksSha256":"{base_locks_sha}","rrdp":{{"{notify_uri}":{{"kind":"delta","base":{{"transport":"rrdp","session":"{session}","serial":10}},"target":{{"transport":"rrdp","session":"{session}","serial":12}},"delta_count":2,"deltas":[11,12]}},"{fallback_notify}":{{"kind":"fallback-rsync","base":{{"transport":"rsync","session":null,"serial":null}},"target":{{"transport":"rsync","session":null,"serial":null}},"delta_count":0,"deltas":[]}}}},"rsync":{{"{module_uri}":{{"file_count":1,"overlay_only":false}}}}}}"#), - ) - .expect("write delta locks"); - - ( - temp, - base_archive, - base_locks, - delta_archive, - delta_locks, - notify_uri, - fallback_notify, - module_uri, - ) -} - -fn timing_to_json(temp_dir: &std::path::Path, timing: &TimingHandle) -> serde_json::Value { - let timing_path = temp_dir.join("timing_retry.json"); - timing.write_json(&timing_path, 50).expect("write json"); - serde_json::from_slice(&std::fs::read(&timing_path).expect("read json")).expect("parse json") -} - -#[test] -fn rsync_sync_writes_current_store_and_records_counts() { - let temp = tempfile::tempdir().expect("tempdir"); - - let repo_dir = temp.path().join("repo"); - std::fs::create_dir_all(repo_dir.join("sub")).expect("mkdir"); - std::fs::write(repo_dir.join("a.mft"), b"mft").expect("write"); - std::fs::write(repo_dir.join("sub").join("b.roa"), b"roa").expect("write"); - std::fs::write(repo_dir.join("sub").join("c.cer"), b"cer").expect("write"); - - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - - let timing = TimingHandle::new(TimingMeta { - recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), - validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), - tal_url: None, - db_path: Some(store_dir.to_string_lossy().into_owned()), - }); - - let policy = Policy { - sync_preference: SyncPreference::RsyncOnly, - ..Policy::default() - }; - let http = DummyHttpFetcher; - let rsync = LocalDirRsyncFetcher::new(&repo_dir); - - let download_log = DownloadLogHandle::new(); - let out = sync_publication_point( - &store, - &policy, - None, - "rsync://example.test/repo/", - &http, - &rsync, - Some(&timing), - Some(&download_log), - ) - .expect("sync ok"); - - assert_eq!(out.source, RepoSyncSource::Rsync); - assert_eq!(out.objects_written, 3); - - let events = download_log.snapshot_events(); - assert_eq!(events.len(), 1); - assert_eq!(events[0].kind, AuditDownloadKind::Rsync); - assert!(events[0].success); - assert_eq!(events[0].bytes, Some(9)); - let objects = events[0].objects.as_ref().expect("objects stat"); - assert_eq!(objects.objects_count, 3); - assert_eq!(objects.objects_bytes_total, 9); - - assert_current_object(&store, "rsync://example.test/repo/a.mft", b"mft"); - assert_current_object(&store, "rsync://example.test/repo/sub/b.roa", b"roa"); - assert_current_object(&store, "rsync://example.test/repo/sub/c.cer", b"cer"); - - let view = store - .get_repository_view_entry("rsync://example.test/repo/a.mft") - .expect("get repository view") - .expect("repository view entry present"); - assert_eq!( - view.current_hash.as_deref(), - Some(hex::encode(sha2::Sha256::digest(b"mft")).as_str()) - ); - assert_eq!( - view.repository_source.as_deref(), - Some("rsync://example.test/repo/") - ); - - let current_bytes = store - .load_current_object_bytes_by_uri("rsync://example.test/repo/sub/b.roa") - .expect("load current bytes") - .expect("current object bytes exist"); - assert_eq!(current_bytes, b"roa".to_vec()); - assert!( - store - .get_raw_by_hash_entry(hex::encode(sha2::Sha256::digest(b"roa")).as_str()) - .expect("get raw_by_hash") - .is_none() - ); - - let timing_path = temp.path().join("timing.json"); - timing.write_json(&timing_path, 5).expect("write json"); - let v: serde_json::Value = - serde_json::from_slice(&std::fs::read(&timing_path).expect("read json")) - .expect("parse json"); - let counts = v.get("counts").expect("counts"); - assert_eq!( - counts - .get("rsync_objects_fetched_total") - .and_then(|v| v.as_u64()), - Some(3) - ); - assert_eq!( - counts - .get("rsync_objects_bytes_total") - .and_then(|v| v.as_u64()), - Some(3 * 3) - ); -} - -#[test] -fn rsync_second_sync_marks_missing_repository_view_entries_withdrawn() { - let temp = tempfile::tempdir().expect("tempdir"); - - let repo_dir = temp.path().join("repo"); - std::fs::create_dir_all(repo_dir.join("sub")).expect("mkdir"); - std::fs::write(repo_dir.join("a.mft"), b"mft-v1").expect("write a"); - std::fs::write(repo_dir.join("sub").join("b.roa"), b"roa-v1").expect("write b"); - - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - let policy = Policy { - sync_preference: SyncPreference::RsyncOnly, - ..Policy::default() - }; - let http = DummyHttpFetcher; - let rsync = LocalDirRsyncFetcher::new(&repo_dir); - - sync_publication_point( - &store, - &policy, - None, - "rsync://example.test/repo/", - &http, - &rsync, - None, - None, - ) - .expect("first sync ok"); - - std::fs::remove_file(repo_dir.join("sub").join("b.roa")).expect("remove b"); - std::fs::write(repo_dir.join("c.crl"), b"crl-v2").expect("write c"); - - sync_publication_point( - &store, - &policy, - None, - "rsync://example.test/repo/", - &http, - &rsync, - None, - None, - ) - .expect("second sync ok"); - - let withdrawn = store - .get_repository_view_entry("rsync://example.test/repo/sub/b.roa") - .expect("get withdrawn repo view") - .expect("withdrawn entry exists"); - assert_eq!( - withdrawn.state, - crate::storage::RepositoryViewState::Withdrawn - ); - assert_eq!( - withdrawn.repository_source.as_deref(), - Some("rsync://example.test/repo/") - ); - - let added = store - .get_repository_view_entry("rsync://example.test/repo/c.crl") - .expect("get added repo view") - .expect("added entry exists"); - assert_eq!(added.state, crate::storage::RepositoryViewState::Present); -} - -#[test] -fn rrdp_fetch_error_falls_back_to_rsync_without_retry() { - let temp = tempfile::tempdir().expect("tempdir"); - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - - let timing = TimingHandle::new(TimingMeta { - recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), - validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), - tal_url: None, - db_path: Some(store_dir.to_string_lossy().into_owned()), - }); - - let notification_uri = "https://example.test/notification.xml"; - let published_uri = "rsync://example.test/repo/a.mft"; - let published_bytes = b"x"; - struct AlwaysFailHttp { - notification_calls: AtomicUsize, - } - - impl HttpFetcher for AlwaysFailHttp { - fn fetch(&self, _uri: &str) -> Result, String> { - self.notification_calls.fetch_add(1, Ordering::SeqCst); - Err("http request failed: simulated transient".to_string()) - } - } - - struct SingleObjectRsync { - uri: String, - bytes: Vec, - } - impl RsyncFetcher for SingleObjectRsync { - fn fetch_objects( - &self, - _rsync_base_uri: &str, - ) -> Result)>, RsyncFetchError> { - Ok(vec![(self.uri.clone(), self.bytes.clone())]) - } - } - - let http = AlwaysFailHttp { - notification_calls: AtomicUsize::new(0), - }; - - let policy = Policy { - sync_preference: SyncPreference::RrdpThenRsync, - ..Policy::default() - }; - - let download_log = DownloadLogHandle::new(); - let out = sync_publication_point( - &store, - &policy, - Some(notification_uri), - "rsync://example.test/repo/", - &http, - &SingleObjectRsync { - uri: published_uri.to_string(), - bytes: published_bytes.to_vec(), - }, - Some(&timing), - Some(&download_log), - ) - .expect("sync ok"); - - assert_eq!(out.source, RepoSyncSource::Rsync); - assert_current_object(&store, published_uri, published_bytes); - assert_eq!(http.notification_calls.load(Ordering::SeqCst), 1); - - let events = download_log.snapshot_events(); - assert_eq!(events.len(), 2, "expected 1x notification + 1x rsync"); - assert_eq!( - events - .iter() - .filter(|e| e.kind == AuditDownloadKind::RrdpNotification) - .count(), - 1 - ); - assert_eq!( - events - .iter() - .filter(|e| e.kind == AuditDownloadKind::RrdpNotification && !e.success) - .count(), - 1 - ); - assert_eq!( - events - .iter() - .filter(|e| e.kind == AuditDownloadKind::Rsync) - .count(), - 1 - ); - - let v = timing_to_json(temp.path(), &timing); - let counts = v.get("counts").expect("counts"); - assert_eq!( - counts - .get("rrdp_retry_attempt_total") - .and_then(|v| v.as_u64()), - Some(1) - ); - assert_eq!( - counts - .get("repo_sync_rrdp_failed_total") - .and_then(|v| v.as_u64()), - Some(1) - ); - assert_eq!( - counts - .get("repo_sync_rsync_fallback_ok_total") - .and_then(|v| v.as_u64()), - Some(1) - ); -} - -#[test] -fn rrdp_protocol_error_does_not_retry_and_falls_back_to_rsync() { - let temp = tempfile::tempdir().expect("tempdir"); - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - - let timing = TimingHandle::new(TimingMeta { - recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), - validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), - tal_url: None, - db_path: Some(store_dir.to_string_lossy().into_owned()), - }); - - let notification_uri = "https://example.test/notification.xml"; - let snapshot_uri = "https://example.test/snapshot.xml"; - let published_uri = "rsync://example.test/repo/a.mft"; - let published_bytes = b"x"; - - let snapshot = snapshot_xml( - "9df4b597-af9e-4dca-bdda-719cce2c4e28", - 1, - &[(published_uri, published_bytes)], - ); - // Intentionally wrong hash to trigger protocol error (SnapshotHashMismatch). - let wrong_hash = "00".repeat(32); - let notif = notification_xml( - "9df4b597-af9e-4dca-bdda-719cce2c4e28", - 1, - snapshot_uri, - &wrong_hash, - ); - - let mut map = HashMap::new(); - map.insert(notification_uri.to_string(), notif); - map.insert(snapshot_uri.to_string(), snapshot); - let http = MapFetcher { map }; - - struct EmptyRsyncFetcher; - impl RsyncFetcher for EmptyRsyncFetcher { - fn fetch_objects( - &self, - _rsync_base_uri: &str, - ) -> Result)>, RsyncFetchError> { - Ok(Vec::new()) - } - } - - let policy = Policy { - sync_preference: SyncPreference::RrdpThenRsync, - ..Policy::default() - }; - - let download_log = DownloadLogHandle::new(); - let out = sync_publication_point( - &store, - &policy, - Some(notification_uri), - "rsync://example.test/repo/", - &http, - &EmptyRsyncFetcher, - Some(&timing), - Some(&download_log), - ) - .expect("sync ok"); - - assert_eq!(out.source, RepoSyncSource::Rsync); - assert!( - out.warnings - .iter() - .any(|w| w.message.contains("RRDP failed; falling back to rsync")), - "expected RRDP fallback warning" - ); - - let events = download_log.snapshot_events(); - assert_eq!( - events.len(), - 3, - "expected notification + snapshot + rsync fallback" - ); - assert_eq!(events[0].kind, AuditDownloadKind::RrdpNotification); - assert!(events[0].success); - assert_eq!(events[1].kind, AuditDownloadKind::RrdpSnapshot); - assert!(!events[1].success); - assert_eq!(events[2].kind, AuditDownloadKind::Rsync); - assert!(events[2].success); - - let v = timing_to_json(temp.path(), &timing); - let counts = v.get("counts").expect("counts"); - assert_eq!( - counts - .get("rrdp_retry_attempt_total") - .and_then(|v| v.as_u64()), - Some(1) - ); - assert_eq!( - counts - .get("rrdp_failed_protocol_total") - .and_then(|v| v.as_u64()), - Some(1) - ); - assert_eq!( - counts - .get("repo_sync_rrdp_failed_total") - .and_then(|v| v.as_u64()), - Some(1) - ); - assert_eq!( - counts - .get("repo_sync_rsync_fallback_ok_total") - .and_then(|v| v.as_u64()), - Some(1) - ); -} - -#[test] -fn rrdp_delta_fetches_are_logged_even_if_snapshot_fallback_is_used() { - let temp = tempfile::tempdir().expect("tempdir"); - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - - let timing = TimingHandle::new(TimingMeta { - recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), - validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), - tal_url: None, - db_path: Some(store_dir.to_string_lossy().into_owned()), - }); - - let notification_uri = "https://example.test/notification.xml"; - let snapshot_uri = "https://example.test/snapshot.xml"; - let delta_2_uri = "https://example.test/delta_2.xml"; - let delta_3_uri = "https://example.test/delta_3.xml"; - let published_uri = "rsync://example.test/repo/a.mft"; - let published_bytes = b"x"; - - let sid = "9df4b597-af9e-4dca-bdda-719cce2c4e28"; - - // Seed old RRDP state so sync_from_notification tries deltas (RFC 8182 §3.4.1). - let state = RrdpState { - session_id: sid.to_string(), - serial: 1, - }; - persist_rrdp_local_state( - &store, - notification_uri, - &state, - RrdpSourceSyncState::DeltaReady, - Some(snapshot_uri), - None, - ) - .expect("seed state"); - - let delta_2 = format!( - r#""# - ) - .into_bytes(); - let delta_3 = format!( - r#""# - ) - .into_bytes(); - let delta_2_hash = hex::encode(sha2::Sha256::digest(&delta_2)); - let delta_3_hash = hex::encode(sha2::Sha256::digest(&delta_3)); - - let snapshot = snapshot_xml(sid, 3, &[(published_uri, published_bytes)]); - let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); - let notif = format!( - r#""# - ) - .into_bytes(); - - let mut map = HashMap::new(); - map.insert(notification_uri.to_string(), notif); - map.insert(snapshot_uri.to_string(), snapshot); - map.insert(delta_2_uri.to_string(), delta_2); - map.insert(delta_3_uri.to_string(), delta_3); - let http = MapFetcher { map }; - - let policy = Policy { - sync_preference: SyncPreference::RrdpThenRsync, - ..Policy::default() - }; - - let download_log = DownloadLogHandle::new(); - let out = sync_publication_point( - &store, - &policy, - Some(notification_uri), - "rsync://example.test/repo/", - &http, - &PanicRsyncFetcher, - Some(&timing), - Some(&download_log), - ) - .expect("sync ok"); - - assert_eq!(out.source, RepoSyncSource::Rrdp); - assert_eq!(out.objects_written, 1); - assert_current_object(&store, published_uri, published_bytes); - - let events = download_log.snapshot_events(); - assert_eq!(events.len(), 4); - assert_eq!( - events - .iter() - .filter(|e| e.kind == AuditDownloadKind::RrdpNotification) - .count(), - 1 - ); - assert_eq!( - events - .iter() - .filter(|e| e.kind == AuditDownloadKind::RrdpDelta) - .count(), - 2 - ); - assert_eq!( - events - .iter() - .filter(|e| e.kind == AuditDownloadKind::RrdpSnapshot) - .count(), - 1 - ); - assert!(events.iter().all(|e| e.success)); -} - -#[test] -fn replay_sync_uses_rrdp_when_locked_to_rrdp() { - let temp = tempfile::tempdir().expect("tempdir"); - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - - let ( - _archive_temp, - archive_root, - locks_path, - notify_uri, - _rsync_locked_notify, - _rsync_base_uri, - published_uri, - ) = build_replay_archive_fixture(); - let replay_index = - ReplayArchiveIndex::load(&archive_root, &locks_path).expect("load replay index"); - let http = PayloadReplayHttpFetcher::from_paths(&archive_root, &locks_path) - .expect("build replay http fetcher"); - let rsync = PayloadReplayRsyncFetcher::from_paths(&archive_root, &locks_path) - .expect("build replay rsync fetcher"); - - let out = sync_publication_point_replay( - &store, - &replay_index, - Some(¬ify_uri), - "rsync://example.test/repo/", - &http, - &rsync, - None, - None, - ) - .expect("replay sync ok"); - - assert_eq!(out.source, RepoSyncSource::Rrdp); - assert_eq!(out.objects_written, 1); - assert_current_object(&store, &published_uri, b"mft"); -} - -#[test] -fn replay_sync_uses_rsync_when_notification_is_locked_to_rsync() { - let temp = tempfile::tempdir().expect("tempdir"); - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - - let ( - _archive_temp, - archive_root, - locks_path, - _notify_uri, - rsync_locked_notify, - rsync_base_uri, - _published_uri, - ) = build_replay_archive_fixture(); - let replay_index = - ReplayArchiveIndex::load(&archive_root, &locks_path).expect("load replay index"); - let http = PayloadReplayHttpFetcher::from_paths(&archive_root, &locks_path) - .expect("build replay http fetcher"); - let rsync = PayloadReplayRsyncFetcher::from_paths(&archive_root, &locks_path) - .expect("build replay rsync fetcher"); - - let out = sync_publication_point_replay( - &store, - &replay_index, - Some(&rsync_locked_notify), - &rsync_base_uri, - &http, - &rsync, - None, - None, - ) - .expect("replay rsync sync ok"); - - assert_eq!(out.source, RepoSyncSource::Rsync); - assert_eq!(out.objects_written, 1); - assert_eq!(out.warnings.len(), 0); - assert_current_object( - &store, - "rsync://rsync.example.test/repo/sub/fallback.cer", - b"cer", - ); -} - -#[test] -fn replay_sync_errors_when_lock_is_missing() { - let temp = tempfile::tempdir().expect("tempdir"); - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - - let ( - _archive_temp, - archive_root, - locks_path, - _notify_uri, - _rsync_locked_notify, - _rsync_base_uri, - _published_uri, - ) = build_replay_archive_fixture(); - let replay_index = - ReplayArchiveIndex::load(&archive_root, &locks_path).expect("load replay index"); - let http = PayloadReplayHttpFetcher::from_paths(&archive_root, &locks_path) - .expect("build replay http fetcher"); - let rsync = PayloadReplayRsyncFetcher::from_paths(&archive_root, &locks_path) - .expect("build replay rsync fetcher"); - - let err = sync_publication_point_replay( - &store, - &replay_index, - Some("https://missing.example/notification.xml"), - "rsync://missing.example/repo/", - &http, - &rsync, - None, - None, - ) - .unwrap_err(); - assert!(matches!(err, RepoSyncError::Replay(_)), "{err}"); -} - -#[test] -fn delta_replay_sync_applies_rrdp_deltas_when_base_state_matches() { - let temp = tempfile::tempdir().expect("tempdir"); - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - let ( - _fixture, - base_archive, - base_locks, - delta_archive, - delta_locks, - notify_uri, - _fallback_notify, - module_uri, - ) = build_delta_replay_fixture(); - let base_index = - Arc::new(ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index")); - let delta_index = Arc::new( - ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"), - ); - let http = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) - .expect("build delta http fetcher"); - let rsync = PayloadDeltaReplayRsyncFetcher::new(base_index, delta_index.clone()); - - let state = RrdpState { - session_id: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".to_string(), - serial: 10, - }; - persist_rrdp_local_state( - &store, - ¬ify_uri, - &state, - RrdpSourceSyncState::DeltaReady, - None, - None, - ) - .expect("seed base state"); - - let out = sync_publication_point_replay_delta( - &store, - &delta_index, - Some(¬ify_uri), - &module_uri, - &http, - &rsync, - None, - None, - ) - .expect("delta sync ok"); - - assert_eq!(out.source, RepoSyncSource::Rrdp); - assert_eq!(out.objects_written, 2); - assert_current_object(&store, "rsync://example.test/repo/a.mft", b"delta-a"); - assert_current_object(&store, "rsync://example.test/repo/sub/b.roa", b"delta-b"); - let new_state = load_rrdp_local_state(&store, ¬ify_uri) - .expect("load current state") - .expect("rrdp state present"); - assert_eq!(new_state.serial, 12); -} - -#[test] -fn delta_replay_sync_rejects_base_state_mismatch() { - let temp = tempfile::tempdir().expect("tempdir"); - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - let ( - _fixture, - base_archive, - base_locks, - delta_archive, - delta_locks, - notify_uri, - _fallback_notify, - module_uri, - ) = build_delta_replay_fixture(); - let base_index = - Arc::new(ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index")); - let delta_index = Arc::new( - ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"), - ); - let http = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) - .expect("build delta http fetcher"); - let rsync = PayloadDeltaReplayRsyncFetcher::new(base_index, delta_index.clone()); - - let err = sync_publication_point_replay_delta( - &store, - &delta_index, - Some(¬ify_uri), - &module_uri, - &http, - &rsync, - None, - None, - ) - .unwrap_err(); - assert!(matches!(err, RepoSyncError::Replay(_)), "{err}"); -} - -#[test] -fn delta_replay_sync_noops_unchanged_rrdp_repo() { - let temp = tempfile::tempdir().expect("tempdir"); - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - let ( - _fixture, - base_archive, - base_locks, - delta_archive, - delta_locks, - notify_uri, - _fallback_notify, - module_uri, - ) = build_delta_replay_fixture(); - let state = RrdpState { - session_id: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".to_string(), - serial: 10, - }; - persist_rrdp_local_state( - &store, - ¬ify_uri, - &state, - RrdpSourceSyncState::DeltaReady, - None, - None, - ) - .expect("seed base state"); - - let base_locks_body = std::fs::read_to_string(&base_locks).expect("read base locks"); - let base_locks_sha = sha256_hex(base_locks_body.as_bytes()); - std::fs::write( - &delta_locks, - format!(r#"{{"version":1,"capture":"delta-cap","baseCapture":"base-cap","baseLocksSha256":"{base_locks_sha}","rrdp":{{"{notify_uri}":{{"kind":"unchanged","base":{{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":10}},"target":{{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":10}},"delta_count":0,"deltas":[]}},"https://rrdp-fallback.example.test/notification.xml":{{"kind":"fallback-rsync","base":{{"transport":"rsync","session":null,"serial":null}},"target":{{"transport":"rsync","session":null,"serial":null}},"delta_count":0,"deltas":[]}}}},"rsync":{{"rsync://rsync.example.test/repo/":{{"file_count":1,"overlay_only":true}}}}}}"#), - ) - .expect("rewrite delta locks"); - let repo_hash = sha256_hex(notify_uri.as_bytes()); - let repo_dir = delta_archive - .join("v1/captures/delta-cap/rrdp/repos") - .join(&repo_hash); - std::fs::write( - repo_dir.join("transition.json"), - r#"{"kind":"unchanged","base":{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":10},"target":{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":10},"delta_count":0,"deltas":[]}"#, - ).expect("rewrite transition"); - let delta_index = - ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"); - let http = PayloadDeltaReplayHttpFetcher::from_index(Arc::new(delta_index.clone())) - .expect("build delta http fetcher"); - let base_index = - Arc::new(ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index")); - let rsync = PayloadDeltaReplayRsyncFetcher::new(base_index, Arc::new(delta_index.clone())); - - let out = sync_publication_point_replay_delta( - &store, - &delta_index, - Some(¬ify_uri), - &module_uri, - &http, - &rsync, - None, - None, - ) - .expect("unchanged delta sync ok"); - assert_eq!(out.source, RepoSyncSource::Rrdp); - assert_eq!(out.objects_written, 0); -} - -#[test] -fn delta_replay_sync_uses_rsync_overlay_for_fallback_rsync_kind() { - let temp = tempfile::tempdir().expect("tempdir"); - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - let ( - _fixture, - base_archive, - base_locks, - delta_archive, - delta_locks, - _notify_uri, - fallback_notify, - module_uri, - ) = build_delta_replay_fixture(); - let base_index = - Arc::new(ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index")); - let delta_index = Arc::new( - ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"), - ); - let http = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) - .expect("build delta http fetcher"); - let rsync = PayloadDeltaReplayRsyncFetcher::new(base_index, delta_index.clone()); - - let out = sync_publication_point_replay_delta( - &store, - &delta_index, - Some(&fallback_notify), - &module_uri, - &http, - &rsync, - None, - None, - ) - .expect("fallback-rsync delta sync ok"); - assert_eq!(out.source, RepoSyncSource::Rsync); - assert_eq!(out.objects_written, 2); - assert_current_object(&store, "rsync://rsync.example.test/repo/a.mft", b"base"); - assert_current_object( - &store, - "rsync://rsync.example.test/repo/sub/x.cer", - b"overlay-cer", - ); -} - -#[test] -fn delta_replay_sync_rejects_session_reset_and_gap() { - for kind in ["session-reset", "gap"] { - let temp = tempfile::tempdir().expect("tempdir"); - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - let ( - _fixture, - base_archive, - base_locks, - delta_archive, - delta_locks, - notify_uri, - _fallback_notify, - module_uri, - ) = build_delta_replay_fixture(); - let base_index = Arc::new( - ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index"), - ); - let state = RrdpState { - session_id: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".to_string(), - serial: 10, - }; - persist_rrdp_local_state( - &store, - ¬ify_uri, - &state, - RrdpSourceSyncState::DeltaReady, - None, - None, - ) - .expect("seed base state"); - - let locks_body = std::fs::read_to_string(&delta_locks).expect("read delta locks"); - let rewritten = locks_body.replace("\"kind\":\"delta\"", &format!("\"kind\":\"{}\"", kind)); - std::fs::write(&delta_locks, rewritten).expect("rewrite locks kind"); - let repo_hash = sha256_hex(notify_uri.as_bytes()); - let repo_dir = delta_archive - .join("v1/captures/delta-cap/rrdp/repos") - .join(&repo_hash); - std::fs::write( - repo_dir.join("transition.json"), - format!( - r#"{{"kind":"{kind}","base":{{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":10}},"target":{{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":12}},"delta_count":2,"deltas":[11,12]}}"#, - ), - ) - .expect("rewrite transition kind"); - let delta_index = Arc::new( - ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"), - ); - let http = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) - .expect("build delta http fetcher"); - let rsync = PayloadDeltaReplayRsyncFetcher::new(base_index.clone(), delta_index.clone()); - let err = sync_publication_point_replay_delta( - &store, - &delta_index, - Some(¬ify_uri), - &module_uri, - &http, - &rsync, - None, - None, - ) - .unwrap_err(); - assert!(matches!(err, RepoSyncError::Replay(_)), "{err}"); - } -} +// Repository sync tests are grouped by transport and replay behavior. +include!("tests_parts/setup_and_sync.rs"); +include!("tests_parts/fallback_and_replay.rs"); +include!("tests_parts/delta_replay.rs"); diff --git a/crates/panda-rpki-validator/src/sync/repo/tests_parts/delta_replay.rs b/crates/panda-rpki-validator/src/sync/repo/tests_parts/delta_replay.rs new file mode 100644 index 0000000..192ed72 --- /dev/null +++ b/crates/panda-rpki-validator/src/sync/repo/tests_parts/delta_replay.rs @@ -0,0 +1,279 @@ +// Repository sync test group: delta replay. + +#[test] +fn delta_replay_sync_applies_rrdp_deltas_when_base_state_matches() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + let ( + _fixture, + base_archive, + base_locks, + delta_archive, + delta_locks, + notify_uri, + _fallback_notify, + module_uri, + ) = build_delta_replay_fixture(); + let base_index = + Arc::new(ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index")); + let delta_index = Arc::new( + ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"), + ); + let http = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) + .expect("build delta http fetcher"); + let rsync = PayloadDeltaReplayRsyncFetcher::new(base_index, delta_index.clone()); + + let state = RrdpState { + session_id: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".to_string(), + serial: 10, + }; + persist_rrdp_local_state( + &store, + ¬ify_uri, + &state, + RrdpSourceSyncState::DeltaReady, + None, + None, + ) + .expect("seed base state"); + + let out = sync_publication_point_replay_delta( + &store, + &delta_index, + Some(¬ify_uri), + &module_uri, + &http, + &rsync, + None, + None, + ) + .expect("delta sync ok"); + + assert_eq!(out.source, RepoSyncSource::Rrdp); + assert_eq!(out.objects_written, 2); + assert_current_object(&store, "rsync://example.test/repo/a.mft", b"delta-a"); + assert_current_object(&store, "rsync://example.test/repo/sub/b.roa", b"delta-b"); + let new_state = load_rrdp_local_state(&store, ¬ify_uri) + .expect("load current state") + .expect("rrdp state present"); + assert_eq!(new_state.serial, 12); +} + +#[test] +fn delta_replay_sync_rejects_base_state_mismatch() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + let ( + _fixture, + base_archive, + base_locks, + delta_archive, + delta_locks, + notify_uri, + _fallback_notify, + module_uri, + ) = build_delta_replay_fixture(); + let base_index = + Arc::new(ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index")); + let delta_index = Arc::new( + ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"), + ); + let http = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) + .expect("build delta http fetcher"); + let rsync = PayloadDeltaReplayRsyncFetcher::new(base_index, delta_index.clone()); + + let err = sync_publication_point_replay_delta( + &store, + &delta_index, + Some(¬ify_uri), + &module_uri, + &http, + &rsync, + None, + None, + ) + .unwrap_err(); + assert!(matches!(err, RepoSyncError::Replay(_)), "{err}"); +} + +#[test] +fn delta_replay_sync_noops_unchanged_rrdp_repo() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + let ( + _fixture, + base_archive, + base_locks, + delta_archive, + delta_locks, + notify_uri, + _fallback_notify, + module_uri, + ) = build_delta_replay_fixture(); + let state = RrdpState { + session_id: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".to_string(), + serial: 10, + }; + persist_rrdp_local_state( + &store, + ¬ify_uri, + &state, + RrdpSourceSyncState::DeltaReady, + None, + None, + ) + .expect("seed base state"); + + let base_locks_body = std::fs::read_to_string(&base_locks).expect("read base locks"); + let base_locks_sha = sha256_hex(base_locks_body.as_bytes()); + std::fs::write( + &delta_locks, + format!(r#"{{"version":1,"capture":"delta-cap","baseCapture":"base-cap","baseLocksSha256":"{base_locks_sha}","rrdp":{{"{notify_uri}":{{"kind":"unchanged","base":{{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":10}},"target":{{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":10}},"delta_count":0,"deltas":[]}},"https://rrdp-fallback.example.test/notification.xml":{{"kind":"fallback-rsync","base":{{"transport":"rsync","session":null,"serial":null}},"target":{{"transport":"rsync","session":null,"serial":null}},"delta_count":0,"deltas":[]}}}},"rsync":{{"rsync://rsync.example.test/repo/":{{"file_count":1,"overlay_only":true}}}}}}"#), + ) + .expect("rewrite delta locks"); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let repo_dir = delta_archive + .join("v1/captures/delta-cap/rrdp/repos") + .join(&repo_hash); + std::fs::write( + repo_dir.join("transition.json"), + r#"{"kind":"unchanged","base":{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":10},"target":{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":10},"delta_count":0,"deltas":[]}"#, + ).expect("rewrite transition"); + let delta_index = + ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"); + let http = PayloadDeltaReplayHttpFetcher::from_index(Arc::new(delta_index.clone())) + .expect("build delta http fetcher"); + let base_index = + Arc::new(ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index")); + let rsync = PayloadDeltaReplayRsyncFetcher::new(base_index, Arc::new(delta_index.clone())); + + let out = sync_publication_point_replay_delta( + &store, + &delta_index, + Some(¬ify_uri), + &module_uri, + &http, + &rsync, + None, + None, + ) + .expect("unchanged delta sync ok"); + assert_eq!(out.source, RepoSyncSource::Rrdp); + assert_eq!(out.objects_written, 0); +} + +#[test] +fn delta_replay_sync_uses_rsync_overlay_for_fallback_rsync_kind() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + let ( + _fixture, + base_archive, + base_locks, + delta_archive, + delta_locks, + _notify_uri, + fallback_notify, + module_uri, + ) = build_delta_replay_fixture(); + let base_index = + Arc::new(ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index")); + let delta_index = Arc::new( + ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"), + ); + let http = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) + .expect("build delta http fetcher"); + let rsync = PayloadDeltaReplayRsyncFetcher::new(base_index, delta_index.clone()); + + let out = sync_publication_point_replay_delta( + &store, + &delta_index, + Some(&fallback_notify), + &module_uri, + &http, + &rsync, + None, + None, + ) + .expect("fallback-rsync delta sync ok"); + assert_eq!(out.source, RepoSyncSource::Rsync); + assert_eq!(out.objects_written, 2); + assert_current_object(&store, "rsync://rsync.example.test/repo/a.mft", b"base"); + assert_current_object( + &store, + "rsync://rsync.example.test/repo/sub/x.cer", + b"overlay-cer", + ); +} + +#[test] +fn delta_replay_sync_rejects_session_reset_and_gap() { + for kind in ["session-reset", "gap"] { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + let ( + _fixture, + base_archive, + base_locks, + delta_archive, + delta_locks, + notify_uri, + _fallback_notify, + module_uri, + ) = build_delta_replay_fixture(); + let base_index = Arc::new( + ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index"), + ); + let state = RrdpState { + session_id: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".to_string(), + serial: 10, + }; + persist_rrdp_local_state( + &store, + ¬ify_uri, + &state, + RrdpSourceSyncState::DeltaReady, + None, + None, + ) + .expect("seed base state"); + + let locks_body = std::fs::read_to_string(&delta_locks).expect("read delta locks"); + let rewritten = locks_body.replace("\"kind\":\"delta\"", &format!("\"kind\":\"{}\"", kind)); + std::fs::write(&delta_locks, rewritten).expect("rewrite locks kind"); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let repo_dir = delta_archive + .join("v1/captures/delta-cap/rrdp/repos") + .join(&repo_hash); + std::fs::write( + repo_dir.join("transition.json"), + format!( + r#"{{"kind":"{kind}","base":{{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":10}},"target":{{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":12}},"delta_count":2,"deltas":[11,12]}}"#, + ), + ) + .expect("rewrite transition kind"); + let delta_index = Arc::new( + ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"), + ); + let http = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) + .expect("build delta http fetcher"); + let rsync = PayloadDeltaReplayRsyncFetcher::new(base_index.clone(), delta_index.clone()); + let err = sync_publication_point_replay_delta( + &store, + &delta_index, + Some(¬ify_uri), + &module_uri, + &http, + &rsync, + None, + None, + ) + .unwrap_err(); + assert!(matches!(err, RepoSyncError::Replay(_)), "{err}"); + } +} diff --git a/crates/panda-rpki-validator/src/sync/repo/tests_parts/fallback_and_replay.rs b/crates/panda-rpki-validator/src/sync/repo/tests_parts/fallback_and_replay.rs new file mode 100644 index 0000000..097a546 --- /dev/null +++ b/crates/panda-rpki-validator/src/sync/repo/tests_parts/fallback_and_replay.rs @@ -0,0 +1,344 @@ +// Repository sync test group: fallback and replay. + +#[test] +fn rrdp_protocol_error_does_not_retry_and_falls_back_to_rsync() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + + let timing = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + tal_url: None, + db_path: Some(store_dir.to_string_lossy().into_owned()), + }); + + let notification_uri = "https://example.test/notification.xml"; + let snapshot_uri = "https://example.test/snapshot.xml"; + let published_uri = "rsync://example.test/repo/a.mft"; + let published_bytes = b"x"; + + let snapshot = snapshot_xml( + "9df4b597-af9e-4dca-bdda-719cce2c4e28", + 1, + &[(published_uri, published_bytes)], + ); + // Intentionally wrong hash to trigger protocol error (SnapshotHashMismatch). + let wrong_hash = "00".repeat(32); + let notif = notification_xml( + "9df4b597-af9e-4dca-bdda-719cce2c4e28", + 1, + snapshot_uri, + &wrong_hash, + ); + + let mut map = HashMap::new(); + map.insert(notification_uri.to_string(), notif); + map.insert(snapshot_uri.to_string(), snapshot); + let http = MapFetcher { map }; + + struct EmptyRsyncFetcher; + impl RsyncFetcher for EmptyRsyncFetcher { + fn fetch_objects( + &self, + _rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + Ok(Vec::new()) + } + } + + let policy = Policy { + sync_preference: SyncPreference::RrdpThenRsync, + ..Policy::default() + }; + + let download_log = DownloadLogHandle::new(); + let out = sync_publication_point( + &store, + &policy, + Some(notification_uri), + "rsync://example.test/repo/", + &http, + &EmptyRsyncFetcher, + Some(&timing), + Some(&download_log), + ) + .expect("sync ok"); + + assert_eq!(out.source, RepoSyncSource::Rsync); + assert!( + out.warnings + .iter() + .any(|w| w.message.contains("RRDP failed; falling back to rsync")), + "expected RRDP fallback warning" + ); + + let events = download_log.snapshot_events(); + assert_eq!( + events.len(), + 3, + "expected notification + snapshot + rsync fallback" + ); + assert_eq!(events[0].kind, AuditDownloadKind::RrdpNotification); + assert!(events[0].success); + assert_eq!(events[1].kind, AuditDownloadKind::RrdpSnapshot); + assert!(!events[1].success); + assert_eq!(events[2].kind, AuditDownloadKind::Rsync); + assert!(events[2].success); + + let v = timing_to_json(temp.path(), &timing); + let counts = v.get("counts").expect("counts"); + assert_eq!( + counts + .get("rrdp_retry_attempt_total") + .and_then(|v| v.as_u64()), + Some(1) + ); + assert_eq!( + counts + .get("rrdp_failed_protocol_total") + .and_then(|v| v.as_u64()), + Some(1) + ); + assert_eq!( + counts + .get("repo_sync_rrdp_failed_total") + .and_then(|v| v.as_u64()), + Some(1) + ); + assert_eq!( + counts + .get("repo_sync_rsync_fallback_ok_total") + .and_then(|v| v.as_u64()), + Some(1) + ); +} + +#[test] +fn rrdp_delta_fetches_are_logged_even_if_snapshot_fallback_is_used() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + + let timing = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + tal_url: None, + db_path: Some(store_dir.to_string_lossy().into_owned()), + }); + + let notification_uri = "https://example.test/notification.xml"; + let snapshot_uri = "https://example.test/snapshot.xml"; + let delta_2_uri = "https://example.test/delta_2.xml"; + let delta_3_uri = "https://example.test/delta_3.xml"; + let published_uri = "rsync://example.test/repo/a.mft"; + let published_bytes = b"x"; + + let sid = "9df4b597-af9e-4dca-bdda-719cce2c4e28"; + + // Seed old RRDP state so sync_from_notification tries deltas (RFC 8182 §3.4.1). + let state = RrdpState { + session_id: sid.to_string(), + serial: 1, + }; + persist_rrdp_local_state( + &store, + notification_uri, + &state, + RrdpSourceSyncState::DeltaReady, + Some(snapshot_uri), + None, + ) + .expect("seed state"); + + let delta_2 = format!( + r#""# + ) + .into_bytes(); + let delta_3 = format!( + r#""# + ) + .into_bytes(); + let delta_2_hash = hex::encode(sha2::Sha256::digest(&delta_2)); + let delta_3_hash = hex::encode(sha2::Sha256::digest(&delta_3)); + + let snapshot = snapshot_xml(sid, 3, &[(published_uri, published_bytes)]); + let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); + let notif = format!( + r#""# + ) + .into_bytes(); + + let mut map = HashMap::new(); + map.insert(notification_uri.to_string(), notif); + map.insert(snapshot_uri.to_string(), snapshot); + map.insert(delta_2_uri.to_string(), delta_2); + map.insert(delta_3_uri.to_string(), delta_3); + let http = MapFetcher { map }; + + let policy = Policy { + sync_preference: SyncPreference::RrdpThenRsync, + ..Policy::default() + }; + + let download_log = DownloadLogHandle::new(); + let out = sync_publication_point( + &store, + &policy, + Some(notification_uri), + "rsync://example.test/repo/", + &http, + &PanicRsyncFetcher, + Some(&timing), + Some(&download_log), + ) + .expect("sync ok"); + + assert_eq!(out.source, RepoSyncSource::Rrdp); + assert_eq!(out.objects_written, 1); + assert_current_object(&store, published_uri, published_bytes); + + let events = download_log.snapshot_events(); + assert_eq!(events.len(), 4); + assert_eq!( + events + .iter() + .filter(|e| e.kind == AuditDownloadKind::RrdpNotification) + .count(), + 1 + ); + assert_eq!( + events + .iter() + .filter(|e| e.kind == AuditDownloadKind::RrdpDelta) + .count(), + 2 + ); + assert_eq!( + events + .iter() + .filter(|e| e.kind == AuditDownloadKind::RrdpSnapshot) + .count(), + 1 + ); + assert!(events.iter().all(|e| e.success)); +} + +#[test] +fn replay_sync_uses_rrdp_when_locked_to_rrdp() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + + let ( + _archive_temp, + archive_root, + locks_path, + notify_uri, + _rsync_locked_notify, + _rsync_base_uri, + published_uri, + ) = build_replay_archive_fixture(); + let replay_index = + ReplayArchiveIndex::load(&archive_root, &locks_path).expect("load replay index"); + let http = PayloadReplayHttpFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay http fetcher"); + let rsync = PayloadReplayRsyncFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay rsync fetcher"); + + let out = sync_publication_point_replay( + &store, + &replay_index, + Some(¬ify_uri), + "rsync://example.test/repo/", + &http, + &rsync, + None, + None, + ) + .expect("replay sync ok"); + + assert_eq!(out.source, RepoSyncSource::Rrdp); + assert_eq!(out.objects_written, 1); + assert_current_object(&store, &published_uri, b"mft"); +} + +#[test] +fn replay_sync_uses_rsync_when_notification_is_locked_to_rsync() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + + let ( + _archive_temp, + archive_root, + locks_path, + _notify_uri, + rsync_locked_notify, + rsync_base_uri, + _published_uri, + ) = build_replay_archive_fixture(); + let replay_index = + ReplayArchiveIndex::load(&archive_root, &locks_path).expect("load replay index"); + let http = PayloadReplayHttpFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay http fetcher"); + let rsync = PayloadReplayRsyncFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay rsync fetcher"); + + let out = sync_publication_point_replay( + &store, + &replay_index, + Some(&rsync_locked_notify), + &rsync_base_uri, + &http, + &rsync, + None, + None, + ) + .expect("replay rsync sync ok"); + + assert_eq!(out.source, RepoSyncSource::Rsync); + assert_eq!(out.objects_written, 1); + assert_eq!(out.warnings.len(), 0); + assert_current_object( + &store, + "rsync://rsync.example.test/repo/sub/fallback.cer", + b"cer", + ); +} + +#[test] +fn replay_sync_errors_when_lock_is_missing() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + + let ( + _archive_temp, + archive_root, + locks_path, + _notify_uri, + _rsync_locked_notify, + _rsync_base_uri, + _published_uri, + ) = build_replay_archive_fixture(); + let replay_index = + ReplayArchiveIndex::load(&archive_root, &locks_path).expect("load replay index"); + let http = PayloadReplayHttpFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay http fetcher"); + let rsync = PayloadReplayRsyncFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay rsync fetcher"); + + let err = sync_publication_point_replay( + &store, + &replay_index, + Some("https://missing.example/notification.xml"), + "rsync://missing.example/repo/", + &http, + &rsync, + None, + None, + ) + .unwrap_err(); + assert!(matches!(err, RepoSyncError::Replay(_)), "{err}"); +} diff --git a/crates/panda-rpki-validator/src/sync/repo/tests_parts/setup_and_sync.rs b/crates/panda-rpki-validator/src/sync/repo/tests_parts/setup_and_sync.rs new file mode 100644 index 0000000..204a730 --- /dev/null +++ b/crates/panda-rpki-validator/src/sync/repo/tests_parts/setup_and_sync.rs @@ -0,0 +1,713 @@ +// Repository sync test group: setup and sync. + +use super::*; +use crate::analysis::timing::{TimingHandle, TimingMeta}; +use crate::fetch::rsync::LocalDirRsyncFetcher; +use crate::replay::archive::{ReplayArchiveIndex, sha256_hex}; +use crate::replay::delta_archive::ReplayDeltaArchiveIndex; +use crate::replay::delta_fetch_http::PayloadDeltaReplayHttpFetcher; +use crate::replay::delta_fetch_rsync::PayloadDeltaReplayRsyncFetcher; +use crate::replay::fetch_http::PayloadReplayHttpFetcher; +use crate::replay::fetch_rsync::PayloadReplayRsyncFetcher; +use crate::storage::RepositoryViewState; +use crate::sync::rrdp::Fetcher as HttpFetcher; +use crate::sync::rrdp::RrdpState; +use crate::sync::store_projection::{build_repository_view_present_entry, compute_sha256_hex}; +use base64::Engine; +use sha2::Digest; +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +struct DummyHttpFetcher; + +impl HttpFetcher for DummyHttpFetcher { + fn fetch(&self, _url: &str) -> Result, String> { + panic!("http fetcher must not be used in rsync-only mode") + } +} + +struct PanicRsyncFetcher; +impl RsyncFetcher for PanicRsyncFetcher { + fn fetch_objects( + &self, + _rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + panic!("rsync must not be used in this test") + } +} + +struct MapFetcher { + map: HashMap>, +} + +impl HttpFetcher for MapFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + self.map + .get(uri) + .cloned() + .ok_or_else(|| format!("not found: {uri}")) + } +} + +fn assert_current_object(store: &RocksStore, uri: &str, expected: &[u8]) { + assert_eq!( + store + .load_current_object_bytes_by_uri(uri) + .expect("load current object"), + Some(expected.to_vec()) + ); +} + +#[test] +fn rsync_sync_uses_fetcher_dedup_scope_for_repository_view_projection() { + struct ScopeFetcher; + impl RsyncFetcher for ScopeFetcher { + fn fetch_objects( + &self, + _rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + Ok(vec![( + "rsync://example.net/repo/child/a.mft".to_string(), + b"manifest".to_vec(), + )]) + } + + fn dedup_key(&self, _rsync_base_uri: &str) -> String { + "rsync://example.net/repo/".to_string() + } + } + + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let seeded = build_repository_view_present_entry( + "rsync://example.net/repo/", + "rsync://example.net/repo/sibling/old.roa", + &compute_sha256_hex(b"old"), + ); + store + .put_projection_batch(&[seeded], &[], &[]) + .expect("seed repository view"); + + let fetcher = ScopeFetcher; + let written = rsync_sync_into_current_store( + &store, + "rsync://example.net/repo/child/", + None, + &fetcher, + None, + None, + ) + .expect("sync ok"); + assert_eq!(written, 1); + + let entries = store + .list_repository_view_entries_with_prefix("rsync://example.net/repo/") + .expect("list repository view"); + let sibling = entries + .iter() + .find(|entry| entry.rsync_uri == "rsync://example.net/repo/sibling/old.roa") + .expect("sibling entry exists"); + assert_eq!(sibling.state, RepositoryViewState::Withdrawn); + let child = entries + .iter() + .find(|entry| entry.rsync_uri == "rsync://example.net/repo/child/a.mft") + .expect("child entry exists"); + assert_eq!(child.state, RepositoryViewState::Present); +} + +fn notification_xml( + session_id: &str, + serial: u64, + snapshot_uri: &str, + snapshot_hash: &str, +) -> Vec { + format!( + r#""# + ) + .into_bytes() +} + +fn snapshot_xml(session_id: &str, serial: u64, published: &[(&str, &[u8])]) -> Vec { + let mut out = format!( + r#""# + ); + for (uri, bytes) in published { + let b64 = base64::engine::general_purpose::STANDARD.encode(bytes); + out.push_str(&format!(r#"{b64}"#)); + } + out.push_str(""); + out.into_bytes() +} + +fn build_replay_archive_fixture() -> ( + tempfile::TempDir, + std::path::PathBuf, + std::path::PathBuf, + String, + String, + String, + String, +) { + let temp = tempfile::tempdir().expect("tempdir"); + let archive_root = temp.path().join("payload-archive"); + let capture = "repo-replay"; + let capture_root = archive_root.join("v1").join("captures").join(capture); + std::fs::create_dir_all(&capture_root).expect("mkdir capture root"); + std::fs::write( + capture_root.join("capture.json"), + format!( + r#"{{"version":1,"captureId":"{capture}","createdAt":"2026-03-13T00:00:00Z","notes":""}}"# + ), + ) + .expect("write capture json"); + + let notify_uri = "https://rrdp.example.test/notification.xml".to_string(); + let snapshot_uri = "https://rrdp.example.test/snapshot.xml".to_string(); + let session = "00000000-0000-0000-0000-000000000001".to_string(); + let serial = 7u64; + let published_uri = "rsync://example.test/repo/a.mft".to_string(); + let published_bytes = b"mft"; + let snapshot = snapshot_xml(&session, serial, &[(&published_uri, published_bytes)]); + let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); + let notification = notification_xml(&session, serial, &snapshot_uri, &snapshot_hash); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let session_dir = capture_root + .join("rrdp/repos") + .join(&repo_hash) + .join(&session); + std::fs::create_dir_all(&session_dir).expect("mkdir session dir"); + std::fs::write( + session_dir.parent().unwrap().join("meta.json"), + format!( + r#"{{"version":1,"rpkiNotify":"{notify_uri}","createdAt":"2026-03-13T00:00:00Z","lastSeenAt":"2026-03-13T00:00:01Z"}}"# + ), + ) + .expect("write repo meta"); + std::fs::write(session_dir.join("notification-7.xml"), notification) + .expect("write notification"); + std::fs::write( + session_dir.join(format!("snapshot-7-{snapshot_hash}.xml")), + &snapshot, + ) + .expect("write snapshot"); + + let rsync_base_uri = "rsync://rsync.example.test/repo/".to_string(); + let rsync_locked_notify = "https://rrdp-fallback.example.test/notification.xml".to_string(); + let mod_hash = sha256_hex(rsync_base_uri.as_bytes()); + let module_bucket_dir = capture_root.join("rsync/modules").join(&mod_hash); + let module_root = module_bucket_dir + .join("tree") + .join("rsync.example.test") + .join("repo"); + std::fs::create_dir_all(module_root.join("sub")).expect("mkdir module tree"); + std::fs::write( + module_bucket_dir.join("meta.json"), + format!( + r#"{{"version":1,"module":"{rsync_base_uri}","createdAt":"2026-03-13T00:00:00Z","lastSeenAt":"2026-03-13T00:00:01Z"}}"# + ), + ) + .expect("write rsync meta"); + std::fs::write(module_root.join("sub").join("fallback.cer"), b"cer") + .expect("write rsync object"); + + let locks_path = temp.path().join("locks.json"); + std::fs::write( + &locks_path, + format!( + r#"{{ + "version":1, + "capture":"{capture}", + "rrdp":{{ + "{notify_uri}":{{"transport":"rrdp","session":"{session}","serial":{serial}}}, + "{rsync_locked_notify}":{{"transport":"rsync","session":null,"serial":null}} + }}, + "rsync":{{ + "{rsync_base_uri}":{{"transport":"rsync"}} + }} +}}"# + ), + ) + .expect("write locks"); + + ( + temp, + archive_root, + locks_path, + notify_uri, + rsync_locked_notify, + rsync_base_uri, + published_uri, + ) +} + +fn build_delta_replay_fixture() -> ( + tempfile::TempDir, + std::path::PathBuf, + std::path::PathBuf, + std::path::PathBuf, + std::path::PathBuf, + String, + String, + String, +) { + let temp = tempfile::tempdir().expect("tempdir"); + + let base_archive = temp.path().join("payload-archive"); + let base_capture_root = base_archive.join("v1/captures/base-cap"); + std::fs::create_dir_all(&base_capture_root).expect("mkdir base capture"); + std::fs::write( + base_capture_root.join("capture.json"), + r#"{"version":1,"captureId":"base-cap","createdAt":"2026-03-16T00:00:00Z","notes":""}"#, + ) + .expect("write base capture meta"); + + let notify_uri = "https://rrdp.example.test/notification.xml".to_string(); + let snapshot_uri = "https://rrdp.example.test/snapshot.xml".to_string(); + let session = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".to_string(); + let base_serial = 10u64; + let delta1_uri = "https://rrdp.example.test/d1.xml".to_string(); + let delta2_uri = "https://rrdp.example.test/d2.xml".to_string(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let base_session_dir = base_capture_root + .join("rrdp/repos") + .join(&repo_hash) + .join(&session); + std::fs::create_dir_all(&base_session_dir).expect("mkdir base session dir"); + std::fs::write( + base_session_dir.parent().unwrap().join("meta.json"), + format!(r#"{{"version":1,"rpkiNotify":"{notify_uri}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), + ) + .expect("write base rrdp meta"); + let base_snapshot = snapshot_xml( + &session, + base_serial, + &[("rsync://example.test/repo/a.mft", b"base")], + ); + let base_snapshot_hash = hex::encode(sha2::Sha256::digest(&base_snapshot)); + let base_notification = + notification_xml(&session, base_serial, &snapshot_uri, &base_snapshot_hash); + std::fs::write( + base_session_dir.join("notification-10.xml"), + base_notification, + ) + .expect("write base notif"); + std::fs::write( + base_session_dir.join(format!("snapshot-10-{base_snapshot_hash}.xml")), + base_snapshot, + ) + .expect("write base snapshot"); + + let module_uri = "rsync://rsync.example.test/repo/".to_string(); + let module_hash = sha256_hex(module_uri.as_bytes()); + let base_module_bucket = base_capture_root.join("rsync/modules").join(&module_hash); + let base_module_tree = base_module_bucket.join("tree/rsync.example.test/repo"); + std::fs::create_dir_all(base_module_tree.join("sub")).expect("mkdir base rsync tree"); + std::fs::write( + base_module_bucket.join("meta.json"), + format!(r#"{{"version":1,"module":"{module_uri}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), + ) + .expect("write base module meta"); + std::fs::write(base_module_tree.join("a.mft"), b"base").expect("write base a.mft"); + std::fs::write(base_module_tree.join("sub").join("x.cer"), b"base-cer") + .expect("write base x.cer"); + + let base_locks = temp.path().join("base-locks.json"); + let fallback_notify = "https://rrdp-fallback.example.test/notification.xml".to_string(); + let base_locks_body = format!( + r#"{{"version":1,"capture":"base-cap","rrdp":{{"{notify_uri}":{{"transport":"rrdp","session":"{session}","serial":10}},"{fallback_notify}":{{"transport":"rsync","session":null,"serial":null}}}},"rsync":{{"{module_uri}":{{"transport":"rsync"}}}}}}"# + ); + std::fs::write(&base_locks, &base_locks_body).expect("write base locks"); + let base_locks_sha = sha256_hex(base_locks_body.as_bytes()); + + let delta_archive = temp.path().join("payload-delta-archive"); + let delta_capture_root = delta_archive.join("v1/captures/delta-cap"); + std::fs::create_dir_all(&delta_capture_root).expect("mkdir delta capture"); + std::fs::write( + delta_capture_root.join("capture.json"), + r#"{"version":1,"captureId":"delta-cap","createdAt":"2026-03-16T00:00:00Z","notes":""}"#, + ) + .expect("write delta capture meta"); + std::fs::write( + delta_capture_root.join("base.json"), + format!(r#"{{"version":1,"baseCapture":"base-cap","baseLocksSha256":"{base_locks_sha}","createdAt":"2026-03-16T00:00:00Z"}}"#), + ) + .expect("write delta base meta"); + + let delta_session_dir = delta_capture_root + .join("rrdp/repos") + .join(&repo_hash) + .join(&session); + let delta_deltas_dir = delta_session_dir.join("deltas"); + std::fs::create_dir_all(&delta_deltas_dir).expect("mkdir delta deltas"); + std::fs::write( + delta_session_dir.parent().unwrap().join("meta.json"), + format!(r#"{{"version":1,"rpkiNotify":"{notify_uri}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), + ) + .expect("write delta meta"); + std::fs::write( + delta_session_dir.parent().unwrap().join("transition.json"), + format!(r#"{{"kind":"delta","base":{{"transport":"rrdp","session":"{session}","serial":10}},"target":{{"transport":"rrdp","session":"{session}","serial":12}},"delta_count":2,"deltas":[11,12]}}"#), + ) + .expect("write delta transition"); + let delta1 = format!( + r#"{}"#, + base64::engine::general_purpose::STANDARD.encode(b"delta-a") + ); + let delta2 = format!( + r#"{}"#, + base64::engine::general_purpose::STANDARD.encode(b"delta-b") + ); + let delta1_hash = hex::encode(sha2::Sha256::digest(delta1.as_bytes())); + let delta2_hash = hex::encode(sha2::Sha256::digest(delta2.as_bytes())); + let target_notification = format!( + r#" + + + + +"# + ); + std::fs::write( + delta_session_dir.join("notification-target-12.xml"), + target_notification, + ) + .expect("write target notification"); + std::fs::write(delta_deltas_dir.join("delta-11-aaaa.xml"), delta1).expect("write delta11"); + std::fs::write(delta_deltas_dir.join("delta-12-bbbb.xml"), delta2).expect("write delta12"); + + let delta_module_bucket = delta_capture_root.join("rsync/modules").join(&module_hash); + let delta_module_tree = delta_module_bucket.join("tree/rsync.example.test/repo"); + std::fs::create_dir_all(delta_module_tree.join("sub")).expect("mkdir delta rsync tree"); + std::fs::write( + delta_module_bucket.join("meta.json"), + format!(r#"{{"version":1,"module":"{module_uri}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), + ) + .expect("write delta rsync meta"); + std::fs::write( + delta_module_bucket.join("files.json"), + format!(r#"{{"version":1,"module":"{module_uri}","fileCount":1,"files":["{module_uri}sub/x.cer"]}}"#), + ) + .expect("write delta files"); + std::fs::write(delta_module_tree.join("sub").join("x.cer"), b"overlay-cer") + .expect("write overlay file"); + + let fallback_hash = sha256_hex(fallback_notify.as_bytes()); + let fallback_repo_dir = delta_capture_root.join("rrdp/repos").join(&fallback_hash); + std::fs::create_dir_all(&fallback_repo_dir).expect("mkdir fallback repo dir"); + std::fs::write( + fallback_repo_dir.join("meta.json"), + format!(r#"{{"version":1,"rpkiNotify":"{fallback_notify}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), + ) + .expect("write fallback meta"); + std::fs::write( + fallback_repo_dir.join("transition.json"), + r#"{"kind":"fallback-rsync","base":{"transport":"rsync","session":null,"serial":null},"target":{"transport":"rsync","session":null,"serial":null},"delta_count":0,"deltas":[]}"#, + ) + .expect("write fallback transition"); + + let delta_locks = temp.path().join("locks-delta.json"); + std::fs::write( + &delta_locks, + format!(r#"{{"version":1,"capture":"delta-cap","baseCapture":"base-cap","baseLocksSha256":"{base_locks_sha}","rrdp":{{"{notify_uri}":{{"kind":"delta","base":{{"transport":"rrdp","session":"{session}","serial":10}},"target":{{"transport":"rrdp","session":"{session}","serial":12}},"delta_count":2,"deltas":[11,12]}},"{fallback_notify}":{{"kind":"fallback-rsync","base":{{"transport":"rsync","session":null,"serial":null}},"target":{{"transport":"rsync","session":null,"serial":null}},"delta_count":0,"deltas":[]}}}},"rsync":{{"{module_uri}":{{"file_count":1,"overlay_only":false}}}}}}"#), + ) + .expect("write delta locks"); + + ( + temp, + base_archive, + base_locks, + delta_archive, + delta_locks, + notify_uri, + fallback_notify, + module_uri, + ) +} + +fn timing_to_json(temp_dir: &std::path::Path, timing: &TimingHandle) -> serde_json::Value { + let timing_path = temp_dir.join("timing_retry.json"); + timing.write_json(&timing_path, 50).expect("write json"); + serde_json::from_slice(&std::fs::read(&timing_path).expect("read json")).expect("parse json") +} + +#[test] +fn rsync_sync_writes_current_store_and_records_counts() { + let temp = tempfile::tempdir().expect("tempdir"); + + let repo_dir = temp.path().join("repo"); + std::fs::create_dir_all(repo_dir.join("sub")).expect("mkdir"); + std::fs::write(repo_dir.join("a.mft"), b"mft").expect("write"); + std::fs::write(repo_dir.join("sub").join("b.roa"), b"roa").expect("write"); + std::fs::write(repo_dir.join("sub").join("c.cer"), b"cer").expect("write"); + + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + + let timing = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + tal_url: None, + db_path: Some(store_dir.to_string_lossy().into_owned()), + }); + + let policy = Policy { + sync_preference: SyncPreference::RsyncOnly, + ..Policy::default() + }; + let http = DummyHttpFetcher; + let rsync = LocalDirRsyncFetcher::new(&repo_dir); + + let download_log = DownloadLogHandle::new(); + let out = sync_publication_point( + &store, + &policy, + None, + "rsync://example.test/repo/", + &http, + &rsync, + Some(&timing), + Some(&download_log), + ) + .expect("sync ok"); + + assert_eq!(out.source, RepoSyncSource::Rsync); + assert_eq!(out.objects_written, 3); + + let events = download_log.snapshot_events(); + assert_eq!(events.len(), 1); + assert_eq!(events[0].kind, AuditDownloadKind::Rsync); + assert!(events[0].success); + assert_eq!(events[0].bytes, Some(9)); + let objects = events[0].objects.as_ref().expect("objects stat"); + assert_eq!(objects.objects_count, 3); + assert_eq!(objects.objects_bytes_total, 9); + + assert_current_object(&store, "rsync://example.test/repo/a.mft", b"mft"); + assert_current_object(&store, "rsync://example.test/repo/sub/b.roa", b"roa"); + assert_current_object(&store, "rsync://example.test/repo/sub/c.cer", b"cer"); + + let view = store + .get_repository_view_entry("rsync://example.test/repo/a.mft") + .expect("get repository view") + .expect("repository view entry present"); + assert_eq!( + view.current_hash.as_deref(), + Some(hex::encode(sha2::Sha256::digest(b"mft")).as_str()) + ); + assert_eq!( + view.repository_source.as_deref(), + Some("rsync://example.test/repo/") + ); + + let current_bytes = store + .load_current_object_bytes_by_uri("rsync://example.test/repo/sub/b.roa") + .expect("load current bytes") + .expect("current object bytes exist"); + assert_eq!(current_bytes, b"roa".to_vec()); + assert!( + store + .get_raw_by_hash_entry(hex::encode(sha2::Sha256::digest(b"roa")).as_str()) + .expect("get raw_by_hash") + .is_none() + ); + + let timing_path = temp.path().join("timing.json"); + timing.write_json(&timing_path, 5).expect("write json"); + let v: serde_json::Value = + serde_json::from_slice(&std::fs::read(&timing_path).expect("read json")) + .expect("parse json"); + let counts = v.get("counts").expect("counts"); + assert_eq!( + counts + .get("rsync_objects_fetched_total") + .and_then(|v| v.as_u64()), + Some(3) + ); + assert_eq!( + counts + .get("rsync_objects_bytes_total") + .and_then(|v| v.as_u64()), + Some(3 * 3) + ); +} + +#[test] +fn rsync_second_sync_marks_missing_repository_view_entries_withdrawn() { + let temp = tempfile::tempdir().expect("tempdir"); + + let repo_dir = temp.path().join("repo"); + std::fs::create_dir_all(repo_dir.join("sub")).expect("mkdir"); + std::fs::write(repo_dir.join("a.mft"), b"mft-v1").expect("write a"); + std::fs::write(repo_dir.join("sub").join("b.roa"), b"roa-v1").expect("write b"); + + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + let policy = Policy { + sync_preference: SyncPreference::RsyncOnly, + ..Policy::default() + }; + let http = DummyHttpFetcher; + let rsync = LocalDirRsyncFetcher::new(&repo_dir); + + sync_publication_point( + &store, + &policy, + None, + "rsync://example.test/repo/", + &http, + &rsync, + None, + None, + ) + .expect("first sync ok"); + + std::fs::remove_file(repo_dir.join("sub").join("b.roa")).expect("remove b"); + std::fs::write(repo_dir.join("c.crl"), b"crl-v2").expect("write c"); + + sync_publication_point( + &store, + &policy, + None, + "rsync://example.test/repo/", + &http, + &rsync, + None, + None, + ) + .expect("second sync ok"); + + let withdrawn = store + .get_repository_view_entry("rsync://example.test/repo/sub/b.roa") + .expect("get withdrawn repo view") + .expect("withdrawn entry exists"); + assert_eq!( + withdrawn.state, + crate::storage::RepositoryViewState::Withdrawn + ); + assert_eq!( + withdrawn.repository_source.as_deref(), + Some("rsync://example.test/repo/") + ); + + let added = store + .get_repository_view_entry("rsync://example.test/repo/c.crl") + .expect("get added repo view") + .expect("added entry exists"); + assert_eq!(added.state, crate::storage::RepositoryViewState::Present); +} + +#[test] +fn rrdp_fetch_error_falls_back_to_rsync_without_retry() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + + let timing = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + tal_url: None, + db_path: Some(store_dir.to_string_lossy().into_owned()), + }); + + let notification_uri = "https://example.test/notification.xml"; + let published_uri = "rsync://example.test/repo/a.mft"; + let published_bytes = b"x"; + struct AlwaysFailHttp { + notification_calls: AtomicUsize, + } + + impl HttpFetcher for AlwaysFailHttp { + fn fetch(&self, _uri: &str) -> Result, String> { + self.notification_calls.fetch_add(1, Ordering::SeqCst); + Err("http request failed: simulated transient".to_string()) + } + } + + struct SingleObjectRsync { + uri: String, + bytes: Vec, + } + impl RsyncFetcher for SingleObjectRsync { + fn fetch_objects( + &self, + _rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + Ok(vec![(self.uri.clone(), self.bytes.clone())]) + } + } + + let http = AlwaysFailHttp { + notification_calls: AtomicUsize::new(0), + }; + + let policy = Policy { + sync_preference: SyncPreference::RrdpThenRsync, + ..Policy::default() + }; + + let download_log = DownloadLogHandle::new(); + let out = sync_publication_point( + &store, + &policy, + Some(notification_uri), + "rsync://example.test/repo/", + &http, + &SingleObjectRsync { + uri: published_uri.to_string(), + bytes: published_bytes.to_vec(), + }, + Some(&timing), + Some(&download_log), + ) + .expect("sync ok"); + + assert_eq!(out.source, RepoSyncSource::Rsync); + assert_current_object(&store, published_uri, published_bytes); + assert_eq!(http.notification_calls.load(Ordering::SeqCst), 1); + + let events = download_log.snapshot_events(); + assert_eq!(events.len(), 2, "expected 1x notification + 1x rsync"); + assert_eq!( + events + .iter() + .filter(|e| e.kind == AuditDownloadKind::RrdpNotification) + .count(), + 1 + ); + assert_eq!( + events + .iter() + .filter(|e| e.kind == AuditDownloadKind::RrdpNotification && !e.success) + .count(), + 1 + ); + assert_eq!( + events + .iter() + .filter(|e| e.kind == AuditDownloadKind::Rsync) + .count(), + 1 + ); + + let v = timing_to_json(temp.path(), &timing); + let counts = v.get("counts").expect("counts"); + assert_eq!( + counts + .get("rrdp_retry_attempt_total") + .and_then(|v| v.as_u64()), + Some(1) + ); + assert_eq!( + counts + .get("repo_sync_rrdp_failed_total") + .and_then(|v| v.as_u64()), + Some(1) + ); + assert_eq!( + counts + .get("repo_sync_rsync_fallback_ok_total") + .and_then(|v| v.as_u64()), + Some(1) + ); +} diff --git a/crates/panda-rpki-validator/src/sync/rrdp.rs b/crates/panda-rpki-validator/src/sync/rrdp.rs index 8b13435..469f61e 100644 --- a/crates/panda-rpki-validator/src/sync/rrdp.rs +++ b/crates/panda-rpki-validator/src/sync/rrdp.rs @@ -23,1334 +23,11 @@ use uuid::Uuid; const RRDP_XMLNS: &str = "http://www.ripe.net/rpki/rrdp"; const RRDP_SNAPSHOT_APPLY_BATCH_SIZE: usize = 1024; -#[derive(Debug, thiserror::Error)] -pub enum RrdpError { - #[error("RRDP XML must be US-ASCII encoded (RFC 8182 §3.5.1.3, §3.5.2.3), got non-ASCII bytes")] - NotAscii, - - #[error("RRDP XML parse error: {0} (RFC 8182 §3.5.1.3, §3.5.2.3)")] - Xml(String), - - #[error( - "RRDP root element must be , , or , got <{0}> (RFC 8182 §3.5.1.3, §3.5.2.3, §3.5.3.3)" - )] - UnexpectedRoot(String), - - #[error("RRDP XML namespace must be {RRDP_XMLNS}, got {0} (RFC 8182 §3.5.1.3, §3.5.2.3)")] - InvalidNamespace(String), - - #[error("RRDP version must be 1, got {0} (RFC 8182 §3.5.1.3, §3.5.2.3)")] - InvalidVersion(String), - - #[error("RRDP session_id invalid UUID: {0} (RFC 8182 §3.5.1.3)")] - InvalidSessionId(String), - - #[error("RRDP serial invalid unsigned integer: {0} (RFC 8182 §3.5.1.3)")] - InvalidSerial(String), - - #[error("notification must contain exactly one element (RFC 8182 §3.5.1.3)")] - SnapshotCountInvalid, - - #[error("snapshot/@uri missing (RFC 8182 §3.5.1.3)")] - SnapshotUriMissing, - - #[error("snapshot/@hash missing (RFC 8182 §3.5.1.3)")] - SnapshotHashMissing, - - #[error("snapshot/@hash must be hex encoding of SHA-256, got {0} (RFC 8182 §3.5.1.3)")] - SnapshotHashInvalid(String), - - #[error("delta/@serial missing in notification (RFC 8182 §3.5.1.3)")] - DeltaRefSerialMissing, - - #[error("delta/@uri missing in notification (RFC 8182 §3.5.1.3)")] - DeltaRefUriMissing, - - #[error("delta/@hash missing in notification (RFC 8182 §3.5.1.3)")] - DeltaRefHashMissing, - - #[error("delta/@hash must be hex encoding of SHA-256, got {0} (RFC 8182 §3.5.1.3)")] - DeltaRefHashInvalid(String), - - #[error("delta/@serial duplicates in notification: {0} (RFC 8182 §3.5.1.3)")] - DeltaRefSerialDuplicate(u64), - - #[error( - "delta/@serial must be <= notification/@serial: delta={delta_serial} notification={notification_serial} (RFC 8182 §3.5.1.3)" - )] - DeltaRefSerialTooHigh { - delta_serial: u64, - notification_serial: u64, - }, - - #[error( - "notification contains deltas but does not end at notification/@serial: max_delta={max_delta_serial} notification={notification_serial} (RFC 8182 §3.5.1.3)" - )] - DeltaRefChainDoesNotEndAtNotificationSerial { - max_delta_serial: u64, - notification_serial: u64, - }, - - #[error( - "notification delta chain not contiguous: missing serial={missing_serial} (range {min_serial}..{notification_serial}) (RFC 8182 §3.5.1.3)" - )] - DeltaRefChainNotContiguous { - min_serial: u64, - notification_serial: u64, - missing_serial: u64, - }, - - #[error("snapshot file hash mismatch (RFC 8182 §3.5.1.3)")] - SnapshotHashMismatch, - - #[error("snapshot session_id mismatch: expected {expected}, got {got} (RFC 8182 §3.5.2.3)")] - SnapshotSessionIdMismatch { expected: String, got: String }, - - #[error("snapshot serial mismatch: expected {expected}, got {got} (RFC 8182 §3.5.2.3)")] - SnapshotSerialMismatch { expected: u64, got: u64 }, - - #[error("delta file hash mismatch (RFC 8182 §3.4.2; RFC 8182 §3.5.1.3)")] - DeltaHashMismatch, - - #[error("delta session_id mismatch: expected {expected}, got {got} (RFC 8182 §3.5.3.3)")] - DeltaSessionIdMismatch { expected: String, got: String }, - - #[error("delta serial mismatch: expected {expected}, got {got} (RFC 8182 §3.5.3.3)")] - DeltaSerialMismatch { expected: u64, got: u64 }, - - #[error("notification serial moved backwards: old={old} new={new} (RFC 8182 §3.4.1)")] - NotificationSerialRollback { old: u64, new: u64 }, - - #[error("delta publish without @hash for existing object: {rsync_uri} (RFC 8182 §3.4.2)")] - DeltaPublishWithoutHashForExisting { rsync_uri: String }, - - #[error( - "delta withdraw/replace target not from this repository server: {rsync_uri} (RFC 8182 §3.4.2)" - )] - DeltaTargetNotFromRepository { rsync_uri: String }, - - #[error("delta withdraw/replace target missing in local cache: {rsync_uri} (RFC 8182 §3.4.2)")] - DeltaTargetMissing { rsync_uri: String }, - - #[error("delta withdraw/replace target hash mismatch: {rsync_uri} (RFC 8182 §3.4.2)")] - DeltaTargetHashMismatch { rsync_uri: String }, - - #[error("publish/@uri missing (RFC 8182 §3.5.2.3)")] - PublishUriMissing, - - #[error("publish element missing base64 content (RFC 8182 §3.5.2.3)")] - PublishContentMissing, - - #[error("publish base64 decode failed (RFC 8182 §3.5.2.3): {0}")] - PublishBase64(String), - - #[error("delta file missing @uri (RFC 8182 §3.5.3.3)")] - DeltaPublishUriMissing, - - #[error("delta file base64 content missing (RFC 8182 §3.5.3.3)")] - DeltaPublishContentMissing, - - #[error("delta file base64 decode failed (RFC 8182 §3.5.3.3): {0}")] - DeltaPublishBase64(String), - - #[error( - "delta file @hash must be hex encoding of SHA-256, got {0} (RFC 8182 §3.5.3.3)" - )] - DeltaPublishHashInvalid(String), - - #[error("delta file missing @uri (RFC 8182 §3.5.3.3)")] - DeltaWithdrawUriMissing, - - #[error("delta file missing @hash (RFC 8182 §3.5.3.3)")] - DeltaWithdrawHashMissing, - - #[error( - "delta file @hash must be hex encoding of SHA-256, got {0} (RFC 8182 §3.5.3.3)" - )] - DeltaWithdrawHashInvalid(String), - - #[error("delta file must not contain text content (RFC 8182 §3.5.3.3)")] - DeltaWithdrawUnexpectedContent, - - #[error("delta file must contain at least one publish/withdraw element (RFC 8182 §3.5.3.3)")] - DeltaNoElements, - - #[error("delta file contains unexpected element <{0}> (RFC 8182 §3.5.3.3)")] - DeltaUnexpectedElement(String), -} - -#[derive(Debug, thiserror::Error)] -pub enum RrdpSyncError { - #[error("{0}")] - Rrdp(#[from] RrdpError), - - #[error("fetch failed: {0}")] - Fetch(String), - - #[error("storage error: {0}")] - Storage(String), -} - -pub type RrdpSyncResult = Result; - -pub trait Fetcher: Send + Sync { - fn fetch(&self, uri: &str) -> Result, String>; - - fn fetch_to_writer(&self, uri: &str, out: &mut dyn Write) -> Result { - let bytes = self.fetch(uri)?; - out.write_all(&bytes) - .map_err(|e| format!("write sink failed: {e}"))?; - Ok(bytes.len() as u64) - } -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct RrdpState { - pub session_id: String, - pub serial: u64, -} - -impl RrdpState { - pub fn encode(&self) -> Result, String> { - serde_cbor::to_vec(self).map_err(|e| e.to_string()) - } - - pub fn decode(bytes: &[u8]) -> Result { - serde_cbor::from_slice(bytes).map_err(|e| e.to_string()) - } -} - -pub(crate) fn load_rrdp_local_state( - store: &RocksStore, - notification_uri: &str, -) -> Result, String> { - if let Some(record) = store - .get_rrdp_source_record(notification_uri) - .map_err(|e| e.to_string())? - { - if let (Some(session_id), Some(serial)) = (record.last_session_id, record.last_serial) { - return Ok(Some(RrdpState { session_id, serial })); - } - } - - Ok(None) -} - -pub(crate) fn persist_rrdp_local_state( - store: &RocksStore, - notification_uri: &str, - state: &RrdpState, - sync_state: RrdpSourceSyncState, - last_snapshot_uri: Option<&str>, - last_snapshot_hash_hex: Option<&str>, -) -> Result<(), String> { - update_rrdp_source_record_on_success( - store, - notification_uri, - state.session_id.as_str(), - state.serial, - sync_state, - last_snapshot_uri, - last_snapshot_hash_hex, - ) -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct NotificationSnapshot { - pub session_id: Uuid, - pub serial: u64, - pub snapshot_uri: String, - pub snapshot_hash_sha256: [u8; 32], -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct NotificationDeltaRef { - pub serial: u64, - pub uri: String, - pub hash_sha256: [u8; 32], -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Notification { - pub session_id: Uuid, - pub serial: u64, - pub snapshot_uri: String, - pub snapshot_hash_sha256: [u8; 32], - /// Deltas referenced by the notification file, sorted by serial ascending. - /// - /// If present, this list is guaranteed to be contiguous and to end at `serial` - /// (RFC 8182 §3.5.1.3). - pub deltas: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum DeltaElement { - Publish { - uri: String, - hash_sha256: Option<[u8; 32]>, - bytes: Vec, - }, - Withdraw { - uri: String, - hash_sha256: [u8; 32], - }, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct DeltaFile { - pub session_id: Uuid, - pub serial: u64, - pub elements: Vec, -} - -pub fn parse_notification(xml: &[u8]) -> Result { - let doc = parse_rrdp_xml(xml)?; - let root = doc.root_element(); - if root.tag_name().name() != "notification" { - return Err(RrdpError::UnexpectedRoot( - root.tag_name().name().to_string(), - )); - } - validate_root_common(&root)?; - - let session_id = parse_uuid_attr(&root, "session_id")?; - let serial = parse_u64_attr(&root, "serial")?; - - let snapshots: Vec<_> = root - .children() - .filter(|n| n.is_element() && n.tag_name().name() == "snapshot") - .collect(); - if snapshots.len() != 1 { - return Err(RrdpError::SnapshotCountInvalid); - } - let snapshot = snapshots[0]; - - let snapshot_uri = snapshot - .attribute("uri") - .ok_or(RrdpError::SnapshotUriMissing)? - .to_string(); - let snapshot_hash_hex = snapshot - .attribute("hash") - .ok_or(RrdpError::SnapshotHashMissing)?; - let snapshot_hash_sha256 = parse_sha256_hex_snapshot(snapshot_hash_hex)?; - - let mut deltas: Vec = Vec::new(); - for d in root - .children() - .filter(|n| n.is_element() && n.tag_name().name() == "delta") - { - let delta_serial = d - .attribute("serial") - .ok_or(RrdpError::DeltaRefSerialMissing)?; - let delta_serial = parse_u64_str(delta_serial)?; - if delta_serial > serial { - return Err(RrdpError::DeltaRefSerialTooHigh { - delta_serial, - notification_serial: serial, - }); - } - let uri = d.attribute("uri").ok_or(RrdpError::DeltaRefUriMissing)?; - let hash = d.attribute("hash").ok_or(RrdpError::DeltaRefHashMissing)?; - let hash_sha256 = parse_sha256_hex_delta_ref(hash)?; - - deltas.push(NotificationDeltaRef { - serial: delta_serial, - uri: uri.to_string(), - hash_sha256, - }); - } - - if !deltas.is_empty() { - let mut seen: std::collections::HashSet = std::collections::HashSet::new(); - let mut min_serial = u64::MAX; - let mut max_serial = 0u64; - for d in &deltas { - if !seen.insert(d.serial) { - return Err(RrdpError::DeltaRefSerialDuplicate(d.serial)); - } - min_serial = min_serial.min(d.serial); - max_serial = max_serial.max(d.serial); - } - if max_serial != serial { - return Err(RrdpError::DeltaRefChainDoesNotEndAtNotificationSerial { - max_delta_serial: max_serial, - notification_serial: serial, - }); - } - for s in min_serial..=serial { - if !seen.contains(&s) { - return Err(RrdpError::DeltaRefChainNotContiguous { - min_serial, - notification_serial: serial, - missing_serial: s, - }); - } - } - deltas.sort_by_key(|d| d.serial); - } - - Ok(Notification { - session_id, - serial, - snapshot_uri, - snapshot_hash_sha256, - deltas, - }) -} - -pub fn parse_notification_snapshot(xml: &[u8]) -> Result { - let n = parse_notification(xml)?; - Ok(NotificationSnapshot { - session_id: n.session_id, - serial: n.serial, - snapshot_uri: n.snapshot_uri, - snapshot_hash_sha256: n.snapshot_hash_sha256, - }) -} - -pub fn parse_delta_file(xml: &[u8]) -> Result { - let doc = parse_rrdp_xml(xml)?; - let root = doc.root_element(); - if root.tag_name().name() != "delta" { - return Err(RrdpError::UnexpectedRoot( - root.tag_name().name().to_string(), - )); - } - validate_root_common(&root)?; - - let session_id = parse_uuid_attr(&root, "session_id")?; - let serial = parse_u64_attr(&root, "serial")?; - - let mut elements: Vec = Vec::new(); - for child in root.children().filter(|n| n.is_element()) { - match child.tag_name().name() { - "publish" => { - let uri = child - .attribute("uri") - .ok_or(RrdpError::DeltaPublishUriMissing)? - .to_string(); - let hash_sha256 = child - .attribute("hash") - .map(parse_sha256_hex_delta_publish) - .transpose()?; - - let content_b64 = - collect_element_text(&child).ok_or(RrdpError::DeltaPublishContentMissing)?; - let content_b64 = strip_all_ascii_whitespace(&content_b64); - if content_b64.is_empty() { - return Err(RrdpError::DeltaPublishContentMissing); - } - let bytes = base64::engine::general_purpose::STANDARD - .decode(content_b64.as_bytes()) - .map_err(|e| RrdpError::DeltaPublishBase64(e.to_string()))?; - - elements.push(DeltaElement::Publish { - uri, - hash_sha256, - bytes, - }); - } - "withdraw" => { - let uri = child - .attribute("uri") - .ok_or(RrdpError::DeltaWithdrawUriMissing)? - .to_string(); - let hash = child - .attribute("hash") - .ok_or(RrdpError::DeltaWithdrawHashMissing)?; - let hash_sha256 = parse_sha256_hex_delta_withdraw(hash)?; - - if let Some(s) = collect_element_text(&child) { - if !strip_all_ascii_whitespace(&s).is_empty() { - return Err(RrdpError::DeltaWithdrawUnexpectedContent); - } - } - - elements.push(DeltaElement::Withdraw { uri, hash_sha256 }); - } - other => return Err(RrdpError::DeltaUnexpectedElement(other.to_string())), - } - } - - if elements.is_empty() { - return Err(RrdpError::DeltaNoElements); - } - - Ok(DeltaFile { - session_id, - serial, - elements, - }) -} - -pub fn sync_from_notification_snapshot( - store: &RocksStore, - notification_uri: &str, - notification_xml: &[u8], - fetcher: &dyn Fetcher, -) -> RrdpSyncResult { - sync_from_notification_snapshot_inner( - store, - notification_uri, - notification_xml, - fetcher, - None, - None, - ) -} - -pub fn sync_from_notification_snapshot_with_timing( - store: &RocksStore, - notification_uri: &str, - notification_xml: &[u8], - fetcher: &dyn Fetcher, - timing: Option<&TimingHandle>, -) -> RrdpSyncResult { - sync_from_notification_snapshot_inner( - store, - notification_uri, - notification_xml, - fetcher, - timing, - None, - ) -} - -pub fn sync_from_notification_snapshot_with_timing_and_download_log( - store: &RocksStore, - notification_uri: &str, - notification_xml: &[u8], - fetcher: &dyn Fetcher, - timing: Option<&TimingHandle>, - download_log: Option<&DownloadLogHandle>, -) -> RrdpSyncResult { - sync_from_notification_snapshot_inner( - store, - notification_uri, - notification_xml, - fetcher, - timing, - download_log, - ) -} - -fn sync_from_notification_snapshot_inner( - store: &RocksStore, - notification_uri: &str, - notification_xml: &[u8], - fetcher: &dyn Fetcher, - timing: Option<&TimingHandle>, - download_log: Option<&DownloadLogHandle>, -) -> RrdpSyncResult { - let _parse_step = timing - .as_ref() - .map(|t| t.span_rrdp_repo_step(notification_uri, "parse_notification_snapshot")); - let _parse_total = timing - .as_ref() - .map(|t| t.span_phase("rrdp_parse_notification_total")); - let notif = parse_notification_snapshot(notification_xml)?; - drop(_parse_step); - drop(_parse_total); - - let _fetch_step = timing - .as_ref() - .map(|t| t.span_rrdp_repo_step(notification_uri, "fetch_snapshot")); - let _fetch_total = timing - .as_ref() - .map(|t| t.span_phase("rrdp_fetch_snapshot_total")); - let mut dl_span = download_log - .map(|dl| dl.span_download(AuditDownloadKind::RrdpSnapshot, ¬if.snapshot_uri)); - let (snapshot_file, _snapshot_bytes) = match fetch_snapshot_into_tempfile( - fetcher, - ¬if.snapshot_uri, - ¬if.snapshot_hash_sha256, - ) { - Ok(v) => { - if let Some(t) = timing.as_ref() { - t.record_count("rrdp_snapshot_fetch_ok_total", 1); - t.record_count("rrdp_snapshot_bytes_total", v.1); - } - if let Some(s) = dl_span.as_mut() { - s.set_bytes(v.1); - s.set_ok(); - } - v - } - Err(RrdpSyncError::Fetch(e)) => { - if let Some(t) = timing.as_ref() { - t.record_count("rrdp_snapshot_fetch_fail_total", 1); - } - if let Some(s) = dl_span.as_mut() { - s.set_err(e.clone()); - } - return Err(RrdpSyncError::Fetch(e)); - } - Err(e) => return Err(e), - }; - drop(_fetch_step); - drop(_fetch_total); - - let _apply_step = timing - .as_ref() - .map(|t| t.span_rrdp_repo_step(notification_uri, "apply_snapshot")); - let _apply_total = timing - .as_ref() - .map(|t| t.span_phase("rrdp_apply_snapshot_total")); - let published = apply_snapshot_from_bufread( - store, - notification_uri, - None, - std::io::BufReader::new( - snapshot_file - .reopen() - .map_err(|e| RrdpSyncError::Fetch(format!("tempfile reopen failed: {e}")))?, - ), - notif.session_id, - notif.serial, - )?; - if let Some(t) = timing.as_ref() { - t.record_count("rrdp_snapshot_objects_applied_total", published as u64); - } - drop(_apply_step); - drop(_apply_total); - - let _write_state_step = timing - .as_ref() - .map(|t| t.span_rrdp_repo_step(notification_uri, "write_state")); - let _write_state_total = timing - .as_ref() - .map(|t| t.span_phase("rrdp_write_state_total")); - let state = RrdpState { - session_id: notif.session_id.to_string(), - serial: notif.serial, - }; - persist_rrdp_local_state( - store, - notification_uri, - &state, - RrdpSourceSyncState::SnapshotOnly, - Some(¬if.snapshot_uri), - Some(&hex::encode(notif.snapshot_hash_sha256)), - ) - .map_err(RrdpSyncError::Storage)?; - drop(_write_state_step); - drop(_write_state_total); - - Ok(published) -} - -pub fn sync_from_notification( - store: &RocksStore, - notification_uri: &str, - notification_xml: &[u8], - fetcher: &dyn Fetcher, -) -> RrdpSyncResult { - sync_from_notification_inner( - store, - notification_uri, - None, - notification_xml, - fetcher, - None, - None, - ) -} - -pub fn sync_from_notification_with_timing( - store: &RocksStore, - notification_uri: &str, - notification_xml: &[u8], - fetcher: &dyn Fetcher, - timing: Option<&TimingHandle>, -) -> RrdpSyncResult { - sync_from_notification_inner( - store, - notification_uri, - None, - notification_xml, - fetcher, - timing, - None, - ) -} - -pub fn sync_from_notification_with_timing_and_download_log( - store: &RocksStore, - notification_uri: &str, - current_repo_index: Option<&CurrentRepoIndexHandle>, - notification_xml: &[u8], - fetcher: &dyn Fetcher, - timing: Option<&TimingHandle>, - download_log: Option<&DownloadLogHandle>, -) -> RrdpSyncResult { - sync_from_notification_inner( - store, - notification_uri, - current_repo_index, - notification_xml, - fetcher, - timing, - download_log, - ) -} - -fn sync_from_notification_inner( - store: &RocksStore, - notification_uri: &str, - current_repo_index: Option<&CurrentRepoIndexHandle>, - notification_xml: &[u8], - fetcher: &dyn Fetcher, - timing: Option<&TimingHandle>, - download_log: Option<&DownloadLogHandle>, -) -> RrdpSyncResult { - let _parse_step = timing - .as_ref() - .map(|t| t.span_rrdp_repo_step(notification_uri, "parse_notification")); - let _parse_total = timing - .as_ref() - .map(|t| t.span_phase("rrdp_parse_notification_total")); - let notif = parse_notification(notification_xml)?; - drop(_parse_step); - drop(_parse_total); - if let Some(t) = timing.as_ref() { - t.record_count( - "rrdp_notification_delta_refs_total", - notif.deltas.len() as u64, - ); - } - - let _read_state_step = timing - .as_ref() - .map(|t| t.span_rrdp_repo_step(notification_uri, "read_state")); - let _read_state_total = timing - .as_ref() - .map(|t| t.span_phase("rrdp_read_state_total")); - let state = load_rrdp_local_state(store, notification_uri).map_err(RrdpSyncError::Storage)?; - drop(_read_state_step); - drop(_read_state_total); - - let same_session_state = state - .as_ref() - .filter(|s| s.session_id == notif.session_id.to_string()); - - if let Some(s) = same_session_state { - if s.serial == notif.serial { - let _hydrate_step = timing - .as_ref() - .map(|t| t.span_rrdp_repo_step(notification_uri, "hydrate_current_index")); - let _hydrate_total = timing - .as_ref() - .map(|t| t.span_phase("rrdp_hydrate_current_index_total")); - let hydrated = hydrate_current_repo_index_from_rrdp_members( - store, - notification_uri, - current_repo_index, - )?; - drop(_hydrate_step); - drop(_hydrate_total); - if let Some(t) = timing.as_ref() { - t.record_count("rrdp_current_index_hydrated_objects_total", hydrated as u64); - } - return Ok(0); - } - if s.serial > notif.serial { - return Err(RrdpError::NotificationSerialRollback { - old: s.serial, - new: notif.serial, - } - .into()); - } - } - - if let Some(s) = same_session_state { - // RFC 8182 §3.4.1: if session matches, MAY use deltas when a contiguous chain from the - // last processed serial to the current serial can be processed (i.e. deltas cover the gap). - let want_first = s.serial + 1; - let want_last = notif.serial; - - if want_first <= want_last && !notif.deltas.is_empty() { - let min_serial = notif.deltas[0].serial; - let max_serial = notif.deltas[notif.deltas.len() - 1].serial; - - // `parse_notification` guarantees contiguity and max==notif.serial when deltas exist. - if max_serial == notif.serial && want_first >= min_serial { - // Fetch all required delta files first so a network failure doesn't leave us with - // partially applied deltas and no snapshot fallback. - let _fetch_d_step = timing - .as_ref() - .map(|t| t.span_rrdp_repo_step(notification_uri, "fetch_deltas")); - let _fetch_d_total = timing - .as_ref() - .map(|t| t.span_phase("rrdp_fetch_deltas_total")); - let mut fetched: Vec<(u64, [u8; 32], Vec)> = - Vec::with_capacity((want_last - want_first + 1) as usize); - let mut fetch_ok = true; - for serial in want_first..=want_last { - if let Some(t) = timing.as_ref() { - t.record_count("rrdp_delta_fetch_attempted_total", 1); - } - let idx = (serial - min_serial) as usize; - let dref = match notif.deltas.get(idx) { - Some(v) if v.serial == serial => v, - _ => { - fetch_ok = false; - break; - } - }; - - let mut dl_span = download_log - .map(|dl| dl.span_download(AuditDownloadKind::RrdpDelta, &dref.uri)); - match fetcher.fetch(&dref.uri) { - Ok(bytes) => { - if let Some(t) = timing.as_ref() { - t.record_count("rrdp_delta_fetch_ok_total", 1); - t.record_count("rrdp_delta_bytes_total", bytes.len() as u64); - } - if let Some(s) = dl_span.as_mut() { - s.set_bytes(bytes.len() as u64); - s.set_ok(); - } - fetched.push((serial, dref.hash_sha256, bytes)) - } - Err(e) => { - if let Some(t) = timing.as_ref() { - t.record_count("rrdp_delta_fetch_fail_total", 1); - } - if let Some(s) = dl_span.as_mut() { - s.set_err(e.clone()); - } - fetch_ok = false; - break; - } - } - } - drop(_fetch_d_step); - drop(_fetch_d_total); - - if fetch_ok { - let _apply_d_step = timing - .as_ref() - .map(|t| t.span_rrdp_repo_step(notification_uri, "apply_deltas")); - let _apply_d_total = timing - .as_ref() - .map(|t| t.span_phase("rrdp_apply_deltas_total")); - let mut applied_total = 0usize; - let mut ok = true; - for (serial, expected_hash, bytes) in &fetched { - match apply_delta( - store, - notification_uri, - current_repo_index, - bytes.as_slice(), - *expected_hash, - notif.session_id, - *serial, - ) { - Ok(n) => applied_total += n, - Err(_) => { - ok = false; - break; - } - } - } - drop(_apply_d_step); - drop(_apply_d_total); - - if ok { - let _write_state_step = timing - .as_ref() - .map(|t| t.span_rrdp_repo_step(notification_uri, "write_state")); - let _write_state_total = timing - .as_ref() - .map(|t| t.span_phase("rrdp_write_state_total")); - let new_state = RrdpState { - session_id: notif.session_id.to_string(), - serial: notif.serial, - }; - persist_rrdp_local_state( - store, - notification_uri, - &new_state, - RrdpSourceSyncState::DeltaReady, - Some(¬if.snapshot_uri), - Some(&hex::encode(notif.snapshot_hash_sha256)), - ) - .map_err(RrdpSyncError::Storage)?; - drop(_write_state_step); - drop(_write_state_total); - if let Some(t) = timing.as_ref() { - t.record_count("rrdp_delta_ops_applied_total", applied_total as u64); - } - let _hydrate_step = timing.as_ref().map(|t| { - t.span_rrdp_repo_step(notification_uri, "hydrate_current_index") - }); - let _hydrate_total = timing - .as_ref() - .map(|t| t.span_phase("rrdp_hydrate_current_index_total")); - let hydrated = hydrate_current_repo_index_from_rrdp_members( - store, - notification_uri, - current_repo_index, - )?; - drop(_hydrate_step); - drop(_hydrate_total); - if let Some(t) = timing.as_ref() { - t.record_count( - "rrdp_current_index_hydrated_objects_total", - hydrated as u64, - ); - } - return Ok(applied_total); - } - } - } - } - } - - // Snapshot fallback (RFC 8182 §3.4.3). - let _fetch_step = timing - .as_ref() - .map(|t| t.span_rrdp_repo_step(notification_uri, "fetch_snapshot")); - let _fetch_total = timing - .as_ref() - .map(|t| t.span_phase("rrdp_fetch_snapshot_total")); - let mut dl_span = download_log - .map(|dl| dl.span_download(AuditDownloadKind::RrdpSnapshot, ¬if.snapshot_uri)); - let (snapshot_file, _snapshot_bytes) = match fetch_snapshot_into_tempfile( - fetcher, - ¬if.snapshot_uri, - ¬if.snapshot_hash_sha256, - ) { - Ok(v) => { - if let Some(t) = timing.as_ref() { - t.record_count("rrdp_snapshot_fetch_ok_total", 1); - t.record_count("rrdp_snapshot_bytes_total", v.1); - } - if let Some(s) = dl_span.as_mut() { - s.set_bytes(v.1); - s.set_ok(); - } - v - } - Err(RrdpSyncError::Fetch(e)) => { - if let Some(t) = timing.as_ref() { - t.record_count("rrdp_snapshot_fetch_fail_total", 1); - } - if let Some(s) = dl_span.as_mut() { - s.set_err(e.clone()); - } - return Err(RrdpSyncError::Fetch(e)); - } - Err(e) => return Err(e), - }; - drop(_fetch_step); - drop(_fetch_total); - - let _apply_step = timing - .as_ref() - .map(|t| t.span_rrdp_repo_step(notification_uri, "apply_snapshot")); - let _apply_total = timing - .as_ref() - .map(|t| t.span_phase("rrdp_apply_snapshot_total")); - let published = apply_snapshot_from_bufread( - store, - notification_uri, - current_repo_index, - std::io::BufReader::new( - snapshot_file - .reopen() - .map_err(|e| RrdpSyncError::Fetch(format!("tempfile reopen failed: {e}")))?, - ), - notif.session_id, - notif.serial, - )?; - if let Some(t) = timing.as_ref() { - t.record_count("rrdp_snapshot_objects_applied_total", published as u64); - } - drop(_apply_step); - drop(_apply_total); - - let _write_state_step = timing - .as_ref() - .map(|t| t.span_rrdp_repo_step(notification_uri, "write_state")); - let _write_state_total = timing - .as_ref() - .map(|t| t.span_phase("rrdp_write_state_total")); - let new_state = RrdpState { - session_id: notif.session_id.to_string(), - serial: notif.serial, - }; - persist_rrdp_local_state( - store, - notification_uri, - &new_state, - RrdpSourceSyncState::SnapshotOnly, - Some(¬if.snapshot_uri), - Some(&hex::encode(notif.snapshot_hash_sha256)), - ) - .map_err(RrdpSyncError::Storage)?; - drop(_write_state_step); - drop(_write_state_total); - - Ok(published) -} - -fn hydrate_current_repo_index_from_rrdp_members( - store: &RocksStore, - notification_uri: &str, - current_repo_index: Option<&CurrentRepoIndexHandle>, -) -> Result { - let Some(index) = current_repo_index else { - return Ok(0); - }; - - let members = store - .list_current_rrdp_source_members(notification_uri) - .map_err(|e| RrdpSyncError::Storage(e.to_string()))?; - if members.is_empty() { - return Ok(0); - } - - let mut entries = Vec::with_capacity(members.len()); - for member in members { - let current_hash = member.current_hash.ok_or_else(|| { - RrdpSyncError::Storage(format!( - "rrdp source member missing current_hash for current object {}", - member.rsync_uri - )) - })?; - entries.push(RepositoryViewEntry { - rsync_uri: member.rsync_uri, - current_hash: Some(current_hash), - repository_source: Some(notification_uri.to_string()), - object_type: member.object_type, - state: RepositoryViewState::Present, - }); - } - - index - .write() - .map_err(|_| RrdpSyncError::Storage("current repo index lock poisoned".to_string()))? - .apply_repository_view_entries(&entries) - .map_err(RrdpSyncError::Storage)?; - Ok(entries.len()) -} - -fn apply_delta( - store: &RocksStore, - notification_uri: &str, - current_repo_index: Option<&CurrentRepoIndexHandle>, - delta_xml: &[u8], - expected_hash_sha256: [u8; 32], - expected_session_id: Uuid, - expected_serial: u64, -) -> Result { - let computed = sha2::Sha256::digest(delta_xml); - if computed.as_slice() != expected_hash_sha256.as_slice() { - return Err(RrdpError::DeltaHashMismatch.into()); - } - - let delta = parse_delta_file(delta_xml)?; - if delta.session_id != expected_session_id { - return Err(RrdpError::DeltaSessionIdMismatch { - expected: expected_session_id.to_string(), - got: delta.session_id.to_string(), - } - .into()); - } - if delta.serial != expected_serial { - return Err(RrdpError::DeltaSerialMismatch { - expected: expected_serial, - got: delta.serial, - } - .into()); - } - - enum DeltaProjectionEffect { - Upsert { - rsync_uri: String, - bytes: Vec, - }, - Delete { - rsync_uri: String, - previous_hash: String, - }, - } - - let session_id = expected_session_id.to_string(); - let mut ops: Vec = Vec::with_capacity(delta.elements.len()); - let mut projection: Vec = Vec::with_capacity(delta.elements.len()); - for e in delta.elements { - match e { - DeltaElement::Publish { - uri, - hash_sha256: Some(old_hash), - bytes, - } => { - let is_member = store - .is_current_rrdp_source_member(notification_uri, uri.as_str()) - .map_err(|e| RrdpSyncError::Storage(e.to_string()))?; - if !is_member { - return Err(RrdpError::DeltaTargetNotFromRepository { rsync_uri: uri }.into()); - } - ensure_rrdp_uri_can_be_owned_by(store, notification_uri, uri.as_str()) - .map_err(RrdpSyncError::Storage)?; - let old_bytes = store - .load_current_object_bytes_by_uri(uri.as_str()) - .map_err(|e| RrdpSyncError::Storage(e.to_string()))? - .ok_or_else(|| RrdpError::DeltaTargetMissing { - rsync_uri: uri.clone(), - })?; - let old_computed = sha2::Sha256::digest(old_bytes.as_slice()); - if old_computed.as_slice() != old_hash.as_slice() { - return Err(RrdpError::DeltaTargetHashMismatch { rsync_uri: uri }.into()); - } - - ops.push(RrdpDeltaOp::Upsert { - rsync_uri: uri.clone(), - bytes: bytes.clone(), - }); - projection.push(DeltaProjectionEffect::Upsert { - rsync_uri: uri, - bytes, - }); - } - DeltaElement::Publish { - uri, - hash_sha256: None, - bytes, - } => { - let is_member = store - .is_current_rrdp_source_member(notification_uri, uri.as_str()) - .map_err(|e| RrdpSyncError::Storage(e.to_string()))?; - if is_member { - return Err( - RrdpError::DeltaPublishWithoutHashForExisting { rsync_uri: uri }.into(), - ); - } - ensure_rrdp_uri_can_be_owned_by(store, notification_uri, uri.as_str()) - .map_err(RrdpSyncError::Storage)?; - ops.push(RrdpDeltaOp::Upsert { - rsync_uri: uri.clone(), - bytes: bytes.clone(), - }); - projection.push(DeltaProjectionEffect::Upsert { - rsync_uri: uri, - bytes, - }); - } - DeltaElement::Withdraw { uri, hash_sha256 } => { - let is_member = store - .is_current_rrdp_source_member(notification_uri, uri.as_str()) - .map_err(|e| RrdpSyncError::Storage(e.to_string()))?; - if !is_member { - return Err(RrdpError::DeltaTargetNotFromRepository { rsync_uri: uri }.into()); - } - ensure_rrdp_uri_can_be_owned_by(store, notification_uri, uri.as_str()) - .map_err(RrdpSyncError::Storage)?; - let old_bytes = store - .load_current_object_bytes_by_uri(uri.as_str()) - .map_err(|e| RrdpSyncError::Storage(e.to_string()))? - .ok_or_else(|| RrdpError::DeltaTargetMissing { - rsync_uri: uri.clone(), - })?; - let old_computed = sha2::Sha256::digest(old_bytes.as_slice()); - if old_computed.as_slice() != hash_sha256.as_slice() { - return Err(RrdpError::DeltaTargetHashMismatch { rsync_uri: uri }.into()); - } - let previous_hash = hex::encode(old_computed); - ops.push(RrdpDeltaOp::Delete { - rsync_uri: uri.clone(), - }); - projection.push(DeltaProjectionEffect::Delete { - rsync_uri: uri, - previous_hash, - }); - } - } - } - - for effect in projection { - match effect { - DeltaProjectionEffect::Upsert { rsync_uri, bytes } => { - let current_hash = - upsert_repo_blob_bytes(store, &bytes).map_err(RrdpSyncError::Storage)?; - put_repository_view_present(store, notification_uri, &rsync_uri, ¤t_hash) - .map_err(RrdpSyncError::Storage)?; - if let Some(index) = current_repo_index { - let entry = build_repository_view_present_entry( - notification_uri, - &rsync_uri, - ¤t_hash, - ); - index - .write() - .map_err(|_| { - RrdpSyncError::Storage("current repo index lock poisoned".to_string()) - })? - .apply_repository_view_entries(&[entry]) - .map_err(RrdpSyncError::Storage)?; - } - put_rrdp_source_member_present( - store, - notification_uri, - &session_id, - expected_serial, - &rsync_uri, - ¤t_hash, - ) - .map_err(RrdpSyncError::Storage)?; - put_rrdp_uri_owner_active( - store, - notification_uri, - &session_id, - expected_serial, - &rsync_uri, - ¤t_hash, - ) - .map_err(RrdpSyncError::Storage)?; - } - DeltaProjectionEffect::Delete { - rsync_uri, - previous_hash, - } => { - put_rrdp_source_member_withdrawn( - store, - notification_uri, - &session_id, - expected_serial, - &rsync_uri, - Some(previous_hash.clone()), - ) - .map_err(RrdpSyncError::Storage)?; - if current_rrdp_owner_is(store, notification_uri, &rsync_uri) - .map_err(RrdpSyncError::Storage)? - { - put_repository_view_withdrawn( - store, - notification_uri, - &rsync_uri, - Some(previous_hash.clone()), - ) - .map_err(RrdpSyncError::Storage)?; - if let Some(index) = current_repo_index { - let entry = build_repository_view_withdrawn_entry( - notification_uri, - &rsync_uri, - Some(previous_hash.clone()), - ); - index - .write() - .map_err(|_| { - RrdpSyncError::Storage( - "current repo index lock poisoned".to_string(), - ) - })? - .apply_repository_view_entries(&[entry]) - .map_err(RrdpSyncError::Storage)?; - } - put_rrdp_uri_owner_withdrawn( - store, - notification_uri, - &session_id, - expected_serial, - &rsync_uri, - Some(previous_hash), - ) - .map_err(RrdpSyncError::Storage)?; - } - } - } - } - - Ok(ops.len()) -} - -#[cfg(test)] -use snapshot_apply::apply_snapshot; -use snapshot_apply::{apply_snapshot_from_bufread, fetch_snapshot_into_tempfile}; - -fn parse_rrdp_xml(xml: &[u8]) -> Result, RrdpError> { - if xml.iter().any(|&b| b > 0x7F) { - return Err(RrdpError::NotAscii); - } - let s = std::str::from_utf8(xml).map_err(|e| RrdpError::Xml(e.to_string()))?; - roxmltree::Document::parse(s).map_err(|e| RrdpError::Xml(e.to_string())) -} - -fn validate_root_common(root: &roxmltree::Node<'_, '_>) -> Result<(), RrdpError> { - let ns = root.default_namespace().unwrap_or("").to_string(); - if ns != RRDP_XMLNS { - return Err(RrdpError::InvalidNamespace(ns)); - } - - let version = root.attribute("version").unwrap_or(""); - if version != "1" { - return Err(RrdpError::InvalidVersion(version.to_string())); - } - - Ok(()) -} - -fn parse_uuid_attr(root: &roxmltree::Node<'_, '_>, name: &'static str) -> Result { - let s = root.attribute(name).unwrap_or(""); - Uuid::parse_str(s).map_err(|_e| RrdpError::InvalidSessionId(s.to_string())) -} - -fn parse_u64_attr(root: &roxmltree::Node<'_, '_>, name: &'static str) -> Result { - let s = root.attribute(name).unwrap_or(""); - parse_u64_str(s) -} - -fn parse_u64_str(s: &str) -> Result { - let v = s - .parse::() - .map_err(|_e| RrdpError::InvalidSerial(s.to_string()))?; - if v == 0 { - return Err(RrdpError::InvalidSerial(s.to_string())); - } - Ok(v) -} - -fn parse_sha256_hex_impl(s: &str, invalid: fn(String) -> RrdpError) -> Result<[u8; 32], RrdpError> { - let bytes = hex::decode(s).map_err(|_e| invalid(s.to_string()))?; - if bytes.len() != 32 { - return Err(invalid(s.to_string())); - } - let mut out = [0u8; 32]; - out.copy_from_slice(&bytes); - Ok(out) -} - -fn parse_sha256_hex_snapshot(s: &str) -> Result<[u8; 32], RrdpError> { - parse_sha256_hex_impl(s, RrdpError::SnapshotHashInvalid) -} - -fn parse_sha256_hex_delta_ref(s: &str) -> Result<[u8; 32], RrdpError> { - parse_sha256_hex_impl(s, RrdpError::DeltaRefHashInvalid) -} - -fn parse_sha256_hex_delta_publish(s: &str) -> Result<[u8; 32], RrdpError> { - parse_sha256_hex_impl(s, RrdpError::DeltaPublishHashInvalid) -} - -fn parse_sha256_hex_delta_withdraw(s: &str) -> Result<[u8; 32], RrdpError> { - parse_sha256_hex_impl(s, RrdpError::DeltaWithdrawHashInvalid) -} - -fn collect_element_text(node: &roxmltree::Node<'_, '_>) -> Option { - let mut out = String::new(); - for child in node.children() { - if child.is_text() { - out.push_str(child.text().unwrap_or("")); - } - } - if out.is_empty() { None } else { Some(out) } -} - -fn strip_all_ascii_whitespace(s: &str) -> String { - s.chars().filter(|c| !c.is_ascii_whitespace()).collect() -} +include!("rrdp/models_and_parsing.rs"); +include!("rrdp/snapshot_sync.rs"); +include!("rrdp/notification_sync.rs"); +include!("rrdp/delta.rs"); +include!("rrdp/parse_helpers.rs"); #[cfg(test)] #[path = "rrdp/tests.rs"] diff --git a/crates/panda-rpki-validator/src/sync/rrdp/delta.rs b/crates/panda-rpki-validator/src/sync/rrdp/delta.rs new file mode 100644 index 0000000..1845c3e --- /dev/null +++ b/crates/panda-rpki-validator/src/sync/rrdp/delta.rs @@ -0,0 +1,272 @@ +// Current-repository hydration and delta application. + +fn hydrate_current_repo_index_from_rrdp_members( + store: &RocksStore, + notification_uri: &str, + current_repo_index: Option<&CurrentRepoIndexHandle>, +) -> Result { + let Some(index) = current_repo_index else { + return Ok(0); + }; + + let members = store + .list_current_rrdp_source_members(notification_uri) + .map_err(|e| RrdpSyncError::Storage(e.to_string()))?; + if members.is_empty() { + return Ok(0); + } + + let mut entries = Vec::with_capacity(members.len()); + for member in members { + let current_hash = member.current_hash.ok_or_else(|| { + RrdpSyncError::Storage(format!( + "rrdp source member missing current_hash for current object {}", + member.rsync_uri + )) + })?; + entries.push(RepositoryViewEntry { + rsync_uri: member.rsync_uri, + current_hash: Some(current_hash), + repository_source: Some(notification_uri.to_string()), + object_type: member.object_type, + state: RepositoryViewState::Present, + }); + } + + index + .write() + .map_err(|_| RrdpSyncError::Storage("current repo index lock poisoned".to_string()))? + .apply_repository_view_entries(&entries) + .map_err(RrdpSyncError::Storage)?; + Ok(entries.len()) +} + +fn apply_delta( + store: &RocksStore, + notification_uri: &str, + current_repo_index: Option<&CurrentRepoIndexHandle>, + delta_xml: &[u8], + expected_hash_sha256: [u8; 32], + expected_session_id: Uuid, + expected_serial: u64, +) -> Result { + let computed = sha2::Sha256::digest(delta_xml); + if computed.as_slice() != expected_hash_sha256.as_slice() { + return Err(RrdpError::DeltaHashMismatch.into()); + } + + let delta = parse_delta_file(delta_xml)?; + if delta.session_id != expected_session_id { + return Err(RrdpError::DeltaSessionIdMismatch { + expected: expected_session_id.to_string(), + got: delta.session_id.to_string(), + } + .into()); + } + if delta.serial != expected_serial { + return Err(RrdpError::DeltaSerialMismatch { + expected: expected_serial, + got: delta.serial, + } + .into()); + } + + enum DeltaProjectionEffect { + Upsert { + rsync_uri: String, + bytes: Vec, + }, + Delete { + rsync_uri: String, + previous_hash: String, + }, + } + + let session_id = expected_session_id.to_string(); + let mut ops: Vec = Vec::with_capacity(delta.elements.len()); + let mut projection: Vec = Vec::with_capacity(delta.elements.len()); + for e in delta.elements { + match e { + DeltaElement::Publish { + uri, + hash_sha256: Some(old_hash), + bytes, + } => { + let is_member = store + .is_current_rrdp_source_member(notification_uri, uri.as_str()) + .map_err(|e| RrdpSyncError::Storage(e.to_string()))?; + if !is_member { + return Err(RrdpError::DeltaTargetNotFromRepository { rsync_uri: uri }.into()); + } + ensure_rrdp_uri_can_be_owned_by(store, notification_uri, uri.as_str()) + .map_err(RrdpSyncError::Storage)?; + let old_bytes = store + .load_current_object_bytes_by_uri(uri.as_str()) + .map_err(|e| RrdpSyncError::Storage(e.to_string()))? + .ok_or_else(|| RrdpError::DeltaTargetMissing { + rsync_uri: uri.clone(), + })?; + let old_computed = sha2::Sha256::digest(old_bytes.as_slice()); + if old_computed.as_slice() != old_hash.as_slice() { + return Err(RrdpError::DeltaTargetHashMismatch { rsync_uri: uri }.into()); + } + + ops.push(RrdpDeltaOp::Upsert { + rsync_uri: uri.clone(), + bytes: bytes.clone(), + }); + projection.push(DeltaProjectionEffect::Upsert { + rsync_uri: uri, + bytes, + }); + } + DeltaElement::Publish { + uri, + hash_sha256: None, + bytes, + } => { + let is_member = store + .is_current_rrdp_source_member(notification_uri, uri.as_str()) + .map_err(|e| RrdpSyncError::Storage(e.to_string()))?; + if is_member { + return Err( + RrdpError::DeltaPublishWithoutHashForExisting { rsync_uri: uri }.into(), + ); + } + ensure_rrdp_uri_can_be_owned_by(store, notification_uri, uri.as_str()) + .map_err(RrdpSyncError::Storage)?; + ops.push(RrdpDeltaOp::Upsert { + rsync_uri: uri.clone(), + bytes: bytes.clone(), + }); + projection.push(DeltaProjectionEffect::Upsert { + rsync_uri: uri, + bytes, + }); + } + DeltaElement::Withdraw { uri, hash_sha256 } => { + let is_member = store + .is_current_rrdp_source_member(notification_uri, uri.as_str()) + .map_err(|e| RrdpSyncError::Storage(e.to_string()))?; + if !is_member { + return Err(RrdpError::DeltaTargetNotFromRepository { rsync_uri: uri }.into()); + } + ensure_rrdp_uri_can_be_owned_by(store, notification_uri, uri.as_str()) + .map_err(RrdpSyncError::Storage)?; + let old_bytes = store + .load_current_object_bytes_by_uri(uri.as_str()) + .map_err(|e| RrdpSyncError::Storage(e.to_string()))? + .ok_or_else(|| RrdpError::DeltaTargetMissing { + rsync_uri: uri.clone(), + })?; + let old_computed = sha2::Sha256::digest(old_bytes.as_slice()); + if old_computed.as_slice() != hash_sha256.as_slice() { + return Err(RrdpError::DeltaTargetHashMismatch { rsync_uri: uri }.into()); + } + let previous_hash = hex::encode(old_computed); + ops.push(RrdpDeltaOp::Delete { + rsync_uri: uri.clone(), + }); + projection.push(DeltaProjectionEffect::Delete { + rsync_uri: uri, + previous_hash, + }); + } + } + } + + for effect in projection { + match effect { + DeltaProjectionEffect::Upsert { rsync_uri, bytes } => { + let current_hash = + upsert_repo_blob_bytes(store, &bytes).map_err(RrdpSyncError::Storage)?; + put_repository_view_present(store, notification_uri, &rsync_uri, ¤t_hash) + .map_err(RrdpSyncError::Storage)?; + if let Some(index) = current_repo_index { + let entry = build_repository_view_present_entry( + notification_uri, + &rsync_uri, + ¤t_hash, + ); + index + .write() + .map_err(|_| { + RrdpSyncError::Storage("current repo index lock poisoned".to_string()) + })? + .apply_repository_view_entries(&[entry]) + .map_err(RrdpSyncError::Storage)?; + } + put_rrdp_source_member_present( + store, + notification_uri, + &session_id, + expected_serial, + &rsync_uri, + ¤t_hash, + ) + .map_err(RrdpSyncError::Storage)?; + put_rrdp_uri_owner_active( + store, + notification_uri, + &session_id, + expected_serial, + &rsync_uri, + ¤t_hash, + ) + .map_err(RrdpSyncError::Storage)?; + } + DeltaProjectionEffect::Delete { + rsync_uri, + previous_hash, + } => { + put_rrdp_source_member_withdrawn( + store, + notification_uri, + &session_id, + expected_serial, + &rsync_uri, + Some(previous_hash.clone()), + ) + .map_err(RrdpSyncError::Storage)?; + if current_rrdp_owner_is(store, notification_uri, &rsync_uri) + .map_err(RrdpSyncError::Storage)? + { + put_repository_view_withdrawn( + store, + notification_uri, + &rsync_uri, + Some(previous_hash.clone()), + ) + .map_err(RrdpSyncError::Storage)?; + if let Some(index) = current_repo_index { + let entry = build_repository_view_withdrawn_entry( + notification_uri, + &rsync_uri, + Some(previous_hash.clone()), + ); + index + .write() + .map_err(|_| { + RrdpSyncError::Storage( + "current repo index lock poisoned".to_string(), + ) + })? + .apply_repository_view_entries(&[entry]) + .map_err(RrdpSyncError::Storage)?; + } + put_rrdp_uri_owner_withdrawn( + store, + notification_uri, + &session_id, + expected_serial, + &rsync_uri, + Some(previous_hash), + ) + .map_err(RrdpSyncError::Storage)?; + } + } + } + } + + Ok(ops.len()) +} diff --git a/crates/panda-rpki-validator/src/sync/rrdp/models_and_parsing.rs b/crates/panda-rpki-validator/src/sync/rrdp/models_and_parsing.rs new file mode 100644 index 0000000..b52129a --- /dev/null +++ b/crates/panda-rpki-validator/src/sync/rrdp/models_and_parsing.rs @@ -0,0 +1,461 @@ +// RRDP error/state models and notification/delta parsing. + +#[derive(Debug, thiserror::Error)] +pub enum RrdpError { + #[error("RRDP XML must be US-ASCII encoded (RFC 8182 §3.5.1.3, §3.5.2.3), got non-ASCII bytes")] + NotAscii, + + #[error("RRDP XML parse error: {0} (RFC 8182 §3.5.1.3, §3.5.2.3)")] + Xml(String), + + #[error( + "RRDP root element must be , , or , got <{0}> (RFC 8182 §3.5.1.3, §3.5.2.3, §3.5.3.3)" + )] + UnexpectedRoot(String), + + #[error("RRDP XML namespace must be {RRDP_XMLNS}, got {0} (RFC 8182 §3.5.1.3, §3.5.2.3)")] + InvalidNamespace(String), + + #[error("RRDP version must be 1, got {0} (RFC 8182 §3.5.1.3, §3.5.2.3)")] + InvalidVersion(String), + + #[error("RRDP session_id invalid UUID: {0} (RFC 8182 §3.5.1.3)")] + InvalidSessionId(String), + + #[error("RRDP serial invalid unsigned integer: {0} (RFC 8182 §3.5.1.3)")] + InvalidSerial(String), + + #[error("notification must contain exactly one element (RFC 8182 §3.5.1.3)")] + SnapshotCountInvalid, + + #[error("snapshot/@uri missing (RFC 8182 §3.5.1.3)")] + SnapshotUriMissing, + + #[error("snapshot/@hash missing (RFC 8182 §3.5.1.3)")] + SnapshotHashMissing, + + #[error("snapshot/@hash must be hex encoding of SHA-256, got {0} (RFC 8182 §3.5.1.3)")] + SnapshotHashInvalid(String), + + #[error("delta/@serial missing in notification (RFC 8182 §3.5.1.3)")] + DeltaRefSerialMissing, + + #[error("delta/@uri missing in notification (RFC 8182 §3.5.1.3)")] + DeltaRefUriMissing, + + #[error("delta/@hash missing in notification (RFC 8182 §3.5.1.3)")] + DeltaRefHashMissing, + + #[error("delta/@hash must be hex encoding of SHA-256, got {0} (RFC 8182 §3.5.1.3)")] + DeltaRefHashInvalid(String), + + #[error("delta/@serial duplicates in notification: {0} (RFC 8182 §3.5.1.3)")] + DeltaRefSerialDuplicate(u64), + + #[error( + "delta/@serial must be <= notification/@serial: delta={delta_serial} notification={notification_serial} (RFC 8182 §3.5.1.3)" + )] + DeltaRefSerialTooHigh { + delta_serial: u64, + notification_serial: u64, + }, + + #[error( + "notification contains deltas but does not end at notification/@serial: max_delta={max_delta_serial} notification={notification_serial} (RFC 8182 §3.5.1.3)" + )] + DeltaRefChainDoesNotEndAtNotificationSerial { + max_delta_serial: u64, + notification_serial: u64, + }, + + #[error( + "notification delta chain not contiguous: missing serial={missing_serial} (range {min_serial}..{notification_serial}) (RFC 8182 §3.5.1.3)" + )] + DeltaRefChainNotContiguous { + min_serial: u64, + notification_serial: u64, + missing_serial: u64, + }, + + #[error("snapshot file hash mismatch (RFC 8182 §3.5.1.3)")] + SnapshotHashMismatch, + + #[error("snapshot session_id mismatch: expected {expected}, got {got} (RFC 8182 §3.5.2.3)")] + SnapshotSessionIdMismatch { expected: String, got: String }, + + #[error("snapshot serial mismatch: expected {expected}, got {got} (RFC 8182 §3.5.2.3)")] + SnapshotSerialMismatch { expected: u64, got: u64 }, + + #[error("delta file hash mismatch (RFC 8182 §3.4.2; RFC 8182 §3.5.1.3)")] + DeltaHashMismatch, + + #[error("delta session_id mismatch: expected {expected}, got {got} (RFC 8182 §3.5.3.3)")] + DeltaSessionIdMismatch { expected: String, got: String }, + + #[error("delta serial mismatch: expected {expected}, got {got} (RFC 8182 §3.5.3.3)")] + DeltaSerialMismatch { expected: u64, got: u64 }, + + #[error("notification serial moved backwards: old={old} new={new} (RFC 8182 §3.4.1)")] + NotificationSerialRollback { old: u64, new: u64 }, + + #[error("delta publish without @hash for existing object: {rsync_uri} (RFC 8182 §3.4.2)")] + DeltaPublishWithoutHashForExisting { rsync_uri: String }, + + #[error( + "delta withdraw/replace target not from this repository server: {rsync_uri} (RFC 8182 §3.4.2)" + )] + DeltaTargetNotFromRepository { rsync_uri: String }, + + #[error("delta withdraw/replace target missing in local cache: {rsync_uri} (RFC 8182 §3.4.2)")] + DeltaTargetMissing { rsync_uri: String }, + + #[error("delta withdraw/replace target hash mismatch: {rsync_uri} (RFC 8182 §3.4.2)")] + DeltaTargetHashMismatch { rsync_uri: String }, + + #[error("publish/@uri missing (RFC 8182 §3.5.2.3)")] + PublishUriMissing, + + #[error("publish element missing base64 content (RFC 8182 §3.5.2.3)")] + PublishContentMissing, + + #[error("publish base64 decode failed (RFC 8182 §3.5.2.3): {0}")] + PublishBase64(String), + + #[error("delta file missing @uri (RFC 8182 §3.5.3.3)")] + DeltaPublishUriMissing, + + #[error("delta file base64 content missing (RFC 8182 §3.5.3.3)")] + DeltaPublishContentMissing, + + #[error("delta file base64 decode failed (RFC 8182 §3.5.3.3): {0}")] + DeltaPublishBase64(String), + + #[error( + "delta file @hash must be hex encoding of SHA-256, got {0} (RFC 8182 §3.5.3.3)" + )] + DeltaPublishHashInvalid(String), + + #[error("delta file missing @uri (RFC 8182 §3.5.3.3)")] + DeltaWithdrawUriMissing, + + #[error("delta file missing @hash (RFC 8182 §3.5.3.3)")] + DeltaWithdrawHashMissing, + + #[error( + "delta file @hash must be hex encoding of SHA-256, got {0} (RFC 8182 §3.5.3.3)" + )] + DeltaWithdrawHashInvalid(String), + + #[error("delta file must not contain text content (RFC 8182 §3.5.3.3)")] + DeltaWithdrawUnexpectedContent, + + #[error("delta file must contain at least one publish/withdraw element (RFC 8182 §3.5.3.3)")] + DeltaNoElements, + + #[error("delta file contains unexpected element <{0}> (RFC 8182 §3.5.3.3)")] + DeltaUnexpectedElement(String), +} + +#[derive(Debug, thiserror::Error)] +pub enum RrdpSyncError { + #[error("{0}")] + Rrdp(#[from] RrdpError), + + #[error("fetch failed: {0}")] + Fetch(String), + + #[error("storage error: {0}")] + Storage(String), +} + +pub type RrdpSyncResult = Result; + +pub trait Fetcher: Send + Sync { + fn fetch(&self, uri: &str) -> Result, String>; + + fn fetch_to_writer(&self, uri: &str, out: &mut dyn Write) -> Result { + let bytes = self.fetch(uri)?; + out.write_all(&bytes) + .map_err(|e| format!("write sink failed: {e}"))?; + Ok(bytes.len() as u64) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RrdpState { + pub session_id: String, + pub serial: u64, +} + +impl RrdpState { + pub fn encode(&self) -> Result, String> { + serde_cbor::to_vec(self).map_err(|e| e.to_string()) + } + + pub fn decode(bytes: &[u8]) -> Result { + serde_cbor::from_slice(bytes).map_err(|e| e.to_string()) + } +} + +pub(crate) fn load_rrdp_local_state( + store: &RocksStore, + notification_uri: &str, +) -> Result, String> { + if let Some(record) = store + .get_rrdp_source_record(notification_uri) + .map_err(|e| e.to_string())? + { + if let (Some(session_id), Some(serial)) = (record.last_session_id, record.last_serial) { + return Ok(Some(RrdpState { session_id, serial })); + } + } + + Ok(None) +} + +pub(crate) fn persist_rrdp_local_state( + store: &RocksStore, + notification_uri: &str, + state: &RrdpState, + sync_state: RrdpSourceSyncState, + last_snapshot_uri: Option<&str>, + last_snapshot_hash_hex: Option<&str>, +) -> Result<(), String> { + update_rrdp_source_record_on_success( + store, + notification_uri, + state.session_id.as_str(), + state.serial, + sync_state, + last_snapshot_uri, + last_snapshot_hash_hex, + ) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NotificationSnapshot { + pub session_id: Uuid, + pub serial: u64, + pub snapshot_uri: String, + pub snapshot_hash_sha256: [u8; 32], +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NotificationDeltaRef { + pub serial: u64, + pub uri: String, + pub hash_sha256: [u8; 32], +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Notification { + pub session_id: Uuid, + pub serial: u64, + pub snapshot_uri: String, + pub snapshot_hash_sha256: [u8; 32], + /// Deltas referenced by the notification file, sorted by serial ascending. + /// + /// If present, this list is guaranteed to be contiguous and to end at `serial` + /// (RFC 8182 §3.5.1.3). + pub deltas: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DeltaElement { + Publish { + uri: String, + hash_sha256: Option<[u8; 32]>, + bytes: Vec, + }, + Withdraw { + uri: String, + hash_sha256: [u8; 32], + }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DeltaFile { + pub session_id: Uuid, + pub serial: u64, + pub elements: Vec, +} + +pub fn parse_notification(xml: &[u8]) -> Result { + let doc = parse_rrdp_xml(xml)?; + let root = doc.root_element(); + if root.tag_name().name() != "notification" { + return Err(RrdpError::UnexpectedRoot( + root.tag_name().name().to_string(), + )); + } + validate_root_common(&root)?; + + let session_id = parse_uuid_attr(&root, "session_id")?; + let serial = parse_u64_attr(&root, "serial")?; + + let snapshots: Vec<_> = root + .children() + .filter(|n| n.is_element() && n.tag_name().name() == "snapshot") + .collect(); + if snapshots.len() != 1 { + return Err(RrdpError::SnapshotCountInvalid); + } + let snapshot = snapshots[0]; + + let snapshot_uri = snapshot + .attribute("uri") + .ok_or(RrdpError::SnapshotUriMissing)? + .to_string(); + let snapshot_hash_hex = snapshot + .attribute("hash") + .ok_or(RrdpError::SnapshotHashMissing)?; + let snapshot_hash_sha256 = parse_sha256_hex_snapshot(snapshot_hash_hex)?; + + let mut deltas: Vec = Vec::new(); + for d in root + .children() + .filter(|n| n.is_element() && n.tag_name().name() == "delta") + { + let delta_serial = d + .attribute("serial") + .ok_or(RrdpError::DeltaRefSerialMissing)?; + let delta_serial = parse_u64_str(delta_serial)?; + if delta_serial > serial { + return Err(RrdpError::DeltaRefSerialTooHigh { + delta_serial, + notification_serial: serial, + }); + } + let uri = d.attribute("uri").ok_or(RrdpError::DeltaRefUriMissing)?; + let hash = d.attribute("hash").ok_or(RrdpError::DeltaRefHashMissing)?; + let hash_sha256 = parse_sha256_hex_delta_ref(hash)?; + + deltas.push(NotificationDeltaRef { + serial: delta_serial, + uri: uri.to_string(), + hash_sha256, + }); + } + + if !deltas.is_empty() { + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + let mut min_serial = u64::MAX; + let mut max_serial = 0u64; + for d in &deltas { + if !seen.insert(d.serial) { + return Err(RrdpError::DeltaRefSerialDuplicate(d.serial)); + } + min_serial = min_serial.min(d.serial); + max_serial = max_serial.max(d.serial); + } + if max_serial != serial { + return Err(RrdpError::DeltaRefChainDoesNotEndAtNotificationSerial { + max_delta_serial: max_serial, + notification_serial: serial, + }); + } + for s in min_serial..=serial { + if !seen.contains(&s) { + return Err(RrdpError::DeltaRefChainNotContiguous { + min_serial, + notification_serial: serial, + missing_serial: s, + }); + } + } + deltas.sort_by_key(|d| d.serial); + } + + Ok(Notification { + session_id, + serial, + snapshot_uri, + snapshot_hash_sha256, + deltas, + }) +} + +pub fn parse_notification_snapshot(xml: &[u8]) -> Result { + let n = parse_notification(xml)?; + Ok(NotificationSnapshot { + session_id: n.session_id, + serial: n.serial, + snapshot_uri: n.snapshot_uri, + snapshot_hash_sha256: n.snapshot_hash_sha256, + }) +} + +pub fn parse_delta_file(xml: &[u8]) -> Result { + let doc = parse_rrdp_xml(xml)?; + let root = doc.root_element(); + if root.tag_name().name() != "delta" { + return Err(RrdpError::UnexpectedRoot( + root.tag_name().name().to_string(), + )); + } + validate_root_common(&root)?; + + let session_id = parse_uuid_attr(&root, "session_id")?; + let serial = parse_u64_attr(&root, "serial")?; + + let mut elements: Vec = Vec::new(); + for child in root.children().filter(|n| n.is_element()) { + match child.tag_name().name() { + "publish" => { + let uri = child + .attribute("uri") + .ok_or(RrdpError::DeltaPublishUriMissing)? + .to_string(); + let hash_sha256 = child + .attribute("hash") + .map(parse_sha256_hex_delta_publish) + .transpose()?; + + let content_b64 = + collect_element_text(&child).ok_or(RrdpError::DeltaPublishContentMissing)?; + let content_b64 = strip_all_ascii_whitespace(&content_b64); + if content_b64.is_empty() { + return Err(RrdpError::DeltaPublishContentMissing); + } + let bytes = base64::engine::general_purpose::STANDARD + .decode(content_b64.as_bytes()) + .map_err(|e| RrdpError::DeltaPublishBase64(e.to_string()))?; + + elements.push(DeltaElement::Publish { + uri, + hash_sha256, + bytes, + }); + } + "withdraw" => { + let uri = child + .attribute("uri") + .ok_or(RrdpError::DeltaWithdrawUriMissing)? + .to_string(); + let hash = child + .attribute("hash") + .ok_or(RrdpError::DeltaWithdrawHashMissing)?; + let hash_sha256 = parse_sha256_hex_delta_withdraw(hash)?; + + if let Some(s) = collect_element_text(&child) { + if !strip_all_ascii_whitespace(&s).is_empty() { + return Err(RrdpError::DeltaWithdrawUnexpectedContent); + } + } + + elements.push(DeltaElement::Withdraw { uri, hash_sha256 }); + } + other => return Err(RrdpError::DeltaUnexpectedElement(other.to_string())), + } + } + + if elements.is_empty() { + return Err(RrdpError::DeltaNoElements); + } + + Ok(DeltaFile { + session_id, + serial, + elements, + }) +} diff --git a/crates/panda-rpki-validator/src/sync/rrdp/notification_sync.rs b/crates/panda-rpki-validator/src/sync/rrdp/notification_sync.rs new file mode 100644 index 0000000..af6b343 --- /dev/null +++ b/crates/panda-rpki-validator/src/sync/rrdp/notification_sync.rs @@ -0,0 +1,357 @@ +// RRDP notification, delta-chain, and state synchronization. + +pub fn sync_from_notification( + store: &RocksStore, + notification_uri: &str, + notification_xml: &[u8], + fetcher: &dyn Fetcher, +) -> RrdpSyncResult { + sync_from_notification_inner( + store, + notification_uri, + None, + notification_xml, + fetcher, + None, + None, + ) +} + +pub fn sync_from_notification_with_timing( + store: &RocksStore, + notification_uri: &str, + notification_xml: &[u8], + fetcher: &dyn Fetcher, + timing: Option<&TimingHandle>, +) -> RrdpSyncResult { + sync_from_notification_inner( + store, + notification_uri, + None, + notification_xml, + fetcher, + timing, + None, + ) +} + +pub fn sync_from_notification_with_timing_and_download_log( + store: &RocksStore, + notification_uri: &str, + current_repo_index: Option<&CurrentRepoIndexHandle>, + notification_xml: &[u8], + fetcher: &dyn Fetcher, + timing: Option<&TimingHandle>, + download_log: Option<&DownloadLogHandle>, +) -> RrdpSyncResult { + sync_from_notification_inner( + store, + notification_uri, + current_repo_index, + notification_xml, + fetcher, + timing, + download_log, + ) +} + +fn sync_from_notification_inner( + store: &RocksStore, + notification_uri: &str, + current_repo_index: Option<&CurrentRepoIndexHandle>, + notification_xml: &[u8], + fetcher: &dyn Fetcher, + timing: Option<&TimingHandle>, + download_log: Option<&DownloadLogHandle>, +) -> RrdpSyncResult { + let _parse_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "parse_notification")); + let _parse_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_parse_notification_total")); + let notif = parse_notification(notification_xml)?; + drop(_parse_step); + drop(_parse_total); + if let Some(t) = timing.as_ref() { + t.record_count( + "rrdp_notification_delta_refs_total", + notif.deltas.len() as u64, + ); + } + + let _read_state_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "read_state")); + let _read_state_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_read_state_total")); + let state = load_rrdp_local_state(store, notification_uri).map_err(RrdpSyncError::Storage)?; + drop(_read_state_step); + drop(_read_state_total); + + let same_session_state = state + .as_ref() + .filter(|s| s.session_id == notif.session_id.to_string()); + + if let Some(s) = same_session_state { + if s.serial == notif.serial { + let _hydrate_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "hydrate_current_index")); + let _hydrate_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_hydrate_current_index_total")); + let hydrated = hydrate_current_repo_index_from_rrdp_members( + store, + notification_uri, + current_repo_index, + )?; + drop(_hydrate_step); + drop(_hydrate_total); + if let Some(t) = timing.as_ref() { + t.record_count("rrdp_current_index_hydrated_objects_total", hydrated as u64); + } + return Ok(0); + } + if s.serial > notif.serial { + return Err(RrdpError::NotificationSerialRollback { + old: s.serial, + new: notif.serial, + } + .into()); + } + } + + if let Some(s) = same_session_state { + // RFC 8182 §3.4.1: if session matches, MAY use deltas when a contiguous chain from the + // last processed serial to the current serial can be processed (i.e. deltas cover the gap). + let want_first = s.serial + 1; + let want_last = notif.serial; + + if want_first <= want_last && !notif.deltas.is_empty() { + let min_serial = notif.deltas[0].serial; + let max_serial = notif.deltas[notif.deltas.len() - 1].serial; + + // `parse_notification` guarantees contiguity and max==notif.serial when deltas exist. + if max_serial == notif.serial && want_first >= min_serial { + // Fetch all required delta files first so a network failure doesn't leave us with + // partially applied deltas and no snapshot fallback. + let _fetch_d_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "fetch_deltas")); + let _fetch_d_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_fetch_deltas_total")); + let mut fetched: Vec<(u64, [u8; 32], Vec)> = + Vec::with_capacity((want_last - want_first + 1) as usize); + let mut fetch_ok = true; + for serial in want_first..=want_last { + if let Some(t) = timing.as_ref() { + t.record_count("rrdp_delta_fetch_attempted_total", 1); + } + let idx = (serial - min_serial) as usize; + let dref = match notif.deltas.get(idx) { + Some(v) if v.serial == serial => v, + _ => { + fetch_ok = false; + break; + } + }; + + let mut dl_span = download_log + .map(|dl| dl.span_download(AuditDownloadKind::RrdpDelta, &dref.uri)); + match fetcher.fetch(&dref.uri) { + Ok(bytes) => { + if let Some(t) = timing.as_ref() { + t.record_count("rrdp_delta_fetch_ok_total", 1); + t.record_count("rrdp_delta_bytes_total", bytes.len() as u64); + } + if let Some(s) = dl_span.as_mut() { + s.set_bytes(bytes.len() as u64); + s.set_ok(); + } + fetched.push((serial, dref.hash_sha256, bytes)) + } + Err(e) => { + if let Some(t) = timing.as_ref() { + t.record_count("rrdp_delta_fetch_fail_total", 1); + } + if let Some(s) = dl_span.as_mut() { + s.set_err(e.clone()); + } + fetch_ok = false; + break; + } + } + } + drop(_fetch_d_step); + drop(_fetch_d_total); + + if fetch_ok { + let _apply_d_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "apply_deltas")); + let _apply_d_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_apply_deltas_total")); + let mut applied_total = 0usize; + let mut ok = true; + for (serial, expected_hash, bytes) in &fetched { + match apply_delta( + store, + notification_uri, + current_repo_index, + bytes.as_slice(), + *expected_hash, + notif.session_id, + *serial, + ) { + Ok(n) => applied_total += n, + Err(_) => { + ok = false; + break; + } + } + } + drop(_apply_d_step); + drop(_apply_d_total); + + if ok { + let _write_state_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "write_state")); + let _write_state_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_write_state_total")); + let new_state = RrdpState { + session_id: notif.session_id.to_string(), + serial: notif.serial, + }; + persist_rrdp_local_state( + store, + notification_uri, + &new_state, + RrdpSourceSyncState::DeltaReady, + Some(¬if.snapshot_uri), + Some(&hex::encode(notif.snapshot_hash_sha256)), + ) + .map_err(RrdpSyncError::Storage)?; + drop(_write_state_step); + drop(_write_state_total); + if let Some(t) = timing.as_ref() { + t.record_count("rrdp_delta_ops_applied_total", applied_total as u64); + } + let _hydrate_step = timing.as_ref().map(|t| { + t.span_rrdp_repo_step(notification_uri, "hydrate_current_index") + }); + let _hydrate_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_hydrate_current_index_total")); + let hydrated = hydrate_current_repo_index_from_rrdp_members( + store, + notification_uri, + current_repo_index, + )?; + drop(_hydrate_step); + drop(_hydrate_total); + if let Some(t) = timing.as_ref() { + t.record_count( + "rrdp_current_index_hydrated_objects_total", + hydrated as u64, + ); + } + return Ok(applied_total); + } + } + } + } + } + + // Snapshot fallback (RFC 8182 §3.4.3). + let _fetch_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "fetch_snapshot")); + let _fetch_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_fetch_snapshot_total")); + let mut dl_span = download_log + .map(|dl| dl.span_download(AuditDownloadKind::RrdpSnapshot, ¬if.snapshot_uri)); + let (snapshot_file, _snapshot_bytes) = match fetch_snapshot_into_tempfile( + fetcher, + ¬if.snapshot_uri, + ¬if.snapshot_hash_sha256, + ) { + Ok(v) => { + if let Some(t) = timing.as_ref() { + t.record_count("rrdp_snapshot_fetch_ok_total", 1); + t.record_count("rrdp_snapshot_bytes_total", v.1); + } + if let Some(s) = dl_span.as_mut() { + s.set_bytes(v.1); + s.set_ok(); + } + v + } + Err(RrdpSyncError::Fetch(e)) => { + if let Some(t) = timing.as_ref() { + t.record_count("rrdp_snapshot_fetch_fail_total", 1); + } + if let Some(s) = dl_span.as_mut() { + s.set_err(e.clone()); + } + return Err(RrdpSyncError::Fetch(e)); + } + Err(e) => return Err(e), + }; + drop(_fetch_step); + drop(_fetch_total); + + let _apply_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "apply_snapshot")); + let _apply_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_apply_snapshot_total")); + let published = apply_snapshot_from_bufread( + store, + notification_uri, + current_repo_index, + std::io::BufReader::new( + snapshot_file + .reopen() + .map_err(|e| RrdpSyncError::Fetch(format!("tempfile reopen failed: {e}")))?, + ), + notif.session_id, + notif.serial, + )?; + if let Some(t) = timing.as_ref() { + t.record_count("rrdp_snapshot_objects_applied_total", published as u64); + } + drop(_apply_step); + drop(_apply_total); + + let _write_state_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "write_state")); + let _write_state_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_write_state_total")); + let new_state = RrdpState { + session_id: notif.session_id.to_string(), + serial: notif.serial, + }; + persist_rrdp_local_state( + store, + notification_uri, + &new_state, + RrdpSourceSyncState::SnapshotOnly, + Some(¬if.snapshot_uri), + Some(&hex::encode(notif.snapshot_hash_sha256)), + ) + .map_err(RrdpSyncError::Storage)?; + drop(_write_state_step); + drop(_write_state_total); + + Ok(published) +} diff --git a/crates/panda-rpki-validator/src/sync/rrdp/parse_helpers.rs b/crates/panda-rpki-validator/src/sync/rrdp/parse_helpers.rs new file mode 100644 index 0000000..af97de6 --- /dev/null +++ b/crates/panda-rpki-validator/src/sync/rrdp/parse_helpers.rs @@ -0,0 +1,87 @@ +// Shared XML, URI, and hash parsing helpers. + +#[cfg(test)] +use snapshot_apply::apply_snapshot; +use snapshot_apply::{apply_snapshot_from_bufread, fetch_snapshot_into_tempfile}; + +fn parse_rrdp_xml(xml: &[u8]) -> Result, RrdpError> { + if xml.iter().any(|&b| b > 0x7F) { + return Err(RrdpError::NotAscii); + } + let s = std::str::from_utf8(xml).map_err(|e| RrdpError::Xml(e.to_string()))?; + roxmltree::Document::parse(s).map_err(|e| RrdpError::Xml(e.to_string())) +} + +fn validate_root_common(root: &roxmltree::Node<'_, '_>) -> Result<(), RrdpError> { + let ns = root.default_namespace().unwrap_or("").to_string(); + if ns != RRDP_XMLNS { + return Err(RrdpError::InvalidNamespace(ns)); + } + + let version = root.attribute("version").unwrap_or(""); + if version != "1" { + return Err(RrdpError::InvalidVersion(version.to_string())); + } + + Ok(()) +} + +fn parse_uuid_attr(root: &roxmltree::Node<'_, '_>, name: &'static str) -> Result { + let s = root.attribute(name).unwrap_or(""); + Uuid::parse_str(s).map_err(|_e| RrdpError::InvalidSessionId(s.to_string())) +} + +fn parse_u64_attr(root: &roxmltree::Node<'_, '_>, name: &'static str) -> Result { + let s = root.attribute(name).unwrap_or(""); + parse_u64_str(s) +} + +fn parse_u64_str(s: &str) -> Result { + let v = s + .parse::() + .map_err(|_e| RrdpError::InvalidSerial(s.to_string()))?; + if v == 0 { + return Err(RrdpError::InvalidSerial(s.to_string())); + } + Ok(v) +} + +fn parse_sha256_hex_impl(s: &str, invalid: fn(String) -> RrdpError) -> Result<[u8; 32], RrdpError> { + let bytes = hex::decode(s).map_err(|_e| invalid(s.to_string()))?; + if bytes.len() != 32 { + return Err(invalid(s.to_string())); + } + let mut out = [0u8; 32]; + out.copy_from_slice(&bytes); + Ok(out) +} + +fn parse_sha256_hex_snapshot(s: &str) -> Result<[u8; 32], RrdpError> { + parse_sha256_hex_impl(s, RrdpError::SnapshotHashInvalid) +} + +fn parse_sha256_hex_delta_ref(s: &str) -> Result<[u8; 32], RrdpError> { + parse_sha256_hex_impl(s, RrdpError::DeltaRefHashInvalid) +} + +fn parse_sha256_hex_delta_publish(s: &str) -> Result<[u8; 32], RrdpError> { + parse_sha256_hex_impl(s, RrdpError::DeltaPublishHashInvalid) +} + +fn parse_sha256_hex_delta_withdraw(s: &str) -> Result<[u8; 32], RrdpError> { + parse_sha256_hex_impl(s, RrdpError::DeltaWithdrawHashInvalid) +} + +fn collect_element_text(node: &roxmltree::Node<'_, '_>) -> Option { + let mut out = String::new(); + for child in node.children() { + if child.is_text() { + out.push_str(child.text().unwrap_or("")); + } + } + if out.is_empty() { None } else { Some(out) } +} + +fn strip_all_ascii_whitespace(s: &str) -> String { + s.chars().filter(|c| !c.is_ascii_whitespace()).collect() +} diff --git a/crates/panda-rpki-validator/src/sync/rrdp/snapshot_sync.rs b/crates/panda-rpki-validator/src/sync/rrdp/snapshot_sync.rs new file mode 100644 index 0000000..3038b78 --- /dev/null +++ b/crates/panda-rpki-validator/src/sync/rrdp/snapshot_sync.rs @@ -0,0 +1,157 @@ +// RRDP snapshot synchronization and timing instrumentation. + +pub fn sync_from_notification_snapshot( + store: &RocksStore, + notification_uri: &str, + notification_xml: &[u8], + fetcher: &dyn Fetcher, +) -> RrdpSyncResult { + sync_from_notification_snapshot_inner( + store, + notification_uri, + notification_xml, + fetcher, + None, + None, + ) +} + +pub fn sync_from_notification_snapshot_with_timing( + store: &RocksStore, + notification_uri: &str, + notification_xml: &[u8], + fetcher: &dyn Fetcher, + timing: Option<&TimingHandle>, +) -> RrdpSyncResult { + sync_from_notification_snapshot_inner( + store, + notification_uri, + notification_xml, + fetcher, + timing, + None, + ) +} + +pub fn sync_from_notification_snapshot_with_timing_and_download_log( + store: &RocksStore, + notification_uri: &str, + notification_xml: &[u8], + fetcher: &dyn Fetcher, + timing: Option<&TimingHandle>, + download_log: Option<&DownloadLogHandle>, +) -> RrdpSyncResult { + sync_from_notification_snapshot_inner( + store, + notification_uri, + notification_xml, + fetcher, + timing, + download_log, + ) +} + +fn sync_from_notification_snapshot_inner( + store: &RocksStore, + notification_uri: &str, + notification_xml: &[u8], + fetcher: &dyn Fetcher, + timing: Option<&TimingHandle>, + download_log: Option<&DownloadLogHandle>, +) -> RrdpSyncResult { + let _parse_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "parse_notification_snapshot")); + let _parse_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_parse_notification_total")); + let notif = parse_notification_snapshot(notification_xml)?; + drop(_parse_step); + drop(_parse_total); + + let _fetch_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "fetch_snapshot")); + let _fetch_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_fetch_snapshot_total")); + let mut dl_span = download_log + .map(|dl| dl.span_download(AuditDownloadKind::RrdpSnapshot, ¬if.snapshot_uri)); + let (snapshot_file, _snapshot_bytes) = match fetch_snapshot_into_tempfile( + fetcher, + ¬if.snapshot_uri, + ¬if.snapshot_hash_sha256, + ) { + Ok(v) => { + if let Some(t) = timing.as_ref() { + t.record_count("rrdp_snapshot_fetch_ok_total", 1); + t.record_count("rrdp_snapshot_bytes_total", v.1); + } + if let Some(s) = dl_span.as_mut() { + s.set_bytes(v.1); + s.set_ok(); + } + v + } + Err(RrdpSyncError::Fetch(e)) => { + if let Some(t) = timing.as_ref() { + t.record_count("rrdp_snapshot_fetch_fail_total", 1); + } + if let Some(s) = dl_span.as_mut() { + s.set_err(e.clone()); + } + return Err(RrdpSyncError::Fetch(e)); + } + Err(e) => return Err(e), + }; + drop(_fetch_step); + drop(_fetch_total); + + let _apply_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "apply_snapshot")); + let _apply_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_apply_snapshot_total")); + let published = apply_snapshot_from_bufread( + store, + notification_uri, + None, + std::io::BufReader::new( + snapshot_file + .reopen() + .map_err(|e| RrdpSyncError::Fetch(format!("tempfile reopen failed: {e}")))?, + ), + notif.session_id, + notif.serial, + )?; + if let Some(t) = timing.as_ref() { + t.record_count("rrdp_snapshot_objects_applied_total", published as u64); + } + drop(_apply_step); + drop(_apply_total); + + let _write_state_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "write_state")); + let _write_state_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_write_state_total")); + let state = RrdpState { + session_id: notif.session_id.to_string(), + serial: notif.serial, + }; + persist_rrdp_local_state( + store, + notification_uri, + &state, + RrdpSourceSyncState::SnapshotOnly, + Some(¬if.snapshot_uri), + Some(&hex::encode(notif.snapshot_hash_sha256)), + ) + .map_err(RrdpSyncError::Storage)?; + drop(_write_state_step); + drop(_write_state_total); + + Ok(published) +} diff --git a/crates/panda-rpki-validator/src/sync/rrdp/tests.rs b/crates/panda-rpki-validator/src/sync/rrdp/tests.rs index 39a4c85..7db3d13 100644 --- a/crates/panda-rpki-validator/src/sync/rrdp/tests.rs +++ b/crates/panda-rpki-validator/src/sync/rrdp/tests.rs @@ -1,1316 +1,5 @@ -use super::*; -use crate::analysis::timing::{TimingHandle, TimingMeta}; -use crate::current_repo_index::CurrentRepoIndex; -use crate::storage::RocksStore; -use std::collections::HashMap; -use std::io::Read; -use std::time::Duration; - -struct MapFetcher { - map: HashMap>, -} - -impl Fetcher for MapFetcher { - fn fetch(&self, uri: &str) -> Result, String> { - self.map - .get(uri) - .cloned() - .ok_or_else(|| format!("not found: {uri}")) - } -} - -struct SleepyFetcher { - inner: MapFetcher, - sleep_uri: String, - sleep: Duration, -} - -impl Fetcher for SleepyFetcher { - fn fetch(&self, uri: &str) -> Result, String> { - if uri == self.sleep_uri { - std::thread::sleep(self.sleep); - } - self.inner.fetch(uri) - } -} - -struct WriterOnlyFetcher { - map: HashMap>, -} - -impl Fetcher for WriterOnlyFetcher { - fn fetch(&self, uri: &str) -> Result, String> { - Err(format!("unexpected buffered fetch: {uri}")) - } - - fn fetch_to_writer(&self, uri: &str, out: &mut dyn std::io::Write) -> Result { - let bytes = self - .map - .get(uri) - .ok_or_else(|| format!("not found: {uri}"))?; - out.write_all(bytes) - .map_err(|e| format!("write sink failed: {e}"))?; - Ok(bytes.len() as u64) - } -} - -struct NonAsciiWriterFetcher; - -impl Fetcher for NonAsciiWriterFetcher { - fn fetch(&self, uri: &str) -> Result, String> { - Err(format!("unexpected buffered fetch: {uri}")) - } - - fn fetch_to_writer(&self, _uri: &str, out: &mut dyn std::io::Write) -> Result { - out.write_all(&[0x80]) - .map_err(|e| format!("write sink failed: {e}"))?; - Err("snapshot body contains non-ASCII bytes".to_string()) - } -} - -fn assert_current_object(store: &RocksStore, uri: &str, expected: &[u8]) { - assert_eq!( - store - .load_current_object_bytes_by_uri(uri) - .expect("load current object"), - Some(expected.to_vec()) - ); -} - -fn notification_xml( - session_id: &str, - serial: u64, - snapshot_uri: &str, - snapshot_hash: &str, -) -> Vec { - format!( - r#""# - ) - .into_bytes() -} - -fn notification_xml_with_deltas( - session_id: &str, - serial: u64, - snapshot_uri: &str, - snapshot_hash: &str, - deltas: &[(&str, u64, &str, &str)], -) -> Vec { - let mut out = format!( - r#""# - ); - for (_name, delta_serial, uri, hash) in deltas { - out.push_str(&format!( - r#""# - )); - } - out.push_str(""); - out.into_bytes() -} - -fn snapshot_xml(session_id: &str, serial: u64, published: &[(&str, &[u8])]) -> Vec { - let mut out = format!( - r#""# - ); - for (uri, bytes) in published { - let b64 = base64::engine::general_purpose::STANDARD.encode(bytes); - out.push_str(&format!(r#"{b64}"#)); - } - out.push_str(""); - out.into_bytes() -} - -#[test] -fn fetch_snapshot_into_tempfile_streams_and_validates_hash() { - let snapshot_uri = "https://example.test/snapshot.xml"; - let snapshot = b"".to_vec(); - let mut expected_hash = [0u8; 32]; - expected_hash.copy_from_slice(&sha2::Sha256::digest(&snapshot)); - let fetcher = WriterOnlyFetcher { - map: HashMap::from([(snapshot_uri.to_string(), snapshot.clone())]), - }; - - let (mut file, bytes_written) = - fetch_snapshot_into_tempfile(&fetcher, snapshot_uri, &expected_hash) - .expect("fetch snapshot into tempfile"); - assert_eq!(bytes_written, snapshot.len() as u64); - let mut got = Vec::new(); - file.as_file_mut() - .read_to_end(&mut got) - .expect("read tempfile"); - assert_eq!(got, snapshot); - - let mut wrong_hash = expected_hash; - wrong_hash[0] ^= 0xff; - let err = fetch_snapshot_into_tempfile(&fetcher, snapshot_uri, &wrong_hash).unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::SnapshotHashMismatch) - )); -} - -#[test] -fn fetch_snapshot_into_tempfile_maps_stream_non_ascii_error() { - let err = fetch_snapshot_into_tempfile( - &NonAsciiWriterFetcher, - "https://example.test/snapshot.xml", - &[0u8; 32], - ) - .unwrap_err(); - assert!(matches!(err, RrdpSyncError::Rrdp(RrdpError::NotAscii))); -} - -#[test] -fn timing_rrdp_repo_step_spans_cover_snapshot_fetch_duration() { - let temp = tempfile::tempdir().expect("tempdir"); - let store_dir = temp.path().join("db"); - let store = RocksStore::open(&store_dir).expect("open rocksdb"); - - let notification_uri = "https://example.test/notification.xml"; - let snapshot_uri = "https://example.test/snapshot.xml"; - let published_uri = "rsync://example.test/repo/a.mft"; - let published_bytes = b"x"; - let session_id = "550e8400-e29b-41d4-a716-446655440000"; - - let snapshot = snapshot_xml(session_id, 1, &[(published_uri, published_bytes)]); - let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); - let notif = notification_xml(session_id, 1, snapshot_uri, &snapshot_hash); - - let mut map = HashMap::new(); - map.insert(snapshot_uri.to_string(), snapshot); - let fetcher = SleepyFetcher { - inner: MapFetcher { map }, - sleep_uri: snapshot_uri.to_string(), - sleep: Duration::from_millis(25), - }; - - let timing = TimingHandle::new(TimingMeta { - recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), - validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), - tal_url: None, - db_path: Some(store_dir.to_string_lossy().into_owned()), - }); - - sync_from_notification_snapshot_with_timing( - &store, - notification_uri, - ¬if, - &fetcher, - Some(&timing), - ) - .expect("rrdp snapshot sync ok"); - - let timing_path = temp.path().join("timing.json"); - timing.write_json(&timing_path, 200).expect("write timing"); - let rep: serde_json::Value = - serde_json::from_slice(&std::fs::read(&timing_path).expect("read timing")) - .expect("parse timing"); - - let want = format!("{notification_uri}::fetch_snapshot"); - let steps = rep - .get("top_rrdp_repo_steps") - .and_then(|v| v.as_array()) - .expect("top_rrdp_repo_steps array"); - let entry = steps - .iter() - .find(|e| e.get("key").and_then(|k| k.as_str()) == Some(want.as_str())) - .unwrap_or_else(|| panic!("missing timing step entry for {want}")); - let nanos = entry - .get("total_nanos") - .and_then(|v| v.as_u64()) - .expect("total_nanos"); - assert!( - nanos >= 20_000_000, - "expected fetch_snapshot timing to include the fetch duration; got {nanos}ns" - ); -} - -#[test] -fn parse_notification_snapshot_rejects_non_ascii() { - let mut xml = b"".to_vec(); - xml.push(0x80); - let err = parse_notification_snapshot(&xml).unwrap_err(); - assert!(matches!(err, RrdpError::NotAscii)); -} - -#[test] -fn parse_notification_snapshot_parses_valid_minimal_notification() { - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let snapshot_uri = "https://example.net/snapshot.xml"; - let hash = "00".repeat(32); - let xml = notification_xml(sid, 7, snapshot_uri, &hash); - let n = parse_notification_snapshot(&xml).expect("parse"); - assert_eq!(n.session_id, Uuid::parse_str(sid).unwrap()); - assert_eq!(n.serial, 7); - assert_eq!(n.snapshot_uri, snapshot_uri); - assert_eq!(hex::encode(n.snapshot_hash_sha256), hash); -} - -#[test] -fn parse_notification_parses_deltas_and_validates_contiguity() { - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let snapshot_uri = "https://example.net/snapshot.xml"; - let hash = "00".repeat(32); - let d_hash_2 = "11".repeat(32); - let d_hash_3 = "22".repeat(32); - // Provide deltas in reverse order to ensure we sort. - let xml = notification_xml_with_deltas( - sid, - 3, - snapshot_uri, - &hash, - &[ - ("d3", 3, "https://example.net/delta-3.xml", &d_hash_3), - ("d2", 2, "https://example.net/delta-2.xml", &d_hash_2), - ], - ); - let n = parse_notification(&xml).expect("parse notification"); - assert_eq!(n.serial, 3); - assert_eq!(n.deltas.len(), 2); - assert_eq!(n.deltas[0].serial, 2); - assert_eq!(n.deltas[1].serial, 3); - assert_eq!(n.deltas[0].uri, "https://example.net/delta-2.xml"); - assert_eq!(hex::encode(n.deltas[1].hash_sha256), d_hash_3); -} - -#[test] -fn parse_notification_rejects_non_contiguous_deltas() { - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let snapshot_uri = "https://example.net/snapshot.xml"; - let hash = "00".repeat(32); - let d_hash_1 = "11".repeat(32); - let d_hash_3 = "22".repeat(32); - // Missing delta serial 2. - let xml = notification_xml_with_deltas( - sid, - 3, - snapshot_uri, - &hash, - &[ - ("d3", 3, "https://example.net/delta-3.xml", &d_hash_3), - ("d1", 1, "https://example.net/delta-1.xml", &d_hash_1), - ], - ); - let err = parse_notification(&xml).unwrap_err(); - assert!(matches!(err, RrdpError::DeltaRefChainNotContiguous { .. })); -} - -fn delta_xml(session_id: &str, serial: u64, elements: &[&str]) -> Vec { - let mut out = format!( - r#""# - ); - for e in elements { - out.push_str(e); - } - out.push_str(""); - out.into_bytes() -} - -#[test] -fn parse_delta_file_parses_publish_and_withdraw() { - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let serial = 3u64; - let publish_bytes = b"abc"; - let publish_b64 = base64::engine::general_purpose::STANDARD.encode(publish_bytes); - let withdraw_hash = "33".repeat(32); - - let xml = delta_xml( - sid, - serial, - &[ - &format!(r#"{publish_b64}"#), - &format!(r#""#), - ], - ); - - let d = parse_delta_file(&xml).expect("parse delta"); - assert_eq!(d.session_id, Uuid::parse_str(sid).unwrap()); - assert_eq!(d.serial, serial); - assert_eq!(d.elements.len(), 2); - match &d.elements[0] { - DeltaElement::Publish { - uri, - hash_sha256, - bytes, - } => { - assert_eq!(uri, "rsync://example.net/repo/a.mft"); - assert_eq!(*hash_sha256, None); - assert_eq!(bytes, publish_bytes); - } - _ => panic!("expected publish"), - } - match &d.elements[1] { - DeltaElement::Withdraw { uri, hash_sha256 } => { - assert_eq!(uri, "rsync://example.net/repo/b.cer"); - assert_eq!(hex::encode(hash_sha256), withdraw_hash); - } - _ => panic!("expected withdraw"), - } -} - -#[test] -fn parse_delta_file_rejects_withdraw_with_content() { - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let serial = 1u64; - let withdraw_hash = "33".repeat(32); - let xml = delta_xml( - sid, - serial, - &[&format!( - r#"AA=="# - )], - ); - let err = parse_delta_file(&xml).unwrap_err(); - assert!(matches!(err, RrdpError::DeltaWithdrawUnexpectedContent)); -} - -#[test] -fn apply_delta_applies_publish_replace_and_withdraw_with_membership_checks() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let notif_uri = "https://example.net/notification.xml"; - - // Start from snapshot state with a + b - let snapshot_uri = "https://example.net/snapshot.xml"; - let snapshot = snapshot_xml( - sid, - 1, - &[ - ("rsync://example.net/repo/a.mft", b"a1"), - ("rsync://example.net/repo/b.roa", b"b1"), - ], - ); - let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); - let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash); - let fetcher = MapFetcher { - map: HashMap::from([(snapshot_uri.to_string(), snapshot)]), - }; - sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).expect("sync snapshot"); - - let old_b = store - .load_current_object_bytes_by_uri("rsync://example.net/repo/b.roa") - .expect("load current b") - .expect("b present"); - let old_b_hash = hex::encode(sha2::Sha256::digest(old_b.as_slice())); - - let withdraw_a_hash = hex::encode(sha2::Sha256::digest(b"a1".as_slice())); - let publish_c_b64 = base64::engine::general_purpose::STANDARD.encode(b"c2"); - let replace_b_b64 = base64::engine::general_purpose::STANDARD.encode(b"b2"); - - let delta = delta_xml( - sid, - 2, - &[ - &format!( - r#""# - ), - &format!( - r#"{replace_b_b64}"# - ), - &format!(r#"{publish_c_b64}"#), - ], - ); - let delta_hash = sha2::Sha256::digest(&delta); - let mut expected_hash = [0u8; 32]; - expected_hash.copy_from_slice(delta_hash.as_slice()); - - let applied = apply_delta( - &store, - notif_uri, - None, - &delta, - expected_hash, - Uuid::parse_str(sid).unwrap(), - 2, - ) - .expect("apply delta"); - assert_eq!(applied, 3); - - assert_eq!( - store - .load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft") - .expect("load current a"), - None, - "a withdrawn" - ); - let b = store - .load_current_object_bytes_by_uri("rsync://example.net/repo/b.roa") - .expect("load current b") - .expect("b present"); - assert_eq!(b, b"b2"); - let c = store - .load_current_object_bytes_by_uri("rsync://example.net/repo/c.crl") - .expect("load current c") - .expect("c present"); - assert_eq!(c, b"c2"); - - assert!( - !store - .is_current_rrdp_source_member(notif_uri, "rsync://example.net/repo/a.mft") - .expect("is member"), - "a removed from rrdp repo index" - ); - assert!( - store - .is_current_rrdp_source_member(notif_uri, "rsync://example.net/repo/c.crl") - .expect("is member"), - "c added to rrdp repo index" - ); - - let a_view = store - .get_repository_view_entry("rsync://example.net/repo/a.mft") - .expect("get a view") - .expect("a view exists"); - assert_eq!(a_view.state, crate::storage::RepositoryViewState::Withdrawn); - let b_view = store - .get_repository_view_entry("rsync://example.net/repo/b.roa") - .expect("get b view") - .expect("b view exists"); - assert_eq!(b_view.state, crate::storage::RepositoryViewState::Present); - assert_eq!( - b_view.current_hash.as_deref(), - Some(hex::encode(sha2::Sha256::digest(b"b2")).as_str()) - ); - let c_owner = store - .get_rrdp_uri_owner_record("rsync://example.net/repo/c.crl") - .expect("get c owner") - .expect("c owner exists"); - assert_eq!( - c_owner.owner_state, - crate::storage::RrdpUriOwnerState::Active - ); - let a_member = store - .get_rrdp_source_member_record(notif_uri, "rsync://example.net/repo/a.mft") - .expect("get a member") - .expect("a member exists"); - assert!(!a_member.present); - let current_members = store - .list_current_rrdp_source_members(notif_uri) - .expect("list current members"); - assert_eq!( - current_members - .iter() - .map(|record| record.rsync_uri.as_str()) - .collect::>(), - vec![ - "rsync://example.net/repo/b.roa", - "rsync://example.net/repo/c.crl", - ] - ); -} - -#[test] -fn apply_delta_rejects_hash_mismatch() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); - let notif_uri = "https://example.net/notification.xml"; - - let delta = delta_xml( - sid.to_string().as_str(), - 1, - &[r#"QQ=="#], - ); - let mut wrong = [0u8; 32]; - wrong[0] = 1; - let err = apply_delta(&store, notif_uri, None, &delta, wrong, sid, 1).unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::DeltaHashMismatch) - )); -} - -#[test] -fn apply_delta_rejects_withdraw_of_non_member() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); - let notif_uri = "https://example.net/notification.xml"; - - let withdraw_hash = "00".repeat(32); - let delta = delta_xml( - sid.to_string().as_str(), - 1, - &[&format!( - r#""# - )], - ); - let delta_hash = sha2::Sha256::digest(&delta); - let mut expected_hash = [0u8; 32]; - expected_hash.copy_from_slice(delta_hash.as_slice()); - - let err = apply_delta(&store, notif_uri, None, &delta, expected_hash, sid, 1).unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::DeltaTargetNotFromRepository { .. }) - )); -} - -#[test] -fn apply_delta_rejects_publish_without_hash_for_existing_object() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let notif_uri = "https://example.net/notification.xml"; - - // Seed snapshot with a.mft. - let snapshot_uri = "https://example.net/snapshot.xml"; - let snapshot = snapshot_xml(sid, 1, &[("rsync://example.net/repo/a.mft", b"a1")]); - let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); - let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash); - let fetcher = MapFetcher { - map: HashMap::from([(snapshot_uri.to_string(), snapshot)]), - }; - sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).expect("seed"); - - // Replace publish for an existing URI must have @hash. - let publish_b64 = base64::engine::general_purpose::STANDARD.encode(b"a2"); - let delta = delta_xml( - sid, - 2, - &[&format!( - r#"{publish_b64}"# - )], - ); - let delta_hash = sha2::Sha256::digest(&delta); - let mut expected_hash = [0u8; 32]; - expected_hash.copy_from_slice(delta_hash.as_slice()); - - let err = apply_delta( - &store, - notif_uri, - None, - &delta, - expected_hash, - Uuid::parse_str(sid).unwrap(), - 2, - ) - .unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::DeltaPublishWithoutHashForExisting { .. }) - )); -} - -#[test] -fn apply_delta_rejects_target_missing_and_hash_mismatch() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let notif_uri = "https://example.net/notification.xml"; - - // Seed snapshot with a.mft. - let snapshot_uri = "https://example.net/snapshot.xml"; - let snapshot = snapshot_xml(sid, 1, &[("rsync://example.net/repo/a.mft", b"a1")]); - let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); - let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash); - let fetcher = MapFetcher { - map: HashMap::from([(snapshot_uri.to_string(), snapshot)]), - }; - sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).expect("seed"); - - let old_bytes = store - .load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft") - .expect("get") - .expect("present"); - let old_hash = hex::encode(sha2::Sha256::digest(old_bytes.as_slice())); - - // Hash mismatch on withdraw. - let wrong_hash = "11".repeat(32); - let delta = delta_xml( - sid, - 2, - &[&format!( - r#""# - )], - ); - let delta_hash = sha2::Sha256::digest(&delta); - let mut expected_hash = [0u8; 32]; - expected_hash.copy_from_slice(delta_hash.as_slice()); - let err = apply_delta( - &store, - notif_uri, - None, - &delta, - expected_hash, - Uuid::parse_str(sid).unwrap(), - 2, - ) - .unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::DeltaTargetHashMismatch { .. }) - )); - - // Target missing in local cache (index still says it's a member). - store - .delete_repository_view_entry("rsync://example.net/repo/a.mft") - .expect("delete current repository view entry"); - let delta = delta_xml( - sid, - 2, - &[&format!( - r#""# - )], - ); - let delta_hash = sha2::Sha256::digest(&delta); - let mut expected_hash = [0u8; 32]; - expected_hash.copy_from_slice(delta_hash.as_slice()); - let err = apply_delta( - &store, - notif_uri, - None, - &delta, - expected_hash, - Uuid::parse_str(sid).unwrap(), - 2, - ) - .unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::DeltaTargetMissing { .. }) - )); -} - -#[test] -fn apply_delta_rejects_session_and_serial_mismatch() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let notif_uri = "https://example.net/notification.xml"; - - let publish_b64 = base64::engine::general_purpose::STANDARD.encode(b"x"); - let delta = delta_xml( - sid, - 2, - &[&format!( - r#"{publish_b64}"# - )], - ); - let delta_hash = sha2::Sha256::digest(&delta); - let mut expected_hash = [0u8; 32]; - expected_hash.copy_from_slice(delta_hash.as_slice()); - - // Session mismatch. - let other_sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440001").unwrap(); - let err = - apply_delta(&store, notif_uri, None, &delta, expected_hash, other_sid, 2).unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::DeltaSessionIdMismatch { .. }) - )); - - // Serial mismatch. - let err = apply_delta( - &store, - notif_uri, - None, - &delta, - expected_hash, - Uuid::parse_str(sid).unwrap(), - 3, - ) - .unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::DeltaSerialMismatch { .. }) - )); -} - -#[test] -fn sync_from_notification_snapshot_rejects_cross_source_owner_conflict() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - - let sid_a = "550e8400-e29b-41d4-a716-446655440000"; - let sid_b = "550e8400-e29b-41d4-a716-446655440001"; - let uri = "rsync://example.net/repo/a.mft"; - - let notif_a_uri = "https://example.net/a/notification.xml"; - let snapshot_a_uri = "https://example.net/a/snapshot.xml"; - let snapshot_a = snapshot_xml(sid_a, 1, &[(uri, b"a1")]); - let snapshot_a_hash = hex::encode(sha2::Sha256::digest(&snapshot_a)); - let notif_a = notification_xml(sid_a, 1, snapshot_a_uri, &snapshot_a_hash); - let fetcher_a = MapFetcher { - map: HashMap::from([(snapshot_a_uri.to_string(), snapshot_a)]), - }; - sync_from_notification_snapshot(&store, notif_a_uri, ¬if_a, &fetcher_a) - .expect("seed source a"); - - let notif_b_uri = "https://example.net/b/notification.xml"; - let snapshot_b_uri = "https://example.net/b/snapshot.xml"; - let snapshot_b = snapshot_xml(sid_b, 1, &[(uri, b"b1")]); - let snapshot_b_hash = hex::encode(sha2::Sha256::digest(&snapshot_b)); - let notif_b = notification_xml(sid_b, 1, snapshot_b_uri, &snapshot_b_hash); - let fetcher_b = MapFetcher { - map: HashMap::from([(snapshot_b_uri.to_string(), snapshot_b)]), - }; - - let err = sync_from_notification_snapshot(&store, notif_b_uri, ¬if_b, &fetcher_b) - .expect_err("cross-source overwrite must fail"); - assert!(matches!(err, RrdpSyncError::Storage(_))); - assert!(err.to_string().contains("owner conflict"), "{err}"); -} - -#[test] -fn sync_from_notification_snapshot_applies_snapshot_and_stores_state() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let serial = 9u64; - let notif_uri = "https://example.net/notification.xml"; - let snapshot_uri = "https://example.net/snapshot.xml"; - - let snapshot = snapshot_xml( - sid, - serial, - &[ - ("rsync://example.net/repo/a.mft", b"mft-bytes"), - ("rsync://example.net/repo/b.roa", b"roa-bytes"), - ], - ); - let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); - let notif = notification_xml(sid, serial, snapshot_uri, &snapshot_hash); - - let fetcher = MapFetcher { - map: HashMap::from([(snapshot_uri.to_string(), snapshot.clone())]), - }; - - let published = - sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).expect("sync"); - assert_eq!(published, 2); - - assert_current_object(&store, "rsync://example.net/repo/a.mft", b"mft-bytes"); - assert_current_object(&store, "rsync://example.net/repo/b.roa", b"roa-bytes"); - - let state = load_rrdp_local_state(&store, notif_uri) - .expect("get rrdp state") - .expect("state present"); - assert_eq!(state.session_id, sid); - assert_eq!(state.serial, serial); - - let source = store - .get_rrdp_source_record(notif_uri) - .expect("get rrdp source") - .expect("rrdp source exists"); - assert_eq!(source.last_session_id.as_deref(), Some(sid)); - assert_eq!(source.last_serial, Some(serial)); - assert_eq!( - source.sync_state, - crate::storage::RrdpSourceSyncState::SnapshotOnly - ); - - let view = store - .get_repository_view_entry("rsync://example.net/repo/a.mft") - .expect("get repository view") - .expect("repository view exists"); - assert_eq!(view.state, crate::storage::RepositoryViewState::Present); - assert_eq!(view.repository_source.as_deref(), Some(notif_uri)); - - let current_bytes = store - .load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft") - .expect("load current bytes") - .expect("current object bytes exist"); - assert_eq!(current_bytes, b"mft-bytes".to_vec()); - assert!( - store - .get_raw_by_hash_entry(hex::encode(sha2::Sha256::digest(b"mft-bytes")).as_str()) - .expect("get raw_by_hash") - .is_none() - ); - - let member = store - .get_rrdp_source_member_record(notif_uri, "rsync://example.net/repo/a.mft") - .expect("get member") - .expect("member exists"); - assert!(member.present); - let owner = store - .get_rrdp_uri_owner_record("rsync://example.net/repo/a.mft") - .expect("get owner") - .expect("owner exists"); - assert_eq!(owner.notify_uri, notif_uri); - assert_eq!(owner.owner_state, crate::storage::RrdpUriOwnerState::Active); -} - -#[test] -fn sync_from_notification_snapshot_deletes_objects_not_in_new_snapshot() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let notif_uri = "https://example.net/notification.xml"; - - // serial 1: publish a + b - let snapshot_uri_1 = "https://example.net/snapshot-1.xml"; - let snapshot_1 = snapshot_xml( - sid, - 1, - &[ - ("rsync://example.net/repo/a.mft", b"a1"), - ("rsync://example.net/repo/b.roa", b"b1"), - ], - ); - let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1)); - let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1); - - let fetcher_1 = MapFetcher { - map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]), - }; - sync_from_notification_snapshot(&store, notif_uri, ¬if_1, &fetcher_1).expect("sync 1"); - - // serial 2: publish b (new bytes) + c, and drop a - let snapshot_uri_2 = "https://example.net/snapshot-2.xml"; - let snapshot_2 = snapshot_xml( - sid, - 2, - &[ - ("rsync://example.net/repo/b.roa", b"b2"), - ("rsync://example.net/repo/c.crl", b"c2"), - ], - ); - let snapshot_hash_2 = hex::encode(sha2::Sha256::digest(&snapshot_2)); - let notif_2 = notification_xml(sid, 2, snapshot_uri_2, &snapshot_hash_2); - - let fetcher_2 = MapFetcher { - map: HashMap::from([(snapshot_uri_2.to_string(), snapshot_2)]), - }; - sync_from_notification_snapshot(&store, notif_uri, ¬if_2, &fetcher_2).expect("sync 2"); - - assert!( - store - .load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft") - .expect("get current a") - .is_none(), - "a should be deleted by full-state snapshot apply" - ); - - assert_current_object(&store, "rsync://example.net/repo/b.roa", b"b2"); - assert_current_object(&store, "rsync://example.net/repo/c.crl", b"c2"); -} - -#[test] -fn sync_from_notification_uses_deltas_when_available_for_local_state() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let notif_uri = "https://example.net/notification.xml"; - - // Seed state with snapshot serial=1 containing a+b. - let snapshot_uri_1 = "https://example.net/snapshot-1.xml"; - let snapshot_1 = snapshot_xml( - sid, - 1, - &[ - ("rsync://example.net/repo/a.mft", b"a1"), - ("rsync://example.net/repo/b.roa", b"b1"), - ], - ); - let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1)); - let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1); - let fetcher_1 = MapFetcher { - map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]), - }; - sync_from_notification_snapshot(&store, notif_uri, ¬if_1, &fetcher_1).expect("seed"); - - // Notification serial=3 with deltas 2 and 3. Snapshot URI is intentionally not fetchable - // to assert we really use deltas. - let snapshot_uri_3 = "https://example.net/snapshot-3.xml"; - let snapshot_hash_3 = "00".repeat(32); - - let publish_c_b64 = base64::engine::general_purpose::STANDARD.encode(b"c2"); - let delta_2 = delta_xml( - sid, - 2, - &[&format!( - r#"{publish_c_b64}"# - )], - ); - let delta_2_hash_hex = hex::encode(sha2::Sha256::digest(&delta_2)); - - let c_hash_hex = hex::encode(sha2::Sha256::digest(b"c2".as_slice())); - let delta_3 = delta_xml( - sid, - 3, - &[&format!( - r#""# - )], - ); - let delta_3_hash_hex = hex::encode(sha2::Sha256::digest(&delta_3)); - - let notif_3 = notification_xml_with_deltas( - sid, - 3, - snapshot_uri_3, - &snapshot_hash_3, - &[ - ( - "d3", - 3, - "https://example.net/delta-3.xml", - &delta_3_hash_hex, - ), - ( - "d2", - 2, - "https://example.net/delta-2.xml", - &delta_2_hash_hex, - ), - ], - ); - - let fetcher = MapFetcher { - map: HashMap::from([ - ("https://example.net/delta-2.xml".to_string(), delta_2), - ("https://example.net/delta-3.xml".to_string(), delta_3), - ]), - }; - - let applied = sync_from_notification(&store, notif_uri, ¬if_3, &fetcher).expect("sync"); - assert!(applied > 0); - - // Delta 2 publishes c then delta 3 withdraws it => final state should not contain c. - assert!( - store - .load_current_object_bytes_by_uri("rsync://example.net/repo/c.crl") - .expect("get current") - .is_none() - ); - - let state = load_rrdp_local_state(&store, notif_uri) - .expect("get rrdp state") - .expect("state present"); - assert_eq!(state.session_id, Uuid::parse_str(sid).unwrap().to_string()); - assert_eq!(state.serial, 3); -} - -#[test] -fn sync_from_notification_same_serial_hydrates_current_repo_index() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let notif_uri = "https://example.net/notification.xml"; - let snapshot_uri = "https://example.net/snapshot.xml"; - let uri_a = "rsync://example.net/repo/a.mft"; - let uri_b = "rsync://example.net/repo/b.roa"; - - let snapshot = snapshot_xml(sid, 1, &[(uri_a, b"a1"), (uri_b, b"b1")]); - let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); - let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash); - let fetcher_1 = MapFetcher { - map: HashMap::from([(snapshot_uri.to_string(), snapshot)]), - }; - sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher_1).expect("seed"); - - let index = CurrentRepoIndex::shared(); - let no_fetcher = MapFetcher { - map: HashMap::new(), - }; - let applied = sync_from_notification_with_timing_and_download_log( - &store, - notif_uri, - Some(&index), - ¬if, - &no_fetcher, - None, - None, - ) - .expect("same serial no-op"); - assert_eq!(applied, 0); - - let index = index.read().expect("read-lock index"); - assert_eq!(index.active_uri_count(), 2); - assert!(index.get_by_uri(uri_a).is_some()); - assert!(index.get_by_uri(uri_b).is_some()); -} - -#[test] -fn sync_from_notification_delta_hydrates_unchanged_current_repo_entries() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let notif_uri = "https://example.net/notification.xml"; - let snapshot_uri_1 = "https://example.net/snapshot-1.xml"; - let uri_a = "rsync://example.net/repo/a.mft"; - let uri_b = "rsync://example.net/repo/b.roa"; - let uri_c = "rsync://example.net/repo/c.crl"; - - let snapshot_1 = snapshot_xml(sid, 1, &[(uri_a, b"a1"), (uri_b, b"b1")]); - let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1)); - let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1); - let fetcher_1 = MapFetcher { - map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]), - }; - sync_from_notification_snapshot(&store, notif_uri, ¬if_1, &fetcher_1).expect("seed"); - - let publish_c_b64 = base64::engine::general_purpose::STANDARD.encode(b"c2"); - let delta_2 = delta_xml( - sid, - 2, - &[&format!( - r#"{publish_c_b64}"# - )], - ); - let delta_2_hash_hex = hex::encode(sha2::Sha256::digest(&delta_2)); - let notif_2 = notification_xml_with_deltas( - sid, - 2, - "https://example.net/snapshot-2.xml", - &"00".repeat(32), - &[( - "d2", - 2, - "https://example.net/delta-2.xml", - &delta_2_hash_hex, - )], - ); - let fetcher_2 = MapFetcher { - map: HashMap::from([("https://example.net/delta-2.xml".to_string(), delta_2)]), - }; - - let index = CurrentRepoIndex::shared(); - let applied = sync_from_notification_with_timing_and_download_log( - &store, - notif_uri, - Some(&index), - ¬if_2, - &fetcher_2, - None, - None, - ) - .expect("delta sync"); - assert_eq!(applied, 1); - - let index = index.read().expect("read-lock index"); - assert_eq!(index.active_uri_count(), 3); - assert!( - index.get_by_uri(uri_a).is_some(), - "unchanged object from the previous serial must be visible" - ); - assert!( - index.get_by_uri(uri_b).is_some(), - "unchanged object from the previous serial must be visible" - ); - assert!(index.get_by_uri(uri_c).is_some(), "delta publish visible"); -} - -#[test] -fn load_rrdp_local_state_uses_source_record_only() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - let notif_uri = "https://example.net/notification.xml"; - assert_eq!( - load_rrdp_local_state(&store, notif_uri).expect("load empty"), - None - ); - - update_rrdp_source_record_on_success( - &store, - notif_uri, - "source-session", - 9, - crate::storage::RrdpSourceSyncState::DeltaReady, - Some("https://example.net/snapshot.xml"), - Some(&hex::encode([0x11; 32])), - ) - .expect("write source record"); - - let got = load_rrdp_local_state(&store, notif_uri) - .expect("load source preferred") - .expect("source present"); - assert_eq!( - got, - RrdpState { - session_id: "source-session".to_string(), - serial: 9, - } - ); -} - -#[test] -fn sync_from_notification_falls_back_to_snapshot_if_missing_required_deltas() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let notif_uri = "https://example.net/notification.xml"; - - // Seed state serial=1 with a only. - let snapshot_uri_1 = "https://example.net/snapshot-1.xml"; - let snapshot_1 = snapshot_xml(sid, 1, &[("rsync://example.net/repo/a.mft", b"a1")]); - let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1)); - let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1); - let fetcher_1 = MapFetcher { - map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]), - }; - sync_from_notification_snapshot(&store, notif_uri, ¬if_1, &fetcher_1).expect("seed"); - - // Notification serial=3 only includes delta serial=3 (contiguous per RFC, but missing - // serial=2 relative to our local state, so we must use snapshot). - let snapshot_uri_3 = "https://example.net/snapshot-3.xml"; - let snapshot_3 = snapshot_xml(sid, 3, &[("rsync://example.net/repo/z.roa", b"z3")]); - let snapshot_hash_3 = hex::encode(sha2::Sha256::digest(&snapshot_3)); - let delta_3_hash_hex = "11".repeat(32); - let notif_3 = notification_xml_with_deltas( - sid, - 3, - snapshot_uri_3, - &snapshot_hash_3, - &[( - "d3", - 3, - "https://example.net/delta-3.xml", - &delta_3_hash_hex, - )], - ); - - let fetcher = MapFetcher { - map: HashMap::from([(snapshot_uri_3.to_string(), snapshot_3)]), - }; - - let published = sync_from_notification(&store, notif_uri, ¬if_3, &fetcher).expect("sync"); - assert_eq!(published, 1); - assert!( - store - .load_current_object_bytes_by_uri("rsync://example.net/repo/z.roa") - .expect("get current") - .is_some() - ); -} - -#[test] -fn sync_from_notification_snapshot_rejects_snapshot_hash_mismatch() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - - let sid = "550e8400-e29b-41d4-a716-446655440000"; - let serial = 1u64; - let notif_uri = "https://example.net/notification.xml"; - let snapshot_uri = "https://example.net/snapshot.xml"; - - let snapshot = snapshot_xml(sid, serial, &[("rsync://example.net/repo/a.mft", b"x")]); - let notif = notification_xml(sid, serial, snapshot_uri, &"00".repeat(32)); - - let fetcher = MapFetcher { - map: HashMap::from([(snapshot_uri.to_string(), snapshot)]), - }; - let err = sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::SnapshotHashMismatch) - )); -} - -#[test] -fn apply_snapshot_rejects_session_id_and_serial_mismatch() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - let notif_uri = "https://example.net/notification.xml"; - - let expected_sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); - let got_sid = "550e8400-e29b-41d4-a716-446655440001"; - - let snapshot = snapshot_xml(got_sid, 2, &[("rsync://example.net/repo/a.mft", b"x")]); - let err = apply_snapshot(&store, notif_uri, None, &snapshot, expected_sid, 2).unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::SnapshotSessionIdMismatch { .. }) - )); - - let snapshot = snapshot_xml( - expected_sid.to_string().as_str(), - 3, - &[("rsync://example.net/repo/a.mft", b"x")], - ); - let err = apply_snapshot(&store, notif_uri, None, &snapshot, expected_sid, 2).unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::SnapshotSerialMismatch { .. }) - )); -} - -#[test] -fn strip_all_ascii_whitespace_removes_newlines_and_spaces() { - assert_eq!(strip_all_ascii_whitespace(" a \n b\tc "), "abc"); -} - -#[test] -fn apply_snapshot_reports_publish_errors() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - let notif_uri = "https://example.net/notification.xml"; - let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); - - // Missing publish/@uri - let xml = format!( - r#"AA=="# - ) - .into_bytes(); - let err = apply_snapshot(&store, notif_uri, None, &xml, sid, 1).unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::PublishUriMissing) - )); - - // Missing base64 content (no text nodes). - let xml = format!( - r#""# - ) - .into_bytes(); - let err = apply_snapshot(&store, notif_uri, None, &xml, sid, 1).unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::PublishContentMissing) - )); - - // Invalid base64 content. - let xml = format!( - r#"!!!"# - ) - .into_bytes(); - let err = apply_snapshot(&store, notif_uri, None, &xml, sid, 1).unwrap_err(); - assert!(matches!( - err, - RrdpSyncError::Rrdp(RrdpError::PublishBase64(_)) - )); -} - -#[test] -fn apply_snapshot_handles_multiple_publish_batches() { - let tmp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(tmp.path()).expect("open rocksdb"); - let notif_uri = "https://example.net/notification.xml"; - let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); - - let total = RRDP_SNAPSHOT_APPLY_BATCH_SIZE + 7; - let mut xml = - format!(r#""#); - for i in 0..total { - let uri = format!("rsync://example.net/repo/{i:04}.roa"); - let bytes = format!("payload-{i}").into_bytes(); - let b64 = base64::engine::general_purpose::STANDARD.encode(bytes); - xml.push_str(&format!(r#"{b64}"#)); - } - xml.push_str(""); - - let published = - apply_snapshot(&store, notif_uri, None, xml.as_bytes(), sid, 1).expect("apply snapshot"); - assert_eq!(published, total); - - for idx in [0usize, RRDP_SNAPSHOT_APPLY_BATCH_SIZE - 1, total - 1] { - let uri = format!("rsync://example.net/repo/{idx:04}.roa"); - let got = store - .load_current_object_bytes_by_uri(&uri) - .expect("load object") - .expect("object exists"); - assert_eq!(got, format!("payload-{idx}").into_bytes()); - } -} +// RRDP tests are grouped by parsing, delta application, and synchronization. +include!("tests_parts/parsing.rs"); +include!("tests_parts/delta_apply.rs"); +include!("tests_parts/sync.rs"); +include!("tests_parts/edge_cases.rs"); diff --git a/crates/panda-rpki-validator/src/sync/rrdp/tests_parts/delta_apply.rs b/crates/panda-rpki-validator/src/sync/rrdp/tests_parts/delta_apply.rs new file mode 100644 index 0000000..b0d3808 --- /dev/null +++ b/crates/panda-rpki-validator/src/sync/rrdp/tests_parts/delta_apply.rs @@ -0,0 +1,359 @@ +// RRDP test group: delta apply. + +#[test] +fn apply_delta_applies_publish_replace_and_withdraw_with_membership_checks() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + + // Start from snapshot state with a + b + let snapshot_uri = "https://example.net/snapshot.xml"; + let snapshot = snapshot_xml( + sid, + 1, + &[ + ("rsync://example.net/repo/a.mft", b"a1"), + ("rsync://example.net/repo/b.roa", b"b1"), + ], + ); + let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); + let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash); + let fetcher = MapFetcher { + map: HashMap::from([(snapshot_uri.to_string(), snapshot)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).expect("sync snapshot"); + + let old_b = store + .load_current_object_bytes_by_uri("rsync://example.net/repo/b.roa") + .expect("load current b") + .expect("b present"); + let old_b_hash = hex::encode(sha2::Sha256::digest(old_b.as_slice())); + + let withdraw_a_hash = hex::encode(sha2::Sha256::digest(b"a1".as_slice())); + let publish_c_b64 = base64::engine::general_purpose::STANDARD.encode(b"c2"); + let replace_b_b64 = base64::engine::general_purpose::STANDARD.encode(b"b2"); + + let delta = delta_xml( + sid, + 2, + &[ + &format!( + r#""# + ), + &format!( + r#"{replace_b_b64}"# + ), + &format!(r#"{publish_c_b64}"#), + ], + ); + let delta_hash = sha2::Sha256::digest(&delta); + let mut expected_hash = [0u8; 32]; + expected_hash.copy_from_slice(delta_hash.as_slice()); + + let applied = apply_delta( + &store, + notif_uri, + None, + &delta, + expected_hash, + Uuid::parse_str(sid).unwrap(), + 2, + ) + .expect("apply delta"); + assert_eq!(applied, 3); + + assert_eq!( + store + .load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft") + .expect("load current a"), + None, + "a withdrawn" + ); + let b = store + .load_current_object_bytes_by_uri("rsync://example.net/repo/b.roa") + .expect("load current b") + .expect("b present"); + assert_eq!(b, b"b2"); + let c = store + .load_current_object_bytes_by_uri("rsync://example.net/repo/c.crl") + .expect("load current c") + .expect("c present"); + assert_eq!(c, b"c2"); + + assert!( + !store + .is_current_rrdp_source_member(notif_uri, "rsync://example.net/repo/a.mft") + .expect("is member"), + "a removed from rrdp repo index" + ); + assert!( + store + .is_current_rrdp_source_member(notif_uri, "rsync://example.net/repo/c.crl") + .expect("is member"), + "c added to rrdp repo index" + ); + + let a_view = store + .get_repository_view_entry("rsync://example.net/repo/a.mft") + .expect("get a view") + .expect("a view exists"); + assert_eq!(a_view.state, crate::storage::RepositoryViewState::Withdrawn); + let b_view = store + .get_repository_view_entry("rsync://example.net/repo/b.roa") + .expect("get b view") + .expect("b view exists"); + assert_eq!(b_view.state, crate::storage::RepositoryViewState::Present); + assert_eq!( + b_view.current_hash.as_deref(), + Some(hex::encode(sha2::Sha256::digest(b"b2")).as_str()) + ); + let c_owner = store + .get_rrdp_uri_owner_record("rsync://example.net/repo/c.crl") + .expect("get c owner") + .expect("c owner exists"); + assert_eq!( + c_owner.owner_state, + crate::storage::RrdpUriOwnerState::Active + ); + let a_member = store + .get_rrdp_source_member_record(notif_uri, "rsync://example.net/repo/a.mft") + .expect("get a member") + .expect("a member exists"); + assert!(!a_member.present); + let current_members = store + .list_current_rrdp_source_members(notif_uri) + .expect("list current members"); + assert_eq!( + current_members + .iter() + .map(|record| record.rsync_uri.as_str()) + .collect::>(), + vec![ + "rsync://example.net/repo/b.roa", + "rsync://example.net/repo/c.crl", + ] + ); +} + +#[test] +fn apply_delta_rejects_hash_mismatch() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); + let notif_uri = "https://example.net/notification.xml"; + + let delta = delta_xml( + sid.to_string().as_str(), + 1, + &[r#"QQ=="#], + ); + let mut wrong = [0u8; 32]; + wrong[0] = 1; + let err = apply_delta(&store, notif_uri, None, &delta, wrong, sid, 1).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::DeltaHashMismatch) + )); +} + +#[test] +fn apply_delta_rejects_withdraw_of_non_member() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); + let notif_uri = "https://example.net/notification.xml"; + + let withdraw_hash = "00".repeat(32); + let delta = delta_xml( + sid.to_string().as_str(), + 1, + &[&format!( + r#""# + )], + ); + let delta_hash = sha2::Sha256::digest(&delta); + let mut expected_hash = [0u8; 32]; + expected_hash.copy_from_slice(delta_hash.as_slice()); + + let err = apply_delta(&store, notif_uri, None, &delta, expected_hash, sid, 1).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::DeltaTargetNotFromRepository { .. }) + )); +} + +#[test] +fn apply_delta_rejects_publish_without_hash_for_existing_object() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + + // Seed snapshot with a.mft. + let snapshot_uri = "https://example.net/snapshot.xml"; + let snapshot = snapshot_xml(sid, 1, &[("rsync://example.net/repo/a.mft", b"a1")]); + let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); + let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash); + let fetcher = MapFetcher { + map: HashMap::from([(snapshot_uri.to_string(), snapshot)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).expect("seed"); + + // Replace publish for an existing URI must have @hash. + let publish_b64 = base64::engine::general_purpose::STANDARD.encode(b"a2"); + let delta = delta_xml( + sid, + 2, + &[&format!( + r#"{publish_b64}"# + )], + ); + let delta_hash = sha2::Sha256::digest(&delta); + let mut expected_hash = [0u8; 32]; + expected_hash.copy_from_slice(delta_hash.as_slice()); + + let err = apply_delta( + &store, + notif_uri, + None, + &delta, + expected_hash, + Uuid::parse_str(sid).unwrap(), + 2, + ) + .unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::DeltaPublishWithoutHashForExisting { .. }) + )); +} + +#[test] +fn apply_delta_rejects_target_missing_and_hash_mismatch() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + + // Seed snapshot with a.mft. + let snapshot_uri = "https://example.net/snapshot.xml"; + let snapshot = snapshot_xml(sid, 1, &[("rsync://example.net/repo/a.mft", b"a1")]); + let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); + let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash); + let fetcher = MapFetcher { + map: HashMap::from([(snapshot_uri.to_string(), snapshot)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).expect("seed"); + + let old_bytes = store + .load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft") + .expect("get") + .expect("present"); + let old_hash = hex::encode(sha2::Sha256::digest(old_bytes.as_slice())); + + // Hash mismatch on withdraw. + let wrong_hash = "11".repeat(32); + let delta = delta_xml( + sid, + 2, + &[&format!( + r#""# + )], + ); + let delta_hash = sha2::Sha256::digest(&delta); + let mut expected_hash = [0u8; 32]; + expected_hash.copy_from_slice(delta_hash.as_slice()); + let err = apply_delta( + &store, + notif_uri, + None, + &delta, + expected_hash, + Uuid::parse_str(sid).unwrap(), + 2, + ) + .unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::DeltaTargetHashMismatch { .. }) + )); + + // Target missing in local cache (index still says it's a member). + store + .delete_repository_view_entry("rsync://example.net/repo/a.mft") + .expect("delete current repository view entry"); + let delta = delta_xml( + sid, + 2, + &[&format!( + r#""# + )], + ); + let delta_hash = sha2::Sha256::digest(&delta); + let mut expected_hash = [0u8; 32]; + expected_hash.copy_from_slice(delta_hash.as_slice()); + let err = apply_delta( + &store, + notif_uri, + None, + &delta, + expected_hash, + Uuid::parse_str(sid).unwrap(), + 2, + ) + .unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::DeltaTargetMissing { .. }) + )); +} + +#[test] +fn apply_delta_rejects_session_and_serial_mismatch() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + + let publish_b64 = base64::engine::general_purpose::STANDARD.encode(b"x"); + let delta = delta_xml( + sid, + 2, + &[&format!( + r#"{publish_b64}"# + )], + ); + let delta_hash = sha2::Sha256::digest(&delta); + let mut expected_hash = [0u8; 32]; + expected_hash.copy_from_slice(delta_hash.as_slice()); + + // Session mismatch. + let other_sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440001").unwrap(); + let err = + apply_delta(&store, notif_uri, None, &delta, expected_hash, other_sid, 2).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::DeltaSessionIdMismatch { .. }) + )); + + // Serial mismatch. + let err = apply_delta( + &store, + notif_uri, + None, + &delta, + expected_hash, + Uuid::parse_str(sid).unwrap(), + 3, + ) + .unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::DeltaSerialMismatch { .. }) + )); +} diff --git a/crates/panda-rpki-validator/src/sync/rrdp/tests_parts/edge_cases.rs b/crates/panda-rpki-validator/src/sync/rrdp/tests_parts/edge_cases.rs new file mode 100644 index 0000000..2dc0107 --- /dev/null +++ b/crates/panda-rpki-validator/src/sync/rrdp/tests_parts/edge_cases.rs @@ -0,0 +1,214 @@ +// RRDP test group: edge cases. + +#[test] +fn load_rrdp_local_state_uses_source_record_only() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + let notif_uri = "https://example.net/notification.xml"; + assert_eq!( + load_rrdp_local_state(&store, notif_uri).expect("load empty"), + None + ); + + update_rrdp_source_record_on_success( + &store, + notif_uri, + "source-session", + 9, + crate::storage::RrdpSourceSyncState::DeltaReady, + Some("https://example.net/snapshot.xml"), + Some(&hex::encode([0x11; 32])), + ) + .expect("write source record"); + + let got = load_rrdp_local_state(&store, notif_uri) + .expect("load source preferred") + .expect("source present"); + assert_eq!( + got, + RrdpState { + session_id: "source-session".to_string(), + serial: 9, + } + ); +} + +#[test] +fn sync_from_notification_falls_back_to_snapshot_if_missing_required_deltas() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + + // Seed state serial=1 with a only. + let snapshot_uri_1 = "https://example.net/snapshot-1.xml"; + let snapshot_1 = snapshot_xml(sid, 1, &[("rsync://example.net/repo/a.mft", b"a1")]); + let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1)); + let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1); + let fetcher_1 = MapFetcher { + map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if_1, &fetcher_1).expect("seed"); + + // Notification serial=3 only includes delta serial=3 (contiguous per RFC, but missing + // serial=2 relative to our local state, so we must use snapshot). + let snapshot_uri_3 = "https://example.net/snapshot-3.xml"; + let snapshot_3 = snapshot_xml(sid, 3, &[("rsync://example.net/repo/z.roa", b"z3")]); + let snapshot_hash_3 = hex::encode(sha2::Sha256::digest(&snapshot_3)); + let delta_3_hash_hex = "11".repeat(32); + let notif_3 = notification_xml_with_deltas( + sid, + 3, + snapshot_uri_3, + &snapshot_hash_3, + &[( + "d3", + 3, + "https://example.net/delta-3.xml", + &delta_3_hash_hex, + )], + ); + + let fetcher = MapFetcher { + map: HashMap::from([(snapshot_uri_3.to_string(), snapshot_3)]), + }; + + let published = sync_from_notification(&store, notif_uri, ¬if_3, &fetcher).expect("sync"); + assert_eq!(published, 1); + assert!( + store + .load_current_object_bytes_by_uri("rsync://example.net/repo/z.roa") + .expect("get current") + .is_some() + ); +} + +#[test] +fn sync_from_notification_snapshot_rejects_snapshot_hash_mismatch() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let serial = 1u64; + let notif_uri = "https://example.net/notification.xml"; + let snapshot_uri = "https://example.net/snapshot.xml"; + + let snapshot = snapshot_xml(sid, serial, &[("rsync://example.net/repo/a.mft", b"x")]); + let notif = notification_xml(sid, serial, snapshot_uri, &"00".repeat(32)); + + let fetcher = MapFetcher { + map: HashMap::from([(snapshot_uri.to_string(), snapshot)]), + }; + let err = sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::SnapshotHashMismatch) + )); +} + +#[test] +fn apply_snapshot_rejects_session_id_and_serial_mismatch() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + let notif_uri = "https://example.net/notification.xml"; + + let expected_sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); + let got_sid = "550e8400-e29b-41d4-a716-446655440001"; + + let snapshot = snapshot_xml(got_sid, 2, &[("rsync://example.net/repo/a.mft", b"x")]); + let err = apply_snapshot(&store, notif_uri, None, &snapshot, expected_sid, 2).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::SnapshotSessionIdMismatch { .. }) + )); + + let snapshot = snapshot_xml( + expected_sid.to_string().as_str(), + 3, + &[("rsync://example.net/repo/a.mft", b"x")], + ); + let err = apply_snapshot(&store, notif_uri, None, &snapshot, expected_sid, 2).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::SnapshotSerialMismatch { .. }) + )); +} + +#[test] +fn strip_all_ascii_whitespace_removes_newlines_and_spaces() { + assert_eq!(strip_all_ascii_whitespace(" a \n b\tc "), "abc"); +} + +#[test] +fn apply_snapshot_reports_publish_errors() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + let notif_uri = "https://example.net/notification.xml"; + let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); + + // Missing publish/@uri + let xml = format!( + r#"AA=="# + ) + .into_bytes(); + let err = apply_snapshot(&store, notif_uri, None, &xml, sid, 1).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::PublishUriMissing) + )); + + // Missing base64 content (no text nodes). + let xml = format!( + r#""# + ) + .into_bytes(); + let err = apply_snapshot(&store, notif_uri, None, &xml, sid, 1).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::PublishContentMissing) + )); + + // Invalid base64 content. + let xml = format!( + r#"!!!"# + ) + .into_bytes(); + let err = apply_snapshot(&store, notif_uri, None, &xml, sid, 1).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::PublishBase64(_)) + )); +} + +#[test] +fn apply_snapshot_handles_multiple_publish_batches() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + let notif_uri = "https://example.net/notification.xml"; + let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); + + let total = RRDP_SNAPSHOT_APPLY_BATCH_SIZE + 7; + let mut xml = + format!(r#""#); + for i in 0..total { + let uri = format!("rsync://example.net/repo/{i:04}.roa"); + let bytes = format!("payload-{i}").into_bytes(); + let b64 = base64::engine::general_purpose::STANDARD.encode(bytes); + xml.push_str(&format!(r#"{b64}"#)); + } + xml.push_str(""); + + let published = + apply_snapshot(&store, notif_uri, None, xml.as_bytes(), sid, 1).expect("apply snapshot"); + assert_eq!(published, total); + + for idx in [0usize, RRDP_SNAPSHOT_APPLY_BATCH_SIZE - 1, total - 1] { + let uri = format!("rsync://example.net/repo/{idx:04}.roa"); + let got = store + .load_current_object_bytes_by_uri(&uri) + .expect("load object") + .expect("object exists"); + assert_eq!(got, format!("payload-{idx}").into_bytes()); + } +} diff --git a/crates/panda-rpki-validator/src/sync/rrdp/tests_parts/parsing.rs b/crates/panda-rpki-validator/src/sync/rrdp/tests_parts/parsing.rs new file mode 100644 index 0000000..c932a87 --- /dev/null +++ b/crates/panda-rpki-validator/src/sync/rrdp/tests_parts/parsing.rs @@ -0,0 +1,367 @@ +// RRDP test group: parsing. + +use super::*; +use crate::analysis::timing::{TimingHandle, TimingMeta}; +use crate::current_repo_index::CurrentRepoIndex; +use crate::storage::RocksStore; +use std::collections::HashMap; +use std::io::Read; +use std::time::Duration; + +struct MapFetcher { + map: HashMap>, +} + +impl Fetcher for MapFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + self.map + .get(uri) + .cloned() + .ok_or_else(|| format!("not found: {uri}")) + } +} + +struct SleepyFetcher { + inner: MapFetcher, + sleep_uri: String, + sleep: Duration, +} + +impl Fetcher for SleepyFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + if uri == self.sleep_uri { + std::thread::sleep(self.sleep); + } + self.inner.fetch(uri) + } +} + +struct WriterOnlyFetcher { + map: HashMap>, +} + +impl Fetcher for WriterOnlyFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + Err(format!("unexpected buffered fetch: {uri}")) + } + + fn fetch_to_writer(&self, uri: &str, out: &mut dyn std::io::Write) -> Result { + let bytes = self + .map + .get(uri) + .ok_or_else(|| format!("not found: {uri}"))?; + out.write_all(bytes) + .map_err(|e| format!("write sink failed: {e}"))?; + Ok(bytes.len() as u64) + } +} + +struct NonAsciiWriterFetcher; + +impl Fetcher for NonAsciiWriterFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + Err(format!("unexpected buffered fetch: {uri}")) + } + + fn fetch_to_writer(&self, _uri: &str, out: &mut dyn std::io::Write) -> Result { + out.write_all(&[0x80]) + .map_err(|e| format!("write sink failed: {e}"))?; + Err("snapshot body contains non-ASCII bytes".to_string()) + } +} + +fn assert_current_object(store: &RocksStore, uri: &str, expected: &[u8]) { + assert_eq!( + store + .load_current_object_bytes_by_uri(uri) + .expect("load current object"), + Some(expected.to_vec()) + ); +} + +fn notification_xml( + session_id: &str, + serial: u64, + snapshot_uri: &str, + snapshot_hash: &str, +) -> Vec { + format!( + r#""# + ) + .into_bytes() +} + +fn notification_xml_with_deltas( + session_id: &str, + serial: u64, + snapshot_uri: &str, + snapshot_hash: &str, + deltas: &[(&str, u64, &str, &str)], +) -> Vec { + let mut out = format!( + r#""# + ); + for (_name, delta_serial, uri, hash) in deltas { + out.push_str(&format!( + r#""# + )); + } + out.push_str(""); + out.into_bytes() +} + +fn snapshot_xml(session_id: &str, serial: u64, published: &[(&str, &[u8])]) -> Vec { + let mut out = format!( + r#""# + ); + for (uri, bytes) in published { + let b64 = base64::engine::general_purpose::STANDARD.encode(bytes); + out.push_str(&format!(r#"{b64}"#)); + } + out.push_str(""); + out.into_bytes() +} + +#[test] +fn fetch_snapshot_into_tempfile_streams_and_validates_hash() { + let snapshot_uri = "https://example.test/snapshot.xml"; + let snapshot = b"".to_vec(); + let mut expected_hash = [0u8; 32]; + expected_hash.copy_from_slice(&sha2::Sha256::digest(&snapshot)); + let fetcher = WriterOnlyFetcher { + map: HashMap::from([(snapshot_uri.to_string(), snapshot.clone())]), + }; + + let (mut file, bytes_written) = + fetch_snapshot_into_tempfile(&fetcher, snapshot_uri, &expected_hash) + .expect("fetch snapshot into tempfile"); + assert_eq!(bytes_written, snapshot.len() as u64); + let mut got = Vec::new(); + file.as_file_mut() + .read_to_end(&mut got) + .expect("read tempfile"); + assert_eq!(got, snapshot); + + let mut wrong_hash = expected_hash; + wrong_hash[0] ^= 0xff; + let err = fetch_snapshot_into_tempfile(&fetcher, snapshot_uri, &wrong_hash).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::SnapshotHashMismatch) + )); +} + +#[test] +fn fetch_snapshot_into_tempfile_maps_stream_non_ascii_error() { + let err = fetch_snapshot_into_tempfile( + &NonAsciiWriterFetcher, + "https://example.test/snapshot.xml", + &[0u8; 32], + ) + .unwrap_err(); + assert!(matches!(err, RrdpSyncError::Rrdp(RrdpError::NotAscii))); +} + +#[test] +fn timing_rrdp_repo_step_spans_cover_snapshot_fetch_duration() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + + let notification_uri = "https://example.test/notification.xml"; + let snapshot_uri = "https://example.test/snapshot.xml"; + let published_uri = "rsync://example.test/repo/a.mft"; + let published_bytes = b"x"; + let session_id = "550e8400-e29b-41d4-a716-446655440000"; + + let snapshot = snapshot_xml(session_id, 1, &[(published_uri, published_bytes)]); + let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); + let notif = notification_xml(session_id, 1, snapshot_uri, &snapshot_hash); + + let mut map = HashMap::new(); + map.insert(snapshot_uri.to_string(), snapshot); + let fetcher = SleepyFetcher { + inner: MapFetcher { map }, + sleep_uri: snapshot_uri.to_string(), + sleep: Duration::from_millis(25), + }; + + let timing = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + tal_url: None, + db_path: Some(store_dir.to_string_lossy().into_owned()), + }); + + sync_from_notification_snapshot_with_timing( + &store, + notification_uri, + ¬if, + &fetcher, + Some(&timing), + ) + .expect("rrdp snapshot sync ok"); + + let timing_path = temp.path().join("timing.json"); + timing.write_json(&timing_path, 200).expect("write timing"); + let rep: serde_json::Value = + serde_json::from_slice(&std::fs::read(&timing_path).expect("read timing")) + .expect("parse timing"); + + let want = format!("{notification_uri}::fetch_snapshot"); + let steps = rep + .get("top_rrdp_repo_steps") + .and_then(|v| v.as_array()) + .expect("top_rrdp_repo_steps array"); + let entry = steps + .iter() + .find(|e| e.get("key").and_then(|k| k.as_str()) == Some(want.as_str())) + .unwrap_or_else(|| panic!("missing timing step entry for {want}")); + let nanos = entry + .get("total_nanos") + .and_then(|v| v.as_u64()) + .expect("total_nanos"); + assert!( + nanos >= 20_000_000, + "expected fetch_snapshot timing to include the fetch duration; got {nanos}ns" + ); +} + +#[test] +fn parse_notification_snapshot_rejects_non_ascii() { + let mut xml = b"".to_vec(); + xml.push(0x80); + let err = parse_notification_snapshot(&xml).unwrap_err(); + assert!(matches!(err, RrdpError::NotAscii)); +} + +#[test] +fn parse_notification_snapshot_parses_valid_minimal_notification() { + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let snapshot_uri = "https://example.net/snapshot.xml"; + let hash = "00".repeat(32); + let xml = notification_xml(sid, 7, snapshot_uri, &hash); + let n = parse_notification_snapshot(&xml).expect("parse"); + assert_eq!(n.session_id, Uuid::parse_str(sid).unwrap()); + assert_eq!(n.serial, 7); + assert_eq!(n.snapshot_uri, snapshot_uri); + assert_eq!(hex::encode(n.snapshot_hash_sha256), hash); +} + +#[test] +fn parse_notification_parses_deltas_and_validates_contiguity() { + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let snapshot_uri = "https://example.net/snapshot.xml"; + let hash = "00".repeat(32); + let d_hash_2 = "11".repeat(32); + let d_hash_3 = "22".repeat(32); + // Provide deltas in reverse order to ensure we sort. + let xml = notification_xml_with_deltas( + sid, + 3, + snapshot_uri, + &hash, + &[ + ("d3", 3, "https://example.net/delta-3.xml", &d_hash_3), + ("d2", 2, "https://example.net/delta-2.xml", &d_hash_2), + ], + ); + let n = parse_notification(&xml).expect("parse notification"); + assert_eq!(n.serial, 3); + assert_eq!(n.deltas.len(), 2); + assert_eq!(n.deltas[0].serial, 2); + assert_eq!(n.deltas[1].serial, 3); + assert_eq!(n.deltas[0].uri, "https://example.net/delta-2.xml"); + assert_eq!(hex::encode(n.deltas[1].hash_sha256), d_hash_3); +} + +#[test] +fn parse_notification_rejects_non_contiguous_deltas() { + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let snapshot_uri = "https://example.net/snapshot.xml"; + let hash = "00".repeat(32); + let d_hash_1 = "11".repeat(32); + let d_hash_3 = "22".repeat(32); + // Missing delta serial 2. + let xml = notification_xml_with_deltas( + sid, + 3, + snapshot_uri, + &hash, + &[ + ("d3", 3, "https://example.net/delta-3.xml", &d_hash_3), + ("d1", 1, "https://example.net/delta-1.xml", &d_hash_1), + ], + ); + let err = parse_notification(&xml).unwrap_err(); + assert!(matches!(err, RrdpError::DeltaRefChainNotContiguous { .. })); +} + +fn delta_xml(session_id: &str, serial: u64, elements: &[&str]) -> Vec { + let mut out = format!( + r#""# + ); + for e in elements { + out.push_str(e); + } + out.push_str(""); + out.into_bytes() +} + +#[test] +fn parse_delta_file_parses_publish_and_withdraw() { + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let serial = 3u64; + let publish_bytes = b"abc"; + let publish_b64 = base64::engine::general_purpose::STANDARD.encode(publish_bytes); + let withdraw_hash = "33".repeat(32); + + let xml = delta_xml( + sid, + serial, + &[ + &format!(r#"{publish_b64}"#), + &format!(r#""#), + ], + ); + + let d = parse_delta_file(&xml).expect("parse delta"); + assert_eq!(d.session_id, Uuid::parse_str(sid).unwrap()); + assert_eq!(d.serial, serial); + assert_eq!(d.elements.len(), 2); + match &d.elements[0] { + DeltaElement::Publish { + uri, + hash_sha256, + bytes, + } => { + assert_eq!(uri, "rsync://example.net/repo/a.mft"); + assert_eq!(*hash_sha256, None); + assert_eq!(bytes, publish_bytes); + } + _ => panic!("expected publish"), + } + match &d.elements[1] { + DeltaElement::Withdraw { uri, hash_sha256 } => { + assert_eq!(uri, "rsync://example.net/repo/b.cer"); + assert_eq!(hex::encode(hash_sha256), withdraw_hash); + } + _ => panic!("expected withdraw"), + } +} + +#[test] +fn parse_delta_file_rejects_withdraw_with_content() { + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let serial = 1u64; + let withdraw_hash = "33".repeat(32); + let xml = delta_xml( + sid, + serial, + &[&format!( + r#"AA=="# + )], + ); + let err = parse_delta_file(&xml).unwrap_err(); + assert!(matches!(err, RrdpError::DeltaWithdrawUnexpectedContent)); +} diff --git a/crates/panda-rpki-validator/src/sync/rrdp/tests_parts/sync.rs b/crates/panda-rpki-validator/src/sync/rrdp/tests_parts/sync.rs new file mode 100644 index 0000000..f04f6f4 --- /dev/null +++ b/crates/panda-rpki-validator/src/sync/rrdp/tests_parts/sync.rs @@ -0,0 +1,381 @@ +// RRDP test group: sync. + +#[test] +fn sync_from_notification_snapshot_rejects_cross_source_owner_conflict() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid_a = "550e8400-e29b-41d4-a716-446655440000"; + let sid_b = "550e8400-e29b-41d4-a716-446655440001"; + let uri = "rsync://example.net/repo/a.mft"; + + let notif_a_uri = "https://example.net/a/notification.xml"; + let snapshot_a_uri = "https://example.net/a/snapshot.xml"; + let snapshot_a = snapshot_xml(sid_a, 1, &[(uri, b"a1")]); + let snapshot_a_hash = hex::encode(sha2::Sha256::digest(&snapshot_a)); + let notif_a = notification_xml(sid_a, 1, snapshot_a_uri, &snapshot_a_hash); + let fetcher_a = MapFetcher { + map: HashMap::from([(snapshot_a_uri.to_string(), snapshot_a)]), + }; + sync_from_notification_snapshot(&store, notif_a_uri, ¬if_a, &fetcher_a) + .expect("seed source a"); + + let notif_b_uri = "https://example.net/b/notification.xml"; + let snapshot_b_uri = "https://example.net/b/snapshot.xml"; + let snapshot_b = snapshot_xml(sid_b, 1, &[(uri, b"b1")]); + let snapshot_b_hash = hex::encode(sha2::Sha256::digest(&snapshot_b)); + let notif_b = notification_xml(sid_b, 1, snapshot_b_uri, &snapshot_b_hash); + let fetcher_b = MapFetcher { + map: HashMap::from([(snapshot_b_uri.to_string(), snapshot_b)]), + }; + + let err = sync_from_notification_snapshot(&store, notif_b_uri, ¬if_b, &fetcher_b) + .expect_err("cross-source overwrite must fail"); + assert!(matches!(err, RrdpSyncError::Storage(_))); + assert!(err.to_string().contains("owner conflict"), "{err}"); +} + +#[test] +fn sync_from_notification_snapshot_applies_snapshot_and_stores_state() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let serial = 9u64; + let notif_uri = "https://example.net/notification.xml"; + let snapshot_uri = "https://example.net/snapshot.xml"; + + let snapshot = snapshot_xml( + sid, + serial, + &[ + ("rsync://example.net/repo/a.mft", b"mft-bytes"), + ("rsync://example.net/repo/b.roa", b"roa-bytes"), + ], + ); + let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); + let notif = notification_xml(sid, serial, snapshot_uri, &snapshot_hash); + + let fetcher = MapFetcher { + map: HashMap::from([(snapshot_uri.to_string(), snapshot.clone())]), + }; + + let published = + sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).expect("sync"); + assert_eq!(published, 2); + + assert_current_object(&store, "rsync://example.net/repo/a.mft", b"mft-bytes"); + assert_current_object(&store, "rsync://example.net/repo/b.roa", b"roa-bytes"); + + let state = load_rrdp_local_state(&store, notif_uri) + .expect("get rrdp state") + .expect("state present"); + assert_eq!(state.session_id, sid); + assert_eq!(state.serial, serial); + + let source = store + .get_rrdp_source_record(notif_uri) + .expect("get rrdp source") + .expect("rrdp source exists"); + assert_eq!(source.last_session_id.as_deref(), Some(sid)); + assert_eq!(source.last_serial, Some(serial)); + assert_eq!( + source.sync_state, + crate::storage::RrdpSourceSyncState::SnapshotOnly + ); + + let view = store + .get_repository_view_entry("rsync://example.net/repo/a.mft") + .expect("get repository view") + .expect("repository view exists"); + assert_eq!(view.state, crate::storage::RepositoryViewState::Present); + assert_eq!(view.repository_source.as_deref(), Some(notif_uri)); + + let current_bytes = store + .load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft") + .expect("load current bytes") + .expect("current object bytes exist"); + assert_eq!(current_bytes, b"mft-bytes".to_vec()); + assert!( + store + .get_raw_by_hash_entry(hex::encode(sha2::Sha256::digest(b"mft-bytes")).as_str()) + .expect("get raw_by_hash") + .is_none() + ); + + let member = store + .get_rrdp_source_member_record(notif_uri, "rsync://example.net/repo/a.mft") + .expect("get member") + .expect("member exists"); + assert!(member.present); + let owner = store + .get_rrdp_uri_owner_record("rsync://example.net/repo/a.mft") + .expect("get owner") + .expect("owner exists"); + assert_eq!(owner.notify_uri, notif_uri); + assert_eq!(owner.owner_state, crate::storage::RrdpUriOwnerState::Active); +} + +#[test] +fn sync_from_notification_snapshot_deletes_objects_not_in_new_snapshot() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + + // serial 1: publish a + b + let snapshot_uri_1 = "https://example.net/snapshot-1.xml"; + let snapshot_1 = snapshot_xml( + sid, + 1, + &[ + ("rsync://example.net/repo/a.mft", b"a1"), + ("rsync://example.net/repo/b.roa", b"b1"), + ], + ); + let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1)); + let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1); + + let fetcher_1 = MapFetcher { + map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if_1, &fetcher_1).expect("sync 1"); + + // serial 2: publish b (new bytes) + c, and drop a + let snapshot_uri_2 = "https://example.net/snapshot-2.xml"; + let snapshot_2 = snapshot_xml( + sid, + 2, + &[ + ("rsync://example.net/repo/b.roa", b"b2"), + ("rsync://example.net/repo/c.crl", b"c2"), + ], + ); + let snapshot_hash_2 = hex::encode(sha2::Sha256::digest(&snapshot_2)); + let notif_2 = notification_xml(sid, 2, snapshot_uri_2, &snapshot_hash_2); + + let fetcher_2 = MapFetcher { + map: HashMap::from([(snapshot_uri_2.to_string(), snapshot_2)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if_2, &fetcher_2).expect("sync 2"); + + assert!( + store + .load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft") + .expect("get current a") + .is_none(), + "a should be deleted by full-state snapshot apply" + ); + + assert_current_object(&store, "rsync://example.net/repo/b.roa", b"b2"); + assert_current_object(&store, "rsync://example.net/repo/c.crl", b"c2"); +} + +#[test] +fn sync_from_notification_uses_deltas_when_available_for_local_state() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + + // Seed state with snapshot serial=1 containing a+b. + let snapshot_uri_1 = "https://example.net/snapshot-1.xml"; + let snapshot_1 = snapshot_xml( + sid, + 1, + &[ + ("rsync://example.net/repo/a.mft", b"a1"), + ("rsync://example.net/repo/b.roa", b"b1"), + ], + ); + let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1)); + let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1); + let fetcher_1 = MapFetcher { + map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if_1, &fetcher_1).expect("seed"); + + // Notification serial=3 with deltas 2 and 3. Snapshot URI is intentionally not fetchable + // to assert we really use deltas. + let snapshot_uri_3 = "https://example.net/snapshot-3.xml"; + let snapshot_hash_3 = "00".repeat(32); + + let publish_c_b64 = base64::engine::general_purpose::STANDARD.encode(b"c2"); + let delta_2 = delta_xml( + sid, + 2, + &[&format!( + r#"{publish_c_b64}"# + )], + ); + let delta_2_hash_hex = hex::encode(sha2::Sha256::digest(&delta_2)); + + let c_hash_hex = hex::encode(sha2::Sha256::digest(b"c2".as_slice())); + let delta_3 = delta_xml( + sid, + 3, + &[&format!( + r#""# + )], + ); + let delta_3_hash_hex = hex::encode(sha2::Sha256::digest(&delta_3)); + + let notif_3 = notification_xml_with_deltas( + sid, + 3, + snapshot_uri_3, + &snapshot_hash_3, + &[ + ( + "d3", + 3, + "https://example.net/delta-3.xml", + &delta_3_hash_hex, + ), + ( + "d2", + 2, + "https://example.net/delta-2.xml", + &delta_2_hash_hex, + ), + ], + ); + + let fetcher = MapFetcher { + map: HashMap::from([ + ("https://example.net/delta-2.xml".to_string(), delta_2), + ("https://example.net/delta-3.xml".to_string(), delta_3), + ]), + }; + + let applied = sync_from_notification(&store, notif_uri, ¬if_3, &fetcher).expect("sync"); + assert!(applied > 0); + + // Delta 2 publishes c then delta 3 withdraws it => final state should not contain c. + assert!( + store + .load_current_object_bytes_by_uri("rsync://example.net/repo/c.crl") + .expect("get current") + .is_none() + ); + + let state = load_rrdp_local_state(&store, notif_uri) + .expect("get rrdp state") + .expect("state present"); + assert_eq!(state.session_id, Uuid::parse_str(sid).unwrap().to_string()); + assert_eq!(state.serial, 3); +} + +#[test] +fn sync_from_notification_same_serial_hydrates_current_repo_index() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + let snapshot_uri = "https://example.net/snapshot.xml"; + let uri_a = "rsync://example.net/repo/a.mft"; + let uri_b = "rsync://example.net/repo/b.roa"; + + let snapshot = snapshot_xml(sid, 1, &[(uri_a, b"a1"), (uri_b, b"b1")]); + let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); + let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash); + let fetcher_1 = MapFetcher { + map: HashMap::from([(snapshot_uri.to_string(), snapshot)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher_1).expect("seed"); + + let index = CurrentRepoIndex::shared(); + let no_fetcher = MapFetcher { + map: HashMap::new(), + }; + let applied = sync_from_notification_with_timing_and_download_log( + &store, + notif_uri, + Some(&index), + ¬if, + &no_fetcher, + None, + None, + ) + .expect("same serial no-op"); + assert_eq!(applied, 0); + + let index = index.read().expect("read-lock index"); + assert_eq!(index.active_uri_count(), 2); + assert!(index.get_by_uri(uri_a).is_some()); + assert!(index.get_by_uri(uri_b).is_some()); +} + +#[test] +fn sync_from_notification_delta_hydrates_unchanged_current_repo_entries() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + let snapshot_uri_1 = "https://example.net/snapshot-1.xml"; + let uri_a = "rsync://example.net/repo/a.mft"; + let uri_b = "rsync://example.net/repo/b.roa"; + let uri_c = "rsync://example.net/repo/c.crl"; + + let snapshot_1 = snapshot_xml(sid, 1, &[(uri_a, b"a1"), (uri_b, b"b1")]); + let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1)); + let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1); + let fetcher_1 = MapFetcher { + map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if_1, &fetcher_1).expect("seed"); + + let publish_c_b64 = base64::engine::general_purpose::STANDARD.encode(b"c2"); + let delta_2 = delta_xml( + sid, + 2, + &[&format!( + r#"{publish_c_b64}"# + )], + ); + let delta_2_hash_hex = hex::encode(sha2::Sha256::digest(&delta_2)); + let notif_2 = notification_xml_with_deltas( + sid, + 2, + "https://example.net/snapshot-2.xml", + &"00".repeat(32), + &[( + "d2", + 2, + "https://example.net/delta-2.xml", + &delta_2_hash_hex, + )], + ); + let fetcher_2 = MapFetcher { + map: HashMap::from([("https://example.net/delta-2.xml".to_string(), delta_2)]), + }; + + let index = CurrentRepoIndex::shared(); + let applied = sync_from_notification_with_timing_and_download_log( + &store, + notif_uri, + Some(&index), + ¬if_2, + &fetcher_2, + None, + None, + ) + .expect("delta sync"); + assert_eq!(applied, 1); + + let index = index.read().expect("read-lock index"); + assert_eq!(index.active_uri_count(), 3); + assert!( + index.get_by_uri(uri_a).is_some(), + "unchanged object from the previous serial must be visible" + ); + assert!( + index.get_by_uri(uri_b).is_some(), + "unchanged object from the previous serial must be visible" + ); + assert!(index.get_by_uri(uri_c).is_some(), "delta publish visible"); +} diff --git a/crates/panda-rpki-validator/src/ta_constraints.rs b/crates/panda-rpki-validator/src/ta_constraints.rs index cf9dea1..6abe861 100644 --- a/crates/panda-rpki-validator/src/ta_constraints.rs +++ b/crates/panda-rpki-validator/src/ta_constraints.rs @@ -1,1139 +1,9 @@ //! Locally configured constraints for the resources carried by RPKI EE -//! certificates. The configuration format follows +//! certificates. The configuration format follows //! draft-ietf-sidrops-constraining-rpki-trust-anchors. -use std::collections::{BTreeMap, BTreeSet}; -use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; -use std::ops::Deref; -use std::path::{Path, PathBuf}; - -use sha2::Digest; - -use crate::data_model::rc::{ - Afi, AsIdOrRange, AsIdentifierChoice, IpAddressChoice, IpAddressOrRange, IpResourceSet, - ResourceCertificate, -}; -use crate::parallel::types::{TalInputSpec, TalSource}; - -const LINEAR_INTERVAL_THRESHOLD: usize = 10; - -/// A normalized rule set with a small-set linear path and a large-set tree path. -/// -/// The parser constructs this only after sorting and merging overlapping or -/// adjacent intervals. The tree lookup therefore only needs to inspect the -/// predecessor of a target interval; normalized intervals are disjoint and -/// sorted by their lower bound. -#[derive(Clone)] -struct IntervalIndex { - rules: Vec, - lookup: IntervalLookup, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -enum IntervalLookup { - Linear, - Tree(BTreeMap), -} - -impl std::fmt::Debug for IntervalIndex { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - // Keep derived acceleration data out of Debug so diagnostics continue - // to describe canonical rules rather than their index layout. - self.rules.fmt(formatter) - } -} - -impl PartialEq for IntervalIndex { - fn eq(&self, other: &Self) -> bool { - self.rules == other.rules - } -} - -impl Eq for IntervalIndex {} - -impl Deref for IntervalIndex { - type Target = [I]; - - fn deref(&self) -> &Self::Target { - &self.rules - } -} - -trait IntervalValue { - fn start(&self) -> u128; - fn end(&self) -> u128; - fn overlaps(&self, other: &Self) -> bool; -} - -impl IntervalIndex { - fn new(rules: Vec) -> Self { - debug_assert!( - rules - .windows(2) - .all(|window| { window[0].end().saturating_add(1) < window[1].start() }) - ); - let lookup = if rules.len() > LINEAR_INTERVAL_THRESHOLD { - let tree = rules - .iter() - .map(|rule| (rule.start(), rule.end())) - .collect(); - IntervalLookup::Tree(tree) - } else { - IntervalLookup::Linear - }; - Self { rules, lookup } - } - - fn any_overlaps(&self, target: &I) -> bool { - match &self.lookup { - IntervalLookup::Linear => self.rules.iter().any(|entry| entry.overlaps(target)), - IntervalLookup::Tree(tree) => tree - .range(..=target.end()) - .next_back() - .map(|(_, end)| *end >= target.start()) - .unwrap_or(false), - } - } - - fn fully_covers(&self, target: &I) -> bool { - match &self.lookup { - IntervalLookup::Linear => self.fully_covers_linear(target), - IntervalLookup::Tree(tree) => tree - .range(..=target.start()) - .next_back() - .map(|(_, end)| *end >= target.end()) - .unwrap_or(false), - } - } - - fn fully_covers_linear(&self, target: &I) -> bool { - let mut cursor = target.start(); - for entry in &self.rules { - if entry.end() < cursor { - continue; - } - if entry.start() > cursor { - return false; - } - if entry.end() >= target.end() { - return true; - } - cursor = entry.end().saturating_add(1); - } - false - } - - #[cfg(test)] - fn uses_tree(&self) -> bool { - matches!(self.lookup, IntervalLookup::Tree(_)) - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct TaConstraintsByTal { - by_tal_id: BTreeMap>, - /// Stable semantic digest of the normalized rules. This is computed once - /// while loading the run policy so cache identity checks do not serialize - /// every rule for every publication point. - fingerprint: [u8; 32], -} - -impl TaConstraintsByTal { - pub fn load_for_tals( - tal_inputs: &[TalInputSpec], - explicit_specs: &[String], - ) -> Result { - let tal_ids = tal_inputs - .iter() - .map(|input| input.tal_id.as_str()) - .collect::>(); - let mut explicit_paths = BTreeMap::::new(); - for spec in explicit_specs { - let (tal_id, path) = spec - .split_once('=') - .ok_or_else(|| format!("--ta-constraints expects =, got '{spec}'"))?; - let tal_id = tal_id.trim(); - let path = path.trim(); - if tal_id.is_empty() || path.is_empty() { - return Err(format!( - "--ta-constraints expects non-empty =, got '{spec}'" - )); - } - if !tal_ids.contains(tal_id) { - return Err(format!( - "--ta-constraints references unknown TAL id '{tal_id}'" - )); - } - if explicit_paths - .insert(tal_id.to_string(), PathBuf::from(path)) - .is_some() - { - return Err(format!( - "--ta-constraints specifies TAL id '{tal_id}' more than once" - )); - } - } - - let mut by_tal_id = BTreeMap::new(); - for input in tal_inputs { - let path = explicit_paths - .get(&input.tal_id) - .cloned() - .or_else(|| adjacent_constraints_path(&input.source).filter(|path| path.is_file())); - if let Some(path) = path { - let constraints = TaConstraints::from_file(&path).map_err(|error| { - format!( - "load TA constraints for '{}' from {} failed: {error}", - input.tal_id, - path.display() - ) - })?; - by_tal_id.insert(input.tal_id.clone(), std::sync::Arc::new(constraints)); - } - } - let fingerprint = constraints_fingerprint(&by_tal_id); - Ok(Self { - by_tal_id, - fingerprint, - }) - } - - pub fn for_tal(&self, tal_id: &str) -> Option<&TaConstraints> { - self.by_tal_id.get(tal_id).map(std::sync::Arc::as_ref) - } - - /// Return the immutable, process-local snapshot for a TAL without cloning - /// the rule trees. Phase-2 object workers own this `Arc` in their task - /// payload, while scoped stage workers borrow the same policy map. - pub(crate) fn shared_for_tal(&self, tal_id: &str) -> Option> { - self.by_tal_id.get(tal_id).cloned() - } - - pub fn is_empty(&self) -> bool { - self.by_tal_id.is_empty() - } - - pub fn configuration_warnings(&self) -> Vec { - self.by_tal_id - .iter() - .flat_map(|(tal_id, constraints)| { - constraints.warnings().iter().map(move |warning| { - format!( - "TA constraints for TAL '{tal_id}' ({}): {warning}", - constraints.source().display() - ) - }) - }) - .collect() - } - - /// Return the precomputed SHA-256 digest of the normalized semantic rules. - /// - /// The returned bytes are stable across source paths and derived index - /// layouts, and are intentionally borrowed so hot-path cache lookups do - /// not allocate. - pub fn fingerprint_bytes(&self) -> &[u8] { - &self.fingerprint - } - - pub fn fingerprint_sha256_hex(&self) -> String { - hex::encode(self.fingerprint) - } -} - -impl Default for TaConstraintsByTal { - fn default() -> Self { - let by_tal_id = BTreeMap::new(); - let fingerprint = constraints_fingerprint(&by_tal_id); - Self { - by_tal_id, - fingerprint, - } - } -} - -const CONSTRAINTS_FINGERPRINT_VERSION: &[u8] = b"ta-constraints-semantic-v1"; - -fn constraints_fingerprint( - by_tal_id: &BTreeMap>, -) -> [u8; 32] { - let mut hasher = sha2::Sha256::new(); - hasher.update(CONSTRAINTS_FINGERPRINT_VERSION); - hasher.update((by_tal_id.len() as u64).to_be_bytes()); - for (tal_id, constraints) in by_tal_id { - update_length_prefixed(&mut hasher, tal_id.as_bytes()); - update_ip_intervals(&mut hasher, b"allow-v4", &constraints.allow_v4); - update_ip_intervals(&mut hasher, b"deny-v4", &constraints.deny_v4); - update_ip_intervals(&mut hasher, b"allow-v6", &constraints.allow_v6); - update_ip_intervals(&mut hasher, b"deny-v6", &constraints.deny_v6); - update_as_intervals(&mut hasher, b"allow-asn", &constraints.allow_asn); - update_as_intervals(&mut hasher, b"deny-asn", &constraints.deny_asn); - } - let digest = hasher.finalize(); - let mut out = [0u8; 32]; - out.copy_from_slice(&digest); - out -} - -fn update_length_prefixed(hasher: &mut sha2::Sha256, value: &[u8]) { - hasher.update((value.len() as u64).to_be_bytes()); - hasher.update(value); -} - -fn update_ip_intervals( - hasher: &mut sha2::Sha256, - label: &[u8], - intervals: &IntervalIndex, -) { - update_length_prefixed(hasher, label); - hasher.update((intervals.rules.len() as u64).to_be_bytes()); - for interval in &intervals.rules { - hasher.update([match interval.afi { - Afi::Ipv4 => 4, - Afi::Ipv6 => 6, - }]); - hasher.update(interval.min.to_be_bytes()); - hasher.update(interval.max.to_be_bytes()); - } -} - -fn update_as_intervals( - hasher: &mut sha2::Sha256, - label: &[u8], - intervals: &IntervalIndex, -) { - update_length_prefixed(hasher, label); - hasher.update((intervals.rules.len() as u64).to_be_bytes()); - for interval in &intervals.rules { - hasher.update(interval.min.to_be_bytes()); - hasher.update(interval.max.to_be_bytes()); - } -} - -fn adjacent_constraints_path(source: &TalSource) -> Option { - match source { - TalSource::FilePath(path) => Some(path.with_extension("constraints")), - TalSource::FilePathWithTa { tal_path, .. } => Some(tal_path.with_extension("constraints")), - TalSource::Url(_) | TalSource::DerBytes { .. } => None, - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct TaConstraints { - source: PathBuf, - allow_v4: IntervalIndex, - deny_v4: IntervalIndex, - allow_v6: IntervalIndex, - deny_v6: IntervalIndex, - allow_asn: IntervalIndex, - deny_asn: IntervalIndex, - warnings: Vec, -} - -impl TaConstraints { - pub fn from_file(path: &Path) -> Result { - let contents = std::fs::read_to_string(path) - .map_err(|error| format!("read {}: {error}", path.display()))?; - Self::parse(path.to_path_buf(), &contents) - } - - pub fn warnings(&self) -> &[String] { - &self.warnings - } - - pub fn source(&self) -> &Path { - &self.source - } - - pub fn validate_ee_certificate( - &self, - certificate: &ResourceCertificate, - ) -> Result<(), TaConstraintsViolation> { - if let Some(ip_resources) = certificate.tbs.extensions.ip_resources.as_ref() { - self.validate_ip_resources(ip_resources)?; - } - if let Some(as_resources) = certificate.tbs.extensions.as_resources.as_ref() { - self.validate_as_choice("AS", as_resources.asnum.as_ref())?; - self.validate_as_choice("RDI", as_resources.rdi.as_ref())?; - } - Ok(()) - } - - fn parse(source: PathBuf, contents: &str) -> Result { - let mut allow_v4 = Vec::new(); - let mut deny_v4 = Vec::new(); - let mut allow_v6 = Vec::new(); - let mut deny_v6 = Vec::new(); - let mut allow_asn = Vec::new(); - let mut deny_asn = Vec::new(); - - for (index, raw_line) in contents.lines().enumerate() { - let line_number = index + 1; - let line = raw_line.split('#').next().unwrap_or("").trim(); - if line.is_empty() { - continue; - } - let mut words = line.split_whitespace(); - let action = words.next().expect("non-empty line has first word"); - let resource = words.collect::>().join(" "); - if resource.is_empty() { - return Err(format!( - "line {line_number}: missing resource after '{action}'" - )); - } - let allow = match action { - "allow" => true, - "deny" => false, - _ => { - return Err(format!( - "line {line_number}: expected 'allow' or 'deny', got '{action}'" - )); - } - }; - if looks_like_ip_resource(&resource) { - let interval = parse_ip_interval(&resource).map_err(|error| { - format!("line {line_number}: invalid IP resource '{resource}': {error}") - })?; - match (interval.afi, allow) { - (Afi::Ipv4, true) => allow_v4.push(interval), - (Afi::Ipv4, false) => deny_v4.push(interval), - (Afi::Ipv6, true) => allow_v6.push(interval), - (Afi::Ipv6, false) => deny_v6.push(interval), - } - } else { - let interval = parse_as_interval(&resource).map_err(|error| { - format!("line {line_number}: invalid AS resource '{resource}': {error}") - })?; - if allow { - allow_asn.push(interval); - } else { - deny_asn.push(interval); - } - } - } - - let mut warnings = Vec::new(); - normalize_ip_intervals("allow IPv4", &mut allow_v4, &mut warnings); - normalize_ip_intervals("deny IPv4", &mut deny_v4, &mut warnings); - normalize_ip_intervals("allow IPv6", &mut allow_v6, &mut warnings); - normalize_ip_intervals("deny IPv6", &mut deny_v6, &mut warnings); - normalize_as_intervals("allow AS", &mut allow_asn, &mut warnings); - normalize_as_intervals("deny AS", &mut deny_asn, &mut warnings); - - Ok(Self { - source, - allow_v4: IntervalIndex::new(allow_v4), - deny_v4: IntervalIndex::new(deny_v4), - allow_v6: IntervalIndex::new(allow_v6), - deny_v6: IntervalIndex::new(deny_v6), - allow_asn: IntervalIndex::new(allow_asn), - deny_asn: IntervalIndex::new(deny_asn), - warnings, - }) - } - - fn validate_ip_resources( - &self, - resources: &IpResourceSet, - ) -> Result<(), TaConstraintsViolation> { - for family in &resources.families { - let items = match &family.choice { - // Constraints apply to explicit INR listings. EE profiles for - // constrained signed objects already reject inappropriate inherit. - IpAddressChoice::Inherit => continue, - IpAddressChoice::AddressesOrRanges(items) => items, - }; - let (allow, deny) = match family.afi { - Afi::Ipv4 => (&self.allow_v4, &self.deny_v4), - Afi::Ipv6 => (&self.allow_v6, &self.deny_v6), - }; - for item in items { - let interval = ip_item_to_interval(family.afi, item)?; - if deny.any_overlaps(&interval) { - return Err(TaConstraintsViolation(format!( - "{} {} intersects a deny rule in {}", - afi_name(family.afi), - interval, - self.source.display() - ))); - } - if !allow.fully_covers(&interval) { - return Err(TaConstraintsViolation(format!( - "{} {} is not fully contained in allow rules in {}", - afi_name(family.afi), - interval, - self.source.display() - ))); - } - } - } - Ok(()) - } - - fn validate_as_choice( - &self, - kind: &str, - choice: Option<&AsIdentifierChoice>, - ) -> Result<(), TaConstraintsViolation> { - let Some(choice) = choice else { - return Ok(()); - }; - let items = match choice { - AsIdentifierChoice::Inherit => return Ok(()), - AsIdentifierChoice::AsIdsOrRanges(items) => items, - }; - for item in items { - let interval = match item { - AsIdOrRange::Id(value) => AsInterval::new(*value, *value), - AsIdOrRange::Range { min, max } => AsInterval::new(*min, *max), - }; - if self.deny_asn.any_overlaps(&interval) { - return Err(TaConstraintsViolation(format!( - "{kind} {interval} intersects a deny rule in {}", - self.source.display() - ))); - } - if !self.allow_asn.fully_covers(&interval) { - return Err(TaConstraintsViolation(format!( - "{kind} {interval} is not fully contained in allow rules in {}", - self.source.display() - ))); - } - } - Ok(()) - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct TaConstraintsViolation(pub String); - -impl std::fmt::Display for TaConstraintsViolation { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.0.fmt(formatter) - } -} - -impl std::error::Error for TaConstraintsViolation {} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct IpInterval { - afi: Afi, - min: u128, - max: u128, -} - -impl IpInterval { - fn new(afi: Afi, min: u128, max: u128) -> Self { - Self { afi, min, max } - } - - fn overlaps(&self, other: &Self) -> bool { - self.afi == other.afi && self.min <= other.max && other.min <= self.max - } -} - -impl IntervalValue for IpInterval { - fn start(&self) -> u128 { - self.min - } - - fn end(&self) -> u128 { - self.max - } - - fn overlaps(&self, other: &Self) -> bool { - IpInterval::overlaps(self, other) - } -} - -impl std::fmt::Display for IpInterval { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let min = u128_to_ip(self.afi, self.min); - let max = u128_to_ip(self.afi, self.max); - if min == max { - write!(formatter, "{min}") - } else { - write!(formatter, "{min} - {max}") - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct AsInterval { - min: u32, - max: u32, -} - -impl AsInterval { - fn new(min: u32, max: u32) -> Self { - Self { min, max } - } - - fn overlaps(&self, other: &Self) -> bool { - self.min <= other.max && other.min <= self.max - } -} - -impl IntervalValue for AsInterval { - fn start(&self) -> u128 { - self.min.into() - } - - fn end(&self) -> u128 { - self.max.into() - } - - fn overlaps(&self, other: &Self) -> bool { - AsInterval::overlaps(self, other) - } -} - -impl std::fmt::Display for AsInterval { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - if self.min == self.max { - write!(formatter, "{}", self.min) - } else { - write!(formatter, "{} - {}", self.min, self.max) - } - } -} - -fn looks_like_ip_resource(resource: &str) -> bool { - resource.contains('.') || resource.contains(':') || resource.contains('/') -} - -fn parse_ip_interval(resource: &str) -> Result { - if let Some((raw_min, raw_max)) = resource.split_once('-') { - let min: IpAddr = raw_min.trim().parse().map_err(|_| "invalid range start")?; - let max: IpAddr = raw_max.trim().parse().map_err(|_| "invalid range end")?; - let (afi, min) = ip_to_u128(min); - let (max_afi, max) = ip_to_u128(max); - if afi != max_afi { - return Err("range endpoints use different address families".to_string()); - } - if min > max { - return Err("range start is greater than range end".to_string()); - } - return Ok(IpInterval::new(afi, min, max)); - } - - let (raw_address, raw_prefix_len) = resource - .split_once('/') - .ok_or_else(|| "expected CIDR prefix or range".to_string())?; - let address: IpAddr = raw_address.trim().parse().map_err(|_| "invalid address")?; - let prefix_len: u16 = raw_prefix_len - .trim() - .parse() - .map_err(|_| "invalid prefix length")?; - let (afi, address) = ip_to_u128(address); - let width = match afi { - Afi::Ipv4 => 32, - Afi::Ipv6 => 128, - }; - if prefix_len > width { - return Err(format!("prefix length must be <= {width}")); - } - let host_bits = width - prefix_len; - let mask = if prefix_len == 0 { - 0 - } else { - width_mask(width) << host_bits - }; - let min = address & mask; - let max = min | (!mask & width_mask(width)); - Ok(IpInterval::new(afi, min, max)) -} - -fn parse_as_interval(resource: &str) -> Result { - let parse_asn = |raw: &str| -> Result { - raw.trim() - .strip_prefix("AS") - .or_else(|| raw.trim().strip_prefix("as")) - .unwrap_or(raw.trim()) - .parse::() - .map_err(|_| "expected an ASN in the range 0..4294967295".to_string()) - }; - if let Some((raw_min, raw_max)) = resource.split_once('-') { - let min = parse_asn(raw_min)?; - let max = parse_asn(raw_max)?; - if min > max { - return Err("range start is greater than range end".to_string()); - } - Ok(AsInterval::new(min, max)) - } else { - let value = parse_asn(resource)?; - Ok(AsInterval::new(value, value)) - } -} - -fn normalize_ip_intervals(label: &str, entries: &mut Vec, warnings: &mut Vec) { - entries.sort_by_key(|entry| (entry.min, entry.max)); - let mut normalized = Vec::with_capacity(entries.len()); - for entry in entries.drain(..) { - let Some(last) = normalized.last_mut() else { - normalized.push(entry); - continue; - }; - if entry.min <= last.max { - warnings.push(format!( - "TA constraints {label} rules overlap; normalized without blocking startup" - )); - last.max = last.max.max(entry.max); - } else if entry.min == last.max.saturating_add(1) { - last.max = entry.max; - } else { - normalized.push(entry); - } - } - warnings.sort(); - warnings.dedup(); - *entries = normalized; -} - -fn normalize_as_intervals(label: &str, entries: &mut Vec, warnings: &mut Vec) { - entries.sort_by_key(|entry| (entry.min, entry.max)); - let mut normalized = Vec::with_capacity(entries.len()); - for entry in entries.drain(..) { - let Some(last) = normalized.last_mut() else { - normalized.push(entry); - continue; - }; - if entry.min <= last.max { - warnings.push(format!( - "TA constraints {label} rules overlap; normalized without blocking startup" - )); - last.max = last.max.max(entry.max); - } else if entry.min == last.max.saturating_add(1) { - last.max = entry.max; - } else { - normalized.push(entry); - } - } - warnings.sort(); - warnings.dedup(); - *entries = normalized; -} - -fn ip_item_to_interval( - afi: Afi, - item: &IpAddressOrRange, -) -> Result { - let (min, max) = match item { - IpAddressOrRange::Prefix(prefix) => { - let width = prefix.afi.ub(); - let address = ip_bytes_to_u128(&prefix.addr); - let prefix_len = prefix.prefix_len.min(width); - let host_bits = width - prefix_len; - let mask = if prefix_len == 0 { - 0 - } else { - width_mask(width) << host_bits - }; - let min = address & mask; - (min, min | (!mask & width_mask(width))) - } - IpAddressOrRange::Range(range) => { - (ip_bytes_to_u128(&range.min), ip_bytes_to_u128(&range.max)) - } - }; - if min > max { - return Err(TaConstraintsViolation( - "EE certificate carries an invalid IP range".to_string(), - )); - } - Ok(IpInterval::new(afi, min, max)) -} - -fn ip_to_u128(address: IpAddr) -> (Afi, u128) { - match address { - IpAddr::V4(address) => (Afi::Ipv4, u32::from(address) as u128), - IpAddr::V6(address) => (Afi::Ipv6, u128::from(address)), - } -} - -fn ip_bytes_to_u128(bytes: &[u8]) -> u128 { - bytes - .iter() - .fold(0u128, |value, byte| (value << 8) | u128::from(*byte)) -} - -fn width_mask(width: u16) -> u128 { - if width == 128 { - u128::MAX - } else { - (1u128 << width) - 1 - } -} - -fn u128_to_ip(afi: Afi, value: u128) -> IpAddr { - match afi { - Afi::Ipv4 => IpAddr::V4(Ipv4Addr::from(value as u32)), - Afi::Ipv6 => IpAddr::V6(Ipv6Addr::from(value)), - } -} - -fn afi_name(afi: Afi) -> &'static str { - match afi { - Afi::Ipv4 => "IPv4", - Afi::Ipv6 => "IPv6", - } -} +include!("ta_constraints/implementation.rs"); #[cfg(test)] -mod tests { - use super::{IntervalIndex, IntervalLookup, TaConstraints, TaConstraintsByTal}; - - fn parse(body: &str) -> TaConstraints { - TaConstraints::parse("test.constraints".into(), body).expect("parse constraints") - } - - fn indexed_ipv4_rules(action: &str, count: usize) -> String { - (0..count) - .map(|index| format!("{action} 10.{}.0.0/16\n", index * 2)) - .collect() - } - - fn indexed_ipv6_rules(action: &str, count: usize) -> String { - (0..count) - .map(|index| format!("{action} 2001:db8:{}::/48\n", index * 2)) - .collect() - } - - fn indexed_asn_rules(action: &str, count: usize) -> String { - (0..count) - .map(|index| format!("{action} {}\n", 65000 + index * 2)) - .collect() - } - - #[test] - fn deny_has_precedence_and_unlisted_resources_are_denied() { - let constraints = parse("allow 192.0.2.0/24\ndeny 192.0.2.128/25\n"); - assert!( - constraints - .allow_v4 - .fully_covers(&super::parse_ip_interval("192.0.2.0/25").unwrap()) - ); - assert!( - constraints - .deny_v4 - .iter() - .any(|entry| entry.overlaps(&super::parse_ip_interval("192.0.2.128/25").unwrap())) - ); - assert!( - !constraints - .allow_v4 - .fully_covers(&super::parse_ip_interval("198.51.100.0/24").unwrap()) - ); - } - - #[test] - fn same_list_overlap_warns_and_normalizes() { - let constraints = parse("allow 10.0.0.0/8\nallow 10.1.0.0/16\n"); - assert_eq!(constraints.allow_v4.len(), 1); - assert_eq!(constraints.warnings.len(), 1); - assert!(constraints.warnings[0].contains("overlap")); - } - - #[test] - fn adjacent_allow_ranges_cover_one_interval() { - let constraints = parse("allow 64496 - 64500\nallow 64501 - 64511\n"); - assert!( - constraints - .allow_asn - .fully_covers(&super::parse_as_interval("64496 - 64511").unwrap()) - ); - } - - #[test] - fn interval_index_uses_linear_for_at_most_ten_and_tree_above_ten() { - for count in [0, 1, 9, 10] { - let constraints = parse(&indexed_ipv4_rules("deny", count)); - assert_eq!(constraints.deny_v4.len(), count); - assert!(!constraints.deny_v4.uses_tree(), "count={count}"); - } - - let constraints = parse(&indexed_ipv4_rules("deny", 11)); - assert_eq!(constraints.deny_v4.len(), 11); - assert!(constraints.deny_v4.uses_tree()); - } - - #[test] - fn tree_ip_queries_match_linear_reference() { - let body = format!( - "{}{}", - indexed_ipv4_rules("allow", 11), - indexed_ipv4_rules("deny", 11) - ); - let constraints = parse(&body); - assert!(constraints.allow_v4.uses_tree()); - assert!(constraints.deny_v4.uses_tree()); - - let linear_allow = IntervalIndex { - rules: constraints.allow_v4.rules.clone(), - lookup: IntervalLookup::Linear, - }; - let linear_deny = IntervalIndex { - rules: constraints.deny_v4.rules.clone(), - lookup: IntervalLookup::Linear, - }; - let targets = [ - super::parse_ip_interval("10.0.0.0/16").unwrap(), - super::parse_ip_interval("10.0.1.0/24").unwrap(), - super::parse_ip_interval("10.1.0.0/16").unwrap(), - super::parse_ip_interval("10.20.0.0/16").unwrap(), - super::parse_ip_interval("10.21.0.0/16").unwrap(), - super::parse_ip_interval("10.0.0.0 - 10.2.255.255").unwrap(), - super::parse_ip_interval("9.0.0.0/8").unwrap(), - super::parse_ip_interval("11.0.0.0/8").unwrap(), - ]; - for target in targets { - assert_eq!( - constraints.allow_v4.any_overlaps(&target), - linear_allow.any_overlaps(&target), - "allow overlap for {target}" - ); - assert_eq!( - constraints.allow_v4.fully_covers(&target), - linear_allow.fully_covers(&target), - "allow coverage for {target}" - ); - assert_eq!( - constraints.deny_v4.any_overlaps(&target), - linear_deny.any_overlaps(&target), - "deny overlap for {target}" - ); - } - } - - #[test] - fn tree_ipv6_queries_match_linear_reference() { - let body = format!( - "{}{}", - indexed_ipv6_rules("allow", 11), - indexed_ipv6_rules("deny", 11) - ); - let constraints = parse(&body); - assert!(constraints.allow_v6.uses_tree()); - assert!(constraints.deny_v6.uses_tree()); - - let linear_allow = IntervalIndex { - rules: constraints.allow_v6.rules.clone(), - lookup: IntervalLookup::Linear, - }; - let linear_deny = IntervalIndex { - rules: constraints.deny_v6.rules.clone(), - lookup: IntervalLookup::Linear, - }; - let targets = [ - super::parse_ip_interval("2001:db8:0::/48").unwrap(), - super::parse_ip_interval("2001:db8:1::/64").unwrap(), - super::parse_ip_interval("2001:db8:2::/48").unwrap(), - super::parse_ip_interval("2001:db8:20::/48").unwrap(), - super::parse_ip_interval("2001:db8:21::/48").unwrap(), - super::parse_ip_interval("2001:db8:0:: - 2001:db8:2:ffff:ffff:ffff:ffff:ffff").unwrap(), - super::parse_ip_interval("2001:db7::/32").unwrap(), - super::parse_ip_interval("2001:db9::/32").unwrap(), - ]; - for target in targets { - assert_eq!( - constraints.allow_v6.any_overlaps(&target), - linear_allow.any_overlaps(&target), - "allow overlap for {target}" - ); - assert_eq!( - constraints.allow_v6.fully_covers(&target), - linear_allow.fully_covers(&target), - "allow coverage for {target}" - ); - assert_eq!( - constraints.deny_v6.any_overlaps(&target), - linear_deny.any_overlaps(&target), - "deny overlap for {target}" - ); - } - } - - #[test] - fn tree_asn_queries_match_linear_reference() { - let body = format!( - "{}{}", - indexed_asn_rules("allow", 11), - indexed_asn_rules("deny", 11) - ); - let constraints = parse(&body); - assert!(constraints.allow_asn.uses_tree()); - assert!(constraints.deny_asn.uses_tree()); - - let linear_allow = IntervalIndex { - rules: constraints.allow_asn.rules.clone(), - lookup: IntervalLookup::Linear, - }; - let linear_deny = IntervalIndex { - rules: constraints.deny_asn.rules.clone(), - lookup: IntervalLookup::Linear, - }; - let targets = [ - super::AsInterval::new(65000, 65000), - super::AsInterval::new(65001, 65001), - super::AsInterval::new(65000, 65002), - super::AsInterval::new(65020, 65020), - super::AsInterval::new(65021, 65021), - super::AsInterval::new(64999, 65000), - ]; - for target in targets { - assert_eq!( - constraints.allow_asn.any_overlaps(&target), - linear_allow.any_overlaps(&target), - "allow overlap for {target}" - ); - assert_eq!( - constraints.allow_asn.fully_covers(&target), - linear_allow.fully_covers(&target), - "allow coverage for {target}" - ); - assert_eq!( - constraints.deny_asn.any_overlaps(&target), - linear_deny.any_overlaps(&target), - "deny overlap for {target}" - ); - } - } - - #[test] - fn index_equality_and_debug_ignore_derived_lookup() { - let constraints = parse(&indexed_ipv4_rules("deny", 11)); - let tree = constraints.deny_v4.clone(); - let linear = IntervalIndex { - rules: tree.rules.clone(), - lookup: IntervalLookup::Linear, - }; - assert_eq!(tree, linear); - assert_eq!(format!("{tree:?}"), format!("{linear:?}")); - } - - #[test] - fn adjacent_file_is_discovered_by_tal_stem() { - let dir = tempfile::tempdir().expect("tmpdir"); - let tal_path = dir.path().join("example.tal"); - std::fs::write(&tal_path, "placeholder").expect("write TAL"); - std::fs::write( - tal_path.with_extension("constraints"), - "allow 192.0.2.0/24\n", - ) - .expect("write constraints"); - let inputs = vec![crate::parallel::types::TalInputSpec::from_file_path( - tal_path, - )]; - let loaded = TaConstraintsByTal::load_for_tals(&inputs, &[]).expect("load constraints"); - assert!(loaded.for_tal("example").is_some()); - let first = loaded - .shared_for_tal("example") - .expect("shared constraints snapshot"); - let second = loaded - .shared_for_tal("example") - .expect("shared constraints snapshot"); - assert!(std::sync::Arc::ptr_eq(&first, &second)); - } - - #[test] - fn shared_snapshots_remain_tal_specific() { - let dir = tempfile::tempdir().expect("tmpdir"); - let tal_a = dir.path().join("alpha.tal"); - let tal_b = dir.path().join("bravo.tal"); - std::fs::write(&tal_a, "placeholder").expect("write alpha TAL"); - std::fs::write(&tal_b, "placeholder").expect("write bravo TAL"); - std::fs::write(tal_a.with_extension("constraints"), "allow 192.0.2.0/24\n") - .expect("write alpha constraints"); - std::fs::write( - tal_b.with_extension("constraints"), - "allow 198.51.100.0/24\n", - ) - .expect("write bravo constraints"); - - let inputs = vec![ - crate::parallel::types::TalInputSpec::from_file_path(tal_a), - crate::parallel::types::TalInputSpec::from_file_path(tal_b), - ]; - let loaded = TaConstraintsByTal::load_for_tals(&inputs, &[]).expect("load constraints"); - let alpha = loaded.shared_for_tal("alpha").expect("alpha snapshot"); - let bravo = loaded.shared_for_tal("bravo").expect("bravo snapshot"); - assert!(!std::sync::Arc::ptr_eq(&alpha, &bravo)); - assert_ne!(alpha.as_ref(), bravo.as_ref()); - } - - #[test] - fn semantic_fingerprint_is_cached_and_ignores_source_path() { - let first_dir = tempfile::tempdir().expect("first tmpdir"); - let second_dir = tempfile::tempdir().expect("second tmpdir"); - let first_tal = first_dir.path().join("example.tal"); - let second_tal = second_dir.path().join("example.tal"); - for tal_path in [&first_tal, &second_tal] { - std::fs::write(tal_path, "placeholder").expect("write TAL"); - std::fs::write( - tal_path.with_extension("constraints"), - "allow 192.0.2.0/24\ndeny 192.0.2.128/25\n", - ) - .expect("write constraints"); - } - - let first = TaConstraintsByTal::load_for_tals( - &[crate::parallel::types::TalInputSpec::from_file_path( - first_tal, - )], - &[], - ) - .expect("load first constraints"); - let second = TaConstraintsByTal::load_for_tals( - &[crate::parallel::types::TalInputSpec::from_file_path( - second_tal, - )], - &[], - ) - .expect("load second constraints"); - - assert_eq!(first.fingerprint_bytes().len(), 32); - assert_eq!(first.fingerprint_bytes(), second.fingerprint_bytes()); - assert_eq!(first.fingerprint_sha256_hex().len(), 64); - } - - #[test] - fn full_afrinic_ipv4_fixture_is_parseable() { - let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/ta_constraints/afrinic-full-ipv4-deny.constraints"); - let raw = std::fs::read_to_string(&path).expect("read full AFRINIC fixture"); - let source_ipv4_rules = raw - .lines() - .filter(|line| line.trim_start().starts_with("deny ")) - .count(); - assert_eq!(source_ipv4_rules, 383); - - let constraints = TaConstraints::from_file(&path).expect("parse full AFRINIC fixture"); - assert_eq!(constraints.allow_v4.len(), 1); - assert_eq!(constraints.allow_v6.len(), 1); - assert_eq!(constraints.allow_asn.len(), 1); - assert!(!constraints.deny_v4.is_empty()); - } - - #[test] - fn current_afrinic_ipv4_fixture_is_parseable_and_canonical() { - let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/ta_constraints/afrinic-current-ipv4-deny.constraints"); - let raw = std::fs::read_to_string(&path).expect("read current AFRINIC fixture"); - let source_ipv4_rules = raw - .lines() - .filter(|line| line.trim_start().starts_with("deny ")) - .count(); - assert_eq!(source_ipv4_rules, 1825); - - let constraints = TaConstraints::from_file(&path).expect("parse current AFRINIC fixture"); - assert_eq!(constraints.allow_v4.len(), 1); - assert_eq!(constraints.allow_v6.len(), 1); - assert_eq!(constraints.allow_asn.len(), 1); - assert_eq!(constraints.deny_v4.len(), 815); - assert!(constraints.deny_v4.uses_tree()); - assert!(!constraints.allow_v4.uses_tree()); - assert!(!constraints.allow_v6.uses_tree()); - assert!(!constraints.allow_asn.uses_tree()); - assert!(constraints.warnings.is_empty()); - } -} +#[path = "ta_constraints/tests.rs"] +mod tests; diff --git a/crates/panda-rpki-validator/src/ta_constraints/implementation.rs b/crates/panda-rpki-validator/src/ta_constraints/implementation.rs new file mode 100644 index 0000000..d59dc3b --- /dev/null +++ b/crates/panda-rpki-validator/src/ta_constraints/implementation.rs @@ -0,0 +1,779 @@ +// Interval indexes and trust-anchor resource constraint evaluation. + + +use std::collections::{BTreeMap, BTreeSet}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::ops::Deref; +use std::path::{Path, PathBuf}; + +use sha2::Digest; + +use crate::data_model::rc::{ + Afi, AsIdOrRange, AsIdentifierChoice, IpAddressChoice, IpAddressOrRange, IpResourceSet, + ResourceCertificate, +}; +use crate::parallel::types::{TalInputSpec, TalSource}; + +const LINEAR_INTERVAL_THRESHOLD: usize = 10; + +/// A normalized rule set with a small-set linear path and a large-set tree path. +/// +/// The parser constructs this only after sorting and merging overlapping or +/// adjacent intervals. The tree lookup therefore only needs to inspect the +/// predecessor of a target interval; normalized intervals are disjoint and +/// sorted by their lower bound. +#[derive(Clone)] +struct IntervalIndex { + rules: Vec, + lookup: IntervalLookup, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum IntervalLookup { + Linear, + Tree(BTreeMap), +} + +impl std::fmt::Debug for IntervalIndex { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Keep derived acceleration data out of Debug so diagnostics continue + // to describe canonical rules rather than their index layout. + self.rules.fmt(formatter) + } +} + +impl PartialEq for IntervalIndex { + fn eq(&self, other: &Self) -> bool { + self.rules == other.rules + } +} + +impl Eq for IntervalIndex {} + +impl Deref for IntervalIndex { + type Target = [I]; + + fn deref(&self) -> &Self::Target { + &self.rules + } +} + +trait IntervalValue { + fn start(&self) -> u128; + fn end(&self) -> u128; + fn overlaps(&self, other: &Self) -> bool; +} + +impl IntervalIndex { + fn new(rules: Vec) -> Self { + debug_assert!( + rules + .windows(2) + .all(|window| { window[0].end().saturating_add(1) < window[1].start() }) + ); + let lookup = if rules.len() > LINEAR_INTERVAL_THRESHOLD { + let tree = rules + .iter() + .map(|rule| (rule.start(), rule.end())) + .collect(); + IntervalLookup::Tree(tree) + } else { + IntervalLookup::Linear + }; + Self { rules, lookup } + } + + fn any_overlaps(&self, target: &I) -> bool { + match &self.lookup { + IntervalLookup::Linear => self.rules.iter().any(|entry| entry.overlaps(target)), + IntervalLookup::Tree(tree) => tree + .range(..=target.end()) + .next_back() + .map(|(_, end)| *end >= target.start()) + .unwrap_or(false), + } + } + + fn fully_covers(&self, target: &I) -> bool { + match &self.lookup { + IntervalLookup::Linear => self.fully_covers_linear(target), + IntervalLookup::Tree(tree) => tree + .range(..=target.start()) + .next_back() + .map(|(_, end)| *end >= target.end()) + .unwrap_or(false), + } + } + + fn fully_covers_linear(&self, target: &I) -> bool { + let mut cursor = target.start(); + for entry in &self.rules { + if entry.end() < cursor { + continue; + } + if entry.start() > cursor { + return false; + } + if entry.end() >= target.end() { + return true; + } + cursor = entry.end().saturating_add(1); + } + false + } + + #[cfg(test)] + fn uses_tree(&self) -> bool { + matches!(self.lookup, IntervalLookup::Tree(_)) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TaConstraintsByTal { + by_tal_id: BTreeMap>, + /// Stable semantic digest of the normalized rules. This is computed once + /// while loading the run policy so cache identity checks do not serialize + /// every rule for every publication point. + fingerprint: [u8; 32], +} + +impl TaConstraintsByTal { + pub fn load_for_tals( + tal_inputs: &[TalInputSpec], + explicit_specs: &[String], + ) -> Result { + let tal_ids = tal_inputs + .iter() + .map(|input| input.tal_id.as_str()) + .collect::>(); + let mut explicit_paths = BTreeMap::::new(); + for spec in explicit_specs { + let (tal_id, path) = spec + .split_once('=') + .ok_or_else(|| format!("--ta-constraints expects =, got '{spec}'"))?; + let tal_id = tal_id.trim(); + let path = path.trim(); + if tal_id.is_empty() || path.is_empty() { + return Err(format!( + "--ta-constraints expects non-empty =, got '{spec}'" + )); + } + if !tal_ids.contains(tal_id) { + return Err(format!( + "--ta-constraints references unknown TAL id '{tal_id}'" + )); + } + if explicit_paths + .insert(tal_id.to_string(), PathBuf::from(path)) + .is_some() + { + return Err(format!( + "--ta-constraints specifies TAL id '{tal_id}' more than once" + )); + } + } + + let mut by_tal_id = BTreeMap::new(); + for input in tal_inputs { + let path = explicit_paths + .get(&input.tal_id) + .cloned() + .or_else(|| adjacent_constraints_path(&input.source).filter(|path| path.is_file())); + if let Some(path) = path { + let constraints = TaConstraints::from_file(&path).map_err(|error| { + format!( + "load TA constraints for '{}' from {} failed: {error}", + input.tal_id, + path.display() + ) + })?; + by_tal_id.insert(input.tal_id.clone(), std::sync::Arc::new(constraints)); + } + } + let fingerprint = constraints_fingerprint(&by_tal_id); + Ok(Self { + by_tal_id, + fingerprint, + }) + } + + pub fn for_tal(&self, tal_id: &str) -> Option<&TaConstraints> { + self.by_tal_id.get(tal_id).map(std::sync::Arc::as_ref) + } + + /// Return the immutable, process-local snapshot for a TAL without cloning + /// the rule trees. Phase-2 object workers own this `Arc` in their task + /// payload, while scoped stage workers borrow the same policy map. + pub(crate) fn shared_for_tal(&self, tal_id: &str) -> Option> { + self.by_tal_id.get(tal_id).cloned() + } + + pub fn is_empty(&self) -> bool { + self.by_tal_id.is_empty() + } + + pub fn configuration_warnings(&self) -> Vec { + self.by_tal_id + .iter() + .flat_map(|(tal_id, constraints)| { + constraints.warnings().iter().map(move |warning| { + format!( + "TA constraints for TAL '{tal_id}' ({}): {warning}", + constraints.source().display() + ) + }) + }) + .collect() + } + + /// Return the precomputed SHA-256 digest of the normalized semantic rules. + /// + /// The returned bytes are stable across source paths and derived index + /// layouts, and are intentionally borrowed so hot-path cache lookups do + /// not allocate. + pub fn fingerprint_bytes(&self) -> &[u8] { + &self.fingerprint + } + + pub fn fingerprint_sha256_hex(&self) -> String { + hex::encode(self.fingerprint) + } +} + +impl Default for TaConstraintsByTal { + fn default() -> Self { + let by_tal_id = BTreeMap::new(); + let fingerprint = constraints_fingerprint(&by_tal_id); + Self { + by_tal_id, + fingerprint, + } + } +} + +const CONSTRAINTS_FINGERPRINT_VERSION: &[u8] = b"ta-constraints-semantic-v1"; + +fn constraints_fingerprint( + by_tal_id: &BTreeMap>, +) -> [u8; 32] { + let mut hasher = sha2::Sha256::new(); + hasher.update(CONSTRAINTS_FINGERPRINT_VERSION); + hasher.update((by_tal_id.len() as u64).to_be_bytes()); + for (tal_id, constraints) in by_tal_id { + update_length_prefixed(&mut hasher, tal_id.as_bytes()); + update_ip_intervals(&mut hasher, b"allow-v4", &constraints.allow_v4); + update_ip_intervals(&mut hasher, b"deny-v4", &constraints.deny_v4); + update_ip_intervals(&mut hasher, b"allow-v6", &constraints.allow_v6); + update_ip_intervals(&mut hasher, b"deny-v6", &constraints.deny_v6); + update_as_intervals(&mut hasher, b"allow-asn", &constraints.allow_asn); + update_as_intervals(&mut hasher, b"deny-asn", &constraints.deny_asn); + } + let digest = hasher.finalize(); + let mut out = [0u8; 32]; + out.copy_from_slice(&digest); + out +} + +fn update_length_prefixed(hasher: &mut sha2::Sha256, value: &[u8]) { + hasher.update((value.len() as u64).to_be_bytes()); + hasher.update(value); +} + +fn update_ip_intervals( + hasher: &mut sha2::Sha256, + label: &[u8], + intervals: &IntervalIndex, +) { + update_length_prefixed(hasher, label); + hasher.update((intervals.rules.len() as u64).to_be_bytes()); + for interval in &intervals.rules { + hasher.update([match interval.afi { + Afi::Ipv4 => 4, + Afi::Ipv6 => 6, + }]); + hasher.update(interval.min.to_be_bytes()); + hasher.update(interval.max.to_be_bytes()); + } +} + +fn update_as_intervals( + hasher: &mut sha2::Sha256, + label: &[u8], + intervals: &IntervalIndex, +) { + update_length_prefixed(hasher, label); + hasher.update((intervals.rules.len() as u64).to_be_bytes()); + for interval in &intervals.rules { + hasher.update(interval.min.to_be_bytes()); + hasher.update(interval.max.to_be_bytes()); + } +} + +fn adjacent_constraints_path(source: &TalSource) -> Option { + match source { + TalSource::FilePath(path) => Some(path.with_extension("constraints")), + TalSource::FilePathWithTa { tal_path, .. } => Some(tal_path.with_extension("constraints")), + TalSource::Url(_) | TalSource::DerBytes { .. } => None, + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TaConstraints { + source: PathBuf, + allow_v4: IntervalIndex, + deny_v4: IntervalIndex, + allow_v6: IntervalIndex, + deny_v6: IntervalIndex, + allow_asn: IntervalIndex, + deny_asn: IntervalIndex, + warnings: Vec, +} + +impl TaConstraints { + pub fn from_file(path: &Path) -> Result { + let contents = std::fs::read_to_string(path) + .map_err(|error| format!("read {}: {error}", path.display()))?; + Self::parse(path.to_path_buf(), &contents) + } + + pub fn warnings(&self) -> &[String] { + &self.warnings + } + + pub fn source(&self) -> &Path { + &self.source + } + + pub fn validate_ee_certificate( + &self, + certificate: &ResourceCertificate, + ) -> Result<(), TaConstraintsViolation> { + if let Some(ip_resources) = certificate.tbs.extensions.ip_resources.as_ref() { + self.validate_ip_resources(ip_resources)?; + } + if let Some(as_resources) = certificate.tbs.extensions.as_resources.as_ref() { + self.validate_as_choice("AS", as_resources.asnum.as_ref())?; + self.validate_as_choice("RDI", as_resources.rdi.as_ref())?; + } + Ok(()) + } + + fn parse(source: PathBuf, contents: &str) -> Result { + let mut allow_v4 = Vec::new(); + let mut deny_v4 = Vec::new(); + let mut allow_v6 = Vec::new(); + let mut deny_v6 = Vec::new(); + let mut allow_asn = Vec::new(); + let mut deny_asn = Vec::new(); + + for (index, raw_line) in contents.lines().enumerate() { + let line_number = index + 1; + let line = raw_line.split('#').next().unwrap_or("").trim(); + if line.is_empty() { + continue; + } + let mut words = line.split_whitespace(); + let action = words.next().expect("non-empty line has first word"); + let resource = words.collect::>().join(" "); + if resource.is_empty() { + return Err(format!( + "line {line_number}: missing resource after '{action}'" + )); + } + let allow = match action { + "allow" => true, + "deny" => false, + _ => { + return Err(format!( + "line {line_number}: expected 'allow' or 'deny', got '{action}'" + )); + } + }; + if looks_like_ip_resource(&resource) { + let interval = parse_ip_interval(&resource).map_err(|error| { + format!("line {line_number}: invalid IP resource '{resource}': {error}") + })?; + match (interval.afi, allow) { + (Afi::Ipv4, true) => allow_v4.push(interval), + (Afi::Ipv4, false) => deny_v4.push(interval), + (Afi::Ipv6, true) => allow_v6.push(interval), + (Afi::Ipv6, false) => deny_v6.push(interval), + } + } else { + let interval = parse_as_interval(&resource).map_err(|error| { + format!("line {line_number}: invalid AS resource '{resource}': {error}") + })?; + if allow { + allow_asn.push(interval); + } else { + deny_asn.push(interval); + } + } + } + + let mut warnings = Vec::new(); + normalize_ip_intervals("allow IPv4", &mut allow_v4, &mut warnings); + normalize_ip_intervals("deny IPv4", &mut deny_v4, &mut warnings); + normalize_ip_intervals("allow IPv6", &mut allow_v6, &mut warnings); + normalize_ip_intervals("deny IPv6", &mut deny_v6, &mut warnings); + normalize_as_intervals("allow AS", &mut allow_asn, &mut warnings); + normalize_as_intervals("deny AS", &mut deny_asn, &mut warnings); + + Ok(Self { + source, + allow_v4: IntervalIndex::new(allow_v4), + deny_v4: IntervalIndex::new(deny_v4), + allow_v6: IntervalIndex::new(allow_v6), + deny_v6: IntervalIndex::new(deny_v6), + allow_asn: IntervalIndex::new(allow_asn), + deny_asn: IntervalIndex::new(deny_asn), + warnings, + }) + } + + fn validate_ip_resources( + &self, + resources: &IpResourceSet, + ) -> Result<(), TaConstraintsViolation> { + for family in &resources.families { + let items = match &family.choice { + // Constraints apply to explicit INR listings. EE profiles for + // constrained signed objects already reject inappropriate inherit. + IpAddressChoice::Inherit => continue, + IpAddressChoice::AddressesOrRanges(items) => items, + }; + let (allow, deny) = match family.afi { + Afi::Ipv4 => (&self.allow_v4, &self.deny_v4), + Afi::Ipv6 => (&self.allow_v6, &self.deny_v6), + }; + for item in items { + let interval = ip_item_to_interval(family.afi, item)?; + if deny.any_overlaps(&interval) { + return Err(TaConstraintsViolation(format!( + "{} {} intersects a deny rule in {}", + afi_name(family.afi), + interval, + self.source.display() + ))); + } + if !allow.fully_covers(&interval) { + return Err(TaConstraintsViolation(format!( + "{} {} is not fully contained in allow rules in {}", + afi_name(family.afi), + interval, + self.source.display() + ))); + } + } + } + Ok(()) + } + + fn validate_as_choice( + &self, + kind: &str, + choice: Option<&AsIdentifierChoice>, + ) -> Result<(), TaConstraintsViolation> { + let Some(choice) = choice else { + return Ok(()); + }; + let items = match choice { + AsIdentifierChoice::Inherit => return Ok(()), + AsIdentifierChoice::AsIdsOrRanges(items) => items, + }; + for item in items { + let interval = match item { + AsIdOrRange::Id(value) => AsInterval::new(*value, *value), + AsIdOrRange::Range { min, max } => AsInterval::new(*min, *max), + }; + if self.deny_asn.any_overlaps(&interval) { + return Err(TaConstraintsViolation(format!( + "{kind} {interval} intersects a deny rule in {}", + self.source.display() + ))); + } + if !self.allow_asn.fully_covers(&interval) { + return Err(TaConstraintsViolation(format!( + "{kind} {interval} is not fully contained in allow rules in {}", + self.source.display() + ))); + } + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TaConstraintsViolation(pub String); + +impl std::fmt::Display for TaConstraintsViolation { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(formatter) + } +} + +impl std::error::Error for TaConstraintsViolation {} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct IpInterval { + afi: Afi, + min: u128, + max: u128, +} + +impl IpInterval { + fn new(afi: Afi, min: u128, max: u128) -> Self { + Self { afi, min, max } + } + + fn overlaps(&self, other: &Self) -> bool { + self.afi == other.afi && self.min <= other.max && other.min <= self.max + } +} + +impl IntervalValue for IpInterval { + fn start(&self) -> u128 { + self.min + } + + fn end(&self) -> u128 { + self.max + } + + fn overlaps(&self, other: &Self) -> bool { + IpInterval::overlaps(self, other) + } +} + +impl std::fmt::Display for IpInterval { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let min = u128_to_ip(self.afi, self.min); + let max = u128_to_ip(self.afi, self.max); + if min == max { + write!(formatter, "{min}") + } else { + write!(formatter, "{min} - {max}") + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct AsInterval { + min: u32, + max: u32, +} + +impl AsInterval { + fn new(min: u32, max: u32) -> Self { + Self { min, max } + } + + fn overlaps(&self, other: &Self) -> bool { + self.min <= other.max && other.min <= self.max + } +} + +impl IntervalValue for AsInterval { + fn start(&self) -> u128 { + self.min.into() + } + + fn end(&self) -> u128 { + self.max.into() + } + + fn overlaps(&self, other: &Self) -> bool { + AsInterval::overlaps(self, other) + } +} + +impl std::fmt::Display for AsInterval { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.min == self.max { + write!(formatter, "{}", self.min) + } else { + write!(formatter, "{} - {}", self.min, self.max) + } + } +} + +fn looks_like_ip_resource(resource: &str) -> bool { + resource.contains('.') || resource.contains(':') || resource.contains('/') +} + +fn parse_ip_interval(resource: &str) -> Result { + if let Some((raw_min, raw_max)) = resource.split_once('-') { + let min: IpAddr = raw_min.trim().parse().map_err(|_| "invalid range start")?; + let max: IpAddr = raw_max.trim().parse().map_err(|_| "invalid range end")?; + let (afi, min) = ip_to_u128(min); + let (max_afi, max) = ip_to_u128(max); + if afi != max_afi { + return Err("range endpoints use different address families".to_string()); + } + if min > max { + return Err("range start is greater than range end".to_string()); + } + return Ok(IpInterval::new(afi, min, max)); + } + + let (raw_address, raw_prefix_len) = resource + .split_once('/') + .ok_or_else(|| "expected CIDR prefix or range".to_string())?; + let address: IpAddr = raw_address.trim().parse().map_err(|_| "invalid address")?; + let prefix_len: u16 = raw_prefix_len + .trim() + .parse() + .map_err(|_| "invalid prefix length")?; + let (afi, address) = ip_to_u128(address); + let width = match afi { + Afi::Ipv4 => 32, + Afi::Ipv6 => 128, + }; + if prefix_len > width { + return Err(format!("prefix length must be <= {width}")); + } + let host_bits = width - prefix_len; + let mask = if prefix_len == 0 { + 0 + } else { + width_mask(width) << host_bits + }; + let min = address & mask; + let max = min | (!mask & width_mask(width)); + Ok(IpInterval::new(afi, min, max)) +} + +fn parse_as_interval(resource: &str) -> Result { + let parse_asn = |raw: &str| -> Result { + raw.trim() + .strip_prefix("AS") + .or_else(|| raw.trim().strip_prefix("as")) + .unwrap_or(raw.trim()) + .parse::() + .map_err(|_| "expected an ASN in the range 0..4294967295".to_string()) + }; + if let Some((raw_min, raw_max)) = resource.split_once('-') { + let min = parse_asn(raw_min)?; + let max = parse_asn(raw_max)?; + if min > max { + return Err("range start is greater than range end".to_string()); + } + Ok(AsInterval::new(min, max)) + } else { + let value = parse_asn(resource)?; + Ok(AsInterval::new(value, value)) + } +} + +fn normalize_ip_intervals(label: &str, entries: &mut Vec, warnings: &mut Vec) { + entries.sort_by_key(|entry| (entry.min, entry.max)); + let mut normalized = Vec::with_capacity(entries.len()); + for entry in entries.drain(..) { + let Some(last) = normalized.last_mut() else { + normalized.push(entry); + continue; + }; + if entry.min <= last.max { + warnings.push(format!( + "TA constraints {label} rules overlap; normalized without blocking startup" + )); + last.max = last.max.max(entry.max); + } else if entry.min == last.max.saturating_add(1) { + last.max = entry.max; + } else { + normalized.push(entry); + } + } + warnings.sort(); + warnings.dedup(); + *entries = normalized; +} + +fn normalize_as_intervals(label: &str, entries: &mut Vec, warnings: &mut Vec) { + entries.sort_by_key(|entry| (entry.min, entry.max)); + let mut normalized = Vec::with_capacity(entries.len()); + for entry in entries.drain(..) { + let Some(last) = normalized.last_mut() else { + normalized.push(entry); + continue; + }; + if entry.min <= last.max { + warnings.push(format!( + "TA constraints {label} rules overlap; normalized without blocking startup" + )); + last.max = last.max.max(entry.max); + } else if entry.min == last.max.saturating_add(1) { + last.max = entry.max; + } else { + normalized.push(entry); + } + } + warnings.sort(); + warnings.dedup(); + *entries = normalized; +} + +fn ip_item_to_interval( + afi: Afi, + item: &IpAddressOrRange, +) -> Result { + let (min, max) = match item { + IpAddressOrRange::Prefix(prefix) => { + let width = prefix.afi.ub(); + let address = ip_bytes_to_u128(&prefix.addr); + let prefix_len = prefix.prefix_len.min(width); + let host_bits = width - prefix_len; + let mask = if prefix_len == 0 { + 0 + } else { + width_mask(width) << host_bits + }; + let min = address & mask; + (min, min | (!mask & width_mask(width))) + } + IpAddressOrRange::Range(range) => { + (ip_bytes_to_u128(&range.min), ip_bytes_to_u128(&range.max)) + } + }; + if min > max { + return Err(TaConstraintsViolation( + "EE certificate carries an invalid IP range".to_string(), + )); + } + Ok(IpInterval::new(afi, min, max)) +} + +fn ip_to_u128(address: IpAddr) -> (Afi, u128) { + match address { + IpAddr::V4(address) => (Afi::Ipv4, u32::from(address) as u128), + IpAddr::V6(address) => (Afi::Ipv6, u128::from(address)), + } +} + +fn ip_bytes_to_u128(bytes: &[u8]) -> u128 { + bytes + .iter() + .fold(0u128, |value, byte| (value << 8) | u128::from(*byte)) +} + +fn width_mask(width: u16) -> u128 { + if width == 128 { + u128::MAX + } else { + (1u128 << width) - 1 + } +} + +fn u128_to_ip(afi: Afi, value: u128) -> IpAddr { + match afi { + Afi::Ipv4 => IpAddr::V4(Ipv4Addr::from(value as u32)), + Afi::Ipv6 => IpAddr::V6(Ipv6Addr::from(value)), + } +} + +fn afi_name(afi: Afi) -> &'static str { + match afi { + Afi::Ipv4 => "IPv4", + Afi::Ipv6 => "IPv6", + } +} diff --git a/crates/panda-rpki-validator/src/ta_constraints/tests.rs b/crates/panda-rpki-validator/src/ta_constraints/tests.rs new file mode 100644 index 0000000..5f1ec66 --- /dev/null +++ b/crates/panda-rpki-validator/src/ta_constraints/tests.rs @@ -0,0 +1,357 @@ +// Resource constraint parser and interval-index tests. + +use super::{IntervalIndex, IntervalLookup, TaConstraints, TaConstraintsByTal}; + +fn parse(body: &str) -> TaConstraints { + TaConstraints::parse("test.constraints".into(), body).expect("parse constraints") +} + +fn indexed_ipv4_rules(action: &str, count: usize) -> String { + (0..count) + .map(|index| format!("{action} 10.{}.0.0/16\n", index * 2)) + .collect() +} + +fn indexed_ipv6_rules(action: &str, count: usize) -> String { + (0..count) + .map(|index| format!("{action} 2001:db8:{}::/48\n", index * 2)) + .collect() +} + +fn indexed_asn_rules(action: &str, count: usize) -> String { + (0..count) + .map(|index| format!("{action} {}\n", 65000 + index * 2)) + .collect() +} + +#[test] +fn deny_has_precedence_and_unlisted_resources_are_denied() { + let constraints = parse("allow 192.0.2.0/24\ndeny 192.0.2.128/25\n"); + assert!( + constraints + .allow_v4 + .fully_covers(&super::parse_ip_interval("192.0.2.0/25").unwrap()) + ); + assert!( + constraints + .deny_v4 + .iter() + .any(|entry| entry.overlaps(&super::parse_ip_interval("192.0.2.128/25").unwrap())) + ); + assert!( + !constraints + .allow_v4 + .fully_covers(&super::parse_ip_interval("198.51.100.0/24").unwrap()) + ); +} + +#[test] +fn same_list_overlap_warns_and_normalizes() { + let constraints = parse("allow 10.0.0.0/8\nallow 10.1.0.0/16\n"); + assert_eq!(constraints.allow_v4.len(), 1); + assert_eq!(constraints.warnings.len(), 1); + assert!(constraints.warnings[0].contains("overlap")); +} + +#[test] +fn adjacent_allow_ranges_cover_one_interval() { + let constraints = parse("allow 64496 - 64500\nallow 64501 - 64511\n"); + assert!( + constraints + .allow_asn + .fully_covers(&super::parse_as_interval("64496 - 64511").unwrap()) + ); +} + +#[test] +fn interval_index_uses_linear_for_at_most_ten_and_tree_above_ten() { + for count in [0, 1, 9, 10] { + let constraints = parse(&indexed_ipv4_rules("deny", count)); + assert_eq!(constraints.deny_v4.len(), count); + assert!(!constraints.deny_v4.uses_tree(), "count={count}"); + } + + let constraints = parse(&indexed_ipv4_rules("deny", 11)); + assert_eq!(constraints.deny_v4.len(), 11); + assert!(constraints.deny_v4.uses_tree()); +} + +#[test] +fn tree_ip_queries_match_linear_reference() { + let body = format!( + "{}{}", + indexed_ipv4_rules("allow", 11), + indexed_ipv4_rules("deny", 11) + ); + let constraints = parse(&body); + assert!(constraints.allow_v4.uses_tree()); + assert!(constraints.deny_v4.uses_tree()); + + let linear_allow = IntervalIndex { + rules: constraints.allow_v4.rules.clone(), + lookup: IntervalLookup::Linear, + }; + let linear_deny = IntervalIndex { + rules: constraints.deny_v4.rules.clone(), + lookup: IntervalLookup::Linear, + }; + let targets = [ + super::parse_ip_interval("10.0.0.0/16").unwrap(), + super::parse_ip_interval("10.0.1.0/24").unwrap(), + super::parse_ip_interval("10.1.0.0/16").unwrap(), + super::parse_ip_interval("10.20.0.0/16").unwrap(), + super::parse_ip_interval("10.21.0.0/16").unwrap(), + super::parse_ip_interval("10.0.0.0 - 10.2.255.255").unwrap(), + super::parse_ip_interval("9.0.0.0/8").unwrap(), + super::parse_ip_interval("11.0.0.0/8").unwrap(), + ]; + for target in targets { + assert_eq!( + constraints.allow_v4.any_overlaps(&target), + linear_allow.any_overlaps(&target), + "allow overlap for {target}" + ); + assert_eq!( + constraints.allow_v4.fully_covers(&target), + linear_allow.fully_covers(&target), + "allow coverage for {target}" + ); + assert_eq!( + constraints.deny_v4.any_overlaps(&target), + linear_deny.any_overlaps(&target), + "deny overlap for {target}" + ); + } +} + +#[test] +fn tree_ipv6_queries_match_linear_reference() { + let body = format!( + "{}{}", + indexed_ipv6_rules("allow", 11), + indexed_ipv6_rules("deny", 11) + ); + let constraints = parse(&body); + assert!(constraints.allow_v6.uses_tree()); + assert!(constraints.deny_v6.uses_tree()); + + let linear_allow = IntervalIndex { + rules: constraints.allow_v6.rules.clone(), + lookup: IntervalLookup::Linear, + }; + let linear_deny = IntervalIndex { + rules: constraints.deny_v6.rules.clone(), + lookup: IntervalLookup::Linear, + }; + let targets = [ + super::parse_ip_interval("2001:db8:0::/48").unwrap(), + super::parse_ip_interval("2001:db8:1::/64").unwrap(), + super::parse_ip_interval("2001:db8:2::/48").unwrap(), + super::parse_ip_interval("2001:db8:20::/48").unwrap(), + super::parse_ip_interval("2001:db8:21::/48").unwrap(), + super::parse_ip_interval("2001:db8:0:: - 2001:db8:2:ffff:ffff:ffff:ffff:ffff").unwrap(), + super::parse_ip_interval("2001:db7::/32").unwrap(), + super::parse_ip_interval("2001:db9::/32").unwrap(), + ]; + for target in targets { + assert_eq!( + constraints.allow_v6.any_overlaps(&target), + linear_allow.any_overlaps(&target), + "allow overlap for {target}" + ); + assert_eq!( + constraints.allow_v6.fully_covers(&target), + linear_allow.fully_covers(&target), + "allow coverage for {target}" + ); + assert_eq!( + constraints.deny_v6.any_overlaps(&target), + linear_deny.any_overlaps(&target), + "deny overlap for {target}" + ); + } +} + +#[test] +fn tree_asn_queries_match_linear_reference() { + let body = format!( + "{}{}", + indexed_asn_rules("allow", 11), + indexed_asn_rules("deny", 11) + ); + let constraints = parse(&body); + assert!(constraints.allow_asn.uses_tree()); + assert!(constraints.deny_asn.uses_tree()); + + let linear_allow = IntervalIndex { + rules: constraints.allow_asn.rules.clone(), + lookup: IntervalLookup::Linear, + }; + let linear_deny = IntervalIndex { + rules: constraints.deny_asn.rules.clone(), + lookup: IntervalLookup::Linear, + }; + let targets = [ + super::AsInterval::new(65000, 65000), + super::AsInterval::new(65001, 65001), + super::AsInterval::new(65000, 65002), + super::AsInterval::new(65020, 65020), + super::AsInterval::new(65021, 65021), + super::AsInterval::new(64999, 65000), + ]; + for target in targets { + assert_eq!( + constraints.allow_asn.any_overlaps(&target), + linear_allow.any_overlaps(&target), + "allow overlap for {target}" + ); + assert_eq!( + constraints.allow_asn.fully_covers(&target), + linear_allow.fully_covers(&target), + "allow coverage for {target}" + ); + assert_eq!( + constraints.deny_asn.any_overlaps(&target), + linear_deny.any_overlaps(&target), + "deny overlap for {target}" + ); + } +} + +#[test] +fn index_equality_and_debug_ignore_derived_lookup() { + let constraints = parse(&indexed_ipv4_rules("deny", 11)); + let tree = constraints.deny_v4.clone(); + let linear = IntervalIndex { + rules: tree.rules.clone(), + lookup: IntervalLookup::Linear, + }; + assert_eq!(tree, linear); + assert_eq!(format!("{tree:?}"), format!("{linear:?}")); +} + +#[test] +fn adjacent_file_is_discovered_by_tal_stem() { + let dir = tempfile::tempdir().expect("tmpdir"); + let tal_path = dir.path().join("example.tal"); + std::fs::write(&tal_path, "placeholder").expect("write TAL"); + std::fs::write( + tal_path.with_extension("constraints"), + "allow 192.0.2.0/24\n", + ) + .expect("write constraints"); + let inputs = vec![crate::parallel::types::TalInputSpec::from_file_path( + tal_path, + )]; + let loaded = TaConstraintsByTal::load_for_tals(&inputs, &[]).expect("load constraints"); + assert!(loaded.for_tal("example").is_some()); + let first = loaded + .shared_for_tal("example") + .expect("shared constraints snapshot"); + let second = loaded + .shared_for_tal("example") + .expect("shared constraints snapshot"); + assert!(std::sync::Arc::ptr_eq(&first, &second)); +} + +#[test] +fn shared_snapshots_remain_tal_specific() { + let dir = tempfile::tempdir().expect("tmpdir"); + let tal_a = dir.path().join("alpha.tal"); + let tal_b = dir.path().join("bravo.tal"); + std::fs::write(&tal_a, "placeholder").expect("write alpha TAL"); + std::fs::write(&tal_b, "placeholder").expect("write bravo TAL"); + std::fs::write(tal_a.with_extension("constraints"), "allow 192.0.2.0/24\n") + .expect("write alpha constraints"); + std::fs::write( + tal_b.with_extension("constraints"), + "allow 198.51.100.0/24\n", + ) + .expect("write bravo constraints"); + + let inputs = vec![ + crate::parallel::types::TalInputSpec::from_file_path(tal_a), + crate::parallel::types::TalInputSpec::from_file_path(tal_b), + ]; + let loaded = TaConstraintsByTal::load_for_tals(&inputs, &[]).expect("load constraints"); + let alpha = loaded.shared_for_tal("alpha").expect("alpha snapshot"); + let bravo = loaded.shared_for_tal("bravo").expect("bravo snapshot"); + assert!(!std::sync::Arc::ptr_eq(&alpha, &bravo)); + assert_ne!(alpha.as_ref(), bravo.as_ref()); +} + +#[test] +fn semantic_fingerprint_is_cached_and_ignores_source_path() { + let first_dir = tempfile::tempdir().expect("first tmpdir"); + let second_dir = tempfile::tempdir().expect("second tmpdir"); + let first_tal = first_dir.path().join("example.tal"); + let second_tal = second_dir.path().join("example.tal"); + for tal_path in [&first_tal, &second_tal] { + std::fs::write(tal_path, "placeholder").expect("write TAL"); + std::fs::write( + tal_path.with_extension("constraints"), + "allow 192.0.2.0/24\ndeny 192.0.2.128/25\n", + ) + .expect("write constraints"); + } + + let first = TaConstraintsByTal::load_for_tals( + &[crate::parallel::types::TalInputSpec::from_file_path( + first_tal, + )], + &[], + ) + .expect("load first constraints"); + let second = TaConstraintsByTal::load_for_tals( + &[crate::parallel::types::TalInputSpec::from_file_path( + second_tal, + )], + &[], + ) + .expect("load second constraints"); + + assert_eq!(first.fingerprint_bytes().len(), 32); + assert_eq!(first.fingerprint_bytes(), second.fingerprint_bytes()); + assert_eq!(first.fingerprint_sha256_hex().len(), 64); +} + +#[test] +fn full_afrinic_ipv4_fixture_is_parseable() { + let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/ta_constraints/afrinic-full-ipv4-deny.constraints"); + let raw = std::fs::read_to_string(&path).expect("read full AFRINIC fixture"); + let source_ipv4_rules = raw + .lines() + .filter(|line| line.trim_start().starts_with("deny ")) + .count(); + assert_eq!(source_ipv4_rules, 383); + + let constraints = TaConstraints::from_file(&path).expect("parse full AFRINIC fixture"); + assert_eq!(constraints.allow_v4.len(), 1); + assert_eq!(constraints.allow_v6.len(), 1); + assert_eq!(constraints.allow_asn.len(), 1); + assert!(!constraints.deny_v4.is_empty()); +} + +#[test] +fn current_afrinic_ipv4_fixture_is_parseable_and_canonical() { + let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/ta_constraints/afrinic-current-ipv4-deny.constraints"); + let raw = std::fs::read_to_string(&path).expect("read current AFRINIC fixture"); + let source_ipv4_rules = raw + .lines() + .filter(|line| line.trim_start().starts_with("deny ")) + .count(); + assert_eq!(source_ipv4_rules, 1825); + + let constraints = TaConstraints::from_file(&path).expect("parse current AFRINIC fixture"); + assert_eq!(constraints.allow_v4.len(), 1); + assert_eq!(constraints.allow_v6.len(), 1); + assert_eq!(constraints.allow_asn.len(), 1); + assert_eq!(constraints.deny_v4.len(), 815); + assert!(constraints.deny_v4.uses_tree()); + assert!(!constraints.allow_v4.uses_tree()); + assert!(!constraints.allow_v6.uses_tree()); + assert!(!constraints.allow_asn.uses_tree()); + assert!(constraints.warnings.is_empty()); +} diff --git a/crates/panda-rpki-validator/src/validation/ca_path.rs b/crates/panda-rpki-validator/src/validation/ca_path.rs index d67c7c2..658b82c 100644 --- a/crates/panda-rpki-validator/src/validation/ca_path.rs +++ b/crates/panda-rpki-validator/src/validation/ca_path.rs @@ -1,2409 +1,9 @@ -use crate::data_model::common::BigUnsigned; -use crate::data_model::crl::{CrlDecodeError, CrlVerifyError, RpkixCrl}; -use crate::data_model::oid::OID_KEY_USAGE_RAW; -use crate::data_model::rc::{ - AsIdentifierChoice, AsResourceSet, IpAddressChoice, IpResourceSet, ResourceCertKind, - ResourceCertificate, ResourceCertificateDecodeError, ResourceCertificateProfileError, - ResourceCertificateRole, -}; -use crate::policy::ResourceValidationMode; -use x509_parser::prelude::{FromDer, X509Certificate}; - -use crate::validation::x509_name::x509_names_equivalent; -use std::collections::{BTreeMap, HashMap, HashSet}; -use x509_parser::x509::SubjectPublicKeyInfo; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ValidatedSubordinateCa { - pub child_ca: ResourceCertificate, - pub issuer_ca: ResourceCertificate, - pub issuer_crl: RpkixCrl, - pub effective_ip_resources: Option, - pub effective_as_resources: Option, - pub resource_warnings: ResourceValidationWarnings, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ValidatedSubordinateCaLite { - pub child_ca: ResourceCertificate, - pub effective_ip_resources: Option, - pub effective_as_resources: Option, - pub resource_warnings: ResourceValidationWarnings, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct ResourceValidationWarnings { - pub ip_reduced_by_vrs: bool, - pub as_reduced_by_vrs: bool, - pub ip_vrs_empty: bool, - pub as_vrs_empty: bool, -} - -impl ResourceValidationWarnings { - pub fn is_empty(&self) -> bool { - !self.ip_reduced_by_vrs - && !self.as_reduced_by_vrs - && !self.ip_vrs_empty - && !self.as_vrs_empty - } - - pub fn summary(&self) -> String { - let mut parts = Vec::new(); - if self.ip_reduced_by_vrs { - parts.push("ip_reduced_by_vrs"); - } - if self.as_reduced_by_vrs { - parts.push("as_reduced_by_vrs"); - } - if self.ip_vrs_empty { - parts.push("ip_vrs_empty"); - } - if self.as_vrs_empty { - parts.push("as_vrs_empty"); - } - parts.join(",") - } -} - -#[derive(Clone, Debug, Default)] -pub struct IssuerEffectiveResourcesIndex { - parent_ip_by_afi_items: - Option>>, - parent_ip_merged_intervals: HashMap, Vec)>>, - parent_asnum_intervals: Option>, - parent_rdi_intervals: Option>, -} - -impl IssuerEffectiveResourcesIndex { - pub fn from_effective_resources( - issuer_effective_ip: Option<&IpResourceSet>, - issuer_effective_as: Option<&AsResourceSet>, - ) -> Result { - let parent_ip_by_afi_items = issuer_effective_ip - .map(ip_resources_by_afi_items) - .transpose()?; - - let parent_ip_merged_intervals = issuer_effective_ip - .map(ip_resources_to_merged_intervals_by_afi) - .unwrap_or_default(); - - let parent_asnum_intervals = issuer_effective_as - .and_then(|resources| resources.asnum.as_ref().map(as_choice_to_merged_intervals)); - let parent_rdi_intervals = issuer_effective_as - .and_then(|resources| resources.rdi.as_ref().map(as_choice_to_merged_intervals)); - - Ok(Self { - parent_ip_by_afi_items, - parent_ip_merged_intervals, - parent_asnum_intervals, - parent_rdi_intervals, - }) - } -} - -#[derive(Debug, thiserror::Error)] -pub enum CaPathError { - #[error("child CA certificate decode failed: {0} (RFC 6487 §4; RFC 5280 §4.1)")] - ChildDecode(#[from] ResourceCertificateDecodeError), - - #[error("issuer CA certificate decode failed: {0} (RFC 6487 §4; RFC 5280 §4.1)")] - IssuerDecode(ResourceCertificateDecodeError), - - #[error("child CA certificate profile validation failed: {0} (RFC 6487 §4.8)")] - ChildProfile(ResourceCertificateProfileError), - - #[error("issuer CA certificate profile validation failed: {0} (RFC 6487 §4.8)")] - IssuerProfile(ResourceCertificateProfileError), - - #[error("issuer CRL decode failed: {0} (RFC 6487 §5; RFC 9829 §3.1; RFC 5280 §5.1)")] - CrlDecode(#[from] CrlDecodeError), - - #[error( - "child certificate must be a CA resource certificate (RFC 6487 §4.8.1; RFC 5280 §4.2.1.9)" - )] - ChildNotCa, - - #[error( - "issuer certificate must be a CA resource certificate (RFC 6487 §4.8.1; RFC 5280 §4.2.1.9)" - )] - IssuerNotCa, - - #[error( - "child issuer DN does not match issuer CA subject DN: child.issuer={child_issuer_dn} issuer.subject={issuer_subject_dn} (RFC 5280 §6.1)" - )] - IssuerSubjectMismatch { - child_issuer_dn: String, - issuer_subject_dn: String, - }, - - #[error("child CA certificate signature verification failed: {0} (RFC 5280 §6.1)")] - ChildSignatureInvalid(String), - - #[error("issuer SubjectPublicKeyInfo parse error: {0} (RFC 5280 §4.1.2.7)")] - IssuerSpkiParse(String), - - #[error( - "trailing bytes after issuer SubjectPublicKeyInfo DER: {0} bytes (DER; RFC 5280 §4.1.2.7)" - )] - IssuerSpkiTrailingBytes(usize), - - #[error("certificate not valid at validation_time (RFC 5280 §4.1.2.5; RFC 5280 §6.1)")] - CertificateNotValidAtTime, - - #[error("child CA KeyUsage extension missing (RFC 6487 §4.8.4; RFC 5280 §4.2.1.3)")] - KeyUsageMissing, - - #[error("child CA KeyUsage criticality must be critical (RFC 6487 §4.8.4; RFC 5280 §4.2.1.3)")] - KeyUsageNotCritical, - - #[error("child CA KeyUsage must have only keyCertSign and cRLSign set (RFC 6487 §4.8.4)")] - KeyUsageInvalidBits, - - #[error( - "CRL signature/binding verification failed: {0} (RFC 5280 §6.3.3; RFC 6487 §5; RFC 9829 §3.1)" - )] - CrlVerify(#[from] CrlVerifyError), - - #[error( - "CRL not valid at validation_time (RFC 5280 §6.3.3(g); RFC 5280 §5.1.2.4-§5.1.2.5; RFC 6487 §5)" - )] - CrlNotValidAtTime, - - #[error("child CA certificate is revoked by issuer CRL (RFC 5280 §6.3.3; RFC 6487 §5)")] - ChildRevoked, - - #[error( - "child CA certificate must contain at least one RFC 3779 resource extension (IP or AS) (RFC 6487 §4.8.10-§4.8.11)" - )] - ResourcesMissing, - - #[error( - "resource extension inheritance cannot be resolved (parent missing resources) (RFC 6487 §7.2)" - )] - InheritWithoutParentResources, - - #[error("child CA resources are not a subset of issuer resources (RFC 6487 §7.2)")] - ResourcesNotSubset, - - #[error("issuer CA subjectKeyIdentifier missing (RFC 6487 §4.8.2)")] - IssuerSkiMissing, - - #[error("child CA authorityKeyIdentifier missing (RFC 6487 §4.8.3; RFC 5280 §4.2.1.1)")] - ChildAkiMissing, - - #[error( - "child CA authorityKeyIdentifier does not match issuer subjectKeyIdentifier (RFC 6487 §4.8.3)" - )] - ChildAkiMismatch, - - #[error("child CA authorityInfoAccess missing (RFC 6487 §4.8.7; RFC 5280 §4.2.2.1)")] - ChildAiaMissing, - - #[error( - "child CA authorityInfoAccess does not reference issuer certificate rsync URI (RFC 6487 §4.8.7)" - )] - ChildAiaIssuerUriMismatch, - - #[error("child CA CRLDistributionPoints missing (RFC 6487 §4.8.6; RFC 5280 §4.2.1.13)")] - ChildCrlDpMissing, - - #[error( - "child CA CRLDistributionPoints does not reference issuer CRL rsync URI (RFC 6487 §4.8.6)" - )] - ChildCrlDpUriMismatch, -} - -pub fn validate_subordinate_ca_cert( - child_ca_der: &[u8], - issuer_ca_der: &[u8], - issuer_crl_der: &[u8], - issuer_ca_rsync_uri: Option<&str>, - issuer_crl_rsync_uri: &str, - issuer_effective_ip: Option<&IpResourceSet>, - issuer_effective_as: Option<&AsResourceSet>, - validation_time: time::OffsetDateTime, -) -> Result { - validate_subordinate_ca_cert_with_resource_validation_mode( - child_ca_der, - issuer_ca_der, - issuer_crl_der, - issuer_ca_rsync_uri, - issuer_crl_rsync_uri, - issuer_effective_ip, - issuer_effective_as, - validation_time, - ResourceValidationMode::Rfc6487, - ) -} - -pub fn validate_subordinate_ca_cert_with_resource_validation_mode( - child_ca_der: &[u8], - issuer_ca_der: &[u8], - issuer_crl_der: &[u8], - issuer_ca_rsync_uri: Option<&str>, - issuer_crl_rsync_uri: &str, - issuer_effective_ip: Option<&IpResourceSet>, - issuer_effective_as: Option<&AsResourceSet>, - validation_time: time::OffsetDateTime, - resource_validation_mode: ResourceValidationMode, -) -> Result { - let child_ca = ResourceCertificate::decode_der(child_ca_der)?; - if child_ca.kind != ResourceCertKind::Ca { - return Err(CaPathError::ChildNotCa); - } - child_ca - .validate_rfc6487_profile(ResourceCertificateRole::Ca) - .map_err(CaPathError::ChildProfile)?; - - let issuer_ca = - ResourceCertificate::decode_der(issuer_ca_der).map_err(CaPathError::IssuerDecode)?; - if issuer_ca.kind != ResourceCertKind::Ca { - return Err(CaPathError::IssuerNotCa); - } - issuer_ca - .validate_rfc6487_profile(ResourceCertificateRole::Ca) - .map_err(CaPathError::IssuerProfile)?; - let issuer_spki = parse_subject_pki_from_der(&issuer_ca.tbs.subject_public_key_info)?; - - if !x509_names_equivalent(&child_ca.tbs.issuer_name, &issuer_ca.tbs.subject_name) { - return Err(CaPathError::IssuerSubjectMismatch { - child_issuer_dn: child_ca.tbs.issuer_name.to_string(), - issuer_subject_dn: issuer_ca.tbs.subject_name.to_string(), - }); - } - - validate_child_aki_matches_issuer_ski(&child_ca, &issuer_ca)?; - if let Some(expected_issuer_uri) = issuer_ca_rsync_uri { - validate_child_aia_points_to_issuer_uri(&child_ca, expected_issuer_uri)?; - } - validate_child_crldp_contains_issuer_crl_uri(&child_ca, issuer_crl_rsync_uri)?; - - if !time_within_validity( - validation_time, - child_ca.tbs.validity_not_before, - child_ca.tbs.validity_not_after, - ) || !time_within_validity( - validation_time, - issuer_ca.tbs.validity_not_before, - issuer_ca.tbs.validity_not_after, - ) { - return Err(CaPathError::CertificateNotValidAtTime); - } - - let child_x509 = parse_x509_cert(child_ca_der)?; - verify_child_signature(&child_x509, &issuer_spki)?; - validate_child_ca_key_usage(&child_x509)?; - - let issuer_crl = RpkixCrl::decode_der(issuer_crl_der)?; - issuer_crl.verify_signature_with_issuer_certificate_der(issuer_ca_der)?; - if !crl_valid_at_time(&issuer_crl, validation_time) { - return Err(CaPathError::CrlNotValidAtTime); - } - - if is_serial_revoked_by_crl(&child_ca, &issuer_crl) { - return Err(CaPathError::ChildRevoked); - } - - let ResourceResolution { - effective_ip_resources, - effective_as_resources, - warnings: resource_warnings, - } = resolve_child_resources( - child_ca.tbs.extensions.ip_resources.as_ref(), - issuer_effective_ip, - child_ca.tbs.extensions.as_resources.as_ref(), - issuer_effective_as, - &IssuerEffectiveResourcesIndex::from_effective_resources( - issuer_effective_ip, - issuer_effective_as, - )?, - resource_validation_mode, - )?; - if effective_ip_resources.is_none() && effective_as_resources.is_none() { - return Err(CaPathError::ResourcesMissing); - } - - Ok(ValidatedSubordinateCa { - child_ca, - issuer_ca, - issuer_crl, - effective_ip_resources, - effective_as_resources, - resource_warnings, - }) -} - -/// Validate a subordinate child CA using *pre-decoded issuer CA* and *pre-decoded+verified issuer CRL*. -/// -/// This avoids repeating issuer CA decode and issuer CRL decode+signature verification for every -/// child CA certificate discovered in a publication point. -pub fn validate_subordinate_ca_cert_with_prevalidated_issuer( - child_ca_der: &[u8], - child_ca: ResourceCertificate, - issuer_ca: &ResourceCertificate, - issuer_spki: &SubjectPublicKeyInfo<'_>, - issuer_crl: &RpkixCrl, - issuer_crl_revoked_serials: &HashSet>, - issuer_ca_rsync_uri: Option<&str>, - issuer_crl_rsync_uri: &str, - issuer_effective_ip: Option<&IpResourceSet>, - issuer_effective_as: Option<&AsResourceSet>, - validation_time: time::OffsetDateTime, -) -> Result { - let issuer_resources_index = IssuerEffectiveResourcesIndex::from_effective_resources( - issuer_effective_ip, - issuer_effective_as, - )?; - validate_subordinate_ca_cert_with_prevalidated_issuer_and_resources( - child_ca_der, - child_ca, - issuer_ca, - issuer_spki, - issuer_crl, - issuer_crl_revoked_serials, - issuer_ca_rsync_uri, - issuer_crl_rsync_uri, - issuer_effective_ip, - issuer_effective_as, - &issuer_resources_index, - validation_time, - ResourceValidationMode::Rfc6487, - ) -} - -pub fn validate_subordinate_ca_cert_with_prevalidated_issuer_and_resources( - child_ca_der: &[u8], - child_ca: ResourceCertificate, - issuer_ca: &ResourceCertificate, - issuer_spki: &SubjectPublicKeyInfo<'_>, - issuer_crl: &RpkixCrl, - issuer_crl_revoked_serials: &HashSet>, - issuer_ca_rsync_uri: Option<&str>, - issuer_crl_rsync_uri: &str, - issuer_effective_ip: Option<&IpResourceSet>, - issuer_effective_as: Option<&AsResourceSet>, - issuer_resources_index: &IssuerEffectiveResourcesIndex, - validation_time: time::OffsetDateTime, - resource_validation_mode: ResourceValidationMode, -) -> Result { - if child_ca.kind != ResourceCertKind::Ca { - return Err(CaPathError::ChildNotCa); - } - if issuer_ca.kind != ResourceCertKind::Ca { - return Err(CaPathError::IssuerNotCa); - } - child_ca - .validate_rfc6487_profile(ResourceCertificateRole::Ca) - .map_err(CaPathError::ChildProfile)?; - issuer_ca - .validate_rfc6487_profile(ResourceCertificateRole::Ca) - .map_err(CaPathError::IssuerProfile)?; - - if !x509_names_equivalent(&child_ca.tbs.issuer_name, &issuer_ca.tbs.subject_name) { - return Err(CaPathError::IssuerSubjectMismatch { - child_issuer_dn: child_ca.tbs.issuer_name.to_string(), - issuer_subject_dn: issuer_ca.tbs.subject_name.to_string(), - }); - } - - validate_child_aki_matches_issuer_ski(&child_ca, issuer_ca)?; - if let Some(expected_issuer_uri) = issuer_ca_rsync_uri { - validate_child_aia_points_to_issuer_uri(&child_ca, expected_issuer_uri)?; - } - validate_child_crldp_contains_issuer_crl_uri(&child_ca, issuer_crl_rsync_uri)?; - - if !time_within_validity( - validation_time, - child_ca.tbs.validity_not_before, - child_ca.tbs.validity_not_after, - ) || !time_within_validity( - validation_time, - issuer_ca.tbs.validity_not_before, - issuer_ca.tbs.validity_not_after, - ) { - return Err(CaPathError::CertificateNotValidAtTime); - } - - let child_x509 = parse_x509_cert(child_ca_der)?; - verify_child_signature(&child_x509, issuer_spki)?; - validate_child_ca_key_usage(&child_x509)?; - - if !crl_valid_at_time(issuer_crl, validation_time) { - return Err(CaPathError::CrlNotValidAtTime); - } - - let serial = BigUnsigned::from_biguint(&child_ca.tbs.serial_number); - if issuer_crl_revoked_serials.contains(&serial.bytes_be) { - return Err(CaPathError::ChildRevoked); - } - - let ResourceResolution { - effective_ip_resources, - effective_as_resources, - warnings: resource_warnings, - } = resolve_child_resources( - child_ca.tbs.extensions.ip_resources.as_ref(), - issuer_effective_ip, - child_ca.tbs.extensions.as_resources.as_ref(), - issuer_effective_as, - issuer_resources_index, - resource_validation_mode, - )?; - if effective_ip_resources.is_none() && effective_as_resources.is_none() { - return Err(CaPathError::ResourcesMissing); - } - - Ok(ValidatedSubordinateCaLite { - child_ca, - effective_ip_resources, - effective_as_resources, - resource_warnings, - }) -} - -fn parse_subject_pki_from_der(der: &[u8]) -> Result, CaPathError> { - let (rem, spki) = SubjectPublicKeyInfo::from_der(der) - .map_err(|e| CaPathError::IssuerSpkiParse(e.to_string()))?; - if !rem.is_empty() { - return Err(CaPathError::IssuerSpkiTrailingBytes(rem.len())); - } - Ok(spki) -} - -fn parse_x509_cert(der: &[u8]) -> Result, CaPathError> { - let (rem, cert) = X509Certificate::from_der(der) - .map_err(|e| CaPathError::ChildSignatureInvalid(e.to_string()))?; - if !rem.is_empty() { - return Err(CaPathError::ChildSignatureInvalid( - "trailing bytes after child certificate".to_string(), - )); - } - Ok(cert) -} - -fn verify_child_signature( - child: &X509Certificate<'_>, - issuer_spki: &SubjectPublicKeyInfo<'_>, -) -> Result<(), CaPathError> { - crate::crypto_sig_cache::verify_with_cache( - crate::crypto_sig_cache::CryptoSigVerifyPoint::ChildCaCert, - child.tbs_certificate.as_ref(), - child.signature_value.data.as_ref(), - issuer_spki.raw, - || { - child - .verify_signature(Some(issuer_spki)) - .map_err(|e| CaPathError::ChildSignatureInvalid(e.to_string())) - }, - ) -} - -fn validate_child_aki_matches_issuer_ski( - child: &ResourceCertificate, - issuer: &ResourceCertificate, -) -> Result<(), CaPathError> { - let Some(issuer_ski) = issuer.tbs.extensions.subject_key_identifier.as_deref() else { - return Err(CaPathError::IssuerSkiMissing); - }; - let Some(child_aki) = child.tbs.extensions.authority_key_identifier.as_deref() else { - return Err(CaPathError::ChildAkiMissing); - }; - if child_aki != issuer_ski { - return Err(CaPathError::ChildAkiMismatch); - } - Ok(()) -} - -fn validate_child_aia_points_to_issuer_uri( - child: &ResourceCertificate, - issuer_ca_rsync_uri: &str, -) -> Result<(), CaPathError> { - let Some(uris) = child.tbs.extensions.ca_issuers_uris.as_ref() else { - return Err(CaPathError::ChildAiaMissing); - }; - if !uris.iter().any(|u| u.as_str() == issuer_ca_rsync_uri) { - return Err(CaPathError::ChildAiaIssuerUriMismatch); - } - Ok(()) -} - -fn validate_child_crldp_contains_issuer_crl_uri( - child: &ResourceCertificate, - issuer_crl_rsync_uri: &str, -) -> Result<(), CaPathError> { - let Some(uris) = child.tbs.extensions.crl_distribution_points_uris.as_ref() else { - return Err(CaPathError::ChildCrlDpMissing); - }; - if !uris.iter().any(|u| u.as_str() == issuer_crl_rsync_uri) { - return Err(CaPathError::ChildCrlDpUriMismatch); - } - Ok(()) -} - -fn validate_child_ca_key_usage(cert: &X509Certificate<'_>) -> Result<(), CaPathError> { - let mut ku_critical: Option = None; - for ext in cert.extensions() { - if ext.oid.as_bytes() == OID_KEY_USAGE_RAW { - ku_critical = Some(ext.critical); - break; - } - } - - let Some(critical) = ku_critical else { - return Err(CaPathError::KeyUsageMissing); - }; - if !critical { - return Err(CaPathError::KeyUsageNotCritical); - } - - let Some(ku) = cert - .key_usage() - .map_err(|e| CaPathError::ChildSignatureInvalid(e.to_string()))? - else { - return Err(CaPathError::KeyUsageMissing); - }; - - let v = &ku.value; - let ok = v.key_cert_sign() - && v.crl_sign() - && !v.digital_signature() - && !v.non_repudiation() - && !v.key_encipherment() - && !v.data_encipherment() - && !v.key_agreement() - && !v.encipher_only() - && !v.decipher_only(); - if !ok { - return Err(CaPathError::KeyUsageInvalidBits); - } - - Ok(()) -} - -fn time_within_validity( - t: time::OffsetDateTime, - not_before: time::OffsetDateTime, - not_after: time::OffsetDateTime, -) -> bool { - let t = t.to_offset(time::UtcOffset::UTC); - let not_before = not_before.to_offset(time::UtcOffset::UTC); - let not_after = not_after.to_offset(time::UtcOffset::UTC); - t >= not_before && t <= not_after -} - -fn crl_valid_at_time(crl: &RpkixCrl, t: time::OffsetDateTime) -> bool { - let t = t.to_offset(time::UtcOffset::UTC); - let this_update = crl.this_update.utc.to_offset(time::UtcOffset::UTC); - let next_update = crl.next_update.utc.to_offset(time::UtcOffset::UTC); - t >= this_update && t < next_update -} - -fn is_serial_revoked_by_crl(cert: &ResourceCertificate, crl: &RpkixCrl) -> bool { - let serial = BigUnsigned::from_biguint(&cert.tbs.serial_number); - crl.revoked_certs - .iter() - .any(|rc| rc.serial_number == serial) -} - -#[derive(Clone, Debug)] -struct ResourceResolution { - effective_ip_resources: Option, - effective_as_resources: Option, - warnings: ResourceValidationWarnings, -} - -fn resolve_child_resources( - child_ip: Option<&IpResourceSet>, - issuer_effective_ip: Option<&IpResourceSet>, - child_as: Option<&AsResourceSet>, - issuer_effective_as: Option<&AsResourceSet>, - issuer_resources_index: &IssuerEffectiveResourcesIndex, - mode: ResourceValidationMode, -) -> Result { - match mode { - ResourceValidationMode::Rfc6487 => Ok(ResourceResolution { - effective_ip_resources: resolve_child_ip_resources_indexed( - child_ip, - issuer_effective_ip, - issuer_resources_index.parent_ip_by_afi_items.as_ref(), - &issuer_resources_index.parent_ip_merged_intervals, - )?, - effective_as_resources: resolve_child_as_resources_indexed( - child_as, - issuer_effective_as, - issuer_resources_index.parent_asnum_intervals.as_deref(), - issuer_resources_index.parent_rdi_intervals.as_deref(), - )?, - warnings: ResourceValidationWarnings::default(), - }), - ResourceValidationMode::ValidationUpdate03 => { - let (effective_ip_resources, ip_reduced_by_vrs, ip_vrs_empty) = - resolve_child_ip_resources_vrs( - child_ip, - issuer_effective_ip, - issuer_resources_index.parent_ip_by_afi_items.as_ref(), - &issuer_resources_index.parent_ip_merged_intervals, - )?; - let (effective_as_resources, as_reduced_by_vrs, as_vrs_empty) = - resolve_child_as_resources_vrs( - child_as, - issuer_effective_as, - issuer_resources_index.parent_asnum_intervals.as_deref(), - issuer_resources_index.parent_rdi_intervals.as_deref(), - )?; - Ok(ResourceResolution { - effective_ip_resources, - effective_as_resources, - warnings: ResourceValidationWarnings { - ip_reduced_by_vrs, - as_reduced_by_vrs, - ip_vrs_empty, - as_vrs_empty, - }, - }) - } - } -} - -fn resolve_child_ip_resources( - child_ip: Option<&IpResourceSet>, - issuer_effective: Option<&IpResourceSet>, -) -> Result, CaPathError> { - let precomputed_parent_by_afi = issuer_effective - .map(ip_resources_by_afi_items) - .transpose()?; - let precomputed_parent_intervals = issuer_effective - .map(ip_resources_to_merged_intervals_by_afi) - .unwrap_or_default(); - resolve_child_ip_resources_indexed( - child_ip, - issuer_effective, - precomputed_parent_by_afi.as_ref(), - &precomputed_parent_intervals, - ) -} - -fn resolve_child_ip_resources_indexed( - child_ip: Option<&IpResourceSet>, - issuer_effective: Option<&IpResourceSet>, - parent_by_afi: Option< - &BTreeMap>, - >, - parent_intervals_by_afi: &HashMap, Vec)>>, -) -> Result, CaPathError> { - let Some(child_ip) = child_ip else { - return Ok(None); - }; - - let Some(_parent) = issuer_effective else { - if child_ip.has_any_inherit() { - return Err(CaPathError::InheritWithoutParentResources); - } - // With no parent effective resources, we cannot validate subset. - return Err(CaPathError::ResourcesNotSubset); - }; - - // Resolve per-AFI inherit, producing an effective set with no inherit. - let parent_by_afi = parent_by_afi.ok_or(CaPathError::InheritWithoutParentResources)?; - let mut out_families: Vec = Vec::new(); - - for fam in &child_ip.families { - match &fam.choice { - IpAddressChoice::Inherit => { - let items = parent_by_afi - .get(&fam.afi) - .ok_or(CaPathError::InheritWithoutParentResources)?; - out_families.push(crate::data_model::rc::IpAddressFamily { - afi: fam.afi, - choice: IpAddressChoice::AddressesOrRanges(items.clone()), - }); - } - IpAddressChoice::AddressesOrRanges(items) => { - // Subset check against parent union for that AFI. - let parent_intervals = parent_intervals_by_afi - .get(&fam.afi) - .map(Vec::as_slice) - .unwrap_or(&[]); - if !ip_family_items_subset_with_parent_intervals(items, parent_intervals) { - return Err(CaPathError::ResourcesNotSubset); - } - out_families.push(crate::data_model::rc::IpAddressFamily { - afi: fam.afi, - choice: IpAddressChoice::AddressesOrRanges(items.clone()), - }); - } - } - } - - Ok(Some(IpResourceSet { - families: out_families, - })) -} - -fn resolve_child_ip_resources_vrs( - child_ip: Option<&IpResourceSet>, - issuer_effective: Option<&IpResourceSet>, - parent_by_afi: Option< - &BTreeMap>, - >, - parent_intervals_by_afi: &HashMap, Vec)>>, -) -> Result<(Option, bool, bool), CaPathError> { - let Some(child_ip) = child_ip else { - return Ok((None, false, false)); - }; - if child_ip.has_any_inherit() && issuer_effective.is_none() { - return Err(CaPathError::InheritWithoutParentResources); - } - - let parent_by_afi = parent_by_afi.unwrap_or_else(|| { - static EMPTY: std::sync::OnceLock< - BTreeMap>, - > = std::sync::OnceLock::new(); - EMPTY.get_or_init(BTreeMap::new) - }); - let mut out_families = Vec::new(); - let mut reduced = false; - let mut saw_declared = false; - - for fam in &child_ip.families { - match &fam.choice { - IpAddressChoice::Inherit => { - let items = parent_by_afi - .get(&fam.afi) - .ok_or(CaPathError::InheritWithoutParentResources)?; - out_families.push(crate::data_model::rc::IpAddressFamily { - afi: fam.afi, - choice: IpAddressChoice::AddressesOrRanges(items.clone()), - }); - } - IpAddressChoice::AddressesOrRanges(items) => { - saw_declared = saw_declared || !items.is_empty(); - let parent_intervals = parent_intervals_by_afi - .get(&fam.afi) - .map(Vec::as_slice) - .unwrap_or(&[]); - let intersections = - intersect_ip_items_with_parent_intervals(items, parent_intervals); - let child_intervals = ip_items_to_merged_intervals(items); - if intervals_changed(&child_intervals, &intersections) { - reduced = true; - } - if !intersections.is_empty() { - out_families.push(crate::data_model::rc::IpAddressFamily { - afi: fam.afi, - choice: IpAddressChoice::AddressesOrRanges(ip_intervals_to_ranges( - fam.afi, - &intersections, - )), - }); - } - } - } - } - - let empty = saw_declared && out_families.is_empty(); - Ok(( - Some(IpResourceSet { - families: out_families, - }), - reduced, - empty, - )) -} - -fn resolve_child_as_resources( - child_as: Option<&AsResourceSet>, - issuer_effective: Option<&AsResourceSet>, -) -> Result, CaPathError> { - let precomputed_asnum = issuer_effective - .and_then(|resources| resources.asnum.as_ref().map(as_choice_to_merged_intervals)); - let precomputed_rdi = issuer_effective - .and_then(|resources| resources.rdi.as_ref().map(as_choice_to_merged_intervals)); - resolve_child_as_resources_indexed( - child_as, - issuer_effective, - precomputed_asnum.as_deref(), - precomputed_rdi.as_deref(), - ) -} - -fn resolve_child_as_resources_indexed( - child_as: Option<&AsResourceSet>, - issuer_effective: Option<&AsResourceSet>, - parent_asnum_intervals: Option<&[(u32, u32)]>, - parent_rdi_intervals: Option<&[(u32, u32)]>, -) -> Result, CaPathError> { - let Some(child_as) = child_as else { - return Ok(None); - }; - let Some(parent) = issuer_effective else { - if matches!(child_as.asnum, Some(AsIdentifierChoice::Inherit)) - || matches!(child_as.rdi, Some(AsIdentifierChoice::Inherit)) - { - return Err(CaPathError::InheritWithoutParentResources); - } - return Err(CaPathError::ResourcesNotSubset); - }; - - let asnum = match child_as.asnum.as_ref() { - None => None, - Some(AsIdentifierChoice::Inherit) => parent - .asnum - .clone() - .ok_or(CaPathError::InheritWithoutParentResources) - .map(Some)?, - Some(_) => { - if !as_choice_subset_with_parent_intervals( - child_as.asnum.as_ref(), - parent.asnum.as_ref(), - parent_asnum_intervals, - ) { - return Err(CaPathError::ResourcesNotSubset); - } - child_as.asnum.clone() - } - }; - - let rdi = match child_as.rdi.as_ref() { - None => None, - Some(AsIdentifierChoice::Inherit) => parent - .rdi - .clone() - .ok_or(CaPathError::InheritWithoutParentResources) - .map(Some)?, - Some(_) => { - if !as_choice_subset_with_parent_intervals( - child_as.rdi.as_ref(), - parent.rdi.as_ref(), - parent_rdi_intervals, - ) { - return Err(CaPathError::ResourcesNotSubset); - } - child_as.rdi.clone() - } - }; - - Ok(Some(AsResourceSet { asnum, rdi })) -} - -fn resolve_child_as_resources_vrs( - child_as: Option<&AsResourceSet>, - issuer_effective: Option<&AsResourceSet>, - parent_asnum_intervals: Option<&[(u32, u32)]>, - parent_rdi_intervals: Option<&[(u32, u32)]>, -) -> Result<(Option, bool, bool), CaPathError> { - let Some(child_as) = child_as else { - return Ok((None, false, false)); - }; - if issuer_effective.is_none() - && (matches!(child_as.asnum, Some(AsIdentifierChoice::Inherit)) - || matches!(child_as.rdi, Some(AsIdentifierChoice::Inherit))) - { - return Err(CaPathError::InheritWithoutParentResources); - } - - let (asnum, asnum_reduced, asnum_empty) = resolve_as_choice_vrs( - child_as.asnum.as_ref(), - issuer_effective.and_then(|p| p.asnum.as_ref()), - parent_asnum_intervals, - )?; - let (rdi, rdi_reduced, rdi_empty) = resolve_as_choice_vrs( - child_as.rdi.as_ref(), - issuer_effective.and_then(|p| p.rdi.as_ref()), - parent_rdi_intervals, - )?; - Ok(( - Some(AsResourceSet { asnum, rdi }), - asnum_reduced || rdi_reduced, - asnum_empty || rdi_empty, - )) -} - -fn resolve_as_choice_vrs( - child: Option<&AsIdentifierChoice>, - parent: Option<&AsIdentifierChoice>, - parent_intervals_hint: Option<&[(u32, u32)]>, -) -> Result<(Option, bool, bool), CaPathError> { - let Some(child) = child else { - return Ok((None, false, false)); - }; - match child { - AsIdentifierChoice::Inherit => { - let parent = parent - .cloned() - .ok_or(CaPathError::InheritWithoutParentResources)?; - Ok((Some(parent), false, false)) - } - AsIdentifierChoice::AsIdsOrRanges(_) => { - let child_intervals = as_choice_to_merged_intervals(child); - let parent_intervals; - let parent_intervals = match parent_intervals_hint { - Some(v) => v, - None => { - parent_intervals = parent - .map(as_choice_to_merged_intervals) - .unwrap_or_default(); - parent_intervals.as_slice() - } - }; - let intersections = intersect_as_intervals(&child_intervals, parent_intervals); - let reduced = child_intervals != intersections; - let empty = !child_intervals.is_empty() && intersections.is_empty(); - Ok(( - Some(AsIdentifierChoice::AsIdsOrRanges(as_intervals_to_items( - &intersections, - ))), - reduced, - empty, - )) - } - } -} - -fn as_choice_subset( - child: Option<&AsIdentifierChoice>, - parent: Option<&AsIdentifierChoice>, -) -> bool { - as_choice_subset_with_parent_intervals(child, parent, None) -} - -fn as_choice_subset_with_parent_intervals( - child: Option<&AsIdentifierChoice>, - parent: Option<&AsIdentifierChoice>, - parent_intervals_hint: Option<&[(u32, u32)]>, -) -> bool { - let Some(child) = child else { - return true; - }; - let Some(parent) = parent else { - return false; - }; - - // Treat inherit as "all of parent" here; actual resolution is handled elsewhere. - if matches!(child, AsIdentifierChoice::Inherit) { - return true; - } - if matches!(parent, AsIdentifierChoice::Inherit) { - return true; - } - - let child_intervals = as_choice_to_merged_intervals(child); - let owned_parent_intervals; - let parent_intervals = match parent_intervals_hint { - Some(intervals) => intervals, - None => { - owned_parent_intervals = as_choice_to_merged_intervals(parent); - owned_parent_intervals.as_slice() - } - }; - for (cmin, cmax) in &child_intervals { - if !as_interval_is_covered(parent_intervals, *cmin, *cmax) { - return false; - } - } - true -} - -fn as_choice_to_merged_intervals(choice: &AsIdentifierChoice) -> Vec<(u32, u32)> { - let mut v = Vec::new(); - match choice { - AsIdentifierChoice::Inherit => {} - AsIdentifierChoice::AsIdsOrRanges(items) => { - for item in items { - match item { - crate::data_model::rc::AsIdOrRange::Id(id) => v.push((*id, *id)), - crate::data_model::rc::AsIdOrRange::Range { min, max } => v.push((*min, *max)), - } - } - } - } - v.sort_by_key(|(a, _b)| *a); - merge_as_intervals(&v) -} - -fn merge_as_intervals(v: &[(u32, u32)]) -> Vec<(u32, u32)> { - let mut out: Vec<(u32, u32)> = Vec::new(); - for (min, max) in v { - let Some(last) = out.last_mut() else { - out.push((*min, *max)); - continue; - }; - if *min <= last.1.saturating_add(1) { - last.1 = last.1.max(*max); - continue; - } - out.push((*min, *max)); - } - out -} - -fn as_interval_is_covered(parent: &[(u32, u32)], min: u32, max: u32) -> bool { - for (pmin, pmax) in parent { - if *pmin <= min && max <= *pmax { - return true; - } - if *pmin > min { - break; - } - } - false -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -enum AfiKey { - V4, - V6, -} - -fn ip_resources_by_afi_items( - set: &IpResourceSet, -) -> Result< - std::collections::BTreeMap< - crate::data_model::rc::Afi, - Vec, - >, - CaPathError, -> { - let mut m: std::collections::BTreeMap< - crate::data_model::rc::Afi, - Vec, - > = std::collections::BTreeMap::new(); - for fam in &set.families { - match &fam.choice { - IpAddressChoice::Inherit => return Err(CaPathError::InheritWithoutParentResources), - IpAddressChoice::AddressesOrRanges(items) => { - m.insert(fam.afi, items.clone()); - } - } - } - Ok(m) -} - -fn ip_resources_single_afi( - parent: &IpResourceSet, - afi: crate::data_model::rc::Afi, - items: Option<&Vec>, -) -> IpResourceSet { - let mut families = Vec::new(); - if let Some(items) = items { - families.push(crate::data_model::rc::IpAddressFamily { - afi, - choice: IpAddressChoice::AddressesOrRanges(items.clone()), - }); - } else { - // If parent doesn't mention this AFI explicitly, treat as empty. - // The subset check will fail. - let _ = parent; - } - IpResourceSet { families } -} - -fn ip_family_items_subset( - child_items: &[crate::data_model::rc::IpAddressOrRange], - parent_set: &IpResourceSet, -) -> bool { - let parent_by_afi = ip_resources_to_merged_intervals(parent_set); - // parent_set should contain exactly one AFI. - let (afi_key, parent_intervals) = match parent_by_afi.into_iter().next() { - None => return false, - Some(v) => v, - }; - - let mut child_intervals: Vec<(Vec, Vec)> = Vec::new(); - for item in child_items { - match item { - crate::data_model::rc::IpAddressOrRange::Prefix(p) => { - child_intervals.push(prefix_to_range(p)) - } - crate::data_model::rc::IpAddressOrRange::Range(r) => { - child_intervals.push((r.min.clone(), r.max.clone())) - } - } - } - child_intervals.sort_by(|(a, _), (b, _)| a.cmp(b)); - let child_intervals = merge_ip_intervals(&child_intervals); - - let _ = afi_key; - for (cmin, cmax) in &child_intervals { - if !interval_is_covered(&parent_intervals, cmin, cmax) { - return false; - } - } - true -} - -fn ip_resources_to_merged_intervals( - set: &IpResourceSet, -) -> std::collections::HashMap, Vec)>> { - let m = ip_resources_to_merged_intervals_by_afi(set); - m.into_iter() - .map(|(afi, ranges)| { - ( - match afi { - crate::data_model::rc::Afi::Ipv4 => AfiKey::V4, - crate::data_model::rc::Afi::Ipv6 => AfiKey::V6, - }, - ranges, - ) - }) - .collect() -} - -fn ip_resources_to_merged_intervals_by_afi( - set: &IpResourceSet, -) -> HashMap, Vec)>> { - let mut m: HashMap, Vec)>> = HashMap::new(); - - for fam in &set.families { - match &fam.choice { - IpAddressChoice::Inherit => { - // When used in subset checks, treat inherit as "all" by leaving it absent. - // Resolution should have happened earlier. - } - IpAddressChoice::AddressesOrRanges(items) => { - let ent = m.entry(fam.afi).or_default(); - for item in items { - match item { - crate::data_model::rc::IpAddressOrRange::Prefix(p) => { - let (min, max) = prefix_to_range(p); - ent.push((min, max)); - } - crate::data_model::rc::IpAddressOrRange::Range(r) => { - ent.push((r.min.clone(), r.max.clone())); - } - } - } - } - } - } - - for (_afi, v) in m.iter_mut() { - v.sort_by(|(a, _), (b, _)| a.cmp(b)); - *v = merge_ip_intervals(v); - } - - m -} - -fn ip_family_items_subset_with_parent_intervals( - child_items: &[crate::data_model::rc::IpAddressOrRange], - parent_intervals: &[(Vec, Vec)], -) -> bool { - if parent_intervals.is_empty() { - return false; - } - - let mut child_intervals: Vec<(Vec, Vec)> = Vec::new(); - for item in child_items { - match item { - crate::data_model::rc::IpAddressOrRange::Prefix(p) => { - child_intervals.push(prefix_to_range(p)) - } - crate::data_model::rc::IpAddressOrRange::Range(r) => { - child_intervals.push((r.min.clone(), r.max.clone())) - } - } - } - child_intervals.sort_by(|(a, _), (b, _)| a.cmp(b)); - let child_intervals = merge_ip_intervals(&child_intervals); - - for (cmin, cmax) in &child_intervals { - if !interval_is_covered(parent_intervals, cmin, cmax) { - return false; - } - } - true -} - -fn ip_items_to_merged_intervals( - items: &[crate::data_model::rc::IpAddressOrRange], -) -> Vec<(Vec, Vec)> { - let mut intervals = Vec::new(); - for item in items { - match item { - crate::data_model::rc::IpAddressOrRange::Prefix(p) => { - intervals.push(prefix_to_range(p)) - } - crate::data_model::rc::IpAddressOrRange::Range(r) => { - intervals.push((r.min.clone(), r.max.clone())) - } - } - } - intervals.sort_by(|(a, _), (b, _)| a.cmp(b)); - merge_ip_intervals(&intervals) -} - -fn intersect_ip_items_with_parent_intervals( - items: &[crate::data_model::rc::IpAddressOrRange], - parent_intervals: &[(Vec, Vec)], -) -> Vec<(Vec, Vec)> { - let child_intervals = ip_items_to_merged_intervals(items); - intersect_ip_intervals(&child_intervals, parent_intervals) -} - -fn intersect_ip_intervals( - child_intervals: &[(Vec, Vec)], - parent_intervals: &[(Vec, Vec)], -) -> Vec<(Vec, Vec)> { - let mut out = Vec::new(); - let mut parent_index = 0usize; - for (child_min, child_max) in child_intervals { - while parent_index < parent_intervals.len() - && parent_intervals[parent_index].1.as_slice() < child_min.as_slice() - { - parent_index += 1; - } - let mut scan = parent_index; - while scan < parent_intervals.len() - && parent_intervals[scan].0.as_slice() <= child_max.as_slice() - { - let (parent_min, parent_max) = &parent_intervals[scan]; - let min = if bytes_leq(child_min, parent_min) { - parent_min.clone() - } else { - child_min.clone() - }; - let max = if bytes_leq(child_max, parent_max) { - child_max.clone() - } else { - parent_max.clone() - }; - if bytes_leq(&min, &max) { - out.push((min, max)); - } - scan += 1; - } - } - merge_ip_intervals(&out) -} - -fn ip_intervals_to_ranges( - afi: crate::data_model::rc::Afi, - intervals: &[(Vec, Vec)], -) -> Vec { - intervals - .iter() - .map(|(min, max)| { - crate::data_model::rc::IpAddressOrRange::Range(crate::data_model::rc::IpAddressRange { - min: normalize_ip_bytes(afi, min), - max: normalize_ip_bytes(afi, max), - }) - }) - .collect() -} - -fn normalize_ip_bytes(afi: crate::data_model::rc::Afi, bytes: &[u8]) -> Vec { - let target_len = afi.octets_len(); - if bytes.len() == target_len { - return bytes.to_vec(); - } - let mut out = vec![0u8; target_len]; - let copy_len = bytes.len().min(target_len); - out[..copy_len].copy_from_slice(&bytes[..copy_len]); - out -} - -fn intervals_changed(left: &[T], right: &[T]) -> bool { - left != right -} - -fn intersect_as_intervals(child: &[(u32, u32)], parent: &[(u32, u32)]) -> Vec<(u32, u32)> { - let mut out = Vec::new(); - let mut parent_index = 0usize; - for (child_min, child_max) in child { - while parent_index < parent.len() && parent[parent_index].1 < *child_min { - parent_index += 1; - } - let mut scan = parent_index; - while scan < parent.len() && parent[scan].0 <= *child_max { - let min = (*child_min).max(parent[scan].0); - let max = (*child_max).min(parent[scan].1); - if min <= max { - out.push((min, max)); - } - scan += 1; - } - } - merge_as_intervals(&out) -} - -fn as_intervals_to_items(intervals: &[(u32, u32)]) -> Vec { - intervals - .iter() - .map(|(min, max)| { - if min == max { - crate::data_model::rc::AsIdOrRange::Id(*min) - } else { - crate::data_model::rc::AsIdOrRange::Range { - min: *min, - max: *max, - } - } - }) - .collect() -} - -fn merge_ip_intervals(v: &[(Vec, Vec)]) -> Vec<(Vec, Vec)> { - let mut out: Vec<(Vec, Vec)> = Vec::new(); - for (min, max) in v { - let Some(last) = out.last_mut() else { - out.push((min.clone(), max.clone())); - continue; - }; - - if bytes_leq(min, &increment_bytes(&last.1)) { - if bytes_leq(&last.1, max) { - last.1 = max.clone(); - } - continue; - } - - out.push((min.clone(), max.clone())); - } - out -} - -fn interval_is_covered(parent: &[(Vec, Vec)], min: &[u8], max: &[u8]) -> bool { - for (pmin, pmax) in parent { - if bytes_leq(pmin, min) && bytes_leq(max, pmax) { - return true; - } - if pmin.as_slice() > min { - break; - } - } - false -} - -fn prefix_to_range(prefix: &crate::data_model::rc::IpPrefix) -> (Vec, Vec) { - let mut min = prefix.addr.clone(); - let mut max = prefix.addr.clone(); - - let bitlen = match prefix.afi { - crate::data_model::rc::Afi::Ipv4 => 32u16, - crate::data_model::rc::Afi::Ipv6 => 128u16, - }; - let plen = prefix.prefix_len.min(bitlen); - for bit in plen..bitlen { - let byte = (bit / 8) as usize; - let offset = 7 - (bit % 8); - let mask = 1u8 << offset; - min[byte] &= !mask; - max[byte] |= mask; - } - (min, max) -} - -fn bytes_leq(a: &[u8], b: &[u8]) -> bool { - a <= b -} +include!("ca_path/types_and_validation.rs"); +include!("ca_path/certificate_checks.rs"); +include!("ca_path/resource_resolution.rs"); +include!("ca_path/ip_resources.rs"); +include!("ca_path/increment.rs"); #[cfg(test)] -mod tests { - use super::*; - use crate::data_model::common::X509NameDer; - use crate::data_model::oid::OID_CP_IPADDR_ASNUMBER; - use crate::data_model::rc::{ - Afi, AsIdOrRange, AsIdentifierChoice, AsResourceSet, IpAddressChoice, IpAddressFamily, - IpAddressOrRange, IpPrefix, IpResourceSet, - }; - use crate::data_model::rc::{ - BasicConstraintsProfile, CertificatePoliciesProfile, RcExtensions, ResourceCertKind, - ResourceCertificate, RpkixTbsCertificate, - }; - use der_parser::num_bigint::BigUint; - use std::process::Command; - fn dummy_cert( - kind: ResourceCertKind, - subject_dn: &str, - issuer_dn: &str, - ski: Option>, - aki: Option>, - aia: Option>, - crldp: Option>, - ) -> ResourceCertificate { - let aia = aia.map(|v| v.into_iter().map(|s| s.to_string()).collect::>()); - let crldp = crldp.map(|v| v.into_iter().map(|s| s.to_string()).collect::>()); - - ResourceCertificate { - raw_der: Vec::new(), - kind, - tbs: RpkixTbsCertificate { - version: 2, - serial_number: BigUint::from(1u8), - signature_algorithm: "1.2.840.113549.1.1.11".to_string(), - issuer_name: X509NameDer(issuer_dn.as_bytes().to_vec()), - subject_name: X509NameDer(subject_dn.as_bytes().to_vec()), - validity_not_before: time::OffsetDateTime::UNIX_EPOCH, - validity_not_after: time::OffsetDateTime::UNIX_EPOCH, - subject_public_key_info: Vec::new(), - extensions: RcExtensions { - basic_constraints_ca: kind == ResourceCertKind::Ca, - basic_constraints: (kind == ResourceCertKind::Ca).then_some( - BasicConstraintsProfile { - ca: true, - critical: true, - path_len_constraint: None, - }, - ), - subject_key_identifier: ski, - authority_key_identifier: aki, - crl_distribution_points_uris: crldp, - ca_issuers_uris: aia, - subject_info_access: None, - certificate_policies_oid: (kind == ResourceCertKind::Ca) - .then_some(OID_CP_IPADDR_ASNUMBER.to_string()), - certificate_policies: (kind == ResourceCertKind::Ca).then_some( - CertificatePoliciesProfile { - policy_oid: OID_CP_IPADDR_ASNUMBER.to_string(), - qualifier_oids: Vec::new(), - }, - ), - extension_oids: Vec::new(), - ip_resources: None, - as_resources: None, - }, - }, - } - } - - fn openssl_available() -> bool { - Command::new("openssl") - .arg("version") - .output() - .map(|o| o.status.success()) - .unwrap_or(false) - } - - fn write_cert_der_with_addext(dir: &std::path::Path, addext: Option<&str>) -> Vec { - assert!(openssl_available(), "openssl is required for this test"); - let key = dir.join("k.pem"); - let cert = dir.join("c.pem"); - let der = dir.join("c.der"); - - let mut cmd = Command::new("openssl"); - cmd.arg("req") - .arg("-x509") - .arg("-newkey") - .arg("rsa:2048") - .arg("-nodes") - .arg("-keyout") - .arg(&key) - .arg("-subj") - .arg("/CN=ku") - .arg("-days") - .arg("1") - .arg("-out") - .arg(&cert); - if let Some(ext) = addext { - cmd.arg("-addext").arg(ext); - } - let out = cmd.output().expect("openssl req"); - assert!( - out.status.success(), - "openssl req failed: {}", - String::from_utf8_lossy(&out.stderr) - ); - - let out = Command::new("openssl") - .arg("x509") - .arg("-in") - .arg(&cert) - .arg("-outform") - .arg("DER") - .arg("-out") - .arg(&der) - .output() - .expect("openssl x509"); - assert!( - out.status.success(), - "openssl x509 failed: {}", - String::from_utf8_lossy(&out.stderr) - ); - std::fs::read(&der).expect("read der") - } - - fn gen_issuer_and_child_der(dir: &std::path::Path) -> (Vec, Vec, Vec) { - assert!(openssl_available(), "openssl is required for this test"); - let issuer_key = dir.join("issuer.key"); - let issuer_csr = dir.join("issuer.csr"); - let issuer_pem = dir.join("issuer.pem"); - let issuer_der = dir.join("issuer.der"); - - let child_key = dir.join("child.key"); - let child_csr = dir.join("child.csr"); - let child_pem = dir.join("child.pem"); - let child_der = dir.join("child.der"); - - let other_key = dir.join("other.key"); - let other_csr = dir.join("other.csr"); - let other_pem = dir.join("other.pem"); - let other_der = dir.join("other.der"); - - let run = |cmd: &mut Command| { - let out = cmd.output().expect("run openssl"); - assert!( - out.status.success(), - "command failed: {:?}\nstderr={}", - cmd, - String::from_utf8_lossy(&out.stderr) - ); - }; - - // Issuer self-signed. - run(Command::new("openssl") - .args(["genrsa", "-out"]) - .arg(&issuer_key) - .arg("2048")); - run(Command::new("openssl") - .args(["req", "-new", "-key"]) - .arg(&issuer_key) - .args(["-subj", "/CN=issuer", "-out"]) - .arg(&issuer_csr)); - run(Command::new("openssl") - .args(["x509", "-req", "-in"]) - .arg(&issuer_csr) - .args(["-signkey"]) - .arg(&issuer_key) - .args(["-days", "1", "-out"]) - .arg(&issuer_pem)); - run(Command::new("openssl") - .args(["x509", "-in"]) - .arg(&issuer_pem) - .args(["-outform", "DER", "-out"]) - .arg(&issuer_der)); - - // Child signed by issuer. - run(Command::new("openssl") - .args(["genrsa", "-out"]) - .arg(&child_key) - .arg("2048")); - run(Command::new("openssl") - .args(["req", "-new", "-key"]) - .arg(&child_key) - .args(["-subj", "/CN=child", "-out"]) - .arg(&child_csr)); - run(Command::new("openssl") - .args(["x509", "-req", "-in"]) - .arg(&child_csr) - .args(["-CA"]) - .arg(&issuer_pem) - .args(["-CAkey"]) - .arg(&issuer_key) - .args(["-CAcreateserial", "-days", "1", "-out"]) - .arg(&child_pem)); - run(Command::new("openssl") - .args(["x509", "-in"]) - .arg(&child_pem) - .args(["-outform", "DER", "-out"]) - .arg(&child_der)); - - // Other self-signed issuer. - run(Command::new("openssl") - .args(["genrsa", "-out"]) - .arg(&other_key) - .arg("2048")); - run(Command::new("openssl") - .args(["req", "-new", "-key"]) - .arg(&other_key) - .args(["-subj", "/CN=other", "-out"]) - .arg(&other_csr)); - run(Command::new("openssl") - .args(["x509", "-req", "-in"]) - .arg(&other_csr) - .args(["-signkey"]) - .arg(&other_key) - .args(["-days", "1", "-out"]) - .arg(&other_pem)); - run(Command::new("openssl") - .args(["x509", "-in"]) - .arg(&other_pem) - .args(["-outform", "DER", "-out"]) - .arg(&other_der)); - - ( - std::fs::read(&issuer_der).expect("read issuer der"), - std::fs::read(&child_der).expect("read child der"), - std::fs::read(&other_der).expect("read other der"), - ) - } - - #[test] - fn resolve_child_ip_resources_rejects_inherit_without_parent_effective_resources() { - let child = IpResourceSet { - families: vec![IpAddressFamily { - afi: Afi::Ipv4, - choice: IpAddressChoice::Inherit, - }], - }; - let err = resolve_child_ip_resources(Some(&child), None).unwrap_err(); - assert!(matches!(err, CaPathError::InheritWithoutParentResources)); - } - - #[test] - fn resolve_child_ip_resources_rejects_non_inherit_without_parent_effective_resources() { - let child = IpResourceSet { - families: vec![IpAddressFamily { - afi: Afi::Ipv4, - choice: IpAddressChoice::AddressesOrRanges(vec![]), - }], - }; - let err = resolve_child_ip_resources(Some(&child), None).unwrap_err(); - assert!(matches!(err, CaPathError::ResourcesNotSubset)); - } - - #[test] - fn ip_resources_by_afi_items_rejects_inherit_families() { - let parent = IpResourceSet { - families: vec![IpAddressFamily { - afi: Afi::Ipv6, - choice: IpAddressChoice::Inherit, - }], - }; - let err = ip_resources_by_afi_items(&parent).unwrap_err(); - assert!(matches!(err, CaPathError::InheritWithoutParentResources)); - } - - #[test] - fn resolve_child_as_resources_rejects_inherit_without_parent_effective_resources() { - let child = AsResourceSet { - asnum: Some(AsIdentifierChoice::Inherit), - rdi: None, - }; - let err = resolve_child_as_resources(Some(&child), None).unwrap_err(); - assert!(matches!(err, CaPathError::InheritWithoutParentResources)); - } - - #[test] - fn validation_update_03_intersects_overclaiming_child_ip_resources() { - let issuer = IpResourceSet { - families: vec![IpAddressFamily { - afi: Afi::Ipv4, - choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( - IpPrefix { - afi: Afi::Ipv4, - prefix_len: 24, - addr: vec![10, 0, 0, 0], - }, - )]), - }], - }; - let child = IpResourceSet { - families: vec![IpAddressFamily { - afi: Afi::Ipv4, - choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( - IpPrefix { - afi: Afi::Ipv4, - prefix_len: 23, - addr: vec![10, 0, 0, 0], - }, - )]), - }], - }; - let index = IssuerEffectiveResourcesIndex::from_effective_resources(Some(&issuer), None) - .expect("index"); - let resolved = resolve_child_resources( - Some(&child), - Some(&issuer), - None, - None, - &index, - ResourceValidationMode::ValidationUpdate03, - ) - .expect("vrs resolution"); - assert!(resolved.warnings.ip_reduced_by_vrs); - assert!(!resolved.warnings.ip_vrs_empty); - let effective = resolved.effective_ip_resources.expect("effective ip"); - assert!(effective.families[0].contains_prefix(&IpPrefix { - afi: Afi::Ipv4, - prefix_len: 24, - addr: vec![10, 0, 0, 0], - })); - assert!(!effective.families[0].contains_prefix(&IpPrefix { - afi: Afi::Ipv4, - prefix_len: 24, - addr: vec![10, 0, 1, 0], - })); - } - - #[test] - fn validation_update_03_keeps_empty_vrs_instead_of_rejecting_child_ca() { - let issuer = IpResourceSet { - families: vec![IpAddressFamily { - afi: Afi::Ipv4, - choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( - IpPrefix { - afi: Afi::Ipv4, - prefix_len: 24, - addr: vec![10, 0, 0, 0], - }, - )]), - }], - }; - let child = IpResourceSet { - families: vec![IpAddressFamily { - afi: Afi::Ipv4, - choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( - IpPrefix { - afi: Afi::Ipv4, - prefix_len: 24, - addr: vec![192, 0, 2, 0], - }, - )]), - }], - }; - let index = IssuerEffectiveResourcesIndex::from_effective_resources(Some(&issuer), None) - .expect("index"); - let strict_err = resolve_child_resources( - Some(&child), - Some(&issuer), - None, - None, - &index, - ResourceValidationMode::Rfc6487, - ) - .unwrap_err(); - assert!(matches!(strict_err, CaPathError::ResourcesNotSubset)); - - let resolved = resolve_child_resources( - Some(&child), - Some(&issuer), - None, - None, - &index, - ResourceValidationMode::ValidationUpdate03, - ) - .expect("vrs resolution"); - assert!(resolved.warnings.ip_reduced_by_vrs); - assert!(resolved.warnings.ip_vrs_empty); - assert!( - resolved - .effective_ip_resources - .expect("effective ip") - .families - .is_empty() - ); - } - - #[test] - fn validation_update_03_intersects_overclaiming_child_as_resources() { - let issuer = AsResourceSet { - asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ - AsIdOrRange::Range { - min: 64500, - max: 64510, - }, - ])), - rdi: None, - }; - let child = AsResourceSet { - asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ - AsIdOrRange::Range { - min: 64505, - max: 64520, - }, - ])), - rdi: None, - }; - let index = IssuerEffectiveResourcesIndex::from_effective_resources(None, Some(&issuer)) - .expect("index"); - let resolved = resolve_child_resources( - None, - None, - Some(&child), - Some(&issuer), - &index, - ResourceValidationMode::ValidationUpdate03, - ) - .expect("vrs resolution"); - assert!(resolved.warnings.as_reduced_by_vrs); - assert!(!resolved.warnings.as_vrs_empty); - let effective = resolved.effective_as_resources.expect("effective as"); - assert_eq!( - effective.asnum, - Some(AsIdentifierChoice::AsIdsOrRanges(vec![ - AsIdOrRange::Range { - min: 64505, - max: 64510, - }, - ])) - ); - } - - #[test] - fn validation_update_03_records_all_resource_warning_summary_parts() { - let issuer_ip = IpResourceSet { - families: vec![IpAddressFamily { - afi: Afi::Ipv4, - choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( - IpPrefix { - afi: Afi::Ipv4, - prefix_len: 24, - addr: vec![10, 0, 0, 0], - }, - )]), - }], - }; - let child_ip = IpResourceSet { - families: vec![IpAddressFamily { - afi: Afi::Ipv4, - choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( - IpPrefix { - afi: Afi::Ipv4, - prefix_len: 24, - addr: vec![192, 0, 2, 0], - }, - )]), - }], - }; - let issuer_as = AsResourceSet { - asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ - AsIdOrRange::Range { - min: 64500, - max: 64510, - }, - ])), - rdi: None, - }; - let child_as = AsResourceSet { - asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ - AsIdOrRange::Range { - min: 64520, - max: 64530, - }, - ])), - rdi: None, - }; - let index = IssuerEffectiveResourcesIndex::from_effective_resources( - Some(&issuer_ip), - Some(&issuer_as), - ) - .expect("index"); - - let resolved = resolve_child_resources( - Some(&child_ip), - Some(&issuer_ip), - Some(&child_as), - Some(&issuer_as), - &index, - ResourceValidationMode::ValidationUpdate03, - ) - .expect("vrs resolution"); - - assert!(resolved.warnings.ip_reduced_by_vrs); - assert!(resolved.warnings.ip_vrs_empty); - assert!(resolved.warnings.as_reduced_by_vrs); - assert!(resolved.warnings.as_vrs_empty); - assert_eq!( - resolved.warnings.summary(), - "ip_reduced_by_vrs,as_reduced_by_vrs,ip_vrs_empty,as_vrs_empty" - ); - assert!(!resolved.warnings.is_empty()); - } - - #[test] - fn child_aki_mismatch_is_rejected() { - let issuer = dummy_cert( - ResourceCertKind::Ca, - "CN=issuer", - "CN=issuer", - Some(vec![1]), - None, - None, - None, - ); - let child = dummy_cert( - ResourceCertKind::Ca, - "CN=child", - "CN=issuer", - Some(vec![2]), - Some(vec![9]), - Some(vec!["rsync://example.test/issuer.cer"]), - Some(vec!["rsync://example.test/issuer.crl"]), - ); - let err = validate_child_aki_matches_issuer_ski(&child, &issuer).unwrap_err(); - assert!(matches!(err, CaPathError::ChildAkiMismatch), "{err}"); - } - - #[test] - fn child_aia_missing_is_rejected() { - let _issuer = dummy_cert( - ResourceCertKind::Ca, - "CN=issuer", - "CN=issuer", - Some(vec![1]), - None, - None, - None, - ); - let child = dummy_cert( - ResourceCertKind::Ca, - "CN=child", - "CN=issuer", - Some(vec![2]), - Some(vec![1]), - None, - Some(vec!["rsync://example.test/issuer.crl"]), - ); - let err = - validate_child_aia_points_to_issuer_uri(&child, "rsync://example.test/issuer.cer") - .unwrap_err(); - assert!(matches!(err, CaPathError::ChildAiaMissing), "{err}"); - - // Also cover issuer ski missing. - let issuer_missing_ski = dummy_cert( - ResourceCertKind::Ca, - "CN=issuer", - "CN=issuer", - None, - None, - None, - None, - ); - let err = validate_child_aki_matches_issuer_ski(&child, &issuer_missing_ski).unwrap_err(); - assert!(matches!(err, CaPathError::IssuerSkiMissing), "{err}"); - } - - #[test] - fn child_aia_issuer_uri_mismatch_is_rejected() { - let _issuer = dummy_cert( - ResourceCertKind::Ca, - "CN=issuer", - "CN=issuer", - Some(vec![1]), - None, - None, - None, - ); - let child = dummy_cert( - ResourceCertKind::Ca, - "CN=child", - "CN=issuer", - Some(vec![2]), - Some(vec![1]), - Some(vec!["rsync://example.test/other.cer"]), - Some(vec!["rsync://example.test/issuer.crl"]), - ); - let err = - validate_child_aia_points_to_issuer_uri(&child, "rsync://example.test/issuer.cer") - .unwrap_err(); - assert!( - matches!(err, CaPathError::ChildAiaIssuerUriMismatch), - "{err}" - ); - } - - #[test] - fn child_crldp_mismatch_is_rejected() { - let issuer = dummy_cert( - ResourceCertKind::Ca, - "CN=issuer", - "CN=issuer", - Some(vec![1]), - None, - None, - None, - ); - let child = dummy_cert( - ResourceCertKind::Ca, - "CN=child", - "CN=issuer", - Some(vec![2]), - Some(vec![1]), - Some(vec!["rsync://example.test/issuer.cer"]), - None, - ); - let err = - validate_child_crldp_contains_issuer_crl_uri(&child, "rsync://example.test/issuer.crl") - .unwrap_err(); - assert!(matches!(err, CaPathError::ChildCrlDpMissing), "{err}"); - - let child = dummy_cert( - ResourceCertKind::Ca, - "CN=child", - "CN=issuer", - Some(vec![2]), - Some(vec![1]), - Some(vec!["rsync://example.test/issuer.cer"]), - Some(vec!["rsync://example.test/other.crl"]), - ); - let err = - validate_child_crldp_contains_issuer_crl_uri(&child, "rsync://example.test/issuer.crl") - .unwrap_err(); - assert!(matches!(err, CaPathError::ChildCrlDpUriMismatch), "{err}"); - - // Cover child AKI missing. - let child_missing_aki = dummy_cert( - ResourceCertKind::Ca, - "CN=child", - "CN=issuer", - Some(vec![2]), - None, - Some(vec!["rsync://example.test/issuer.cer"]), - Some(vec!["rsync://example.test/issuer.crl"]), - ); - let err = validate_child_aki_matches_issuer_ski(&child_missing_aki, &issuer).unwrap_err(); - assert!(matches!(err, CaPathError::ChildAkiMissing), "{err}"); - } - - #[test] - fn child_binding_checks_accept_when_matching() { - let issuer = dummy_cert( - ResourceCertKind::Ca, - "CN=issuer", - "CN=issuer", - Some(vec![1]), - None, - None, - None, - ); - let child = dummy_cert( - ResourceCertKind::Ca, - "CN=child", - "CN=issuer", - Some(vec![2]), - Some(vec![1]), - Some(vec!["rsync://example.test/issuer.cer"]), - Some(vec!["rsync://example.test/issuer.crl"]), - ); - validate_child_aki_matches_issuer_ski(&child, &issuer).expect("aki ok"); - validate_child_aia_points_to_issuer_uri(&child, "rsync://example.test/issuer.cer") - .expect("aia ok"); - validate_child_crldp_contains_issuer_crl_uri(&child, "rsync://example.test/issuer.crl") - .expect("crldp ok"); - } - - #[test] - fn validate_child_ca_key_usage_accepts_only_keycertsign_and_crlsign_critical() { - let td = tempfile::tempdir().expect("tempdir"); - let der = write_cert_der_with_addext( - td.path(), - Some("keyUsage = critical, keyCertSign, cRLSign"), - ); - let cert = parse_x509_cert(&der).expect("x509 parse ok"); - validate_child_ca_key_usage(&cert).expect("key usage ok"); - } - - #[test] - fn validate_child_ca_key_usage_rejects_missing_noncritical_and_invalid_bits() { - let td = tempfile::tempdir().expect("tempdir"); - let missing = write_cert_der_with_addext(td.path(), None); - let cert = parse_x509_cert(&missing).expect("x509 parse ok"); - let err = validate_child_ca_key_usage(&cert).unwrap_err(); - assert!(matches!(err, CaPathError::KeyUsageMissing), "{err}"); - - let td = tempfile::tempdir().expect("tempdir"); - let noncritical = - write_cert_der_with_addext(td.path(), Some("keyUsage = keyCertSign, cRLSign")); - let cert = parse_x509_cert(&noncritical).expect("x509 parse ok"); - let err = validate_child_ca_key_usage(&cert).unwrap_err(); - assert!(matches!(err, CaPathError::KeyUsageNotCritical), "{err}"); - - let td = tempfile::tempdir().expect("tempdir"); - let invalid = write_cert_der_with_addext( - td.path(), - Some("keyUsage = critical, keyCertSign, cRLSign, digitalSignature"), - ); - let cert = parse_x509_cert(&invalid).expect("x509 parse ok"); - let err = validate_child_ca_key_usage(&cert).unwrap_err(); - assert!(matches!(err, CaPathError::KeyUsageInvalidBits), "{err}"); - } - - #[test] - fn verify_cert_signature_with_issuer_accepts_valid_chain_and_rejects_wrong_issuer() { - let td = tempfile::tempdir().expect("tempdir"); - let (issuer, child, other) = gen_issuer_and_child_der(td.path()); - let issuer_cert = parse_x509_cert(&issuer).expect("x509 parse issuer"); - let child_cert = parse_x509_cert(&child).expect("x509 parse child"); - verify_child_signature(&child_cert, &issuer_cert.tbs_certificate.subject_pki) - .expect("signature ok"); - let other_cert = parse_x509_cert(&other).expect("x509 parse other"); - let err = verify_child_signature(&child_cert, &other_cert.tbs_certificate.subject_pki) - .unwrap_err(); - assert!( - matches!(err, CaPathError::ChildSignatureInvalid(_)), - "{err}" - ); - } - - #[test] - fn signature_cache_stores_real_verify_success_and_skips_on_hit() { - let _guard = crate::crypto_sig_cache::GLOBAL_TEST_LOCK - .lock() - .expect("test lock"); - crate::crypto_sig_cache::clear_global(); - let td = tempfile::tempdir().expect("tempdir"); - let (issuer, child, other) = gen_issuer_and_child_der(td.path()); - let issuer_cert = parse_x509_cert(&issuer).expect("x509 parse issuer"); - let child_cert = parse_x509_cert(&child).expect("x509 parse child"); - let other_cert = parse_x509_cert(&other).expect("x509 parse other"); - - let cache_dir = tempfile::tempdir().expect("tempdir"); - let cache = std::sync::Arc::new(crate::crypto_sig_cache::CryptoSigCache::load_or_rebuild( - cache_dir.path().join("work-db.crypto-sig-cache"), - true, - )); - crate::crypto_sig_cache::install_global(std::sync::Arc::clone(&cache)); - - // Miss -> real verification runs and passes -> positive conclusion written. - verify_child_signature(&child_cert, &issuer_cert.tbs_certificate.subject_pki) - .expect("signature ok"); - // Hit -> skipped (a second execution would also pass, so assert via stats). - verify_child_signature(&child_cert, &issuer_cert.tbs_certificate.subject_pki) - .expect("signature ok"); - // Wrong issuer: real verification fails, nothing is written (no negative caching), - // so a retry executes the real verification again instead of inheriting anything. - for _ in 0..2 { - let err = verify_child_signature(&child_cert, &other_cert.tbs_certificate.subject_pki) - .unwrap_err(); - assert!( - matches!(err, CaPathError::ChildSignatureInvalid(_)), - "{err}" - ); - } - - let summary = cache.summary(); - let stats = summary - .per_point - .get("child_ca_cert") - .expect("child_ca_cert stats"); - assert_eq!(stats.calls, 4); - assert_eq!(stats.would_hit, 1); - assert_eq!(stats.new_keys, 3); - assert_eq!(stats.verify_executed, 3); - assert_eq!(stats.verify_skipped, 1); - crate::crypto_sig_cache::clear_global(); - } - - #[test] - fn issuer_effective_resources_index_and_indexed_resolvers_cover_success_and_failure_paths() { - use crate::data_model::rc::{AsIdOrRange, IpPrefix}; - - let parent_ip = IpResourceSet { - families: vec![IpAddressFamily { - afi: Afi::Ipv4, - choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( - IpPrefix { - afi: Afi::Ipv4, - prefix_len: 8, - addr: vec![10, 0, 0, 0], - }, - )]), - }], - }; - let parent_as = AsResourceSet { - asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ - AsIdOrRange::Range { - min: 64500, - max: 64599, - }, - ])), - rdi: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id( - 65000, - )])), - }; - let idx = IssuerEffectiveResourcesIndex::from_effective_resources( - Some(&parent_ip), - Some(&parent_as), - ) - .expect("index builds"); - assert_eq!( - idx.parent_ip_by_afi_items.as_ref().map(|v| v.len()), - Some(1) - ); - assert_eq!(idx.parent_ip_merged_intervals.len(), 1); - assert_eq!( - idx.parent_asnum_intervals.as_ref().map(|v| v.len()), - Some(1) - ); - assert_eq!(idx.parent_rdi_intervals.as_ref().map(|v| v.len()), Some(1)); - - let child_ip_subset = IpResourceSet { - families: vec![IpAddressFamily { - afi: Afi::Ipv4, - choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( - IpPrefix { - afi: Afi::Ipv4, - prefix_len: 16, - addr: vec![10, 1, 0, 0], - }, - )]), - }], - }; - assert!( - resolve_child_ip_resources_indexed( - Some(&child_ip_subset), - Some(&parent_ip), - idx.parent_ip_by_afi_items.as_ref(), - &idx.parent_ip_merged_intervals, - ) - .expect("subset should resolve") - .is_some() - ); - - let child_ip_bad = IpResourceSet { - families: vec![IpAddressFamily { - afi: Afi::Ipv4, - choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( - IpPrefix { - afi: Afi::Ipv4, - prefix_len: 16, - addr: vec![11, 0, 0, 0], - }, - )]), - }], - }; - let err = resolve_child_ip_resources_indexed( - Some(&child_ip_bad), - Some(&parent_ip), - idx.parent_ip_by_afi_items.as_ref(), - &idx.parent_ip_merged_intervals, - ) - .unwrap_err(); - assert!(matches!(err, CaPathError::ResourcesNotSubset)); - - let child_as_subset = AsResourceSet { - asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id( - 64542, - )])), - rdi: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id( - 65000, - )])), - }; - assert!( - resolve_child_as_resources_indexed( - Some(&child_as_subset), - Some(&parent_as), - idx.parent_asnum_intervals.as_deref(), - idx.parent_rdi_intervals.as_deref(), - ) - .expect("subset as resolves") - .is_some() - ); - - let child_as_bad = AsResourceSet { - asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id( - 65123, - )])), - rdi: None, - }; - let err = resolve_child_as_resources_indexed( - Some(&child_as_bad), - Some(&parent_as), - idx.parent_asnum_intervals.as_deref(), - idx.parent_rdi_intervals.as_deref(), - ) - .unwrap_err(); - assert!(matches!(err, CaPathError::ResourcesNotSubset)); - } - - #[test] - fn resolve_child_ip_and_as_resources_success_paths() { - use crate::data_model::rc::{AsIdOrRange, IpAddressOrRange, IpPrefix}; - - let parent_ip = IpResourceSet { - families: vec![IpAddressFamily { - afi: Afi::Ipv4, - choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( - IpPrefix { - afi: Afi::Ipv4, - prefix_len: 8, - addr: vec![10, 0, 0, 0], - }, - )]), - }], - }; - - let child_ip_inherit = IpResourceSet { - families: vec![IpAddressFamily { - afi: Afi::Ipv4, - choice: IpAddressChoice::Inherit, - }], - }; - let eff = resolve_child_ip_resources(Some(&child_ip_inherit), Some(&parent_ip)) - .expect("inherit resolves") - .expect("some ip"); - assert_eq!(eff.families.len(), 1); - assert!(matches!( - eff.families[0].choice, - IpAddressChoice::AddressesOrRanges(_) - )); - - let child_ip_subset = IpResourceSet { - families: vec![IpAddressFamily { - afi: Afi::Ipv4, - choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( - IpPrefix { - afi: Afi::Ipv4, - prefix_len: 16, - addr: vec![10, 1, 0, 0], - }, - )]), - }], - }; - resolve_child_ip_resources(Some(&child_ip_subset), Some(&parent_ip)) - .expect("subset ok") - .expect("some"); - - let child_ip_bad = IpResourceSet { - families: vec![IpAddressFamily { - afi: Afi::Ipv4, - choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( - IpPrefix { - afi: Afi::Ipv4, - prefix_len: 16, - addr: vec![11, 0, 0, 0], - }, - )]), - }], - }; - let err = resolve_child_ip_resources(Some(&child_ip_bad), Some(&parent_ip)).unwrap_err(); - assert!(matches!(err, CaPathError::ResourcesNotSubset), "{err}"); - - let parent_as = AsResourceSet { - asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ - AsIdOrRange::Range { min: 1, max: 100 }, - ])), - rdi: None, - }; - let child_as_inherit = AsResourceSet { - asnum: Some(AsIdentifierChoice::Inherit), - rdi: None, - }; - let eff_as = resolve_child_as_resources(Some(&child_as_inherit), Some(&parent_as)) - .expect("inherit as") - .expect("some"); - assert_eq!(eff_as.asnum, parent_as.asnum); - - let child_as_subset = AsResourceSet { - asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id(50)])), - rdi: None, - }; - resolve_child_as_resources(Some(&child_as_subset), Some(&parent_as)) - .expect("subset as") - .expect("some"); - - let child_as_bad = AsResourceSet { - asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id( - 200, - )])), - rdi: None, - }; - let err = resolve_child_as_resources(Some(&child_as_bad), Some(&parent_as)).unwrap_err(); - assert!(matches!(err, CaPathError::ResourcesNotSubset), "{err}"); - } -} - -fn increment_bytes(v: &[u8]) -> Vec { - let mut out = v.to_vec(); - for i in (0..out.len()).rev() { - if out[i] != 0xFF { - out[i] += 1; - for j in i + 1..out.len() { - out[j] = 0; - } - return out; - } - } - vec![0u8; out.len()] -} +#[path = "ca_path/tests.rs"] +mod tests; diff --git a/crates/panda-rpki-validator/src/validation/ca_path/certificate_checks.rs b/crates/panda-rpki-validator/src/validation/ca_path/certificate_checks.rs new file mode 100644 index 0000000..d6ce206 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/ca_path/certificate_checks.rs @@ -0,0 +1,145 @@ +// Certificate profile, signature, CRL, and validity checks. + +fn parse_subject_pki_from_der(der: &[u8]) -> Result, CaPathError> { + let (rem, spki) = SubjectPublicKeyInfo::from_der(der) + .map_err(|e| CaPathError::IssuerSpkiParse(e.to_string()))?; + if !rem.is_empty() { + return Err(CaPathError::IssuerSpkiTrailingBytes(rem.len())); + } + Ok(spki) +} + +fn parse_x509_cert(der: &[u8]) -> Result, CaPathError> { + let (rem, cert) = X509Certificate::from_der(der) + .map_err(|e| CaPathError::ChildSignatureInvalid(e.to_string()))?; + if !rem.is_empty() { + return Err(CaPathError::ChildSignatureInvalid( + "trailing bytes after child certificate".to_string(), + )); + } + Ok(cert) +} + +fn verify_child_signature( + child: &X509Certificate<'_>, + issuer_spki: &SubjectPublicKeyInfo<'_>, +) -> Result<(), CaPathError> { + crate::crypto_sig_cache::verify_with_cache( + crate::crypto_sig_cache::CryptoSigVerifyPoint::ChildCaCert, + child.tbs_certificate.as_ref(), + child.signature_value.data.as_ref(), + issuer_spki.raw, + || { + child + .verify_signature(Some(issuer_spki)) + .map_err(|e| CaPathError::ChildSignatureInvalid(e.to_string())) + }, + ) +} + +fn validate_child_aki_matches_issuer_ski( + child: &ResourceCertificate, + issuer: &ResourceCertificate, +) -> Result<(), CaPathError> { + let Some(issuer_ski) = issuer.tbs.extensions.subject_key_identifier.as_deref() else { + return Err(CaPathError::IssuerSkiMissing); + }; + let Some(child_aki) = child.tbs.extensions.authority_key_identifier.as_deref() else { + return Err(CaPathError::ChildAkiMissing); + }; + if child_aki != issuer_ski { + return Err(CaPathError::ChildAkiMismatch); + } + Ok(()) +} + +fn validate_child_aia_points_to_issuer_uri( + child: &ResourceCertificate, + issuer_ca_rsync_uri: &str, +) -> Result<(), CaPathError> { + let Some(uris) = child.tbs.extensions.ca_issuers_uris.as_ref() else { + return Err(CaPathError::ChildAiaMissing); + }; + if !uris.iter().any(|u| u.as_str() == issuer_ca_rsync_uri) { + return Err(CaPathError::ChildAiaIssuerUriMismatch); + } + Ok(()) +} + +fn validate_child_crldp_contains_issuer_crl_uri( + child: &ResourceCertificate, + issuer_crl_rsync_uri: &str, +) -> Result<(), CaPathError> { + let Some(uris) = child.tbs.extensions.crl_distribution_points_uris.as_ref() else { + return Err(CaPathError::ChildCrlDpMissing); + }; + if !uris.iter().any(|u| u.as_str() == issuer_crl_rsync_uri) { + return Err(CaPathError::ChildCrlDpUriMismatch); + } + Ok(()) +} + +fn validate_child_ca_key_usage(cert: &X509Certificate<'_>) -> Result<(), CaPathError> { + let mut ku_critical: Option = None; + for ext in cert.extensions() { + if ext.oid.as_bytes() == OID_KEY_USAGE_RAW { + ku_critical = Some(ext.critical); + break; + } + } + + let Some(critical) = ku_critical else { + return Err(CaPathError::KeyUsageMissing); + }; + if !critical { + return Err(CaPathError::KeyUsageNotCritical); + } + + let Some(ku) = cert + .key_usage() + .map_err(|e| CaPathError::ChildSignatureInvalid(e.to_string()))? + else { + return Err(CaPathError::KeyUsageMissing); + }; + + let v = &ku.value; + let ok = v.key_cert_sign() + && v.crl_sign() + && !v.digital_signature() + && !v.non_repudiation() + && !v.key_encipherment() + && !v.data_encipherment() + && !v.key_agreement() + && !v.encipher_only() + && !v.decipher_only(); + if !ok { + return Err(CaPathError::KeyUsageInvalidBits); + } + + Ok(()) +} + +fn time_within_validity( + t: time::OffsetDateTime, + not_before: time::OffsetDateTime, + not_after: time::OffsetDateTime, +) -> bool { + let t = t.to_offset(time::UtcOffset::UTC); + let not_before = not_before.to_offset(time::UtcOffset::UTC); + let not_after = not_after.to_offset(time::UtcOffset::UTC); + t >= not_before && t <= not_after +} + +fn crl_valid_at_time(crl: &RpkixCrl, t: time::OffsetDateTime) -> bool { + let t = t.to_offset(time::UtcOffset::UTC); + let this_update = crl.this_update.utc.to_offset(time::UtcOffset::UTC); + let next_update = crl.next_update.utc.to_offset(time::UtcOffset::UTC); + t >= this_update && t < next_update +} + +fn is_serial_revoked_by_crl(cert: &ResourceCertificate, crl: &RpkixCrl) -> bool { + let serial = BigUnsigned::from_biguint(&cert.tbs.serial_number); + crl.revoked_certs + .iter() + .any(|rc| rc.serial_number == serial) +} diff --git a/crates/panda-rpki-validator/src/validation/ca_path/increment.rs b/crates/panda-rpki-validator/src/validation/ca_path/increment.rs new file mode 100644 index 0000000..9605626 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/ca_path/increment.rs @@ -0,0 +1,15 @@ +// Small byte-vector helper used by resource interval tests. + +fn increment_bytes(v: &[u8]) -> Vec { + let mut out = v.to_vec(); + for i in (0..out.len()).rev() { + if out[i] != 0xFF { + out[i] += 1; + for j in i + 1..out.len() { + out[j] = 0; + } + return out; + } + } + vec![0u8; out.len()] +} diff --git a/crates/panda-rpki-validator/src/validation/ca_path/ip_resources.rs b/crates/panda-rpki-validator/src/validation/ca_path/ip_resources.rs new file mode 100644 index 0000000..fb998e9 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/ca_path/ip_resources.rs @@ -0,0 +1,350 @@ +// IP interval normalization, intersection, and coverage helpers. + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +enum AfiKey { + V4, + V6, +} + +fn ip_resources_by_afi_items( + set: &IpResourceSet, +) -> Result< + std::collections::BTreeMap< + crate::data_model::rc::Afi, + Vec, + >, + CaPathError, +> { + let mut m: std::collections::BTreeMap< + crate::data_model::rc::Afi, + Vec, + > = std::collections::BTreeMap::new(); + for fam in &set.families { + match &fam.choice { + IpAddressChoice::Inherit => return Err(CaPathError::InheritWithoutParentResources), + IpAddressChoice::AddressesOrRanges(items) => { + m.insert(fam.afi, items.clone()); + } + } + } + Ok(m) +} + +fn ip_resources_single_afi( + parent: &IpResourceSet, + afi: crate::data_model::rc::Afi, + items: Option<&Vec>, +) -> IpResourceSet { + let mut families = Vec::new(); + if let Some(items) = items { + families.push(crate::data_model::rc::IpAddressFamily { + afi, + choice: IpAddressChoice::AddressesOrRanges(items.clone()), + }); + } else { + // If parent doesn't mention this AFI explicitly, treat as empty. + // The subset check will fail. + let _ = parent; + } + IpResourceSet { families } +} + +fn ip_family_items_subset( + child_items: &[crate::data_model::rc::IpAddressOrRange], + parent_set: &IpResourceSet, +) -> bool { + let parent_by_afi = ip_resources_to_merged_intervals(parent_set); + // parent_set should contain exactly one AFI. + let (afi_key, parent_intervals) = match parent_by_afi.into_iter().next() { + None => return false, + Some(v) => v, + }; + + let mut child_intervals: Vec<(Vec, Vec)> = Vec::new(); + for item in child_items { + match item { + crate::data_model::rc::IpAddressOrRange::Prefix(p) => { + child_intervals.push(prefix_to_range(p)) + } + crate::data_model::rc::IpAddressOrRange::Range(r) => { + child_intervals.push((r.min.clone(), r.max.clone())) + } + } + } + child_intervals.sort_by(|(a, _), (b, _)| a.cmp(b)); + let child_intervals = merge_ip_intervals(&child_intervals); + + let _ = afi_key; + for (cmin, cmax) in &child_intervals { + if !interval_is_covered(&parent_intervals, cmin, cmax) { + return false; + } + } + true +} + +fn ip_resources_to_merged_intervals( + set: &IpResourceSet, +) -> std::collections::HashMap, Vec)>> { + let m = ip_resources_to_merged_intervals_by_afi(set); + m.into_iter() + .map(|(afi, ranges)| { + ( + match afi { + crate::data_model::rc::Afi::Ipv4 => AfiKey::V4, + crate::data_model::rc::Afi::Ipv6 => AfiKey::V6, + }, + ranges, + ) + }) + .collect() +} + +fn ip_resources_to_merged_intervals_by_afi( + set: &IpResourceSet, +) -> HashMap, Vec)>> { + let mut m: HashMap, Vec)>> = HashMap::new(); + + for fam in &set.families { + match &fam.choice { + IpAddressChoice::Inherit => { + // When used in subset checks, treat inherit as "all" by leaving it absent. + // Resolution should have happened earlier. + } + IpAddressChoice::AddressesOrRanges(items) => { + let ent = m.entry(fam.afi).or_default(); + for item in items { + match item { + crate::data_model::rc::IpAddressOrRange::Prefix(p) => { + let (min, max) = prefix_to_range(p); + ent.push((min, max)); + } + crate::data_model::rc::IpAddressOrRange::Range(r) => { + ent.push((r.min.clone(), r.max.clone())); + } + } + } + } + } + } + + for (_afi, v) in m.iter_mut() { + v.sort_by(|(a, _), (b, _)| a.cmp(b)); + *v = merge_ip_intervals(v); + } + + m +} + +fn ip_family_items_subset_with_parent_intervals( + child_items: &[crate::data_model::rc::IpAddressOrRange], + parent_intervals: &[(Vec, Vec)], +) -> bool { + if parent_intervals.is_empty() { + return false; + } + + let mut child_intervals: Vec<(Vec, Vec)> = Vec::new(); + for item in child_items { + match item { + crate::data_model::rc::IpAddressOrRange::Prefix(p) => { + child_intervals.push(prefix_to_range(p)) + } + crate::data_model::rc::IpAddressOrRange::Range(r) => { + child_intervals.push((r.min.clone(), r.max.clone())) + } + } + } + child_intervals.sort_by(|(a, _), (b, _)| a.cmp(b)); + let child_intervals = merge_ip_intervals(&child_intervals); + + for (cmin, cmax) in &child_intervals { + if !interval_is_covered(parent_intervals, cmin, cmax) { + return false; + } + } + true +} + +fn ip_items_to_merged_intervals( + items: &[crate::data_model::rc::IpAddressOrRange], +) -> Vec<(Vec, Vec)> { + let mut intervals = Vec::new(); + for item in items { + match item { + crate::data_model::rc::IpAddressOrRange::Prefix(p) => { + intervals.push(prefix_to_range(p)) + } + crate::data_model::rc::IpAddressOrRange::Range(r) => { + intervals.push((r.min.clone(), r.max.clone())) + } + } + } + intervals.sort_by(|(a, _), (b, _)| a.cmp(b)); + merge_ip_intervals(&intervals) +} + +fn intersect_ip_items_with_parent_intervals( + items: &[crate::data_model::rc::IpAddressOrRange], + parent_intervals: &[(Vec, Vec)], +) -> Vec<(Vec, Vec)> { + let child_intervals = ip_items_to_merged_intervals(items); + intersect_ip_intervals(&child_intervals, parent_intervals) +} + +fn intersect_ip_intervals( + child_intervals: &[(Vec, Vec)], + parent_intervals: &[(Vec, Vec)], +) -> Vec<(Vec, Vec)> { + let mut out = Vec::new(); + let mut parent_index = 0usize; + for (child_min, child_max) in child_intervals { + while parent_index < parent_intervals.len() + && parent_intervals[parent_index].1.as_slice() < child_min.as_slice() + { + parent_index += 1; + } + let mut scan = parent_index; + while scan < parent_intervals.len() + && parent_intervals[scan].0.as_slice() <= child_max.as_slice() + { + let (parent_min, parent_max) = &parent_intervals[scan]; + let min = if bytes_leq(child_min, parent_min) { + parent_min.clone() + } else { + child_min.clone() + }; + let max = if bytes_leq(child_max, parent_max) { + child_max.clone() + } else { + parent_max.clone() + }; + if bytes_leq(&min, &max) { + out.push((min, max)); + } + scan += 1; + } + } + merge_ip_intervals(&out) +} + +fn ip_intervals_to_ranges( + afi: crate::data_model::rc::Afi, + intervals: &[(Vec, Vec)], +) -> Vec { + intervals + .iter() + .map(|(min, max)| { + crate::data_model::rc::IpAddressOrRange::Range(crate::data_model::rc::IpAddressRange { + min: normalize_ip_bytes(afi, min), + max: normalize_ip_bytes(afi, max), + }) + }) + .collect() +} + +fn normalize_ip_bytes(afi: crate::data_model::rc::Afi, bytes: &[u8]) -> Vec { + let target_len = afi.octets_len(); + if bytes.len() == target_len { + return bytes.to_vec(); + } + let mut out = vec![0u8; target_len]; + let copy_len = bytes.len().min(target_len); + out[..copy_len].copy_from_slice(&bytes[..copy_len]); + out +} + +fn intervals_changed(left: &[T], right: &[T]) -> bool { + left != right +} + +fn intersect_as_intervals(child: &[(u32, u32)], parent: &[(u32, u32)]) -> Vec<(u32, u32)> { + let mut out = Vec::new(); + let mut parent_index = 0usize; + for (child_min, child_max) in child { + while parent_index < parent.len() && parent[parent_index].1 < *child_min { + parent_index += 1; + } + let mut scan = parent_index; + while scan < parent.len() && parent[scan].0 <= *child_max { + let min = (*child_min).max(parent[scan].0); + let max = (*child_max).min(parent[scan].1); + if min <= max { + out.push((min, max)); + } + scan += 1; + } + } + merge_as_intervals(&out) +} + +fn as_intervals_to_items(intervals: &[(u32, u32)]) -> Vec { + intervals + .iter() + .map(|(min, max)| { + if min == max { + crate::data_model::rc::AsIdOrRange::Id(*min) + } else { + crate::data_model::rc::AsIdOrRange::Range { + min: *min, + max: *max, + } + } + }) + .collect() +} + +fn merge_ip_intervals(v: &[(Vec, Vec)]) -> Vec<(Vec, Vec)> { + let mut out: Vec<(Vec, Vec)> = Vec::new(); + for (min, max) in v { + let Some(last) = out.last_mut() else { + out.push((min.clone(), max.clone())); + continue; + }; + + if bytes_leq(min, &increment_bytes(&last.1)) { + if bytes_leq(&last.1, max) { + last.1 = max.clone(); + } + continue; + } + + out.push((min.clone(), max.clone())); + } + out +} + +fn interval_is_covered(parent: &[(Vec, Vec)], min: &[u8], max: &[u8]) -> bool { + for (pmin, pmax) in parent { + if bytes_leq(pmin, min) && bytes_leq(max, pmax) { + return true; + } + if pmin.as_slice() > min { + break; + } + } + false +} + +fn prefix_to_range(prefix: &crate::data_model::rc::IpPrefix) -> (Vec, Vec) { + let mut min = prefix.addr.clone(); + let mut max = prefix.addr.clone(); + + let bitlen = match prefix.afi { + crate::data_model::rc::Afi::Ipv4 => 32u16, + crate::data_model::rc::Afi::Ipv6 => 128u16, + }; + let plen = prefix.prefix_len.min(bitlen); + for bit in plen..bitlen { + let byte = (bit / 8) as usize; + let offset = 7 - (bit % 8); + let mask = 1u8 << offset; + min[byte] &= !mask; + max[byte] |= mask; + } + (min, max) +} + +fn bytes_leq(a: &[u8], b: &[u8]) -> bool { + a <= b +} diff --git a/crates/panda-rpki-validator/src/validation/ca_path/resource_resolution.rs b/crates/panda-rpki-validator/src/validation/ca_path/resource_resolution.rs new file mode 100644 index 0000000..d4a710a --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/ca_path/resource_resolution.rs @@ -0,0 +1,445 @@ +// Effective IP/AS resource resolution and interval helpers. + +#[derive(Clone, Debug)] +struct ResourceResolution { + effective_ip_resources: Option, + effective_as_resources: Option, + warnings: ResourceValidationWarnings, +} + +fn resolve_child_resources( + child_ip: Option<&IpResourceSet>, + issuer_effective_ip: Option<&IpResourceSet>, + child_as: Option<&AsResourceSet>, + issuer_effective_as: Option<&AsResourceSet>, + issuer_resources_index: &IssuerEffectiveResourcesIndex, + mode: ResourceValidationMode, +) -> Result { + match mode { + ResourceValidationMode::Rfc6487 => Ok(ResourceResolution { + effective_ip_resources: resolve_child_ip_resources_indexed( + child_ip, + issuer_effective_ip, + issuer_resources_index.parent_ip_by_afi_items.as_ref(), + &issuer_resources_index.parent_ip_merged_intervals, + )?, + effective_as_resources: resolve_child_as_resources_indexed( + child_as, + issuer_effective_as, + issuer_resources_index.parent_asnum_intervals.as_deref(), + issuer_resources_index.parent_rdi_intervals.as_deref(), + )?, + warnings: ResourceValidationWarnings::default(), + }), + ResourceValidationMode::ValidationUpdate03 => { + let (effective_ip_resources, ip_reduced_by_vrs, ip_vrs_empty) = + resolve_child_ip_resources_vrs( + child_ip, + issuer_effective_ip, + issuer_resources_index.parent_ip_by_afi_items.as_ref(), + &issuer_resources_index.parent_ip_merged_intervals, + )?; + let (effective_as_resources, as_reduced_by_vrs, as_vrs_empty) = + resolve_child_as_resources_vrs( + child_as, + issuer_effective_as, + issuer_resources_index.parent_asnum_intervals.as_deref(), + issuer_resources_index.parent_rdi_intervals.as_deref(), + )?; + Ok(ResourceResolution { + effective_ip_resources, + effective_as_resources, + warnings: ResourceValidationWarnings { + ip_reduced_by_vrs, + as_reduced_by_vrs, + ip_vrs_empty, + as_vrs_empty, + }, + }) + } + } +} + +fn resolve_child_ip_resources( + child_ip: Option<&IpResourceSet>, + issuer_effective: Option<&IpResourceSet>, +) -> Result, CaPathError> { + let precomputed_parent_by_afi = issuer_effective + .map(ip_resources_by_afi_items) + .transpose()?; + let precomputed_parent_intervals = issuer_effective + .map(ip_resources_to_merged_intervals_by_afi) + .unwrap_or_default(); + resolve_child_ip_resources_indexed( + child_ip, + issuer_effective, + precomputed_parent_by_afi.as_ref(), + &precomputed_parent_intervals, + ) +} + +fn resolve_child_ip_resources_indexed( + child_ip: Option<&IpResourceSet>, + issuer_effective: Option<&IpResourceSet>, + parent_by_afi: Option< + &BTreeMap>, + >, + parent_intervals_by_afi: &HashMap, Vec)>>, +) -> Result, CaPathError> { + let Some(child_ip) = child_ip else { + return Ok(None); + }; + + let Some(_parent) = issuer_effective else { + if child_ip.has_any_inherit() { + return Err(CaPathError::InheritWithoutParentResources); + } + // With no parent effective resources, we cannot validate subset. + return Err(CaPathError::ResourcesNotSubset); + }; + + // Resolve per-AFI inherit, producing an effective set with no inherit. + let parent_by_afi = parent_by_afi.ok_or(CaPathError::InheritWithoutParentResources)?; + let mut out_families: Vec = Vec::new(); + + for fam in &child_ip.families { + match &fam.choice { + IpAddressChoice::Inherit => { + let items = parent_by_afi + .get(&fam.afi) + .ok_or(CaPathError::InheritWithoutParentResources)?; + out_families.push(crate::data_model::rc::IpAddressFamily { + afi: fam.afi, + choice: IpAddressChoice::AddressesOrRanges(items.clone()), + }); + } + IpAddressChoice::AddressesOrRanges(items) => { + // Subset check against parent union for that AFI. + let parent_intervals = parent_intervals_by_afi + .get(&fam.afi) + .map(Vec::as_slice) + .unwrap_or(&[]); + if !ip_family_items_subset_with_parent_intervals(items, parent_intervals) { + return Err(CaPathError::ResourcesNotSubset); + } + out_families.push(crate::data_model::rc::IpAddressFamily { + afi: fam.afi, + choice: IpAddressChoice::AddressesOrRanges(items.clone()), + }); + } + } + } + + Ok(Some(IpResourceSet { + families: out_families, + })) +} + +fn resolve_child_ip_resources_vrs( + child_ip: Option<&IpResourceSet>, + issuer_effective: Option<&IpResourceSet>, + parent_by_afi: Option< + &BTreeMap>, + >, + parent_intervals_by_afi: &HashMap, Vec)>>, +) -> Result<(Option, bool, bool), CaPathError> { + let Some(child_ip) = child_ip else { + return Ok((None, false, false)); + }; + if child_ip.has_any_inherit() && issuer_effective.is_none() { + return Err(CaPathError::InheritWithoutParentResources); + } + + let parent_by_afi = parent_by_afi.unwrap_or_else(|| { + static EMPTY: std::sync::OnceLock< + BTreeMap>, + > = std::sync::OnceLock::new(); + EMPTY.get_or_init(BTreeMap::new) + }); + let mut out_families = Vec::new(); + let mut reduced = false; + let mut saw_declared = false; + + for fam in &child_ip.families { + match &fam.choice { + IpAddressChoice::Inherit => { + let items = parent_by_afi + .get(&fam.afi) + .ok_or(CaPathError::InheritWithoutParentResources)?; + out_families.push(crate::data_model::rc::IpAddressFamily { + afi: fam.afi, + choice: IpAddressChoice::AddressesOrRanges(items.clone()), + }); + } + IpAddressChoice::AddressesOrRanges(items) => { + saw_declared = saw_declared || !items.is_empty(); + let parent_intervals = parent_intervals_by_afi + .get(&fam.afi) + .map(Vec::as_slice) + .unwrap_or(&[]); + let intersections = + intersect_ip_items_with_parent_intervals(items, parent_intervals); + let child_intervals = ip_items_to_merged_intervals(items); + if intervals_changed(&child_intervals, &intersections) { + reduced = true; + } + if !intersections.is_empty() { + out_families.push(crate::data_model::rc::IpAddressFamily { + afi: fam.afi, + choice: IpAddressChoice::AddressesOrRanges(ip_intervals_to_ranges( + fam.afi, + &intersections, + )), + }); + } + } + } + } + + let empty = saw_declared && out_families.is_empty(); + Ok(( + Some(IpResourceSet { + families: out_families, + }), + reduced, + empty, + )) +} + +fn resolve_child_as_resources( + child_as: Option<&AsResourceSet>, + issuer_effective: Option<&AsResourceSet>, +) -> Result, CaPathError> { + let precomputed_asnum = issuer_effective + .and_then(|resources| resources.asnum.as_ref().map(as_choice_to_merged_intervals)); + let precomputed_rdi = issuer_effective + .and_then(|resources| resources.rdi.as_ref().map(as_choice_to_merged_intervals)); + resolve_child_as_resources_indexed( + child_as, + issuer_effective, + precomputed_asnum.as_deref(), + precomputed_rdi.as_deref(), + ) +} + +fn resolve_child_as_resources_indexed( + child_as: Option<&AsResourceSet>, + issuer_effective: Option<&AsResourceSet>, + parent_asnum_intervals: Option<&[(u32, u32)]>, + parent_rdi_intervals: Option<&[(u32, u32)]>, +) -> Result, CaPathError> { + let Some(child_as) = child_as else { + return Ok(None); + }; + let Some(parent) = issuer_effective else { + if matches!(child_as.asnum, Some(AsIdentifierChoice::Inherit)) + || matches!(child_as.rdi, Some(AsIdentifierChoice::Inherit)) + { + return Err(CaPathError::InheritWithoutParentResources); + } + return Err(CaPathError::ResourcesNotSubset); + }; + + let asnum = match child_as.asnum.as_ref() { + None => None, + Some(AsIdentifierChoice::Inherit) => parent + .asnum + .clone() + .ok_or(CaPathError::InheritWithoutParentResources) + .map(Some)?, + Some(_) => { + if !as_choice_subset_with_parent_intervals( + child_as.asnum.as_ref(), + parent.asnum.as_ref(), + parent_asnum_intervals, + ) { + return Err(CaPathError::ResourcesNotSubset); + } + child_as.asnum.clone() + } + }; + + let rdi = match child_as.rdi.as_ref() { + None => None, + Some(AsIdentifierChoice::Inherit) => parent + .rdi + .clone() + .ok_or(CaPathError::InheritWithoutParentResources) + .map(Some)?, + Some(_) => { + if !as_choice_subset_with_parent_intervals( + child_as.rdi.as_ref(), + parent.rdi.as_ref(), + parent_rdi_intervals, + ) { + return Err(CaPathError::ResourcesNotSubset); + } + child_as.rdi.clone() + } + }; + + Ok(Some(AsResourceSet { asnum, rdi })) +} + +fn resolve_child_as_resources_vrs( + child_as: Option<&AsResourceSet>, + issuer_effective: Option<&AsResourceSet>, + parent_asnum_intervals: Option<&[(u32, u32)]>, + parent_rdi_intervals: Option<&[(u32, u32)]>, +) -> Result<(Option, bool, bool), CaPathError> { + let Some(child_as) = child_as else { + return Ok((None, false, false)); + }; + if issuer_effective.is_none() + && (matches!(child_as.asnum, Some(AsIdentifierChoice::Inherit)) + || matches!(child_as.rdi, Some(AsIdentifierChoice::Inherit))) + { + return Err(CaPathError::InheritWithoutParentResources); + } + + let (asnum, asnum_reduced, asnum_empty) = resolve_as_choice_vrs( + child_as.asnum.as_ref(), + issuer_effective.and_then(|p| p.asnum.as_ref()), + parent_asnum_intervals, + )?; + let (rdi, rdi_reduced, rdi_empty) = resolve_as_choice_vrs( + child_as.rdi.as_ref(), + issuer_effective.and_then(|p| p.rdi.as_ref()), + parent_rdi_intervals, + )?; + Ok(( + Some(AsResourceSet { asnum, rdi }), + asnum_reduced || rdi_reduced, + asnum_empty || rdi_empty, + )) +} + +fn resolve_as_choice_vrs( + child: Option<&AsIdentifierChoice>, + parent: Option<&AsIdentifierChoice>, + parent_intervals_hint: Option<&[(u32, u32)]>, +) -> Result<(Option, bool, bool), CaPathError> { + let Some(child) = child else { + return Ok((None, false, false)); + }; + match child { + AsIdentifierChoice::Inherit => { + let parent = parent + .cloned() + .ok_or(CaPathError::InheritWithoutParentResources)?; + Ok((Some(parent), false, false)) + } + AsIdentifierChoice::AsIdsOrRanges(_) => { + let child_intervals = as_choice_to_merged_intervals(child); + let parent_intervals; + let parent_intervals = match parent_intervals_hint { + Some(v) => v, + None => { + parent_intervals = parent + .map(as_choice_to_merged_intervals) + .unwrap_or_default(); + parent_intervals.as_slice() + } + }; + let intersections = intersect_as_intervals(&child_intervals, parent_intervals); + let reduced = child_intervals != intersections; + let empty = !child_intervals.is_empty() && intersections.is_empty(); + Ok(( + Some(AsIdentifierChoice::AsIdsOrRanges(as_intervals_to_items( + &intersections, + ))), + reduced, + empty, + )) + } + } +} + +fn as_choice_subset( + child: Option<&AsIdentifierChoice>, + parent: Option<&AsIdentifierChoice>, +) -> bool { + as_choice_subset_with_parent_intervals(child, parent, None) +} + +fn as_choice_subset_with_parent_intervals( + child: Option<&AsIdentifierChoice>, + parent: Option<&AsIdentifierChoice>, + parent_intervals_hint: Option<&[(u32, u32)]>, +) -> bool { + let Some(child) = child else { + return true; + }; + let Some(parent) = parent else { + return false; + }; + + // Treat inherit as "all of parent" here; actual resolution is handled elsewhere. + if matches!(child, AsIdentifierChoice::Inherit) { + return true; + } + if matches!(parent, AsIdentifierChoice::Inherit) { + return true; + } + + let child_intervals = as_choice_to_merged_intervals(child); + let owned_parent_intervals; + let parent_intervals = match parent_intervals_hint { + Some(intervals) => intervals, + None => { + owned_parent_intervals = as_choice_to_merged_intervals(parent); + owned_parent_intervals.as_slice() + } + }; + for (cmin, cmax) in &child_intervals { + if !as_interval_is_covered(parent_intervals, *cmin, *cmax) { + return false; + } + } + true +} + +fn as_choice_to_merged_intervals(choice: &AsIdentifierChoice) -> Vec<(u32, u32)> { + let mut v = Vec::new(); + match choice { + AsIdentifierChoice::Inherit => {} + AsIdentifierChoice::AsIdsOrRanges(items) => { + for item in items { + match item { + crate::data_model::rc::AsIdOrRange::Id(id) => v.push((*id, *id)), + crate::data_model::rc::AsIdOrRange::Range { min, max } => v.push((*min, *max)), + } + } + } + } + v.sort_by_key(|(a, _b)| *a); + merge_as_intervals(&v) +} + +fn merge_as_intervals(v: &[(u32, u32)]) -> Vec<(u32, u32)> { + let mut out: Vec<(u32, u32)> = Vec::new(); + for (min, max) in v { + let Some(last) = out.last_mut() else { + out.push((*min, *max)); + continue; + }; + if *min <= last.1.saturating_add(1) { + last.1 = last.1.max(*max); + continue; + } + out.push((*min, *max)); + } + out +} + +fn as_interval_is_covered(parent: &[(u32, u32)], min: u32, max: u32) -> bool { + for (pmin, pmax) in parent { + if *pmin <= min && max <= *pmax { + return true; + } + if *pmin > min { + break; + } + } + false +} diff --git a/crates/panda-rpki-validator/src/validation/ca_path/tests.rs b/crates/panda-rpki-validator/src/validation/ca_path/tests.rs new file mode 100644 index 0000000..7ab141d --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/ca_path/tests.rs @@ -0,0 +1,962 @@ +// CA path resource and certificate validation tests. + +use super::*; +use crate::data_model::common::X509NameDer; +use crate::data_model::oid::OID_CP_IPADDR_ASNUMBER; +use crate::data_model::rc::{ + Afi, AsIdOrRange, AsIdentifierChoice, AsResourceSet, IpAddressChoice, IpAddressFamily, + IpAddressOrRange, IpPrefix, IpResourceSet, +}; +use crate::data_model::rc::{ + BasicConstraintsProfile, CertificatePoliciesProfile, RcExtensions, ResourceCertKind, + ResourceCertificate, RpkixTbsCertificate, +}; +use der_parser::num_bigint::BigUint; +use std::process::Command; +fn dummy_cert( + kind: ResourceCertKind, + subject_dn: &str, + issuer_dn: &str, + ski: Option>, + aki: Option>, + aia: Option>, + crldp: Option>, +) -> ResourceCertificate { + let aia = aia.map(|v| v.into_iter().map(|s| s.to_string()).collect::>()); + let crldp = crldp.map(|v| v.into_iter().map(|s| s.to_string()).collect::>()); + + ResourceCertificate { + raw_der: Vec::new(), + kind, + tbs: RpkixTbsCertificate { + version: 2, + serial_number: BigUint::from(1u8), + signature_algorithm: "1.2.840.113549.1.1.11".to_string(), + issuer_name: X509NameDer(issuer_dn.as_bytes().to_vec()), + subject_name: X509NameDer(subject_dn.as_bytes().to_vec()), + validity_not_before: time::OffsetDateTime::UNIX_EPOCH, + validity_not_after: time::OffsetDateTime::UNIX_EPOCH, + subject_public_key_info: Vec::new(), + extensions: RcExtensions { + basic_constraints_ca: kind == ResourceCertKind::Ca, + basic_constraints: (kind == ResourceCertKind::Ca).then_some( + BasicConstraintsProfile { + ca: true, + critical: true, + path_len_constraint: None, + }, + ), + subject_key_identifier: ski, + authority_key_identifier: aki, + crl_distribution_points_uris: crldp, + ca_issuers_uris: aia, + subject_info_access: None, + certificate_policies_oid: (kind == ResourceCertKind::Ca) + .then_some(OID_CP_IPADDR_ASNUMBER.to_string()), + certificate_policies: (kind == ResourceCertKind::Ca).then_some( + CertificatePoliciesProfile { + policy_oid: OID_CP_IPADDR_ASNUMBER.to_string(), + qualifier_oids: Vec::new(), + }, + ), + extension_oids: Vec::new(), + ip_resources: None, + as_resources: None, + }, + }, + } +} + +fn openssl_available() -> bool { + Command::new("openssl") + .arg("version") + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +fn write_cert_der_with_addext(dir: &std::path::Path, addext: Option<&str>) -> Vec { + assert!(openssl_available(), "openssl is required for this test"); + let key = dir.join("k.pem"); + let cert = dir.join("c.pem"); + let der = dir.join("c.der"); + + let mut cmd = Command::new("openssl"); + cmd.arg("req") + .arg("-x509") + .arg("-newkey") + .arg("rsa:2048") + .arg("-nodes") + .arg("-keyout") + .arg(&key) + .arg("-subj") + .arg("/CN=ku") + .arg("-days") + .arg("1") + .arg("-out") + .arg(&cert); + if let Some(ext) = addext { + cmd.arg("-addext").arg(ext); + } + let out = cmd.output().expect("openssl req"); + assert!( + out.status.success(), + "openssl req failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + + let out = Command::new("openssl") + .arg("x509") + .arg("-in") + .arg(&cert) + .arg("-outform") + .arg("DER") + .arg("-out") + .arg(&der) + .output() + .expect("openssl x509"); + assert!( + out.status.success(), + "openssl x509 failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + std::fs::read(&der).expect("read der") +} + +fn gen_issuer_and_child_der(dir: &std::path::Path) -> (Vec, Vec, Vec) { + assert!(openssl_available(), "openssl is required for this test"); + let issuer_key = dir.join("issuer.key"); + let issuer_csr = dir.join("issuer.csr"); + let issuer_pem = dir.join("issuer.pem"); + let issuer_der = dir.join("issuer.der"); + + let child_key = dir.join("child.key"); + let child_csr = dir.join("child.csr"); + let child_pem = dir.join("child.pem"); + let child_der = dir.join("child.der"); + + let other_key = dir.join("other.key"); + let other_csr = dir.join("other.csr"); + let other_pem = dir.join("other.pem"); + let other_der = dir.join("other.der"); + + let run = |cmd: &mut Command| { + let out = cmd.output().expect("run openssl"); + assert!( + out.status.success(), + "command failed: {:?}\nstderr={}", + cmd, + String::from_utf8_lossy(&out.stderr) + ); + }; + + // Issuer self-signed. + run(Command::new("openssl") + .args(["genrsa", "-out"]) + .arg(&issuer_key) + .arg("2048")); + run(Command::new("openssl") + .args(["req", "-new", "-key"]) + .arg(&issuer_key) + .args(["-subj", "/CN=issuer", "-out"]) + .arg(&issuer_csr)); + run(Command::new("openssl") + .args(["x509", "-req", "-in"]) + .arg(&issuer_csr) + .args(["-signkey"]) + .arg(&issuer_key) + .args(["-days", "1", "-out"]) + .arg(&issuer_pem)); + run(Command::new("openssl") + .args(["x509", "-in"]) + .arg(&issuer_pem) + .args(["-outform", "DER", "-out"]) + .arg(&issuer_der)); + + // Child signed by issuer. + run(Command::new("openssl") + .args(["genrsa", "-out"]) + .arg(&child_key) + .arg("2048")); + run(Command::new("openssl") + .args(["req", "-new", "-key"]) + .arg(&child_key) + .args(["-subj", "/CN=child", "-out"]) + .arg(&child_csr)); + run(Command::new("openssl") + .args(["x509", "-req", "-in"]) + .arg(&child_csr) + .args(["-CA"]) + .arg(&issuer_pem) + .args(["-CAkey"]) + .arg(&issuer_key) + .args(["-CAcreateserial", "-days", "1", "-out"]) + .arg(&child_pem)); + run(Command::new("openssl") + .args(["x509", "-in"]) + .arg(&child_pem) + .args(["-outform", "DER", "-out"]) + .arg(&child_der)); + + // Other self-signed issuer. + run(Command::new("openssl") + .args(["genrsa", "-out"]) + .arg(&other_key) + .arg("2048")); + run(Command::new("openssl") + .args(["req", "-new", "-key"]) + .arg(&other_key) + .args(["-subj", "/CN=other", "-out"]) + .arg(&other_csr)); + run(Command::new("openssl") + .args(["x509", "-req", "-in"]) + .arg(&other_csr) + .args(["-signkey"]) + .arg(&other_key) + .args(["-days", "1", "-out"]) + .arg(&other_pem)); + run(Command::new("openssl") + .args(["x509", "-in"]) + .arg(&other_pem) + .args(["-outform", "DER", "-out"]) + .arg(&other_der)); + + ( + std::fs::read(&issuer_der).expect("read issuer der"), + std::fs::read(&child_der).expect("read child der"), + std::fs::read(&other_der).expect("read other der"), + ) +} + +#[test] +fn resolve_child_ip_resources_rejects_inherit_without_parent_effective_resources() { + let child = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::Inherit, + }], + }; + let err = resolve_child_ip_resources(Some(&child), None).unwrap_err(); + assert!(matches!(err, CaPathError::InheritWithoutParentResources)); +} + +#[test] +fn resolve_child_ip_resources_rejects_non_inherit_without_parent_effective_resources() { + let child = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![]), + }], + }; + let err = resolve_child_ip_resources(Some(&child), None).unwrap_err(); + assert!(matches!(err, CaPathError::ResourcesNotSubset)); +} + +#[test] +fn ip_resources_by_afi_items_rejects_inherit_families() { + let parent = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv6, + choice: IpAddressChoice::Inherit, + }], + }; + let err = ip_resources_by_afi_items(&parent).unwrap_err(); + assert!(matches!(err, CaPathError::InheritWithoutParentResources)); +} + +#[test] +fn resolve_child_as_resources_rejects_inherit_without_parent_effective_resources() { + let child = AsResourceSet { + asnum: Some(AsIdentifierChoice::Inherit), + rdi: None, + }; + let err = resolve_child_as_resources(Some(&child), None).unwrap_err(); + assert!(matches!(err, CaPathError::InheritWithoutParentResources)); +} + +#[test] +fn validation_update_03_intersects_overclaiming_child_ip_resources() { + let issuer = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix(IpPrefix { + afi: Afi::Ipv4, + prefix_len: 24, + addr: vec![10, 0, 0, 0], + })]), + }], + }; + let child = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix(IpPrefix { + afi: Afi::Ipv4, + prefix_len: 23, + addr: vec![10, 0, 0, 0], + })]), + }], + }; + let index = IssuerEffectiveResourcesIndex::from_effective_resources(Some(&issuer), None) + .expect("index"); + let resolved = resolve_child_resources( + Some(&child), + Some(&issuer), + None, + None, + &index, + ResourceValidationMode::ValidationUpdate03, + ) + .expect("vrs resolution"); + assert!(resolved.warnings.ip_reduced_by_vrs); + assert!(!resolved.warnings.ip_vrs_empty); + let effective = resolved.effective_ip_resources.expect("effective ip"); + assert!(effective.families[0].contains_prefix(&IpPrefix { + afi: Afi::Ipv4, + prefix_len: 24, + addr: vec![10, 0, 0, 0], + })); + assert!(!effective.families[0].contains_prefix(&IpPrefix { + afi: Afi::Ipv4, + prefix_len: 24, + addr: vec![10, 0, 1, 0], + })); +} + +#[test] +fn validation_update_03_keeps_empty_vrs_instead_of_rejecting_child_ca() { + let issuer = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix(IpPrefix { + afi: Afi::Ipv4, + prefix_len: 24, + addr: vec![10, 0, 0, 0], + })]), + }], + }; + let child = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix(IpPrefix { + afi: Afi::Ipv4, + prefix_len: 24, + addr: vec![192, 0, 2, 0], + })]), + }], + }; + let index = IssuerEffectiveResourcesIndex::from_effective_resources(Some(&issuer), None) + .expect("index"); + let strict_err = resolve_child_resources( + Some(&child), + Some(&issuer), + None, + None, + &index, + ResourceValidationMode::Rfc6487, + ) + .unwrap_err(); + assert!(matches!(strict_err, CaPathError::ResourcesNotSubset)); + + let resolved = resolve_child_resources( + Some(&child), + Some(&issuer), + None, + None, + &index, + ResourceValidationMode::ValidationUpdate03, + ) + .expect("vrs resolution"); + assert!(resolved.warnings.ip_reduced_by_vrs); + assert!(resolved.warnings.ip_vrs_empty); + assert!( + resolved + .effective_ip_resources + .expect("effective ip") + .families + .is_empty() + ); +} + +#[test] +fn validation_update_03_intersects_overclaiming_child_as_resources() { + let issuer = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { + min: 64500, + max: 64510, + }, + ])), + rdi: None, + }; + let child = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { + min: 64505, + max: 64520, + }, + ])), + rdi: None, + }; + let index = IssuerEffectiveResourcesIndex::from_effective_resources(None, Some(&issuer)) + .expect("index"); + let resolved = resolve_child_resources( + None, + None, + Some(&child), + Some(&issuer), + &index, + ResourceValidationMode::ValidationUpdate03, + ) + .expect("vrs resolution"); + assert!(resolved.warnings.as_reduced_by_vrs); + assert!(!resolved.warnings.as_vrs_empty); + let effective = resolved.effective_as_resources.expect("effective as"); + assert_eq!( + effective.asnum, + Some(AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { + min: 64505, + max: 64510, + }, + ])) + ); +} + +#[test] +fn validation_update_03_records_all_resource_warning_summary_parts() { + let issuer_ip = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix(IpPrefix { + afi: Afi::Ipv4, + prefix_len: 24, + addr: vec![10, 0, 0, 0], + })]), + }], + }; + let child_ip = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix(IpPrefix { + afi: Afi::Ipv4, + prefix_len: 24, + addr: vec![192, 0, 2, 0], + })]), + }], + }; + let issuer_as = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { + min: 64500, + max: 64510, + }, + ])), + rdi: None, + }; + let child_as = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { + min: 64520, + max: 64530, + }, + ])), + rdi: None, + }; + let index = + IssuerEffectiveResourcesIndex::from_effective_resources(Some(&issuer_ip), Some(&issuer_as)) + .expect("index"); + + let resolved = resolve_child_resources( + Some(&child_ip), + Some(&issuer_ip), + Some(&child_as), + Some(&issuer_as), + &index, + ResourceValidationMode::ValidationUpdate03, + ) + .expect("vrs resolution"); + + assert!(resolved.warnings.ip_reduced_by_vrs); + assert!(resolved.warnings.ip_vrs_empty); + assert!(resolved.warnings.as_reduced_by_vrs); + assert!(resolved.warnings.as_vrs_empty); + assert_eq!( + resolved.warnings.summary(), + "ip_reduced_by_vrs,as_reduced_by_vrs,ip_vrs_empty,as_vrs_empty" + ); + assert!(!resolved.warnings.is_empty()); +} + +#[test] +fn child_aki_mismatch_is_rejected() { + let issuer = dummy_cert( + ResourceCertKind::Ca, + "CN=issuer", + "CN=issuer", + Some(vec![1]), + None, + None, + None, + ); + let child = dummy_cert( + ResourceCertKind::Ca, + "CN=child", + "CN=issuer", + Some(vec![2]), + Some(vec![9]), + Some(vec!["rsync://example.test/issuer.cer"]), + Some(vec!["rsync://example.test/issuer.crl"]), + ); + let err = validate_child_aki_matches_issuer_ski(&child, &issuer).unwrap_err(); + assert!(matches!(err, CaPathError::ChildAkiMismatch), "{err}"); +} + +#[test] +fn child_aia_missing_is_rejected() { + let _issuer = dummy_cert( + ResourceCertKind::Ca, + "CN=issuer", + "CN=issuer", + Some(vec![1]), + None, + None, + None, + ); + let child = dummy_cert( + ResourceCertKind::Ca, + "CN=child", + "CN=issuer", + Some(vec![2]), + Some(vec![1]), + None, + Some(vec!["rsync://example.test/issuer.crl"]), + ); + let err = validate_child_aia_points_to_issuer_uri(&child, "rsync://example.test/issuer.cer") + .unwrap_err(); + assert!(matches!(err, CaPathError::ChildAiaMissing), "{err}"); + + // Also cover issuer ski missing. + let issuer_missing_ski = dummy_cert( + ResourceCertKind::Ca, + "CN=issuer", + "CN=issuer", + None, + None, + None, + None, + ); + let err = validate_child_aki_matches_issuer_ski(&child, &issuer_missing_ski).unwrap_err(); + assert!(matches!(err, CaPathError::IssuerSkiMissing), "{err}"); +} + +#[test] +fn child_aia_issuer_uri_mismatch_is_rejected() { + let _issuer = dummy_cert( + ResourceCertKind::Ca, + "CN=issuer", + "CN=issuer", + Some(vec![1]), + None, + None, + None, + ); + let child = dummy_cert( + ResourceCertKind::Ca, + "CN=child", + "CN=issuer", + Some(vec![2]), + Some(vec![1]), + Some(vec!["rsync://example.test/other.cer"]), + Some(vec!["rsync://example.test/issuer.crl"]), + ); + let err = validate_child_aia_points_to_issuer_uri(&child, "rsync://example.test/issuer.cer") + .unwrap_err(); + assert!( + matches!(err, CaPathError::ChildAiaIssuerUriMismatch), + "{err}" + ); +} + +#[test] +fn child_crldp_mismatch_is_rejected() { + let issuer = dummy_cert( + ResourceCertKind::Ca, + "CN=issuer", + "CN=issuer", + Some(vec![1]), + None, + None, + None, + ); + let child = dummy_cert( + ResourceCertKind::Ca, + "CN=child", + "CN=issuer", + Some(vec![2]), + Some(vec![1]), + Some(vec!["rsync://example.test/issuer.cer"]), + None, + ); + let err = + validate_child_crldp_contains_issuer_crl_uri(&child, "rsync://example.test/issuer.crl") + .unwrap_err(); + assert!(matches!(err, CaPathError::ChildCrlDpMissing), "{err}"); + + let child = dummy_cert( + ResourceCertKind::Ca, + "CN=child", + "CN=issuer", + Some(vec![2]), + Some(vec![1]), + Some(vec!["rsync://example.test/issuer.cer"]), + Some(vec!["rsync://example.test/other.crl"]), + ); + let err = + validate_child_crldp_contains_issuer_crl_uri(&child, "rsync://example.test/issuer.crl") + .unwrap_err(); + assert!(matches!(err, CaPathError::ChildCrlDpUriMismatch), "{err}"); + + // Cover child AKI missing. + let child_missing_aki = dummy_cert( + ResourceCertKind::Ca, + "CN=child", + "CN=issuer", + Some(vec![2]), + None, + Some(vec!["rsync://example.test/issuer.cer"]), + Some(vec!["rsync://example.test/issuer.crl"]), + ); + let err = validate_child_aki_matches_issuer_ski(&child_missing_aki, &issuer).unwrap_err(); + assert!(matches!(err, CaPathError::ChildAkiMissing), "{err}"); +} + +#[test] +fn child_binding_checks_accept_when_matching() { + let issuer = dummy_cert( + ResourceCertKind::Ca, + "CN=issuer", + "CN=issuer", + Some(vec![1]), + None, + None, + None, + ); + let child = dummy_cert( + ResourceCertKind::Ca, + "CN=child", + "CN=issuer", + Some(vec![2]), + Some(vec![1]), + Some(vec!["rsync://example.test/issuer.cer"]), + Some(vec!["rsync://example.test/issuer.crl"]), + ); + validate_child_aki_matches_issuer_ski(&child, &issuer).expect("aki ok"); + validate_child_aia_points_to_issuer_uri(&child, "rsync://example.test/issuer.cer") + .expect("aia ok"); + validate_child_crldp_contains_issuer_crl_uri(&child, "rsync://example.test/issuer.crl") + .expect("crldp ok"); +} + +#[test] +fn validate_child_ca_key_usage_accepts_only_keycertsign_and_crlsign_critical() { + let td = tempfile::tempdir().expect("tempdir"); + let der = + write_cert_der_with_addext(td.path(), Some("keyUsage = critical, keyCertSign, cRLSign")); + let cert = parse_x509_cert(&der).expect("x509 parse ok"); + validate_child_ca_key_usage(&cert).expect("key usage ok"); +} + +#[test] +fn validate_child_ca_key_usage_rejects_missing_noncritical_and_invalid_bits() { + let td = tempfile::tempdir().expect("tempdir"); + let missing = write_cert_der_with_addext(td.path(), None); + let cert = parse_x509_cert(&missing).expect("x509 parse ok"); + let err = validate_child_ca_key_usage(&cert).unwrap_err(); + assert!(matches!(err, CaPathError::KeyUsageMissing), "{err}"); + + let td = tempfile::tempdir().expect("tempdir"); + let noncritical = + write_cert_der_with_addext(td.path(), Some("keyUsage = keyCertSign, cRLSign")); + let cert = parse_x509_cert(&noncritical).expect("x509 parse ok"); + let err = validate_child_ca_key_usage(&cert).unwrap_err(); + assert!(matches!(err, CaPathError::KeyUsageNotCritical), "{err}"); + + let td = tempfile::tempdir().expect("tempdir"); + let invalid = write_cert_der_with_addext( + td.path(), + Some("keyUsage = critical, keyCertSign, cRLSign, digitalSignature"), + ); + let cert = parse_x509_cert(&invalid).expect("x509 parse ok"); + let err = validate_child_ca_key_usage(&cert).unwrap_err(); + assert!(matches!(err, CaPathError::KeyUsageInvalidBits), "{err}"); +} + +#[test] +fn verify_cert_signature_with_issuer_accepts_valid_chain_and_rejects_wrong_issuer() { + let td = tempfile::tempdir().expect("tempdir"); + let (issuer, child, other) = gen_issuer_and_child_der(td.path()); + let issuer_cert = parse_x509_cert(&issuer).expect("x509 parse issuer"); + let child_cert = parse_x509_cert(&child).expect("x509 parse child"); + verify_child_signature(&child_cert, &issuer_cert.tbs_certificate.subject_pki) + .expect("signature ok"); + let other_cert = parse_x509_cert(&other).expect("x509 parse other"); + let err = + verify_child_signature(&child_cert, &other_cert.tbs_certificate.subject_pki).unwrap_err(); + assert!( + matches!(err, CaPathError::ChildSignatureInvalid(_)), + "{err}" + ); +} + +#[test] +fn signature_cache_stores_real_verify_success_and_skips_on_hit() { + let _guard = crate::crypto_sig_cache::GLOBAL_TEST_LOCK + .lock() + .expect("test lock"); + crate::crypto_sig_cache::clear_global(); + let td = tempfile::tempdir().expect("tempdir"); + let (issuer, child, other) = gen_issuer_and_child_der(td.path()); + let issuer_cert = parse_x509_cert(&issuer).expect("x509 parse issuer"); + let child_cert = parse_x509_cert(&child).expect("x509 parse child"); + let other_cert = parse_x509_cert(&other).expect("x509 parse other"); + + let cache_dir = tempfile::tempdir().expect("tempdir"); + let cache = std::sync::Arc::new(crate::crypto_sig_cache::CryptoSigCache::load_or_rebuild( + cache_dir.path().join("work-db.crypto-sig-cache"), + true, + )); + crate::crypto_sig_cache::install_global(std::sync::Arc::clone(&cache)); + + // Miss -> real verification runs and passes -> positive conclusion written. + verify_child_signature(&child_cert, &issuer_cert.tbs_certificate.subject_pki) + .expect("signature ok"); + // Hit -> skipped (a second execution would also pass, so assert via stats). + verify_child_signature(&child_cert, &issuer_cert.tbs_certificate.subject_pki) + .expect("signature ok"); + // Wrong issuer: real verification fails, nothing is written (no negative caching), + // so a retry executes the real verification again instead of inheriting anything. + for _ in 0..2 { + let err = verify_child_signature(&child_cert, &other_cert.tbs_certificate.subject_pki) + .unwrap_err(); + assert!( + matches!(err, CaPathError::ChildSignatureInvalid(_)), + "{err}" + ); + } + + let summary = cache.summary(); + let stats = summary + .per_point + .get("child_ca_cert") + .expect("child_ca_cert stats"); + assert_eq!(stats.calls, 4); + assert_eq!(stats.would_hit, 1); + assert_eq!(stats.new_keys, 3); + assert_eq!(stats.verify_executed, 3); + assert_eq!(stats.verify_skipped, 1); + crate::crypto_sig_cache::clear_global(); +} + +#[test] +fn issuer_effective_resources_index_and_indexed_resolvers_cover_success_and_failure_paths() { + use crate::data_model::rc::{AsIdOrRange, IpPrefix}; + + let parent_ip = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix(IpPrefix { + afi: Afi::Ipv4, + prefix_len: 8, + addr: vec![10, 0, 0, 0], + })]), + }], + }; + let parent_as = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { + min: 64500, + max: 64599, + }, + ])), + rdi: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id( + 65000, + )])), + }; + let idx = + IssuerEffectiveResourcesIndex::from_effective_resources(Some(&parent_ip), Some(&parent_as)) + .expect("index builds"); + assert_eq!( + idx.parent_ip_by_afi_items.as_ref().map(|v| v.len()), + Some(1) + ); + assert_eq!(idx.parent_ip_merged_intervals.len(), 1); + assert_eq!( + idx.parent_asnum_intervals.as_ref().map(|v| v.len()), + Some(1) + ); + assert_eq!(idx.parent_rdi_intervals.as_ref().map(|v| v.len()), Some(1)); + + let child_ip_subset = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix(IpPrefix { + afi: Afi::Ipv4, + prefix_len: 16, + addr: vec![10, 1, 0, 0], + })]), + }], + }; + assert!( + resolve_child_ip_resources_indexed( + Some(&child_ip_subset), + Some(&parent_ip), + idx.parent_ip_by_afi_items.as_ref(), + &idx.parent_ip_merged_intervals, + ) + .expect("subset should resolve") + .is_some() + ); + + let child_ip_bad = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix(IpPrefix { + afi: Afi::Ipv4, + prefix_len: 16, + addr: vec![11, 0, 0, 0], + })]), + }], + }; + let err = resolve_child_ip_resources_indexed( + Some(&child_ip_bad), + Some(&parent_ip), + idx.parent_ip_by_afi_items.as_ref(), + &idx.parent_ip_merged_intervals, + ) + .unwrap_err(); + assert!(matches!(err, CaPathError::ResourcesNotSubset)); + + let child_as_subset = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id( + 64542, + )])), + rdi: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id( + 65000, + )])), + }; + assert!( + resolve_child_as_resources_indexed( + Some(&child_as_subset), + Some(&parent_as), + idx.parent_asnum_intervals.as_deref(), + idx.parent_rdi_intervals.as_deref(), + ) + .expect("subset as resolves") + .is_some() + ); + + let child_as_bad = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id( + 65123, + )])), + rdi: None, + }; + let err = resolve_child_as_resources_indexed( + Some(&child_as_bad), + Some(&parent_as), + idx.parent_asnum_intervals.as_deref(), + idx.parent_rdi_intervals.as_deref(), + ) + .unwrap_err(); + assert!(matches!(err, CaPathError::ResourcesNotSubset)); +} + +#[test] +fn resolve_child_ip_and_as_resources_success_paths() { + use crate::data_model::rc::{AsIdOrRange, IpAddressOrRange, IpPrefix}; + + let parent_ip = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix(IpPrefix { + afi: Afi::Ipv4, + prefix_len: 8, + addr: vec![10, 0, 0, 0], + })]), + }], + }; + + let child_ip_inherit = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::Inherit, + }], + }; + let eff = resolve_child_ip_resources(Some(&child_ip_inherit), Some(&parent_ip)) + .expect("inherit resolves") + .expect("some ip"); + assert_eq!(eff.families.len(), 1); + assert!(matches!( + eff.families[0].choice, + IpAddressChoice::AddressesOrRanges(_) + )); + + let child_ip_subset = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix(IpPrefix { + afi: Afi::Ipv4, + prefix_len: 16, + addr: vec![10, 1, 0, 0], + })]), + }], + }; + resolve_child_ip_resources(Some(&child_ip_subset), Some(&parent_ip)) + .expect("subset ok") + .expect("some"); + + let child_ip_bad = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix(IpPrefix { + afi: Afi::Ipv4, + prefix_len: 16, + addr: vec![11, 0, 0, 0], + })]), + }], + }; + let err = resolve_child_ip_resources(Some(&child_ip_bad), Some(&parent_ip)).unwrap_err(); + assert!(matches!(err, CaPathError::ResourcesNotSubset), "{err}"); + + let parent_as = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { min: 1, max: 100 }, + ])), + rdi: None, + }; + let child_as_inherit = AsResourceSet { + asnum: Some(AsIdentifierChoice::Inherit), + rdi: None, + }; + let eff_as = resolve_child_as_resources(Some(&child_as_inherit), Some(&parent_as)) + .expect("inherit as") + .expect("some"); + assert_eq!(eff_as.asnum, parent_as.asnum); + + let child_as_subset = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id(50)])), + rdi: None, + }; + resolve_child_as_resources(Some(&child_as_subset), Some(&parent_as)) + .expect("subset as") + .expect("some"); + + let child_as_bad = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id( + 200, + )])), + rdi: None, + }; + let err = resolve_child_as_resources(Some(&child_as_bad), Some(&parent_as)).unwrap_err(); + assert!(matches!(err, CaPathError::ResourcesNotSubset), "{err}"); +} diff --git a/crates/panda-rpki-validator/src/validation/ca_path/types_and_validation.rs b/crates/panda-rpki-validator/src/validation/ca_path/types_and_validation.rs new file mode 100644 index 0000000..80ad0b4 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/ca_path/types_and_validation.rs @@ -0,0 +1,464 @@ +// Resource certificate path types and public validation entry points. + +use crate::data_model::common::BigUnsigned; +use crate::data_model::crl::{CrlDecodeError, CrlVerifyError, RpkixCrl}; +use crate::data_model::oid::OID_KEY_USAGE_RAW; +use crate::data_model::rc::{ + AsIdentifierChoice, AsResourceSet, IpAddressChoice, IpResourceSet, ResourceCertKind, + ResourceCertificate, ResourceCertificateDecodeError, ResourceCertificateProfileError, + ResourceCertificateRole, +}; +use crate::policy::ResourceValidationMode; +use x509_parser::prelude::{FromDer, X509Certificate}; + +use crate::validation::x509_name::x509_names_equivalent; +use std::collections::{BTreeMap, HashMap, HashSet}; +use x509_parser::x509::SubjectPublicKeyInfo; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ValidatedSubordinateCa { + pub child_ca: ResourceCertificate, + pub issuer_ca: ResourceCertificate, + pub issuer_crl: RpkixCrl, + pub effective_ip_resources: Option, + pub effective_as_resources: Option, + pub resource_warnings: ResourceValidationWarnings, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ValidatedSubordinateCaLite { + pub child_ca: ResourceCertificate, + pub effective_ip_resources: Option, + pub effective_as_resources: Option, + pub resource_warnings: ResourceValidationWarnings, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ResourceValidationWarnings { + pub ip_reduced_by_vrs: bool, + pub as_reduced_by_vrs: bool, + pub ip_vrs_empty: bool, + pub as_vrs_empty: bool, +} + +impl ResourceValidationWarnings { + pub fn is_empty(&self) -> bool { + !self.ip_reduced_by_vrs + && !self.as_reduced_by_vrs + && !self.ip_vrs_empty + && !self.as_vrs_empty + } + + pub fn summary(&self) -> String { + let mut parts = Vec::new(); + if self.ip_reduced_by_vrs { + parts.push("ip_reduced_by_vrs"); + } + if self.as_reduced_by_vrs { + parts.push("as_reduced_by_vrs"); + } + if self.ip_vrs_empty { + parts.push("ip_vrs_empty"); + } + if self.as_vrs_empty { + parts.push("as_vrs_empty"); + } + parts.join(",") + } +} + +#[derive(Clone, Debug, Default)] +pub struct IssuerEffectiveResourcesIndex { + parent_ip_by_afi_items: + Option>>, + parent_ip_merged_intervals: HashMap, Vec)>>, + parent_asnum_intervals: Option>, + parent_rdi_intervals: Option>, +} + +impl IssuerEffectiveResourcesIndex { + pub fn from_effective_resources( + issuer_effective_ip: Option<&IpResourceSet>, + issuer_effective_as: Option<&AsResourceSet>, + ) -> Result { + let parent_ip_by_afi_items = issuer_effective_ip + .map(ip_resources_by_afi_items) + .transpose()?; + + let parent_ip_merged_intervals = issuer_effective_ip + .map(ip_resources_to_merged_intervals_by_afi) + .unwrap_or_default(); + + let parent_asnum_intervals = issuer_effective_as + .and_then(|resources| resources.asnum.as_ref().map(as_choice_to_merged_intervals)); + let parent_rdi_intervals = issuer_effective_as + .and_then(|resources| resources.rdi.as_ref().map(as_choice_to_merged_intervals)); + + Ok(Self { + parent_ip_by_afi_items, + parent_ip_merged_intervals, + parent_asnum_intervals, + parent_rdi_intervals, + }) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum CaPathError { + #[error("child CA certificate decode failed: {0} (RFC 6487 §4; RFC 5280 §4.1)")] + ChildDecode(#[from] ResourceCertificateDecodeError), + + #[error("issuer CA certificate decode failed: {0} (RFC 6487 §4; RFC 5280 §4.1)")] + IssuerDecode(ResourceCertificateDecodeError), + + #[error("child CA certificate profile validation failed: {0} (RFC 6487 §4.8)")] + ChildProfile(ResourceCertificateProfileError), + + #[error("issuer CA certificate profile validation failed: {0} (RFC 6487 §4.8)")] + IssuerProfile(ResourceCertificateProfileError), + + #[error("issuer CRL decode failed: {0} (RFC 6487 §5; RFC 9829 §3.1; RFC 5280 §5.1)")] + CrlDecode(#[from] CrlDecodeError), + + #[error( + "child certificate must be a CA resource certificate (RFC 6487 §4.8.1; RFC 5280 §4.2.1.9)" + )] + ChildNotCa, + + #[error( + "issuer certificate must be a CA resource certificate (RFC 6487 §4.8.1; RFC 5280 §4.2.1.9)" + )] + IssuerNotCa, + + #[error( + "child issuer DN does not match issuer CA subject DN: child.issuer={child_issuer_dn} issuer.subject={issuer_subject_dn} (RFC 5280 §6.1)" + )] + IssuerSubjectMismatch { + child_issuer_dn: String, + issuer_subject_dn: String, + }, + + #[error("child CA certificate signature verification failed: {0} (RFC 5280 §6.1)")] + ChildSignatureInvalid(String), + + #[error("issuer SubjectPublicKeyInfo parse error: {0} (RFC 5280 §4.1.2.7)")] + IssuerSpkiParse(String), + + #[error( + "trailing bytes after issuer SubjectPublicKeyInfo DER: {0} bytes (DER; RFC 5280 §4.1.2.7)" + )] + IssuerSpkiTrailingBytes(usize), + + #[error("certificate not valid at validation_time (RFC 5280 §4.1.2.5; RFC 5280 §6.1)")] + CertificateNotValidAtTime, + + #[error("child CA KeyUsage extension missing (RFC 6487 §4.8.4; RFC 5280 §4.2.1.3)")] + KeyUsageMissing, + + #[error("child CA KeyUsage criticality must be critical (RFC 6487 §4.8.4; RFC 5280 §4.2.1.3)")] + KeyUsageNotCritical, + + #[error("child CA KeyUsage must have only keyCertSign and cRLSign set (RFC 6487 §4.8.4)")] + KeyUsageInvalidBits, + + #[error( + "CRL signature/binding verification failed: {0} (RFC 5280 §6.3.3; RFC 6487 §5; RFC 9829 §3.1)" + )] + CrlVerify(#[from] CrlVerifyError), + + #[error( + "CRL not valid at validation_time (RFC 5280 §6.3.3(g); RFC 5280 §5.1.2.4-§5.1.2.5; RFC 6487 §5)" + )] + CrlNotValidAtTime, + + #[error("child CA certificate is revoked by issuer CRL (RFC 5280 §6.3.3; RFC 6487 §5)")] + ChildRevoked, + + #[error( + "child CA certificate must contain at least one RFC 3779 resource extension (IP or AS) (RFC 6487 §4.8.10-§4.8.11)" + )] + ResourcesMissing, + + #[error( + "resource extension inheritance cannot be resolved (parent missing resources) (RFC 6487 §7.2)" + )] + InheritWithoutParentResources, + + #[error("child CA resources are not a subset of issuer resources (RFC 6487 §7.2)")] + ResourcesNotSubset, + + #[error("issuer CA subjectKeyIdentifier missing (RFC 6487 §4.8.2)")] + IssuerSkiMissing, + + #[error("child CA authorityKeyIdentifier missing (RFC 6487 §4.8.3; RFC 5280 §4.2.1.1)")] + ChildAkiMissing, + + #[error( + "child CA authorityKeyIdentifier does not match issuer subjectKeyIdentifier (RFC 6487 §4.8.3)" + )] + ChildAkiMismatch, + + #[error("child CA authorityInfoAccess missing (RFC 6487 §4.8.7; RFC 5280 §4.2.2.1)")] + ChildAiaMissing, + + #[error( + "child CA authorityInfoAccess does not reference issuer certificate rsync URI (RFC 6487 §4.8.7)" + )] + ChildAiaIssuerUriMismatch, + + #[error("child CA CRLDistributionPoints missing (RFC 6487 §4.8.6; RFC 5280 §4.2.1.13)")] + ChildCrlDpMissing, + + #[error( + "child CA CRLDistributionPoints does not reference issuer CRL rsync URI (RFC 6487 §4.8.6)" + )] + ChildCrlDpUriMismatch, +} + +pub fn validate_subordinate_ca_cert( + child_ca_der: &[u8], + issuer_ca_der: &[u8], + issuer_crl_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_crl_rsync_uri: &str, + issuer_effective_ip: Option<&IpResourceSet>, + issuer_effective_as: Option<&AsResourceSet>, + validation_time: time::OffsetDateTime, +) -> Result { + validate_subordinate_ca_cert_with_resource_validation_mode( + child_ca_der, + issuer_ca_der, + issuer_crl_der, + issuer_ca_rsync_uri, + issuer_crl_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + ResourceValidationMode::Rfc6487, + ) +} + +pub fn validate_subordinate_ca_cert_with_resource_validation_mode( + child_ca_der: &[u8], + issuer_ca_der: &[u8], + issuer_crl_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_crl_rsync_uri: &str, + issuer_effective_ip: Option<&IpResourceSet>, + issuer_effective_as: Option<&AsResourceSet>, + validation_time: time::OffsetDateTime, + resource_validation_mode: ResourceValidationMode, +) -> Result { + let child_ca = ResourceCertificate::decode_der(child_ca_der)?; + if child_ca.kind != ResourceCertKind::Ca { + return Err(CaPathError::ChildNotCa); + } + child_ca + .validate_rfc6487_profile(ResourceCertificateRole::Ca) + .map_err(CaPathError::ChildProfile)?; + + let issuer_ca = + ResourceCertificate::decode_der(issuer_ca_der).map_err(CaPathError::IssuerDecode)?; + if issuer_ca.kind != ResourceCertKind::Ca { + return Err(CaPathError::IssuerNotCa); + } + issuer_ca + .validate_rfc6487_profile(ResourceCertificateRole::Ca) + .map_err(CaPathError::IssuerProfile)?; + let issuer_spki = parse_subject_pki_from_der(&issuer_ca.tbs.subject_public_key_info)?; + + if !x509_names_equivalent(&child_ca.tbs.issuer_name, &issuer_ca.tbs.subject_name) { + return Err(CaPathError::IssuerSubjectMismatch { + child_issuer_dn: child_ca.tbs.issuer_name.to_string(), + issuer_subject_dn: issuer_ca.tbs.subject_name.to_string(), + }); + } + + validate_child_aki_matches_issuer_ski(&child_ca, &issuer_ca)?; + if let Some(expected_issuer_uri) = issuer_ca_rsync_uri { + validate_child_aia_points_to_issuer_uri(&child_ca, expected_issuer_uri)?; + } + validate_child_crldp_contains_issuer_crl_uri(&child_ca, issuer_crl_rsync_uri)?; + + if !time_within_validity( + validation_time, + child_ca.tbs.validity_not_before, + child_ca.tbs.validity_not_after, + ) || !time_within_validity( + validation_time, + issuer_ca.tbs.validity_not_before, + issuer_ca.tbs.validity_not_after, + ) { + return Err(CaPathError::CertificateNotValidAtTime); + } + + let child_x509 = parse_x509_cert(child_ca_der)?; + verify_child_signature(&child_x509, &issuer_spki)?; + validate_child_ca_key_usage(&child_x509)?; + + let issuer_crl = RpkixCrl::decode_der(issuer_crl_der)?; + issuer_crl.verify_signature_with_issuer_certificate_der(issuer_ca_der)?; + if !crl_valid_at_time(&issuer_crl, validation_time) { + return Err(CaPathError::CrlNotValidAtTime); + } + + if is_serial_revoked_by_crl(&child_ca, &issuer_crl) { + return Err(CaPathError::ChildRevoked); + } + + let ResourceResolution { + effective_ip_resources, + effective_as_resources, + warnings: resource_warnings, + } = resolve_child_resources( + child_ca.tbs.extensions.ip_resources.as_ref(), + issuer_effective_ip, + child_ca.tbs.extensions.as_resources.as_ref(), + issuer_effective_as, + &IssuerEffectiveResourcesIndex::from_effective_resources( + issuer_effective_ip, + issuer_effective_as, + )?, + resource_validation_mode, + )?; + if effective_ip_resources.is_none() && effective_as_resources.is_none() { + return Err(CaPathError::ResourcesMissing); + } + + Ok(ValidatedSubordinateCa { + child_ca, + issuer_ca, + issuer_crl, + effective_ip_resources, + effective_as_resources, + resource_warnings, + }) +} + +/// Validate a subordinate child CA using *pre-decoded issuer CA* and *pre-decoded+verified issuer CRL*. +/// +/// This avoids repeating issuer CA decode and issuer CRL decode+signature verification for every +/// child CA certificate discovered in a publication point. +pub fn validate_subordinate_ca_cert_with_prevalidated_issuer( + child_ca_der: &[u8], + child_ca: ResourceCertificate, + issuer_ca: &ResourceCertificate, + issuer_spki: &SubjectPublicKeyInfo<'_>, + issuer_crl: &RpkixCrl, + issuer_crl_revoked_serials: &HashSet>, + issuer_ca_rsync_uri: Option<&str>, + issuer_crl_rsync_uri: &str, + issuer_effective_ip: Option<&IpResourceSet>, + issuer_effective_as: Option<&AsResourceSet>, + validation_time: time::OffsetDateTime, +) -> Result { + let issuer_resources_index = IssuerEffectiveResourcesIndex::from_effective_resources( + issuer_effective_ip, + issuer_effective_as, + )?; + validate_subordinate_ca_cert_with_prevalidated_issuer_and_resources( + child_ca_der, + child_ca, + issuer_ca, + issuer_spki, + issuer_crl, + issuer_crl_revoked_serials, + issuer_ca_rsync_uri, + issuer_crl_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + &issuer_resources_index, + validation_time, + ResourceValidationMode::Rfc6487, + ) +} + +pub fn validate_subordinate_ca_cert_with_prevalidated_issuer_and_resources( + child_ca_der: &[u8], + child_ca: ResourceCertificate, + issuer_ca: &ResourceCertificate, + issuer_spki: &SubjectPublicKeyInfo<'_>, + issuer_crl: &RpkixCrl, + issuer_crl_revoked_serials: &HashSet>, + issuer_ca_rsync_uri: Option<&str>, + issuer_crl_rsync_uri: &str, + issuer_effective_ip: Option<&IpResourceSet>, + issuer_effective_as: Option<&AsResourceSet>, + issuer_resources_index: &IssuerEffectiveResourcesIndex, + validation_time: time::OffsetDateTime, + resource_validation_mode: ResourceValidationMode, +) -> Result { + if child_ca.kind != ResourceCertKind::Ca { + return Err(CaPathError::ChildNotCa); + } + if issuer_ca.kind != ResourceCertKind::Ca { + return Err(CaPathError::IssuerNotCa); + } + child_ca + .validate_rfc6487_profile(ResourceCertificateRole::Ca) + .map_err(CaPathError::ChildProfile)?; + issuer_ca + .validate_rfc6487_profile(ResourceCertificateRole::Ca) + .map_err(CaPathError::IssuerProfile)?; + + if !x509_names_equivalent(&child_ca.tbs.issuer_name, &issuer_ca.tbs.subject_name) { + return Err(CaPathError::IssuerSubjectMismatch { + child_issuer_dn: child_ca.tbs.issuer_name.to_string(), + issuer_subject_dn: issuer_ca.tbs.subject_name.to_string(), + }); + } + + validate_child_aki_matches_issuer_ski(&child_ca, issuer_ca)?; + if let Some(expected_issuer_uri) = issuer_ca_rsync_uri { + validate_child_aia_points_to_issuer_uri(&child_ca, expected_issuer_uri)?; + } + validate_child_crldp_contains_issuer_crl_uri(&child_ca, issuer_crl_rsync_uri)?; + + if !time_within_validity( + validation_time, + child_ca.tbs.validity_not_before, + child_ca.tbs.validity_not_after, + ) || !time_within_validity( + validation_time, + issuer_ca.tbs.validity_not_before, + issuer_ca.tbs.validity_not_after, + ) { + return Err(CaPathError::CertificateNotValidAtTime); + } + + let child_x509 = parse_x509_cert(child_ca_der)?; + verify_child_signature(&child_x509, issuer_spki)?; + validate_child_ca_key_usage(&child_x509)?; + + if !crl_valid_at_time(issuer_crl, validation_time) { + return Err(CaPathError::CrlNotValidAtTime); + } + + let serial = BigUnsigned::from_biguint(&child_ca.tbs.serial_number); + if issuer_crl_revoked_serials.contains(&serial.bytes_be) { + return Err(CaPathError::ChildRevoked); + } + + let ResourceResolution { + effective_ip_resources, + effective_as_resources, + warnings: resource_warnings, + } = resolve_child_resources( + child_ca.tbs.extensions.ip_resources.as_ref(), + issuer_effective_ip, + child_ca.tbs.extensions.as_resources.as_ref(), + issuer_effective_as, + issuer_resources_index, + resource_validation_mode, + )?; + if effective_ip_resources.is_none() && effective_as_resources.is_none() { + return Err(CaPathError::ResourcesMissing); + } + + Ok(ValidatedSubordinateCaLite { + child_ca, + effective_ip_resources, + effective_as_resources, + resource_warnings, + }) +} diff --git a/crates/panda-rpki-validator/src/validation/cert_path.rs b/crates/panda-rpki-validator/src/validation/cert_path.rs index cd90e26..677d67a 100644 --- a/crates/panda-rpki-validator/src/validation/cert_path.rs +++ b/crates/panda-rpki-validator/src/validation/cert_path.rs @@ -188,7 +188,7 @@ pub fn validate_ee_cert_path( /// Validate the EE certificate path using a *pre-decoded issuer CA* and a *pre-decoded and /// pre-verified issuer CRL*. /// -/// This is a performance-oriented helper for stage2 serial runs: it avoids repeating issuer CA +/// This is a performance-oriented helper for serial runs: it avoids repeating issuer CA /// decode and issuer CRL decode+signature verification for every signed object in a publication point. /// /// The caller must ensure: diff --git a/crates/panda-rpki-validator/src/validation/manifest.rs b/crates/panda-rpki-validator/src/validation/manifest.rs index 802564e..2522e5f 100644 --- a/crates/panda-rpki-validator/src/validation/manifest.rs +++ b/crates/panda-rpki-validator/src/validation/manifest.rs @@ -11,1814 +11,9 @@ use std::cmp::Ordering; use std::collections::HashSet; use x509_parser::prelude::FromDer; -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum PublicationPointSource { - Fresh, - PublicationPointCache, - VcirCurrentInstance, - FailedFetchNoCache, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct PublicationPointResult { - pub source: PublicationPointSource, - pub snapshot: PublicationPointSnapshot, - pub warnings: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct FreshValidatedPublicationPoint { - pub manifest_rsync_uri: String, - pub publication_point_rsync_uri: String, - pub manifest_number_be: Vec, - pub this_update: PackTime, - pub next_update: PackTime, - pub verified_at: PackTime, - pub manifest_bytes: Vec, - pub files: Vec, -} - -pub trait PublicationPointData { - fn manifest_rsync_uri(&self) -> &str; - fn publication_point_rsync_uri(&self) -> &str; - fn manifest_number_be(&self) -> &[u8]; - fn this_update(&self) -> &PackTime; - fn next_update(&self) -> &PackTime; - fn verified_at(&self) -> &PackTime; - fn manifest_bytes(&self) -> &[u8]; - fn files(&self) -> &[PackFile]; -} - -impl FreshValidatedPublicationPoint { - pub fn to_publication_point_snapshot(&self) -> PublicationPointSnapshot { - PublicationPointSnapshot { - format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, - manifest_rsync_uri: self.manifest_rsync_uri.clone(), - publication_point_rsync_uri: self.publication_point_rsync_uri.clone(), - manifest_number_be: self.manifest_number_be.clone(), - this_update: self.this_update.clone(), - next_update: self.next_update.clone(), - verified_at: self.verified_at.clone(), - manifest_bytes: self.manifest_bytes.clone(), - files: self.files.clone(), - } - } -} - -impl PublicationPointData for FreshValidatedPublicationPoint { - fn manifest_rsync_uri(&self) -> &str { - &self.manifest_rsync_uri - } - - fn publication_point_rsync_uri(&self) -> &str { - &self.publication_point_rsync_uri - } - - fn manifest_number_be(&self) -> &[u8] { - self.manifest_number_be.as_slice() - } - - fn this_update(&self) -> &PackTime { - &self.this_update - } - - fn next_update(&self) -> &PackTime { - &self.next_update - } - - fn verified_at(&self) -> &PackTime { - &self.verified_at - } - - fn manifest_bytes(&self) -> &[u8] { - self.manifest_bytes.as_slice() - } - - fn files(&self) -> &[PackFile] { - self.files.as_slice() - } -} - -impl PublicationPointData for PublicationPointSnapshot { - fn manifest_rsync_uri(&self) -> &str { - &self.manifest_rsync_uri - } - - fn publication_point_rsync_uri(&self) -> &str { - &self.publication_point_rsync_uri - } - - fn manifest_number_be(&self) -> &[u8] { - self.manifest_number_be.as_slice() - } - - fn this_update(&self) -> &PackTime { - &self.this_update - } - - fn next_update(&self) -> &PackTime { - &self.next_update - } - - fn verified_at(&self) -> &PackTime { - &self.verified_at - } - - fn manifest_bytes(&self) -> &[u8] { - self.manifest_bytes.as_slice() - } - - fn files(&self) -> &[PackFile] { - self.files.as_slice() - } -} - -#[derive(Debug, thiserror::Error)] -pub enum ManifestFreshError { - #[error("repo sync failed: {detail} (RFC 8182 §3.4.5; RFC 9286 §6.6)")] - RepoSyncFailed { detail: String }, - - #[error( - "manifest not found in current repository view: {manifest_rsync_uri} (RFC 9286 §6.2; RFC 9286 §6.6)" - )] - MissingManifest { manifest_rsync_uri: String }, - - #[error("manifest decode failed: {0} (RFC 9286 §4; RFC 9286 §6.2; RFC 9286 §6.6)")] - Decode(#[from] ManifestDecodeError), - - #[error( - "manifest embedded EE certificate resources invalid: {0} (RFC 9286 §5.1; RFC 9286 §6.2; RFC 9286 §6.6)" - )] - EeResources(#[from] ManifestValidateError), - - #[error( - "manifest CMS signature verification failed: {0} (RFC 6488 §3; RFC 9589 §4; RFC 9286 §6.2; RFC 9286 §6.6)" - )] - Signature(#[from] SignedObjectVerifyError), - - #[error( - "manifest embedded EE certificate path validation failed: {0} (RFC 6488 §3; RFC 9286 §6.2; RFC 9286 §6.6)" - )] - EeCertPath(#[from] CertPathError), - - #[error( - "manifest embedded EE certificate CRLDistributionPoints missing (cannot validate EE certificate) (RFC 6487 §4.8.6; RFC 6488 §3; RFC 9286 §6.2; RFC 9286 §6.6)" - )] - EeCrlDpMissing, - - #[error( - "publication point contains no CRL files (cannot validate manifest EE certificate) (RFC 9286 §7; RFC 6487 §4.8.6; RFC 6488 §3; RFC 9286 §6.2; RFC 9286 §6.6)" - )] - NoCrlFiles, - - #[error( - "CRL referenced by manifest embedded EE certificate CRLDistributionPoints not found at publication point: {0} (RFC 6487 §4.8.6; RFC 9286 §4.2.1; RFC 9286 §6.2; RFC 9286 §6.6)" - )] - EeCrlNotFound(String), - - #[error( - "manifest is not valid at validation_time: this_update={this_update_rfc3339_utc} next_update={next_update_rfc3339_utc} validation_time={validation_time_rfc3339_utc} (RFC 9286 §6.3; RFC 9286 §6.6)" - )] - StaleOrEarly { - this_update_rfc3339_utc: String, - next_update_rfc3339_utc: String, - validation_time_rfc3339_utc: String, - }, - - #[error( - "manifest must reside at the same publication point as id-ad-caRepository: manifest={manifest_rsync_uri} publication_point={publication_point_rsync_uri} (RFC 9286 §6.1; RFC 9286 §6.6)" - )] - ManifestOutsidePublicationPoint { - manifest_rsync_uri: String, - publication_point_rsync_uri: String, - }, - - #[error( - "manifestNumber not higher than previously validated manifest: old={old_hex} new={new_hex} (RFC 9286 §4.2.1; RFC 9286 §6.6)" - )] - ManifestNumberNotIncreasing { old_hex: String, new_hex: String }, - - #[error( - "thisUpdate not more recent than previously validated manifest: old={old_rfc3339_utc} new={new_rfc3339_utc} (RFC 9286 §4.2.1; RFC 9286 §6.6)" - )] - ThisUpdateNotIncreasing { - old_rfc3339_utc: String, - new_rfc3339_utc: String, - }, - - #[error( - "manifest referenced file missing in current repository view: {rsync_uri} (RFC 9286 §6.4; RFC 9286 §6.6)" - )] - MissingFile { rsync_uri: String }, - - #[error("manifest file hash mismatch: {rsync_uri} (RFC 9286 §6.5; RFC 9286 §6.6)")] - HashMismatch { rsync_uri: String }, - - #[error("issuer CA certificate bytes unavailable: {detail} (RFC 6487 §4; RFC 9286 §6.2)")] - IssuerCaLoadFailed { detail: String }, -} - -impl ManifestFreshError { - pub(crate) fn should_warn_when_current_instance_reused(&self) -> bool { - !matches!( - self, - ManifestFreshError::RepoSyncFailed { .. } - | ManifestFreshError::MissingManifest { .. } - | ManifestFreshError::MissingFile { .. } - ) - } -} - -#[derive(Debug, thiserror::Error)] -pub enum ManifestReuseError { - #[error("latest current-instance VCIR missing: {0} (RFC 9286 §6.6)")] - MissingCurrentInstanceVcir(String), - - #[error( - "latest current-instance VCIR is not marked failed-fetch eligible: {0} (RFC 9286 §6.6)" - )] - IneligibleCurrentInstanceVcir(String), - - #[error( - "latest current-instance VCIR instance_gate expired: manifest={manifest_rsync_uri} effective_until={effective_until_rfc3339_utc} validation_time={validation_time_rfc3339_utc} (RFC 9286 §6.6)" - )] - CurrentInstanceVcirExpired { - manifest_rsync_uri: String, - effective_until_rfc3339_utc: String, - validation_time_rfc3339_utc: String, - }, - - #[error( - "current-instance VCIR current_manifest_rsync_uri does not match requested manifest URI: expected={expected} actual={actual}" - )] - ManifestUriMismatch { expected: String, actual: String }, - - #[error("manifest raw bytes missing for current-instance VCIR reconstruction: {0}")] - MissingManifestRaw(String), - - #[error("artifact raw bytes missing for current-instance VCIR reconstruction: {rsync_uri}")] - MissingArtifactRaw { rsync_uri: String }, - - #[error("invalid current-instance VCIR: {0}")] - InvalidCurrentInstanceVcir(String), - - #[error("storage error during current-instance VCIR reuse: {0}")] - Storage(#[from] StorageError), -} - -#[derive(Debug, thiserror::Error)] -pub enum ManifestProcessError { - #[error("manifest processing failed and cache use is disabled: {0}")] - StopAllOutput(#[from] ManifestFreshError), - - #[error( - "manifest processing failed and no reusable current-instance validated result is available: fresh={fresh}; reused={reused}" - )] - NoUsableCache { - fresh: ManifestFreshError, - reused: ManifestReuseError, - }, - - #[error("storage error during manifest processing: {0}")] - Storage(#[from] StorageError), -} - -pub fn process_manifest_publication_point( - store: &RocksStore, - policy: &Policy, - manifest_rsync_uri: &str, - publication_point_rsync_uri: &str, - issuer_ca_der: &[u8], - issuer_ca_rsync_uri: Option<&str>, - validation_time: time::OffsetDateTime, -) -> Result { - process_manifest_publication_point_after_repo_sync( - store, - policy, - manifest_rsync_uri, - publication_point_rsync_uri, - issuer_ca_der, - issuer_ca_rsync_uri, - validation_time, - true, - None, - ) -} - -pub fn process_manifest_publication_point_fresh_after_repo_sync( - store: &RocksStore, - manifest_rsync_uri: &str, - publication_point_rsync_uri: &str, - issuer_ca_der: &[u8], - issuer_ca_rsync_uri: Option<&str>, - validation_time: time::OffsetDateTime, - repo_sync_ok: bool, - repo_sync_error: Option<&str>, -) -> Result { - process_manifest_publication_point_fresh_after_repo_sync_with_timing( - store, - manifest_rsync_uri, - publication_point_rsync_uri, - None, - issuer_ca_der, - issuer_ca_rsync_uri, - validation_time, - repo_sync_ok, - repo_sync_error, - ) - .map(|(fresh, _timing)| fresh) -} - -#[derive(Clone, Debug, Default)] -pub struct FreshPublicationPointTimingBreakdown { - pub current_index_lock_ms: u64, - pub manifest_load_ms: u64, - pub manifest_index_lookup_ms: u64, - pub manifest_blob_load_ms: u64, - pub manifest_decode_ms: u64, - pub replay_guard_ms: u64, - pub replay_meta_hit: bool, - pub replay_meta_miss: bool, - pub manifest_entries_ms: u64, - pub pack_files_ms: u64, - pub pack_files_index_lookup_ms: u64, - pub pack_files_blob_load_ms: u64, - pub ee_path_validate_ms: u64, - pub manifest_file_count: usize, -} - -pub fn process_manifest_publication_point_fresh_after_repo_sync_with_timing( - store: &RocksStore, - manifest_rsync_uri: &str, - publication_point_rsync_uri: &str, - current_repo_index: Option<&CurrentRepoIndexHandle>, - issuer_ca_der: &[u8], - issuer_ca_rsync_uri: Option<&str>, - validation_time: time::OffsetDateTime, - repo_sync_ok: bool, - repo_sync_error: Option<&str>, -) -> Result< - ( - FreshValidatedPublicationPoint, - FreshPublicationPointTimingBreakdown, - ), - ManifestFreshError, -> { - if repo_sync_ok { - try_build_fresh_publication_point_with_timing( - store, - manifest_rsync_uri, - publication_point_rsync_uri, - current_repo_index, - issuer_ca_der, - issuer_ca_rsync_uri, - validation_time, - ) - } else { - Err(ManifestFreshError::RepoSyncFailed { - detail: repo_sync_error.unwrap_or("repo sync failed").to_string(), - }) - } -} - -pub fn process_manifest_publication_point_after_repo_sync( - store: &RocksStore, - policy: &Policy, - manifest_rsync_uri: &str, - publication_point_rsync_uri: &str, - issuer_ca_der: &[u8], - issuer_ca_rsync_uri: Option<&str>, - validation_time: time::OffsetDateTime, - repo_sync_ok: bool, - repo_sync_error: Option<&str>, -) -> Result { - let fresh = if repo_sync_ok { - try_build_fresh_publication_point( - store, - manifest_rsync_uri, - publication_point_rsync_uri, - issuer_ca_der, - issuer_ca_rsync_uri, - validation_time, - ) - } else { - Err(ManifestFreshError::RepoSyncFailed { - detail: repo_sync_error.unwrap_or("repo sync failed").to_string(), - }) - }; - - match fresh { - Ok(fresh_point) => { - let snapshot = fresh_point.to_publication_point_snapshot(); - Ok(PublicationPointResult { - source: PublicationPointSource::Fresh, - snapshot, - warnings: Vec::new(), - }) - } - Err(fresh_err) => match policy.ca_failed_fetch_policy { - CaFailedFetchPolicy::StopAllOutput => { - Err(ManifestProcessError::StopAllOutput(fresh_err)) - } - CaFailedFetchPolicy::ReuseCurrentInstanceVcir => { - match load_current_instance_vcir_publication_point( - store, - manifest_rsync_uri, - publication_point_rsync_uri, - validation_time, - ) { - Ok(snapshot) => { - let mut warnings = Vec::new(); - if fresh_err.should_warn_when_current_instance_reused() { - warnings.push( - Warning::new(format!("manifest failed fetch: {fresh_err}")) - .with_rfc_refs(&[RfcRef("RFC 9286 §6.6")]) - .with_context(manifest_rsync_uri), - ); - } - Ok(PublicationPointResult { - source: PublicationPointSource::VcirCurrentInstance, - snapshot, - warnings, - }) - } - Err(reused) => Err(ManifestProcessError::NoUsableCache { - fresh: fresh_err, - reused, - }), - } - } - }, - } -} - -pub fn load_current_instance_vcir_publication_point( - store: &RocksStore, - manifest_rsync_uri: &str, - publication_point_rsync_uri: &str, - validation_time: time::OffsetDateTime, -) -> Result { - let vcir = store.get_vcir(manifest_rsync_uri)?.ok_or_else(|| { - ManifestReuseError::MissingCurrentInstanceVcir(manifest_rsync_uri.to_string()) - })?; - - if vcir.current_manifest_rsync_uri != manifest_rsync_uri { - return Err(ManifestReuseError::ManifestUriMismatch { - expected: manifest_rsync_uri.to_string(), - actual: vcir.current_manifest_rsync_uri.clone(), - }); - } - - if !vcir.audit_summary.failed_fetch_eligible { - return Err(ManifestReuseError::IneligibleCurrentInstanceVcir( - manifest_rsync_uri.to_string(), - )); - } - - let instance_effective_until = vcir - .instance_gate - .instance_effective_until - .parse() - .map_err(|e| { - ManifestReuseError::InvalidCurrentInstanceVcir(format!( - "instance_gate.instance_effective_until parse failed: {e}" - )) - })?; - if validation_time > instance_effective_until { - use time::format_description::well_known::Rfc3339; - return Err(ManifestReuseError::CurrentInstanceVcirExpired { - manifest_rsync_uri: manifest_rsync_uri.to_string(), - effective_until_rfc3339_utc: instance_effective_until - .to_offset(time::UtcOffset::UTC) - .format(&Rfc3339) - .expect("format VCIR instance_effective_until"), - validation_time_rfc3339_utc: validation_time - .to_offset(time::UtcOffset::UTC) - .format(&Rfc3339) - .expect("format validation_time"), - }); - } - - let manifest_artifact = vcir - .related_artifacts - .iter() - .find(|artifact| { - artifact.artifact_role == VcirArtifactRole::Manifest - && artifact.uri.as_deref() == Some(manifest_rsync_uri) - }) - .ok_or_else(|| { - ManifestReuseError::InvalidCurrentInstanceVcir( - "missing manifest artifact matching manifest_rsync_uri".to_string(), - ) - })?; - - let manifest_bytes = store - .get_blob_bytes(&manifest_artifact.sha256)? - .ok_or_else(|| ManifestReuseError::MissingManifestRaw(manifest_artifact.sha256.clone()))?; - - let mut seen = HashSet::new(); - let mut files = Vec::new(); - for artifact in &vcir.related_artifacts { - let Some(uri) = artifact.uri.as_ref() else { - continue; - }; - if artifact.artifact_role == VcirArtifactRole::Manifest - || artifact.artifact_role == VcirArtifactRole::IssuerCert - || artifact.artifact_role == VcirArtifactRole::Tal - || artifact.artifact_role == VcirArtifactRole::TrustAnchorCert - { - continue; - } - if !seen.insert(uri.clone()) { - continue; - } - let entry_bytes = store.get_blob_bytes(&artifact.sha256)?.ok_or_else(|| { - ManifestReuseError::MissingArtifactRaw { - rsync_uri: uri.clone(), - } - })?; - files.push(PackFile::from_bytes_compute_sha256(uri, entry_bytes)); - } - - Ok(PublicationPointSnapshot { - format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, - manifest_rsync_uri: manifest_rsync_uri.to_string(), - publication_point_rsync_uri: publication_point_rsync_uri.to_string(), - manifest_number_be: vcir - .validated_manifest_meta - .validated_manifest_number - .clone(), - this_update: vcir - .validated_manifest_meta - .validated_manifest_this_update - .clone(), - next_update: vcir - .validated_manifest_meta - .validated_manifest_next_update - .clone(), - verified_at: vcir.last_successful_validation_time.clone(), - manifest_bytes, - files, - }) -} - -fn decode_and_validate_manifest_with_current_time( - manifest_bytes: &[u8], - validation_time: time::OffsetDateTime, -) -> Result { - let manifest = ManifestObject::decode_der(manifest_bytes)?; - manifest.validate_embedded_ee_cert()?; - manifest.signed_object.verify()?; - - let this_update = manifest - .manifest - .this_update - .to_offset(time::UtcOffset::UTC); - let next_update = manifest - .manifest - .next_update - .to_offset(time::UtcOffset::UTC); - let now = validation_time.to_offset(time::UtcOffset::UTC); - if now < this_update || now > next_update { - return Err(ManifestFreshError::StaleOrEarly { - this_update_rfc3339_utc: this_update - .format(&time::format_description::well_known::Rfc3339) - .expect("format thisUpdate"), - next_update_rfc3339_utc: next_update - .format(&time::format_description::well_known::Rfc3339) - .expect("format nextUpdate"), - validation_time_rfc3339_utc: now - .format(&time::format_description::well_known::Rfc3339) - .expect("format validation_time"), - }); - } - - Ok(manifest) -} - -pub(crate) fn try_build_fresh_publication_point( - store: &RocksStore, - manifest_rsync_uri: &str, - publication_point_rsync_uri: &str, - issuer_ca_der: &[u8], - issuer_ca_rsync_uri: Option<&str>, - validation_time: time::OffsetDateTime, -) -> Result { - try_build_fresh_publication_point_with_timing( - store, - manifest_rsync_uri, - publication_point_rsync_uri, - None, - issuer_ca_der, - issuer_ca_rsync_uri, - validation_time, - ) - .map(|(fresh, _timing)| fresh) -} - -pub(crate) fn try_build_fresh_publication_point_with_timing( - store: &RocksStore, - manifest_rsync_uri: &str, - publication_point_rsync_uri: &str, - current_repo_index: Option<&CurrentRepoIndexHandle>, - issuer_ca_der: &[u8], - issuer_ca_rsync_uri: Option<&str>, - validation_time: time::OffsetDateTime, -) -> Result< - ( - FreshValidatedPublicationPoint, - FreshPublicationPointTimingBreakdown, - ), - ManifestFreshError, -> { - let mut timing = FreshPublicationPointTimingBreakdown::default(); - let current_index_lock_started = std::time::Instant::now(); - let current_index_guard = current_repo_index.and_then(|handle| handle.read().ok()); - timing.current_index_lock_ms = current_index_lock_started.elapsed().as_millis() as u64; - - if !rsync_uri_is_under_publication_point(manifest_rsync_uri, publication_point_rsync_uri) { - return Err(ManifestFreshError::ManifestOutsidePublicationPoint { - manifest_rsync_uri: manifest_rsync_uri.to_string(), - publication_point_rsync_uri: publication_point_rsync_uri.to_string(), - }); - } - - let manifest_load_started = std::time::Instant::now(); - let manifest_bytes = if let Some(index) = current_index_guard.as_ref() { - let manifest_lookup_started = std::time::Instant::now(); - let current = index.get_by_uri(manifest_rsync_uri).ok_or_else(|| { - ManifestFreshError::MissingManifest { - manifest_rsync_uri: manifest_rsync_uri.to_string(), - } - })?; - timing.manifest_index_lookup_ms = manifest_lookup_started.elapsed().as_millis() as u64; - let manifest_blob_load_started = std::time::Instant::now(); - store - .get_blob_bytes(¤t.current_hash_hex) - .map_err(|e| ManifestFreshError::MissingManifest { - manifest_rsync_uri: format!("{manifest_rsync_uri} ({e})"), - })? - .ok_or_else(|| ManifestFreshError::MissingManifest { - manifest_rsync_uri: manifest_rsync_uri.to_string(), - }) - .inspect(|_| { - timing.manifest_blob_load_ms = - manifest_blob_load_started.elapsed().as_millis() as u64; - })? - } else { - let manifest_blob_load_started = std::time::Instant::now(); - store - .load_current_object_bytes_by_uri(manifest_rsync_uri) - .map_err(|e| ManifestFreshError::MissingManifest { - manifest_rsync_uri: format!("{manifest_rsync_uri} ({e})"), - })? - .ok_or_else(|| ManifestFreshError::MissingManifest { - manifest_rsync_uri: manifest_rsync_uri.to_string(), - }) - .inspect(|_| { - timing.manifest_blob_load_ms = - manifest_blob_load_started.elapsed().as_millis() as u64; - })? - }; - timing.manifest_load_ms = manifest_load_started.elapsed().as_millis() as u64; - - let manifest_decode_started = std::time::Instant::now(); - let manifest = - decode_and_validate_manifest_with_current_time(&manifest_bytes, validation_time)?; - timing.manifest_decode_ms = manifest_decode_started.elapsed().as_millis() as u64; - - let this_update = manifest - .manifest - .this_update - .to_offset(time::UtcOffset::UTC); - let next_update = manifest - .manifest - .next_update - .to_offset(time::UtcOffset::UTC); - let now = validation_time.to_offset(time::UtcOffset::UTC); - - // RFC 9286 §4.2.1: replay/rollback detection for manifestNumber and thisUpdate. - // - // Important nuance for revalidation across runs: - // - If the manifestNumber is equal to the previously validated manifestNumber *and* the - // manifest bytes are identical, then this is the same manifest being revalidated and MUST - // be accepted (otherwise, RPs would incorrectly treat stable repositories as "failed fetch" - // and fall back to the current-instance VCIR snapshot). - // - If manifestNumber is equal but the manifest bytes differ, treat this as invalid (a - // repository is not allowed to change the manifest while keeping the manifestNumber). - // - If manifestNumber is lower, treat as rollback and reject. - // - If manifestNumber is higher, require thisUpdate to be more recent than the previously - // validated thisUpdate. - let replay_guard_started = std::time::Instant::now(); - if let Some(old_meta) = store - .get_manifest_replay_meta(manifest_rsync_uri) - .ok() - .flatten() - { - timing.replay_meta_hit = true; - if old_meta.manifest_rsync_uri == manifest_rsync_uri { - let new_num = manifest.manifest.manifest_number.bytes_be.as_slice(); - let old_num = old_meta.manifest_number_be.as_slice(); - match cmp_minimal_be_unsigned(new_num, old_num) { - Ordering::Greater => { - let old_this_update = old_meta - .manifest_this_update - .parse() - .expect("manifest replay meta validation ensures thisUpdate parses"); - if this_update <= old_this_update { - use time::format_description::well_known::Rfc3339; - return Err(ManifestFreshError::ThisUpdateNotIncreasing { - old_rfc3339_utc: old_this_update - .to_offset(time::UtcOffset::UTC) - .format(&Rfc3339) - .expect("format old thisUpdate"), - new_rfc3339_utc: this_update - .format(&Rfc3339) - .expect("format new thisUpdate"), - }); - } - } - Ordering::Equal => { - let new_manifest_hash = sha2::Sha256::digest(&manifest_bytes); - if old_meta.manifest_sha256.as_slice() != new_manifest_hash.as_slice() { - return Err(ManifestFreshError::ManifestNumberNotIncreasing { - old_hex: hex::encode_upper(old_num), - new_hex: hex::encode_upper(new_num), - }); - } - } - Ordering::Less => { - return Err(ManifestFreshError::ManifestNumberNotIncreasing { - old_hex: hex::encode_upper(old_num), - new_hex: hex::encode_upper(new_num), - }); - } - } - } - } else { - timing.replay_meta_miss = true; - } - timing.replay_guard_ms = replay_guard_started.elapsed().as_millis() as u64; - - let manifest_entries_started = std::time::Instant::now(); - let entries = manifest - .manifest - .parse_files() - .map_err(ManifestDecodeError::Validate)?; - timing.manifest_entries_ms = manifest_entries_started.elapsed().as_millis() as u64; - timing.manifest_file_count = entries.len(); - let mut files = Vec::with_capacity(manifest.manifest.file_count()); - let pack_files_started = std::time::Instant::now(); - let external_raw_store = store - .external_raw_store_ref() - .cloned() - .map(std::sync::Arc::new); - let external_repo_bytes = store - .external_repo_bytes_ref() - .cloned() - .map(std::sync::Arc::new); - let mut pack_files_index_lookup_duration = std::time::Duration::ZERO; - let mut pack_files_blob_load_duration = std::time::Duration::ZERO; - for entry in &entries { - let rsync_uri = - join_rsync_dir_and_file(publication_point_rsync_uri, entry.file_name.as_str()); - let current_object = if let Some(index) = current_index_guard.as_ref() { - let index_lookup_started = std::time::Instant::now(); - let current = - index - .get_by_uri(&rsync_uri) - .ok_or_else(|| ManifestFreshError::MissingFile { - rsync_uri: rsync_uri.clone(), - })?; - pack_files_index_lookup_duration += index_lookup_started.elapsed(); - crate::storage::CurrentObjectWithHash { - current_hash_hex: current.current_hash_hex.clone(), - current_hash: current.current_hash, - bytes: Vec::new(), - } - } else { - let blob_load_started = std::time::Instant::now(); - store - .load_current_object_with_hash_by_uri(&rsync_uri) - .map_err(|_e| ManifestFreshError::MissingFile { - rsync_uri: rsync_uri.clone(), - })? - .ok_or_else(|| ManifestFreshError::MissingFile { - rsync_uri: rsync_uri.clone(), - }) - .inspect(|_| { - pack_files_blob_load_duration += blob_load_started.elapsed(); - })? - }; - - if current_object.current_hash != entry.hash_bytes { - return Err(ManifestFreshError::HashMismatch { rsync_uri }); - } - - if let (Some(_), Some(repo_bytes)) = - (current_index_guard.as_ref(), external_repo_bytes.as_ref()) - { - files.push(PackFile::from_lazy_repo_bytes( - rsync_uri, - current_object.current_hash_hex, - current_object.current_hash, - repo_bytes.clone(), - )); - } else if let (Some(_), Some(raw_store)) = - (current_index_guard.as_ref(), external_raw_store.as_ref()) - { - files.push(PackFile::from_lazy_external_raw_store( - rsync_uri, - current_object.current_hash_hex, - current_object.current_hash, - raw_store.clone(), - )); - } else { - let bytes = if current_object.bytes.is_empty() { - let blob_load_started = std::time::Instant::now(); - store - .get_blob_bytes(¤t_object.current_hash_hex) - .map_err(|_e| ManifestFreshError::MissingFile { - rsync_uri: rsync_uri.clone(), - })? - .ok_or_else(|| ManifestFreshError::MissingFile { - rsync_uri: rsync_uri.clone(), - }) - .inspect(|_| { - pack_files_blob_load_duration += blob_load_started.elapsed(); - })? - } else { - current_object.bytes - }; - files.push(PackFile::from_bytes_with_sha256( - rsync_uri, - bytes, - current_object.current_hash, - )); - } - } - timing.pack_files_index_lookup_ms = pack_files_index_lookup_duration.as_millis() as u64; - timing.pack_files_blob_load_ms = pack_files_blob_load_duration.as_millis() as u64; - timing.pack_files_ms = pack_files_started.elapsed().as_millis() as u64; - - // RFC 6488 §3: manifest (signed object) validity includes a valid EE cert path. - // We validate this after §6.4/§6.5 so the issuer CRL can be selected from the publication point. - let ee_path_validate_started = std::time::Instant::now(); - validate_manifest_embedded_ee_cert_path( - &manifest, - &files, - issuer_ca_der, - issuer_ca_rsync_uri, - validation_time, - )?; - timing.ee_path_validate_ms = ee_path_validate_started.elapsed().as_millis() as u64; - - Ok(( - FreshValidatedPublicationPoint { - manifest_rsync_uri: manifest_rsync_uri.to_string(), - publication_point_rsync_uri: publication_point_rsync_uri.to_string(), - manifest_number_be: manifest.manifest.manifest_number.bytes_be.clone(), - this_update: PackTime::from_utc_offset_datetime(this_update), - next_update: PackTime::from_utc_offset_datetime(next_update), - verified_at: PackTime::from_utc_offset_datetime(now), - manifest_bytes, - files, - }, - timing, - )) -} - -fn cmp_minimal_be_unsigned(a: &[u8], b: &[u8]) -> Ordering { - // Compare two minimal big-endian byte strings as unsigned integers. - // (Leading zeros are not expected; callers store minimal big-endian.) - a.len().cmp(&b.len()).then_with(|| a.cmp(b)) -} - -fn join_rsync_dir_and_file(base: &str, file_name: &str) -> String { - if base.ends_with('/') { - format!("{base}{file_name}") - } else { - format!("{base}/{file_name}") - } -} - -fn rsync_uri_is_under_publication_point(uri: &str, publication_point_rsync_uri: &str) -> bool { - let pp = if publication_point_rsync_uri.ends_with('/') { - publication_point_rsync_uri.to_string() - } else { - format!("{publication_point_rsync_uri}/") - }; - uri.starts_with(&pp) -} - -fn validate_manifest_embedded_ee_cert_path( - manifest: &ManifestObject, - files: &[crate::storage::PackFile], - issuer_ca_der: &[u8], - issuer_ca_rsync_uri: Option<&str>, - validation_time: time::OffsetDateTime, -) -> Result<(), ManifestFreshError> { - let ee = &manifest.signed_object.signed_data.certificates[0]; - - let crl_files = files - .iter() - .filter(|f| f.rsync_uri.ends_with(".crl")) - .collect::>(); - if crl_files.is_empty() { - return Err(ManifestFreshError::NoCrlFiles); - } - - let Some(crldp_uris) = ee - .resource_cert - .tbs - .extensions - .crl_distribution_points_uris - .as_ref() - else { - return Err(ManifestFreshError::EeCrlDpMissing); - }; - - for u in crldp_uris { - let s = u.as_str(); - if let Some(f) = crl_files.iter().find(|f| f.rsync_uri == s) { - let crl_bytes = f.bytes().map_err(|e| ManifestFreshError::MissingFile { - rsync_uri: format!("{s} ({e})"), - })?; - let issuer_ca = crate::data_model::rc::ResourceCertificate::decode_der(issuer_ca_der) - .map_err(CertPathError::IssuerDecode)?; - let (rem, issuer_spki) = x509_parser::x509::SubjectPublicKeyInfo::from_der( - &issuer_ca.tbs.subject_public_key_info, - ) - .map_err(|e| CertPathError::IssuerSpkiParse(e.to_string()))?; - if !rem.is_empty() { - return Err(CertPathError::IssuerSpkiTrailingBytes(rem.len()).into()); - } - let issuer_crl = crate::data_model::crl::RpkixCrl::decode_der(crl_bytes) - .map_err(CertPathError::from)?; - let revoked_serials = issuer_crl - .revoked_certs - .iter() - .map(|rc| rc.serial_number.bytes_be.clone()) - .collect::>(); - validate_signed_object_ee_cert_path_fast( - ee, - &issuer_ca, - &issuer_spki, - &issuer_crl, - &revoked_serials, - issuer_ca_rsync_uri, - Some(f.rsync_uri.as_str()), - validation_time, - )?; - return Ok(()); - } - } - - Err(ManifestFreshError::EeCrlNotFound( - crldp_uris - .iter() - .map(|u| u.as_str()) - .collect::>() - .join(", "), - )) -} +include!("manifest/models_and_process.rs"); +include!("manifest/helpers.rs"); #[cfg(test)] -mod tests { - use super::*; - use crate::current_repo_index::CurrentRepoIndex; - use crate::data_model::manifest::ManifestObject; - use crate::storage::{ - PackFile, PackTime, RawByHashEntry, RocksStore, ValidatedCaInstanceResult, - ValidatedManifestMeta, VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus, - VcirAuditSummary, VcirCcrManifestProjection, VcirInstanceGate, VcirRelatedArtifact, - VcirSummary, - }; - use std::path::Path; - - fn manifest_fixture_path() -> &'static Path { - Path::new( - "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft", - ) - } - - fn issuer_ca_fixture_der() -> Vec { - std::fs::read( - "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ) - .expect("read issuer ca fixture") - } - - fn issuer_ca_rsync_uri() -> &'static str { - "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer" - } - - fn fixture_to_rsync_uri(path: &Path) -> String { - let rel = path - .strip_prefix("tests/fixtures/repository") - .expect("path under fixture repository"); - let mut it = rel.components(); - let host = it - .next() - .expect("host component") - .as_os_str() - .to_string_lossy(); - let rest = it.as_path().to_string_lossy(); - format!("rsync://{host}/{rest}") - } - - fn fixture_dir_to_rsync_uri(dir: &Path) -> String { - let mut s = fixture_to_rsync_uri(dir); - if !s.ends_with('/') { - s.push('/'); - } - s - } - - fn load_manifest_fixture() -> ( - ManifestObject, - Vec, - String, - String, - time::OffsetDateTime, - ) { - let manifest_path = manifest_fixture_path(); - let manifest_bytes = std::fs::read(manifest_path).expect("read manifest fixture"); - let manifest = ManifestObject::decode_der(&manifest_bytes).expect("decode manifest"); - let manifest_rsync_uri = fixture_to_rsync_uri(manifest_path); - let publication_point_rsync_uri = fixture_dir_to_rsync_uri(manifest_path.parent().unwrap()); - let validation_time = manifest.manifest.this_update + time::Duration::seconds(1); - ( - manifest, - manifest_bytes, - manifest_rsync_uri, - publication_point_rsync_uri, - validation_time, - ) - } - - fn raw_by_hash_entry(uri: &str, bytes: Vec, object_type: &str) -> RawByHashEntry { - let mut entry = - RawByHashEntry::from_bytes(hex::encode(sha2::Sha256::digest(&bytes)), bytes); - entry.origin_uris.push(uri.to_string()); - entry.object_type = Some(object_type.to_string()); - entry.encoding = Some("der".to_string()); - entry - } - - fn put_current_object(store: &RocksStore, rsync_uri: &str, bytes: Vec, object_type: &str) { - let hash = hex::encode(sha2::Sha256::digest(&bytes)); - store - .put_raw_by_hash_entry(&raw_by_hash_entry(rsync_uri, bytes, object_type)) - .expect("put raw_by_hash entry"); - store - .put_repository_view_entry(&crate::storage::RepositoryViewEntry { - rsync_uri: rsync_uri.to_string(), - current_hash: Some(hash), - repository_source: Some("https://example.test/notification.xml".to_string()), - object_type: Some(object_type.to_string()), - state: crate::storage::RepositoryViewState::Present, - }) - .expect("put repository view entry"); - } - - fn put_complete_publication_point_current_objects( - store: &RocksStore, - manifest: &ManifestObject, - manifest_rsync_uri: &str, - manifest_bytes: Vec, - publication_point_rsync_uri: &str, - ) { - put_current_object(store, manifest_rsync_uri, manifest_bytes, "mft"); - for entry in manifest.manifest.parse_files().expect("parse files") { - let file_path = manifest_fixture_path() - .parent() - .unwrap() - .join(entry.file_name.as_str()); - let bytes = std::fs::read(&file_path).expect("read fixture file"); - let rsync_uri = format!("{publication_point_rsync_uri}{}", entry.file_name); - let object_type = rsync_uri.rsplit('.').next().unwrap_or("bin"); - put_current_object(store, &rsync_uri, bytes, object_type); - } - } - - fn put_raw_only(store: &RocksStore, rsync_uri: &str, bytes: Vec, object_type: &str) { - store - .put_raw_by_hash_entry(&raw_by_hash_entry(rsync_uri, bytes, object_type)) - .expect("put raw_by_hash entry"); - } - - fn sample_vcir_for_manifest_replay_meta( - manifest: &ManifestObject, - manifest_rsync_uri: &str, - publication_point_rsync_uri: &str, - manifest_bytes: &[u8], - validation_time: time::OffsetDateTime, - ) -> ValidatedCaInstanceResult { - let manifest_hash = hex::encode(sha2::Sha256::digest(manifest_bytes)); - let this_update = manifest - .manifest - .this_update - .to_offset(time::UtcOffset::UTC); - let mut vcir = sample_current_instance_vcir( - manifest_rsync_uri, - publication_point_rsync_uri, - &manifest_hash, - "rsync://example.test/repo/object.roa", - &hex::encode(sha2::Sha256::digest(b"object")), - validation_time, - true, - ); - vcir.validated_manifest_meta.validated_manifest_number = - manifest.manifest.manifest_number.bytes_be.clone(); - vcir.validated_manifest_meta.validated_manifest_this_update = - PackTime::from_utc_offset_datetime(this_update); - vcir.ccr_manifest_projection.manifest_number_be = - manifest.manifest.manifest_number.bytes_be.clone(); - vcir.ccr_manifest_projection.manifest_this_update = - PackTime::from_utc_offset_datetime(this_update); - vcir.ccr_manifest_projection.manifest_sha256 = - hex::decode(manifest_hash).expect("decode manifest hash"); - vcir.ccr_manifest_projection.manifest_size = manifest_bytes.len() as u64; - vcir - } - - fn sample_current_instance_vcir( - manifest_rsync_uri: &str, - publication_point_rsync_uri: &str, - manifest_sha256: &str, - locked_object_uri: &str, - locked_object_sha256: &str, - validation_time: time::OffsetDateTime, - failed_fetch_eligible: bool, - ) -> ValidatedCaInstanceResult { - let gate_time = - PackTime::from_utc_offset_datetime(validation_time + time::Duration::hours(1)); - let ccr_manifest_projection = VcirCcrManifestProjection { - manifest_rsync_uri: manifest_rsync_uri.to_string(), - manifest_sha256: hex::decode(manifest_sha256).expect("decode manifest sha256"), - manifest_size: 2048, - manifest_ee_aki: vec![0x11; 20], - manifest_number_be: vec![1], - manifest_this_update: PackTime::from_utc_offset_datetime(validation_time), - manifest_sia_locations_der: vec![vec![ - 0x30, 0x11, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x05, 0x86, 0x05, - b'r', b's', b'y', b'n', b'c', - ]], - subordinate_skis: Vec::new(), - }; - ValidatedCaInstanceResult { - manifest_rsync_uri: manifest_rsync_uri.to_string(), - parent_manifest_rsync_uri: None, - tal_id: "test-tal".to_string(), - ca_subject_name: "CN=test".to_string(), - ca_ski: "00112233445566778899aabbccddeeff00112233".to_string(), - issuer_ski: "00112233445566778899aabbccddeeff00112233".to_string(), - last_successful_validation_time: PackTime::from_utc_offset_datetime(validation_time), - current_manifest_rsync_uri: manifest_rsync_uri.to_string(), - current_crl_rsync_uri: format!("{publication_point_rsync_uri}current.crl"), - validated_manifest_meta: ValidatedManifestMeta { - validated_manifest_number: vec![1], - validated_manifest_this_update: PackTime::from_utc_offset_datetime(validation_time), - validated_manifest_next_update: gate_time.clone(), - }, - ccr_manifest_projection, - instance_gate: VcirInstanceGate { - manifest_next_update: gate_time.clone(), - current_crl_next_update: gate_time.clone(), - self_ca_not_after: gate_time.clone(), - instance_effective_until: gate_time, - }, - child_entries: Vec::new(), - local_outputs: Vec::new(), - related_artifacts: vec![ - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::Manifest, - artifact_kind: VcirArtifactKind::Mft, - uri: Some(manifest_rsync_uri.to_string()), - sha256: manifest_sha256.to_string(), - object_type: Some("mft".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - reject_reason: None, - }, - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::SignedObject, - artifact_kind: VcirArtifactKind::Roa, - uri: Some(locked_object_uri.to_string()), - sha256: locked_object_sha256.to_string(), - object_type: Some("roa".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - reject_reason: None, - }, - ], - summary: VcirSummary { - local_vrp_count: 0, - local_aspa_count: 0, - local_router_key_count: 0, - child_count: 0, - accepted_object_count: 2, - rejected_object_count: 0, - }, - audit_summary: VcirAuditSummary { - failed_fetch_eligible, - last_failed_fetch_reason: None, - warning_count: 0, - audit_flags: Vec::new(), - }, - } - } - fn locked_files_for_manifest( - manifest: &ManifestObject, - publication_point_rsync_uri: &str, - ) -> Vec { - let manifest_path = manifest_fixture_path(); - manifest - .manifest - .parse_files() - .expect("parse files") - .into_iter() - .map(|entry| { - let file_path = manifest_path - .parent() - .unwrap() - .join(entry.file_name.as_str()); - let bytes = std::fs::read(&file_path).unwrap_or_else(|_| { - panic!("read fixture file referenced by manifest: {file_path:?}") - }); - PackFile::from_bytes_compute_sha256( - format!("{publication_point_rsync_uri}{}", entry.file_name), - bytes, - ) - }) - .collect() - } - - #[test] - fn try_build_fresh_publication_point_rejects_manifest_outside_publication_point() { - let temp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(temp.path()).expect("open rocksdb"); - let (_, _, manifest_rsync_uri, _, validation_time) = load_manifest_fixture(); - let err = try_build_fresh_publication_point( - &store, - &manifest_rsync_uri, - "rsync://example.test/other/", - &issuer_ca_fixture_der(), - Some(issuer_ca_rsync_uri()), - validation_time, - ) - .unwrap_err(); - assert!( - matches!( - err, - ManifestFreshError::ManifestOutsidePublicationPoint { .. } - ), - "{err}" - ); - } - - #[test] - fn try_build_fresh_publication_point_reports_missing_manifest_when_raw_store_is_empty() { - let temp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(temp.path()).expect("open rocksdb"); - let (_, _, manifest_rsync_uri, publication_point_rsync_uri, validation_time) = - load_manifest_fixture(); - let err = try_build_fresh_publication_point( - &store, - &manifest_rsync_uri, - &publication_point_rsync_uri, - &issuer_ca_fixture_der(), - Some(issuer_ca_rsync_uri()), - validation_time, - ) - .unwrap_err(); - assert!( - matches!(err, ManifestFreshError::MissingManifest { .. }), - "{err}" - ); - } - - #[test] - fn try_build_fresh_publication_point_reports_missing_locked_file() { - let temp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(temp.path()).expect("open rocksdb"); - let ( - manifest, - manifest_bytes, - manifest_rsync_uri, - publication_point_rsync_uri, - validation_time, - ) = load_manifest_fixture(); - put_current_object(&store, &manifest_rsync_uri, manifest_bytes, "mft"); - let first_non_crl = manifest - .manifest - .parse_files() - .expect("parse files") - .into_iter() - .find(|entry| !entry.file_name.ends_with(".crl")) - .expect("fixture non-crl entry"); - let file_path = manifest_fixture_path() - .parent() - .unwrap() - .join(first_non_crl.file_name.as_str()); - let bytes = std::fs::read(&file_path).expect("read fixture file"); - let rsync_uri = format!("{publication_point_rsync_uri}{}", first_non_crl.file_name); - let object_type = rsync_uri.rsplit('.').next().unwrap_or("bin"); - put_current_object(&store, &rsync_uri, bytes, object_type); - - let err = try_build_fresh_publication_point( - &store, - &manifest_rsync_uri, - &publication_point_rsync_uri, - &issuer_ca_fixture_der(), - Some(issuer_ca_rsync_uri()), - validation_time, - ) - .unwrap_err(); - assert!( - matches!(err, ManifestFreshError::MissingFile { .. }), - "{err}" - ); - } - - #[test] - fn try_build_fresh_publication_point_detects_hash_mismatch_via_repository_view_hash() { - let temp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(temp.path()).expect("open rocksdb"); - let ( - manifest, - manifest_bytes, - manifest_rsync_uri, - publication_point_rsync_uri, - validation_time, - ) = load_manifest_fixture(); - put_current_object(&store, &manifest_rsync_uri, manifest_bytes, "mft"); - - let non_crl_entries = manifest - .manifest - .parse_files() - .expect("parse files") - .into_iter() - .filter(|entry| !entry.file_name.ends_with(".crl")) - .collect::>(); - let first = &non_crl_entries[0]; - let second = &non_crl_entries[1]; - - let first_uri = format!("{publication_point_rsync_uri}{}", first.file_name); - let second_path = manifest_fixture_path() - .parent() - .unwrap() - .join(second.file_name.as_str()); - let wrong_bytes = std::fs::read(&second_path).expect("read wrong fixture file"); - let object_type = first_uri.rsplit('.').next().unwrap_or("bin"); - put_current_object(&store, &first_uri, wrong_bytes, object_type); - - for entry in non_crl_entries.iter().skip(1) { - let file_path = manifest_fixture_path() - .parent() - .unwrap() - .join(entry.file_name.as_str()); - let bytes = std::fs::read(&file_path).expect("read fixture file"); - let rsync_uri = format!("{publication_point_rsync_uri}{}", entry.file_name); - let object_type = rsync_uri.rsplit('.').next().unwrap_or("bin"); - put_current_object(&store, &rsync_uri, bytes, object_type); - } - - let err = try_build_fresh_publication_point( - &store, - &manifest_rsync_uri, - &publication_point_rsync_uri, - &issuer_ca_fixture_der(), - Some(issuer_ca_rsync_uri()), - validation_time, - ) - .unwrap_err(); - assert!( - matches!(err, ManifestFreshError::HashMismatch { .. }), - "{err}" - ); - } - - #[test] - fn try_build_fresh_publication_point_uses_current_repo_index_without_repository_view() { - let temp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(temp.path()).expect("open rocksdb"); - let ( - manifest, - manifest_bytes, - manifest_rsync_uri, - publication_point_rsync_uri, - validation_time, - ) = load_manifest_fixture(); - - put_raw_only(&store, &manifest_rsync_uri, manifest_bytes.clone(), "mft"); - let current_index = CurrentRepoIndex::shared(); - let mut entries = vec![crate::storage::RepositoryViewEntry { - rsync_uri: manifest_rsync_uri.clone(), - current_hash: Some(hex::encode(sha2::Sha256::digest(&manifest_bytes))), - repository_source: Some("https://example.test/notification.xml".to_string()), - object_type: Some("mft".to_string()), - state: crate::storage::RepositoryViewState::Present, - }]; - - for entry in manifest.manifest.parse_files().expect("parse files") { - let file_path = manifest_fixture_path() - .parent() - .unwrap() - .join(entry.file_name.as_str()); - let bytes = std::fs::read(&file_path).expect("read fixture file"); - let rsync_uri = format!("{publication_point_rsync_uri}{}", entry.file_name); - let object_type = rsync_uri.rsplit('.').next().unwrap_or("bin").to_string(); - put_raw_only(&store, &rsync_uri, bytes.clone(), &object_type); - entries.push(crate::storage::RepositoryViewEntry { - rsync_uri, - current_hash: Some(hex::encode(sha2::Sha256::digest(&bytes))), - repository_source: Some("https://example.test/notification.xml".to_string()), - object_type: Some(object_type), - state: crate::storage::RepositoryViewState::Present, - }); - } - - current_index - .write() - .expect("index write lock") - .apply_repository_view_entries(&entries) - .expect("apply current index"); - - assert!( - store - .get_repository_view_entry(&manifest_rsync_uri) - .expect("get repository view") - .is_none() - ); - - let (fresh, _timing) = try_build_fresh_publication_point_with_timing( - &store, - &manifest_rsync_uri, - &publication_point_rsync_uri, - Some(¤t_index), - &issuer_ca_fixture_der(), - Some(issuer_ca_rsync_uri()), - validation_time, - ) - .expect("fresh publication point via current index"); - - assert_eq!(fresh.manifest_rsync_uri, manifest_rsync_uri); - assert_eq!(fresh.files.len(), manifest.manifest.file_count()); - } - - #[test] - fn try_build_fresh_publication_point_records_replay_meta_miss() { - let temp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(temp.path()).expect("open rocksdb"); - let ( - manifest, - manifest_bytes, - manifest_rsync_uri, - publication_point_rsync_uri, - validation_time, - ) = load_manifest_fixture(); - put_complete_publication_point_current_objects( - &store, - &manifest, - &manifest_rsync_uri, - manifest_bytes, - &publication_point_rsync_uri, - ); - - let (_fresh, timing) = try_build_fresh_publication_point_with_timing( - &store, - &manifest_rsync_uri, - &publication_point_rsync_uri, - None, - &issuer_ca_fixture_der(), - Some(issuer_ca_rsync_uri()), - validation_time, - ) - .expect("fresh publication point without replay meta"); - - assert!(timing.replay_meta_miss); - assert!(!timing.replay_meta_hit); - } - - #[test] - fn try_build_fresh_publication_point_uses_replay_meta_hit_for_same_manifest() { - let temp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(temp.path()).expect("open rocksdb"); - let ( - manifest, - manifest_bytes, - manifest_rsync_uri, - publication_point_rsync_uri, - validation_time, - ) = load_manifest_fixture(); - let previous_vcir = sample_vcir_for_manifest_replay_meta( - &manifest, - &manifest_rsync_uri, - &publication_point_rsync_uri, - &manifest_bytes, - validation_time, - ); - store.put_vcir(&previous_vcir).expect("put previous vcir"); - put_complete_publication_point_current_objects( - &store, - &manifest, - &manifest_rsync_uri, - manifest_bytes, - &publication_point_rsync_uri, - ); - - let (_fresh, timing) = try_build_fresh_publication_point_with_timing( - &store, - &manifest_rsync_uri, - &publication_point_rsync_uri, - None, - &issuer_ca_fixture_der(), - Some(issuer_ca_rsync_uri()), - validation_time, - ) - .expect("fresh publication point with matching replay meta"); - - assert!(timing.replay_meta_hit); - assert!(!timing.replay_meta_miss); - } - - #[test] - fn validate_manifest_embedded_ee_cert_path_rejects_missing_crl_files() { - let (manifest, _, _, publication_point_rsync_uri, validation_time) = - load_manifest_fixture(); - let files = locked_files_for_manifest(&manifest, &publication_point_rsync_uri) - .into_iter() - .filter(|f| !f.rsync_uri.ends_with(".crl")) - .collect::>(); - - let err = validate_manifest_embedded_ee_cert_path( - &manifest, - &files, - &issuer_ca_fixture_der(), - Some(issuer_ca_rsync_uri()), - validation_time, - ) - .unwrap_err(); - assert!(matches!(err, ManifestFreshError::NoCrlFiles), "{err}"); - } - - #[test] - fn validate_manifest_embedded_ee_cert_path_rejects_missing_ee_crldp() { - let (mut manifest, _, _, publication_point_rsync_uri, validation_time) = - load_manifest_fixture(); - manifest.signed_object.signed_data.certificates[0] - .resource_cert - .tbs - .extensions - .crl_distribution_points_uris = None; - let files = locked_files_for_manifest(&manifest, &publication_point_rsync_uri); - - let err = validate_manifest_embedded_ee_cert_path( - &manifest, - &files, - &issuer_ca_fixture_der(), - Some(issuer_ca_rsync_uri()), - validation_time, - ) - .unwrap_err(); - assert!(matches!(err, ManifestFreshError::EeCrlDpMissing), "{err}"); - } - - #[test] - fn validate_manifest_embedded_ee_cert_path_rejects_unlisted_crldp_uri() { - let (manifest, _, _, publication_point_rsync_uri, validation_time) = - load_manifest_fixture(); - let mut files = locked_files_for_manifest(&manifest, &publication_point_rsync_uri) - .into_iter() - .filter(|f| !f.rsync_uri.ends_with(".crl")) - .collect::>(); - files.push(PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/unrelated.crl", - b"dummy".to_vec(), - )); - - let err = validate_manifest_embedded_ee_cert_path( - &manifest, - &files, - &issuer_ca_fixture_der(), - Some(issuer_ca_rsync_uri()), - validation_time, - ) - .unwrap_err(); - assert!(matches!(err, ManifestFreshError::EeCrlNotFound(_)), "{err}"); - } - - #[test] - fn validate_manifest_embedded_ee_cert_path_rejects_expired_crl() { - let (manifest, _, _, publication_point_rsync_uri, _) = load_manifest_fixture(); - let files = locked_files_for_manifest(&manifest, &publication_point_rsync_uri); - let ee = &manifest.signed_object.signed_data.certificates[0]; - let crldp_uri = ee - .resource_cert - .tbs - .extensions - .crl_distribution_points_uris - .as_ref() - .and_then(|uris| uris.first()) - .expect("fixture manifest EE CRLDP") - .as_str() - .to_string(); - let crl_file = files - .iter() - .find(|file| file.rsync_uri == crldp_uri) - .expect("fixture CRL referenced by manifest EE"); - let crl = crate::data_model::crl::RpkixCrl::decode_der( - crl_file.bytes().expect("read fixture crl bytes"), - ) - .expect("decode fixture crl"); - let validation_time = crl.next_update.utc; - - let err = validate_manifest_embedded_ee_cert_path( - &manifest, - &files, - &issuer_ca_fixture_der(), - Some(issuer_ca_rsync_uri()), - validation_time, - ) - .unwrap_err(); - - assert!( - matches!( - err, - ManifestFreshError::EeCertPath( - crate::validation::cert_path::CertPathError::CrlNotValidAtTime - ) - ), - "{err}" - ); - } - - #[test] - fn load_current_instance_vcir_publication_point_returns_manifest_and_locked_files() { - let temp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(temp.path()).expect("open rocksdb"); - let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::hours(1); - let manifest_uri = "rsync://example.test/repo/current.mft"; - let publication_point_uri = "rsync://example.test/repo/"; - let locked_uri = "rsync://example.test/repo/object.roa"; - let manifest_bytes = vec![0x30, 0x31, 0x32]; - let locked_bytes = vec![0x01, 0x02, 0x03, 0x04]; - let manifest_entry = raw_by_hash_entry(manifest_uri, manifest_bytes.clone(), "mft"); - let locked_entry = raw_by_hash_entry(locked_uri, locked_bytes.clone(), "roa"); - store - .put_raw_by_hash_entry(&manifest_entry) - .expect("put manifest raw_by_hash"); - store - .put_raw_by_hash_entry(&locked_entry) - .expect("put locked raw_by_hash"); - let vcir = sample_current_instance_vcir( - manifest_uri, - publication_point_uri, - &manifest_entry.sha256_hex, - locked_uri, - &locked_entry.sha256_hex, - validation_time, - true, - ); - store.put_vcir(&vcir).expect("put vcir"); - - let point = load_current_instance_vcir_publication_point( - &store, - manifest_uri, - publication_point_uri, - validation_time, - ) - .expect("load current-instance vcir publication point"); - assert_eq!(point.manifest_bytes, manifest_bytes); - assert_eq!(point.files.len(), 1); - assert_eq!(point.files[0].rsync_uri, locked_uri); - assert_eq!( - point.files[0].bytes_cloned().expect("locked bytes"), - locked_bytes - ); - } - - #[test] - fn load_current_instance_vcir_publication_point_rejects_ineligible_vcir() { - let temp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(temp.path()).expect("open rocksdb"); - let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::hours(1); - let manifest_uri = "rsync://example.test/repo/current.mft"; - let publication_point_uri = "rsync://example.test/repo/"; - let locked_uri = "rsync://example.test/repo/object.roa"; - let manifest_entry = raw_by_hash_entry(manifest_uri, vec![0x30], "mft"); - let locked_entry = raw_by_hash_entry(locked_uri, vec![0x01], "roa"); - store - .put_raw_by_hash_entry(&manifest_entry) - .expect("put manifest raw_by_hash"); - store - .put_raw_by_hash_entry(&locked_entry) - .expect("put locked raw_by_hash"); - let vcir = sample_current_instance_vcir( - manifest_uri, - publication_point_uri, - &manifest_entry.sha256_hex, - locked_uri, - &locked_entry.sha256_hex, - validation_time, - false, - ); - store.put_vcir(&vcir).expect("put vcir"); - - let err = load_current_instance_vcir_publication_point( - &store, - manifest_uri, - publication_point_uri, - validation_time, - ) - .unwrap_err(); - assert!( - matches!(err, ManifestReuseError::IneligibleCurrentInstanceVcir(_)), - "{err}" - ); - } - - #[test] - fn load_current_instance_vcir_publication_point_rejects_expired_vcir() { - let temp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(temp.path()).expect("open rocksdb"); - let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::hours(2); - let manifest_uri = "rsync://example.test/repo/current.mft"; - let publication_point_uri = "rsync://example.test/repo/"; - let locked_uri = "rsync://example.test/repo/object.roa"; - let manifest_entry = raw_by_hash_entry(manifest_uri, vec![0x30], "mft"); - let locked_entry = raw_by_hash_entry(locked_uri, vec![0x01], "roa"); - store - .put_raw_by_hash_entry(&manifest_entry) - .expect("put manifest raw_by_hash"); - store - .put_raw_by_hash_entry(&locked_entry) - .expect("put locked raw_by_hash"); - let mut vcir = sample_current_instance_vcir( - manifest_uri, - publication_point_uri, - &manifest_entry.sha256_hex, - locked_uri, - &locked_entry.sha256_hex, - validation_time - time::Duration::hours(2), - true, - ); - let expired = - PackTime::from_utc_offset_datetime(validation_time - time::Duration::minutes(1)); - vcir.instance_gate.manifest_next_update = expired.clone(); - vcir.instance_gate.current_crl_next_update = expired.clone(); - vcir.instance_gate.self_ca_not_after = expired.clone(); - vcir.instance_gate.instance_effective_until = expired; - vcir.validated_manifest_meta.validated_manifest_next_update = - vcir.instance_gate.instance_effective_until.clone(); - store.put_vcir(&vcir).expect("put expired vcir"); - - let err = load_current_instance_vcir_publication_point( - &store, - manifest_uri, - publication_point_uri, - validation_time, - ) - .unwrap_err(); - assert!( - matches!(err, ManifestReuseError::CurrentInstanceVcirExpired { .. }), - "{err}" - ); - } - - #[test] - fn load_current_instance_vcir_publication_point_rejects_missing_locked_artifact_raw() { - let temp = tempfile::tempdir().expect("tempdir"); - let store = RocksStore::open(temp.path()).expect("open rocksdb"); - let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::hours(1); - let manifest_uri = "rsync://example.test/repo/current.mft"; - let publication_point_uri = "rsync://example.test/repo/"; - let locked_uri = "rsync://example.test/repo/object.roa"; - let manifest_entry = raw_by_hash_entry(manifest_uri, vec![0x30], "mft"); - store - .put_raw_by_hash_entry(&manifest_entry) - .expect("put manifest raw_by_hash"); - let vcir = sample_current_instance_vcir( - manifest_uri, - publication_point_uri, - &manifest_entry.sha256_hex, - locked_uri, - &hex::encode(sha2::Sha256::digest([0x01, 0x02])), - validation_time, - true, - ); - store.put_vcir(&vcir).expect("put vcir"); - - let err = load_current_instance_vcir_publication_point( - &store, - manifest_uri, - publication_point_uri, - validation_time, - ) - .unwrap_err(); - assert!( - matches!(err, ManifestReuseError::MissingArtifactRaw { .. }), - "{err}" - ); - } -} +#[path = "manifest/tests.rs"] +mod tests; diff --git a/crates/panda-rpki-validator/src/validation/manifest/helpers.rs b/crates/panda-rpki-validator/src/validation/manifest/helpers.rs new file mode 100644 index 0000000..cd78954 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/manifest/helpers.rs @@ -0,0 +1,96 @@ +// Manifest URI and embedded certificate validation helpers. + +fn cmp_minimal_be_unsigned(a: &[u8], b: &[u8]) -> Ordering { + // Compare two minimal big-endian byte strings as unsigned integers. + // (Leading zeros are not expected; callers store minimal big-endian.) + a.len().cmp(&b.len()).then_with(|| a.cmp(b)) +} + +fn join_rsync_dir_and_file(base: &str, file_name: &str) -> String { + if base.ends_with('/') { + format!("{base}{file_name}") + } else { + format!("{base}/{file_name}") + } +} + +fn rsync_uri_is_under_publication_point(uri: &str, publication_point_rsync_uri: &str) -> bool { + let pp = if publication_point_rsync_uri.ends_with('/') { + publication_point_rsync_uri.to_string() + } else { + format!("{publication_point_rsync_uri}/") + }; + uri.starts_with(&pp) +} + +fn validate_manifest_embedded_ee_cert_path( + manifest: &ManifestObject, + files: &[crate::storage::PackFile], + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + validation_time: time::OffsetDateTime, +) -> Result<(), ManifestFreshError> { + let ee = &manifest.signed_object.signed_data.certificates[0]; + + let crl_files = files + .iter() + .filter(|f| f.rsync_uri.ends_with(".crl")) + .collect::>(); + if crl_files.is_empty() { + return Err(ManifestFreshError::NoCrlFiles); + } + + let Some(crldp_uris) = ee + .resource_cert + .tbs + .extensions + .crl_distribution_points_uris + .as_ref() + else { + return Err(ManifestFreshError::EeCrlDpMissing); + }; + + for u in crldp_uris { + let s = u.as_str(); + if let Some(f) = crl_files.iter().find(|f| f.rsync_uri == s) { + let crl_bytes = f.bytes().map_err(|e| ManifestFreshError::MissingFile { + rsync_uri: format!("{s} ({e})"), + })?; + let issuer_ca = crate::data_model::rc::ResourceCertificate::decode_der(issuer_ca_der) + .map_err(CertPathError::IssuerDecode)?; + let (rem, issuer_spki) = x509_parser::x509::SubjectPublicKeyInfo::from_der( + &issuer_ca.tbs.subject_public_key_info, + ) + .map_err(|e| CertPathError::IssuerSpkiParse(e.to_string()))?; + if !rem.is_empty() { + return Err(CertPathError::IssuerSpkiTrailingBytes(rem.len()).into()); + } + let issuer_crl = crate::data_model::crl::RpkixCrl::decode_der(crl_bytes) + .map_err(CertPathError::from)?; + let revoked_serials = issuer_crl + .revoked_certs + .iter() + .map(|rc| rc.serial_number.bytes_be.clone()) + .collect::>(); + validate_signed_object_ee_cert_path_fast( + ee, + &issuer_ca, + &issuer_spki, + &issuer_crl, + &revoked_serials, + issuer_ca_rsync_uri, + Some(f.rsync_uri.as_str()), + validation_time, + )?; + return Ok(()); + } + } + + Err(ManifestFreshError::EeCrlNotFound( + crldp_uris + .iter() + .map(|u| u.as_str()) + .collect::>() + .join(", "), + )) +} diff --git a/crates/panda-rpki-validator/src/validation/manifest/models_and_process.rs b/crates/panda-rpki-validator/src/validation/manifest/models_and_process.rs new file mode 100644 index 0000000..8e80610 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/manifest/models_and_process.rs @@ -0,0 +1,878 @@ +// Publication-point models and manifest processing pipeline. + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PublicationPointSource { + Fresh, + PublicationPointCache, + VcirCurrentInstance, + FailedFetchNoCache, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PublicationPointResult { + pub source: PublicationPointSource, + pub snapshot: PublicationPointSnapshot, + pub warnings: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FreshValidatedPublicationPoint { + pub manifest_rsync_uri: String, + pub publication_point_rsync_uri: String, + pub manifest_number_be: Vec, + pub this_update: PackTime, + pub next_update: PackTime, + pub verified_at: PackTime, + pub manifest_bytes: Vec, + pub files: Vec, +} + +pub trait PublicationPointData { + fn manifest_rsync_uri(&self) -> &str; + fn publication_point_rsync_uri(&self) -> &str; + fn manifest_number_be(&self) -> &[u8]; + fn this_update(&self) -> &PackTime; + fn next_update(&self) -> &PackTime; + fn verified_at(&self) -> &PackTime; + fn manifest_bytes(&self) -> &[u8]; + fn files(&self) -> &[PackFile]; +} + +impl FreshValidatedPublicationPoint { + pub fn to_publication_point_snapshot(&self) -> PublicationPointSnapshot { + PublicationPointSnapshot { + format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, + manifest_rsync_uri: self.manifest_rsync_uri.clone(), + publication_point_rsync_uri: self.publication_point_rsync_uri.clone(), + manifest_number_be: self.manifest_number_be.clone(), + this_update: self.this_update.clone(), + next_update: self.next_update.clone(), + verified_at: self.verified_at.clone(), + manifest_bytes: self.manifest_bytes.clone(), + files: self.files.clone(), + } + } +} + +impl PublicationPointData for FreshValidatedPublicationPoint { + fn manifest_rsync_uri(&self) -> &str { + &self.manifest_rsync_uri + } + + fn publication_point_rsync_uri(&self) -> &str { + &self.publication_point_rsync_uri + } + + fn manifest_number_be(&self) -> &[u8] { + self.manifest_number_be.as_slice() + } + + fn this_update(&self) -> &PackTime { + &self.this_update + } + + fn next_update(&self) -> &PackTime { + &self.next_update + } + + fn verified_at(&self) -> &PackTime { + &self.verified_at + } + + fn manifest_bytes(&self) -> &[u8] { + self.manifest_bytes.as_slice() + } + + fn files(&self) -> &[PackFile] { + self.files.as_slice() + } +} + +impl PublicationPointData for PublicationPointSnapshot { + fn manifest_rsync_uri(&self) -> &str { + &self.manifest_rsync_uri + } + + fn publication_point_rsync_uri(&self) -> &str { + &self.publication_point_rsync_uri + } + + fn manifest_number_be(&self) -> &[u8] { + self.manifest_number_be.as_slice() + } + + fn this_update(&self) -> &PackTime { + &self.this_update + } + + fn next_update(&self) -> &PackTime { + &self.next_update + } + + fn verified_at(&self) -> &PackTime { + &self.verified_at + } + + fn manifest_bytes(&self) -> &[u8] { + self.manifest_bytes.as_slice() + } + + fn files(&self) -> &[PackFile] { + self.files.as_slice() + } +} + +#[derive(Debug, thiserror::Error)] +pub enum ManifestFreshError { + #[error("repo sync failed: {detail} (RFC 8182 §3.4.5; RFC 9286 §6.6)")] + RepoSyncFailed { detail: String }, + + #[error( + "manifest not found in current repository view: {manifest_rsync_uri} (RFC 9286 §6.2; RFC 9286 §6.6)" + )] + MissingManifest { manifest_rsync_uri: String }, + + #[error("manifest decode failed: {0} (RFC 9286 §4; RFC 9286 §6.2; RFC 9286 §6.6)")] + Decode(#[from] ManifestDecodeError), + + #[error( + "manifest embedded EE certificate resources invalid: {0} (RFC 9286 §5.1; RFC 9286 §6.2; RFC 9286 §6.6)" + )] + EeResources(#[from] ManifestValidateError), + + #[error( + "manifest CMS signature verification failed: {0} (RFC 6488 §3; RFC 9589 §4; RFC 9286 §6.2; RFC 9286 §6.6)" + )] + Signature(#[from] SignedObjectVerifyError), + + #[error( + "manifest embedded EE certificate path validation failed: {0} (RFC 6488 §3; RFC 9286 §6.2; RFC 9286 §6.6)" + )] + EeCertPath(#[from] CertPathError), + + #[error( + "manifest embedded EE certificate CRLDistributionPoints missing (cannot validate EE certificate) (RFC 6487 §4.8.6; RFC 6488 §3; RFC 9286 §6.2; RFC 9286 §6.6)" + )] + EeCrlDpMissing, + + #[error( + "publication point contains no CRL files (cannot validate manifest EE certificate) (RFC 9286 §7; RFC 6487 §4.8.6; RFC 6488 §3; RFC 9286 §6.2; RFC 9286 §6.6)" + )] + NoCrlFiles, + + #[error( + "CRL referenced by manifest embedded EE certificate CRLDistributionPoints not found at publication point: {0} (RFC 6487 §4.8.6; RFC 9286 §4.2.1; RFC 9286 §6.2; RFC 9286 §6.6)" + )] + EeCrlNotFound(String), + + #[error( + "manifest is not valid at validation_time: this_update={this_update_rfc3339_utc} next_update={next_update_rfc3339_utc} validation_time={validation_time_rfc3339_utc} (RFC 9286 §6.3; RFC 9286 §6.6)" + )] + StaleOrEarly { + this_update_rfc3339_utc: String, + next_update_rfc3339_utc: String, + validation_time_rfc3339_utc: String, + }, + + #[error( + "manifest must reside at the same publication point as id-ad-caRepository: manifest={manifest_rsync_uri} publication_point={publication_point_rsync_uri} (RFC 9286 §6.1; RFC 9286 §6.6)" + )] + ManifestOutsidePublicationPoint { + manifest_rsync_uri: String, + publication_point_rsync_uri: String, + }, + + #[error( + "manifestNumber not higher than previously validated manifest: old={old_hex} new={new_hex} (RFC 9286 §4.2.1; RFC 9286 §6.6)" + )] + ManifestNumberNotIncreasing { old_hex: String, new_hex: String }, + + #[error( + "thisUpdate not more recent than previously validated manifest: old={old_rfc3339_utc} new={new_rfc3339_utc} (RFC 9286 §4.2.1; RFC 9286 §6.6)" + )] + ThisUpdateNotIncreasing { + old_rfc3339_utc: String, + new_rfc3339_utc: String, + }, + + #[error( + "manifest referenced file missing in current repository view: {rsync_uri} (RFC 9286 §6.4; RFC 9286 §6.6)" + )] + MissingFile { rsync_uri: String }, + + #[error("manifest file hash mismatch: {rsync_uri} (RFC 9286 §6.5; RFC 9286 §6.6)")] + HashMismatch { rsync_uri: String }, + + #[error("issuer CA certificate bytes unavailable: {detail} (RFC 6487 §4; RFC 9286 §6.2)")] + IssuerCaLoadFailed { detail: String }, +} + +impl ManifestFreshError { + pub(crate) fn should_warn_when_current_instance_reused(&self) -> bool { + !matches!( + self, + ManifestFreshError::RepoSyncFailed { .. } + | ManifestFreshError::MissingManifest { .. } + | ManifestFreshError::MissingFile { .. } + ) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum ManifestReuseError { + #[error("latest current-instance VCIR missing: {0} (RFC 9286 §6.6)")] + MissingCurrentInstanceVcir(String), + + #[error( + "latest current-instance VCIR is not marked failed-fetch eligible: {0} (RFC 9286 §6.6)" + )] + IneligibleCurrentInstanceVcir(String), + + #[error( + "latest current-instance VCIR instance_gate expired: manifest={manifest_rsync_uri} effective_until={effective_until_rfc3339_utc} validation_time={validation_time_rfc3339_utc} (RFC 9286 §6.6)" + )] + CurrentInstanceVcirExpired { + manifest_rsync_uri: String, + effective_until_rfc3339_utc: String, + validation_time_rfc3339_utc: String, + }, + + #[error( + "current-instance VCIR current_manifest_rsync_uri does not match requested manifest URI: expected={expected} actual={actual}" + )] + ManifestUriMismatch { expected: String, actual: String }, + + #[error("manifest raw bytes missing for current-instance VCIR reconstruction: {0}")] + MissingManifestRaw(String), + + #[error("artifact raw bytes missing for current-instance VCIR reconstruction: {rsync_uri}")] + MissingArtifactRaw { rsync_uri: String }, + + #[error("invalid current-instance VCIR: {0}")] + InvalidCurrentInstanceVcir(String), + + #[error("storage error during current-instance VCIR reuse: {0}")] + Storage(#[from] StorageError), +} + +#[derive(Debug, thiserror::Error)] +pub enum ManifestProcessError { + #[error("manifest processing failed and cache use is disabled: {0}")] + StopAllOutput(#[from] ManifestFreshError), + + #[error( + "manifest processing failed and no reusable current-instance validated result is available: fresh={fresh}; reused={reused}" + )] + NoUsableCache { + fresh: ManifestFreshError, + reused: ManifestReuseError, + }, + + #[error("storage error during manifest processing: {0}")] + Storage(#[from] StorageError), +} + +pub fn process_manifest_publication_point( + store: &RocksStore, + policy: &Policy, + manifest_rsync_uri: &str, + publication_point_rsync_uri: &str, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + validation_time: time::OffsetDateTime, +) -> Result { + process_manifest_publication_point_after_repo_sync( + store, + policy, + manifest_rsync_uri, + publication_point_rsync_uri, + issuer_ca_der, + issuer_ca_rsync_uri, + validation_time, + true, + None, + ) +} + +pub fn process_manifest_publication_point_fresh_after_repo_sync( + store: &RocksStore, + manifest_rsync_uri: &str, + publication_point_rsync_uri: &str, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + validation_time: time::OffsetDateTime, + repo_sync_ok: bool, + repo_sync_error: Option<&str>, +) -> Result { + process_manifest_publication_point_fresh_after_repo_sync_with_timing( + store, + manifest_rsync_uri, + publication_point_rsync_uri, + None, + issuer_ca_der, + issuer_ca_rsync_uri, + validation_time, + repo_sync_ok, + repo_sync_error, + ) + .map(|(fresh, _timing)| fresh) +} + +#[derive(Clone, Debug, Default)] +pub struct FreshPublicationPointTimingBreakdown { + pub current_index_lock_ms: u64, + pub manifest_load_ms: u64, + pub manifest_index_lookup_ms: u64, + pub manifest_blob_load_ms: u64, + pub manifest_decode_ms: u64, + pub replay_guard_ms: u64, + pub replay_meta_hit: bool, + pub replay_meta_miss: bool, + pub manifest_entries_ms: u64, + pub pack_files_ms: u64, + pub pack_files_index_lookup_ms: u64, + pub pack_files_blob_load_ms: u64, + pub ee_path_validate_ms: u64, + pub manifest_file_count: usize, +} + +pub fn process_manifest_publication_point_fresh_after_repo_sync_with_timing( + store: &RocksStore, + manifest_rsync_uri: &str, + publication_point_rsync_uri: &str, + current_repo_index: Option<&CurrentRepoIndexHandle>, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + validation_time: time::OffsetDateTime, + repo_sync_ok: bool, + repo_sync_error: Option<&str>, +) -> Result< + ( + FreshValidatedPublicationPoint, + FreshPublicationPointTimingBreakdown, + ), + ManifestFreshError, +> { + if repo_sync_ok { + try_build_fresh_publication_point_with_timing( + store, + manifest_rsync_uri, + publication_point_rsync_uri, + current_repo_index, + issuer_ca_der, + issuer_ca_rsync_uri, + validation_time, + ) + } else { + Err(ManifestFreshError::RepoSyncFailed { + detail: repo_sync_error.unwrap_or("repo sync failed").to_string(), + }) + } +} + +pub fn process_manifest_publication_point_after_repo_sync( + store: &RocksStore, + policy: &Policy, + manifest_rsync_uri: &str, + publication_point_rsync_uri: &str, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + validation_time: time::OffsetDateTime, + repo_sync_ok: bool, + repo_sync_error: Option<&str>, +) -> Result { + let fresh = if repo_sync_ok { + try_build_fresh_publication_point( + store, + manifest_rsync_uri, + publication_point_rsync_uri, + issuer_ca_der, + issuer_ca_rsync_uri, + validation_time, + ) + } else { + Err(ManifestFreshError::RepoSyncFailed { + detail: repo_sync_error.unwrap_or("repo sync failed").to_string(), + }) + }; + + match fresh { + Ok(fresh_point) => { + let snapshot = fresh_point.to_publication_point_snapshot(); + Ok(PublicationPointResult { + source: PublicationPointSource::Fresh, + snapshot, + warnings: Vec::new(), + }) + } + Err(fresh_err) => match policy.ca_failed_fetch_policy { + CaFailedFetchPolicy::StopAllOutput => { + Err(ManifestProcessError::StopAllOutput(fresh_err)) + } + CaFailedFetchPolicy::ReuseCurrentInstanceVcir => { + match load_current_instance_vcir_publication_point( + store, + manifest_rsync_uri, + publication_point_rsync_uri, + validation_time, + ) { + Ok(snapshot) => { + let mut warnings = Vec::new(); + if fresh_err.should_warn_when_current_instance_reused() { + warnings.push( + Warning::new(format!("manifest failed fetch: {fresh_err}")) + .with_rfc_refs(&[RfcRef("RFC 9286 §6.6")]) + .with_context(manifest_rsync_uri), + ); + } + Ok(PublicationPointResult { + source: PublicationPointSource::VcirCurrentInstance, + snapshot, + warnings, + }) + } + Err(reused) => Err(ManifestProcessError::NoUsableCache { + fresh: fresh_err, + reused, + }), + } + } + }, + } +} + +pub fn load_current_instance_vcir_publication_point( + store: &RocksStore, + manifest_rsync_uri: &str, + publication_point_rsync_uri: &str, + validation_time: time::OffsetDateTime, +) -> Result { + let vcir = store.get_vcir(manifest_rsync_uri)?.ok_or_else(|| { + ManifestReuseError::MissingCurrentInstanceVcir(manifest_rsync_uri.to_string()) + })?; + + if vcir.current_manifest_rsync_uri != manifest_rsync_uri { + return Err(ManifestReuseError::ManifestUriMismatch { + expected: manifest_rsync_uri.to_string(), + actual: vcir.current_manifest_rsync_uri.clone(), + }); + } + + if !vcir.audit_summary.failed_fetch_eligible { + return Err(ManifestReuseError::IneligibleCurrentInstanceVcir( + manifest_rsync_uri.to_string(), + )); + } + + let instance_effective_until = vcir + .instance_gate + .instance_effective_until + .parse() + .map_err(|e| { + ManifestReuseError::InvalidCurrentInstanceVcir(format!( + "instance_gate.instance_effective_until parse failed: {e}" + )) + })?; + if validation_time > instance_effective_until { + use time::format_description::well_known::Rfc3339; + return Err(ManifestReuseError::CurrentInstanceVcirExpired { + manifest_rsync_uri: manifest_rsync_uri.to_string(), + effective_until_rfc3339_utc: instance_effective_until + .to_offset(time::UtcOffset::UTC) + .format(&Rfc3339) + .expect("format VCIR instance_effective_until"), + validation_time_rfc3339_utc: validation_time + .to_offset(time::UtcOffset::UTC) + .format(&Rfc3339) + .expect("format validation_time"), + }); + } + + let manifest_artifact = vcir + .related_artifacts + .iter() + .find(|artifact| { + artifact.artifact_role == VcirArtifactRole::Manifest + && artifact.uri.as_deref() == Some(manifest_rsync_uri) + }) + .ok_or_else(|| { + ManifestReuseError::InvalidCurrentInstanceVcir( + "missing manifest artifact matching manifest_rsync_uri".to_string(), + ) + })?; + + let manifest_bytes = store + .get_blob_bytes(&manifest_artifact.sha256)? + .ok_or_else(|| ManifestReuseError::MissingManifestRaw(manifest_artifact.sha256.clone()))?; + + let mut seen = HashSet::new(); + let mut files = Vec::new(); + for artifact in &vcir.related_artifacts { + let Some(uri) = artifact.uri.as_ref() else { + continue; + }; + if artifact.artifact_role == VcirArtifactRole::Manifest + || artifact.artifact_role == VcirArtifactRole::IssuerCert + || artifact.artifact_role == VcirArtifactRole::Tal + || artifact.artifact_role == VcirArtifactRole::TrustAnchorCert + { + continue; + } + if !seen.insert(uri.clone()) { + continue; + } + let entry_bytes = store.get_blob_bytes(&artifact.sha256)?.ok_or_else(|| { + ManifestReuseError::MissingArtifactRaw { + rsync_uri: uri.clone(), + } + })?; + files.push(PackFile::from_bytes_compute_sha256(uri, entry_bytes)); + } + + Ok(PublicationPointSnapshot { + format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, + manifest_rsync_uri: manifest_rsync_uri.to_string(), + publication_point_rsync_uri: publication_point_rsync_uri.to_string(), + manifest_number_be: vcir + .validated_manifest_meta + .validated_manifest_number + .clone(), + this_update: vcir + .validated_manifest_meta + .validated_manifest_this_update + .clone(), + next_update: vcir + .validated_manifest_meta + .validated_manifest_next_update + .clone(), + verified_at: vcir.last_successful_validation_time.clone(), + manifest_bytes, + files, + }) +} + +fn decode_and_validate_manifest_with_current_time( + manifest_bytes: &[u8], + validation_time: time::OffsetDateTime, +) -> Result { + let manifest = ManifestObject::decode_der(manifest_bytes)?; + manifest.validate_embedded_ee_cert()?; + manifest.signed_object.verify()?; + + let this_update = manifest + .manifest + .this_update + .to_offset(time::UtcOffset::UTC); + let next_update = manifest + .manifest + .next_update + .to_offset(time::UtcOffset::UTC); + let now = validation_time.to_offset(time::UtcOffset::UTC); + if now < this_update || now > next_update { + return Err(ManifestFreshError::StaleOrEarly { + this_update_rfc3339_utc: this_update + .format(&time::format_description::well_known::Rfc3339) + .expect("format thisUpdate"), + next_update_rfc3339_utc: next_update + .format(&time::format_description::well_known::Rfc3339) + .expect("format nextUpdate"), + validation_time_rfc3339_utc: now + .format(&time::format_description::well_known::Rfc3339) + .expect("format validation_time"), + }); + } + + Ok(manifest) +} + +pub(crate) fn try_build_fresh_publication_point( + store: &RocksStore, + manifest_rsync_uri: &str, + publication_point_rsync_uri: &str, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + validation_time: time::OffsetDateTime, +) -> Result { + try_build_fresh_publication_point_with_timing( + store, + manifest_rsync_uri, + publication_point_rsync_uri, + None, + issuer_ca_der, + issuer_ca_rsync_uri, + validation_time, + ) + .map(|(fresh, _timing)| fresh) +} + +pub(crate) fn try_build_fresh_publication_point_with_timing( + store: &RocksStore, + manifest_rsync_uri: &str, + publication_point_rsync_uri: &str, + current_repo_index: Option<&CurrentRepoIndexHandle>, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + validation_time: time::OffsetDateTime, +) -> Result< + ( + FreshValidatedPublicationPoint, + FreshPublicationPointTimingBreakdown, + ), + ManifestFreshError, +> { + let mut timing = FreshPublicationPointTimingBreakdown::default(); + let current_index_lock_started = std::time::Instant::now(); + let current_index_guard = current_repo_index.and_then(|handle| handle.read().ok()); + timing.current_index_lock_ms = current_index_lock_started.elapsed().as_millis() as u64; + + if !rsync_uri_is_under_publication_point(manifest_rsync_uri, publication_point_rsync_uri) { + return Err(ManifestFreshError::ManifestOutsidePublicationPoint { + manifest_rsync_uri: manifest_rsync_uri.to_string(), + publication_point_rsync_uri: publication_point_rsync_uri.to_string(), + }); + } + + let manifest_load_started = std::time::Instant::now(); + let manifest_bytes = if let Some(index) = current_index_guard.as_ref() { + let manifest_lookup_started = std::time::Instant::now(); + let current = index.get_by_uri(manifest_rsync_uri).ok_or_else(|| { + ManifestFreshError::MissingManifest { + manifest_rsync_uri: manifest_rsync_uri.to_string(), + } + })?; + timing.manifest_index_lookup_ms = manifest_lookup_started.elapsed().as_millis() as u64; + let manifest_blob_load_started = std::time::Instant::now(); + store + .get_blob_bytes(¤t.current_hash_hex) + .map_err(|e| ManifestFreshError::MissingManifest { + manifest_rsync_uri: format!("{manifest_rsync_uri} ({e})"), + })? + .ok_or_else(|| ManifestFreshError::MissingManifest { + manifest_rsync_uri: manifest_rsync_uri.to_string(), + }) + .inspect(|_| { + timing.manifest_blob_load_ms = + manifest_blob_load_started.elapsed().as_millis() as u64; + })? + } else { + let manifest_blob_load_started = std::time::Instant::now(); + store + .load_current_object_bytes_by_uri(manifest_rsync_uri) + .map_err(|e| ManifestFreshError::MissingManifest { + manifest_rsync_uri: format!("{manifest_rsync_uri} ({e})"), + })? + .ok_or_else(|| ManifestFreshError::MissingManifest { + manifest_rsync_uri: manifest_rsync_uri.to_string(), + }) + .inspect(|_| { + timing.manifest_blob_load_ms = + manifest_blob_load_started.elapsed().as_millis() as u64; + })? + }; + timing.manifest_load_ms = manifest_load_started.elapsed().as_millis() as u64; + + let manifest_decode_started = std::time::Instant::now(); + let manifest = + decode_and_validate_manifest_with_current_time(&manifest_bytes, validation_time)?; + timing.manifest_decode_ms = manifest_decode_started.elapsed().as_millis() as u64; + + let this_update = manifest + .manifest + .this_update + .to_offset(time::UtcOffset::UTC); + let next_update = manifest + .manifest + .next_update + .to_offset(time::UtcOffset::UTC); + let now = validation_time.to_offset(time::UtcOffset::UTC); + + // RFC 9286 §4.2.1: replay/rollback detection for manifestNumber and thisUpdate. + // + // Important nuance for revalidation across runs: + // - If the manifestNumber is equal to the previously validated manifestNumber *and* the + // manifest bytes are identical, then this is the same manifest being revalidated and MUST + // be accepted (otherwise, RPs would incorrectly treat stable repositories as "failed fetch" + // and fall back to the current-instance VCIR snapshot). + // - If manifestNumber is equal but the manifest bytes differ, treat this as invalid (a + // repository is not allowed to change the manifest while keeping the manifestNumber). + // - If manifestNumber is lower, treat as rollback and reject. + // - If manifestNumber is higher, require thisUpdate to be more recent than the previously + // validated thisUpdate. + let replay_guard_started = std::time::Instant::now(); + if let Some(old_meta) = store + .get_manifest_replay_meta(manifest_rsync_uri) + .ok() + .flatten() + { + timing.replay_meta_hit = true; + if old_meta.manifest_rsync_uri == manifest_rsync_uri { + let new_num = manifest.manifest.manifest_number.bytes_be.as_slice(); + let old_num = old_meta.manifest_number_be.as_slice(); + match cmp_minimal_be_unsigned(new_num, old_num) { + Ordering::Greater => { + let old_this_update = old_meta + .manifest_this_update + .parse() + .expect("manifest replay meta validation ensures thisUpdate parses"); + if this_update <= old_this_update { + use time::format_description::well_known::Rfc3339; + return Err(ManifestFreshError::ThisUpdateNotIncreasing { + old_rfc3339_utc: old_this_update + .to_offset(time::UtcOffset::UTC) + .format(&Rfc3339) + .expect("format old thisUpdate"), + new_rfc3339_utc: this_update + .format(&Rfc3339) + .expect("format new thisUpdate"), + }); + } + } + Ordering::Equal => { + let new_manifest_hash = sha2::Sha256::digest(&manifest_bytes); + if old_meta.manifest_sha256.as_slice() != new_manifest_hash.as_slice() { + return Err(ManifestFreshError::ManifestNumberNotIncreasing { + old_hex: hex::encode_upper(old_num), + new_hex: hex::encode_upper(new_num), + }); + } + } + Ordering::Less => { + return Err(ManifestFreshError::ManifestNumberNotIncreasing { + old_hex: hex::encode_upper(old_num), + new_hex: hex::encode_upper(new_num), + }); + } + } + } + } else { + timing.replay_meta_miss = true; + } + timing.replay_guard_ms = replay_guard_started.elapsed().as_millis() as u64; + + let manifest_entries_started = std::time::Instant::now(); + let entries = manifest + .manifest + .parse_files() + .map_err(ManifestDecodeError::Validate)?; + timing.manifest_entries_ms = manifest_entries_started.elapsed().as_millis() as u64; + timing.manifest_file_count = entries.len(); + let mut files = Vec::with_capacity(manifest.manifest.file_count()); + let pack_files_started = std::time::Instant::now(); + let external_raw_store = store + .external_raw_store_ref() + .cloned() + .map(std::sync::Arc::new); + let external_repo_bytes = store + .external_repo_bytes_ref() + .cloned() + .map(std::sync::Arc::new); + let mut pack_files_index_lookup_duration = std::time::Duration::ZERO; + let mut pack_files_blob_load_duration = std::time::Duration::ZERO; + for entry in &entries { + let rsync_uri = + join_rsync_dir_and_file(publication_point_rsync_uri, entry.file_name.as_str()); + let current_object = if let Some(index) = current_index_guard.as_ref() { + let index_lookup_started = std::time::Instant::now(); + let current = + index + .get_by_uri(&rsync_uri) + .ok_or_else(|| ManifestFreshError::MissingFile { + rsync_uri: rsync_uri.clone(), + })?; + pack_files_index_lookup_duration += index_lookup_started.elapsed(); + crate::storage::CurrentObjectWithHash { + current_hash_hex: current.current_hash_hex.clone(), + current_hash: current.current_hash, + bytes: Vec::new(), + } + } else { + let blob_load_started = std::time::Instant::now(); + store + .load_current_object_with_hash_by_uri(&rsync_uri) + .map_err(|_e| ManifestFreshError::MissingFile { + rsync_uri: rsync_uri.clone(), + })? + .ok_or_else(|| ManifestFreshError::MissingFile { + rsync_uri: rsync_uri.clone(), + }) + .inspect(|_| { + pack_files_blob_load_duration += blob_load_started.elapsed(); + })? + }; + + if current_object.current_hash != entry.hash_bytes { + return Err(ManifestFreshError::HashMismatch { rsync_uri }); + } + + if let (Some(_), Some(repo_bytes)) = + (current_index_guard.as_ref(), external_repo_bytes.as_ref()) + { + files.push(PackFile::from_lazy_repo_bytes( + rsync_uri, + current_object.current_hash_hex, + current_object.current_hash, + repo_bytes.clone(), + )); + } else if let (Some(_), Some(raw_store)) = + (current_index_guard.as_ref(), external_raw_store.as_ref()) + { + files.push(PackFile::from_lazy_external_raw_store( + rsync_uri, + current_object.current_hash_hex, + current_object.current_hash, + raw_store.clone(), + )); + } else { + let bytes = if current_object.bytes.is_empty() { + let blob_load_started = std::time::Instant::now(); + store + .get_blob_bytes(¤t_object.current_hash_hex) + .map_err(|_e| ManifestFreshError::MissingFile { + rsync_uri: rsync_uri.clone(), + })? + .ok_or_else(|| ManifestFreshError::MissingFile { + rsync_uri: rsync_uri.clone(), + }) + .inspect(|_| { + pack_files_blob_load_duration += blob_load_started.elapsed(); + })? + } else { + current_object.bytes + }; + files.push(PackFile::from_bytes_with_sha256( + rsync_uri, + bytes, + current_object.current_hash, + )); + } + } + timing.pack_files_index_lookup_ms = pack_files_index_lookup_duration.as_millis() as u64; + timing.pack_files_blob_load_ms = pack_files_blob_load_duration.as_millis() as u64; + timing.pack_files_ms = pack_files_started.elapsed().as_millis() as u64; + + // RFC 6488 §3: manifest (signed object) validity includes a valid EE cert path. + // We validate this after §6.4/§6.5 so the issuer CRL can be selected from the publication point. + let ee_path_validate_started = std::time::Instant::now(); + validate_manifest_embedded_ee_cert_path( + &manifest, + &files, + issuer_ca_der, + issuer_ca_rsync_uri, + validation_time, + )?; + timing.ee_path_validate_ms = ee_path_validate_started.elapsed().as_millis() as u64; + + Ok(( + FreshValidatedPublicationPoint { + manifest_rsync_uri: manifest_rsync_uri.to_string(), + publication_point_rsync_uri: publication_point_rsync_uri.to_string(), + manifest_number_be: manifest.manifest.manifest_number.bytes_be.clone(), + this_update: PackTime::from_utc_offset_datetime(this_update), + next_update: PackTime::from_utc_offset_datetime(next_update), + verified_at: PackTime::from_utc_offset_datetime(now), + manifest_bytes, + files, + }, + timing, + )) +} diff --git a/crates/panda-rpki-validator/src/validation/manifest/tests.rs b/crates/panda-rpki-validator/src/validation/manifest/tests.rs new file mode 100644 index 0000000..2b672e1 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/manifest/tests.rs @@ -0,0 +1,833 @@ +// Manifest processing and reuse tests. + +use super::*; +use crate::current_repo_index::CurrentRepoIndex; +use crate::data_model::manifest::ManifestObject; +use crate::storage::{ + PackFile, PackTime, RawByHashEntry, RocksStore, ValidatedCaInstanceResult, + ValidatedManifestMeta, VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus, + VcirAuditSummary, VcirCcrManifestProjection, VcirInstanceGate, VcirRelatedArtifact, + VcirSummary, +}; +use std::path::Path; + +fn manifest_fixture_path() -> &'static Path { + Path::new( + "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft", + ) +} + +fn issuer_ca_fixture_der() -> Vec { + std::fs::read( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ) + .expect("read issuer ca fixture") +} + +fn issuer_ca_rsync_uri() -> &'static str { + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer" +} + +fn fixture_to_rsync_uri(path: &Path) -> String { + let rel = path + .strip_prefix("tests/fixtures/repository") + .expect("path under fixture repository"); + let mut it = rel.components(); + let host = it + .next() + .expect("host component") + .as_os_str() + .to_string_lossy(); + let rest = it.as_path().to_string_lossy(); + format!("rsync://{host}/{rest}") +} + +fn fixture_dir_to_rsync_uri(dir: &Path) -> String { + let mut s = fixture_to_rsync_uri(dir); + if !s.ends_with('/') { + s.push('/'); + } + s +} + +fn load_manifest_fixture() -> ( + ManifestObject, + Vec, + String, + String, + time::OffsetDateTime, +) { + let manifest_path = manifest_fixture_path(); + let manifest_bytes = std::fs::read(manifest_path).expect("read manifest fixture"); + let manifest = ManifestObject::decode_der(&manifest_bytes).expect("decode manifest"); + let manifest_rsync_uri = fixture_to_rsync_uri(manifest_path); + let publication_point_rsync_uri = fixture_dir_to_rsync_uri(manifest_path.parent().unwrap()); + let validation_time = manifest.manifest.this_update + time::Duration::seconds(1); + ( + manifest, + manifest_bytes, + manifest_rsync_uri, + publication_point_rsync_uri, + validation_time, + ) +} + +fn raw_by_hash_entry(uri: &str, bytes: Vec, object_type: &str) -> RawByHashEntry { + let mut entry = RawByHashEntry::from_bytes(hex::encode(sha2::Sha256::digest(&bytes)), bytes); + entry.origin_uris.push(uri.to_string()); + entry.object_type = Some(object_type.to_string()); + entry.encoding = Some("der".to_string()); + entry +} + +fn put_current_object(store: &RocksStore, rsync_uri: &str, bytes: Vec, object_type: &str) { + let hash = hex::encode(sha2::Sha256::digest(&bytes)); + store + .put_raw_by_hash_entry(&raw_by_hash_entry(rsync_uri, bytes, object_type)) + .expect("put raw_by_hash entry"); + store + .put_repository_view_entry(&crate::storage::RepositoryViewEntry { + rsync_uri: rsync_uri.to_string(), + current_hash: Some(hash), + repository_source: Some("https://example.test/notification.xml".to_string()), + object_type: Some(object_type.to_string()), + state: crate::storage::RepositoryViewState::Present, + }) + .expect("put repository view entry"); +} + +fn put_complete_publication_point_current_objects( + store: &RocksStore, + manifest: &ManifestObject, + manifest_rsync_uri: &str, + manifest_bytes: Vec, + publication_point_rsync_uri: &str, +) { + put_current_object(store, manifest_rsync_uri, manifest_bytes, "mft"); + for entry in manifest.manifest.parse_files().expect("parse files") { + let file_path = manifest_fixture_path() + .parent() + .unwrap() + .join(entry.file_name.as_str()); + let bytes = std::fs::read(&file_path).expect("read fixture file"); + let rsync_uri = format!("{publication_point_rsync_uri}{}", entry.file_name); + let object_type = rsync_uri.rsplit('.').next().unwrap_or("bin"); + put_current_object(store, &rsync_uri, bytes, object_type); + } +} + +fn put_raw_only(store: &RocksStore, rsync_uri: &str, bytes: Vec, object_type: &str) { + store + .put_raw_by_hash_entry(&raw_by_hash_entry(rsync_uri, bytes, object_type)) + .expect("put raw_by_hash entry"); +} + +fn sample_vcir_for_manifest_replay_meta( + manifest: &ManifestObject, + manifest_rsync_uri: &str, + publication_point_rsync_uri: &str, + manifest_bytes: &[u8], + validation_time: time::OffsetDateTime, +) -> ValidatedCaInstanceResult { + let manifest_hash = hex::encode(sha2::Sha256::digest(manifest_bytes)); + let this_update = manifest + .manifest + .this_update + .to_offset(time::UtcOffset::UTC); + let mut vcir = sample_current_instance_vcir( + manifest_rsync_uri, + publication_point_rsync_uri, + &manifest_hash, + "rsync://example.test/repo/object.roa", + &hex::encode(sha2::Sha256::digest(b"object")), + validation_time, + true, + ); + vcir.validated_manifest_meta.validated_manifest_number = + manifest.manifest.manifest_number.bytes_be.clone(); + vcir.validated_manifest_meta.validated_manifest_this_update = + PackTime::from_utc_offset_datetime(this_update); + vcir.ccr_manifest_projection.manifest_number_be = + manifest.manifest.manifest_number.bytes_be.clone(); + vcir.ccr_manifest_projection.manifest_this_update = + PackTime::from_utc_offset_datetime(this_update); + vcir.ccr_manifest_projection.manifest_sha256 = + hex::decode(manifest_hash).expect("decode manifest hash"); + vcir.ccr_manifest_projection.manifest_size = manifest_bytes.len() as u64; + vcir +} + +fn sample_current_instance_vcir( + manifest_rsync_uri: &str, + publication_point_rsync_uri: &str, + manifest_sha256: &str, + locked_object_uri: &str, + locked_object_sha256: &str, + validation_time: time::OffsetDateTime, + failed_fetch_eligible: bool, +) -> ValidatedCaInstanceResult { + let gate_time = PackTime::from_utc_offset_datetime(validation_time + time::Duration::hours(1)); + let ccr_manifest_projection = VcirCcrManifestProjection { + manifest_rsync_uri: manifest_rsync_uri.to_string(), + manifest_sha256: hex::decode(manifest_sha256).expect("decode manifest sha256"), + manifest_size: 2048, + manifest_ee_aki: vec![0x11; 20], + manifest_number_be: vec![1], + manifest_this_update: PackTime::from_utc_offset_datetime(validation_time), + manifest_sia_locations_der: vec![vec![ + 0x30, 0x11, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x05, 0x86, 0x05, + b'r', b's', b'y', b'n', b'c', + ]], + subordinate_skis: Vec::new(), + }; + ValidatedCaInstanceResult { + manifest_rsync_uri: manifest_rsync_uri.to_string(), + parent_manifest_rsync_uri: None, + tal_id: "test-tal".to_string(), + ca_subject_name: "CN=test".to_string(), + ca_ski: "00112233445566778899aabbccddeeff00112233".to_string(), + issuer_ski: "00112233445566778899aabbccddeeff00112233".to_string(), + last_successful_validation_time: PackTime::from_utc_offset_datetime(validation_time), + current_manifest_rsync_uri: manifest_rsync_uri.to_string(), + current_crl_rsync_uri: format!("{publication_point_rsync_uri}current.crl"), + validated_manifest_meta: ValidatedManifestMeta { + validated_manifest_number: vec![1], + validated_manifest_this_update: PackTime::from_utc_offset_datetime(validation_time), + validated_manifest_next_update: gate_time.clone(), + }, + ccr_manifest_projection, + instance_gate: VcirInstanceGate { + manifest_next_update: gate_time.clone(), + current_crl_next_update: gate_time.clone(), + self_ca_not_after: gate_time.clone(), + instance_effective_until: gate_time, + }, + child_entries: Vec::new(), + local_outputs: Vec::new(), + related_artifacts: vec![ + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::Manifest, + artifact_kind: VcirArtifactKind::Mft, + uri: Some(manifest_rsync_uri.to_string()), + sha256: manifest_sha256.to_string(), + object_type: Some("mft".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: VcirArtifactKind::Roa, + uri: Some(locked_object_uri.to_string()), + sha256: locked_object_sha256.to_string(), + object_type: Some("roa".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }, + ], + summary: VcirSummary { + local_vrp_count: 0, + local_aspa_count: 0, + local_router_key_count: 0, + child_count: 0, + accepted_object_count: 2, + rejected_object_count: 0, + }, + audit_summary: VcirAuditSummary { + failed_fetch_eligible, + last_failed_fetch_reason: None, + warning_count: 0, + audit_flags: Vec::new(), + }, + } +} +fn locked_files_for_manifest( + manifest: &ManifestObject, + publication_point_rsync_uri: &str, +) -> Vec { + let manifest_path = manifest_fixture_path(); + manifest + .manifest + .parse_files() + .expect("parse files") + .into_iter() + .map(|entry| { + let file_path = manifest_path + .parent() + .unwrap() + .join(entry.file_name.as_str()); + let bytes = std::fs::read(&file_path).unwrap_or_else(|_| { + panic!("read fixture file referenced by manifest: {file_path:?}") + }); + PackFile::from_bytes_compute_sha256( + format!("{publication_point_rsync_uri}{}", entry.file_name), + bytes, + ) + }) + .collect() +} + +#[test] +fn try_build_fresh_publication_point_rejects_manifest_outside_publication_point() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(temp.path()).expect("open rocksdb"); + let (_, _, manifest_rsync_uri, _, validation_time) = load_manifest_fixture(); + let err = try_build_fresh_publication_point( + &store, + &manifest_rsync_uri, + "rsync://example.test/other/", + &issuer_ca_fixture_der(), + Some(issuer_ca_rsync_uri()), + validation_time, + ) + .unwrap_err(); + assert!( + matches!( + err, + ManifestFreshError::ManifestOutsidePublicationPoint { .. } + ), + "{err}" + ); +} + +#[test] +fn try_build_fresh_publication_point_reports_missing_manifest_when_raw_store_is_empty() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(temp.path()).expect("open rocksdb"); + let (_, _, manifest_rsync_uri, publication_point_rsync_uri, validation_time) = + load_manifest_fixture(); + let err = try_build_fresh_publication_point( + &store, + &manifest_rsync_uri, + &publication_point_rsync_uri, + &issuer_ca_fixture_der(), + Some(issuer_ca_rsync_uri()), + validation_time, + ) + .unwrap_err(); + assert!( + matches!(err, ManifestFreshError::MissingManifest { .. }), + "{err}" + ); +} + +#[test] +fn try_build_fresh_publication_point_reports_missing_locked_file() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(temp.path()).expect("open rocksdb"); + let ( + manifest, + manifest_bytes, + manifest_rsync_uri, + publication_point_rsync_uri, + validation_time, + ) = load_manifest_fixture(); + put_current_object(&store, &manifest_rsync_uri, manifest_bytes, "mft"); + let first_non_crl = manifest + .manifest + .parse_files() + .expect("parse files") + .into_iter() + .find(|entry| !entry.file_name.ends_with(".crl")) + .expect("fixture non-crl entry"); + let file_path = manifest_fixture_path() + .parent() + .unwrap() + .join(first_non_crl.file_name.as_str()); + let bytes = std::fs::read(&file_path).expect("read fixture file"); + let rsync_uri = format!("{publication_point_rsync_uri}{}", first_non_crl.file_name); + let object_type = rsync_uri.rsplit('.').next().unwrap_or("bin"); + put_current_object(&store, &rsync_uri, bytes, object_type); + + let err = try_build_fresh_publication_point( + &store, + &manifest_rsync_uri, + &publication_point_rsync_uri, + &issuer_ca_fixture_der(), + Some(issuer_ca_rsync_uri()), + validation_time, + ) + .unwrap_err(); + assert!( + matches!(err, ManifestFreshError::MissingFile { .. }), + "{err}" + ); +} + +#[test] +fn try_build_fresh_publication_point_detects_hash_mismatch_via_repository_view_hash() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(temp.path()).expect("open rocksdb"); + let ( + manifest, + manifest_bytes, + manifest_rsync_uri, + publication_point_rsync_uri, + validation_time, + ) = load_manifest_fixture(); + put_current_object(&store, &manifest_rsync_uri, manifest_bytes, "mft"); + + let non_crl_entries = manifest + .manifest + .parse_files() + .expect("parse files") + .into_iter() + .filter(|entry| !entry.file_name.ends_with(".crl")) + .collect::>(); + let first = &non_crl_entries[0]; + let second = &non_crl_entries[1]; + + let first_uri = format!("{publication_point_rsync_uri}{}", first.file_name); + let second_path = manifest_fixture_path() + .parent() + .unwrap() + .join(second.file_name.as_str()); + let wrong_bytes = std::fs::read(&second_path).expect("read wrong fixture file"); + let object_type = first_uri.rsplit('.').next().unwrap_or("bin"); + put_current_object(&store, &first_uri, wrong_bytes, object_type); + + for entry in non_crl_entries.iter().skip(1) { + let file_path = manifest_fixture_path() + .parent() + .unwrap() + .join(entry.file_name.as_str()); + let bytes = std::fs::read(&file_path).expect("read fixture file"); + let rsync_uri = format!("{publication_point_rsync_uri}{}", entry.file_name); + let object_type = rsync_uri.rsplit('.').next().unwrap_or("bin"); + put_current_object(&store, &rsync_uri, bytes, object_type); + } + + let err = try_build_fresh_publication_point( + &store, + &manifest_rsync_uri, + &publication_point_rsync_uri, + &issuer_ca_fixture_der(), + Some(issuer_ca_rsync_uri()), + validation_time, + ) + .unwrap_err(); + assert!( + matches!(err, ManifestFreshError::HashMismatch { .. }), + "{err}" + ); +} + +#[test] +fn try_build_fresh_publication_point_uses_current_repo_index_without_repository_view() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(temp.path()).expect("open rocksdb"); + let ( + manifest, + manifest_bytes, + manifest_rsync_uri, + publication_point_rsync_uri, + validation_time, + ) = load_manifest_fixture(); + + put_raw_only(&store, &manifest_rsync_uri, manifest_bytes.clone(), "mft"); + let current_index = CurrentRepoIndex::shared(); + let mut entries = vec![crate::storage::RepositoryViewEntry { + rsync_uri: manifest_rsync_uri.clone(), + current_hash: Some(hex::encode(sha2::Sha256::digest(&manifest_bytes))), + repository_source: Some("https://example.test/notification.xml".to_string()), + object_type: Some("mft".to_string()), + state: crate::storage::RepositoryViewState::Present, + }]; + + for entry in manifest.manifest.parse_files().expect("parse files") { + let file_path = manifest_fixture_path() + .parent() + .unwrap() + .join(entry.file_name.as_str()); + let bytes = std::fs::read(&file_path).expect("read fixture file"); + let rsync_uri = format!("{publication_point_rsync_uri}{}", entry.file_name); + let object_type = rsync_uri.rsplit('.').next().unwrap_or("bin").to_string(); + put_raw_only(&store, &rsync_uri, bytes.clone(), &object_type); + entries.push(crate::storage::RepositoryViewEntry { + rsync_uri, + current_hash: Some(hex::encode(sha2::Sha256::digest(&bytes))), + repository_source: Some("https://example.test/notification.xml".to_string()), + object_type: Some(object_type), + state: crate::storage::RepositoryViewState::Present, + }); + } + + current_index + .write() + .expect("index write lock") + .apply_repository_view_entries(&entries) + .expect("apply current index"); + + assert!( + store + .get_repository_view_entry(&manifest_rsync_uri) + .expect("get repository view") + .is_none() + ); + + let (fresh, _timing) = try_build_fresh_publication_point_with_timing( + &store, + &manifest_rsync_uri, + &publication_point_rsync_uri, + Some(¤t_index), + &issuer_ca_fixture_der(), + Some(issuer_ca_rsync_uri()), + validation_time, + ) + .expect("fresh publication point via current index"); + + assert_eq!(fresh.manifest_rsync_uri, manifest_rsync_uri); + assert_eq!(fresh.files.len(), manifest.manifest.file_count()); +} + +#[test] +fn try_build_fresh_publication_point_records_replay_meta_miss() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(temp.path()).expect("open rocksdb"); + let ( + manifest, + manifest_bytes, + manifest_rsync_uri, + publication_point_rsync_uri, + validation_time, + ) = load_manifest_fixture(); + put_complete_publication_point_current_objects( + &store, + &manifest, + &manifest_rsync_uri, + manifest_bytes, + &publication_point_rsync_uri, + ); + + let (_fresh, timing) = try_build_fresh_publication_point_with_timing( + &store, + &manifest_rsync_uri, + &publication_point_rsync_uri, + None, + &issuer_ca_fixture_der(), + Some(issuer_ca_rsync_uri()), + validation_time, + ) + .expect("fresh publication point without replay meta"); + + assert!(timing.replay_meta_miss); + assert!(!timing.replay_meta_hit); +} + +#[test] +fn try_build_fresh_publication_point_uses_replay_meta_hit_for_same_manifest() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(temp.path()).expect("open rocksdb"); + let ( + manifest, + manifest_bytes, + manifest_rsync_uri, + publication_point_rsync_uri, + validation_time, + ) = load_manifest_fixture(); + let previous_vcir = sample_vcir_for_manifest_replay_meta( + &manifest, + &manifest_rsync_uri, + &publication_point_rsync_uri, + &manifest_bytes, + validation_time, + ); + store.put_vcir(&previous_vcir).expect("put previous vcir"); + put_complete_publication_point_current_objects( + &store, + &manifest, + &manifest_rsync_uri, + manifest_bytes, + &publication_point_rsync_uri, + ); + + let (_fresh, timing) = try_build_fresh_publication_point_with_timing( + &store, + &manifest_rsync_uri, + &publication_point_rsync_uri, + None, + &issuer_ca_fixture_der(), + Some(issuer_ca_rsync_uri()), + validation_time, + ) + .expect("fresh publication point with matching replay meta"); + + assert!(timing.replay_meta_hit); + assert!(!timing.replay_meta_miss); +} + +#[test] +fn validate_manifest_embedded_ee_cert_path_rejects_missing_crl_files() { + let (manifest, _, _, publication_point_rsync_uri, validation_time) = load_manifest_fixture(); + let files = locked_files_for_manifest(&manifest, &publication_point_rsync_uri) + .into_iter() + .filter(|f| !f.rsync_uri.ends_with(".crl")) + .collect::>(); + + let err = validate_manifest_embedded_ee_cert_path( + &manifest, + &files, + &issuer_ca_fixture_der(), + Some(issuer_ca_rsync_uri()), + validation_time, + ) + .unwrap_err(); + assert!(matches!(err, ManifestFreshError::NoCrlFiles), "{err}"); +} + +#[test] +fn validate_manifest_embedded_ee_cert_path_rejects_missing_ee_crldp() { + let (mut manifest, _, _, publication_point_rsync_uri, validation_time) = + load_manifest_fixture(); + manifest.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .extensions + .crl_distribution_points_uris = None; + let files = locked_files_for_manifest(&manifest, &publication_point_rsync_uri); + + let err = validate_manifest_embedded_ee_cert_path( + &manifest, + &files, + &issuer_ca_fixture_der(), + Some(issuer_ca_rsync_uri()), + validation_time, + ) + .unwrap_err(); + assert!(matches!(err, ManifestFreshError::EeCrlDpMissing), "{err}"); +} + +#[test] +fn validate_manifest_embedded_ee_cert_path_rejects_unlisted_crldp_uri() { + let (manifest, _, _, publication_point_rsync_uri, validation_time) = load_manifest_fixture(); + let mut files = locked_files_for_manifest(&manifest, &publication_point_rsync_uri) + .into_iter() + .filter(|f| !f.rsync_uri.ends_with(".crl")) + .collect::>(); + files.push(PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/unrelated.crl", + b"dummy".to_vec(), + )); + + let err = validate_manifest_embedded_ee_cert_path( + &manifest, + &files, + &issuer_ca_fixture_der(), + Some(issuer_ca_rsync_uri()), + validation_time, + ) + .unwrap_err(); + assert!(matches!(err, ManifestFreshError::EeCrlNotFound(_)), "{err}"); +} + +#[test] +fn validate_manifest_embedded_ee_cert_path_rejects_expired_crl() { + let (manifest, _, _, publication_point_rsync_uri, _) = load_manifest_fixture(); + let files = locked_files_for_manifest(&manifest, &publication_point_rsync_uri); + let ee = &manifest.signed_object.signed_data.certificates[0]; + let crldp_uri = ee + .resource_cert + .tbs + .extensions + .crl_distribution_points_uris + .as_ref() + .and_then(|uris| uris.first()) + .expect("fixture manifest EE CRLDP") + .as_str() + .to_string(); + let crl_file = files + .iter() + .find(|file| file.rsync_uri == crldp_uri) + .expect("fixture CRL referenced by manifest EE"); + let crl = crate::data_model::crl::RpkixCrl::decode_der( + crl_file.bytes().expect("read fixture crl bytes"), + ) + .expect("decode fixture crl"); + let validation_time = crl.next_update.utc; + + let err = validate_manifest_embedded_ee_cert_path( + &manifest, + &files, + &issuer_ca_fixture_der(), + Some(issuer_ca_rsync_uri()), + validation_time, + ) + .unwrap_err(); + + assert!( + matches!( + err, + ManifestFreshError::EeCertPath( + crate::validation::cert_path::CertPathError::CrlNotValidAtTime + ) + ), + "{err}" + ); +} + +#[test] +fn load_current_instance_vcir_publication_point_returns_manifest_and_locked_files() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(temp.path()).expect("open rocksdb"); + let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::hours(1); + let manifest_uri = "rsync://example.test/repo/current.mft"; + let publication_point_uri = "rsync://example.test/repo/"; + let locked_uri = "rsync://example.test/repo/object.roa"; + let manifest_bytes = vec![0x30, 0x31, 0x32]; + let locked_bytes = vec![0x01, 0x02, 0x03, 0x04]; + let manifest_entry = raw_by_hash_entry(manifest_uri, manifest_bytes.clone(), "mft"); + let locked_entry = raw_by_hash_entry(locked_uri, locked_bytes.clone(), "roa"); + store + .put_raw_by_hash_entry(&manifest_entry) + .expect("put manifest raw_by_hash"); + store + .put_raw_by_hash_entry(&locked_entry) + .expect("put locked raw_by_hash"); + let vcir = sample_current_instance_vcir( + manifest_uri, + publication_point_uri, + &manifest_entry.sha256_hex, + locked_uri, + &locked_entry.sha256_hex, + validation_time, + true, + ); + store.put_vcir(&vcir).expect("put vcir"); + + let point = load_current_instance_vcir_publication_point( + &store, + manifest_uri, + publication_point_uri, + validation_time, + ) + .expect("load current-instance vcir publication point"); + assert_eq!(point.manifest_bytes, manifest_bytes); + assert_eq!(point.files.len(), 1); + assert_eq!(point.files[0].rsync_uri, locked_uri); + assert_eq!( + point.files[0].bytes_cloned().expect("locked bytes"), + locked_bytes + ); +} + +#[test] +fn load_current_instance_vcir_publication_point_rejects_ineligible_vcir() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(temp.path()).expect("open rocksdb"); + let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::hours(1); + let manifest_uri = "rsync://example.test/repo/current.mft"; + let publication_point_uri = "rsync://example.test/repo/"; + let locked_uri = "rsync://example.test/repo/object.roa"; + let manifest_entry = raw_by_hash_entry(manifest_uri, vec![0x30], "mft"); + let locked_entry = raw_by_hash_entry(locked_uri, vec![0x01], "roa"); + store + .put_raw_by_hash_entry(&manifest_entry) + .expect("put manifest raw_by_hash"); + store + .put_raw_by_hash_entry(&locked_entry) + .expect("put locked raw_by_hash"); + let vcir = sample_current_instance_vcir( + manifest_uri, + publication_point_uri, + &manifest_entry.sha256_hex, + locked_uri, + &locked_entry.sha256_hex, + validation_time, + false, + ); + store.put_vcir(&vcir).expect("put vcir"); + + let err = load_current_instance_vcir_publication_point( + &store, + manifest_uri, + publication_point_uri, + validation_time, + ) + .unwrap_err(); + assert!( + matches!(err, ManifestReuseError::IneligibleCurrentInstanceVcir(_)), + "{err}" + ); +} + +#[test] +fn load_current_instance_vcir_publication_point_rejects_expired_vcir() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(temp.path()).expect("open rocksdb"); + let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::hours(2); + let manifest_uri = "rsync://example.test/repo/current.mft"; + let publication_point_uri = "rsync://example.test/repo/"; + let locked_uri = "rsync://example.test/repo/object.roa"; + let manifest_entry = raw_by_hash_entry(manifest_uri, vec![0x30], "mft"); + let locked_entry = raw_by_hash_entry(locked_uri, vec![0x01], "roa"); + store + .put_raw_by_hash_entry(&manifest_entry) + .expect("put manifest raw_by_hash"); + store + .put_raw_by_hash_entry(&locked_entry) + .expect("put locked raw_by_hash"); + let mut vcir = sample_current_instance_vcir( + manifest_uri, + publication_point_uri, + &manifest_entry.sha256_hex, + locked_uri, + &locked_entry.sha256_hex, + validation_time - time::Duration::hours(2), + true, + ); + let expired = PackTime::from_utc_offset_datetime(validation_time - time::Duration::minutes(1)); + vcir.instance_gate.manifest_next_update = expired.clone(); + vcir.instance_gate.current_crl_next_update = expired.clone(); + vcir.instance_gate.self_ca_not_after = expired.clone(); + vcir.instance_gate.instance_effective_until = expired; + vcir.validated_manifest_meta.validated_manifest_next_update = + vcir.instance_gate.instance_effective_until.clone(); + store.put_vcir(&vcir).expect("put expired vcir"); + + let err = load_current_instance_vcir_publication_point( + &store, + manifest_uri, + publication_point_uri, + validation_time, + ) + .unwrap_err(); + assert!( + matches!(err, ManifestReuseError::CurrentInstanceVcirExpired { .. }), + "{err}" + ); +} + +#[test] +fn load_current_instance_vcir_publication_point_rejects_missing_locked_artifact_raw() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(temp.path()).expect("open rocksdb"); + let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::hours(1); + let manifest_uri = "rsync://example.test/repo/current.mft"; + let publication_point_uri = "rsync://example.test/repo/"; + let locked_uri = "rsync://example.test/repo/object.roa"; + let manifest_entry = raw_by_hash_entry(manifest_uri, vec![0x30], "mft"); + store + .put_raw_by_hash_entry(&manifest_entry) + .expect("put manifest raw_by_hash"); + let vcir = sample_current_instance_vcir( + manifest_uri, + publication_point_uri, + &manifest_entry.sha256_hex, + locked_uri, + &hex::encode(sha2::Sha256::digest([0x01, 0x02])), + validation_time, + true, + ); + store.put_vcir(&vcir).expect("put vcir"); + + let err = load_current_instance_vcir_publication_point( + &store, + manifest_uri, + publication_point_uri, + validation_time, + ) + .unwrap_err(); + assert!( + matches!(err, ManifestReuseError::MissingArtifactRaw { .. }), + "{err}" + ); +} diff --git a/crates/panda-rpki-validator/src/validation/objects.rs b/crates/panda-rpki-validator/src/validation/objects.rs index f7ef122..a6cff3d 100644 --- a/crates/panda-rpki-validator/src/validation/objects.rs +++ b/crates/panda-rpki-validator/src/validation/objects.rs @@ -29,5680 +29,16 @@ use std::time::{Duration, Instant}; use x509_parser::prelude::FromDer; use x509_parser::x509::SubjectPublicKeyInfo; -const RFC_NONE: &[RfcRef] = &[]; -const RFC_CRLDP: &[RfcRef] = &[RfcRef("RFC 6487 §4.8.6")]; -const RFC_CRLDP_AND_LOCKED_PACK: &[RfcRef] = - &[RfcRef("RFC 6487 §4.8.6"), RfcRef("RFC 9286 §4.2.1")]; - -fn ber_compatible_cms_warning(der: &[u8], rsync_uri: &str, object_kind: &str) -> Option { - let strict_error = RpkiSignedObject::strict_cms_der_error(der)?; - Some( - Warning::new(format!( - "accepted BER-compatible CMS encoding for {object_kind}: {rsync_uri}: {strict_error}" - )) - .with_category(WarningCategory::BerCompatibleCmsEncoding) - .with_rfc_refs(&[RfcRef("X.690 §10"), RfcRef("RFC 6488 §2")]) - .with_context(rsync_uri), - ) -} - -fn ber_compatible_cms_warning_for_file(file: &PackFile, object_kind: &str) -> Option { - ber_compatible_cms_warning(file.bytes().ok()?, &file.rsync_uri, object_kind) -} - -fn sha256_hex_to_32(hex_value: &str) -> [u8; 32] { - let bytes = hex::decode(hex_value).expect("internal sha256 hex should decode"); - let mut out = [0u8; 32]; - out.copy_from_slice(&bytes); - out -} - -fn sha256_hex(bytes: &[u8]) -> String { - hex::encode(sha2::Sha256::digest(bytes)) -} - -fn decode_resource_certificate_with_policy( - der: &[u8], - policy: &Policy, -) -> Result { - if policy.strict.name { - ResourceCertificate::decode_der_with_strict_name(der) - } else { - ResourceCertificate::decode_der(der) - } -} - -#[derive(Clone, Debug)] -pub(crate) struct VerifiedIssuerCrl { - crl: crate::data_model::crl::RpkixCrl, - revoked_serials: std::collections::HashSet>, - sha256_hex: String, -} - -#[derive(Clone, Debug)] -pub(crate) enum CachedIssuerCrl { - Pending { - bytes: Vec, - sha256_hex: Option, - }, - Ok(Arc), -} - -impl CachedIssuerCrl { - fn current_sha256_hex(&mut self) -> &str { - match self { - CachedIssuerCrl::Pending { bytes, sha256_hex } => { - if sha256_hex.is_none() { - *sha256_hex = Some(crate::audit::sha256_hex(bytes)); - } - sha256_hex - .as_deref() - .expect("pending CRL sha256 must be populated") - } - CachedIssuerCrl::Ok(verified) => verified.sha256_hex.as_str(), - } - } -} - -#[derive(Clone, Debug, Default)] -pub(crate) struct IssuerResourcesIndex { - ip_v4: Option, Vec)>>, - ip_v6: Option, Vec)>>, - asnum: Option>, - rdi: Option>, -} - -fn extra_rfc_refs_for_crl_selection(e: &ObjectValidateError) -> &'static [RfcRef] { - match e { - ObjectValidateError::MissingCrlDpUris => RFC_CRLDP, - ObjectValidateError::CrlNotFound(_) => RFC_CRLDP_AND_LOCKED_PACK, - _ => RFC_NONE, - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Vrp { - pub asn: u32, - pub prefix: IpPrefix, - pub max_length: u16, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct AspaAttestation { - pub customer_as_id: u32, - pub provider_as_ids: Vec, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RouterKeyPayload { - pub as_id: u32, - pub ski: Vec, - pub spki_der: Vec, - pub source_object_uri: String, - pub source_object_hash: String, - pub source_ee_cert_hash: String, - pub item_effective_until: PackTime, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct ObjectsOutput { - pub vrps: Vec, - pub aspas: Vec, - pub router_keys: Vec, - pub local_outputs_cache: Vec, - pub warnings: Vec, - pub stats: ObjectsStats, - pub audit: Vec, - pub roa_cache_stats: RoaValidationCacheStats, - pub roa_cache_object_meta: Vec, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct ObjectsStats { - pub roa_total: usize, - pub roa_ok: usize, - pub aspa_total: usize, - pub aspa_ok: usize, - /// Whether this publication point was dropped due to an unrecoverable objects-processing error - /// (e.g., missing issuer CRL in the pack, or `signed_object_failure_policy=drop_publication_point`). - pub publication_point_dropped: bool, -} - -#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize)] -pub struct RoaValidationCacheStats { - pub enabled_publication_points: usize, - pub vcir_hit_publication_points: usize, - pub vcir_miss_publication_points: usize, - pub hit_roas: usize, - pub miss_roas: usize, - pub blocked_roas: usize, - pub fresh_roas: usize, - pub context_blocked_roas: usize, - pub crl_recheck_hit_roas: usize, - pub hash_blocked_roas: usize, - pub expired_blocked_roas: usize, - pub revoked_blocked_roas: usize, - pub metadata_blocked_roas: usize, - pub context_gate_nanos: u64, - pub lookup_nanos: u64, - pub lookup_entry_gate_nanos: u64, - pub lookup_crl_gate_nanos: u64, - pub lookup_materialize_nanos: u64, - pub lookup_crl_gate_verified_crls: usize, - pub lookup_crl_gate_reused_crls: usize, -} - -#[derive(Clone, Copy, Debug)] -pub struct RoaValidationCacheInput<'a> { - enabled: bool, - view: Option<&'a RoaValidationCacheView>, - ca_validation_context_digest: Option<[u8; 32]>, - policy_fingerprint: Option<[u8; 32]>, -} - -impl<'a> RoaValidationCacheInput<'a> { - pub fn disabled() -> Self { - Self { - enabled: false, - view: None, - ca_validation_context_digest: None, - policy_fingerprint: None, - } - } - - pub fn enabled(view: Option<&'a RoaValidationCacheView>) -> Self { - Self { - enabled: true, - view, - ca_validation_context_digest: None, - policy_fingerprint: None, - } - } - - pub fn enabled_with_context( - view: Option<&'a RoaValidationCacheView>, - ca_validation_context_digest: [u8; 32], - policy_fingerprint: [u8; 32], - ) -> Self { - Self { - enabled: true, - view, - ca_validation_context_digest: Some(ca_validation_context_digest), - policy_fingerprint: Some(policy_fingerprint), - } - } -} - -impl RoaValidationCacheStats { - pub fn add_assign(&mut self, other: &Self) { - self.enabled_publication_points += other.enabled_publication_points; - self.vcir_hit_publication_points += other.vcir_hit_publication_points; - self.vcir_miss_publication_points += other.vcir_miss_publication_points; - self.hit_roas += other.hit_roas; - self.miss_roas += other.miss_roas; - self.blocked_roas += other.blocked_roas; - self.fresh_roas += other.fresh_roas; - self.context_blocked_roas += other.context_blocked_roas; - self.crl_recheck_hit_roas += other.crl_recheck_hit_roas; - self.hash_blocked_roas += other.hash_blocked_roas; - self.expired_blocked_roas += other.expired_blocked_roas; - self.revoked_blocked_roas += other.revoked_blocked_roas; - self.metadata_blocked_roas += other.metadata_blocked_roas; - self.context_gate_nanos = self - .context_gate_nanos - .saturating_add(other.context_gate_nanos); - self.lookup_nanos = self.lookup_nanos.saturating_add(other.lookup_nanos); - self.lookup_entry_gate_nanos = self - .lookup_entry_gate_nanos - .saturating_add(other.lookup_entry_gate_nanos); - self.lookup_crl_gate_nanos = self - .lookup_crl_gate_nanos - .saturating_add(other.lookup_crl_gate_nanos); - self.lookup_materialize_nanos = self - .lookup_materialize_nanos - .saturating_add(other.lookup_materialize_nanos); - self.lookup_crl_gate_verified_crls += other.lookup_crl_gate_verified_crls; - self.lookup_crl_gate_reused_crls += other.lookup_crl_gate_reused_crls; - } - - fn for_input(input: RoaValidationCacheInput<'_>, roa_total: usize) -> Self { - let mut stats = Self::default(); - if !input.enabled { - return stats; - } - - stats.enabled_publication_points = 1; - if input.view.is_some() { - stats.vcir_hit_publication_points = 1; - } else { - stats.vcir_miss_publication_points = 1; - stats.miss_roas = roa_total; - stats.fresh_roas = roa_total; - } - stats - } - - fn record_lookup(&mut self, total_nanos: u64, metrics: RoaCacheLookupMetrics) { - self.lookup_nanos = self.lookup_nanos.saturating_add(total_nanos); - self.lookup_entry_gate_nanos = self - .lookup_entry_gate_nanos - .saturating_add(metrics.entry_gate_nanos); - self.lookup_crl_gate_nanos = self - .lookup_crl_gate_nanos - .saturating_add(metrics.crl_gate_nanos); - self.lookup_materialize_nanos = self - .lookup_materialize_nanos - .saturating_add(metrics.materialize_nanos); - self.lookup_crl_gate_verified_crls += metrics.crl_gate_verified_crls; - self.lookup_crl_gate_reused_crls += metrics.crl_gate_reused_crls; - } - - fn record_to_timing(&self, timing: Option<&TimingHandle>) { - let Some(timing) = timing else { - return; - }; - record_non_zero( - timing, - "roa_validation_cache_enabled_publication_points", - self.enabled_publication_points, - ); - record_non_zero( - timing, - "roa_validation_cache_vcir_hit_publication_points", - self.vcir_hit_publication_points, - ); - record_non_zero( - timing, - "roa_validation_cache_vcir_miss_publication_points", - self.vcir_miss_publication_points, - ); - record_non_zero(timing, "roa_validation_cache_hit_roas", self.hit_roas); - record_non_zero(timing, "roa_validation_cache_miss_roas", self.miss_roas); - record_non_zero( - timing, - "roa_validation_cache_blocked_roas", - self.blocked_roas, - ); - record_non_zero(timing, "roa_validation_cache_fresh_roas", self.fresh_roas); - record_non_zero( - timing, - "roa_validation_cache_context_blocked_roas", - self.context_blocked_roas, - ); - record_non_zero( - timing, - "roa_validation_cache_crl_recheck_hit_roas", - self.crl_recheck_hit_roas, - ); - record_non_zero( - timing, - "roa_validation_cache_hash_blocked_roas", - self.hash_blocked_roas, - ); - record_non_zero( - timing, - "roa_validation_cache_expired_blocked_roas", - self.expired_blocked_roas, - ); - record_non_zero( - timing, - "roa_validation_cache_revoked_blocked_roas", - self.revoked_blocked_roas, - ); - record_non_zero( - timing, - "roa_validation_cache_metadata_blocked_roas", - self.metadata_blocked_roas, - ); - record_non_zero( - timing, - "roa_validation_cache_context_gate_nanos", - self.context_gate_nanos as usize, - ); - record_non_zero( - timing, - "roa_validation_cache_lookup_nanos", - self.lookup_nanos as usize, - ); - record_non_zero( - timing, - "roa_validation_cache_lookup_entry_gate_nanos", - self.lookup_entry_gate_nanos as usize, - ); - record_non_zero( - timing, - "roa_validation_cache_lookup_crl_gate_nanos", - self.lookup_crl_gate_nanos as usize, - ); - record_non_zero( - timing, - "roa_validation_cache_lookup_materialize_nanos", - self.lookup_materialize_nanos as usize, - ); - record_non_zero( - timing, - "roa_validation_cache_lookup_crl_gate_verified_crls", - self.lookup_crl_gate_verified_crls, - ); - record_non_zero( - timing, - "roa_validation_cache_lookup_crl_gate_reused_crls", - self.lookup_crl_gate_reused_crls, - ); - timing.record_phase_nanos( - "roa_validation_cache_context_gate_total", - self.context_gate_nanos, - ); - timing.record_phase_nanos("roa_validation_cache_lookup_total", self.lookup_nanos); - timing.record_phase_nanos( - "roa_validation_cache_lookup_entry_gate_total", - self.lookup_entry_gate_nanos, - ); - timing.record_phase_nanos( - "roa_validation_cache_lookup_crl_gate_total", - self.lookup_crl_gate_nanos, - ); - timing.record_phase_nanos( - "roa_validation_cache_lookup_materialize_total", - self.lookup_materialize_nanos, - ); - } -} - -fn record_non_zero(timing: &TimingHandle, key: &'static str, value: usize) { - if value > 0 { - timing.record_count(key, value as u64); - } -} - -fn elapsed_nanos_u64(started: Instant) -> u64 { - started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64 -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -struct RoaCacheLookupMetrics { - entry_gate_nanos: u64, - crl_gate_nanos: u64, - materialize_nanos: u64, - crl_gate_verified_crls: usize, - crl_gate_reused_crls: usize, -} - -#[derive(Clone, Debug)] -enum RoaCacheCrlGate { - Unchanged, - ChangedValid(Arc), - Expired, - Invalid, - Missing, -} - -#[derive(Debug, Default)] -struct RoaCacheCrlGateSet { - gates_by_uri: HashMap, -} - -impl RoaCacheCrlGateSet { - fn evaluate( - &mut self, - crl_uri: &str, - expected_crl_sha256_by_uri: &HashMap, - crl_cache: &mut std::collections::HashMap, - issuer_ca_der: &[u8], - validation_time: time::OffsetDateTime, - metrics: &mut RoaCacheLookupMetrics, - ) -> RoaCacheCrlGate { - if let Some(gate) = self.gates_by_uri.get(crl_uri) { - metrics.crl_gate_reused_crls += 1; - return gate.clone(); - } - - let gate = evaluate_roa_cache_crl_gate( - crl_uri, - expected_crl_sha256_by_uri, - crl_cache, - issuer_ca_der, - validation_time, - metrics, - ); - self.gates_by_uri.insert(crl_uri.to_string(), gate.clone()); - gate - } -} - -fn evaluate_roa_cache_crl_gate( - crl_uri: &str, - expected_crl_sha256_by_uri: &HashMap, - crl_cache: &mut std::collections::HashMap, - issuer_ca_der: &[u8], - validation_time: time::OffsetDateTime, - metrics: &mut RoaCacheLookupMetrics, -) -> RoaCacheCrlGate { - let crl_unchanged = { - let Some(current_crl_hash) = crl_cache - .get_mut(crl_uri) - .map(CachedIssuerCrl::current_sha256_hex) - else { - return RoaCacheCrlGate::Missing; - }; - expected_crl_sha256_by_uri - .get(crl_uri) - .map(|expected| expected == current_crl_hash) - .unwrap_or(false) - }; - if crl_unchanged { - return RoaCacheCrlGate::Unchanged; - } - - let verified_crl = match ensure_issuer_crl_verified(crl_uri, crl_cache, issuer_ca_der) { - Ok(verified_crl) => { - metrics.crl_gate_verified_crls += 1; - verified_crl - } - Err(_) => return RoaCacheCrlGate::Invalid, - }; - if !crl_valid_at_time(&verified_crl.crl, validation_time) { - return RoaCacheCrlGate::Expired; - } - RoaCacheCrlGate::ChangedValid(verified_crl) -} - -#[derive(Clone, Debug)] -pub struct RoaValidationCacheView { - entries_by_uri: HashMap, - issuer_ca_sha256_hex: Option, - ca_validation_context_digest: Option<[u8; 32]>, - policy_fingerprint: Option<[u8; 32]>, - crl_sha256_by_uri: HashMap, - blocked: bool, -} - -#[derive(Clone, Debug)] -pub struct CachedRoaValidationResult { - source_object_hash: [u8; 32], - ee_serial: Option>, - crl_uri: Option, - earliest_safe_reuse_time_unix: i64, - outputs_effective_until_unix: i64, - outputs: Vec, -} - -impl RoaValidationCacheView { - pub fn from_projection( - projection: &RoaCacheProjection, - validation_time: time::OffsetDateTime, - ) -> Self { - let mut entries_by_uri: HashMap = - HashMap::with_capacity(projection.entries.len()); - let issuer_ca_sha256_hex = projection.issuer_ca_sha256_hex.clone(); - let ca_validation_context_digest = projection.ca_validation_context_digest; - let policy_fingerprint = projection.policy_fingerprint; - let crl_sha256_by_uri = projection - .crl_sha256_by_uri - .iter() - .map(|crl| (crl.uri.clone(), crl.sha256.clone())) - .collect::>(); - let blocked = projection - .instance_effective_until - .parse() - .map(|effective_until| effective_until <= validation_time) - .unwrap_or(true); - - if blocked { - return Self { - entries_by_uri, - issuer_ca_sha256_hex, - ca_validation_context_digest, - policy_fingerprint, - crl_sha256_by_uri, - blocked, - }; - } - - for entry in &projection.entries { - let Some(earliest_safe_reuse_time_unix) = entry.earliest_safe_reuse_time_unix else { - continue; - }; - if time::OffsetDateTime::from_unix_timestamp(earliest_safe_reuse_time_unix).is_err() { - continue; - } - let outputs = entry - .outputs - .iter() - .map(|output| VcirLocalOutput { - output_type: VcirOutputType::Vrp, - item_effective_until: output.item_effective_until.clone(), - source_object_uri: entry.source_object_uri.clone(), - source_object_type: VcirSourceObjectType::Roa, - source_object_hash: entry.source_object_hash, - source_ee_cert_hash: output.source_ee_cert_hash, - payload: output.payload.clone(), - rule_hash: output.rule_hash, - }) - .collect::>(); - entries_by_uri.insert( - entry.source_object_uri.clone(), - CachedRoaValidationResult { - source_object_hash: entry.source_object_hash, - ee_serial: entry.ee_serial.clone(), - crl_uri: entry.crl_uri.clone(), - earliest_safe_reuse_time_unix, - outputs_effective_until_unix: entry.outputs_effective_until_unix, - outputs, - }, - ); - } - - Self { - entries_by_uri, - issuer_ca_sha256_hex, - ca_validation_context_digest, - policy_fingerprint, - crl_sha256_by_uri, - blocked, - } - } - - fn matches_current_context( - &self, - issuer_ca_der: &[u8], - ca_validation_context_digest: Option<[u8; 32]>, - policy_fingerprint: Option<[u8; 32]>, - ) -> bool { - if self.blocked { - return false; - } - - let Some(expected_issuer_hash) = self.issuer_ca_sha256_hex.as_ref() else { - return false; - }; - if expected_issuer_hash != &sha256_hex(issuer_ca_der) { - return false; - } - - if let Some(expected_ca_validation_context) = self.ca_validation_context_digest { - if Some(expected_ca_validation_context) != ca_validation_context_digest { - return false; - } - } else { - return false; - } - - if let Some(expected_policy) = self.policy_fingerprint { - if Some(expected_policy) != policy_fingerprint { - return false; - } - } else { - return false; - } - - true - } - - fn lookup( - &self, - file: &PackFile, - crl_cache: &mut std::collections::HashMap, - issuer_ca_der: &[u8], - validation_time: time::OffsetDateTime, - ) -> RoaCacheLookupResult { - self.lookup_with_metrics(file, crl_cache, issuer_ca_der, validation_time, None) - .0 - } - - fn lookup_with_metrics( - &self, - file: &PackFile, - crl_cache: &mut std::collections::HashMap, - issuer_ca_der: &[u8], - validation_time: time::OffsetDateTime, - crl_gate_set: Option<&mut RoaCacheCrlGateSet>, - ) -> (RoaCacheLookupResult, RoaCacheLookupMetrics) { - let mut metrics = RoaCacheLookupMetrics::default(); - let entry_gate_started = Instant::now(); - macro_rules! return_entry_gate { - ($result:expr) => {{ - metrics.entry_gate_nanos = metrics - .entry_gate_nanos - .saturating_add(elapsed_nanos_u64(entry_gate_started)); - return ($result, metrics); - }}; - } - macro_rules! return_crl_gate { - ($started:expr, $result:expr) => {{ - metrics.crl_gate_nanos = metrics - .crl_gate_nanos - .saturating_add(elapsed_nanos_u64($started)); - return ($result, metrics); - }}; - } - macro_rules! return_materialize { - ($started:expr, $result:expr) => {{ - metrics.materialize_nanos = metrics - .materialize_nanos - .saturating_add(elapsed_nanos_u64($started)); - return ($result, metrics); - }}; - } - - if self.blocked { - return_entry_gate!(RoaCacheLookupResult::ExpiredBlocked); - } - - let Some(cached) = self.entries_by_uri.get(file.rsync_uri.as_str()) else { - return_entry_gate!(RoaCacheLookupResult::Miss); - }; - if cached.source_object_hash != file.sha256 { - return_entry_gate!(RoaCacheLookupResult::HashBlocked); - } - if cached.outputs.is_empty() { - return_entry_gate!(RoaCacheLookupResult::Miss); - } - if validation_time.unix_timestamp() < cached.earliest_safe_reuse_time_unix - || cached.outputs_effective_until_unix <= validation_time.unix_timestamp() - { - return_entry_gate!(RoaCacheLookupResult::ExpiredBlocked); - } - - let Some(crl_uri) = cached.crl_uri.as_deref() else { - return_entry_gate!(RoaCacheLookupResult::MetadataBlocked); - }; - let Some(ee_serial) = cached.ee_serial.as_ref() else { - return_entry_gate!(RoaCacheLookupResult::MetadataBlocked); - }; - metrics.entry_gate_nanos = metrics - .entry_gate_nanos - .saturating_add(elapsed_nanos_u64(entry_gate_started)); - - let crl_gate_started = Instant::now(); - let crl_gate = if let Some(crl_gate_set) = crl_gate_set { - crl_gate_set.evaluate( - crl_uri, - &self.crl_sha256_by_uri, - crl_cache, - issuer_ca_der, - validation_time, - &mut metrics, - ) - } else { - evaluate_roa_cache_crl_gate( - crl_uri, - &self.crl_sha256_by_uri, - crl_cache, - issuer_ca_der, - validation_time, - &mut metrics, - ) - }; - let crl_rechecked = match crl_gate { - RoaCacheCrlGate::Unchanged => false, - RoaCacheCrlGate::ChangedValid(verified_crl) => { - if verified_crl.revoked_serials.contains(ee_serial) { - return_crl_gate!(crl_gate_started, RoaCacheLookupResult::RevokedBlocked); - } - true - } - RoaCacheCrlGate::Expired => { - return_crl_gate!(crl_gate_started, RoaCacheLookupResult::ExpiredBlocked); - } - RoaCacheCrlGate::Invalid | RoaCacheCrlGate::Missing => { - return_crl_gate!(crl_gate_started, RoaCacheLookupResult::MetadataBlocked); - } - }; - metrics.crl_gate_nanos = metrics - .crl_gate_nanos - .saturating_add(elapsed_nanos_u64(crl_gate_started)); - - let materialize_started = Instant::now(); - let mut vrps = Vec::with_capacity(cached.outputs.len()); - for output in &cached.outputs { - let VcirLocalOutputPayload::Vrp { - asn, - afi, - prefix_len, - addr, - max_length, - } = &output.payload - else { - return_materialize!(materialize_started, RoaCacheLookupResult::MetadataBlocked); - }; - vrps.push(Vrp { - asn: *asn, - prefix: IpPrefix { - afi: *afi, - prefix_len: *prefix_len, - addr: *addr, - }, - max_length: *max_length, - }); - } - - let ok = RoaTaskOk { - vrps, - local_outputs: cached.outputs.clone(), - reused_from_cache: true, - cache_object_meta: Some(RoaCacheObjectMeta { - source_object_uri: file.rsync_uri.clone(), - source_object_hash: file.sha256, - ee_serial: ee_serial.clone(), - crl_uri: crl_uri.to_string(), - earliest_safe_reuse_time: PackTime::from_utc_offset_datetime( - time::OffsetDateTime::from_unix_timestamp(cached.earliest_safe_reuse_time_unix) - .expect("cached ROA safe reuse time must be valid"), - ), - }), - }; - metrics.materialize_nanos = metrics - .materialize_nanos - .saturating_add(elapsed_nanos_u64(materialize_started)); - if crl_rechecked { - (RoaCacheLookupResult::CrlRecheckHit(ok), metrics) - } else { - (RoaCacheLookupResult::Hit(ok), metrics) - } - } -} - -fn active_roa_cache_view<'a>( - roa_cache: RoaValidationCacheInput<'a>, - issuer_ca_der: &[u8], - stats: &mut RoaValidationCacheStats, - roa_total: usize, -) -> Option<&'a RoaValidationCacheView> { - let view = roa_cache.view?; - let gate_started = Instant::now(); - if view.matches_current_context( - issuer_ca_der, - roa_cache.ca_validation_context_digest, - roa_cache.policy_fingerprint, - ) { - stats.context_gate_nanos = stats - .context_gate_nanos - .saturating_add(gate_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); - Some(view) - } else { - stats.context_gate_nanos = stats - .context_gate_nanos - .saturating_add(gate_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); - stats.blocked_roas += roa_total; - stats.context_blocked_roas += roa_total; - stats.fresh_roas += roa_total; - None - } -} - -#[derive(Debug)] -enum RoaCacheLookupResult { - Hit(RoaTaskOk), - CrlRecheckHit(RoaTaskOk), - Miss, - HashBlocked, - ExpiredBlocked, - RevokedBlocked, - MetadataBlocked, -} - -fn crl_valid_at_time( - crl: &crate::data_model::crl::RpkixCrl, - validation_time: time::OffsetDateTime, -) -> bool { - let this_update = crl.this_update.utc.to_offset(time::UtcOffset::UTC); - let next_update = crl.next_update.utc.to_offset(time::UtcOffset::UTC); - validation_time >= this_update && validation_time < next_update -} - -fn roa_cache_earliest_safe_reuse_time( - ee_not_before: time::OffsetDateTime, - crl_this_update: time::OffsetDateTime, - validation_time: time::OffsetDateTime, -) -> PackTime { - PackTime::from_utc_offset_datetime(ee_not_before.max(crl_this_update).max(validation_time)) -} - -fn record_roa_cache_block( - stats: &mut RoaValidationCacheStats, - lookup_result: &RoaCacheLookupResult, -) { - stats.blocked_roas += 1; - stats.fresh_roas += 1; - match lookup_result { - RoaCacheLookupResult::HashBlocked => stats.hash_blocked_roas += 1, - RoaCacheLookupResult::ExpiredBlocked => stats.expired_blocked_roas += 1, - RoaCacheLookupResult::RevokedBlocked => stats.revoked_blocked_roas += 1, - RoaCacheLookupResult::MetadataBlocked => stats.metadata_blocked_roas += 1, - RoaCacheLookupResult::Hit(_) - | RoaCacheLookupResult::CrlRecheckHit(_) - | RoaCacheLookupResult::Miss => {} - } -} - -#[derive(Clone, Copy)] -pub(crate) struct RoaTask<'a> { - pub(crate) index: usize, - pub(crate) file: &'a PackFile, -} - -#[derive(Debug)] -pub(crate) struct RoaTaskOk { - pub(crate) vrps: Vec, - pub(crate) local_outputs: Vec, - pub(crate) reused_from_cache: bool, - pub(crate) cache_object_meta: Option, -} - -#[derive(Debug)] -pub(crate) struct RoaTaskResult { - pub(crate) publication_point_id: u64, - pub(crate) index: usize, - pub(crate) worker_index: usize, - pub(crate) queue_wait_ms: u64, - pub(crate) worker_ms: u64, - pub(crate) outcome: Result, -} - -/// Process objects from a publication point snapshot using a known issuer CA certificate -/// and its effective resources (resolved via the resource-path, RFC 6487 §7.2). -pub fn process_publication_point_for_issuer( - publication_point: &P, - policy: &Policy, - issuer_ca_der: &[u8], - issuer_ca_rsync_uri: Option<&str>, - issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, - issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, - validation_time: time::OffsetDateTime, - timing: Option<&TimingHandle>, -) -> ObjectsOutput { - process_publication_point_for_issuer_with_options( - publication_point, - policy, - issuer_ca_der, - issuer_ca_rsync_uri, - issuer_effective_ip, - issuer_effective_as, - validation_time, - timing, - true, - ) -} - -pub fn process_publication_point_for_issuer_with_options( - publication_point: &P, - policy: &Policy, - issuer_ca_der: &[u8], - issuer_ca_rsync_uri: Option<&str>, - issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, - issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, - validation_time: time::OffsetDateTime, - timing: Option<&TimingHandle>, - collect_vcir_local_outputs: bool, -) -> ObjectsOutput { - process_publication_point_for_issuer_with_cache_options( - publication_point, - policy, - issuer_ca_der, - issuer_ca_rsync_uri, - issuer_effective_ip, - issuer_effective_as, - validation_time, - timing, - collect_vcir_local_outputs, - RoaValidationCacheInput::disabled(), - ) -} - -pub fn process_publication_point_for_issuer_with_cache_options( - publication_point: &P, - policy: &Policy, - issuer_ca_der: &[u8], - issuer_ca_rsync_uri: Option<&str>, - issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, - issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, - validation_time: time::OffsetDateTime, - timing: Option<&TimingHandle>, - collect_vcir_local_outputs: bool, - roa_cache: RoaValidationCacheInput<'_>, -) -> ObjectsOutput { - process_publication_point_for_issuer_with_cache_options_and_ta_constraints( - publication_point, - policy, - issuer_ca_der, - issuer_ca_rsync_uri, - issuer_effective_ip, - issuer_effective_as, - validation_time, - timing, - collect_vcir_local_outputs, - roa_cache, - None, - ) -} - -/// Serial signed-object processing with an optional, locally configured -/// constraint set for the TA that owns this publication-point tree. -pub fn process_publication_point_for_issuer_with_cache_options_and_ta_constraints< - P: PublicationPointData, ->( - publication_point: &P, - policy: &Policy, - issuer_ca_der: &[u8], - issuer_ca_rsync_uri: Option<&str>, - issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, - issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, - validation_time: time::OffsetDateTime, - timing: Option<&TimingHandle>, - collect_vcir_local_outputs: bool, - roa_cache: RoaValidationCacheInput<'_>, - ta_constraints: Option<&crate::ta_constraints::TaConstraints>, -) -> ObjectsOutput { - let manifest_rsync_uri = publication_point.manifest_rsync_uri(); - let manifest_bytes = publication_point.manifest_bytes(); - let locked_files = publication_point.files(); - let mut warnings: Vec = Vec::new(); - let mut stats = ObjectsStats::default(); - stats.roa_total = locked_files - .iter() - .filter(|f| f.rsync_uri.ends_with(".roa")) - .count(); - stats.aspa_total = locked_files - .iter() - .filter(|f| f.rsync_uri.ends_with(".asa")) - .count(); - let mut roa_cache_stats = RoaValidationCacheStats::for_input(roa_cache, stats.roa_total); - let mut audit: Vec = Vec::new(); - - // Enforce that `manifest_bytes` is actually a manifest object. - let _manifest = match ManifestObject::decode_der_with_strict_options( - manifest_bytes, - policy.strict.cms_der, - policy.strict.name, - ) { - Ok(manifest) => manifest, - Err(e) => { - stats.publication_point_dropped = true; - warnings.push( - Warning::new(format!( - "dropping publication point: manifest decode failed: {e}" - )) - .with_rfc_refs(&[ - RfcRef("RFC 9286 §4"), - RfcRef("RFC 9286 §6.2"), - RfcRef("RFC 9286 §6.6"), - ]) - .with_context(manifest_rsync_uri), - ); - for f in locked_files { - if f.rsync_uri.ends_with(".roa") { - audit.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::Roa, - result: AuditObjectResult::Skipped, - detail: Some("skipped: manifest decode failed".to_string()), - }); - } else if f.rsync_uri.ends_with(".asa") { - audit.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::Aspa, - result: AuditObjectResult::Skipped, - detail: Some("skipped: manifest decode failed".to_string()), - }); - } - } - return ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings, - stats, - audit, - roa_cache_stats, - roa_cache_object_meta: Vec::new(), - }; - } - }; - - if let Some(warning) = - ber_compatible_cms_warning(manifest_bytes, manifest_rsync_uri, "manifest") - { - warnings.push(warning); - } - - // Decode issuer CA once; if it fails we cannot validate ROA/ASPA EE certificates. - let issuer_ca = match decode_resource_certificate_with_policy(issuer_ca_der, policy) { - Ok(v) => v, - Err(e) => { - stats.publication_point_dropped = true; - warnings.push( - Warning::new(format!( - "dropping publication point: issuer CA decode failed: {e}" - )) - .with_rfc_refs(&[RfcRef("RFC 6487 §7.2"), RfcRef("RFC 5280 §6.1")]) - .with_context(manifest_rsync_uri), - ); - for f in locked_files { - if f.rsync_uri.ends_with(".roa") { - audit.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::Roa, - result: AuditObjectResult::Skipped, - detail: Some("skipped: issuer CA decode failed".to_string()), - }); - } else if f.rsync_uri.ends_with(".asa") { - audit.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::Aspa, - result: AuditObjectResult::Skipped, - detail: Some("skipped: issuer CA decode failed".to_string()), - }); - } - } - return ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings, - stats, - audit, - roa_cache_stats, - roa_cache_object_meta: Vec::new(), - }; - } - }; - - // Parse issuer SubjectPublicKeyInfo once and reuse for all EE certificate signature checks. - let issuer_spki = match SubjectPublicKeyInfo::from_der(&issuer_ca.tbs.subject_public_key_info) { - Ok((rem, spki)) if rem.is_empty() => spki, - Ok((rem, _)) => { - stats.publication_point_dropped = true; - warnings.push( - Warning::new(format!( - "dropping publication point: trailing bytes after issuer SPKI DER: {} bytes", - rem.len() - )) - .with_rfc_refs(&[RfcRef("RFC 5280 §4.1.2.7")]) - .with_context(manifest_rsync_uri), - ); - return ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings, - stats, - audit, - roa_cache_stats, - roa_cache_object_meta: Vec::new(), - }; - } - Err(e) => { - stats.publication_point_dropped = true; - warnings.push( - Warning::new(format!( - "dropping publication point: issuer SPKI parse failed: {e}" - )) - .with_rfc_refs(&[RfcRef("RFC 5280 §4.1.2.7")]) - .with_context(manifest_rsync_uri), - ); - return ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings, - stats, - audit, - roa_cache_stats, - roa_cache_object_meta: Vec::new(), - }; - } - }; - - let mut crl_cache: std::collections::HashMap = locked_files - .iter() - .filter(|f| f.rsync_uri.ends_with(".crl")) - .map(|f| { - let bytes = f - .bytes_cloned() - .expect("snapshot CRL bytes must be loadable"); - ( - f.rsync_uri.clone(), - CachedIssuerCrl::Pending { - bytes, - sha256_hex: None, - }, - ) - }) - .collect(); - - let issuer_resources_index = - build_issuer_resources_index(issuer_effective_ip, issuer_effective_as); - - // If the snapshot has signed objects but no CRLs at all, we cannot validate any embedded EE - // certificate paths deterministically (EE CRLDP must reference an rsync URI in the snapshot). - if crl_cache.is_empty() && (stats.roa_total > 0 || stats.aspa_total > 0) { - stats.publication_point_dropped = true; - warnings.push( - Warning::new("dropping publication point: no CRL files in validated publication point") - .with_rfc_refs(&[RfcRef("RFC 6487 §4.8.6"), RfcRef("RFC 9286 §7")]) - .with_context(manifest_rsync_uri), - ); - for f in locked_files { - if f.rsync_uri.ends_with(".roa") { - audit.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::Roa, - result: AuditObjectResult::Skipped, - detail: Some( - "skipped due to missing CRL files in validated publication point" - .to_string(), - ), - }); - } else if f.rsync_uri.ends_with(".asa") { - audit.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::Aspa, - result: AuditObjectResult::Skipped, - detail: Some( - "skipped due to missing CRL files in validated publication point" - .to_string(), - ), - }); - } - } - return ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings, - stats, - audit, - roa_cache_stats, - roa_cache_object_meta: Vec::new(), - }; - } - - let mut vrps: Vec = Vec::new(); - let mut aspas: Vec = Vec::new(); - let mut local_outputs_cache: Vec = Vec::new(); - let mut roa_cache_object_meta: Vec = Vec::new(); - let active_cache_view = active_roa_cache_view( - roa_cache, - issuer_ca_der, - &mut roa_cache_stats, - stats.roa_total, - ); - let mut crl_gate_set = if active_cache_view.is_some() { - Some(RoaCacheCrlGateSet::default()) - } else { - None - }; - - for (idx, file) in locked_files.iter().enumerate() { - if file.rsync_uri.ends_with(".roa") { - let result = if let Some(cache_view) = active_cache_view { - let lookup_started = Instant::now(); - let (lookup_result, lookup_metrics) = cache_view.lookup_with_metrics( - file, - &mut crl_cache, - issuer_ca_der, - validation_time, - crl_gate_set.as_mut(), - ); - roa_cache_stats.record_lookup(elapsed_nanos_u64(lookup_started), lookup_metrics); - match lookup_result { - RoaCacheLookupResult::Hit(ok) => { - roa_cache_stats.hit_roas += 1; - RoaTaskResult { - publication_point_id: 0, - index: idx, - worker_index: 0, - queue_wait_ms: 0, - worker_ms: 0, - outcome: Ok(ok), - } - } - RoaCacheLookupResult::CrlRecheckHit(ok) => { - roa_cache_stats.hit_roas += 1; - roa_cache_stats.crl_recheck_hit_roas += 1; - RoaTaskResult { - publication_point_id: 0, - index: idx, - worker_index: 0, - queue_wait_ms: 0, - worker_ms: 0, - outcome: Ok(ok), - } - } - RoaCacheLookupResult::Miss => { - roa_cache_stats.miss_roas += 1; - roa_cache_stats.fresh_roas += 1; - let task = RoaTask { index: idx, file }; - let _t = timing.as_ref().map(|t| t.span_phase("objects_roa_total")); - validate_roa_task_serial( - task, - manifest_rsync_uri, - issuer_ca_der, - &issuer_ca, - &issuer_spki, - issuer_ca_rsync_uri, - &mut crl_cache, - &issuer_resources_index, - issuer_effective_ip, - issuer_effective_as, - validation_time, - timing, - collect_vcir_local_outputs, - policy.strict.cms_der, - policy.strict.name, - policy.resource_validation_mode, - ta_constraints, - ) - } - blocked @ (RoaCacheLookupResult::HashBlocked - | RoaCacheLookupResult::ExpiredBlocked - | RoaCacheLookupResult::RevokedBlocked - | RoaCacheLookupResult::MetadataBlocked) => { - record_roa_cache_block(&mut roa_cache_stats, &blocked); - let task = RoaTask { index: idx, file }; - let _t = timing.as_ref().map(|t| t.span_phase("objects_roa_total")); - validate_roa_task_serial( - task, - manifest_rsync_uri, - issuer_ca_der, - &issuer_ca, - &issuer_spki, - issuer_ca_rsync_uri, - &mut crl_cache, - &issuer_resources_index, - issuer_effective_ip, - issuer_effective_as, - validation_time, - timing, - collect_vcir_local_outputs, - policy.strict.cms_der, - policy.strict.name, - policy.resource_validation_mode, - ta_constraints, - ) - } - } - } else { - let task = RoaTask { index: idx, file }; - let _t = timing.as_ref().map(|t| t.span_phase("objects_roa_total")); - validate_roa_task_serial( - task, - manifest_rsync_uri, - issuer_ca_der, - &issuer_ca, - &issuer_spki, - issuer_ca_rsync_uri, - &mut crl_cache, - &issuer_resources_index, - issuer_effective_ip, - issuer_effective_as, - validation_time, - timing, - collect_vcir_local_outputs, - policy.strict.cms_der, - policy.strict.name, - policy.resource_validation_mode, - ta_constraints, - ) - }; - match result.outcome { - Ok(mut ok) => { - stats.roa_ok += 1; - vrps.append(&mut ok.vrps); - if collect_vcir_local_outputs || ok.reused_from_cache { - local_outputs_cache.extend(ok.local_outputs); - } - if let Some(meta) = ok.cache_object_meta.take() { - roa_cache_object_meta.push(meta); - } - audit.push(ObjectAuditEntry { - rsync_uri: file.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&file.sha256), - kind: AuditObjectKind::Roa, - result: AuditObjectResult::Ok, - detail: None, - }); - if let Some(warning) = ber_compatible_cms_warning_for_file(file, "ROA") { - warnings.push(warning); - } - } - Err(e) => match policy.signed_object_failure_policy { - SignedObjectFailurePolicy::DropObject => { - audit.push(ObjectAuditEntry { - rsync_uri: file.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&file.sha256), - kind: AuditObjectKind::Roa, - result: AuditObjectResult::Error, - detail: Some(e.to_string()), - }); - let mut refs = vec![RfcRef("RFC 6488 §3"), RfcRef("RFC 9582 §4-§5")]; - refs.extend_from_slice(extra_rfc_refs_for_crl_selection(&e)); - warnings.push( - Warning::new(format!("dropping invalid ROA: {}: {e}", file.rsync_uri)) - .with_rfc_refs(&refs) - .with_context(&file.rsync_uri), - ) - } - SignedObjectFailurePolicy::DropPublicationPoint => { - stats.publication_point_dropped = true; - audit.push(ObjectAuditEntry { - rsync_uri: file.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&file.sha256), - kind: AuditObjectKind::Roa, - result: AuditObjectResult::Error, - detail: Some(e.to_string()), - }); - for f in locked_files.iter().skip(idx + 1) { - if f.rsync_uri.ends_with(".roa") { - audit.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::Roa, - result: AuditObjectResult::Skipped, - detail: Some( - "skipped due to policy=signed_object_failure_policy=drop_publication_point" - .to_string(), - ), - }); - } else if f.rsync_uri.ends_with(".asa") { - audit.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::Aspa, - result: AuditObjectResult::Skipped, - detail: Some( - "skipped due to policy=signed_object_failure_policy=drop_publication_point" - .to_string(), - ), - }); - } - } - let mut refs = vec![RfcRef("RFC 6488 §3"), RfcRef("RFC 9582 §4-§5")]; - refs.extend_from_slice(extra_rfc_refs_for_crl_selection(&e)); - warnings.push( - Warning::new(format!( - "dropping publication point due to invalid ROA: {}: {e}", - file.rsync_uri - )) - .with_rfc_refs(&refs) - .with_context(manifest_rsync_uri), - ); - return ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings, - stats, - audit, - roa_cache_stats, - roa_cache_object_meta: Vec::new(), - }; - } - }, - } - } else if file.rsync_uri.ends_with(".asa") { - let _t = timing.as_ref().map(|t| t.span_phase("objects_aspa_total")); - match process_aspa_with_issuer( - file, - manifest_rsync_uri, - issuer_ca_der, - &issuer_ca, - &issuer_spki, - issuer_ca_rsync_uri, - &mut crl_cache, - &issuer_resources_index, - issuer_effective_ip, - issuer_effective_as, - validation_time, - timing, - collect_vcir_local_outputs, - policy.strict.cms_der, - policy.strict.name, - policy.resource_validation_mode, - ta_constraints, - ) { - Ok((att, local_output)) => { - stats.aspa_ok += 1; - aspas.push(att); - if let Some(local_output) = local_output { - local_outputs_cache.push(local_output); - } - audit.push(ObjectAuditEntry { - rsync_uri: file.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&file.sha256), - kind: AuditObjectKind::Aspa, - result: AuditObjectResult::Ok, - detail: None, - }); - if let Some(warning) = ber_compatible_cms_warning_for_file(file, "ASPA") { - warnings.push(warning); - } - } - Err(e) => match policy.signed_object_failure_policy { - SignedObjectFailurePolicy::DropObject => { - audit.push(ObjectAuditEntry { - rsync_uri: file.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&file.sha256), - kind: AuditObjectKind::Aspa, - result: AuditObjectResult::Error, - detail: Some(e.to_string()), - }); - let mut refs = vec![RfcRef("RFC 6488 §3")]; - refs.extend_from_slice(extra_rfc_refs_for_crl_selection(&e)); - warnings.push( - Warning::new(format!("dropping invalid ASPA: {}: {e}", file.rsync_uri)) - .with_rfc_refs(&refs) - .with_context(&file.rsync_uri), - ) - } - SignedObjectFailurePolicy::DropPublicationPoint => { - stats.publication_point_dropped = true; - audit.push(ObjectAuditEntry { - rsync_uri: file.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&file.sha256), - kind: AuditObjectKind::Aspa, - result: AuditObjectResult::Error, - detail: Some(e.to_string()), - }); - for f in locked_files.iter().skip(idx + 1) { - if f.rsync_uri.ends_with(".roa") { - audit.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::Roa, - result: AuditObjectResult::Skipped, - detail: Some( - "skipped due to policy=signed_object_failure_policy=drop_publication_point" - .to_string(), - ), - }); - } else if f.rsync_uri.ends_with(".asa") { - audit.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::Aspa, - result: AuditObjectResult::Skipped, - detail: Some( - "skipped due to policy=signed_object_failure_policy=drop_publication_point" - .to_string(), - ), - }); - } - } - let mut refs = vec![RfcRef("RFC 6488 §3")]; - refs.extend_from_slice(extra_rfc_refs_for_crl_selection(&e)); - warnings.push( - Warning::new(format!( - "dropping publication point due to invalid ASPA: {}: {e}", - file.rsync_uri - )) - .with_rfc_refs(&refs) - .with_context(manifest_rsync_uri), - ); - return ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings, - stats, - audit, - roa_cache_stats, - roa_cache_object_meta: Vec::new(), - }; - } - }, - } - } - } - - roa_cache_stats.record_to_timing(timing); - - ObjectsOutput { - vrps, - aspas, - router_keys: Vec::new(), - local_outputs_cache, - warnings, - stats, - audit, - roa_cache_stats, - roa_cache_object_meta, - } -} - -pub fn process_publication_point_for_issuer_parallel_roa( - publication_point: &P, - policy: &Policy, - issuer_ca_der: &[u8], - issuer_ca_rsync_uri: Option<&str>, - issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, - issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, - validation_time: time::OffsetDateTime, - timing: Option<&TimingHandle>, - config: &ParallelPhase2Config, -) -> ObjectsOutput { - process_publication_point_for_issuer_parallel_roa_with_options( - publication_point, - policy, - issuer_ca_der, - issuer_ca_rsync_uri, - issuer_effective_ip, - issuer_effective_as, - validation_time, - timing, - config, - true, - ) -} - -pub fn process_publication_point_for_issuer_parallel_roa_with_options( - publication_point: &P, - policy: &Policy, - issuer_ca_der: &[u8], - issuer_ca_rsync_uri: Option<&str>, - issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, - issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, - validation_time: time::OffsetDateTime, - timing: Option<&TimingHandle>, - config: &ParallelPhase2Config, - collect_vcir_local_outputs: bool, -) -> ObjectsOutput { - process_publication_point_for_issuer_parallel_roa_with_cache_options( - publication_point, - policy, - issuer_ca_der, - issuer_ca_rsync_uri, - issuer_effective_ip, - issuer_effective_as, - validation_time, - timing, - config, - collect_vcir_local_outputs, - RoaValidationCacheInput::disabled(), - ) -} - -pub fn process_publication_point_for_issuer_parallel_roa_with_cache_options< - P: PublicationPointData, ->( - publication_point: &P, - policy: &Policy, - issuer_ca_der: &[u8], - issuer_ca_rsync_uri: Option<&str>, - issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, - issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, - validation_time: time::OffsetDateTime, - timing: Option<&TimingHandle>, - config: &ParallelPhase2Config, - collect_vcir_local_outputs: bool, - roa_cache: RoaValidationCacheInput<'_>, -) -> ObjectsOutput { - if config.object_workers <= 1 - || policy.signed_object_failure_policy == SignedObjectFailurePolicy::DropPublicationPoint - { - return process_publication_point_for_issuer_with_cache_options( - publication_point, - policy, - issuer_ca_der, - issuer_ca_rsync_uri, - issuer_effective_ip, - issuer_effective_as, - validation_time, - timing, - collect_vcir_local_outputs, - roa_cache, - ); - } - - let pool = match ParallelRoaWorkerPool::new(config) { - Ok(pool) => pool, - Err(_) => { - return process_publication_point_for_issuer_with_cache_options( - publication_point, - policy, - issuer_ca_der, - issuer_ca_rsync_uri, - issuer_effective_ip, - issuer_effective_as, - validation_time, - timing, - collect_vcir_local_outputs, - roa_cache, - ); - } - }; - - process_publication_point_for_issuer_parallel_roa_with_pool_cache_options( - publication_point, - policy, - issuer_ca_der, - issuer_ca_rsync_uri, - issuer_effective_ip, - issuer_effective_as, - validation_time, - timing, - &pool, - collect_vcir_local_outputs, - roa_cache, - ) -} - -pub fn process_publication_point_for_issuer_parallel_roa_with_pool( - publication_point: &P, - policy: &Policy, - issuer_ca_der: &[u8], - issuer_ca_rsync_uri: Option<&str>, - issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, - issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, - validation_time: time::OffsetDateTime, - timing: Option<&TimingHandle>, - pool: &ParallelRoaWorkerPool, -) -> ObjectsOutput { - process_publication_point_for_issuer_parallel_roa_with_pool_options( - publication_point, - policy, - issuer_ca_der, - issuer_ca_rsync_uri, - issuer_effective_ip, - issuer_effective_as, - validation_time, - timing, - pool, - true, - ) -} - -pub fn process_publication_point_for_issuer_parallel_roa_with_pool_options< - P: PublicationPointData, ->( - publication_point: &P, - policy: &Policy, - issuer_ca_der: &[u8], - issuer_ca_rsync_uri: Option<&str>, - issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, - issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, - validation_time: time::OffsetDateTime, - timing: Option<&TimingHandle>, - pool: &ParallelRoaWorkerPool, - collect_vcir_local_outputs: bool, -) -> ObjectsOutput { - process_publication_point_for_issuer_parallel_roa_with_pool_cache_options( - publication_point, - policy, - issuer_ca_der, - issuer_ca_rsync_uri, - issuer_effective_ip, - issuer_effective_as, - validation_time, - timing, - pool, - collect_vcir_local_outputs, - RoaValidationCacheInput::disabled(), - ) -} - -pub fn process_publication_point_for_issuer_parallel_roa_with_pool_cache_options< - P: PublicationPointData, ->( - publication_point: &P, - policy: &Policy, - issuer_ca_der: &[u8], - issuer_ca_rsync_uri: Option<&str>, - issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, - issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, - validation_time: time::OffsetDateTime, - timing: Option<&TimingHandle>, - pool: &ParallelRoaWorkerPool, - collect_vcir_local_outputs: bool, - roa_cache: RoaValidationCacheInput<'_>, -) -> ObjectsOutput { - if policy.signed_object_failure_policy == SignedObjectFailurePolicy::DropPublicationPoint { - return process_publication_point_for_issuer_with_options( - publication_point, - policy, - issuer_ca_der, - issuer_ca_rsync_uri, - issuer_effective_ip, - issuer_effective_as, - validation_time, - timing, - collect_vcir_local_outputs, - ); - } - - process_publication_point_for_issuer_parallel_roa_inner( - publication_point, - policy, - issuer_ca_der, - issuer_ca_rsync_uri, - issuer_effective_ip, - issuer_effective_as, - validation_time, - timing, - pool, - collect_vcir_local_outputs, - roa_cache, - ) - .unwrap_or_else(|_| { - process_publication_point_for_issuer_with_cache_options( - publication_point, - policy, - issuer_ca_der, - issuer_ca_rsync_uri, - issuer_effective_ip, - issuer_effective_as, - validation_time, - timing, - collect_vcir_local_outputs, - roa_cache, - ) - }) -} - -#[derive(Clone)] -pub(crate) struct RoaTaskShared { - locked_files: Arc<[PackFile]>, - manifest_rsync_uri: Arc, - issuer_ca_der: Arc<[u8]>, - issuer_ca: Arc, - issuer_spki_der: Arc<[u8]>, - issuer_ca_rsync_uri: Option>, - crl_cache: Arc>>, - issuer_resources_index: Arc, - issuer_effective_ip: Option>, - issuer_effective_as: Option>, - resource_validation_mode: ResourceValidationMode, - /// Immutable constraints snapshot selected by the owning TAL. Every - /// ROA task for a publication point shares this Arc, so workers do not - /// need a mutable/global policy lookup or a copy of the interval rules. - ta_constraints: Option>, -} - -#[derive(Clone)] -pub(crate) struct OwnedRoaTask { - pub(crate) publication_point_id: u64, - index: usize, - shared: Arc, - validation_time: time::OffsetDateTime, - collect_vcir_local_outputs: bool, - strict_cms_der: bool, - strict_name: bool, - resource_validation_mode: ResourceValidationMode, - pub(crate) submitted_at: Option, -} - -#[derive(Clone)] -struct RoaTaskExecutor; - -impl ObjectTaskExecutor for RoaTaskExecutor { - fn execute(&self, worker_index: usize, task: OwnedRoaTask) -> RoaTaskResult { - validate_owned_roa_task(worker_index, task) - } -} - -pub struct ParallelRoaWorkerPool { - pool: Mutex>, -} - -impl ParallelRoaWorkerPool { - pub fn new(config: &ParallelPhase2Config) -> Result { - if config.object_workers <= 1 { - return Err("parallel ROA worker pool requires object_workers > 1".to_string()); - } - - Ok(Self { - pool: Mutex::new(ObjectWorkerPool::new( - config.object_workers, - config.worker_queue_capacity, - RoaTaskExecutor, - )?), - }) - } - - pub(crate) fn try_submit_round_robin( - &self, - task: OwnedRoaTask, - ) -> Result> { - self.pool - .lock() - .expect("parallel ROA worker pool lock") - .try_submit_round_robin(task) - } - - pub(crate) fn recv_result_timeout( - &self, - timeout: Duration, - ) -> Result, String> { - self.pool - .lock() - .expect("parallel ROA worker pool lock") - .recv_result_timeout(timeout) - } -} - -fn validate_owned_roa_task(worker_index: usize, task: OwnedRoaTask) -> RoaTaskResult { - let worker_started = Instant::now(); - let queue_wait_ms = task - .submitted_at - .map(|submitted_at| worker_started.saturating_duration_since(submitted_at)) - .map(|duration| duration.as_millis() as u64) - .unwrap_or(0); - let shared = task.shared.as_ref(); - let file = task - .shared - .locked_files - .get(task.index) - .expect("ROA task index must reference locked file"); - let issuer_spki = match SubjectPublicKeyInfo::from_der(shared.issuer_spki_der.as_ref()) { - Ok((rem, spki)) if rem.is_empty() => spki, - Ok((rem, _)) => { - return RoaTaskResult { - publication_point_id: task.publication_point_id, - index: task.index, - worker_index, - queue_wait_ms, - worker_ms: worker_started.elapsed().as_millis() as u64, - outcome: Err(ObjectValidateError::CertPath( - CertPathError::IssuerSpkiTrailingBytes(rem.len()), - )), - }; - } - Err(e) => { - return RoaTaskResult { - publication_point_id: task.publication_point_id, - index: task.index, - worker_index, - queue_wait_ms, - worker_ms: worker_started.elapsed().as_millis() as u64, - outcome: Err(ObjectValidateError::CertPath( - CertPathError::IssuerSpkiParse(e.to_string()), - )), - }; - } - }; - let outcome = process_roa_with_issuer_parallel_cached( - file, - shared.manifest_rsync_uri.as_ref(), - shared.issuer_ca_der.as_ref(), - shared.issuer_ca.as_ref(), - &issuer_spki, - shared.issuer_ca_rsync_uri.as_deref(), - shared.crl_cache.as_ref(), - shared.issuer_resources_index.as_ref(), - shared.issuer_effective_ip.as_deref(), - shared.issuer_effective_as.as_deref(), - task.validation_time, - None, - task.collect_vcir_local_outputs, - task.strict_cms_der, - task.strict_name, - task.resource_validation_mode, - shared.ta_constraints.as_deref(), - ) - .map(|(vrps, local_outputs, cache_object_meta)| RoaTaskOk { - vrps, - local_outputs, - reused_from_cache: false, - cache_object_meta, - }); - - RoaTaskResult { - publication_point_id: task.publication_point_id, - index: task.index, - worker_index, - queue_wait_ms, - worker_ms: worker_started.elapsed().as_millis() as u64, - outcome, - } -} - -pub(crate) enum ParallelObjectsPrepare { - Complete(ObjectsOutput), - Staged(ParallelObjectsStage), -} - -pub(crate) struct ParallelObjectsStage { - pub(crate) publication_point_id: u64, - shared: Arc, - validation_time: time::OffsetDateTime, - collect_vcir_local_outputs: bool, - strict_cms_der: bool, - strict_name: bool, - resource_validation_mode: ResourceValidationMode, - roa_task_indices: Vec, - cached_roa_results: Vec, - roa_cache_stats: RoaValidationCacheStats, - warnings: Vec, - stats: ObjectsStats, - audit: Vec, -} - -impl ParallelObjectsStage { - #[cfg(test)] - pub(crate) fn build_roa_tasks(&self) -> Vec { - let mut tasks = Vec::with_capacity(self.roa_task_count()); - self.extend_roa_tasks(|task| tasks.push(task)); - tasks - } - - pub(crate) fn append_roa_tasks_to( - &self, - pending: &mut std::collections::VecDeque, - ) { - self.extend_roa_tasks(|task| pending.push_back(task)); - } - - fn extend_roa_tasks(&self, mut push: F) - where - F: FnMut(OwnedRoaTask), - { - let shared = self.shared.clone(); - self.roa_task_indices.iter().for_each(|index| { - push(OwnedRoaTask { - publication_point_id: self.publication_point_id, - index: *index, - shared: shared.clone(), - validation_time: self.validation_time, - collect_vcir_local_outputs: self.collect_vcir_local_outputs, - strict_cms_der: self.strict_cms_der, - strict_name: self.strict_name, - resource_validation_mode: self.resource_validation_mode, - submitted_at: None, - }); - }); - } - - pub(crate) fn roa_task_count(&self) -> usize { - self.roa_task_indices.len() - } - - pub(crate) fn aspa_task_count(&self) -> usize { - self.stats.aspa_total - } - - pub(crate) fn locked_file_count(&self) -> usize { - self.shared.locked_files.len() - } -} - -pub(crate) fn prepare_publication_point_for_parallel_roa_with_cache( - publication_point_id: u64, - publication_point: &P, - policy: &Policy, - issuer_ca_der: &[u8], - issuer_ca_rsync_uri: Option<&str>, - issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, - issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, - validation_time: time::OffsetDateTime, - collect_vcir_local_outputs: bool, - roa_cache: RoaValidationCacheInput<'_>, -) -> ParallelObjectsPrepare { - prepare_publication_point_for_parallel_roa_with_cache_and_ta_constraints( - publication_point_id, - publication_point, - policy, - issuer_ca_der, - issuer_ca_rsync_uri, - issuer_effective_ip, - issuer_effective_as, - validation_time, - collect_vcir_local_outputs, - roa_cache, - None, - ) -} - -/// Prepare a publication point for parallel ROA validation with the -/// immutable constraints snapshot belonging to its TAL. The snapshot is -/// moved into the stage-owned shared payload and therefore remains available -/// to detached ROA workers after the scoped phase-2 stage worker returns. -pub(crate) fn prepare_publication_point_for_parallel_roa_with_cache_and_ta_constraints< - P: PublicationPointData, ->( - publication_point_id: u64, - publication_point: &P, - policy: &Policy, - issuer_ca_der: &[u8], - issuer_ca_rsync_uri: Option<&str>, - issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, - issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, - validation_time: time::OffsetDateTime, - collect_vcir_local_outputs: bool, - roa_cache: RoaValidationCacheInput<'_>, - ta_constraints: Option>, -) -> ParallelObjectsPrepare { - let manifest_rsync_uri = publication_point.manifest_rsync_uri(); - let manifest_bytes = publication_point.manifest_bytes(); - let locked_files = publication_point.files(); - let mut warnings: Vec = Vec::new(); - let mut stats = ObjectsStats::default(); - stats.roa_total = locked_files - .iter() - .filter(|f| f.rsync_uri.ends_with(".roa")) - .count(); - stats.aspa_total = locked_files - .iter() - .filter(|f| f.rsync_uri.ends_with(".asa")) - .count(); - let mut roa_cache_stats = RoaValidationCacheStats::for_input(roa_cache, stats.roa_total); - let mut audit: Vec = Vec::new(); - - let _manifest = match ManifestObject::decode_der_with_strict_options( - manifest_bytes, - policy.strict.cms_der, - policy.strict.name, - ) { - Ok(manifest) => manifest, - Err(e) => { - stats.publication_point_dropped = true; - warnings.push( - Warning::new(format!( - "dropping publication point: manifest decode failed: {e}" - )) - .with_rfc_refs(&[ - RfcRef("RFC 9286 §4"), - RfcRef("RFC 9286 §6.2"), - RfcRef("RFC 9286 §6.6"), - ]) - .with_context(manifest_rsync_uri), - ); - for f in locked_files { - if f.rsync_uri.ends_with(".roa") { - audit.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::Roa, - result: AuditObjectResult::Skipped, - detail: Some("skipped: manifest decode failed".to_string()), - }); - } else if f.rsync_uri.ends_with(".asa") { - audit.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::Aspa, - result: AuditObjectResult::Skipped, - detail: Some("skipped: manifest decode failed".to_string()), - }); - } - } - return ParallelObjectsPrepare::Complete(ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings, - stats, - audit, - roa_cache_stats, - roa_cache_object_meta: Vec::new(), - }); - } - }; - - if let Some(warning) = - ber_compatible_cms_warning(manifest_bytes, manifest_rsync_uri, "manifest") - { - warnings.push(warning); - } - - let issuer_ca = match decode_resource_certificate_with_policy(issuer_ca_der, policy) { - Ok(v) => v, - Err(e) => { - stats.publication_point_dropped = true; - warnings.push( - Warning::new(format!( - "dropping publication point: issuer CA decode failed: {e}" - )) - .with_rfc_refs(&[RfcRef("RFC 6487 §7.2"), RfcRef("RFC 5280 §6.1")]) - .with_context(manifest_rsync_uri), - ); - for f in locked_files { - if f.rsync_uri.ends_with(".roa") { - audit.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::Roa, - result: AuditObjectResult::Skipped, - detail: Some("skipped: issuer CA decode failed".to_string()), - }); - } else if f.rsync_uri.ends_with(".asa") { - audit.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::Aspa, - result: AuditObjectResult::Skipped, - detail: Some("skipped: issuer CA decode failed".to_string()), - }); - } - } - return ParallelObjectsPrepare::Complete(ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings, - stats, - audit, - roa_cache_stats, - roa_cache_object_meta: Vec::new(), - }); - } - }; - - match SubjectPublicKeyInfo::from_der(&issuer_ca.tbs.subject_public_key_info) { - Ok((rem, _)) if rem.is_empty() => {} - Ok((rem, _)) => { - stats.publication_point_dropped = true; - warnings.push( - Warning::new(format!( - "dropping publication point: trailing bytes after issuer SPKI DER: {} bytes", - rem.len() - )) - .with_rfc_refs(&[RfcRef("RFC 5280 §4.1.2.7")]) - .with_context(manifest_rsync_uri), - ); - return ParallelObjectsPrepare::Complete(ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings, - stats, - audit, - roa_cache_stats, - roa_cache_object_meta: Vec::new(), - }); - } - Err(e) => { - stats.publication_point_dropped = true; - warnings.push( - Warning::new(format!( - "dropping publication point: issuer SPKI parse failed: {e}" - )) - .with_rfc_refs(&[RfcRef("RFC 5280 §4.1.2.7")]) - .with_context(manifest_rsync_uri), - ); - return ParallelObjectsPrepare::Complete(ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings, - stats, - audit, - roa_cache_stats, - roa_cache_object_meta: Vec::new(), - }); - } - } - - let mut crl_cache: std::collections::HashMap = locked_files - .iter() - .filter(|f| f.rsync_uri.ends_with(".crl")) - .map(|f| { - let bytes = f - .bytes_cloned() - .expect("snapshot CRL bytes must be loadable"); - ( - f.rsync_uri.clone(), - CachedIssuerCrl::Pending { - bytes, - sha256_hex: None, - }, - ) - }) - .collect(); - - if crl_cache.is_empty() && (stats.roa_total > 0 || stats.aspa_total > 0) { - stats.publication_point_dropped = true; - warnings.push( - Warning::new("dropping publication point: no CRL files in validated publication point") - .with_rfc_refs(&[RfcRef("RFC 6487 §4.8.6"), RfcRef("RFC 9286 §7")]) - .with_context(manifest_rsync_uri), - ); - for f in locked_files { - if f.rsync_uri.ends_with(".roa") { - audit.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::Roa, - result: AuditObjectResult::Skipped, - detail: Some( - "skipped due to missing CRL files in validated publication point" - .to_string(), - ), - }); - } else if f.rsync_uri.ends_with(".asa") { - audit.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::Aspa, - result: AuditObjectResult::Skipped, - detail: Some( - "skipped due to missing CRL files in validated publication point" - .to_string(), - ), - }); - } - } - return ParallelObjectsPrepare::Complete(ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings, - stats, - audit, - roa_cache_stats, - roa_cache_object_meta: Vec::new(), - }); - } - - let active_cache_view = active_roa_cache_view( - roa_cache, - issuer_ca_der, - &mut roa_cache_stats, - stats.roa_total, - ); - let mut crl_gate_set = if active_cache_view.is_some() { - Some(RoaCacheCrlGateSet::default()) - } else { - None - }; - let mut roa_task_indices = Vec::new(); - let mut cached_roa_results = Vec::new(); - for (index, file) in locked_files.iter().enumerate() { - if !file.rsync_uri.ends_with(".roa") { - continue; - } - if let Some(cache_view) = active_cache_view { - let lookup_started = Instant::now(); - let (lookup_result, lookup_metrics) = cache_view.lookup_with_metrics( - file, - &mut crl_cache, - issuer_ca_der, - validation_time, - crl_gate_set.as_mut(), - ); - roa_cache_stats.record_lookup(elapsed_nanos_u64(lookup_started), lookup_metrics); - match lookup_result { - RoaCacheLookupResult::Hit(ok) => { - roa_cache_stats.hit_roas += 1; - cached_roa_results.push(RoaTaskResult { - publication_point_id, - index, - worker_index: usize::MAX, - queue_wait_ms: 0, - worker_ms: 0, - outcome: Ok(ok), - }); - } - RoaCacheLookupResult::CrlRecheckHit(ok) => { - roa_cache_stats.hit_roas += 1; - roa_cache_stats.crl_recheck_hit_roas += 1; - cached_roa_results.push(RoaTaskResult { - publication_point_id, - index, - worker_index: usize::MAX, - queue_wait_ms: 0, - worker_ms: 0, - outcome: Ok(ok), - }); - } - RoaCacheLookupResult::Miss => { - roa_cache_stats.miss_roas += 1; - roa_cache_stats.fresh_roas += 1; - roa_task_indices.push(index); - } - blocked @ (RoaCacheLookupResult::HashBlocked - | RoaCacheLookupResult::ExpiredBlocked - | RoaCacheLookupResult::RevokedBlocked - | RoaCacheLookupResult::MetadataBlocked) => { - record_roa_cache_block(&mut roa_cache_stats, &blocked); - roa_task_indices.push(index); - } - } - } else { - roa_task_indices.push(index); - } - } - - ParallelObjectsPrepare::Staged(ParallelObjectsStage { - publication_point_id, - shared: Arc::new(RoaTaskShared { - locked_files: Arc::<[PackFile]>::from(locked_files.to_vec()), - manifest_rsync_uri: Arc::::from(manifest_rsync_uri), - issuer_ca_der: Arc::<[u8]>::from(issuer_ca_der.to_vec()), - issuer_spki_der: Arc::<[u8]>::from(issuer_ca.tbs.subject_public_key_info.clone()), - issuer_ca: Arc::new(issuer_ca), - issuer_ca_rsync_uri: issuer_ca_rsync_uri.map(Arc::::from), - crl_cache: Arc::new(Mutex::new(crl_cache)), - issuer_resources_index: Arc::new(build_issuer_resources_index( - issuer_effective_ip, - issuer_effective_as, - )), - issuer_effective_ip: issuer_effective_ip.cloned().map(Arc::new), - issuer_effective_as: issuer_effective_as.cloned().map(Arc::new), - resource_validation_mode: policy.resource_validation_mode, - ta_constraints, - }), - validation_time, - collect_vcir_local_outputs, - strict_cms_der: policy.strict.cms_der, - strict_name: policy.strict.name, - resource_validation_mode: policy.resource_validation_mode, - roa_task_indices, - cached_roa_results, - roa_cache_stats, - warnings, - stats, - audit, - }) -} - -pub(crate) fn reduce_parallel_roa_stage( - stage: ParallelObjectsStage, - mut roa_results: Vec, - timing: Option<&TimingHandle>, -) -> Result { - roa_results.extend(stage.cached_roa_results); - roa_results.sort_by_key(|result| result.index); - let mut roa_results = roa_results.into_iter().peekable(); - let shared = stage.shared.clone(); - let mut aspa_crl_cache = shared - .crl_cache - .lock() - .expect("parallel ROA CRL cache lock") - .clone(); - let issuer_spki = SubjectPublicKeyInfo::from_der(shared.issuer_spki_der.as_ref()) - .map_err(|e| e.to_string())? - .1; - let collect_vcir_local_outputs = stage.collect_vcir_local_outputs; - let validation_time = stage.validation_time; - let strict_cms_der = stage.strict_cms_der; - let strict_name = stage.strict_name; - let roa_cache_stats = stage.roa_cache_stats; - let mut stats = stage.stats; - let mut warnings = stage.warnings; - let mut audit = stage.audit; - let mut vrps: Vec = Vec::new(); - let mut aspas: Vec = Vec::new(); - let mut local_outputs_cache: Vec = Vec::new(); - let mut roa_cache_object_meta: Vec = Vec::new(); - - for (idx, file) in shared.locked_files.iter().enumerate() { - if file.rsync_uri.ends_with(".roa") { - let result = match roa_results.peek() { - Some(result) if result.index == idx => roa_results - .next() - .expect("peeked ROA task result must be present"), - Some(result) => { - return Err(format!( - "unexpected ROA task result index {} while reducing {} at index {}", - result.index, file.rsync_uri, idx - )); - } - None => { - return Err(format!("missing ROA task result for {}", file.rsync_uri)); - } - }; - match result.outcome { - Ok(mut ok) => { - stats.roa_ok += 1; - vrps.append(&mut ok.vrps); - if collect_vcir_local_outputs || ok.reused_from_cache { - local_outputs_cache.extend(ok.local_outputs); - } - if let Some(meta) = ok.cache_object_meta.take() { - roa_cache_object_meta.push(meta); - } - audit.push(ObjectAuditEntry { - rsync_uri: file.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&file.sha256), - kind: AuditObjectKind::Roa, - result: AuditObjectResult::Ok, - detail: None, - }); - if let Some(warning) = ber_compatible_cms_warning_for_file(file, "ROA") { - warnings.push(warning); - } - } - Err(e) => { - audit.push(ObjectAuditEntry { - rsync_uri: file.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&file.sha256), - kind: AuditObjectKind::Roa, - result: AuditObjectResult::Error, - detail: Some(e.to_string()), - }); - let mut refs = vec![RfcRef("RFC 6488 §3"), RfcRef("RFC 9582 §4-§5")]; - refs.extend_from_slice(extra_rfc_refs_for_crl_selection(&e)); - warnings.push( - Warning::new(format!("dropping invalid ROA: {}: {e}", file.rsync_uri)) - .with_rfc_refs(&refs) - .with_context(&file.rsync_uri), - ) - } - } - } else if file.rsync_uri.ends_with(".asa") { - let _t = timing.as_ref().map(|t| t.span_phase("objects_aspa_total")); - match process_aspa_with_issuer( - file, - shared.manifest_rsync_uri.as_ref(), - shared.issuer_ca_der.as_ref(), - shared.issuer_ca.as_ref(), - &issuer_spki, - shared.issuer_ca_rsync_uri.as_deref(), - &mut aspa_crl_cache, - shared.issuer_resources_index.as_ref(), - shared.issuer_effective_ip.as_deref(), - shared.issuer_effective_as.as_deref(), - validation_time, - timing, - collect_vcir_local_outputs, - strict_cms_der, - strict_name, - shared.resource_validation_mode, - shared.ta_constraints.as_deref(), - ) { - Ok((att, local_output)) => { - stats.aspa_ok += 1; - aspas.push(att); - if let Some(local_output) = local_output { - local_outputs_cache.push(local_output); - } - audit.push(ObjectAuditEntry { - rsync_uri: file.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&file.sha256), - kind: AuditObjectKind::Aspa, - result: AuditObjectResult::Ok, - detail: None, - }); - if let Some(warning) = ber_compatible_cms_warning_for_file(file, "ASPA") { - warnings.push(warning); - } - } - Err(e) => { - audit.push(ObjectAuditEntry { - rsync_uri: file.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&file.sha256), - kind: AuditObjectKind::Aspa, - result: AuditObjectResult::Error, - detail: Some(e.to_string()), - }); - let mut refs = vec![RfcRef("RFC 6488 §3")]; - refs.extend_from_slice(extra_rfc_refs_for_crl_selection(&e)); - warnings.push( - Warning::new(format!("dropping invalid ASPA: {}: {e}", file.rsync_uri)) - .with_rfc_refs(&refs) - .with_context(&file.rsync_uri), - ) - } - } - } - } - if let Some(result) = roa_results.next() { - return Err(format!( - "unexpected trailing ROA task result at index {}", - result.index - )); - } - - roa_cache_stats.record_to_timing(timing); - - Ok(ObjectsOutput { - vrps, - aspas, - router_keys: Vec::new(), - local_outputs_cache, - warnings, - stats, - audit, - roa_cache_stats, - roa_cache_object_meta, - }) -} - -fn process_publication_point_for_issuer_parallel_roa_inner( - publication_point: &P, - _policy: &Policy, - issuer_ca_der: &[u8], - issuer_ca_rsync_uri: Option<&str>, - issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, - issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, - validation_time: time::OffsetDateTime, - timing: Option<&TimingHandle>, - pool: &ParallelRoaWorkerPool, - collect_vcir_local_outputs: bool, - roa_cache: RoaValidationCacheInput<'_>, -) -> Result { - let stage = match prepare_publication_point_for_parallel_roa_with_cache( - 0, - publication_point, - _policy, - issuer_ca_der, - issuer_ca_rsync_uri, - issuer_effective_ip, - issuer_effective_as, - validation_time, - collect_vcir_local_outputs, - roa_cache, - ) { - ParallelObjectsPrepare::Complete(out) => return Ok(out), - ParallelObjectsPrepare::Staged(stage) => stage, - }; - - let roa_task_count = stage.roa_task_count(); - let mut pending = std::collections::VecDeque::with_capacity(roa_task_count); - stage.append_roa_tasks_to(&mut pending); - let mut worker_pool = pool - .pool - .lock() - .map_err(|_| "parallel ROA worker pool lock poisoned".to_string())?; - - while let Some(task) = pending.pop_front() { - match worker_pool.try_submit_round_robin(task) { - Ok(_) => {} - Err(ObjectWorkerSubmitError::QueueFull { task, .. }) => { - pending.push_front(task); - std::thread::yield_now(); - } - Err(ObjectWorkerSubmitError::Disconnected { .. }) => { - return Err("parallel ROA worker queue disconnected".to_string()); - } - } - } - - let mut roa_results = Vec::with_capacity(roa_task_count); - while roa_results.len() < roa_task_count { - let Some(result) = worker_pool.recv_result_timeout(Duration::from_secs(30))? else { - return Err("parallel ROA worker timed out".to_string()); - }; - roa_results.push(result); - } - drop(worker_pool); - - reduce_parallel_roa_stage(stage, roa_results, timing) -} -/// Compatibility wrapper that processes a publication point snapshot. -pub fn process_publication_point_snapshot_for_issuer( - pack: &PublicationPointSnapshot, - policy: &Policy, - issuer_ca_der: &[u8], - issuer_ca_rsync_uri: Option<&str>, - issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, - issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, - validation_time: time::OffsetDateTime, - timing: Option<&TimingHandle>, -) -> ObjectsOutput { - process_publication_point_for_issuer( - pack, - policy, - issuer_ca_der, - issuer_ca_rsync_uri, - issuer_effective_ip, - issuer_effective_as, - validation_time, - timing, - ) -} - -pub fn process_publication_point_snapshot_for_issuer_parallel_roa( - pack: &PublicationPointSnapshot, - policy: &Policy, - issuer_ca_der: &[u8], - issuer_ca_rsync_uri: Option<&str>, - issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, - issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, - validation_time: time::OffsetDateTime, - timing: Option<&TimingHandle>, - config: &ParallelPhase2Config, -) -> ObjectsOutput { - process_publication_point_for_issuer_parallel_roa( - pack, - policy, - issuer_ca_der, - issuer_ca_rsync_uri, - issuer_effective_ip, - issuer_effective_as, - validation_time, - timing, - config, - ) -} - -#[derive(Debug, thiserror::Error)] -pub(crate) enum ObjectValidateError { - #[error("object bytes load failed: {0}")] - BytesLoad(String), - - #[error("ROA decode failed: {0}")] - RoaDecode(#[from] RoaDecodeError), - - #[error("ROA embedded EE resource validation failed: {0}")] - RoaEeResources(#[from] RoaValidateError), - - #[error("ASPA decode failed: {0}")] - AspaDecode(#[from] AspaDecodeError), - - #[error("ASPA embedded EE resource validation failed: {0}")] - AspaEeResources(#[from] AspaValidateError), - - #[error("CMS signature verification failed: {0}")] - Signature(#[from] SignedObjectVerifyError), - - #[error("EE certificate path validation failed: {0}")] - CertPath(#[from] CertPathError), - - #[error( - "certificate CRLDistributionPoints URIs missing (cannot select issuer CRL) (RFC 6487 §4.8.6)" - )] - MissingCrlDpUris, - - #[error( - "no CRL available in publication point snapshot (cannot validate certificates) (RFC 9286 §7; RFC 6487 §4.8.6)" - )] - MissingCrlInPack, - - #[error( - "CRL referenced by CRLDistributionPoints not found in publication point snapshot: {0} (RFC 6487 §4.8.6; RFC 9286 §4.2.1)" - )] - CrlNotFound(String), - - #[error( - "issuer effective IP resources missing (cannot validate EE IP resources subset) (RFC 6487 §7.2; RFC 3779 §2.3)" - )] - MissingIssuerEffectiveIp, - - #[error( - "issuer effective AS resources missing (cannot validate EE AS resources subset) (RFC 6487 §7.2; RFC 3779 §3.3)" - )] - MissingIssuerEffectiveAs, - - #[error( - "EE certificate resources are not a subset of issuer effective resources (RFC 6487 §7.2; RFC 3779)" - )] - EeResourcesNotSubset, - - #[error("EE certificate violates locally configured TA constraints: {0}")] - TaConstraints(#[from] crate::ta_constraints::TaConstraintsViolation), -} - -pub(crate) fn validate_roa_task_serial( - task: RoaTask<'_>, - manifest_rsync_uri: &str, - issuer_ca_der: &[u8], - issuer_ca: &ResourceCertificate, - issuer_spki: &SubjectPublicKeyInfo<'_>, - issuer_ca_rsync_uri: Option<&str>, - crl_cache: &mut std::collections::HashMap, - issuer_resources_index: &IssuerResourcesIndex, - issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, - issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, - validation_time: time::OffsetDateTime, - timing: Option<&TimingHandle>, - collect_vcir_local_outputs: bool, - strict_cms_der: bool, - strict_name: bool, - resource_validation_mode: ResourceValidationMode, - ta_constraints: Option<&crate::ta_constraints::TaConstraints>, -) -> RoaTaskResult { - let outcome = process_roa_with_issuer( - task.file, - manifest_rsync_uri, - issuer_ca_der, - issuer_ca, - issuer_spki, - issuer_ca_rsync_uri, - crl_cache, - issuer_resources_index, - issuer_effective_ip, - issuer_effective_as, - validation_time, - timing, - collect_vcir_local_outputs, - strict_cms_der, - strict_name, - resource_validation_mode, - ta_constraints, - ) - .map(|(vrps, local_outputs, cache_object_meta)| RoaTaskOk { - vrps, - local_outputs, - reused_from_cache: false, - cache_object_meta, - }); - - RoaTaskResult { - publication_point_id: 0, - index: task.index, - worker_index: 0, - queue_wait_ms: 0, - worker_ms: 0, - outcome, - } -} - -fn process_roa_with_issuer( - file: &PackFile, - _manifest_rsync_uri: &str, - issuer_ca_der: &[u8], - issuer_ca: &ResourceCertificate, - issuer_spki: &SubjectPublicKeyInfo<'_>, - issuer_ca_rsync_uri: Option<&str>, - crl_cache: &mut std::collections::HashMap, - issuer_resources_index: &IssuerResourcesIndex, - issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, - issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, - validation_time: time::OffsetDateTime, - timing: Option<&TimingHandle>, - collect_vcir_local_outputs: bool, - strict_cms_der: bool, - strict_name: bool, - resource_validation_mode: ResourceValidationMode, - ta_constraints: Option<&crate::ta_constraints::TaConstraints>, -) -> Result<(Vec, Vec, Option), ObjectValidateError> { - let _decode = timing - .as_ref() - .map(|t| t.span_phase("objects_roa_decode_and_validate_total")); - let roa = RoaObject::decode_der_with_strict_options( - file.bytes().map_err(ObjectValidateError::BytesLoad)?, - strict_cms_der, - strict_name, - )?; - drop(_decode); - - let _ee_profile = timing - .as_ref() - .map(|t| t.span_phase("objects_roa_validate_embedded_ee_total")); - roa.validate_embedded_ee_cert()?; - drop(_ee_profile); - - let _verify = timing - .as_ref() - .map(|t| t.span_phase("objects_roa_verify_signature_total")); - roa.signed_object.verify()?; - drop(_verify); - - let ee = &roa.signed_object.signed_data.certificates[0]; - let ee_crldp_uris = ee - .resource_cert - .tbs - .extensions - .crl_distribution_points_uris - .as_ref(); - let issuer_crl_rsync_uri = choose_crl_uri_for_certificate(ee_crldp_uris, crl_cache)?; - let verified_crl = ensure_issuer_crl_verified(issuer_crl_rsync_uri, crl_cache, issuer_ca_der)?; - - let _cert_path = timing - .as_ref() - .map(|t| t.span_phase("objects_roa_validate_ee_cert_path_total")); - validate_signed_object_ee_cert_path_fast( - ee, - issuer_ca, - issuer_spki, - &verified_crl.crl, - &verified_crl.revoked_serials, - issuer_ca_rsync_uri, - Some(issuer_crl_rsync_uri), - validation_time, - )?; - drop(_cert_path); - - let _subset = timing - .as_ref() - .map(|t| t.span_phase("objects_roa_validate_ee_resources_subset_total")); - let ee_vrs = validate_ee_resources_for_mode( - &ee.resource_cert, - issuer_effective_ip, - issuer_effective_as, - issuer_resources_index, - resource_validation_mode, - )?; - drop(_subset); - - if let Some(ta_constraints) = ta_constraints { - ta_constraints.validate_ee_certificate(&ee.resource_cert)?; - } - - let vrps = roa_to_vrps_with_vrs(&roa, ee_vrs.ip.as_ref())?; - let cache_object_meta = RoaCacheObjectMeta { - source_object_uri: file.rsync_uri.clone(), - source_object_hash: file.sha256, - ee_serial: BigUnsigned::from_biguint(&ee.resource_cert.tbs.serial_number).bytes_be, - crl_uri: issuer_crl_rsync_uri.to_string(), - earliest_safe_reuse_time: roa_cache_earliest_safe_reuse_time( - ee.resource_cert.tbs.validity_not_before, - verified_crl.crl.this_update.utc, - validation_time, - ), - }; - if !collect_vcir_local_outputs { - return Ok((vrps, Vec::new(), Some(cache_object_meta))); - } - let source_object_hash = sha256_hex_from_32(&file.sha256); - let source_ee_cert_hash = crate::audit::sha256_hex(ee.raw_der.as_slice()); - let item_effective_until = - PackTime::from_utc_offset_datetime(ee.resource_cert.tbs.validity_not_after); - let local_outputs = vrps - .iter() - .map(|vrp| { - let prefix = vrp_prefix_to_string(vrp); - let rule_hash = crate::audit::sha256_hex( - format!( - "roa-rule:{}:{}:{}:{}", - source_object_hash, vrp.asn, prefix, vrp.max_length - ) - .as_bytes(), - ); - VcirLocalOutput { - output_type: VcirOutputType::Vrp, - item_effective_until: item_effective_until.clone(), - source_object_uri: file.rsync_uri.clone(), - source_object_type: VcirSourceObjectType::Roa, - source_object_hash: file.sha256, - source_ee_cert_hash: sha256_hex_to_32(&source_ee_cert_hash), - payload: VcirLocalOutputPayload::Vrp { - asn: vrp.asn, - afi: vrp.prefix.afi, - prefix_len: vrp.prefix.prefix_len, - addr: vrp.prefix.addr, - max_length: vrp.max_length, - }, - rule_hash: sha256_hex_to_32(&rule_hash), - } - }) - .collect(); - - Ok((vrps, local_outputs, Some(cache_object_meta))) -} - -fn process_roa_with_issuer_parallel_cached( - file: &PackFile, - _manifest_rsync_uri: &str, - issuer_ca_der: &[u8], - issuer_ca: &ResourceCertificate, - issuer_spki: &SubjectPublicKeyInfo<'_>, - issuer_ca_rsync_uri: Option<&str>, - crl_cache: &Mutex>, - issuer_resources_index: &IssuerResourcesIndex, - issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, - issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, - validation_time: time::OffsetDateTime, - timing: Option<&TimingHandle>, - collect_vcir_local_outputs: bool, - strict_cms_der: bool, - strict_name: bool, - resource_validation_mode: ResourceValidationMode, - ta_constraints: Option<&crate::ta_constraints::TaConstraints>, -) -> Result<(Vec, Vec, Option), ObjectValidateError> { - let _decode = timing - .as_ref() - .map(|t| t.span_phase("objects_roa_decode_and_validate_total")); - let roa = RoaObject::decode_der_with_strict_options( - file.bytes().map_err(ObjectValidateError::BytesLoad)?, - strict_cms_der, - strict_name, - )?; - drop(_decode); - - let _ee_profile = timing - .as_ref() - .map(|t| t.span_phase("objects_roa_validate_embedded_ee_total")); - roa.validate_embedded_ee_cert()?; - drop(_ee_profile); - - let _verify = timing - .as_ref() - .map(|t| t.span_phase("objects_roa_verify_signature_total")); - roa.signed_object.verify()?; - drop(_verify); - - let ee = &roa.signed_object.signed_data.certificates[0]; - let ee_crldp_uris = ee - .resource_cert - .tbs - .extensions - .crl_distribution_points_uris - .as_ref(); - let (issuer_crl_rsync_uri, verified_crl) = { - let mut crl_cache = crl_cache.lock().expect("parallel ROA CRL cache lock"); - let issuer_crl_rsync_uri = - choose_crl_uri_for_certificate(ee_crldp_uris, &crl_cache)?.to_string(); - let verified_crl = - ensure_issuer_crl_verified(&issuer_crl_rsync_uri, &mut crl_cache, issuer_ca_der)?; - (issuer_crl_rsync_uri, verified_crl) - }; - - let _cert_path = timing - .as_ref() - .map(|t| t.span_phase("objects_roa_validate_ee_cert_path_total")); - validate_signed_object_ee_cert_path_fast( - ee, - issuer_ca, - issuer_spki, - &verified_crl.crl, - &verified_crl.revoked_serials, - issuer_ca_rsync_uri, - Some(issuer_crl_rsync_uri.as_str()), - validation_time, - )?; - drop(_cert_path); - - let _subset = timing - .as_ref() - .map(|t| t.span_phase("objects_roa_validate_ee_resources_subset_total")); - let ee_vrs = validate_ee_resources_for_mode( - &ee.resource_cert, - issuer_effective_ip, - issuer_effective_as, - issuer_resources_index, - resource_validation_mode, - )?; - drop(_subset); - - if let Some(ta_constraints) = ta_constraints { - ta_constraints.validate_ee_certificate(&ee.resource_cert)?; - } - - let vrps = roa_to_vrps_with_vrs(&roa, ee_vrs.ip.as_ref())?; - let cache_object_meta = RoaCacheObjectMeta { - source_object_uri: file.rsync_uri.clone(), - source_object_hash: file.sha256, - ee_serial: BigUnsigned::from_biguint(&ee.resource_cert.tbs.serial_number).bytes_be, - crl_uri: issuer_crl_rsync_uri.clone(), - earliest_safe_reuse_time: roa_cache_earliest_safe_reuse_time( - ee.resource_cert.tbs.validity_not_before, - verified_crl.crl.this_update.utc, - validation_time, - ), - }; - if !collect_vcir_local_outputs { - return Ok((vrps, Vec::new(), Some(cache_object_meta))); - } - let source_object_hash = sha256_hex_from_32(&file.sha256); - let source_ee_cert_hash = crate::audit::sha256_hex(ee.raw_der.as_slice()); - let item_effective_until = - PackTime::from_utc_offset_datetime(ee.resource_cert.tbs.validity_not_after); - let local_outputs = vrps - .iter() - .map(|vrp| { - let prefix = vrp_prefix_to_string(vrp); - let rule_hash = crate::audit::sha256_hex( - format!( - "roa-rule:{}:{}:{}:{}", - source_object_hash, vrp.asn, prefix, vrp.max_length - ) - .as_bytes(), - ); - VcirLocalOutput { - output_type: VcirOutputType::Vrp, - item_effective_until: item_effective_until.clone(), - source_object_uri: file.rsync_uri.clone(), - source_object_type: VcirSourceObjectType::Roa, - source_object_hash: file.sha256, - source_ee_cert_hash: sha256_hex_to_32(&source_ee_cert_hash), - payload: VcirLocalOutputPayload::Vrp { - asn: vrp.asn, - afi: vrp.prefix.afi, - prefix_len: vrp.prefix.prefix_len, - addr: vrp.prefix.addr, - max_length: vrp.max_length, - }, - rule_hash: sha256_hex_to_32(&rule_hash), - } - }) - .collect(); - - Ok((vrps, local_outputs, Some(cache_object_meta))) -} - -fn process_aspa_with_issuer( - file: &PackFile, - _manifest_rsync_uri: &str, - issuer_ca_der: &[u8], - issuer_ca: &ResourceCertificate, - issuer_spki: &SubjectPublicKeyInfo<'_>, - issuer_ca_rsync_uri: Option<&str>, - crl_cache: &mut std::collections::HashMap, - issuer_resources_index: &IssuerResourcesIndex, - issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, - issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, - validation_time: time::OffsetDateTime, - timing: Option<&TimingHandle>, - collect_vcir_local_outputs: bool, - strict_cms_der: bool, - strict_name: bool, - resource_validation_mode: ResourceValidationMode, - ta_constraints: Option<&crate::ta_constraints::TaConstraints>, -) -> Result<(AspaAttestation, Option), ObjectValidateError> { - let _decode = timing - .as_ref() - .map(|t| t.span_phase("objects_aspa_decode_and_validate_total")); - let aspa = AspaObject::decode_der_with_strict_options( - file.bytes().map_err(ObjectValidateError::BytesLoad)?, - strict_cms_der, - strict_name, - )?; - drop(_decode); - - let _ee_profile = timing - .as_ref() - .map(|t| t.span_phase("objects_aspa_validate_embedded_ee_total")); - aspa.validate_embedded_ee_cert()?; - drop(_ee_profile); - - let _verify = timing - .as_ref() - .map(|t| t.span_phase("objects_aspa_verify_signature_total")); - aspa.signed_object.verify()?; - drop(_verify); - - let ee = &aspa.signed_object.signed_data.certificates[0]; - let ee_crldp_uris = ee - .resource_cert - .tbs - .extensions - .crl_distribution_points_uris - .as_ref(); - let issuer_crl_rsync_uri = choose_crl_uri_for_certificate(ee_crldp_uris, crl_cache)?; - let verified_crl = ensure_issuer_crl_verified(issuer_crl_rsync_uri, crl_cache, issuer_ca_der)?; - - let _cert_path = timing - .as_ref() - .map(|t| t.span_phase("objects_aspa_validate_ee_cert_path_total")); - validate_signed_object_ee_cert_path_fast( - ee, - issuer_ca, - issuer_spki, - &verified_crl.crl, - &verified_crl.revoked_serials, - issuer_ca_rsync_uri, - Some(issuer_crl_rsync_uri), - validation_time, - )?; - drop(_cert_path); - - let _subset = timing - .as_ref() - .map(|t| t.span_phase("objects_aspa_validate_ee_resources_subset_total")); - let ee_vrs = validate_ee_resources_for_mode( - &ee.resource_cert, - issuer_effective_ip, - issuer_effective_as, - issuer_resources_index, - resource_validation_mode, - )?; - drop(_subset); - - if let Some(ta_constraints) = ta_constraints { - ta_constraints.validate_ee_certificate(&ee.resource_cert)?; - } - - validate_aspa_customer_in_vrs(&aspa, ee_vrs.asn.as_ref())?; - - let attestation = AspaAttestation { - customer_as_id: aspa.aspa.customer_as_id, - provider_as_ids: aspa.aspa.provider_as_ids.clone(), - }; - if !collect_vcir_local_outputs { - return Ok((attestation, None)); - } - let source_object_hash = sha256_hex_from_32(&file.sha256); - let source_ee_cert_hash = crate::audit::sha256_hex(ee.raw_der.as_slice()); - let item_effective_until = - PackTime::from_utc_offset_datetime(ee.resource_cert.tbs.validity_not_after); - let providers = attestation - .provider_as_ids - .iter() - .map(u32::to_string) - .collect::>() - .join(","); - let rule_hash = crate::audit::sha256_hex( - format!( - "aspa-rule:{}:{}:{}", - source_object_hash, attestation.customer_as_id, providers - ) - .as_bytes(), - ); - let local_output = VcirLocalOutput { - output_type: VcirOutputType::Aspa, - item_effective_until, - source_object_uri: file.rsync_uri.clone(), - source_object_type: VcirSourceObjectType::Aspa, - source_object_hash: file.sha256, - source_ee_cert_hash: sha256_hex_to_32(&source_ee_cert_hash), - payload: VcirLocalOutputPayload::Aspa { - customer_as_id: attestation.customer_as_id, - provider_as_ids: attestation.provider_as_ids.clone(), - }, - rule_hash: sha256_hex_to_32(&rule_hash), - }; - - Ok((attestation, Some(local_output))) -} - -fn vrp_prefix_to_string(vrp: &Vrp) -> String { - let prefix = &vrp.prefix; - match prefix.afi { - RoaAfi::Ipv4 => { - let addr = std::net::Ipv4Addr::new( - prefix.addr[0], - prefix.addr[1], - prefix.addr[2], - prefix.addr[3], - ); - format!("{addr}/{}", prefix.prefix_len) - } - RoaAfi::Ipv6 => { - let mut octets = [0u8; 16]; - octets.copy_from_slice(&prefix.addr[..16]); - let addr = std::net::Ipv6Addr::from(octets); - format!("{addr}/{}", prefix.prefix_len) - } - } -} - -fn choose_crl_uri_for_certificate<'a>( - crldp_uris: Option<&'a Vec>, - crl_cache: &std::collections::HashMap, -) -> Result<&'a str, ObjectValidateError> { - if crl_cache.is_empty() { - return Err(ObjectValidateError::MissingCrlInPack); - } - - let Some(crldp_uris) = crldp_uris else { - return Err(ObjectValidateError::MissingCrlDpUris); - }; - - for u in crldp_uris { - let s = u.as_str(); - if crl_cache.contains_key(s) { - return Ok(s); - } - } - Err(ObjectValidateError::CrlNotFound( - crldp_uris - .iter() - .map(|u| u.as_str()) - .collect::>() - .join(", "), - )) -} - -fn ensure_issuer_crl_verified<'a>( - crl_rsync_uri: &str, - crl_cache: &'a mut std::collections::HashMap, - issuer_ca_der: &[u8], -) -> Result, CertPathError> { - let entry = crl_cache - .get_mut(crl_rsync_uri) - .expect("CRL must exist in cache"); - match entry { - CachedIssuerCrl::Ok(v) => Ok(Arc::clone(v)), - CachedIssuerCrl::Pending { bytes, sha256_hex } => { - let der = std::mem::take(bytes); - let current_sha256_hex = sha256_hex - .take() - .unwrap_or_else(|| crate::audit::sha256_hex(&der)); - let crl = crate::data_model::crl::RpkixCrl::decode_der(&der) - .map_err(CertPathError::CrlDecode)?; - crl.verify_signature_with_issuer_certificate_der(issuer_ca_der) - .map_err(CertPathError::CrlVerify)?; - - let mut revoked_serials: std::collections::HashSet> = - std::collections::HashSet::with_capacity(crl.revoked_certs.len()); - for rc in &crl.revoked_certs { - revoked_serials.insert(rc.serial_number.bytes_be.clone()); - } - - *entry = CachedIssuerCrl::Ok(Arc::new(VerifiedIssuerCrl { - crl, - revoked_serials, - sha256_hex: current_sha256_hex, - })); - match entry { - CachedIssuerCrl::Ok(v) => Ok(Arc::clone(v)), - _ => unreachable!(), - } - } - } -} - -fn validate_ee_resources_subset( - ee: &ResourceCertificate, - issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, - issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, - issuer_resources_index: &IssuerResourcesIndex, -) -> Result<(), ObjectValidateError> { - if let Some(child_ip) = ee.tbs.extensions.ip_resources.as_ref() { - let Some(parent_ip) = issuer_effective_ip else { - return Err(ObjectValidateError::MissingIssuerEffectiveIp); - }; - if !ip_resources_is_subset_indexed(child_ip, parent_ip, issuer_resources_index) { - return Err(ObjectValidateError::EeResourcesNotSubset); - } - } - - if let Some(child_as) = ee.tbs.extensions.as_resources.as_ref() { - let Some(parent_as) = issuer_effective_as else { - return Err(ObjectValidateError::MissingIssuerEffectiveAs); - }; - if !as_resources_is_subset_indexed(child_as, parent_as, issuer_resources_index) { - return Err(ObjectValidateError::EeResourcesNotSubset); - } - } - - Ok(()) -} - -#[derive(Debug)] -struct EeVerifiedResources { - ip: Option, - asn: Option, -} - -fn validate_ee_resources_for_mode( - ee: &ResourceCertificate, - issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, - issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, - issuer_resources_index: &IssuerResourcesIndex, - mode: ResourceValidationMode, -) -> Result { - match mode { - ResourceValidationMode::Rfc6487 => { - validate_ee_resources_subset( - ee, - issuer_effective_ip, - issuer_effective_as, - issuer_resources_index, - )?; - Ok(EeVerifiedResources { - ip: ee.tbs.extensions.ip_resources.clone(), - asn: ee.tbs.extensions.as_resources.clone(), - }) - } - ResourceValidationMode::ValidationUpdate03 => { - let ip = match ee.tbs.extensions.ip_resources.as_ref() { - Some(child_ip) => Some(intersect_ee_ip_resources_vrs( - child_ip, - issuer_effective_ip, - issuer_resources_index, - )?), - None => None, - }; - let asn = match ee.tbs.extensions.as_resources.as_ref() { - Some(child_as) => Some(intersect_ee_as_resources_vrs( - child_as, - issuer_effective_as, - issuer_resources_index, - )?), - None => None, - }; - Ok(EeVerifiedResources { ip, asn }) - } - } -} - -fn intersect_ee_ip_resources_vrs( - child_ip: &crate::data_model::rc::IpResourceSet, - issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, - issuer_resources_index: &IssuerResourcesIndex, -) -> Result { - if child_ip.has_any_inherit() { - return Err(ObjectValidateError::EeResourcesNotSubset); - } - let _ = issuer_effective_ip; - let mut families = Vec::new(); - for fam in &child_ip.families { - let parent_intervals = match fam.afi { - crate::data_model::rc::Afi::Ipv4 => issuer_resources_index.ip_v4.as_deref(), - crate::data_model::rc::Afi::Ipv6 => issuer_resources_index.ip_v6.as_deref(), - } - .unwrap_or(&[]); - let items = match &fam.choice { - IpAddressChoice::Inherit => return Err(ObjectValidateError::EeResourcesNotSubset), - IpAddressChoice::AddressesOrRanges(items) => items, - }; - let intersections = intersect_ip_items_with_parent_intervals(items, parent_intervals); - if !intersections.is_empty() { - families.push(crate::data_model::rc::IpAddressFamily { - afi: fam.afi, - choice: IpAddressChoice::AddressesOrRanges(ip_intervals_to_ranges( - fam.afi, - &intersections, - )), - }); - } - } - Ok(crate::data_model::rc::IpResourceSet { families }) -} - -fn intersect_ee_as_resources_vrs( - child_as: &crate::data_model::rc::AsResourceSet, - issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, - issuer_resources_index: &IssuerResourcesIndex, -) -> Result { - let _ = issuer_effective_as; - if matches!(child_as.asnum, Some(AsIdentifierChoice::Inherit)) - || matches!(child_as.rdi, Some(AsIdentifierChoice::Inherit)) - { - return Err(ObjectValidateError::EeResourcesNotSubset); - } - let asnum = child_as.asnum.as_ref().map(|choice| { - let child_intervals = as_choice_to_merged_intervals(choice); - AsIdentifierChoice::AsIdsOrRanges(as_intervals_to_items(&intersect_as_intervals( - &child_intervals, - issuer_resources_index.asnum.as_deref().unwrap_or(&[]), - ))) - }); - let rdi = child_as.rdi.as_ref().map(|choice| { - let child_intervals = as_choice_to_merged_intervals(choice); - AsIdentifierChoice::AsIdsOrRanges(as_intervals_to_items(&intersect_as_intervals( - &child_intervals, - issuer_resources_index.rdi.as_deref().unwrap_or(&[]), - ))) - }); - Ok(crate::data_model::rc::AsResourceSet { asnum, rdi }) -} - -fn roa_to_vrps_with_vrs( - roa: &RoaObject, - ee_vrs_ip: Option<&crate::data_model::rc::IpResourceSet>, -) -> Result, ObjectValidateError> { - let vrps = roa_to_vrps(roa); - let Some(ee_vrs_ip) = ee_vrs_ip else { - return Err(ObjectValidateError::EeResourcesNotSubset); - }; - for vrp in &vrps { - let rc_prefix = roa_prefix_to_rc_prefix(&vrp.prefix); - if !ee_vrs_ip.contains_prefix(&rc_prefix) { - return Err(ObjectValidateError::EeResourcesNotSubset); - } - } - Ok(vrps) -} - -fn validate_aspa_customer_in_vrs( - aspa: &AspaObject, - ee_vrs_as: Option<&crate::data_model::rc::AsResourceSet>, -) -> Result<(), ObjectValidateError> { - let Some(ee_vrs_as) = ee_vrs_as else { - return Err(ObjectValidateError::EeResourcesNotSubset); - }; - if !as_resource_set_contains_asn(ee_vrs_as, aspa.aspa.customer_as_id) { - return Err(ObjectValidateError::EeResourcesNotSubset); - } - Ok(()) -} - -fn as_resource_set_contains_asn( - resources: &crate::data_model::rc::AsResourceSet, - asn: u32, -) -> bool { - let Some(choice) = resources.asnum.as_ref() else { - return false; - }; - match choice { - AsIdentifierChoice::Inherit => false, - AsIdentifierChoice::AsIdsOrRanges(items) => items.iter().any(|item| match item { - crate::data_model::rc::AsIdOrRange::Id(id) => *id == asn, - crate::data_model::rc::AsIdOrRange::Range { min, max } => *min <= asn && asn <= *max, - }), - } -} - -fn roa_prefix_to_rc_prefix(prefix: &IpPrefix) -> RcIpPrefix { - let afi = match prefix.afi { - RoaAfi::Ipv4 => crate::data_model::rc::Afi::Ipv4, - RoaAfi::Ipv6 => crate::data_model::rc::Afi::Ipv6, - }; - let mut addr = prefix.addr.to_vec(); - addr.truncate(afi.octets_len()); - RcIpPrefix { - afi, - prefix_len: prefix.prefix_len, - addr, - } -} - -fn as_resources_is_subset(child: &AsResourceSet, parent: &AsResourceSet) -> bool { - as_choice_subset(child.asnum.as_ref(), parent.asnum.as_ref()) - && as_choice_subset(child.rdi.as_ref(), parent.rdi.as_ref()) -} - -fn as_resources_is_subset_indexed( - child: &AsResourceSet, - parent: &AsResourceSet, - idx: &IssuerResourcesIndex, -) -> bool { - let _ = parent; - as_choice_subset_indexed(child.asnum.as_ref(), idx.asnum.as_deref()) - && as_choice_subset_indexed(child.rdi.as_ref(), idx.rdi.as_deref()) -} - -fn as_choice_subset_indexed( - child: Option<&AsIdentifierChoice>, - parent_intervals: Option<&[(u32, u32)]>, -) -> bool { - let Some(child) = child else { - return true; - }; - let Some(parent_intervals) = parent_intervals else { - return false; - }; - - if matches!(child, AsIdentifierChoice::Inherit) { - return false; - } - - let child_intervals = as_choice_to_merged_intervals(child); - for (cmin, cmax) in &child_intervals { - if !as_interval_is_covered(parent_intervals, *cmin, *cmax) { - return false; - } - } - true -} - -fn as_choice_subset( - child: Option<&AsIdentifierChoice>, - parent: Option<&AsIdentifierChoice>, -) -> bool { - let Some(child) = child else { - return true; - }; - let Some(parent) = parent else { - return false; - }; - - match (child, parent) { - (AsIdentifierChoice::Inherit, _) => return false, - (_, AsIdentifierChoice::Inherit) => return false, - _ => {} - } - - let child_intervals = as_choice_to_merged_intervals(child); - let parent_intervals = as_choice_to_merged_intervals(parent); - for (cmin, cmax) in &child_intervals { - if !as_interval_is_covered(&parent_intervals, *cmin, *cmax) { - return false; - } - } - true -} - -fn as_choice_to_merged_intervals(choice: &AsIdentifierChoice) -> Vec<(u32, u32)> { - let mut v = Vec::new(); - match choice { - AsIdentifierChoice::Inherit => {} - AsIdentifierChoice::AsIdsOrRanges(items) => { - for item in items { - match item { - crate::data_model::rc::AsIdOrRange::Id(id) => v.push((*id, *id)), - crate::data_model::rc::AsIdOrRange::Range { min, max } => v.push((*min, *max)), - } - } - } - } - v.sort_by_key(|(a, _)| *a); - merge_as_intervals(&v) -} - -fn merge_as_intervals(v: &[(u32, u32)]) -> Vec<(u32, u32)> { - let mut out: Vec<(u32, u32)> = Vec::new(); - for (min, max) in v { - let Some(last) = out.last_mut() else { - out.push((*min, *max)); - continue; - }; - if *min <= last.1.saturating_add(1) { - last.1 = last.1.max(*max); - continue; - } - out.push((*min, *max)); - } - out -} - -fn as_interval_is_covered(parent: &[(u32, u32)], min: u32, max: u32) -> bool { - for (pmin, pmax) in parent { - if *pmin <= min && max <= *pmax { - return true; - } - if *pmin > min { - break; - } - } - false -} - -fn ip_resources_is_subset( - child: &crate::data_model::rc::IpResourceSet, - parent: &crate::data_model::rc::IpResourceSet, -) -> bool { - let parent_by_afi = ip_resources_to_merged_intervals(parent); - let child_by_afi = match ip_resources_to_merged_intervals_strict(child) { - Ok(v) => v, - Err(()) => return false, - }; - - for (afi, child_intervals) in child_by_afi { - let Some(parent_intervals) = parent_by_afi.get(&afi) else { - return false; - }; - for (cmin, cmax) in &child_intervals { - if !interval_is_covered(parent_intervals, cmin, cmax) { - return false; - } - } - } - true -} - -fn ip_resources_is_subset_indexed( - child: &crate::data_model::rc::IpResourceSet, - parent: &crate::data_model::rc::IpResourceSet, - idx: &IssuerResourcesIndex, -) -> bool { - let _ = parent; - - for fam in &child.families { - let parent_intervals = match fam.afi { - crate::data_model::rc::Afi::Ipv4 => idx.ip_v4.as_deref(), - crate::data_model::rc::Afi::Ipv6 => idx.ip_v6.as_deref(), - }; - let Some(parent_intervals) = parent_intervals else { - return false; - }; - let items = match &fam.choice { - IpAddressChoice::Inherit => return false, - IpAddressChoice::AddressesOrRanges(items) => items, - }; - - let mut child_intervals: Vec<(Vec, Vec)> = Vec::new(); - for item in items { - match item { - IpAddressOrRange::Prefix(p) => child_intervals.push(prefix_to_range(p)), - IpAddressOrRange::Range(r) => child_intervals.push((r.min.clone(), r.max.clone())), - } - } - if child_intervals.is_empty() { - continue; - } - child_intervals.sort_by(|(a, _), (b, _)| a.cmp(b)); - merge_ip_intervals_in_place(&mut child_intervals); - if !intervals_are_covered(parent_intervals, &child_intervals) { - return false; - } - } - true -} - -fn ip_items_to_merged_intervals( - items: &[crate::data_model::rc::IpAddressOrRange], -) -> Vec<(Vec, Vec)> { - let mut intervals = Vec::new(); - for item in items { - match item { - IpAddressOrRange::Prefix(p) => intervals.push(prefix_to_range(p)), - IpAddressOrRange::Range(r) => intervals.push((r.min.clone(), r.max.clone())), - } - } - intervals.sort_by(|(a, _), (b, _)| a.cmp(b)); - merge_ip_intervals_in_place(&mut intervals); - intervals -} - -fn intersect_ip_items_with_parent_intervals( - items: &[crate::data_model::rc::IpAddressOrRange], - parent_intervals: &[(Vec, Vec)], -) -> Vec<(Vec, Vec)> { - let child_intervals = ip_items_to_merged_intervals(items); - let mut out = Vec::new(); - let mut parent_index = 0usize; - for (child_min, child_max) in &child_intervals { - while parent_index < parent_intervals.len() - && parent_intervals[parent_index].1.as_slice() < child_min.as_slice() - { - parent_index += 1; - } - let mut scan = parent_index; - while scan < parent_intervals.len() - && parent_intervals[scan].0.as_slice() <= child_max.as_slice() - { - let (parent_min, parent_max) = &parent_intervals[scan]; - let min = if bytes_leq(child_min, parent_min) { - parent_min.clone() - } else { - child_min.clone() - }; - let max = if bytes_leq(child_max, parent_max) { - child_max.clone() - } else { - parent_max.clone() - }; - if bytes_leq(&min, &max) { - out.push((min, max)); - } - scan += 1; - } - } - merge_ip_intervals_in_place(&mut out); - out -} - -fn ip_intervals_to_ranges( - afi: crate::data_model::rc::Afi, - intervals: &[(Vec, Vec)], -) -> Vec { - intervals - .iter() - .map(|(min, max)| { - IpAddressOrRange::Range(crate::data_model::rc::IpAddressRange { - min: normalize_ip_bytes(afi, min), - max: normalize_ip_bytes(afi, max), - }) - }) - .collect() -} - -fn normalize_ip_bytes(afi: crate::data_model::rc::Afi, bytes: &[u8]) -> Vec { - let target_len = afi.octets_len(); - if bytes.len() == target_len { - return bytes.to_vec(); - } - let mut out = vec![0u8; target_len]; - let copy_len = bytes.len().min(target_len); - out[..copy_len].copy_from_slice(&bytes[..copy_len]); - out -} - -fn intersect_as_intervals(child: &[(u32, u32)], parent: &[(u32, u32)]) -> Vec<(u32, u32)> { - let mut out = Vec::new(); - let mut parent_index = 0usize; - for (child_min, child_max) in child { - while parent_index < parent.len() && parent[parent_index].1 < *child_min { - parent_index += 1; - } - let mut scan = parent_index; - while scan < parent.len() && parent[scan].0 <= *child_max { - let min = (*child_min).max(parent[scan].0); - let max = (*child_max).min(parent[scan].1); - if min <= max { - out.push((min, max)); - } - scan += 1; - } - } - merge_as_intervals(&out) -} - -fn as_intervals_to_items(intervals: &[(u32, u32)]) -> Vec { - intervals - .iter() - .map(|(min, max)| { - if min == max { - crate::data_model::rc::AsIdOrRange::Id(*min) - } else { - crate::data_model::rc::AsIdOrRange::Range { - min: *min, - max: *max, - } - } - }) - .collect() -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -enum AfiKey { - V4, - V6, -} - -fn ip_resources_to_merged_intervals( - set: &crate::data_model::rc::IpResourceSet, -) -> std::collections::HashMap, Vec)>> { - let mut m: std::collections::HashMap, Vec)>> = - std::collections::HashMap::new(); - - for fam in &set.families { - let afi = match fam.afi { - crate::data_model::rc::Afi::Ipv4 => AfiKey::V4, - crate::data_model::rc::Afi::Ipv6 => AfiKey::V6, - }; - match &fam.choice { - IpAddressChoice::Inherit => { - // Effective resource sets should not contain inherit, but if they do we treat it - // as "unknown" by leaving it empty here (subset checks will fail). - } - IpAddressChoice::AddressesOrRanges(items) => { - let ent = m.entry(afi).or_default(); - for item in items { - match item { - IpAddressOrRange::Prefix(p) => ent.push(prefix_to_range(p)), - IpAddressOrRange::Range(r) => ent.push((r.min.clone(), r.max.clone())), - } - } - } - } - } - - for (_afi, v) in m.iter_mut() { - v.sort_by(|(a, _), (b, _)| a.cmp(b)); - merge_ip_intervals_in_place(v); - } - - m -} - -fn ip_resources_to_merged_intervals_strict( - set: &crate::data_model::rc::IpResourceSet, -) -> Result, Vec)>>, ()> { - let mut m: std::collections::HashMap, Vec)>> = - std::collections::HashMap::new(); - - for fam in &set.families { - let afi = match fam.afi { - crate::data_model::rc::Afi::Ipv4 => AfiKey::V4, - crate::data_model::rc::Afi::Ipv6 => AfiKey::V6, - }; - match &fam.choice { - IpAddressChoice::Inherit => return Err(()), - IpAddressChoice::AddressesOrRanges(items) => { - let ent = m.entry(afi).or_default(); - for item in items { - match item { - IpAddressOrRange::Prefix(p) => ent.push(prefix_to_range(p)), - IpAddressOrRange::Range(r) => ent.push((r.min.clone(), r.max.clone())), - } - } - } - } - } - - for (_afi, v) in m.iter_mut() { - v.sort_by(|(a, _), (b, _)| a.cmp(b)); - merge_ip_intervals_in_place(v); - } - - Ok(m) -} - -fn merge_ip_intervals_in_place(v: &mut Vec<(Vec, Vec)>) { - if v.is_empty() { - return; - } - let mut out: Vec<(Vec, Vec)> = Vec::with_capacity(v.len()); - for (min, max) in v.drain(..) { - let Some(last) = out.last_mut() else { - out.push((min, max)); - continue; - }; - if bytes_leq(&min, &last.1) || bytes_is_next(&min, &last.1) { - if bytes_leq(&last.1, &max) { - last.1 = max; - } - continue; - } - out.push((min, max)); - } - *v = out; -} - -fn interval_is_covered(parent: &[(Vec, Vec)], min: &[u8], max: &[u8]) -> bool { - for (pmin, pmax) in parent { - if bytes_leq(pmin, min) && bytes_leq(max, pmax) { - return true; - } - if pmin.as_slice() > min { - break; - } - } - false -} - -fn intervals_are_covered(parent: &[(Vec, Vec)], child: &[(Vec, Vec)]) -> bool { - let mut i = 0usize; - for (cmin, cmax) in child { - while i < parent.len() && parent[i].1.as_slice() < cmin.as_slice() { - i += 1; - } - if i >= parent.len() { - return false; - } - let (pmin, pmax) = &parent[i]; - if !bytes_leq(pmin, cmin) || !bytes_leq(cmax, pmax) { - return false; - } - } - true -} - -fn prefix_to_range(prefix: &RcIpPrefix) -> (Vec, Vec) { - let mut min = prefix.addr.clone(); - let mut max = prefix.addr.clone(); - let bitlen = prefix.afi.ub(); - let plen = prefix.prefix_len.min(bitlen); - for bit in plen..bitlen { - let byte = (bit / 8) as usize; - let offset = 7 - (bit % 8); - let mask = 1u8 << offset; - min[byte] &= !mask; - max[byte] |= mask; - } - (min, max) -} - -fn bytes_leq(a: &[u8], b: &[u8]) -> bool { - a <= b -} - -fn increment_bytes(v: &[u8]) -> Vec { - let mut out = v.to_vec(); - for i in (0..out.len()).rev() { - if out[i] != 0xFF { - out[i] += 1; - for j in i + 1..out.len() { - out[j] = 0; - } - return out; - } - } - vec![0u8; out.len()] -} - -fn bytes_is_next(a: &[u8], b: &[u8]) -> bool { - if a.len() != b.len() { - return false; - } - let mut carry: u16 = 1; - for i in (0..b.len()).rev() { - let sum = (b[i] as u16) + carry; - let expected = (sum & 0xFF) as u8; - carry = sum >> 8; - if a[i] != expected { - return false; - } - } - true -} - -fn build_issuer_resources_index( - issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, - issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, -) -> IssuerResourcesIndex { - let mut idx = IssuerResourcesIndex::default(); - - if let Some(ip) = issuer_effective_ip { - let mut v4: Vec<(Vec, Vec)> = Vec::new(); - let mut v6: Vec<(Vec, Vec)> = Vec::new(); - for fam in &ip.families { - let ent = match fam.afi { - crate::data_model::rc::Afi::Ipv4 => &mut v4, - crate::data_model::rc::Afi::Ipv6 => &mut v6, - }; - match &fam.choice { - IpAddressChoice::Inherit => { - // Effective resources should not contain inherit; leave empty so subset fails. - } - IpAddressChoice::AddressesOrRanges(items) => { - for item in items { - match item { - IpAddressOrRange::Prefix(p) => ent.push(prefix_to_range(p)), - IpAddressOrRange::Range(r) => ent.push((r.min.clone(), r.max.clone())), - } - } - } - } - } - if !v4.is_empty() { - v4.sort_by(|(a, _), (b, _)| a.cmp(b)); - merge_ip_intervals_in_place(&mut v4); - idx.ip_v4 = Some(v4); - } - if !v6.is_empty() { - v6.sort_by(|(a, _), (b, _)| a.cmp(b)); - merge_ip_intervals_in_place(&mut v6); - idx.ip_v6 = Some(v6); - } - } - - if let Some(asr) = issuer_effective_as { - if let Some(choice) = asr.asnum.as_ref() { - if !matches!(choice, AsIdentifierChoice::Inherit) { - idx.asnum = Some(as_choice_to_merged_intervals(choice)); - } - } - if let Some(choice) = asr.rdi.as_ref() { - if !matches!(choice, AsIdentifierChoice::Inherit) { - idx.rdi = Some(as_choice_to_merged_intervals(choice)); - } - } - } - - idx -} - -fn roa_to_vrps(roa: &RoaObject) -> Vec { - let asn = roa.roa.as_id; - let mut out = Vec::new(); - for fam in &roa.roa.ip_addr_blocks { - for entry in &fam.addresses { - let max_length = entry.max_length.unwrap_or(entry.prefix.prefix_len); - out.push(Vrp { - asn, - prefix: entry.prefix.clone(), - max_length, - }); - } - } - out -} - -#[allow(dead_code)] -fn roa_afi_to_string(afi: RoaAfi) -> &'static str { - match afi { - RoaAfi::Ipv4 => "ipv4", - RoaAfi::Ipv6 => "ipv6", - } -} +include!("objects/cache.rs"); +include!("objects/serial_processing.rs"); +include!("objects/parallel_processing.rs"); +include!("objects/parallel_stage.rs"); +include!("objects/object_validation.rs"); +include!("objects/resource_validation.rs"); #[cfg(test)] mod tests { - use super::*; - use crate::analysis::timing::{TimingHandle, TimingMeta}; - use crate::data_model::rc::{ - Afi, AsIdOrRange, AsIdentifierChoice, IpAddressFamily, IpAddressOrRange, IpAddressRange, - IpPrefix, IpResourceSet, - }; - use crate::policy::Policy; - use crate::storage::{ - PackTime, RoaCacheObjectMeta, RoaCacheProjection, RoaCacheProjectionContext, - ValidatedCaInstanceResult, ValidatedManifestMeta, VcirArtifactKind, VcirArtifactRole, - VcirArtifactValidationStatus, VcirAuditSummary, VcirCcrManifestProjection, - VcirInstanceGate, VcirRelatedArtifact, VcirSummary, - }; - use crate::validation::publication_point::PublicationPointSnapshot; - use std::collections::HashMap; - use time::OffsetDateTime; - use time::format_description::well_known::Rfc3339; - - fn fixture_bytes(path: &str) -> Vec { - std::fs::read(std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(path)) - .unwrap_or_else(|e| panic!("read fixture {path}: {e}")) - } - - fn fixed_time(value: &str) -> OffsetDateTime { - OffsetDateTime::parse(value, &Rfc3339).expect("parse fixed test time") - } - - const TEST_CA_VALIDATION_CONTEXT: [u8; 32] = [0x70; 32]; - const TEST_POLICY_FINGERPRINT: [u8; 32] = [0x71; 32]; - const TEST_CRL_URI: &str = "rsync://example.test/repo/current.crl"; - const TEST_ROA_URI: &str = "rsync://example.test/repo/a.roa"; - - fn sha256_32(bytes: &[u8]) -> [u8; 32] { - let digest = sha2::Sha256::digest(bytes); - let mut out = [0u8; 32]; - out.copy_from_slice(&digest); - out - } - - fn sample_roa_cache_projection( - vcir: &ValidatedCaInstanceResult, - roa_hash: [u8; 32], - ) -> RoaCacheProjection { - RoaCacheProjection::from_vcir_with_context( - vcir, - Some(&RoaCacheProjectionContext { - ca_validation_context_digest: TEST_CA_VALIDATION_CONTEXT, - policy_fingerprint: TEST_POLICY_FINGERPRINT, - object_meta: vec![RoaCacheObjectMeta { - source_object_uri: TEST_ROA_URI.to_string(), - source_object_hash: roa_hash, - ee_serial: vec![0x01], - crl_uri: TEST_CRL_URI.to_string(), - earliest_safe_reuse_time: vcir.last_successful_validation_time.clone(), - }], - }), - ) - .expect("build projection") - .expect("projection exists") - } - - fn sample_crl_cache(crl_bytes: Vec) -> HashMap { - HashMap::from([( - TEST_CRL_URI.to_string(), - CachedIssuerCrl::Pending { - bytes: crl_bytes, - sha256_hex: None, - }, - )]) - } - - fn sample_roa_cache_vcir( - issuer_der: &[u8], - crl_hash: [u8; 32], - roa_hash: [u8; 32], - item_effective_until: OffsetDateTime, - instance_effective_until: OffsetDateTime, - ) -> ValidatedCaInstanceResult { - let manifest_time = PackTime::from_utc_offset_datetime(fixed_time("2026-06-04T00:00:00Z")); - let effective_until = PackTime::from_utc_offset_datetime(instance_effective_until); - ValidatedCaInstanceResult { - manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(), - parent_manifest_rsync_uri: Some("rsync://example.test/repo/parent.mft".to_string()), - tal_id: "test-tal".to_string(), - ca_subject_name: "CN=example".to_string(), - ca_ski: "001122".to_string(), - issuer_ski: "334455".to_string(), - last_successful_validation_time: manifest_time.clone(), - current_manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(), - current_crl_rsync_uri: "rsync://example.test/repo/current.crl".to_string(), - validated_manifest_meta: ValidatedManifestMeta { - validated_manifest_number: vec![1], - validated_manifest_this_update: manifest_time.clone(), - validated_manifest_next_update: effective_until.clone(), - }, - ccr_manifest_projection: VcirCcrManifestProjection { - manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(), - manifest_sha256: vec![0xaa; 32], - manifest_size: 2048, - manifest_ee_aki: vec![0xbb; 20], - manifest_number_be: vec![1], - manifest_this_update: manifest_time.clone(), - manifest_sia_locations_der: vec![vec![0x30, 0x00]], - subordinate_skis: Vec::new(), - }, - instance_gate: VcirInstanceGate { - manifest_next_update: effective_until.clone(), - current_crl_next_update: effective_until.clone(), - self_ca_not_after: effective_until.clone(), - instance_effective_until: effective_until, - }, - child_entries: Vec::new(), - local_outputs: vec![VcirLocalOutput { - output_type: VcirOutputType::Vrp, - item_effective_until: PackTime::from_utc_offset_datetime(item_effective_until), - source_object_uri: "rsync://example.test/repo/a.roa".to_string(), - source_object_type: VcirSourceObjectType::Roa, - source_object_hash: roa_hash, - source_ee_cert_hash: [0xcc; 32], - payload: VcirLocalOutputPayload::Vrp { - asn: 64500, - afi: RoaAfi::Ipv4, - prefix_len: 24, - addr: [192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - max_length: 24, - }, - rule_hash: [0xdd; 32], - }], - related_artifacts: vec![ - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::IssuerCert, - artifact_kind: VcirArtifactKind::Cer, - uri: Some("rsync://example.test/repo/ca.cer".to_string()), - sha256: sha256_hex(issuer_der), - object_type: Some("cer".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - reject_reason: None, - }, - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::CurrentCrl, - artifact_kind: VcirArtifactKind::Crl, - uri: Some("rsync://example.test/repo/current.crl".to_string()), - sha256: sha256_hex_from_32(&crl_hash), - object_type: Some("crl".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - reject_reason: None, - }, - ], - summary: VcirSummary { - local_vrp_count: 1, - local_aspa_count: 0, - local_router_key_count: 0, - child_count: 0, - accepted_object_count: 2, - rejected_object_count: 0, - }, - audit_summary: VcirAuditSummary { - failed_fetch_eligible: true, - last_failed_fetch_reason: None, - warning_count: 0, - audit_flags: Vec::new(), - }, - } - } - - #[test] - fn roa_validation_cache_view_hits_when_context_and_hash_match() { - let validation_time = fixed_time("2026-06-05T00:00:00Z"); - let issuer_der = b"issuer-ca"; - let crl_bytes = b"current-crl".to_vec(); - let crl_hash = sha256_32(&crl_bytes); - let roa_hash = [0x11; 32]; - let vcir = sample_roa_cache_vcir( - issuer_der, - crl_hash, - roa_hash, - fixed_time("2026-06-07T00:00:00Z"), - fixed_time("2026-06-08T00:00:00Z"), - ); - let projection = sample_roa_cache_projection(&vcir, roa_hash); - let view = RoaValidationCacheView::from_projection(&projection, validation_time); - let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); - let mut crl_cache = sample_crl_cache(crl_bytes); - - assert!(view.matches_current_context( - issuer_der, - Some(TEST_CA_VALIDATION_CONTEXT), - Some(TEST_POLICY_FINGERPRINT) - )); - let hit = view.lookup(&file, &mut crl_cache, issuer_der, validation_time); - let RoaCacheLookupResult::Hit(ok) = hit else { - panic!("expected cache hit, got {hit:?}"); - }; - assert!(ok.reused_from_cache); - assert_eq!(ok.vrps.len(), 1); - assert_eq!(ok.vrps[0].asn, 64500); - assert_eq!(ok.local_outputs.len(), 1); - assert!(ok.cache_object_meta.is_some()); - } - - #[test] - fn roa_validation_cache_lookup_memoizes_current_crl_hash() { - let validation_time = fixed_time("2026-06-05T00:00:00Z"); - let issuer_der = b"issuer-ca"; - let crl_bytes = b"current-crl".to_vec(); - let crl_hash = sha256_32(&crl_bytes); - let crl_hash_hex = sha256_hex(&crl_bytes); - let roa_hash = [0x11; 32]; - let vcir = sample_roa_cache_vcir( - issuer_der, - crl_hash, - roa_hash, - fixed_time("2026-06-07T00:00:00Z"), - fixed_time("2026-06-08T00:00:00Z"), - ); - let projection = sample_roa_cache_projection(&vcir, roa_hash); - let view = RoaValidationCacheView::from_projection(&projection, validation_time); - let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); - let mut crl_cache = sample_crl_cache(crl_bytes); - - let first = view.lookup(&file, &mut crl_cache, issuer_der, validation_time); - assert!(matches!(first, RoaCacheLookupResult::Hit(_))); - match crl_cache.get(TEST_CRL_URI).expect("test CRL cache entry") { - CachedIssuerCrl::Pending { - sha256_hex: Some(cached), - .. - } => assert_eq!(cached, &crl_hash_hex), - other => panic!("expected pending CRL with memoized hash, got {other:?}"), - } - - let second = view.lookup(&file, &mut crl_cache, issuer_der, validation_time); - assert!(matches!(second, RoaCacheLookupResult::Hit(_))); - } - - #[test] - fn roa_validation_cache_lookup_reuses_publication_point_crl_gate() { - let validation_time = fixed_time("2026-06-05T00:00:00Z"); - let issuer_der = b"issuer-ca"; - let crl_bytes = b"current-crl".to_vec(); - let crl_hash = sha256_32(&crl_bytes); - let roa_hash = [0x11; 32]; - let vcir = sample_roa_cache_vcir( - issuer_der, - crl_hash, - roa_hash, - fixed_time("2026-06-07T00:00:00Z"), - fixed_time("2026-06-08T00:00:00Z"), - ); - let projection = sample_roa_cache_projection(&vcir, roa_hash); - let view = RoaValidationCacheView::from_projection(&projection, validation_time); - let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); - let mut crl_cache = sample_crl_cache(crl_bytes); - let mut gate_set = RoaCacheCrlGateSet::default(); - - let (first, first_metrics) = view.lookup_with_metrics( - &file, - &mut crl_cache, - issuer_der, - validation_time, - Some(&mut gate_set), - ); - assert!(matches!(first, RoaCacheLookupResult::Hit(_))); - assert_eq!(first_metrics.crl_gate_reused_crls, 0); - assert_eq!(gate_set.gates_by_uri.len(), 1); - - let (second, second_metrics) = view.lookup_with_metrics( - &file, - &mut crl_cache, - issuer_der, - validation_time, - Some(&mut gate_set), - ); - assert!(matches!(second, RoaCacheLookupResult::Hit(_))); - assert_eq!(second_metrics.crl_gate_reused_crls, 1); - assert_eq!(second_metrics.crl_gate_verified_crls, 0); - assert_eq!(gate_set.gates_by_uri.len(), 1); - } - - #[test] - fn cached_verified_crl_reports_hash_and_validity_window() { - let crl_bytes = fixture_bytes( - "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.crl", - ); - let crl = crate::data_model::crl::RpkixCrl::decode_der(&crl_bytes).expect("decode CRL"); - let verified = Arc::new(VerifiedIssuerCrl { - crl, - revoked_serials: std::collections::HashSet::new(), - sha256_hex: sha256_hex(&crl_bytes), - }); - let mut cached = CachedIssuerCrl::Ok(Arc::clone(&verified)); - - assert_eq!(cached.current_sha256_hex(), verified.sha256_hex); - assert!(crl_valid_at_time( - &verified.crl, - fixed_time("2026-01-21T00:00:00Z") - )); - assert!(!crl_valid_at_time( - &verified.crl, - fixed_time("2026-01-22T00:00:00Z") - )); - } - - #[test] - fn roa_validation_cache_view_from_projection_preserves_metadata() { - let validation_time = fixed_time("2026-06-05T00:00:00Z"); - let issuer_der = b"issuer-ca"; - let crl_bytes = b"current-crl".to_vec(); - let crl_hash = sha256_32(&crl_bytes); - let roa_hash = [0x11; 32]; - let vcir = sample_roa_cache_vcir( - issuer_der, - crl_hash, - roa_hash, - fixed_time("2026-06-07T00:00:00Z"), - fixed_time("2026-06-08T00:00:00Z"), - ); - let projection = sample_roa_cache_projection(&vcir, roa_hash); - assert_eq!( - projection.ca_validation_context_digest, - Some(TEST_CA_VALIDATION_CONTEXT) - ); - assert_eq!(projection.policy_fingerprint, Some(TEST_POLICY_FINGERPRINT)); - assert_eq!( - projection.entries[0].outputs_effective_until_unix, - fixed_time("2026-06-07T00:00:00Z").unix_timestamp() - ); - assert_eq!( - projection.entries[0].ee_serial.as_deref(), - Some(&[0x01][..]) - ); - assert_eq!(projection.entries[0].crl_uri.as_deref(), Some(TEST_CRL_URI)); - - let view = RoaValidationCacheView::from_projection(&projection, validation_time); - let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); - let mut crl_cache = sample_crl_cache(crl_bytes); - let hit = view.lookup(&file, &mut crl_cache, issuer_der, validation_time); - let RoaCacheLookupResult::Hit(ok) = hit else { - panic!("expected projection cache hit, got {hit:?}"); - }; - assert_eq!(ok.local_outputs[0].source_object_uri, TEST_ROA_URI); - } - - #[test] - fn roa_validation_cache_view_blocks_on_context_hash_and_expiry_gates() { - let validation_time = fixed_time("2026-06-05T00:00:00Z"); - let issuer_der = b"issuer-ca"; - let crl_bytes = b"current-crl".to_vec(); - let crl_hash = sha256_32(&crl_bytes); - let roa_hash = [0x11; 32]; - let vcir = sample_roa_cache_vcir( - issuer_der, - crl_hash, - roa_hash, - fixed_time("2026-06-07T00:00:00Z"), - fixed_time("2026-06-08T00:00:00Z"), - ); - let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); - - let mut projection = sample_roa_cache_projection(&vcir, roa_hash); - projection.issuer_ca_sha256_hex = Some("00".repeat(32)); - let issuer_changed = RoaValidationCacheView::from_projection(&projection, validation_time); - assert!(!issuer_changed.matches_current_context( - issuer_der, - Some(TEST_CA_VALIDATION_CONTEXT), - Some(TEST_POLICY_FINGERPRINT) - )); - - let projection = sample_roa_cache_projection(&vcir, roa_hash); - let parent_changed = RoaValidationCacheView::from_projection(&projection, validation_time); - assert!(!parent_changed.matches_current_context( - issuer_der, - Some([0x99; 32]), - Some(TEST_POLICY_FINGERPRINT) - )); - assert!(!parent_changed.matches_current_context( - issuer_der, - Some(TEST_CA_VALIDATION_CONTEXT), - Some([0x98; 32]) - )); - - let mut projection = sample_roa_cache_projection(&vcir, roa_hash); - projection.entries[0].source_object_hash = [0xff; 32]; - let roa_changed = RoaValidationCacheView::from_projection(&projection, validation_time); - let mut crl_cache = sample_crl_cache(crl_bytes); - assert!(matches!( - roa_changed.lookup(&file, &mut crl_cache, issuer_der, validation_time), - RoaCacheLookupResult::HashBlocked - )); - - let expired_vcir = sample_roa_cache_vcir( - issuer_der, - crl_hash, - roa_hash, - fixed_time("2026-06-07T00:00:00Z"), - fixed_time("2026-06-04T00:00:00Z"), - ); - let expired_projection = sample_roa_cache_projection(&expired_vcir, roa_hash); - let expired_view = - RoaValidationCacheView::from_projection(&expired_projection, validation_time); - assert!(!expired_view.matches_current_context( - issuer_der, - Some(TEST_CA_VALIDATION_CONTEXT), - Some(TEST_POLICY_FINGERPRINT) - )); - let mut crl_cache = sample_crl_cache(b"current-crl".to_vec()); - assert!(matches!( - expired_view.lookup(&file, &mut crl_cache, issuer_der, validation_time), - RoaCacheLookupResult::ExpiredBlocked - )); - } - - #[test] - fn active_roa_cache_view_records_context_block() { - let validation_time = fixed_time("2026-06-05T00:00:00Z"); - let issuer_der = b"issuer-ca"; - let crl_hash = [0x22; 32]; - let roa_hash = [0x11; 32]; - let vcir = sample_roa_cache_vcir( - issuer_der, - crl_hash, - roa_hash, - fixed_time("2026-06-07T00:00:00Z"), - fixed_time("2026-06-08T00:00:00Z"), - ); - let projection = sample_roa_cache_projection(&vcir, roa_hash); - let view = RoaValidationCacheView::from_projection(&projection, validation_time); - let input = RoaValidationCacheInput::enabled_with_context( - Some(&view), - [0x99; 32], - TEST_POLICY_FINGERPRINT, - ); - let mut stats = RoaValidationCacheStats::for_input(input, 1); - - assert!(active_roa_cache_view(input, issuer_der, &mut stats, 1).is_none()); - assert_eq!(stats.blocked_roas, 1); - assert_eq!(stats.context_blocked_roas, 1); - assert_eq!(stats.fresh_roas, 1); - } - - #[test] - fn roa_validation_cache_view_blocks_expired_output() { - let validation_time = fixed_time("2026-06-05T00:00:00Z"); - let issuer_der = b"issuer-ca"; - let crl_bytes = b"current-crl".to_vec(); - let crl_hash = sha256_32(&crl_bytes); - let roa_hash = [0x11; 32]; - let vcir = sample_roa_cache_vcir( - issuer_der, - crl_hash, - roa_hash, - fixed_time("2026-06-04T01:00:00Z"), - fixed_time("2026-06-08T00:00:00Z"), - ); - let projection = sample_roa_cache_projection(&vcir, roa_hash); - let view = RoaValidationCacheView::from_projection(&projection, validation_time); - let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); - let mut crl_cache = sample_crl_cache(crl_bytes); - - assert!(matches!( - view.lookup(&file, &mut crl_cache, issuer_der, validation_time), - RoaCacheLookupResult::ExpiredBlocked - )); - } - - #[test] - fn roa_validation_cache_view_blocks_before_safe_reuse_time() { - let validation_time = fixed_time("2026-06-05T00:00:00Z"); - let issuer_der = b"issuer-ca"; - let crl_bytes = b"current-crl".to_vec(); - let crl_hash = sha256_32(&crl_bytes); - let roa_hash = [0x11; 32]; - let vcir = sample_roa_cache_vcir( - issuer_der, - crl_hash, - roa_hash, - fixed_time("2026-06-07T00:00:00Z"), - fixed_time("2026-06-08T00:00:00Z"), - ); - let mut projection = sample_roa_cache_projection(&vcir, roa_hash); - projection.entries[0].earliest_safe_reuse_time_unix = - Some(fixed_time("2026-06-06T00:00:00Z").unix_timestamp()); - let view = RoaValidationCacheView::from_projection(&projection, validation_time); - let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); - let mut crl_cache = sample_crl_cache(crl_bytes); - - assert!(matches!( - view.lookup(&file, &mut crl_cache, issuer_der, validation_time), - RoaCacheLookupResult::ExpiredBlocked - )); - } - - #[test] - fn roa_validation_cache_view_treats_legacy_entry_without_lower_bound_as_miss() { - let validation_time = fixed_time("2026-06-05T00:00:00Z"); - let issuer_der = b"issuer-ca"; - let crl_hash = sha256_32(b"current-crl"); - let roa_hash = [0x11; 32]; - let vcir = sample_roa_cache_vcir( - issuer_der, - crl_hash, - roa_hash, - fixed_time("2026-06-07T00:00:00Z"), - fixed_time("2026-06-08T00:00:00Z"), - ); - let mut projection = sample_roa_cache_projection(&vcir, roa_hash); - projection.entries[0].earliest_safe_reuse_time_unix = None; - let view = RoaValidationCacheView::from_projection(&projection, validation_time); - let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); - let mut crl_cache = sample_crl_cache(b"current-crl".to_vec()); - - assert!(matches!( - view.lookup(&file, &mut crl_cache, issuer_der, validation_time), - RoaCacheLookupResult::Miss - )); - } - - #[test] - fn roa_validation_cache_stats_records_vcir_miss_to_timing() { - let stats = RoaValidationCacheStats::for_input(RoaValidationCacheInput::enabled(None), 3); - assert_eq!(stats.enabled_publication_points, 1); - assert_eq!(stats.vcir_miss_publication_points, 1); - assert_eq!(stats.miss_roas, 3); - assert_eq!(stats.fresh_roas, 3); - - let timing = TimingHandle::new(TimingMeta { - recorded_at_utc_rfc3339: "2026-06-05T00:00:00Z".to_string(), - validation_time_utc_rfc3339: "2026-06-05T00:00:00Z".to_string(), - tal_url: None, - db_path: None, - }); - stats.record_to_timing(Some(&timing)); - let dir = tempfile::tempdir().expect("timing dir"); - let path = dir.path().join("timing.json"); - timing.write_json(&path, 10).expect("write timing"); - let report: serde_json::Value = - serde_json::from_slice(&std::fs::read(path).expect("read timing")) - .expect("parse timing"); - - assert_eq!( - report["counts"]["roa_validation_cache_enabled_publication_points"], - 1 - ); - assert_eq!( - report["counts"]["roa_validation_cache_vcir_miss_publication_points"], - 1 - ); - assert_eq!(report["counts"]["roa_validation_cache_miss_roas"], 3); - assert_eq!(report["counts"]["roa_validation_cache_fresh_roas"], 3); - } - - #[test] - fn roa_validation_cache_stats_records_block_breakdown_to_timing() { - let mut stats = RoaValidationCacheStats::default(); - record_roa_cache_block(&mut stats, &RoaCacheLookupResult::HashBlocked); - record_roa_cache_block(&mut stats, &RoaCacheLookupResult::ExpiredBlocked); - record_roa_cache_block(&mut stats, &RoaCacheLookupResult::RevokedBlocked); - record_roa_cache_block(&mut stats, &RoaCacheLookupResult::MetadataBlocked); - stats.crl_recheck_hit_roas = 2; - stats.context_blocked_roas = 3; - stats.context_gate_nanos = 4; - stats.lookup_nanos = 5; - stats.lookup_entry_gate_nanos = 6; - stats.lookup_crl_gate_nanos = 7; - stats.lookup_materialize_nanos = 8; - stats.lookup_crl_gate_verified_crls = 9; - stats.lookup_crl_gate_reused_crls = 10; - - let timing = TimingHandle::new(TimingMeta { - recorded_at_utc_rfc3339: "2026-06-05T00:00:00Z".to_string(), - validation_time_utc_rfc3339: "2026-06-05T00:00:00Z".to_string(), - tal_url: None, - db_path: None, - }); - stats.record_to_timing(Some(&timing)); - let dir = tempfile::tempdir().expect("timing dir"); - let path = dir.path().join("timing.json"); - timing.write_json(&path, 10).expect("write timing"); - let report: serde_json::Value = - serde_json::from_slice(&std::fs::read(path).expect("read timing")) - .expect("parse timing"); - - assert_eq!(stats.blocked_roas, 4); - assert_eq!(stats.fresh_roas, 4); - assert_eq!(report["counts"]["roa_validation_cache_blocked_roas"], 4); - assert_eq!( - report["counts"]["roa_validation_cache_hash_blocked_roas"], - 1 - ); - assert_eq!( - report["counts"]["roa_validation_cache_expired_blocked_roas"], - 1 - ); - assert_eq!( - report["counts"]["roa_validation_cache_revoked_blocked_roas"], - 1 - ); - assert_eq!( - report["counts"]["roa_validation_cache_metadata_blocked_roas"], - 1 - ); - assert_eq!( - report["counts"]["roa_validation_cache_crl_recheck_hit_roas"], - 2 - ); - assert_eq!( - report["counts"]["roa_validation_cache_context_blocked_roas"], - 3 - ); - assert_eq!( - report["counts"]["roa_validation_cache_context_gate_nanos"], - 4 - ); - assert_eq!(report["counts"]["roa_validation_cache_lookup_nanos"], 5); - assert_eq!( - report["counts"]["roa_validation_cache_lookup_entry_gate_nanos"], - 6 - ); - assert_eq!( - report["counts"]["roa_validation_cache_lookup_crl_gate_nanos"], - 7 - ); - assert_eq!( - report["counts"]["roa_validation_cache_lookup_materialize_nanos"], - 8 - ); - assert_eq!( - report["counts"]["roa_validation_cache_lookup_crl_gate_verified_crls"], - 9 - ); - assert_eq!( - report["counts"]["roa_validation_cache_lookup_crl_gate_reused_crls"], - 10 - ); - } - - #[test] - fn roa_validation_cache_view_ignores_rejected_artifacts_and_non_roa_outputs() { - let validation_time = fixed_time("2026-06-05T00:00:00Z"); - let issuer_der = b"issuer-ca"; - let crl_bytes = b"current-crl".to_vec(); - let crl_hash = sha256_32(&crl_bytes); - let roa_hash = [0x11; 32]; - let mut vcir = sample_roa_cache_vcir( - issuer_der, - crl_hash, - roa_hash, - fixed_time("2026-06-07T00:00:00Z"), - fixed_time("2026-06-08T00:00:00Z"), - ); - vcir.related_artifacts.insert( - 0, - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::IssuerCert, - artifact_kind: VcirArtifactKind::Cer, - uri: Some("rsync://example.test/repo/rejected.cer".to_string()), - sha256: "00".repeat(32), - object_type: Some("cer".to_string()), - validation_status: VcirArtifactValidationStatus::Rejected, - reject_reason: None, - }, - ); - vcir.local_outputs.push(VcirLocalOutput { - output_type: VcirOutputType::Aspa, - item_effective_until: PackTime::from_utc_offset_datetime(fixed_time( - "2026-06-07T00:00:00Z", - )), - source_object_uri: "rsync://example.test/repo/a.asa".to_string(), - source_object_type: VcirSourceObjectType::Aspa, - source_object_hash: [0x44; 32], - source_ee_cert_hash: [0x55; 32], - payload: VcirLocalOutputPayload::Aspa { - customer_as_id: 64500, - provider_as_ids: vec![64501], - }, - rule_hash: [0x66; 32], - }); - let projection = sample_roa_cache_projection(&vcir, roa_hash); - let view = RoaValidationCacheView::from_projection(&projection, validation_time); - let mut crl_cache = sample_crl_cache(crl_bytes); - - assert!(view.matches_current_context( - issuer_der, - Some(TEST_CA_VALIDATION_CONTEXT), - Some(TEST_POLICY_FINGERPRINT) - )); - assert!(matches!( - view.lookup( - &PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash), - &mut crl_cache, - issuer_der, - validation_time - ), - RoaCacheLookupResult::Hit(_) - )); - assert!(matches!( - view.lookup( - &PackFile::from_bytes_with_sha256( - "rsync://example.test/repo/a.asa", - vec![0x03], - [0x44; 32], - ), - &mut crl_cache, - issuer_der, - validation_time - ), - RoaCacheLookupResult::Miss - )); - } - - #[test] - fn roa_validation_cache_context_requires_issuer_parent_and_policy() { - let mut view = RoaValidationCacheView { - entries_by_uri: HashMap::new(), - issuer_ca_sha256_hex: None, - ca_validation_context_digest: Some(TEST_CA_VALIDATION_CONTEXT), - policy_fingerprint: Some(TEST_POLICY_FINGERPRINT), - crl_sha256_by_uri: HashMap::new(), - blocked: false, - }; - assert!(!view.matches_current_context( - b"issuer-ca", - Some(TEST_CA_VALIDATION_CONTEXT), - Some(TEST_POLICY_FINGERPRINT) - )); - - view.issuer_ca_sha256_hex = Some("00".repeat(32)); - assert!(!view.matches_current_context( - b"issuer-ca", - Some(TEST_CA_VALIDATION_CONTEXT), - Some(TEST_POLICY_FINGERPRINT) - )); - - view.issuer_ca_sha256_hex = Some(sha256_hex(b"issuer-ca")); - assert!(view.matches_current_context( - b"issuer-ca", - Some(TEST_CA_VALIDATION_CONTEXT), - Some(TEST_POLICY_FINGERPRINT) - )); - assert!(!view.matches_current_context(b"issuer-ca", None, Some(TEST_POLICY_FINGERPRINT))); - - view.ca_validation_context_digest = None; - assert!(!view.matches_current_context( - b"issuer-ca", - Some(TEST_CA_VALIDATION_CONTEXT), - Some(TEST_POLICY_FINGERPRINT) - )); - view.ca_validation_context_digest = Some(TEST_CA_VALIDATION_CONTEXT); - view.policy_fingerprint = None; - assert!(!view.matches_current_context( - b"issuer-ca", - Some(TEST_CA_VALIDATION_CONTEXT), - Some(TEST_POLICY_FINGERPRINT) - )); - } - - #[test] - fn roa_validation_cache_lookup_classifies_hash_empty_payload_and_missing_uri() { - let validation_time = fixed_time("2026-06-05T00:00:00Z"); - let issuer_der = b"issuer-ca"; - let crl_bytes = b"current-crl".to_vec(); - let crl_hash_hex = sha256_hex(&crl_bytes); - let roa_hash = [0x11; 32]; - let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); - let mut output = sample_roa_cache_vcir( - issuer_der, - sha256_32(&crl_bytes), - roa_hash, - fixed_time("2026-06-07T00:00:00Z"), - fixed_time("2026-06-08T00:00:00Z"), - ) - .local_outputs - .remove(0); - let outputs_effective_until_unix = output - .item_effective_until - .parse() - .expect("parse output effective until") - .unix_timestamp(); - let mut view = RoaValidationCacheView { - entries_by_uri: HashMap::new(), - issuer_ca_sha256_hex: Some(sha256_hex(issuer_der)), - ca_validation_context_digest: Some(TEST_CA_VALIDATION_CONTEXT), - policy_fingerprint: Some(TEST_POLICY_FINGERPRINT), - crl_sha256_by_uri: HashMap::from([(TEST_CRL_URI.to_string(), crl_hash_hex.clone())]), - blocked: false, - }; - let mut crl_cache = sample_crl_cache(crl_bytes.clone()); - - assert!(matches!( - view.lookup(&file, &mut crl_cache, issuer_der, validation_time), - RoaCacheLookupResult::Miss - )); - - view.entries_by_uri.insert( - file.rsync_uri.clone(), - CachedRoaValidationResult { - source_object_hash: roa_hash, - ee_serial: Some(vec![0x01]), - crl_uri: None, - earliest_safe_reuse_time_unix: validation_time.unix_timestamp(), - outputs_effective_until_unix, - outputs: vec![output.clone()], - }, - ); - let mut crl_cache = sample_crl_cache(crl_bytes.clone()); - assert!(matches!( - view.lookup(&file, &mut crl_cache, issuer_der, validation_time), - RoaCacheLookupResult::MetadataBlocked - )); - - view.entries_by_uri.insert( - file.rsync_uri.clone(), - CachedRoaValidationResult { - source_object_hash: roa_hash, - ee_serial: None, - crl_uri: Some(TEST_CRL_URI.to_string()), - earliest_safe_reuse_time_unix: validation_time.unix_timestamp(), - outputs_effective_until_unix, - outputs: vec![output.clone()], - }, - ); - let mut crl_cache = sample_crl_cache(crl_bytes.clone()); - assert!(matches!( - view.lookup(&file, &mut crl_cache, issuer_der, validation_time), - RoaCacheLookupResult::MetadataBlocked - )); - - view.entries_by_uri.insert( - file.rsync_uri.clone(), - CachedRoaValidationResult { - source_object_hash: roa_hash, - ee_serial: Some(vec![0x01]), - crl_uri: Some(TEST_CRL_URI.to_string()), - earliest_safe_reuse_time_unix: validation_time.unix_timestamp(), - outputs_effective_until_unix, - outputs: vec![output.clone()], - }, - ); - let mut crl_cache = HashMap::new(); - assert!(matches!( - view.lookup(&file, &mut crl_cache, issuer_der, validation_time), - RoaCacheLookupResult::MetadataBlocked - )); - - view.crl_sha256_by_uri - .insert(TEST_CRL_URI.to_string(), "00".repeat(32)); - let mut crl_cache = sample_crl_cache(crl_bytes.clone()); - assert!(matches!( - view.lookup(&file, &mut crl_cache, issuer_der, validation_time), - RoaCacheLookupResult::MetadataBlocked - )); - view.crl_sha256_by_uri - .insert(TEST_CRL_URI.to_string(), crl_hash_hex); - - view.entries_by_uri.insert( - file.rsync_uri.clone(), - CachedRoaValidationResult { - source_object_hash: [0xff; 32], - ee_serial: Some(vec![0x01]), - crl_uri: Some(TEST_CRL_URI.to_string()), - earliest_safe_reuse_time_unix: validation_time.unix_timestamp(), - outputs_effective_until_unix, - outputs: vec![output.clone()], - }, - ); - let mut crl_cache = sample_crl_cache(crl_bytes.clone()); - assert!(matches!( - view.lookup(&file, &mut crl_cache, issuer_der, validation_time), - RoaCacheLookupResult::HashBlocked - )); - - view.entries_by_uri.insert( - file.rsync_uri.clone(), - CachedRoaValidationResult { - source_object_hash: roa_hash, - ee_serial: Some(vec![0x01]), - crl_uri: Some(TEST_CRL_URI.to_string()), - earliest_safe_reuse_time_unix: validation_time.unix_timestamp(), - outputs_effective_until_unix, - outputs: Vec::new(), - }, - ); - let mut crl_cache = sample_crl_cache(crl_bytes.clone()); - assert!(matches!( - view.lookup(&file, &mut crl_cache, issuer_der, validation_time), - RoaCacheLookupResult::Miss - )); - - output.payload = VcirLocalOutputPayload::Aspa { - customer_as_id: 64500, - provider_as_ids: vec![64501], - }; - view.entries_by_uri.insert( - file.rsync_uri.clone(), - CachedRoaValidationResult { - source_object_hash: roa_hash, - ee_serial: Some(vec![0x01]), - crl_uri: Some(TEST_CRL_URI.to_string()), - earliest_safe_reuse_time_unix: validation_time.unix_timestamp(), - outputs_effective_until_unix, - outputs: vec![output], - }, - ); - let mut crl_cache = sample_crl_cache(crl_bytes); - assert!(matches!( - view.lookup(&file, &mut crl_cache, issuer_der, validation_time), - RoaCacheLookupResult::MetadataBlocked - )); - } - - #[test] - fn parallel_roa_cache_blocked_falls_back_to_single_fresh_task() { - let manifest_bytes = fixture_bytes( - "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft", - ); - let issuer_ca_der = fixture_bytes( - "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ); - let crl_bytes = fixture_bytes( - "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.crl", - ); - let crl_hash = sha256_32(&crl_bytes); - let actual_roa_hash = [0x11; 32]; - let cached_roa_hash = [0x33; 32]; - let validation_time = fixed_time("2026-06-05T00:00:00Z"); - let publication_point = PublicationPointSnapshot { - format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, - manifest_rsync_uri: - "rsync://rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft" - .to_string(), - publication_point_rsync_uri: "rsync://rpki.cernet.net/repo/cernet/0/".to_string(), - manifest_number_be: vec![1], - this_update: PackTime::from_utc_offset_datetime(validation_time), - next_update: PackTime::from_utc_offset_datetime( - validation_time + time::Duration::days(1), - ), - verified_at: PackTime::from_utc_offset_datetime(validation_time), - manifest_bytes, - files: vec![ - PackFile::from_bytes_with_sha256(TEST_CRL_URI, crl_bytes, crl_hash), - PackFile::from_bytes_with_sha256( - "rsync://example.test/repo/a.roa", - vec![0x01], - actual_roa_hash, - ), - ], - }; - let vcir = sample_roa_cache_vcir( - &issuer_ca_der, - crl_hash, - cached_roa_hash, - fixed_time("2026-06-07T00:00:00Z"), - fixed_time("2026-06-08T00:00:00Z"), - ); - let projection = sample_roa_cache_projection(&vcir, cached_roa_hash); - let view = RoaValidationCacheView::from_projection(&projection, validation_time); - - let stage = match prepare_publication_point_for_parallel_roa_with_cache( - 7, - &publication_point, - &Policy::default(), - &issuer_ca_der, - None, - None, - None, - validation_time, - false, - RoaValidationCacheInput::enabled_with_context( - Some(&view), - TEST_CA_VALIDATION_CONTEXT, - TEST_POLICY_FINGERPRINT, - ), - ) { - ParallelObjectsPrepare::Staged(stage) => stage, - ParallelObjectsPrepare::Complete(_) => panic!("expected staged ROA fallback"), - }; - - assert_eq!(stage.roa_cache_stats.blocked_roas, 1); - assert_eq!(stage.roa_cache_stats.fresh_roas, 1); - assert_eq!(stage.roa_task_indices, vec![1]); - assert_eq!(stage.roa_task_count(), 1); - } - - #[test] - fn merge_as_intervals_merges_overlapping_and_adjacent() { - let v = vec![(1, 2), (3, 5), (10, 10), (11, 12)]; - let merged = merge_as_intervals(&v); - assert_eq!(merged, vec![(1, 5), (10, 12)]); - } - - #[test] - fn as_choice_subset_rejects_inherit() { - let child = Some(&AsIdentifierChoice::Inherit); - let parent = Some(&AsIdentifierChoice::AsIdsOrRanges(vec![ - AsIdOrRange::Range { min: 1, max: 10 }, - ])); - assert!(!as_choice_subset(child, parent)); - } - - #[test] - fn as_choice_subset_checks_ranges() { - let child = Some(&AsIdentifierChoice::AsIdsOrRanges(vec![ - AsIdOrRange::Id(5), - AsIdOrRange::Range { min: 7, max: 9 }, - ])); - let parent = Some(&AsIdentifierChoice::AsIdsOrRanges(vec![ - AsIdOrRange::Range { min: 1, max: 10 }, - ])); - assert!(as_choice_subset(child, parent)); - } - - #[test] - fn ip_resources_is_subset_accepts_prefixes_and_ranges() { - let parent = IpResourceSet { - families: vec![IpAddressFamily { - afi: Afi::Ipv4, - choice: IpAddressChoice::AddressesOrRanges(vec![ - IpAddressOrRange::Prefix(IpPrefix { - afi: Afi::Ipv4, - prefix_len: 8, - addr: vec![10, 0, 0, 0], - }), - IpAddressOrRange::Range(IpAddressRange { - min: vec![192, 0, 2, 0], - max: vec![192, 0, 2, 255], - }), - ]), - }], - }; - let child = IpResourceSet { - families: vec![IpAddressFamily { - afi: Afi::Ipv4, - choice: IpAddressChoice::AddressesOrRanges(vec![ - IpAddressOrRange::Prefix(IpPrefix { - afi: Afi::Ipv4, - prefix_len: 16, - addr: vec![10, 1, 0, 0], - }), - IpAddressOrRange::Range(IpAddressRange { - min: vec![192, 0, 2, 10], - max: vec![192, 0, 2, 20], - }), - ]), - }], - }; - - assert!(ip_resources_is_subset(&child, &parent)); - } - - #[test] - fn ip_resources_is_subset_rejects_inherit_in_child() { - let parent = IpResourceSet { - families: vec![IpAddressFamily { - afi: Afi::Ipv6, - choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( - IpPrefix { - afi: Afi::Ipv6, - prefix_len: 32, - addr: vec![0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - }, - )]), - }], - }; - let child = IpResourceSet { - families: vec![IpAddressFamily { - afi: Afi::Ipv6, - choice: IpAddressChoice::Inherit, - }], - }; - assert!(!ip_resources_is_subset(&child, &parent)); - } - - #[test] - fn increment_bytes_wraps_all_ff_to_zero() { - assert_eq!(increment_bytes(&[0xFF, 0xFF]), vec![0x00, 0x00]); - } - - #[test] - fn merge_ip_intervals_merges_contiguous() { - let mut v = vec![ - (vec![0, 0, 0, 0], vec![0, 0, 0, 10]), - (vec![0, 0, 0, 11], vec![0, 0, 0, 20]), - ]; - merge_ip_intervals_in_place(&mut v); - assert_eq!(v, vec![(vec![0, 0, 0, 0], vec![0, 0, 0, 20])]); - } - - #[test] - fn choose_crl_for_certificate_reports_missing_crl_in_snapshot() { - let roa_der = - fixture_bytes("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"); - let roa = RoaObject::decode_der(&roa_der).expect("decode roa"); - let ee_crldp_uris = roa.signed_object.signed_data.certificates[0] - .resource_cert - .tbs - .extensions - .crl_distribution_points_uris - .as_ref(); - let crl_cache: HashMap = HashMap::new(); - let err = choose_crl_uri_for_certificate(ee_crldp_uris, &crl_cache).unwrap_err(); - assert!(matches!(err, ObjectValidateError::MissingCrlInPack)); - } - - #[test] - fn choose_crl_for_certificate_reports_missing_crldp_uris() { - let mut crl_cache: HashMap = HashMap::new(); - crl_cache.insert( - "rsync://example.test/a.crl".to_string(), - CachedIssuerCrl::Pending { - bytes: vec![0x01], - sha256_hex: None, - }, - ); - let err = choose_crl_uri_for_certificate(None, &crl_cache).unwrap_err(); - assert!(matches!(err, ObjectValidateError::MissingCrlDpUris)); - } - - #[test] - fn choose_crl_for_certificate_prefers_matching_crldp_uri_in_order() { - let roa_der = - fixture_bytes("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"); - let roa = RoaObject::decode_der(&roa_der).expect("decode roa"); - let ee_crldp_uris = roa.signed_object.signed_data.certificates[0] - .resource_cert - .tbs - .extensions - .crl_distribution_points_uris - .as_ref() - .expect("fixture ee has crldp"); - - let matching_uri = ee_crldp_uris[0].as_str().to_string(); - let mut crl_cache: HashMap = HashMap::new(); - crl_cache.insert( - "rsync://example.test/other.crl".to_string(), - CachedIssuerCrl::Pending { - bytes: vec![0x00], - sha256_hex: None, - }, - ); - crl_cache.insert( - matching_uri.clone(), - CachedIssuerCrl::Pending { - bytes: vec![0x01], - sha256_hex: None, - }, - ); - - let uri = choose_crl_uri_for_certificate(Some(ee_crldp_uris), &crl_cache).unwrap(); - assert_eq!(uri, matching_uri); - } - - #[test] - fn choose_crl_for_certificate_reports_not_found_when_crldp_does_not_match_snapshot() { - let roa_der = - fixture_bytes("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"); - let roa = RoaObject::decode_der(&roa_der).expect("decode roa"); - let ee_crldp_uris = roa.signed_object.signed_data.certificates[0] - .resource_cert - .tbs - .extensions - .crl_distribution_points_uris - .as_ref(); - - let mut crl_cache: HashMap = HashMap::new(); - crl_cache.insert( - "rsync://example.test/other.crl".to_string(), - CachedIssuerCrl::Pending { - bytes: vec![0x01], - sha256_hex: None, - }, - ); - let err = choose_crl_uri_for_certificate(ee_crldp_uris, &crl_cache).unwrap_err(); - assert!(matches!(err, ObjectValidateError::CrlNotFound(_))); - } - - #[test] - fn validate_ee_resources_subset_reports_missing_issuer_effective_ip() { - let roa_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"); - let roa_der = std::fs::read(roa_path).expect("read roa"); - let roa = RoaObject::decode_der(&roa_der).expect("decode roa"); - let ee = &roa.signed_object.signed_data.certificates[0].resource_cert; - - let idx = IssuerResourcesIndex::default(); - let err = validate_ee_resources_subset(ee, None, None, &idx).unwrap_err(); - assert!(matches!(err, ObjectValidateError::MissingIssuerEffectiveIp)); - } - - #[test] - fn validate_ee_resources_subset_reports_missing_issuer_effective_as() { - let aspa_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/5m80fwYws_3FiFD7JiQjAqZ1RYQ.asa", - ); - let aspa_der = std::fs::read(aspa_path).expect("read aspa"); - let aspa = AspaObject::decode_der(&aspa_der).expect("decode aspa"); - let ee = &aspa.signed_object.signed_data.certificates[0].resource_cert; - - let issuer_ip = IpResourceSet { - families: vec![IpAddressFamily { - afi: Afi::Ipv6, - choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( - IpPrefix { - afi: Afi::Ipv6, - prefix_len: 32, - addr: vec![0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - }, - )]), - }], - }; - - let idx = build_issuer_resources_index(Some(&issuer_ip), None); - let err = validate_ee_resources_subset(ee, Some(&issuer_ip), None, &idx).unwrap_err(); - assert!(matches!(err, ObjectValidateError::MissingIssuerEffectiveAs)); - } - - #[test] - fn validate_ee_resources_subset_reports_not_subset() { - let roa_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"); - let roa_der = std::fs::read(roa_path).expect("read roa"); - let roa = RoaObject::decode_der(&roa_der).expect("decode roa"); - let ee = &roa.signed_object.signed_data.certificates[0].resource_cert; - - // Unrelated parent resources. - let issuer_ip = IpResourceSet { - families: vec![IpAddressFamily { - afi: Afi::Ipv6, - choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( - IpPrefix { - afi: Afi::Ipv6, - prefix_len: 32, - addr: vec![0x26, 0x20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - }, - )]), - }], - }; - - let idx = build_issuer_resources_index(Some(&issuer_ip), None); - let err = validate_ee_resources_subset(ee, Some(&issuer_ip), None, &idx).unwrap_err(); - assert!(matches!(err, ObjectValidateError::EeResourcesNotSubset)); - } - - #[test] - fn validation_update_03_ee_resources_reduce_to_vrs_for_roa_checks() { - let roa_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"); - let roa_der = std::fs::read(roa_path).expect("read roa"); - let roa = RoaObject::decode_der(&roa_der).expect("decode roa"); - let ee = &roa.signed_object.signed_data.certificates[0].resource_cert; - let ee_ip = ee - .tbs - .extensions - .ip_resources - .as_ref() - .expect("fixture EE has IP resources"); - let first_family = ee_ip.families.first().expect("family"); - let first_item = match &first_family.choice { - IpAddressChoice::AddressesOrRanges(items) => items.first().expect("item").clone(), - IpAddressChoice::Inherit => panic!("fixture should not inherit"), - }; - let issuer_ip = IpResourceSet { - families: vec![IpAddressFamily { - afi: first_family.afi, - choice: IpAddressChoice::AddressesOrRanges(vec![first_item]), - }], - }; - let idx = build_issuer_resources_index(Some(&issuer_ip), None); - - let strict_err = validate_ee_resources_for_mode( - ee, - Some(&issuer_ip), - None, - &idx, - ResourceValidationMode::Rfc6487, - ) - .unwrap_err(); - assert!(matches!( - strict_err, - ObjectValidateError::EeResourcesNotSubset - )); - - let vrs = validate_ee_resources_for_mode( - ee, - Some(&issuer_ip), - None, - &idx, - ResourceValidationMode::ValidationUpdate03, - ) - .expect("vrs resource resolution"); - assert_eq!(vrs.ip.expect("vrs ip").families.len(), 1); - } - - #[test] - fn validation_update_03_ee_as_vrs_filters_aspa_customer_resources() { - let issuer_as = AsResourceSet { - asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ - AsIdOrRange::Range { - min: 64500, - max: 64510, - }, - ])), - rdi: None, - }; - let child_as = AsResourceSet { - asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ - AsIdOrRange::Range { - min: 64505, - max: 64520, - }, - ])), - rdi: None, - }; - let idx = build_issuer_resources_index(None, Some(&issuer_as)); - let vrs = intersect_ee_as_resources_vrs(&child_as, Some(&issuer_as), &idx) - .expect("intersect AS resources"); - assert!(as_resource_set_contains_asn(&vrs, 64505)); - assert!(as_resource_set_contains_asn(&vrs, 64510)); - assert!(!as_resource_set_contains_asn(&vrs, 64511)); - } - - #[test] - fn validation_update_03_empty_ee_vrs_rejects_roa_and_aspa_outputs() { - let roa_der = - fixture_bytes("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"); - let roa = RoaObject::decode_der(&roa_der).expect("decode roa"); - let empty_ip_vrs = IpResourceSet { - families: Vec::new(), - }; - let roa_err = roa_to_vrps_with_vrs(&roa, Some(&empty_ip_vrs)).unwrap_err(); - assert!(matches!(roa_err, ObjectValidateError::EeResourcesNotSubset)); - let missing_ip_err = roa_to_vrps_with_vrs(&roa, None).unwrap_err(); - assert!(matches!( - missing_ip_err, - ObjectValidateError::EeResourcesNotSubset - )); - - let aspa_der = fixture_bytes( - "tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/5m80fwYws_3FiFD7JiQjAqZ1RYQ.asa", - ); - let aspa = AspaObject::decode_der(&aspa_der).expect("decode aspa"); - let empty_as_vrs = AsResourceSet { - asnum: Some(AsIdentifierChoice::AsIdsOrRanges(Vec::new())), - rdi: None, - }; - let aspa_err = validate_aspa_customer_in_vrs(&aspa, Some(&empty_as_vrs)).unwrap_err(); - assert!(matches!( - aspa_err, - ObjectValidateError::EeResourcesNotSubset - )); - let missing_as_err = validate_aspa_customer_in_vrs(&aspa, None).unwrap_err(); - assert!(matches!( - missing_as_err, - ObjectValidateError::EeResourcesNotSubset - )); - } - - #[test] - fn extra_rfc_refs_for_crl_selection_distinguishes_crl_errors() { - assert_eq!( - extra_rfc_refs_for_crl_selection(&ObjectValidateError::MissingCrlDpUris), - RFC_CRLDP - ); - assert_eq!( - extra_rfc_refs_for_crl_selection(&ObjectValidateError::CrlNotFound( - "rsync://example.test/x.crl".to_string(), - )), - RFC_CRLDP_AND_LOCKED_PACK - ); - assert!( - extra_rfc_refs_for_crl_selection(&ObjectValidateError::MissingCrlInPack).is_empty() - ); - } - - #[test] - fn as_subset_helpers_cover_success_and_failure_paths() { - let child = AsIdentifierChoice::AsIdsOrRanges(vec![ - AsIdOrRange::Id(5), - AsIdOrRange::Range { min: 7, max: 9 }, - ]); - let parent = - AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Range { min: 1, max: 10 }]); - let parent_intervals = [(1, 10)]; - - assert!(as_choice_subset(None, Some(&parent))); - assert!(!as_choice_subset(Some(&child), None)); - assert!(!as_choice_subset( - Some(&AsIdentifierChoice::Inherit), - Some(&parent) - )); - assert!(!as_choice_subset( - Some(&child), - Some(&AsIdentifierChoice::Inherit) - )); - assert!(as_choice_subset(Some(&child), Some(&parent))); - - assert!(as_choice_subset_indexed(None, Some(&parent_intervals))); - assert!(!as_choice_subset_indexed(Some(&child), None)); - assert!(!as_choice_subset_indexed( - Some(&AsIdentifierChoice::Inherit), - Some(&parent_intervals), - )); - assert!(as_choice_subset_indexed( - Some(&child), - Some(&parent_intervals) - )); - assert!(!as_choice_subset_indexed( - Some(&AsIdentifierChoice::AsIdsOrRanges(vec![ - AsIdOrRange::Range { min: 11, max: 12 } - ])), - Some(&parent_intervals), - )); - - let child_set = AsResourceSet { - asnum: Some(child.clone()), - rdi: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id(42)])), - }; - let parent_set = AsResourceSet { - asnum: Some(parent.clone()), - rdi: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ - AsIdOrRange::Range { min: 40, max: 50 }, - ])), - }; - assert!(as_resources_is_subset(&child_set, &parent_set)); - assert!(as_resources_is_subset_indexed( - &child_set, - &parent_set, - &IssuerResourcesIndex { - asnum: Some(vec![(1, 10)]), - rdi: Some(vec![(40, 50)]), - ..IssuerResourcesIndex::default() - }, - )); - } - - #[test] - fn ip_subset_helpers_cover_strict_and_indexed_paths() { - let parent = IpResourceSet { - families: vec![ - IpAddressFamily { - afi: Afi::Ipv4, - choice: IpAddressChoice::AddressesOrRanges(vec![ - IpAddressOrRange::Prefix(IpPrefix { - afi: Afi::Ipv4, - prefix_len: 8, - addr: vec![10, 0, 0, 0], - }), - IpAddressOrRange::Range(IpAddressRange { - min: vec![192, 0, 2, 0], - max: vec![192, 0, 2, 255], - }), - ]), - }, - IpAddressFamily { - afi: Afi::Ipv6, - choice: IpAddressChoice::Inherit, - }, - ], - }; - let child = IpResourceSet { - families: vec![IpAddressFamily { - afi: Afi::Ipv4, - choice: IpAddressChoice::AddressesOrRanges(vec![ - IpAddressOrRange::Prefix(IpPrefix { - afi: Afi::Ipv4, - prefix_len: 16, - addr: vec![10, 1, 0, 0], - }), - IpAddressOrRange::Range(IpAddressRange { - min: vec![192, 0, 2, 10], - max: vec![192, 0, 2, 20], - }), - ]), - }], - }; - let strict_bad = IpResourceSet { - families: vec![IpAddressFamily { - afi: Afi::Ipv4, - choice: IpAddressChoice::Inherit, - }], - }; - - assert!(ip_resources_is_subset(&child, &parent)); - assert!(ip_resources_to_merged_intervals(&parent).contains_key(&AfiKey::V4)); - assert!(ip_resources_to_merged_intervals_strict(&child).is_ok()); - assert!(ip_resources_to_merged_intervals_strict(&strict_bad).is_err()); - - let idx = build_issuer_resources_index( - Some(&parent), - Some(&AsResourceSet { - asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ - AsIdOrRange::Range { - min: 64500, - max: 64510, - }, - ])), - rdi: Some(AsIdentifierChoice::Inherit), - }), - ); - assert!(idx.ip_v4.is_some()); - assert!(idx.ip_v6.is_none()); - assert!(idx.asnum.is_some()); - assert!(idx.rdi.is_none()); - assert!(ip_resources_is_subset_indexed(&child, &parent, &idx)); - assert!(!ip_resources_is_subset_indexed(&strict_bad, &parent, &idx)); - } - - #[test] - fn interval_and_byte_helpers_cover_edge_cases() { - let parent = vec![(vec![0, 0, 0, 0], vec![0, 0, 0, 10])]; - assert!(interval_is_covered(&parent, &[0, 0, 0, 1], &[0, 0, 0, 2])); - assert!(!interval_is_covered( - &parent, - &[0, 0, 0, 11], - &[0, 0, 0, 12] - )); - assert!(intervals_are_covered( - &parent, - &[(vec![0, 0, 0, 1], vec![0, 0, 0, 2])] - )); - assert!(!intervals_are_covered( - &parent, - &[(vec![0, 0, 0, 9], vec![0, 0, 0, 11])], - )); - - let prefix = RcIpPrefix { - afi: Afi::Ipv4, - prefix_len: 24, - addr: vec![203, 0, 113, 7], - }; - assert_eq!( - prefix_to_range(&prefix), - (vec![203, 0, 113, 0], vec![203, 0, 113, 255]) - ); - assert_eq!(increment_bytes(&[0, 0, 0, 255]), vec![0, 0, 1, 0]); - assert!(bytes_is_next(&[0, 0, 1, 0], &[0, 0, 0, 255])); - assert!(!bytes_is_next(&[1, 2], &[1])); - } - - #[test] - fn merged_interval_helpers_cover_empty_and_break_paths() { - let mut empty: Vec<(Vec, Vec)> = Vec::new(); - merge_ip_intervals_in_place(&mut empty); - assert!(empty.is_empty()); - - let mut v = vec![ - (vec![0, 0, 0, 20], vec![0, 0, 0, 30]), - (vec![0, 0, 0, 0], vec![0, 0, 0, 10]), - (vec![0, 0, 0, 11], vec![0, 0, 0, 19]), - ]; - v.sort_by(|(a, _), (b, _)| a.cmp(b)); - merge_ip_intervals_in_place(&mut v); - assert_eq!(v, vec![(vec![0, 0, 0, 0], vec![0, 0, 0, 30])]); - - assert_eq!( - as_choice_to_merged_intervals(&AsIdentifierChoice::AsIdsOrRanges(vec![ - AsIdOrRange::Id(1), - AsIdOrRange::Range { min: 2, max: 3 }, - AsIdOrRange::Range { min: 7, max: 9 }, - ])), - vec![(1, 3), (7, 9)] - ); - assert!(as_interval_is_covered(&[(1, 3), (7, 9)], 2, 3)); - assert!(!as_interval_is_covered(&[(7, 9)], 2, 3)); - } - - #[test] - fn roa_output_helpers_cover_vrps_and_afi_strings() { - let roa_der = - fixture_bytes("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"); - let roa = RoaObject::decode_der(&roa_der).expect("decode roa"); - let vrps = roa_to_vrps(&roa); - assert!(!vrps.is_empty()); - assert!(vrps.iter().all(|vrp| vrp.asn == roa.roa.as_id)); - assert_eq!(roa_afi_to_string(RoaAfi::Ipv4), "ipv4"); - assert_eq!(roa_afi_to_string(RoaAfi::Ipv6), "ipv6"); - } - - #[test] - fn parallel_stage_roa_tasks_share_stage_owned_payloads() { - let ta_constraints = Arc::new( - crate::ta_constraints::TaConstraints::from_file( - &std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/ta_constraints/afrinic-current-ipv4-deny.constraints"), - ) - .expect("load constraints fixture"), - ); - let stage = ParallelObjectsStage { - publication_point_id: 7, - shared: Arc::new(RoaTaskShared { - locked_files: Arc::<[PackFile]>::from(vec![ - PackFile::from_bytes_with_sha256( - "rsync://example.test/repo/a.roa", - vec![1, 2, 3], - [1u8; 32], - ), - PackFile::from_bytes_with_sha256( - "rsync://example.test/repo/b.roa", - vec![4, 5, 6], - [2u8; 32], - ), - ]), - manifest_rsync_uri: Arc::::from("rsync://example.test/repo/manifest.mft"), - issuer_ca_der: Arc::from([0x01u8].as_slice()), - issuer_ca: Arc::new( - ResourceCertificate::decode_der(&fixture_bytes( - "tests/fixtures/ta/apnic-ta.cer", - )) - .expect("decode fixture CA certificate"), - ), - issuer_spki_der: Arc::from([0x02u8].as_slice()), - issuer_ca_rsync_uri: Some(Arc::::from("rsync://example.test/repo/ca.cer")), - crl_cache: Arc::new(Mutex::new(HashMap::new())), - issuer_resources_index: Arc::new(IssuerResourcesIndex::default()), - issuer_effective_ip: None, - issuer_effective_as: None, - resource_validation_mode: ResourceValidationMode::default(), - ta_constraints: Some(Arc::clone(&ta_constraints)), - }), - validation_time: OffsetDateTime::now_utc(), - collect_vcir_local_outputs: false, - strict_cms_der: false, - strict_name: false, - resource_validation_mode: ResourceValidationMode::default(), - roa_task_indices: vec![0, 1], - cached_roa_results: Vec::new(), - roa_cache_stats: RoaValidationCacheStats::default(), - warnings: Vec::new(), - stats: ObjectsStats { - roa_total: 2, - ..ObjectsStats::default() - }, - audit: Vec::new(), - }; - let tasks = stage.build_roa_tasks(); - assert_eq!(tasks.len(), 2); - assert!(Arc::ptr_eq(&tasks[0].shared, &tasks[1].shared)); - assert!(Arc::ptr_eq( - tasks[0] - .shared - .ta_constraints - .as_ref() - .expect("task constraints snapshot"), - &ta_constraints - )); - } - - #[test] - fn strict_name_manifest_decode_failure_drops_publication_point() { - let publication_point = PublicationPointSnapshot { - format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, - manifest_rsync_uri: "rsync://example.test/repo/manifest.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), - manifest_number_be: vec![0x01], - this_update: PackTime::from_utc_offset_datetime(OffsetDateTime::now_utc()), - next_update: PackTime::from_utc_offset_datetime(OffsetDateTime::now_utc()), - verified_at: PackTime::from_utc_offset_datetime(OffsetDateTime::now_utc()), - manifest_bytes: vec![0x01, 0x02, 0x03], - files: vec![], - }; - let policy = Policy::default(); - let output = process_publication_point_for_issuer_with_options( - &publication_point, - &policy, - &[], - None, - None, - None, - OffsetDateTime::now_utc(), - None, - false, - ); - assert!(output.stats.publication_point_dropped); - assert!(output.vrps.is_empty()); - assert!( - output - .warnings - .iter() - .any(|warning| warning.message.contains("manifest decode failed")), - "{:?}", - output.warnings - ); - } + include!("objects/tests/cache.rs"); + include!("objects/tests/resource_helpers.rs"); + include!("objects/tests/output.rs"); } diff --git a/crates/panda-rpki-validator/src/validation/objects/cache.rs b/crates/panda-rpki-validator/src/validation/objects/cache.rs new file mode 100644 index 0000000..c6432c9 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/objects/cache.rs @@ -0,0 +1,871 @@ + +const RFC_NONE: &[RfcRef] = &[]; +const RFC_CRLDP: &[RfcRef] = &[RfcRef("RFC 6487 §4.8.6")]; +const RFC_CRLDP_AND_LOCKED_PACK: &[RfcRef] = + &[RfcRef("RFC 6487 §4.8.6"), RfcRef("RFC 9286 §4.2.1")]; + +fn ber_compatible_cms_warning(der: &[u8], rsync_uri: &str, object_kind: &str) -> Option { + let strict_error = RpkiSignedObject::strict_cms_der_error(der)?; + Some( + Warning::new(format!( + "accepted BER-compatible CMS encoding for {object_kind}: {rsync_uri}: {strict_error}" + )) + .with_category(WarningCategory::BerCompatibleCmsEncoding) + .with_rfc_refs(&[RfcRef("X.690 §10"), RfcRef("RFC 6488 §2")]) + .with_context(rsync_uri), + ) +} + +fn ber_compatible_cms_warning_for_file(file: &PackFile, object_kind: &str) -> Option { + ber_compatible_cms_warning(file.bytes().ok()?, &file.rsync_uri, object_kind) +} + +fn sha256_hex_to_32(hex_value: &str) -> [u8; 32] { + let bytes = hex::decode(hex_value).expect("internal sha256 hex should decode"); + let mut out = [0u8; 32]; + out.copy_from_slice(&bytes); + out +} + +fn sha256_hex(bytes: &[u8]) -> String { + hex::encode(sha2::Sha256::digest(bytes)) +} + +fn decode_resource_certificate_with_policy( + der: &[u8], + policy: &Policy, +) -> Result { + if policy.strict.name { + ResourceCertificate::decode_der_with_strict_name(der) + } else { + ResourceCertificate::decode_der(der) + } +} + +#[derive(Clone, Debug)] +pub(crate) struct VerifiedIssuerCrl { + crl: crate::data_model::crl::RpkixCrl, + revoked_serials: std::collections::HashSet>, + sha256_hex: String, +} + +#[derive(Clone, Debug)] +pub(crate) enum CachedIssuerCrl { + Pending { + bytes: Vec, + sha256_hex: Option, + }, + Ok(Arc), +} + +impl CachedIssuerCrl { + fn current_sha256_hex(&mut self) -> &str { + match self { + CachedIssuerCrl::Pending { bytes, sha256_hex } => { + if sha256_hex.is_none() { + *sha256_hex = Some(crate::audit::sha256_hex(bytes)); + } + sha256_hex + .as_deref() + .expect("pending CRL sha256 must be populated") + } + CachedIssuerCrl::Ok(verified) => verified.sha256_hex.as_str(), + } + } +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct IssuerResourcesIndex { + ip_v4: Option, Vec)>>, + ip_v6: Option, Vec)>>, + asnum: Option>, + rdi: Option>, +} + +fn extra_rfc_refs_for_crl_selection(e: &ObjectValidateError) -> &'static [RfcRef] { + match e { + ObjectValidateError::MissingCrlDpUris => RFC_CRLDP, + ObjectValidateError::CrlNotFound(_) => RFC_CRLDP_AND_LOCKED_PACK, + _ => RFC_NONE, + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Vrp { + pub asn: u32, + pub prefix: IpPrefix, + pub max_length: u16, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AspaAttestation { + pub customer_as_id: u32, + pub provider_as_ids: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RouterKeyPayload { + pub as_id: u32, + pub ski: Vec, + pub spki_der: Vec, + pub source_object_uri: String, + pub source_object_hash: String, + pub source_ee_cert_hash: String, + pub item_effective_until: PackTime, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ObjectsOutput { + pub vrps: Vec, + pub aspas: Vec, + pub router_keys: Vec, + pub local_outputs_cache: Vec, + pub warnings: Vec, + pub stats: ObjectsStats, + pub audit: Vec, + pub roa_cache_stats: RoaValidationCacheStats, + pub roa_cache_object_meta: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ObjectsStats { + pub roa_total: usize, + pub roa_ok: usize, + pub aspa_total: usize, + pub aspa_ok: usize, + /// Whether this publication point was dropped due to an unrecoverable objects-processing error + /// (e.g., missing issuer CRL in the pack, or `signed_object_failure_policy=drop_publication_point`). + pub publication_point_dropped: bool, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize)] +pub struct RoaValidationCacheStats { + pub enabled_publication_points: usize, + pub vcir_hit_publication_points: usize, + pub vcir_miss_publication_points: usize, + pub hit_roas: usize, + pub miss_roas: usize, + pub blocked_roas: usize, + pub fresh_roas: usize, + pub context_blocked_roas: usize, + pub crl_recheck_hit_roas: usize, + pub hash_blocked_roas: usize, + pub expired_blocked_roas: usize, + pub revoked_blocked_roas: usize, + pub metadata_blocked_roas: usize, + pub context_gate_nanos: u64, + pub lookup_nanos: u64, + pub lookup_entry_gate_nanos: u64, + pub lookup_crl_gate_nanos: u64, + pub lookup_materialize_nanos: u64, + pub lookup_crl_gate_verified_crls: usize, + pub lookup_crl_gate_reused_crls: usize, +} + +#[derive(Clone, Copy, Debug)] +pub struct RoaValidationCacheInput<'a> { + enabled: bool, + view: Option<&'a RoaValidationCacheView>, + ca_validation_context_digest: Option<[u8; 32]>, + policy_fingerprint: Option<[u8; 32]>, +} + +impl<'a> RoaValidationCacheInput<'a> { + pub fn disabled() -> Self { + Self { + enabled: false, + view: None, + ca_validation_context_digest: None, + policy_fingerprint: None, + } + } + + pub fn enabled(view: Option<&'a RoaValidationCacheView>) -> Self { + Self { + enabled: true, + view, + ca_validation_context_digest: None, + policy_fingerprint: None, + } + } + + pub fn enabled_with_context( + view: Option<&'a RoaValidationCacheView>, + ca_validation_context_digest: [u8; 32], + policy_fingerprint: [u8; 32], + ) -> Self { + Self { + enabled: true, + view, + ca_validation_context_digest: Some(ca_validation_context_digest), + policy_fingerprint: Some(policy_fingerprint), + } + } +} + +impl RoaValidationCacheStats { + pub fn add_assign(&mut self, other: &Self) { + self.enabled_publication_points += other.enabled_publication_points; + self.vcir_hit_publication_points += other.vcir_hit_publication_points; + self.vcir_miss_publication_points += other.vcir_miss_publication_points; + self.hit_roas += other.hit_roas; + self.miss_roas += other.miss_roas; + self.blocked_roas += other.blocked_roas; + self.fresh_roas += other.fresh_roas; + self.context_blocked_roas += other.context_blocked_roas; + self.crl_recheck_hit_roas += other.crl_recheck_hit_roas; + self.hash_blocked_roas += other.hash_blocked_roas; + self.expired_blocked_roas += other.expired_blocked_roas; + self.revoked_blocked_roas += other.revoked_blocked_roas; + self.metadata_blocked_roas += other.metadata_blocked_roas; + self.context_gate_nanos = self + .context_gate_nanos + .saturating_add(other.context_gate_nanos); + self.lookup_nanos = self.lookup_nanos.saturating_add(other.lookup_nanos); + self.lookup_entry_gate_nanos = self + .lookup_entry_gate_nanos + .saturating_add(other.lookup_entry_gate_nanos); + self.lookup_crl_gate_nanos = self + .lookup_crl_gate_nanos + .saturating_add(other.lookup_crl_gate_nanos); + self.lookup_materialize_nanos = self + .lookup_materialize_nanos + .saturating_add(other.lookup_materialize_nanos); + self.lookup_crl_gate_verified_crls += other.lookup_crl_gate_verified_crls; + self.lookup_crl_gate_reused_crls += other.lookup_crl_gate_reused_crls; + } + + fn for_input(input: RoaValidationCacheInput<'_>, roa_total: usize) -> Self { + let mut stats = Self::default(); + if !input.enabled { + return stats; + } + + stats.enabled_publication_points = 1; + if input.view.is_some() { + stats.vcir_hit_publication_points = 1; + } else { + stats.vcir_miss_publication_points = 1; + stats.miss_roas = roa_total; + stats.fresh_roas = roa_total; + } + stats + } + + fn record_lookup(&mut self, total_nanos: u64, metrics: RoaCacheLookupMetrics) { + self.lookup_nanos = self.lookup_nanos.saturating_add(total_nanos); + self.lookup_entry_gate_nanos = self + .lookup_entry_gate_nanos + .saturating_add(metrics.entry_gate_nanos); + self.lookup_crl_gate_nanos = self + .lookup_crl_gate_nanos + .saturating_add(metrics.crl_gate_nanos); + self.lookup_materialize_nanos = self + .lookup_materialize_nanos + .saturating_add(metrics.materialize_nanos); + self.lookup_crl_gate_verified_crls += metrics.crl_gate_verified_crls; + self.lookup_crl_gate_reused_crls += metrics.crl_gate_reused_crls; + } + + fn record_to_timing(&self, timing: Option<&TimingHandle>) { + let Some(timing) = timing else { + return; + }; + record_non_zero( + timing, + "roa_validation_cache_enabled_publication_points", + self.enabled_publication_points, + ); + record_non_zero( + timing, + "roa_validation_cache_vcir_hit_publication_points", + self.vcir_hit_publication_points, + ); + record_non_zero( + timing, + "roa_validation_cache_vcir_miss_publication_points", + self.vcir_miss_publication_points, + ); + record_non_zero(timing, "roa_validation_cache_hit_roas", self.hit_roas); + record_non_zero(timing, "roa_validation_cache_miss_roas", self.miss_roas); + record_non_zero( + timing, + "roa_validation_cache_blocked_roas", + self.blocked_roas, + ); + record_non_zero(timing, "roa_validation_cache_fresh_roas", self.fresh_roas); + record_non_zero( + timing, + "roa_validation_cache_context_blocked_roas", + self.context_blocked_roas, + ); + record_non_zero( + timing, + "roa_validation_cache_crl_recheck_hit_roas", + self.crl_recheck_hit_roas, + ); + record_non_zero( + timing, + "roa_validation_cache_hash_blocked_roas", + self.hash_blocked_roas, + ); + record_non_zero( + timing, + "roa_validation_cache_expired_blocked_roas", + self.expired_blocked_roas, + ); + record_non_zero( + timing, + "roa_validation_cache_revoked_blocked_roas", + self.revoked_blocked_roas, + ); + record_non_zero( + timing, + "roa_validation_cache_metadata_blocked_roas", + self.metadata_blocked_roas, + ); + record_non_zero( + timing, + "roa_validation_cache_context_gate_nanos", + self.context_gate_nanos as usize, + ); + record_non_zero( + timing, + "roa_validation_cache_lookup_nanos", + self.lookup_nanos as usize, + ); + record_non_zero( + timing, + "roa_validation_cache_lookup_entry_gate_nanos", + self.lookup_entry_gate_nanos as usize, + ); + record_non_zero( + timing, + "roa_validation_cache_lookup_crl_gate_nanos", + self.lookup_crl_gate_nanos as usize, + ); + record_non_zero( + timing, + "roa_validation_cache_lookup_materialize_nanos", + self.lookup_materialize_nanos as usize, + ); + record_non_zero( + timing, + "roa_validation_cache_lookup_crl_gate_verified_crls", + self.lookup_crl_gate_verified_crls, + ); + record_non_zero( + timing, + "roa_validation_cache_lookup_crl_gate_reused_crls", + self.lookup_crl_gate_reused_crls, + ); + timing.record_phase_nanos( + "roa_validation_cache_context_gate_total", + self.context_gate_nanos, + ); + timing.record_phase_nanos("roa_validation_cache_lookup_total", self.lookup_nanos); + timing.record_phase_nanos( + "roa_validation_cache_lookup_entry_gate_total", + self.lookup_entry_gate_nanos, + ); + timing.record_phase_nanos( + "roa_validation_cache_lookup_crl_gate_total", + self.lookup_crl_gate_nanos, + ); + timing.record_phase_nanos( + "roa_validation_cache_lookup_materialize_total", + self.lookup_materialize_nanos, + ); + } +} + +fn record_non_zero(timing: &TimingHandle, key: &'static str, value: usize) { + if value > 0 { + timing.record_count(key, value as u64); + } +} + +fn elapsed_nanos_u64(started: Instant) -> u64 { + started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64 +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct RoaCacheLookupMetrics { + entry_gate_nanos: u64, + crl_gate_nanos: u64, + materialize_nanos: u64, + crl_gate_verified_crls: usize, + crl_gate_reused_crls: usize, +} + +#[derive(Clone, Debug)] +enum RoaCacheCrlGate { + Unchanged, + ChangedValid(Arc), + Expired, + Invalid, + Missing, +} + +#[derive(Debug, Default)] +struct RoaCacheCrlGateSet { + gates_by_uri: HashMap, +} + +impl RoaCacheCrlGateSet { + fn evaluate( + &mut self, + crl_uri: &str, + expected_crl_sha256_by_uri: &HashMap, + crl_cache: &mut std::collections::HashMap, + issuer_ca_der: &[u8], + validation_time: time::OffsetDateTime, + metrics: &mut RoaCacheLookupMetrics, + ) -> RoaCacheCrlGate { + if let Some(gate) = self.gates_by_uri.get(crl_uri) { + metrics.crl_gate_reused_crls += 1; + return gate.clone(); + } + + let gate = evaluate_roa_cache_crl_gate( + crl_uri, + expected_crl_sha256_by_uri, + crl_cache, + issuer_ca_der, + validation_time, + metrics, + ); + self.gates_by_uri.insert(crl_uri.to_string(), gate.clone()); + gate + } +} + +fn evaluate_roa_cache_crl_gate( + crl_uri: &str, + expected_crl_sha256_by_uri: &HashMap, + crl_cache: &mut std::collections::HashMap, + issuer_ca_der: &[u8], + validation_time: time::OffsetDateTime, + metrics: &mut RoaCacheLookupMetrics, +) -> RoaCacheCrlGate { + let crl_unchanged = { + let Some(current_crl_hash) = crl_cache + .get_mut(crl_uri) + .map(CachedIssuerCrl::current_sha256_hex) + else { + return RoaCacheCrlGate::Missing; + }; + expected_crl_sha256_by_uri + .get(crl_uri) + .map(|expected| expected == current_crl_hash) + .unwrap_or(false) + }; + if crl_unchanged { + return RoaCacheCrlGate::Unchanged; + } + + let verified_crl = match ensure_issuer_crl_verified(crl_uri, crl_cache, issuer_ca_der) { + Ok(verified_crl) => { + metrics.crl_gate_verified_crls += 1; + verified_crl + } + Err(_) => return RoaCacheCrlGate::Invalid, + }; + if !crl_valid_at_time(&verified_crl.crl, validation_time) { + return RoaCacheCrlGate::Expired; + } + RoaCacheCrlGate::ChangedValid(verified_crl) +} + +#[derive(Clone, Debug)] +pub struct RoaValidationCacheView { + entries_by_uri: HashMap, + issuer_ca_sha256_hex: Option, + ca_validation_context_digest: Option<[u8; 32]>, + policy_fingerprint: Option<[u8; 32]>, + crl_sha256_by_uri: HashMap, + blocked: bool, +} + +#[derive(Clone, Debug)] +pub struct CachedRoaValidationResult { + source_object_hash: [u8; 32], + ee_serial: Option>, + crl_uri: Option, + earliest_safe_reuse_time_unix: i64, + outputs_effective_until_unix: i64, + outputs: Vec, +} + +impl RoaValidationCacheView { + pub fn from_projection( + projection: &RoaCacheProjection, + validation_time: time::OffsetDateTime, + ) -> Self { + let mut entries_by_uri: HashMap = + HashMap::with_capacity(projection.entries.len()); + let issuer_ca_sha256_hex = projection.issuer_ca_sha256_hex.clone(); + let ca_validation_context_digest = projection.ca_validation_context_digest; + let policy_fingerprint = projection.policy_fingerprint; + let crl_sha256_by_uri = projection + .crl_sha256_by_uri + .iter() + .map(|crl| (crl.uri.clone(), crl.sha256.clone())) + .collect::>(); + let blocked = projection + .instance_effective_until + .parse() + .map(|effective_until| effective_until <= validation_time) + .unwrap_or(true); + + if blocked { + return Self { + entries_by_uri, + issuer_ca_sha256_hex, + ca_validation_context_digest, + policy_fingerprint, + crl_sha256_by_uri, + blocked, + }; + } + + for entry in &projection.entries { + let Some(earliest_safe_reuse_time_unix) = entry.earliest_safe_reuse_time_unix else { + continue; + }; + if time::OffsetDateTime::from_unix_timestamp(earliest_safe_reuse_time_unix).is_err() { + continue; + } + let outputs = entry + .outputs + .iter() + .map(|output| VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: output.item_effective_until.clone(), + source_object_uri: entry.source_object_uri.clone(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: entry.source_object_hash, + source_ee_cert_hash: output.source_ee_cert_hash, + payload: output.payload.clone(), + rule_hash: output.rule_hash, + }) + .collect::>(); + entries_by_uri.insert( + entry.source_object_uri.clone(), + CachedRoaValidationResult { + source_object_hash: entry.source_object_hash, + ee_serial: entry.ee_serial.clone(), + crl_uri: entry.crl_uri.clone(), + earliest_safe_reuse_time_unix, + outputs_effective_until_unix: entry.outputs_effective_until_unix, + outputs, + }, + ); + } + + Self { + entries_by_uri, + issuer_ca_sha256_hex, + ca_validation_context_digest, + policy_fingerprint, + crl_sha256_by_uri, + blocked, + } + } + + fn matches_current_context( + &self, + issuer_ca_der: &[u8], + ca_validation_context_digest: Option<[u8; 32]>, + policy_fingerprint: Option<[u8; 32]>, + ) -> bool { + if self.blocked { + return false; + } + + let Some(expected_issuer_hash) = self.issuer_ca_sha256_hex.as_ref() else { + return false; + }; + if expected_issuer_hash != &sha256_hex(issuer_ca_der) { + return false; + } + + if let Some(expected_ca_validation_context) = self.ca_validation_context_digest { + if Some(expected_ca_validation_context) != ca_validation_context_digest { + return false; + } + } else { + return false; + } + + if let Some(expected_policy) = self.policy_fingerprint { + if Some(expected_policy) != policy_fingerprint { + return false; + } + } else { + return false; + } + + true + } + + fn lookup( + &self, + file: &PackFile, + crl_cache: &mut std::collections::HashMap, + issuer_ca_der: &[u8], + validation_time: time::OffsetDateTime, + ) -> RoaCacheLookupResult { + self.lookup_with_metrics(file, crl_cache, issuer_ca_der, validation_time, None) + .0 + } + + fn lookup_with_metrics( + &self, + file: &PackFile, + crl_cache: &mut std::collections::HashMap, + issuer_ca_der: &[u8], + validation_time: time::OffsetDateTime, + crl_gate_set: Option<&mut RoaCacheCrlGateSet>, + ) -> (RoaCacheLookupResult, RoaCacheLookupMetrics) { + let mut metrics = RoaCacheLookupMetrics::default(); + let entry_gate_started = Instant::now(); + macro_rules! return_entry_gate { + ($result:expr) => {{ + metrics.entry_gate_nanos = metrics + .entry_gate_nanos + .saturating_add(elapsed_nanos_u64(entry_gate_started)); + return ($result, metrics); + }}; + } + macro_rules! return_crl_gate { + ($started:expr, $result:expr) => {{ + metrics.crl_gate_nanos = metrics + .crl_gate_nanos + .saturating_add(elapsed_nanos_u64($started)); + return ($result, metrics); + }}; + } + macro_rules! return_materialize { + ($started:expr, $result:expr) => {{ + metrics.materialize_nanos = metrics + .materialize_nanos + .saturating_add(elapsed_nanos_u64($started)); + return ($result, metrics); + }}; + } + + if self.blocked { + return_entry_gate!(RoaCacheLookupResult::ExpiredBlocked); + } + + let Some(cached) = self.entries_by_uri.get(file.rsync_uri.as_str()) else { + return_entry_gate!(RoaCacheLookupResult::Miss); + }; + if cached.source_object_hash != file.sha256 { + return_entry_gate!(RoaCacheLookupResult::HashBlocked); + } + if cached.outputs.is_empty() { + return_entry_gate!(RoaCacheLookupResult::Miss); + } + if validation_time.unix_timestamp() < cached.earliest_safe_reuse_time_unix + || cached.outputs_effective_until_unix <= validation_time.unix_timestamp() + { + return_entry_gate!(RoaCacheLookupResult::ExpiredBlocked); + } + + let Some(crl_uri) = cached.crl_uri.as_deref() else { + return_entry_gate!(RoaCacheLookupResult::MetadataBlocked); + }; + let Some(ee_serial) = cached.ee_serial.as_ref() else { + return_entry_gate!(RoaCacheLookupResult::MetadataBlocked); + }; + metrics.entry_gate_nanos = metrics + .entry_gate_nanos + .saturating_add(elapsed_nanos_u64(entry_gate_started)); + + let crl_gate_started = Instant::now(); + let crl_gate = if let Some(crl_gate_set) = crl_gate_set { + crl_gate_set.evaluate( + crl_uri, + &self.crl_sha256_by_uri, + crl_cache, + issuer_ca_der, + validation_time, + &mut metrics, + ) + } else { + evaluate_roa_cache_crl_gate( + crl_uri, + &self.crl_sha256_by_uri, + crl_cache, + issuer_ca_der, + validation_time, + &mut metrics, + ) + }; + let crl_rechecked = match crl_gate { + RoaCacheCrlGate::Unchanged => false, + RoaCacheCrlGate::ChangedValid(verified_crl) => { + if verified_crl.revoked_serials.contains(ee_serial) { + return_crl_gate!(crl_gate_started, RoaCacheLookupResult::RevokedBlocked); + } + true + } + RoaCacheCrlGate::Expired => { + return_crl_gate!(crl_gate_started, RoaCacheLookupResult::ExpiredBlocked); + } + RoaCacheCrlGate::Invalid | RoaCacheCrlGate::Missing => { + return_crl_gate!(crl_gate_started, RoaCacheLookupResult::MetadataBlocked); + } + }; + metrics.crl_gate_nanos = metrics + .crl_gate_nanos + .saturating_add(elapsed_nanos_u64(crl_gate_started)); + + let materialize_started = Instant::now(); + let mut vrps = Vec::with_capacity(cached.outputs.len()); + for output in &cached.outputs { + let VcirLocalOutputPayload::Vrp { + asn, + afi, + prefix_len, + addr, + max_length, + } = &output.payload + else { + return_materialize!(materialize_started, RoaCacheLookupResult::MetadataBlocked); + }; + vrps.push(Vrp { + asn: *asn, + prefix: IpPrefix { + afi: *afi, + prefix_len: *prefix_len, + addr: *addr, + }, + max_length: *max_length, + }); + } + + let ok = RoaTaskOk { + vrps, + local_outputs: cached.outputs.clone(), + reused_from_cache: true, + cache_object_meta: Some(RoaCacheObjectMeta { + source_object_uri: file.rsync_uri.clone(), + source_object_hash: file.sha256, + ee_serial: ee_serial.clone(), + crl_uri: crl_uri.to_string(), + earliest_safe_reuse_time: PackTime::from_utc_offset_datetime( + time::OffsetDateTime::from_unix_timestamp(cached.earliest_safe_reuse_time_unix) + .expect("cached ROA safe reuse time must be valid"), + ), + }), + }; + metrics.materialize_nanos = metrics + .materialize_nanos + .saturating_add(elapsed_nanos_u64(materialize_started)); + if crl_rechecked { + (RoaCacheLookupResult::CrlRecheckHit(ok), metrics) + } else { + (RoaCacheLookupResult::Hit(ok), metrics) + } + } +} + +fn active_roa_cache_view<'a>( + roa_cache: RoaValidationCacheInput<'a>, + issuer_ca_der: &[u8], + stats: &mut RoaValidationCacheStats, + roa_total: usize, +) -> Option<&'a RoaValidationCacheView> { + let view = roa_cache.view?; + let gate_started = Instant::now(); + if view.matches_current_context( + issuer_ca_der, + roa_cache.ca_validation_context_digest, + roa_cache.policy_fingerprint, + ) { + stats.context_gate_nanos = stats + .context_gate_nanos + .saturating_add(gate_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); + Some(view) + } else { + stats.context_gate_nanos = stats + .context_gate_nanos + .saturating_add(gate_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); + stats.blocked_roas += roa_total; + stats.context_blocked_roas += roa_total; + stats.fresh_roas += roa_total; + None + } +} + +#[derive(Debug)] +enum RoaCacheLookupResult { + Hit(RoaTaskOk), + CrlRecheckHit(RoaTaskOk), + Miss, + HashBlocked, + ExpiredBlocked, + RevokedBlocked, + MetadataBlocked, +} + +fn crl_valid_at_time( + crl: &crate::data_model::crl::RpkixCrl, + validation_time: time::OffsetDateTime, +) -> bool { + let this_update = crl.this_update.utc.to_offset(time::UtcOffset::UTC); + let next_update = crl.next_update.utc.to_offset(time::UtcOffset::UTC); + validation_time >= this_update && validation_time < next_update +} + +fn roa_cache_earliest_safe_reuse_time( + ee_not_before: time::OffsetDateTime, + crl_this_update: time::OffsetDateTime, + validation_time: time::OffsetDateTime, +) -> PackTime { + PackTime::from_utc_offset_datetime(ee_not_before.max(crl_this_update).max(validation_time)) +} + +fn record_roa_cache_block( + stats: &mut RoaValidationCacheStats, + lookup_result: &RoaCacheLookupResult, +) { + stats.blocked_roas += 1; + stats.fresh_roas += 1; + match lookup_result { + RoaCacheLookupResult::HashBlocked => stats.hash_blocked_roas += 1, + RoaCacheLookupResult::ExpiredBlocked => stats.expired_blocked_roas += 1, + RoaCacheLookupResult::RevokedBlocked => stats.revoked_blocked_roas += 1, + RoaCacheLookupResult::MetadataBlocked => stats.metadata_blocked_roas += 1, + RoaCacheLookupResult::Hit(_) + | RoaCacheLookupResult::CrlRecheckHit(_) + | RoaCacheLookupResult::Miss => {} + } +} + +#[derive(Clone, Copy)] +pub(crate) struct RoaTask<'a> { + pub(crate) index: usize, + pub(crate) file: &'a PackFile, +} + +#[derive(Debug)] +pub(crate) struct RoaTaskOk { + pub(crate) vrps: Vec, + pub(crate) local_outputs: Vec, + pub(crate) reused_from_cache: bool, + pub(crate) cache_object_meta: Option, +} + +#[derive(Debug)] +pub(crate) struct RoaTaskResult { + pub(crate) publication_point_id: u64, + pub(crate) index: usize, + pub(crate) worker_index: usize, + pub(crate) queue_wait_ms: u64, + pub(crate) worker_ms: u64, + pub(crate) outcome: Result, +} diff --git a/crates/panda-rpki-validator/src/validation/objects/object_validation.rs b/crates/panda-rpki-validator/src/validation/objects/object_validation.rs new file mode 100644 index 0000000..b2166cb --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/objects/object_validation.rs @@ -0,0 +1,398 @@ +fn process_roa_with_issuer( + file: &PackFile, + _manifest_rsync_uri: &str, + issuer_ca_der: &[u8], + issuer_ca: &ResourceCertificate, + issuer_spki: &SubjectPublicKeyInfo<'_>, + issuer_ca_rsync_uri: Option<&str>, + crl_cache: &mut std::collections::HashMap, + issuer_resources_index: &IssuerResourcesIndex, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + collect_vcir_local_outputs: bool, + strict_cms_der: bool, + strict_name: bool, + resource_validation_mode: ResourceValidationMode, + ta_constraints: Option<&crate::ta_constraints::TaConstraints>, +) -> Result<(Vec, Vec, Option), ObjectValidateError> { + let _decode = timing + .as_ref() + .map(|t| t.span_phase("objects_roa_decode_and_validate_total")); + let roa = RoaObject::decode_der_with_strict_options( + file.bytes().map_err(ObjectValidateError::BytesLoad)?, + strict_cms_der, + strict_name, + )?; + drop(_decode); + + let _ee_profile = timing + .as_ref() + .map(|t| t.span_phase("objects_roa_validate_embedded_ee_total")); + roa.validate_embedded_ee_cert()?; + drop(_ee_profile); + + let _verify = timing + .as_ref() + .map(|t| t.span_phase("objects_roa_verify_signature_total")); + roa.signed_object.verify()?; + drop(_verify); + + let ee = &roa.signed_object.signed_data.certificates[0]; + let ee_crldp_uris = ee + .resource_cert + .tbs + .extensions + .crl_distribution_points_uris + .as_ref(); + let issuer_crl_rsync_uri = choose_crl_uri_for_certificate(ee_crldp_uris, crl_cache)?; + let verified_crl = ensure_issuer_crl_verified(issuer_crl_rsync_uri, crl_cache, issuer_ca_der)?; + + let _cert_path = timing + .as_ref() + .map(|t| t.span_phase("objects_roa_validate_ee_cert_path_total")); + validate_signed_object_ee_cert_path_fast( + ee, + issuer_ca, + issuer_spki, + &verified_crl.crl, + &verified_crl.revoked_serials, + issuer_ca_rsync_uri, + Some(issuer_crl_rsync_uri), + validation_time, + )?; + drop(_cert_path); + + let _subset = timing + .as_ref() + .map(|t| t.span_phase("objects_roa_validate_ee_resources_subset_total")); + let ee_vrs = validate_ee_resources_for_mode( + &ee.resource_cert, + issuer_effective_ip, + issuer_effective_as, + issuer_resources_index, + resource_validation_mode, + )?; + drop(_subset); + + if let Some(ta_constraints) = ta_constraints { + ta_constraints.validate_ee_certificate(&ee.resource_cert)?; + } + + let vrps = roa_to_vrps_with_vrs(&roa, ee_vrs.ip.as_ref())?; + let cache_object_meta = RoaCacheObjectMeta { + source_object_uri: file.rsync_uri.clone(), + source_object_hash: file.sha256, + ee_serial: BigUnsigned::from_biguint(&ee.resource_cert.tbs.serial_number).bytes_be, + crl_uri: issuer_crl_rsync_uri.to_string(), + earliest_safe_reuse_time: roa_cache_earliest_safe_reuse_time( + ee.resource_cert.tbs.validity_not_before, + verified_crl.crl.this_update.utc, + validation_time, + ), + }; + if !collect_vcir_local_outputs { + return Ok((vrps, Vec::new(), Some(cache_object_meta))); + } + let source_object_hash = sha256_hex_from_32(&file.sha256); + let source_ee_cert_hash = crate::audit::sha256_hex(ee.raw_der.as_slice()); + let item_effective_until = + PackTime::from_utc_offset_datetime(ee.resource_cert.tbs.validity_not_after); + let local_outputs = vrps + .iter() + .map(|vrp| { + let prefix = vrp_prefix_to_string(vrp); + let rule_hash = crate::audit::sha256_hex( + format!( + "roa-rule:{}:{}:{}:{}", + source_object_hash, vrp.asn, prefix, vrp.max_length + ) + .as_bytes(), + ); + VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: item_effective_until.clone(), + source_object_uri: file.rsync_uri.clone(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: file.sha256, + source_ee_cert_hash: sha256_hex_to_32(&source_ee_cert_hash), + payload: VcirLocalOutputPayload::Vrp { + asn: vrp.asn, + afi: vrp.prefix.afi, + prefix_len: vrp.prefix.prefix_len, + addr: vrp.prefix.addr, + max_length: vrp.max_length, + }, + rule_hash: sha256_hex_to_32(&rule_hash), + } + }) + .collect(); + + Ok((vrps, local_outputs, Some(cache_object_meta))) +} + +fn process_roa_with_issuer_parallel_cached( + file: &PackFile, + _manifest_rsync_uri: &str, + issuer_ca_der: &[u8], + issuer_ca: &ResourceCertificate, + issuer_spki: &SubjectPublicKeyInfo<'_>, + issuer_ca_rsync_uri: Option<&str>, + crl_cache: &Mutex>, + issuer_resources_index: &IssuerResourcesIndex, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + collect_vcir_local_outputs: bool, + strict_cms_der: bool, + strict_name: bool, + resource_validation_mode: ResourceValidationMode, + ta_constraints: Option<&crate::ta_constraints::TaConstraints>, +) -> Result<(Vec, Vec, Option), ObjectValidateError> { + let _decode = timing + .as_ref() + .map(|t| t.span_phase("objects_roa_decode_and_validate_total")); + let roa = RoaObject::decode_der_with_strict_options( + file.bytes().map_err(ObjectValidateError::BytesLoad)?, + strict_cms_der, + strict_name, + )?; + drop(_decode); + + let _ee_profile = timing + .as_ref() + .map(|t| t.span_phase("objects_roa_validate_embedded_ee_total")); + roa.validate_embedded_ee_cert()?; + drop(_ee_profile); + + let _verify = timing + .as_ref() + .map(|t| t.span_phase("objects_roa_verify_signature_total")); + roa.signed_object.verify()?; + drop(_verify); + + let ee = &roa.signed_object.signed_data.certificates[0]; + let ee_crldp_uris = ee + .resource_cert + .tbs + .extensions + .crl_distribution_points_uris + .as_ref(); + let (issuer_crl_rsync_uri, verified_crl) = { + let mut crl_cache = crl_cache.lock().expect("parallel ROA CRL cache lock"); + let issuer_crl_rsync_uri = + choose_crl_uri_for_certificate(ee_crldp_uris, &crl_cache)?.to_string(); + let verified_crl = + ensure_issuer_crl_verified(&issuer_crl_rsync_uri, &mut crl_cache, issuer_ca_der)?; + (issuer_crl_rsync_uri, verified_crl) + }; + + let _cert_path = timing + .as_ref() + .map(|t| t.span_phase("objects_roa_validate_ee_cert_path_total")); + validate_signed_object_ee_cert_path_fast( + ee, + issuer_ca, + issuer_spki, + &verified_crl.crl, + &verified_crl.revoked_serials, + issuer_ca_rsync_uri, + Some(issuer_crl_rsync_uri.as_str()), + validation_time, + )?; + drop(_cert_path); + + let _subset = timing + .as_ref() + .map(|t| t.span_phase("objects_roa_validate_ee_resources_subset_total")); + let ee_vrs = validate_ee_resources_for_mode( + &ee.resource_cert, + issuer_effective_ip, + issuer_effective_as, + issuer_resources_index, + resource_validation_mode, + )?; + drop(_subset); + + if let Some(ta_constraints) = ta_constraints { + ta_constraints.validate_ee_certificate(&ee.resource_cert)?; + } + + let vrps = roa_to_vrps_with_vrs(&roa, ee_vrs.ip.as_ref())?; + let cache_object_meta = RoaCacheObjectMeta { + source_object_uri: file.rsync_uri.clone(), + source_object_hash: file.sha256, + ee_serial: BigUnsigned::from_biguint(&ee.resource_cert.tbs.serial_number).bytes_be, + crl_uri: issuer_crl_rsync_uri.clone(), + earliest_safe_reuse_time: roa_cache_earliest_safe_reuse_time( + ee.resource_cert.tbs.validity_not_before, + verified_crl.crl.this_update.utc, + validation_time, + ), + }; + if !collect_vcir_local_outputs { + return Ok((vrps, Vec::new(), Some(cache_object_meta))); + } + let source_object_hash = sha256_hex_from_32(&file.sha256); + let source_ee_cert_hash = crate::audit::sha256_hex(ee.raw_der.as_slice()); + let item_effective_until = + PackTime::from_utc_offset_datetime(ee.resource_cert.tbs.validity_not_after); + let local_outputs = vrps + .iter() + .map(|vrp| { + let prefix = vrp_prefix_to_string(vrp); + let rule_hash = crate::audit::sha256_hex( + format!( + "roa-rule:{}:{}:{}:{}", + source_object_hash, vrp.asn, prefix, vrp.max_length + ) + .as_bytes(), + ); + VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: item_effective_until.clone(), + source_object_uri: file.rsync_uri.clone(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: file.sha256, + source_ee_cert_hash: sha256_hex_to_32(&source_ee_cert_hash), + payload: VcirLocalOutputPayload::Vrp { + asn: vrp.asn, + afi: vrp.prefix.afi, + prefix_len: vrp.prefix.prefix_len, + addr: vrp.prefix.addr, + max_length: vrp.max_length, + }, + rule_hash: sha256_hex_to_32(&rule_hash), + } + }) + .collect(); + + Ok((vrps, local_outputs, Some(cache_object_meta))) +} + +fn process_aspa_with_issuer( + file: &PackFile, + _manifest_rsync_uri: &str, + issuer_ca_der: &[u8], + issuer_ca: &ResourceCertificate, + issuer_spki: &SubjectPublicKeyInfo<'_>, + issuer_ca_rsync_uri: Option<&str>, + crl_cache: &mut std::collections::HashMap, + issuer_resources_index: &IssuerResourcesIndex, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + collect_vcir_local_outputs: bool, + strict_cms_der: bool, + strict_name: bool, + resource_validation_mode: ResourceValidationMode, + ta_constraints: Option<&crate::ta_constraints::TaConstraints>, +) -> Result<(AspaAttestation, Option), ObjectValidateError> { + let _decode = timing + .as_ref() + .map(|t| t.span_phase("objects_aspa_decode_and_validate_total")); + let aspa = AspaObject::decode_der_with_strict_options( + file.bytes().map_err(ObjectValidateError::BytesLoad)?, + strict_cms_der, + strict_name, + )?; + drop(_decode); + + let _ee_profile = timing + .as_ref() + .map(|t| t.span_phase("objects_aspa_validate_embedded_ee_total")); + aspa.validate_embedded_ee_cert()?; + drop(_ee_profile); + + let _verify = timing + .as_ref() + .map(|t| t.span_phase("objects_aspa_verify_signature_total")); + aspa.signed_object.verify()?; + drop(_verify); + + let ee = &aspa.signed_object.signed_data.certificates[0]; + let ee_crldp_uris = ee + .resource_cert + .tbs + .extensions + .crl_distribution_points_uris + .as_ref(); + let issuer_crl_rsync_uri = choose_crl_uri_for_certificate(ee_crldp_uris, crl_cache)?; + let verified_crl = ensure_issuer_crl_verified(issuer_crl_rsync_uri, crl_cache, issuer_ca_der)?; + + let _cert_path = timing + .as_ref() + .map(|t| t.span_phase("objects_aspa_validate_ee_cert_path_total")); + validate_signed_object_ee_cert_path_fast( + ee, + issuer_ca, + issuer_spki, + &verified_crl.crl, + &verified_crl.revoked_serials, + issuer_ca_rsync_uri, + Some(issuer_crl_rsync_uri), + validation_time, + )?; + drop(_cert_path); + + let _subset = timing + .as_ref() + .map(|t| t.span_phase("objects_aspa_validate_ee_resources_subset_total")); + let ee_vrs = validate_ee_resources_for_mode( + &ee.resource_cert, + issuer_effective_ip, + issuer_effective_as, + issuer_resources_index, + resource_validation_mode, + )?; + drop(_subset); + + if let Some(ta_constraints) = ta_constraints { + ta_constraints.validate_ee_certificate(&ee.resource_cert)?; + } + + validate_aspa_customer_in_vrs(&aspa, ee_vrs.asn.as_ref())?; + + let attestation = AspaAttestation { + customer_as_id: aspa.aspa.customer_as_id, + provider_as_ids: aspa.aspa.provider_as_ids.clone(), + }; + if !collect_vcir_local_outputs { + return Ok((attestation, None)); + } + let source_object_hash = sha256_hex_from_32(&file.sha256); + let source_ee_cert_hash = crate::audit::sha256_hex(ee.raw_der.as_slice()); + let item_effective_until = + PackTime::from_utc_offset_datetime(ee.resource_cert.tbs.validity_not_after); + let providers = attestation + .provider_as_ids + .iter() + .map(u32::to_string) + .collect::>() + .join(","); + let rule_hash = crate::audit::sha256_hex( + format!( + "aspa-rule:{}:{}:{}", + source_object_hash, attestation.customer_as_id, providers + ) + .as_bytes(), + ); + let local_output = VcirLocalOutput { + output_type: VcirOutputType::Aspa, + item_effective_until, + source_object_uri: file.rsync_uri.clone(), + source_object_type: VcirSourceObjectType::Aspa, + source_object_hash: file.sha256, + source_ee_cert_hash: sha256_hex_to_32(&source_ee_cert_hash), + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: attestation.customer_as_id, + provider_as_ids: attestation.provider_as_ids.clone(), + }, + rule_hash: sha256_hex_to_32(&rule_hash), + }; + + Ok((attestation, Some(local_output))) +} diff --git a/crates/panda-rpki-validator/src/validation/objects/parallel_processing.rs b/crates/panda-rpki-validator/src/validation/objects/parallel_processing.rs new file mode 100644 index 0000000..fee9f65 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/objects/parallel_processing.rs @@ -0,0 +1,406 @@ +pub fn process_publication_point_for_issuer_parallel_roa( + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + config: &ParallelPhase2Config, +) -> ObjectsOutput { + process_publication_point_for_issuer_parallel_roa_with_options( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + config, + true, + ) +} + +pub fn process_publication_point_for_issuer_parallel_roa_with_options( + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + config: &ParallelPhase2Config, + collect_vcir_local_outputs: bool, +) -> ObjectsOutput { + process_publication_point_for_issuer_parallel_roa_with_cache_options( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + config, + collect_vcir_local_outputs, + RoaValidationCacheInput::disabled(), + ) +} + +pub fn process_publication_point_for_issuer_parallel_roa_with_cache_options< + P: PublicationPointData, +>( + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + config: &ParallelPhase2Config, + collect_vcir_local_outputs: bool, + roa_cache: RoaValidationCacheInput<'_>, +) -> ObjectsOutput { + if config.object_workers <= 1 + || policy.signed_object_failure_policy == SignedObjectFailurePolicy::DropPublicationPoint + { + return process_publication_point_for_issuer_with_cache_options( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + roa_cache, + ); + } + + let pool = match ParallelRoaWorkerPool::new(config) { + Ok(pool) => pool, + Err(_) => { + return process_publication_point_for_issuer_with_cache_options( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + roa_cache, + ); + } + }; + + process_publication_point_for_issuer_parallel_roa_with_pool_cache_options( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + &pool, + collect_vcir_local_outputs, + roa_cache, + ) +} + +pub fn process_publication_point_for_issuer_parallel_roa_with_pool( + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + pool: &ParallelRoaWorkerPool, +) -> ObjectsOutput { + process_publication_point_for_issuer_parallel_roa_with_pool_options( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + pool, + true, + ) +} + +pub fn process_publication_point_for_issuer_parallel_roa_with_pool_options< + P: PublicationPointData, +>( + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + pool: &ParallelRoaWorkerPool, + collect_vcir_local_outputs: bool, +) -> ObjectsOutput { + process_publication_point_for_issuer_parallel_roa_with_pool_cache_options( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + pool, + collect_vcir_local_outputs, + RoaValidationCacheInput::disabled(), + ) +} + +pub fn process_publication_point_for_issuer_parallel_roa_with_pool_cache_options< + P: PublicationPointData, +>( + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + pool: &ParallelRoaWorkerPool, + collect_vcir_local_outputs: bool, + roa_cache: RoaValidationCacheInput<'_>, +) -> ObjectsOutput { + if policy.signed_object_failure_policy == SignedObjectFailurePolicy::DropPublicationPoint { + return process_publication_point_for_issuer_with_options( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + ); + } + + process_publication_point_for_issuer_parallel_roa_inner( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + pool, + collect_vcir_local_outputs, + roa_cache, + ) + .unwrap_or_else(|_| { + process_publication_point_for_issuer_with_cache_options( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + roa_cache, + ) + }) +} + +#[derive(Clone)] +pub(crate) struct RoaTaskShared { + locked_files: Arc<[PackFile]>, + manifest_rsync_uri: Arc, + issuer_ca_der: Arc<[u8]>, + issuer_ca: Arc, + issuer_spki_der: Arc<[u8]>, + issuer_ca_rsync_uri: Option>, + crl_cache: Arc>>, + issuer_resources_index: Arc, + issuer_effective_ip: Option>, + issuer_effective_as: Option>, + resource_validation_mode: ResourceValidationMode, + /// Immutable constraints snapshot selected by the owning TAL. Every + /// ROA task for a publication point shares this Arc, so workers do not + /// need a mutable/global policy lookup or a copy of the interval rules. + ta_constraints: Option>, +} + +#[derive(Clone)] +pub(crate) struct OwnedRoaTask { + pub(crate) publication_point_id: u64, + index: usize, + shared: Arc, + validation_time: time::OffsetDateTime, + collect_vcir_local_outputs: bool, + strict_cms_der: bool, + strict_name: bool, + resource_validation_mode: ResourceValidationMode, + pub(crate) submitted_at: Option, +} + +#[derive(Clone)] +struct RoaTaskExecutor; + +impl ObjectTaskExecutor for RoaTaskExecutor { + fn execute(&self, worker_index: usize, task: OwnedRoaTask) -> RoaTaskResult { + validate_owned_roa_task(worker_index, task) + } +} + +pub struct ParallelRoaWorkerPool { + pool: Mutex>, +} + +impl ParallelRoaWorkerPool { + pub fn new(config: &ParallelPhase2Config) -> Result { + if config.object_workers <= 1 { + return Err("parallel ROA worker pool requires object_workers > 1".to_string()); + } + + Ok(Self { + pool: Mutex::new(ObjectWorkerPool::new( + config.object_workers, + config.worker_queue_capacity, + RoaTaskExecutor, + )?), + }) + } + + pub(crate) fn try_submit_round_robin( + &self, + task: OwnedRoaTask, + ) -> Result> { + self.pool + .lock() + .expect("parallel ROA worker pool lock") + .try_submit_round_robin(task) + } + + pub(crate) fn recv_result_timeout( + &self, + timeout: Duration, + ) -> Result, String> { + self.pool + .lock() + .expect("parallel ROA worker pool lock") + .recv_result_timeout(timeout) + } +} + +fn validate_owned_roa_task(worker_index: usize, task: OwnedRoaTask) -> RoaTaskResult { + let worker_started = Instant::now(); + let queue_wait_ms = task + .submitted_at + .map(|submitted_at| worker_started.saturating_duration_since(submitted_at)) + .map(|duration| duration.as_millis() as u64) + .unwrap_or(0); + let shared = task.shared.as_ref(); + let file = task + .shared + .locked_files + .get(task.index) + .expect("ROA task index must reference locked file"); + let issuer_spki = match SubjectPublicKeyInfo::from_der(shared.issuer_spki_der.as_ref()) { + Ok((rem, spki)) if rem.is_empty() => spki, + Ok((rem, _)) => { + return RoaTaskResult { + publication_point_id: task.publication_point_id, + index: task.index, + worker_index, + queue_wait_ms, + worker_ms: worker_started.elapsed().as_millis() as u64, + outcome: Err(ObjectValidateError::CertPath( + CertPathError::IssuerSpkiTrailingBytes(rem.len()), + )), + }; + } + Err(e) => { + return RoaTaskResult { + publication_point_id: task.publication_point_id, + index: task.index, + worker_index, + queue_wait_ms, + worker_ms: worker_started.elapsed().as_millis() as u64, + outcome: Err(ObjectValidateError::CertPath( + CertPathError::IssuerSpkiParse(e.to_string()), + )), + }; + } + }; + let outcome = process_roa_with_issuer_parallel_cached( + file, + shared.manifest_rsync_uri.as_ref(), + shared.issuer_ca_der.as_ref(), + shared.issuer_ca.as_ref(), + &issuer_spki, + shared.issuer_ca_rsync_uri.as_deref(), + shared.crl_cache.as_ref(), + shared.issuer_resources_index.as_ref(), + shared.issuer_effective_ip.as_deref(), + shared.issuer_effective_as.as_deref(), + task.validation_time, + None, + task.collect_vcir_local_outputs, + task.strict_cms_der, + task.strict_name, + task.resource_validation_mode, + shared.ta_constraints.as_deref(), + ) + .map(|(vrps, local_outputs, cache_object_meta)| RoaTaskOk { + vrps, + local_outputs, + reused_from_cache: false, + cache_object_meta, + }); + + RoaTaskResult { + publication_point_id: task.publication_point_id, + index: task.index, + worker_index, + queue_wait_ms, + worker_ms: worker_started.elapsed().as_millis() as u64, + outcome, + } +} + +pub(crate) enum ParallelObjectsPrepare { + Complete(ObjectsOutput), + Staged(ParallelObjectsStage), +} + +pub(crate) struct ParallelObjectsStage { + pub(crate) publication_point_id: u64, + shared: Arc, + validation_time: time::OffsetDateTime, + collect_vcir_local_outputs: bool, + strict_cms_der: bool, + strict_name: bool, + resource_validation_mode: ResourceValidationMode, + roa_task_indices: Vec, + cached_roa_results: Vec, + roa_cache_stats: RoaValidationCacheStats, + warnings: Vec, + stats: ObjectsStats, + audit: Vec, +} diff --git a/crates/panda-rpki-validator/src/validation/objects/parallel_stage.rs b/crates/panda-rpki-validator/src/validation/objects/parallel_stage.rs new file mode 100644 index 0000000..14b917c --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/objects/parallel_stage.rs @@ -0,0 +1,805 @@ +impl ParallelObjectsStage { + #[cfg(test)] + pub(crate) fn build_roa_tasks(&self) -> Vec { + let mut tasks = Vec::with_capacity(self.roa_task_count()); + self.extend_roa_tasks(|task| tasks.push(task)); + tasks + } + + pub(crate) fn append_roa_tasks_to( + &self, + pending: &mut std::collections::VecDeque, + ) { + self.extend_roa_tasks(|task| pending.push_back(task)); + } + + fn extend_roa_tasks(&self, mut push: F) + where + F: FnMut(OwnedRoaTask), + { + let shared = self.shared.clone(); + self.roa_task_indices.iter().for_each(|index| { + push(OwnedRoaTask { + publication_point_id: self.publication_point_id, + index: *index, + shared: shared.clone(), + validation_time: self.validation_time, + collect_vcir_local_outputs: self.collect_vcir_local_outputs, + strict_cms_der: self.strict_cms_der, + strict_name: self.strict_name, + resource_validation_mode: self.resource_validation_mode, + submitted_at: None, + }); + }); + } + + pub(crate) fn roa_task_count(&self) -> usize { + self.roa_task_indices.len() + } + + pub(crate) fn aspa_task_count(&self) -> usize { + self.stats.aspa_total + } + + pub(crate) fn locked_file_count(&self) -> usize { + self.shared.locked_files.len() + } +} + +pub(crate) fn prepare_publication_point_for_parallel_roa_with_cache( + publication_point_id: u64, + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + collect_vcir_local_outputs: bool, + roa_cache: RoaValidationCacheInput<'_>, +) -> ParallelObjectsPrepare { + prepare_publication_point_for_parallel_roa_with_cache_and_ta_constraints( + publication_point_id, + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + collect_vcir_local_outputs, + roa_cache, + None, + ) +} + +/// Prepare a publication point for parallel ROA validation with the +/// immutable constraints snapshot belonging to its TAL. The snapshot is +/// moved into the stage-owned shared payload and therefore remains available +/// to detached ROA workers after the scoped phase-2 stage worker returns. +pub(crate) fn prepare_publication_point_for_parallel_roa_with_cache_and_ta_constraints< + P: PublicationPointData, +>( + publication_point_id: u64, + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + collect_vcir_local_outputs: bool, + roa_cache: RoaValidationCacheInput<'_>, + ta_constraints: Option>, +) -> ParallelObjectsPrepare { + let manifest_rsync_uri = publication_point.manifest_rsync_uri(); + let manifest_bytes = publication_point.manifest_bytes(); + let locked_files = publication_point.files(); + let mut warnings: Vec = Vec::new(); + let mut stats = ObjectsStats::default(); + stats.roa_total = locked_files + .iter() + .filter(|f| f.rsync_uri.ends_with(".roa")) + .count(); + stats.aspa_total = locked_files + .iter() + .filter(|f| f.rsync_uri.ends_with(".asa")) + .count(); + let mut roa_cache_stats = RoaValidationCacheStats::for_input(roa_cache, stats.roa_total); + let mut audit: Vec = Vec::new(); + + let _manifest = match ManifestObject::decode_der_with_strict_options( + manifest_bytes, + policy.strict.cms_der, + policy.strict.name, + ) { + Ok(manifest) => manifest, + Err(e) => { + stats.publication_point_dropped = true; + warnings.push( + Warning::new(format!( + "dropping publication point: manifest decode failed: {e}" + )) + .with_rfc_refs(&[ + RfcRef("RFC 9286 §4"), + RfcRef("RFC 9286 §6.2"), + RfcRef("RFC 9286 §6.6"), + ]) + .with_context(manifest_rsync_uri), + ); + for f in locked_files { + if f.rsync_uri.ends_with(".roa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Skipped, + detail: Some("skipped: manifest decode failed".to_string()), + }); + } else if f.rsync_uri.ends_with(".asa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Skipped, + detail: Some("skipped: manifest decode failed".to_string()), + }); + } + } + return ParallelObjectsPrepare::Complete(ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta: Vec::new(), + }); + } + }; + + if let Some(warning) = + ber_compatible_cms_warning(manifest_bytes, manifest_rsync_uri, "manifest") + { + warnings.push(warning); + } + + let issuer_ca = match decode_resource_certificate_with_policy(issuer_ca_der, policy) { + Ok(v) => v, + Err(e) => { + stats.publication_point_dropped = true; + warnings.push( + Warning::new(format!( + "dropping publication point: issuer CA decode failed: {e}" + )) + .with_rfc_refs(&[RfcRef("RFC 6487 §7.2"), RfcRef("RFC 5280 §6.1")]) + .with_context(manifest_rsync_uri), + ); + for f in locked_files { + if f.rsync_uri.ends_with(".roa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Skipped, + detail: Some("skipped: issuer CA decode failed".to_string()), + }); + } else if f.rsync_uri.ends_with(".asa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Skipped, + detail: Some("skipped: issuer CA decode failed".to_string()), + }); + } + } + return ParallelObjectsPrepare::Complete(ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta: Vec::new(), + }); + } + }; + + match SubjectPublicKeyInfo::from_der(&issuer_ca.tbs.subject_public_key_info) { + Ok((rem, _)) if rem.is_empty() => {} + Ok((rem, _)) => { + stats.publication_point_dropped = true; + warnings.push( + Warning::new(format!( + "dropping publication point: trailing bytes after issuer SPKI DER: {} bytes", + rem.len() + )) + .with_rfc_refs(&[RfcRef("RFC 5280 §4.1.2.7")]) + .with_context(manifest_rsync_uri), + ); + return ParallelObjectsPrepare::Complete(ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta: Vec::new(), + }); + } + Err(e) => { + stats.publication_point_dropped = true; + warnings.push( + Warning::new(format!( + "dropping publication point: issuer SPKI parse failed: {e}" + )) + .with_rfc_refs(&[RfcRef("RFC 5280 §4.1.2.7")]) + .with_context(manifest_rsync_uri), + ); + return ParallelObjectsPrepare::Complete(ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta: Vec::new(), + }); + } + } + + let mut crl_cache: std::collections::HashMap = locked_files + .iter() + .filter(|f| f.rsync_uri.ends_with(".crl")) + .map(|f| { + let bytes = f + .bytes_cloned() + .expect("snapshot CRL bytes must be loadable"); + ( + f.rsync_uri.clone(), + CachedIssuerCrl::Pending { + bytes, + sha256_hex: None, + }, + ) + }) + .collect(); + + if crl_cache.is_empty() && (stats.roa_total > 0 || stats.aspa_total > 0) { + stats.publication_point_dropped = true; + warnings.push( + Warning::new("dropping publication point: no CRL files in validated publication point") + .with_rfc_refs(&[RfcRef("RFC 6487 §4.8.6"), RfcRef("RFC 9286 §7")]) + .with_context(manifest_rsync_uri), + ); + for f in locked_files { + if f.rsync_uri.ends_with(".roa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Skipped, + detail: Some( + "skipped due to missing CRL files in validated publication point" + .to_string(), + ), + }); + } else if f.rsync_uri.ends_with(".asa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Skipped, + detail: Some( + "skipped due to missing CRL files in validated publication point" + .to_string(), + ), + }); + } + } + return ParallelObjectsPrepare::Complete(ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta: Vec::new(), + }); + } + + let active_cache_view = active_roa_cache_view( + roa_cache, + issuer_ca_der, + &mut roa_cache_stats, + stats.roa_total, + ); + let mut crl_gate_set = if active_cache_view.is_some() { + Some(RoaCacheCrlGateSet::default()) + } else { + None + }; + let mut roa_task_indices = Vec::new(); + let mut cached_roa_results = Vec::new(); + for (index, file) in locked_files.iter().enumerate() { + if !file.rsync_uri.ends_with(".roa") { + continue; + } + if let Some(cache_view) = active_cache_view { + let lookup_started = Instant::now(); + let (lookup_result, lookup_metrics) = cache_view.lookup_with_metrics( + file, + &mut crl_cache, + issuer_ca_der, + validation_time, + crl_gate_set.as_mut(), + ); + roa_cache_stats.record_lookup(elapsed_nanos_u64(lookup_started), lookup_metrics); + match lookup_result { + RoaCacheLookupResult::Hit(ok) => { + roa_cache_stats.hit_roas += 1; + cached_roa_results.push(RoaTaskResult { + publication_point_id, + index, + worker_index: usize::MAX, + queue_wait_ms: 0, + worker_ms: 0, + outcome: Ok(ok), + }); + } + RoaCacheLookupResult::CrlRecheckHit(ok) => { + roa_cache_stats.hit_roas += 1; + roa_cache_stats.crl_recheck_hit_roas += 1; + cached_roa_results.push(RoaTaskResult { + publication_point_id, + index, + worker_index: usize::MAX, + queue_wait_ms: 0, + worker_ms: 0, + outcome: Ok(ok), + }); + } + RoaCacheLookupResult::Miss => { + roa_cache_stats.miss_roas += 1; + roa_cache_stats.fresh_roas += 1; + roa_task_indices.push(index); + } + blocked @ (RoaCacheLookupResult::HashBlocked + | RoaCacheLookupResult::ExpiredBlocked + | RoaCacheLookupResult::RevokedBlocked + | RoaCacheLookupResult::MetadataBlocked) => { + record_roa_cache_block(&mut roa_cache_stats, &blocked); + roa_task_indices.push(index); + } + } + } else { + roa_task_indices.push(index); + } + } + + ParallelObjectsPrepare::Staged(ParallelObjectsStage { + publication_point_id, + shared: Arc::new(RoaTaskShared { + locked_files: Arc::<[PackFile]>::from(locked_files.to_vec()), + manifest_rsync_uri: Arc::::from(manifest_rsync_uri), + issuer_ca_der: Arc::<[u8]>::from(issuer_ca_der.to_vec()), + issuer_spki_der: Arc::<[u8]>::from(issuer_ca.tbs.subject_public_key_info.clone()), + issuer_ca: Arc::new(issuer_ca), + issuer_ca_rsync_uri: issuer_ca_rsync_uri.map(Arc::::from), + crl_cache: Arc::new(Mutex::new(crl_cache)), + issuer_resources_index: Arc::new(build_issuer_resources_index( + issuer_effective_ip, + issuer_effective_as, + )), + issuer_effective_ip: issuer_effective_ip.cloned().map(Arc::new), + issuer_effective_as: issuer_effective_as.cloned().map(Arc::new), + resource_validation_mode: policy.resource_validation_mode, + ta_constraints, + }), + validation_time, + collect_vcir_local_outputs, + strict_cms_der: policy.strict.cms_der, + strict_name: policy.strict.name, + resource_validation_mode: policy.resource_validation_mode, + roa_task_indices, + cached_roa_results, + roa_cache_stats, + warnings, + stats, + audit, + }) +} + +pub(crate) fn reduce_parallel_roa_stage( + stage: ParallelObjectsStage, + mut roa_results: Vec, + timing: Option<&TimingHandle>, +) -> Result { + roa_results.extend(stage.cached_roa_results); + roa_results.sort_by_key(|result| result.index); + let mut roa_results = roa_results.into_iter().peekable(); + let shared = stage.shared.clone(); + let mut aspa_crl_cache = shared + .crl_cache + .lock() + .expect("parallel ROA CRL cache lock") + .clone(); + let issuer_spki = SubjectPublicKeyInfo::from_der(shared.issuer_spki_der.as_ref()) + .map_err(|e| e.to_string())? + .1; + let collect_vcir_local_outputs = stage.collect_vcir_local_outputs; + let validation_time = stage.validation_time; + let strict_cms_der = stage.strict_cms_der; + let strict_name = stage.strict_name; + let roa_cache_stats = stage.roa_cache_stats; + let mut stats = stage.stats; + let mut warnings = stage.warnings; + let mut audit = stage.audit; + let mut vrps: Vec = Vec::new(); + let mut aspas: Vec = Vec::new(); + let mut local_outputs_cache: Vec = Vec::new(); + let mut roa_cache_object_meta: Vec = Vec::new(); + + for (idx, file) in shared.locked_files.iter().enumerate() { + if file.rsync_uri.ends_with(".roa") { + let result = match roa_results.peek() { + Some(result) if result.index == idx => roa_results + .next() + .expect("peeked ROA task result must be present"), + Some(result) => { + return Err(format!( + "unexpected ROA task result index {} while reducing {} at index {}", + result.index, file.rsync_uri, idx + )); + } + None => { + return Err(format!("missing ROA task result for {}", file.rsync_uri)); + } + }; + match result.outcome { + Ok(mut ok) => { + stats.roa_ok += 1; + vrps.append(&mut ok.vrps); + if collect_vcir_local_outputs || ok.reused_from_cache { + local_outputs_cache.extend(ok.local_outputs); + } + if let Some(meta) = ok.cache_object_meta.take() { + roa_cache_object_meta.push(meta); + } + audit.push(ObjectAuditEntry { + rsync_uri: file.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&file.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Ok, + detail: None, + }); + if let Some(warning) = ber_compatible_cms_warning_for_file(file, "ROA") { + warnings.push(warning); + } + } + Err(e) => { + audit.push(ObjectAuditEntry { + rsync_uri: file.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&file.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Error, + detail: Some(e.to_string()), + }); + let mut refs = vec![RfcRef("RFC 6488 §3"), RfcRef("RFC 9582 §4-§5")]; + refs.extend_from_slice(extra_rfc_refs_for_crl_selection(&e)); + warnings.push( + Warning::new(format!("dropping invalid ROA: {}: {e}", file.rsync_uri)) + .with_rfc_refs(&refs) + .with_context(&file.rsync_uri), + ) + } + } + } else if file.rsync_uri.ends_with(".asa") { + let _t = timing.as_ref().map(|t| t.span_phase("objects_aspa_total")); + match process_aspa_with_issuer( + file, + shared.manifest_rsync_uri.as_ref(), + shared.issuer_ca_der.as_ref(), + shared.issuer_ca.as_ref(), + &issuer_spki, + shared.issuer_ca_rsync_uri.as_deref(), + &mut aspa_crl_cache, + shared.issuer_resources_index.as_ref(), + shared.issuer_effective_ip.as_deref(), + shared.issuer_effective_as.as_deref(), + validation_time, + timing, + collect_vcir_local_outputs, + strict_cms_der, + strict_name, + shared.resource_validation_mode, + shared.ta_constraints.as_deref(), + ) { + Ok((att, local_output)) => { + stats.aspa_ok += 1; + aspas.push(att); + if let Some(local_output) = local_output { + local_outputs_cache.push(local_output); + } + audit.push(ObjectAuditEntry { + rsync_uri: file.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&file.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Ok, + detail: None, + }); + if let Some(warning) = ber_compatible_cms_warning_for_file(file, "ASPA") { + warnings.push(warning); + } + } + Err(e) => { + audit.push(ObjectAuditEntry { + rsync_uri: file.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&file.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Error, + detail: Some(e.to_string()), + }); + let mut refs = vec![RfcRef("RFC 6488 §3")]; + refs.extend_from_slice(extra_rfc_refs_for_crl_selection(&e)); + warnings.push( + Warning::new(format!("dropping invalid ASPA: {}: {e}", file.rsync_uri)) + .with_rfc_refs(&refs) + .with_context(&file.rsync_uri), + ) + } + } + } + } + if let Some(result) = roa_results.next() { + return Err(format!( + "unexpected trailing ROA task result at index {}", + result.index + )); + } + + roa_cache_stats.record_to_timing(timing); + + Ok(ObjectsOutput { + vrps, + aspas, + router_keys: Vec::new(), + local_outputs_cache, + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta, + }) +} + +fn process_publication_point_for_issuer_parallel_roa_inner( + publication_point: &P, + _policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + pool: &ParallelRoaWorkerPool, + collect_vcir_local_outputs: bool, + roa_cache: RoaValidationCacheInput<'_>, +) -> Result { + let stage = match prepare_publication_point_for_parallel_roa_with_cache( + 0, + publication_point, + _policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + collect_vcir_local_outputs, + roa_cache, + ) { + ParallelObjectsPrepare::Complete(out) => return Ok(out), + ParallelObjectsPrepare::Staged(stage) => stage, + }; + + let roa_task_count = stage.roa_task_count(); + let mut pending = std::collections::VecDeque::with_capacity(roa_task_count); + stage.append_roa_tasks_to(&mut pending); + let mut worker_pool = pool + .pool + .lock() + .map_err(|_| "parallel ROA worker pool lock poisoned".to_string())?; + + while let Some(task) = pending.pop_front() { + match worker_pool.try_submit_round_robin(task) { + Ok(_) => {} + Err(ObjectWorkerSubmitError::QueueFull { task, .. }) => { + pending.push_front(task); + std::thread::yield_now(); + } + Err(ObjectWorkerSubmitError::Disconnected { .. }) => { + return Err("parallel ROA worker queue disconnected".to_string()); + } + } + } + + let mut roa_results = Vec::with_capacity(roa_task_count); + while roa_results.len() < roa_task_count { + let Some(result) = worker_pool.recv_result_timeout(Duration::from_secs(30))? else { + return Err("parallel ROA worker timed out".to_string()); + }; + roa_results.push(result); + } + drop(worker_pool); + + reduce_parallel_roa_stage(stage, roa_results, timing) +} +/// Compatibility wrapper that processes a publication point snapshot. +pub fn process_publication_point_snapshot_for_issuer( + pack: &PublicationPointSnapshot, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, +) -> ObjectsOutput { + process_publication_point_for_issuer( + pack, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + ) +} + +pub fn process_publication_point_snapshot_for_issuer_parallel_roa( + pack: &PublicationPointSnapshot, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + config: &ParallelPhase2Config, +) -> ObjectsOutput { + process_publication_point_for_issuer_parallel_roa( + pack, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + config, + ) +} + +#[derive(Debug, thiserror::Error)] +pub(crate) enum ObjectValidateError { + #[error("object bytes load failed: {0}")] + BytesLoad(String), + + #[error("ROA decode failed: {0}")] + RoaDecode(#[from] RoaDecodeError), + + #[error("ROA embedded EE resource validation failed: {0}")] + RoaEeResources(#[from] RoaValidateError), + + #[error("ASPA decode failed: {0}")] + AspaDecode(#[from] AspaDecodeError), + + #[error("ASPA embedded EE resource validation failed: {0}")] + AspaEeResources(#[from] AspaValidateError), + + #[error("CMS signature verification failed: {0}")] + Signature(#[from] SignedObjectVerifyError), + + #[error("EE certificate path validation failed: {0}")] + CertPath(#[from] CertPathError), + + #[error( + "certificate CRLDistributionPoints URIs missing (cannot select issuer CRL) (RFC 6487 §4.8.6)" + )] + MissingCrlDpUris, + + #[error( + "no CRL available in publication point snapshot (cannot validate certificates) (RFC 9286 §7; RFC 6487 §4.8.6)" + )] + MissingCrlInPack, + + #[error( + "CRL referenced by CRLDistributionPoints not found in publication point snapshot: {0} (RFC 6487 §4.8.6; RFC 9286 §4.2.1)" + )] + CrlNotFound(String), + + #[error( + "issuer effective IP resources missing (cannot validate EE IP resources subset) (RFC 6487 §7.2; RFC 3779 §2.3)" + )] + MissingIssuerEffectiveIp, + + #[error( + "issuer effective AS resources missing (cannot validate EE AS resources subset) (RFC 6487 §7.2; RFC 3779 §3.3)" + )] + MissingIssuerEffectiveAs, + + #[error( + "EE certificate resources are not a subset of issuer effective resources (RFC 6487 §7.2; RFC 3779)" + )] + EeResourcesNotSubset, + + #[error("EE certificate violates locally configured TA constraints: {0}")] + TaConstraints(#[from] crate::ta_constraints::TaConstraintsViolation), +} + +pub(crate) fn validate_roa_task_serial( + task: RoaTask<'_>, + manifest_rsync_uri: &str, + issuer_ca_der: &[u8], + issuer_ca: &ResourceCertificate, + issuer_spki: &SubjectPublicKeyInfo<'_>, + issuer_ca_rsync_uri: Option<&str>, + crl_cache: &mut std::collections::HashMap, + issuer_resources_index: &IssuerResourcesIndex, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + collect_vcir_local_outputs: bool, + strict_cms_der: bool, + strict_name: bool, + resource_validation_mode: ResourceValidationMode, + ta_constraints: Option<&crate::ta_constraints::TaConstraints>, +) -> RoaTaskResult { + let outcome = process_roa_with_issuer( + task.file, + manifest_rsync_uri, + issuer_ca_der, + issuer_ca, + issuer_spki, + issuer_ca_rsync_uri, + crl_cache, + issuer_resources_index, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + strict_cms_der, + strict_name, + resource_validation_mode, + ta_constraints, + ) + .map(|(vrps, local_outputs, cache_object_meta)| RoaTaskOk { + vrps, + local_outputs, + reused_from_cache: false, + cache_object_meta, + }); + + RoaTaskResult { + publication_point_id: 0, + index: task.index, + worker_index: 0, + queue_wait_ms: 0, + worker_ms: 0, + outcome, + } +} diff --git a/crates/panda-rpki-validator/src/validation/objects/resource_validation.rs b/crates/panda-rpki-validator/src/validation/objects/resource_validation.rs new file mode 100644 index 0000000..9602835 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/objects/resource_validation.rs @@ -0,0 +1,825 @@ +fn vrp_prefix_to_string(vrp: &Vrp) -> String { + let prefix = &vrp.prefix; + match prefix.afi { + RoaAfi::Ipv4 => { + let addr = std::net::Ipv4Addr::new( + prefix.addr[0], + prefix.addr[1], + prefix.addr[2], + prefix.addr[3], + ); + format!("{addr}/{}", prefix.prefix_len) + } + RoaAfi::Ipv6 => { + let mut octets = [0u8; 16]; + octets.copy_from_slice(&prefix.addr[..16]); + let addr = std::net::Ipv6Addr::from(octets); + format!("{addr}/{}", prefix.prefix_len) + } + } +} + +fn choose_crl_uri_for_certificate<'a>( + crldp_uris: Option<&'a Vec>, + crl_cache: &std::collections::HashMap, +) -> Result<&'a str, ObjectValidateError> { + if crl_cache.is_empty() { + return Err(ObjectValidateError::MissingCrlInPack); + } + + let Some(crldp_uris) = crldp_uris else { + return Err(ObjectValidateError::MissingCrlDpUris); + }; + + for u in crldp_uris { + let s = u.as_str(); + if crl_cache.contains_key(s) { + return Ok(s); + } + } + Err(ObjectValidateError::CrlNotFound( + crldp_uris + .iter() + .map(|u| u.as_str()) + .collect::>() + .join(", "), + )) +} + +fn ensure_issuer_crl_verified<'a>( + crl_rsync_uri: &str, + crl_cache: &'a mut std::collections::HashMap, + issuer_ca_der: &[u8], +) -> Result, CertPathError> { + let entry = crl_cache + .get_mut(crl_rsync_uri) + .expect("CRL must exist in cache"); + match entry { + CachedIssuerCrl::Ok(v) => Ok(Arc::clone(v)), + CachedIssuerCrl::Pending { bytes, sha256_hex } => { + let der = std::mem::take(bytes); + let current_sha256_hex = sha256_hex + .take() + .unwrap_or_else(|| crate::audit::sha256_hex(&der)); + let crl = crate::data_model::crl::RpkixCrl::decode_der(&der) + .map_err(CertPathError::CrlDecode)?; + crl.verify_signature_with_issuer_certificate_der(issuer_ca_der) + .map_err(CertPathError::CrlVerify)?; + + let mut revoked_serials: std::collections::HashSet> = + std::collections::HashSet::with_capacity(crl.revoked_certs.len()); + for rc in &crl.revoked_certs { + revoked_serials.insert(rc.serial_number.bytes_be.clone()); + } + + *entry = CachedIssuerCrl::Ok(Arc::new(VerifiedIssuerCrl { + crl, + revoked_serials, + sha256_hex: current_sha256_hex, + })); + match entry { + CachedIssuerCrl::Ok(v) => Ok(Arc::clone(v)), + _ => unreachable!(), + } + } + } +} + +fn validate_ee_resources_subset( + ee: &ResourceCertificate, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + issuer_resources_index: &IssuerResourcesIndex, +) -> Result<(), ObjectValidateError> { + if let Some(child_ip) = ee.tbs.extensions.ip_resources.as_ref() { + let Some(parent_ip) = issuer_effective_ip else { + return Err(ObjectValidateError::MissingIssuerEffectiveIp); + }; + if !ip_resources_is_subset_indexed(child_ip, parent_ip, issuer_resources_index) { + return Err(ObjectValidateError::EeResourcesNotSubset); + } + } + + if let Some(child_as) = ee.tbs.extensions.as_resources.as_ref() { + let Some(parent_as) = issuer_effective_as else { + return Err(ObjectValidateError::MissingIssuerEffectiveAs); + }; + if !as_resources_is_subset_indexed(child_as, parent_as, issuer_resources_index) { + return Err(ObjectValidateError::EeResourcesNotSubset); + } + } + + Ok(()) +} + +#[derive(Debug)] +struct EeVerifiedResources { + ip: Option, + asn: Option, +} + +fn validate_ee_resources_for_mode( + ee: &ResourceCertificate, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + issuer_resources_index: &IssuerResourcesIndex, + mode: ResourceValidationMode, +) -> Result { + match mode { + ResourceValidationMode::Rfc6487 => { + validate_ee_resources_subset( + ee, + issuer_effective_ip, + issuer_effective_as, + issuer_resources_index, + )?; + Ok(EeVerifiedResources { + ip: ee.tbs.extensions.ip_resources.clone(), + asn: ee.tbs.extensions.as_resources.clone(), + }) + } + ResourceValidationMode::ValidationUpdate03 => { + let ip = match ee.tbs.extensions.ip_resources.as_ref() { + Some(child_ip) => Some(intersect_ee_ip_resources_vrs( + child_ip, + issuer_effective_ip, + issuer_resources_index, + )?), + None => None, + }; + let asn = match ee.tbs.extensions.as_resources.as_ref() { + Some(child_as) => Some(intersect_ee_as_resources_vrs( + child_as, + issuer_effective_as, + issuer_resources_index, + )?), + None => None, + }; + Ok(EeVerifiedResources { ip, asn }) + } + } +} + +fn intersect_ee_ip_resources_vrs( + child_ip: &crate::data_model::rc::IpResourceSet, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_resources_index: &IssuerResourcesIndex, +) -> Result { + if child_ip.has_any_inherit() { + return Err(ObjectValidateError::EeResourcesNotSubset); + } + let _ = issuer_effective_ip; + let mut families = Vec::new(); + for fam in &child_ip.families { + let parent_intervals = match fam.afi { + crate::data_model::rc::Afi::Ipv4 => issuer_resources_index.ip_v4.as_deref(), + crate::data_model::rc::Afi::Ipv6 => issuer_resources_index.ip_v6.as_deref(), + } + .unwrap_or(&[]); + let items = match &fam.choice { + IpAddressChoice::Inherit => return Err(ObjectValidateError::EeResourcesNotSubset), + IpAddressChoice::AddressesOrRanges(items) => items, + }; + let intersections = intersect_ip_items_with_parent_intervals(items, parent_intervals); + if !intersections.is_empty() { + families.push(crate::data_model::rc::IpAddressFamily { + afi: fam.afi, + choice: IpAddressChoice::AddressesOrRanges(ip_intervals_to_ranges( + fam.afi, + &intersections, + )), + }); + } + } + Ok(crate::data_model::rc::IpResourceSet { families }) +} + +fn intersect_ee_as_resources_vrs( + child_as: &crate::data_model::rc::AsResourceSet, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + issuer_resources_index: &IssuerResourcesIndex, +) -> Result { + let _ = issuer_effective_as; + if matches!(child_as.asnum, Some(AsIdentifierChoice::Inherit)) + || matches!(child_as.rdi, Some(AsIdentifierChoice::Inherit)) + { + return Err(ObjectValidateError::EeResourcesNotSubset); + } + let asnum = child_as.asnum.as_ref().map(|choice| { + let child_intervals = as_choice_to_merged_intervals(choice); + AsIdentifierChoice::AsIdsOrRanges(as_intervals_to_items(&intersect_as_intervals( + &child_intervals, + issuer_resources_index.asnum.as_deref().unwrap_or(&[]), + ))) + }); + let rdi = child_as.rdi.as_ref().map(|choice| { + let child_intervals = as_choice_to_merged_intervals(choice); + AsIdentifierChoice::AsIdsOrRanges(as_intervals_to_items(&intersect_as_intervals( + &child_intervals, + issuer_resources_index.rdi.as_deref().unwrap_or(&[]), + ))) + }); + Ok(crate::data_model::rc::AsResourceSet { asnum, rdi }) +} + +fn roa_to_vrps_with_vrs( + roa: &RoaObject, + ee_vrs_ip: Option<&crate::data_model::rc::IpResourceSet>, +) -> Result, ObjectValidateError> { + let vrps = roa_to_vrps(roa); + let Some(ee_vrs_ip) = ee_vrs_ip else { + return Err(ObjectValidateError::EeResourcesNotSubset); + }; + for vrp in &vrps { + let rc_prefix = roa_prefix_to_rc_prefix(&vrp.prefix); + if !ee_vrs_ip.contains_prefix(&rc_prefix) { + return Err(ObjectValidateError::EeResourcesNotSubset); + } + } + Ok(vrps) +} + +fn validate_aspa_customer_in_vrs( + aspa: &AspaObject, + ee_vrs_as: Option<&crate::data_model::rc::AsResourceSet>, +) -> Result<(), ObjectValidateError> { + let Some(ee_vrs_as) = ee_vrs_as else { + return Err(ObjectValidateError::EeResourcesNotSubset); + }; + if !as_resource_set_contains_asn(ee_vrs_as, aspa.aspa.customer_as_id) { + return Err(ObjectValidateError::EeResourcesNotSubset); + } + Ok(()) +} + +fn as_resource_set_contains_asn( + resources: &crate::data_model::rc::AsResourceSet, + asn: u32, +) -> bool { + let Some(choice) = resources.asnum.as_ref() else { + return false; + }; + match choice { + AsIdentifierChoice::Inherit => false, + AsIdentifierChoice::AsIdsOrRanges(items) => items.iter().any(|item| match item { + crate::data_model::rc::AsIdOrRange::Id(id) => *id == asn, + crate::data_model::rc::AsIdOrRange::Range { min, max } => *min <= asn && asn <= *max, + }), + } +} + +fn roa_prefix_to_rc_prefix(prefix: &IpPrefix) -> RcIpPrefix { + let afi = match prefix.afi { + RoaAfi::Ipv4 => crate::data_model::rc::Afi::Ipv4, + RoaAfi::Ipv6 => crate::data_model::rc::Afi::Ipv6, + }; + let mut addr = prefix.addr.to_vec(); + addr.truncate(afi.octets_len()); + RcIpPrefix { + afi, + prefix_len: prefix.prefix_len, + addr, + } +} + +fn as_resources_is_subset(child: &AsResourceSet, parent: &AsResourceSet) -> bool { + as_choice_subset(child.asnum.as_ref(), parent.asnum.as_ref()) + && as_choice_subset(child.rdi.as_ref(), parent.rdi.as_ref()) +} + +fn as_resources_is_subset_indexed( + child: &AsResourceSet, + parent: &AsResourceSet, + idx: &IssuerResourcesIndex, +) -> bool { + let _ = parent; + as_choice_subset_indexed(child.asnum.as_ref(), idx.asnum.as_deref()) + && as_choice_subset_indexed(child.rdi.as_ref(), idx.rdi.as_deref()) +} + +fn as_choice_subset_indexed( + child: Option<&AsIdentifierChoice>, + parent_intervals: Option<&[(u32, u32)]>, +) -> bool { + let Some(child) = child else { + return true; + }; + let Some(parent_intervals) = parent_intervals else { + return false; + }; + + if matches!(child, AsIdentifierChoice::Inherit) { + return false; + } + + let child_intervals = as_choice_to_merged_intervals(child); + for (cmin, cmax) in &child_intervals { + if !as_interval_is_covered(parent_intervals, *cmin, *cmax) { + return false; + } + } + true +} + +fn as_choice_subset( + child: Option<&AsIdentifierChoice>, + parent: Option<&AsIdentifierChoice>, +) -> bool { + let Some(child) = child else { + return true; + }; + let Some(parent) = parent else { + return false; + }; + + match (child, parent) { + (AsIdentifierChoice::Inherit, _) => return false, + (_, AsIdentifierChoice::Inherit) => return false, + _ => {} + } + + let child_intervals = as_choice_to_merged_intervals(child); + let parent_intervals = as_choice_to_merged_intervals(parent); + for (cmin, cmax) in &child_intervals { + if !as_interval_is_covered(&parent_intervals, *cmin, *cmax) { + return false; + } + } + true +} + +fn as_choice_to_merged_intervals(choice: &AsIdentifierChoice) -> Vec<(u32, u32)> { + let mut v = Vec::new(); + match choice { + AsIdentifierChoice::Inherit => {} + AsIdentifierChoice::AsIdsOrRanges(items) => { + for item in items { + match item { + crate::data_model::rc::AsIdOrRange::Id(id) => v.push((*id, *id)), + crate::data_model::rc::AsIdOrRange::Range { min, max } => v.push((*min, *max)), + } + } + } + } + v.sort_by_key(|(a, _)| *a); + merge_as_intervals(&v) +} + +fn merge_as_intervals(v: &[(u32, u32)]) -> Vec<(u32, u32)> { + let mut out: Vec<(u32, u32)> = Vec::new(); + for (min, max) in v { + let Some(last) = out.last_mut() else { + out.push((*min, *max)); + continue; + }; + if *min <= last.1.saturating_add(1) { + last.1 = last.1.max(*max); + continue; + } + out.push((*min, *max)); + } + out +} + +fn as_interval_is_covered(parent: &[(u32, u32)], min: u32, max: u32) -> bool { + for (pmin, pmax) in parent { + if *pmin <= min && max <= *pmax { + return true; + } + if *pmin > min { + break; + } + } + false +} + +fn ip_resources_is_subset( + child: &crate::data_model::rc::IpResourceSet, + parent: &crate::data_model::rc::IpResourceSet, +) -> bool { + let parent_by_afi = ip_resources_to_merged_intervals(parent); + let child_by_afi = match ip_resources_to_merged_intervals_strict(child) { + Ok(v) => v, + Err(()) => return false, + }; + + for (afi, child_intervals) in child_by_afi { + let Some(parent_intervals) = parent_by_afi.get(&afi) else { + return false; + }; + for (cmin, cmax) in &child_intervals { + if !interval_is_covered(parent_intervals, cmin, cmax) { + return false; + } + } + } + true +} + +fn ip_resources_is_subset_indexed( + child: &crate::data_model::rc::IpResourceSet, + parent: &crate::data_model::rc::IpResourceSet, + idx: &IssuerResourcesIndex, +) -> bool { + let _ = parent; + + for fam in &child.families { + let parent_intervals = match fam.afi { + crate::data_model::rc::Afi::Ipv4 => idx.ip_v4.as_deref(), + crate::data_model::rc::Afi::Ipv6 => idx.ip_v6.as_deref(), + }; + let Some(parent_intervals) = parent_intervals else { + return false; + }; + let items = match &fam.choice { + IpAddressChoice::Inherit => return false, + IpAddressChoice::AddressesOrRanges(items) => items, + }; + + let mut child_intervals: Vec<(Vec, Vec)> = Vec::new(); + for item in items { + match item { + IpAddressOrRange::Prefix(p) => child_intervals.push(prefix_to_range(p)), + IpAddressOrRange::Range(r) => child_intervals.push((r.min.clone(), r.max.clone())), + } + } + if child_intervals.is_empty() { + continue; + } + child_intervals.sort_by(|(a, _), (b, _)| a.cmp(b)); + merge_ip_intervals_in_place(&mut child_intervals); + if !intervals_are_covered(parent_intervals, &child_intervals) { + return false; + } + } + true +} + +fn ip_items_to_merged_intervals( + items: &[crate::data_model::rc::IpAddressOrRange], +) -> Vec<(Vec, Vec)> { + let mut intervals = Vec::new(); + for item in items { + match item { + IpAddressOrRange::Prefix(p) => intervals.push(prefix_to_range(p)), + IpAddressOrRange::Range(r) => intervals.push((r.min.clone(), r.max.clone())), + } + } + intervals.sort_by(|(a, _), (b, _)| a.cmp(b)); + merge_ip_intervals_in_place(&mut intervals); + intervals +} + +fn intersect_ip_items_with_parent_intervals( + items: &[crate::data_model::rc::IpAddressOrRange], + parent_intervals: &[(Vec, Vec)], +) -> Vec<(Vec, Vec)> { + let child_intervals = ip_items_to_merged_intervals(items); + let mut out = Vec::new(); + let mut parent_index = 0usize; + for (child_min, child_max) in &child_intervals { + while parent_index < parent_intervals.len() + && parent_intervals[parent_index].1.as_slice() < child_min.as_slice() + { + parent_index += 1; + } + let mut scan = parent_index; + while scan < parent_intervals.len() + && parent_intervals[scan].0.as_slice() <= child_max.as_slice() + { + let (parent_min, parent_max) = &parent_intervals[scan]; + let min = if bytes_leq(child_min, parent_min) { + parent_min.clone() + } else { + child_min.clone() + }; + let max = if bytes_leq(child_max, parent_max) { + child_max.clone() + } else { + parent_max.clone() + }; + if bytes_leq(&min, &max) { + out.push((min, max)); + } + scan += 1; + } + } + merge_ip_intervals_in_place(&mut out); + out +} + +fn ip_intervals_to_ranges( + afi: crate::data_model::rc::Afi, + intervals: &[(Vec, Vec)], +) -> Vec { + intervals + .iter() + .map(|(min, max)| { + IpAddressOrRange::Range(crate::data_model::rc::IpAddressRange { + min: normalize_ip_bytes(afi, min), + max: normalize_ip_bytes(afi, max), + }) + }) + .collect() +} + +fn normalize_ip_bytes(afi: crate::data_model::rc::Afi, bytes: &[u8]) -> Vec { + let target_len = afi.octets_len(); + if bytes.len() == target_len { + return bytes.to_vec(); + } + let mut out = vec![0u8; target_len]; + let copy_len = bytes.len().min(target_len); + out[..copy_len].copy_from_slice(&bytes[..copy_len]); + out +} + +fn intersect_as_intervals(child: &[(u32, u32)], parent: &[(u32, u32)]) -> Vec<(u32, u32)> { + let mut out = Vec::new(); + let mut parent_index = 0usize; + for (child_min, child_max) in child { + while parent_index < parent.len() && parent[parent_index].1 < *child_min { + parent_index += 1; + } + let mut scan = parent_index; + while scan < parent.len() && parent[scan].0 <= *child_max { + let min = (*child_min).max(parent[scan].0); + let max = (*child_max).min(parent[scan].1); + if min <= max { + out.push((min, max)); + } + scan += 1; + } + } + merge_as_intervals(&out) +} + +fn as_intervals_to_items(intervals: &[(u32, u32)]) -> Vec { + intervals + .iter() + .map(|(min, max)| { + if min == max { + crate::data_model::rc::AsIdOrRange::Id(*min) + } else { + crate::data_model::rc::AsIdOrRange::Range { + min: *min, + max: *max, + } + } + }) + .collect() +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +enum AfiKey { + V4, + V6, +} + +fn ip_resources_to_merged_intervals( + set: &crate::data_model::rc::IpResourceSet, +) -> std::collections::HashMap, Vec)>> { + let mut m: std::collections::HashMap, Vec)>> = + std::collections::HashMap::new(); + + for fam in &set.families { + let afi = match fam.afi { + crate::data_model::rc::Afi::Ipv4 => AfiKey::V4, + crate::data_model::rc::Afi::Ipv6 => AfiKey::V6, + }; + match &fam.choice { + IpAddressChoice::Inherit => { + // Effective resource sets should not contain inherit, but if they do we treat it + // as "unknown" by leaving it empty here (subset checks will fail). + } + IpAddressChoice::AddressesOrRanges(items) => { + let ent = m.entry(afi).or_default(); + for item in items { + match item { + IpAddressOrRange::Prefix(p) => ent.push(prefix_to_range(p)), + IpAddressOrRange::Range(r) => ent.push((r.min.clone(), r.max.clone())), + } + } + } + } + } + + for (_afi, v) in m.iter_mut() { + v.sort_by(|(a, _), (b, _)| a.cmp(b)); + merge_ip_intervals_in_place(v); + } + + m +} + +fn ip_resources_to_merged_intervals_strict( + set: &crate::data_model::rc::IpResourceSet, +) -> Result, Vec)>>, ()> { + let mut m: std::collections::HashMap, Vec)>> = + std::collections::HashMap::new(); + + for fam in &set.families { + let afi = match fam.afi { + crate::data_model::rc::Afi::Ipv4 => AfiKey::V4, + crate::data_model::rc::Afi::Ipv6 => AfiKey::V6, + }; + match &fam.choice { + IpAddressChoice::Inherit => return Err(()), + IpAddressChoice::AddressesOrRanges(items) => { + let ent = m.entry(afi).or_default(); + for item in items { + match item { + IpAddressOrRange::Prefix(p) => ent.push(prefix_to_range(p)), + IpAddressOrRange::Range(r) => ent.push((r.min.clone(), r.max.clone())), + } + } + } + } + } + + for (_afi, v) in m.iter_mut() { + v.sort_by(|(a, _), (b, _)| a.cmp(b)); + merge_ip_intervals_in_place(v); + } + + Ok(m) +} + +fn merge_ip_intervals_in_place(v: &mut Vec<(Vec, Vec)>) { + if v.is_empty() { + return; + } + let mut out: Vec<(Vec, Vec)> = Vec::with_capacity(v.len()); + for (min, max) in v.drain(..) { + let Some(last) = out.last_mut() else { + out.push((min, max)); + continue; + }; + if bytes_leq(&min, &last.1) || bytes_is_next(&min, &last.1) { + if bytes_leq(&last.1, &max) { + last.1 = max; + } + continue; + } + out.push((min, max)); + } + *v = out; +} + +fn interval_is_covered(parent: &[(Vec, Vec)], min: &[u8], max: &[u8]) -> bool { + for (pmin, pmax) in parent { + if bytes_leq(pmin, min) && bytes_leq(max, pmax) { + return true; + } + if pmin.as_slice() > min { + break; + } + } + false +} + +fn intervals_are_covered(parent: &[(Vec, Vec)], child: &[(Vec, Vec)]) -> bool { + let mut i = 0usize; + for (cmin, cmax) in child { + while i < parent.len() && parent[i].1.as_slice() < cmin.as_slice() { + i += 1; + } + if i >= parent.len() { + return false; + } + let (pmin, pmax) = &parent[i]; + if !bytes_leq(pmin, cmin) || !bytes_leq(cmax, pmax) { + return false; + } + } + true +} + +fn prefix_to_range(prefix: &RcIpPrefix) -> (Vec, Vec) { + let mut min = prefix.addr.clone(); + let mut max = prefix.addr.clone(); + let bitlen = prefix.afi.ub(); + let plen = prefix.prefix_len.min(bitlen); + for bit in plen..bitlen { + let byte = (bit / 8) as usize; + let offset = 7 - (bit % 8); + let mask = 1u8 << offset; + min[byte] &= !mask; + max[byte] |= mask; + } + (min, max) +} + +fn bytes_leq(a: &[u8], b: &[u8]) -> bool { + a <= b +} + +fn increment_bytes(v: &[u8]) -> Vec { + let mut out = v.to_vec(); + for i in (0..out.len()).rev() { + if out[i] != 0xFF { + out[i] += 1; + for j in i + 1..out.len() { + out[j] = 0; + } + return out; + } + } + vec![0u8; out.len()] +} + +fn bytes_is_next(a: &[u8], b: &[u8]) -> bool { + if a.len() != b.len() { + return false; + } + let mut carry: u16 = 1; + for i in (0..b.len()).rev() { + let sum = (b[i] as u16) + carry; + let expected = (sum & 0xFF) as u8; + carry = sum >> 8; + if a[i] != expected { + return false; + } + } + true +} + +fn build_issuer_resources_index( + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, +) -> IssuerResourcesIndex { + let mut idx = IssuerResourcesIndex::default(); + + if let Some(ip) = issuer_effective_ip { + let mut v4: Vec<(Vec, Vec)> = Vec::new(); + let mut v6: Vec<(Vec, Vec)> = Vec::new(); + for fam in &ip.families { + let ent = match fam.afi { + crate::data_model::rc::Afi::Ipv4 => &mut v4, + crate::data_model::rc::Afi::Ipv6 => &mut v6, + }; + match &fam.choice { + IpAddressChoice::Inherit => { + // Effective resources should not contain inherit; leave empty so subset fails. + } + IpAddressChoice::AddressesOrRanges(items) => { + for item in items { + match item { + IpAddressOrRange::Prefix(p) => ent.push(prefix_to_range(p)), + IpAddressOrRange::Range(r) => ent.push((r.min.clone(), r.max.clone())), + } + } + } + } + } + if !v4.is_empty() { + v4.sort_by(|(a, _), (b, _)| a.cmp(b)); + merge_ip_intervals_in_place(&mut v4); + idx.ip_v4 = Some(v4); + } + if !v6.is_empty() { + v6.sort_by(|(a, _), (b, _)| a.cmp(b)); + merge_ip_intervals_in_place(&mut v6); + idx.ip_v6 = Some(v6); + } + } + + if let Some(asr) = issuer_effective_as { + if let Some(choice) = asr.asnum.as_ref() { + if !matches!(choice, AsIdentifierChoice::Inherit) { + idx.asnum = Some(as_choice_to_merged_intervals(choice)); + } + } + if let Some(choice) = asr.rdi.as_ref() { + if !matches!(choice, AsIdentifierChoice::Inherit) { + idx.rdi = Some(as_choice_to_merged_intervals(choice)); + } + } + } + + idx +} + +fn roa_to_vrps(roa: &RoaObject) -> Vec { + let asn = roa.roa.as_id; + let mut out = Vec::new(); + for fam in &roa.roa.ip_addr_blocks { + for entry in &fam.addresses { + let max_length = entry.max_length.unwrap_or(entry.prefix.prefix_len); + out.push(Vrp { + asn, + prefix: entry.prefix.clone(), + max_length, + }); + } + } + out +} + +#[allow(dead_code)] +fn roa_afi_to_string(afi: RoaAfi) -> &'static str { + match afi { + RoaAfi::Ipv4 => "ipv4", + RoaAfi::Ipv6 => "ipv6", + } +} diff --git a/crates/panda-rpki-validator/src/validation/objects/serial_processing.rs b/crates/panda-rpki-validator/src/validation/objects/serial_processing.rs new file mode 100644 index 0000000..8b2377a --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/objects/serial_processing.rs @@ -0,0 +1,683 @@ +/// Process objects from a publication point snapshot using a known issuer CA certificate +/// and its effective resources (resolved via the resource-path, RFC 6487 §7.2). +pub fn process_publication_point_for_issuer( + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, +) -> ObjectsOutput { + process_publication_point_for_issuer_with_options( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + true, + ) +} + +pub fn process_publication_point_for_issuer_with_options( + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + collect_vcir_local_outputs: bool, +) -> ObjectsOutput { + process_publication_point_for_issuer_with_cache_options( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + RoaValidationCacheInput::disabled(), + ) +} + +pub fn process_publication_point_for_issuer_with_cache_options( + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + collect_vcir_local_outputs: bool, + roa_cache: RoaValidationCacheInput<'_>, +) -> ObjectsOutput { + process_publication_point_for_issuer_with_cache_options_and_ta_constraints( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + roa_cache, + None, + ) +} + +/// Serial signed-object processing with an optional, locally configured +/// constraint set for the TA that owns this publication-point tree. +pub fn process_publication_point_for_issuer_with_cache_options_and_ta_constraints< + P: PublicationPointData, +>( + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + collect_vcir_local_outputs: bool, + roa_cache: RoaValidationCacheInput<'_>, + ta_constraints: Option<&crate::ta_constraints::TaConstraints>, +) -> ObjectsOutput { + let manifest_rsync_uri = publication_point.manifest_rsync_uri(); + let manifest_bytes = publication_point.manifest_bytes(); + let locked_files = publication_point.files(); + let mut warnings: Vec = Vec::new(); + let mut stats = ObjectsStats::default(); + stats.roa_total = locked_files + .iter() + .filter(|f| f.rsync_uri.ends_with(".roa")) + .count(); + stats.aspa_total = locked_files + .iter() + .filter(|f| f.rsync_uri.ends_with(".asa")) + .count(); + let mut roa_cache_stats = RoaValidationCacheStats::for_input(roa_cache, stats.roa_total); + let mut audit: Vec = Vec::new(); + + // Enforce that `manifest_bytes` is actually a manifest object. + let _manifest = match ManifestObject::decode_der_with_strict_options( + manifest_bytes, + policy.strict.cms_der, + policy.strict.name, + ) { + Ok(manifest) => manifest, + Err(e) => { + stats.publication_point_dropped = true; + warnings.push( + Warning::new(format!( + "dropping publication point: manifest decode failed: {e}" + )) + .with_rfc_refs(&[ + RfcRef("RFC 9286 §4"), + RfcRef("RFC 9286 §6.2"), + RfcRef("RFC 9286 §6.6"), + ]) + .with_context(manifest_rsync_uri), + ); + for f in locked_files { + if f.rsync_uri.ends_with(".roa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Skipped, + detail: Some("skipped: manifest decode failed".to_string()), + }); + } else if f.rsync_uri.ends_with(".asa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Skipped, + detail: Some("skipped: manifest decode failed".to_string()), + }); + } + } + return ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta: Vec::new(), + }; + } + }; + + if let Some(warning) = + ber_compatible_cms_warning(manifest_bytes, manifest_rsync_uri, "manifest") + { + warnings.push(warning); + } + + // Decode issuer CA once; if it fails we cannot validate ROA/ASPA EE certificates. + let issuer_ca = match decode_resource_certificate_with_policy(issuer_ca_der, policy) { + Ok(v) => v, + Err(e) => { + stats.publication_point_dropped = true; + warnings.push( + Warning::new(format!( + "dropping publication point: issuer CA decode failed: {e}" + )) + .with_rfc_refs(&[RfcRef("RFC 6487 §7.2"), RfcRef("RFC 5280 §6.1")]) + .with_context(manifest_rsync_uri), + ); + for f in locked_files { + if f.rsync_uri.ends_with(".roa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Skipped, + detail: Some("skipped: issuer CA decode failed".to_string()), + }); + } else if f.rsync_uri.ends_with(".asa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Skipped, + detail: Some("skipped: issuer CA decode failed".to_string()), + }); + } + } + return ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta: Vec::new(), + }; + } + }; + + // Parse issuer SubjectPublicKeyInfo once and reuse for all EE certificate signature checks. + let issuer_spki = match SubjectPublicKeyInfo::from_der(&issuer_ca.tbs.subject_public_key_info) { + Ok((rem, spki)) if rem.is_empty() => spki, + Ok((rem, _)) => { + stats.publication_point_dropped = true; + warnings.push( + Warning::new(format!( + "dropping publication point: trailing bytes after issuer SPKI DER: {} bytes", + rem.len() + )) + .with_rfc_refs(&[RfcRef("RFC 5280 §4.1.2.7")]) + .with_context(manifest_rsync_uri), + ); + return ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta: Vec::new(), + }; + } + Err(e) => { + stats.publication_point_dropped = true; + warnings.push( + Warning::new(format!( + "dropping publication point: issuer SPKI parse failed: {e}" + )) + .with_rfc_refs(&[RfcRef("RFC 5280 §4.1.2.7")]) + .with_context(manifest_rsync_uri), + ); + return ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta: Vec::new(), + }; + } + }; + + let mut crl_cache: std::collections::HashMap = locked_files + .iter() + .filter(|f| f.rsync_uri.ends_with(".crl")) + .map(|f| { + let bytes = f + .bytes_cloned() + .expect("snapshot CRL bytes must be loadable"); + ( + f.rsync_uri.clone(), + CachedIssuerCrl::Pending { + bytes, + sha256_hex: None, + }, + ) + }) + .collect(); + + let issuer_resources_index = + build_issuer_resources_index(issuer_effective_ip, issuer_effective_as); + + // If the snapshot has signed objects but no CRLs at all, we cannot validate any embedded EE + // certificate paths deterministically (EE CRLDP must reference an rsync URI in the snapshot). + if crl_cache.is_empty() && (stats.roa_total > 0 || stats.aspa_total > 0) { + stats.publication_point_dropped = true; + warnings.push( + Warning::new("dropping publication point: no CRL files in validated publication point") + .with_rfc_refs(&[RfcRef("RFC 6487 §4.8.6"), RfcRef("RFC 9286 §7")]) + .with_context(manifest_rsync_uri), + ); + for f in locked_files { + if f.rsync_uri.ends_with(".roa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Skipped, + detail: Some( + "skipped due to missing CRL files in validated publication point" + .to_string(), + ), + }); + } else if f.rsync_uri.ends_with(".asa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Skipped, + detail: Some( + "skipped due to missing CRL files in validated publication point" + .to_string(), + ), + }); + } + } + return ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta: Vec::new(), + }; + } + + let mut vrps: Vec = Vec::new(); + let mut aspas: Vec = Vec::new(); + let mut local_outputs_cache: Vec = Vec::new(); + let mut roa_cache_object_meta: Vec = Vec::new(); + let active_cache_view = active_roa_cache_view( + roa_cache, + issuer_ca_der, + &mut roa_cache_stats, + stats.roa_total, + ); + let mut crl_gate_set = if active_cache_view.is_some() { + Some(RoaCacheCrlGateSet::default()) + } else { + None + }; + + for (idx, file) in locked_files.iter().enumerate() { + if file.rsync_uri.ends_with(".roa") { + let result = if let Some(cache_view) = active_cache_view { + let lookup_started = Instant::now(); + let (lookup_result, lookup_metrics) = cache_view.lookup_with_metrics( + file, + &mut crl_cache, + issuer_ca_der, + validation_time, + crl_gate_set.as_mut(), + ); + roa_cache_stats.record_lookup(elapsed_nanos_u64(lookup_started), lookup_metrics); + match lookup_result { + RoaCacheLookupResult::Hit(ok) => { + roa_cache_stats.hit_roas += 1; + RoaTaskResult { + publication_point_id: 0, + index: idx, + worker_index: 0, + queue_wait_ms: 0, + worker_ms: 0, + outcome: Ok(ok), + } + } + RoaCacheLookupResult::CrlRecheckHit(ok) => { + roa_cache_stats.hit_roas += 1; + roa_cache_stats.crl_recheck_hit_roas += 1; + RoaTaskResult { + publication_point_id: 0, + index: idx, + worker_index: 0, + queue_wait_ms: 0, + worker_ms: 0, + outcome: Ok(ok), + } + } + RoaCacheLookupResult::Miss => { + roa_cache_stats.miss_roas += 1; + roa_cache_stats.fresh_roas += 1; + let task = RoaTask { index: idx, file }; + let _t = timing.as_ref().map(|t| t.span_phase("objects_roa_total")); + validate_roa_task_serial( + task, + manifest_rsync_uri, + issuer_ca_der, + &issuer_ca, + &issuer_spki, + issuer_ca_rsync_uri, + &mut crl_cache, + &issuer_resources_index, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + policy.strict.cms_der, + policy.strict.name, + policy.resource_validation_mode, + ta_constraints, + ) + } + blocked @ (RoaCacheLookupResult::HashBlocked + | RoaCacheLookupResult::ExpiredBlocked + | RoaCacheLookupResult::RevokedBlocked + | RoaCacheLookupResult::MetadataBlocked) => { + record_roa_cache_block(&mut roa_cache_stats, &blocked); + let task = RoaTask { index: idx, file }; + let _t = timing.as_ref().map(|t| t.span_phase("objects_roa_total")); + validate_roa_task_serial( + task, + manifest_rsync_uri, + issuer_ca_der, + &issuer_ca, + &issuer_spki, + issuer_ca_rsync_uri, + &mut crl_cache, + &issuer_resources_index, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + policy.strict.cms_der, + policy.strict.name, + policy.resource_validation_mode, + ta_constraints, + ) + } + } + } else { + let task = RoaTask { index: idx, file }; + let _t = timing.as_ref().map(|t| t.span_phase("objects_roa_total")); + validate_roa_task_serial( + task, + manifest_rsync_uri, + issuer_ca_der, + &issuer_ca, + &issuer_spki, + issuer_ca_rsync_uri, + &mut crl_cache, + &issuer_resources_index, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + policy.strict.cms_der, + policy.strict.name, + policy.resource_validation_mode, + ta_constraints, + ) + }; + match result.outcome { + Ok(mut ok) => { + stats.roa_ok += 1; + vrps.append(&mut ok.vrps); + if collect_vcir_local_outputs || ok.reused_from_cache { + local_outputs_cache.extend(ok.local_outputs); + } + if let Some(meta) = ok.cache_object_meta.take() { + roa_cache_object_meta.push(meta); + } + audit.push(ObjectAuditEntry { + rsync_uri: file.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&file.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Ok, + detail: None, + }); + if let Some(warning) = ber_compatible_cms_warning_for_file(file, "ROA") { + warnings.push(warning); + } + } + Err(e) => match policy.signed_object_failure_policy { + SignedObjectFailurePolicy::DropObject => { + audit.push(ObjectAuditEntry { + rsync_uri: file.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&file.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Error, + detail: Some(e.to_string()), + }); + let mut refs = vec![RfcRef("RFC 6488 §3"), RfcRef("RFC 9582 §4-§5")]; + refs.extend_from_slice(extra_rfc_refs_for_crl_selection(&e)); + warnings.push( + Warning::new(format!("dropping invalid ROA: {}: {e}", file.rsync_uri)) + .with_rfc_refs(&refs) + .with_context(&file.rsync_uri), + ) + } + SignedObjectFailurePolicy::DropPublicationPoint => { + stats.publication_point_dropped = true; + audit.push(ObjectAuditEntry { + rsync_uri: file.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&file.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Error, + detail: Some(e.to_string()), + }); + for f in locked_files.iter().skip(idx + 1) { + if f.rsync_uri.ends_with(".roa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Skipped, + detail: Some( + "skipped due to policy=signed_object_failure_policy=drop_publication_point" + .to_string(), + ), + }); + } else if f.rsync_uri.ends_with(".asa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Skipped, + detail: Some( + "skipped due to policy=signed_object_failure_policy=drop_publication_point" + .to_string(), + ), + }); + } + } + let mut refs = vec![RfcRef("RFC 6488 §3"), RfcRef("RFC 9582 §4-§5")]; + refs.extend_from_slice(extra_rfc_refs_for_crl_selection(&e)); + warnings.push( + Warning::new(format!( + "dropping publication point due to invalid ROA: {}: {e}", + file.rsync_uri + )) + .with_rfc_refs(&refs) + .with_context(manifest_rsync_uri), + ); + return ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta: Vec::new(), + }; + } + }, + } + } else if file.rsync_uri.ends_with(".asa") { + let _t = timing.as_ref().map(|t| t.span_phase("objects_aspa_total")); + match process_aspa_with_issuer( + file, + manifest_rsync_uri, + issuer_ca_der, + &issuer_ca, + &issuer_spki, + issuer_ca_rsync_uri, + &mut crl_cache, + &issuer_resources_index, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + policy.strict.cms_der, + policy.strict.name, + policy.resource_validation_mode, + ta_constraints, + ) { + Ok((att, local_output)) => { + stats.aspa_ok += 1; + aspas.push(att); + if let Some(local_output) = local_output { + local_outputs_cache.push(local_output); + } + audit.push(ObjectAuditEntry { + rsync_uri: file.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&file.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Ok, + detail: None, + }); + if let Some(warning) = ber_compatible_cms_warning_for_file(file, "ASPA") { + warnings.push(warning); + } + } + Err(e) => match policy.signed_object_failure_policy { + SignedObjectFailurePolicy::DropObject => { + audit.push(ObjectAuditEntry { + rsync_uri: file.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&file.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Error, + detail: Some(e.to_string()), + }); + let mut refs = vec![RfcRef("RFC 6488 §3")]; + refs.extend_from_slice(extra_rfc_refs_for_crl_selection(&e)); + warnings.push( + Warning::new(format!("dropping invalid ASPA: {}: {e}", file.rsync_uri)) + .with_rfc_refs(&refs) + .with_context(&file.rsync_uri), + ) + } + SignedObjectFailurePolicy::DropPublicationPoint => { + stats.publication_point_dropped = true; + audit.push(ObjectAuditEntry { + rsync_uri: file.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&file.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Error, + detail: Some(e.to_string()), + }); + for f in locked_files.iter().skip(idx + 1) { + if f.rsync_uri.ends_with(".roa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Skipped, + detail: Some( + "skipped due to policy=signed_object_failure_policy=drop_publication_point" + .to_string(), + ), + }); + } else if f.rsync_uri.ends_with(".asa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Skipped, + detail: Some( + "skipped due to policy=signed_object_failure_policy=drop_publication_point" + .to_string(), + ), + }); + } + } + let mut refs = vec![RfcRef("RFC 6488 §3")]; + refs.extend_from_slice(extra_rfc_refs_for_crl_selection(&e)); + warnings.push( + Warning::new(format!( + "dropping publication point due to invalid ASPA: {}: {e}", + file.rsync_uri + )) + .with_rfc_refs(&refs) + .with_context(manifest_rsync_uri), + ); + return ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta: Vec::new(), + }; + } + }, + } + } + } + + roa_cache_stats.record_to_timing(timing); + + ObjectsOutput { + vrps, + aspas, + router_keys: Vec::new(), + local_outputs_cache, + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta, + } +} diff --git a/crates/panda-rpki-validator/src/validation/objects/tests/cache.rs b/crates/panda-rpki-validator/src/validation/objects/tests/cache.rs new file mode 100644 index 0000000..5eb1bff --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/objects/tests/cache.rs @@ -0,0 +1,908 @@ + use super::*; + use crate::analysis::timing::{TimingHandle, TimingMeta}; + use crate::data_model::rc::{ + Afi, AsIdOrRange, AsIdentifierChoice, IpAddressFamily, IpAddressOrRange, IpAddressRange, + IpPrefix, IpResourceSet, + }; + use crate::policy::Policy; + use crate::storage::{ + PackTime, RoaCacheObjectMeta, RoaCacheProjection, RoaCacheProjectionContext, + ValidatedCaInstanceResult, ValidatedManifestMeta, VcirArtifactKind, VcirArtifactRole, + VcirArtifactValidationStatus, VcirAuditSummary, VcirCcrManifestProjection, + VcirInstanceGate, VcirRelatedArtifact, VcirSummary, + }; + use crate::validation::publication_point::PublicationPointSnapshot; + use std::collections::HashMap; + use time::OffsetDateTime; + use time::format_description::well_known::Rfc3339; + + fn fixture_bytes(path: &str) -> Vec { + std::fs::read(std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(path)) + .unwrap_or_else(|e| panic!("read fixture {path}: {e}")) + } + + fn fixed_time(value: &str) -> OffsetDateTime { + OffsetDateTime::parse(value, &Rfc3339).expect("parse fixed test time") + } + + const TEST_CA_VALIDATION_CONTEXT: [u8; 32] = [0x70; 32]; + const TEST_POLICY_FINGERPRINT: [u8; 32] = [0x71; 32]; + const TEST_CRL_URI: &str = "rsync://example.test/repo/current.crl"; + const TEST_ROA_URI: &str = "rsync://example.test/repo/a.roa"; + + fn sha256_32(bytes: &[u8]) -> [u8; 32] { + let digest = sha2::Sha256::digest(bytes); + let mut out = [0u8; 32]; + out.copy_from_slice(&digest); + out + } + + fn sample_roa_cache_projection( + vcir: &ValidatedCaInstanceResult, + roa_hash: [u8; 32], + ) -> RoaCacheProjection { + RoaCacheProjection::from_vcir_with_context( + vcir, + Some(&RoaCacheProjectionContext { + ca_validation_context_digest: TEST_CA_VALIDATION_CONTEXT, + policy_fingerprint: TEST_POLICY_FINGERPRINT, + object_meta: vec![RoaCacheObjectMeta { + source_object_uri: TEST_ROA_URI.to_string(), + source_object_hash: roa_hash, + ee_serial: vec![0x01], + crl_uri: TEST_CRL_URI.to_string(), + earliest_safe_reuse_time: vcir.last_successful_validation_time.clone(), + }], + }), + ) + .expect("build projection") + .expect("projection exists") + } + + fn sample_crl_cache(crl_bytes: Vec) -> HashMap { + HashMap::from([( + TEST_CRL_URI.to_string(), + CachedIssuerCrl::Pending { + bytes: crl_bytes, + sha256_hex: None, + }, + )]) + } + + fn sample_roa_cache_vcir( + issuer_der: &[u8], + crl_hash: [u8; 32], + roa_hash: [u8; 32], + item_effective_until: OffsetDateTime, + instance_effective_until: OffsetDateTime, + ) -> ValidatedCaInstanceResult { + let manifest_time = PackTime::from_utc_offset_datetime(fixed_time("2026-06-04T00:00:00Z")); + let effective_until = PackTime::from_utc_offset_datetime(instance_effective_until); + ValidatedCaInstanceResult { + manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(), + parent_manifest_rsync_uri: Some("rsync://example.test/repo/parent.mft".to_string()), + tal_id: "test-tal".to_string(), + ca_subject_name: "CN=example".to_string(), + ca_ski: "001122".to_string(), + issuer_ski: "334455".to_string(), + last_successful_validation_time: manifest_time.clone(), + current_manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(), + current_crl_rsync_uri: "rsync://example.test/repo/current.crl".to_string(), + validated_manifest_meta: ValidatedManifestMeta { + validated_manifest_number: vec![1], + validated_manifest_this_update: manifest_time.clone(), + validated_manifest_next_update: effective_until.clone(), + }, + ccr_manifest_projection: VcirCcrManifestProjection { + manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(), + manifest_sha256: vec![0xaa; 32], + manifest_size: 2048, + manifest_ee_aki: vec![0xbb; 20], + manifest_number_be: vec![1], + manifest_this_update: manifest_time.clone(), + manifest_sia_locations_der: vec![vec![0x30, 0x00]], + subordinate_skis: Vec::new(), + }, + instance_gate: VcirInstanceGate { + manifest_next_update: effective_until.clone(), + current_crl_next_update: effective_until.clone(), + self_ca_not_after: effective_until.clone(), + instance_effective_until: effective_until, + }, + child_entries: Vec::new(), + local_outputs: vec![VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: PackTime::from_utc_offset_datetime(item_effective_until), + source_object_uri: "rsync://example.test/repo/a.roa".to_string(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: roa_hash, + source_ee_cert_hash: [0xcc; 32], + payload: VcirLocalOutputPayload::Vrp { + asn: 64500, + afi: RoaAfi::Ipv4, + prefix_len: 24, + addr: [192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + max_length: 24, + }, + rule_hash: [0xdd; 32], + }], + related_artifacts: vec![ + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::IssuerCert, + artifact_kind: VcirArtifactKind::Cer, + uri: Some("rsync://example.test/repo/ca.cer".to_string()), + sha256: sha256_hex(issuer_der), + object_type: Some("cer".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::CurrentCrl, + artifact_kind: VcirArtifactKind::Crl, + uri: Some("rsync://example.test/repo/current.crl".to_string()), + sha256: sha256_hex_from_32(&crl_hash), + object_type: Some("crl".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }, + ], + summary: VcirSummary { + local_vrp_count: 1, + local_aspa_count: 0, + local_router_key_count: 0, + child_count: 0, + accepted_object_count: 2, + rejected_object_count: 0, + }, + audit_summary: VcirAuditSummary { + failed_fetch_eligible: true, + last_failed_fetch_reason: None, + warning_count: 0, + audit_flags: Vec::new(), + }, + } + } + + #[test] + fn roa_validation_cache_view_hits_when_context_and_hash_match() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_bytes = b"current-crl".to_vec(); + let crl_hash = sha256_32(&crl_bytes); + let roa_hash = [0x11; 32]; + let vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + let projection = sample_roa_cache_projection(&vcir, roa_hash); + let view = RoaValidationCacheView::from_projection(&projection, validation_time); + let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); + let mut crl_cache = sample_crl_cache(crl_bytes); + + assert!(view.matches_current_context( + issuer_der, + Some(TEST_CA_VALIDATION_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); + let hit = view.lookup(&file, &mut crl_cache, issuer_der, validation_time); + let RoaCacheLookupResult::Hit(ok) = hit else { + panic!("expected cache hit, got {hit:?}"); + }; + assert!(ok.reused_from_cache); + assert_eq!(ok.vrps.len(), 1); + assert_eq!(ok.vrps[0].asn, 64500); + assert_eq!(ok.local_outputs.len(), 1); + assert!(ok.cache_object_meta.is_some()); + } + + #[test] + fn roa_validation_cache_lookup_memoizes_current_crl_hash() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_bytes = b"current-crl".to_vec(); + let crl_hash = sha256_32(&crl_bytes); + let crl_hash_hex = sha256_hex(&crl_bytes); + let roa_hash = [0x11; 32]; + let vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + let projection = sample_roa_cache_projection(&vcir, roa_hash); + let view = RoaValidationCacheView::from_projection(&projection, validation_time); + let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); + let mut crl_cache = sample_crl_cache(crl_bytes); + + let first = view.lookup(&file, &mut crl_cache, issuer_der, validation_time); + assert!(matches!(first, RoaCacheLookupResult::Hit(_))); + match crl_cache.get(TEST_CRL_URI).expect("test CRL cache entry") { + CachedIssuerCrl::Pending { + sha256_hex: Some(cached), + .. + } => assert_eq!(cached, &crl_hash_hex), + other => panic!("expected pending CRL with memoized hash, got {other:?}"), + } + + let second = view.lookup(&file, &mut crl_cache, issuer_der, validation_time); + assert!(matches!(second, RoaCacheLookupResult::Hit(_))); + } + + #[test] + fn roa_validation_cache_lookup_reuses_publication_point_crl_gate() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_bytes = b"current-crl".to_vec(); + let crl_hash = sha256_32(&crl_bytes); + let roa_hash = [0x11; 32]; + let vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + let projection = sample_roa_cache_projection(&vcir, roa_hash); + let view = RoaValidationCacheView::from_projection(&projection, validation_time); + let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); + let mut crl_cache = sample_crl_cache(crl_bytes); + let mut gate_set = RoaCacheCrlGateSet::default(); + + let (first, first_metrics) = view.lookup_with_metrics( + &file, + &mut crl_cache, + issuer_der, + validation_time, + Some(&mut gate_set), + ); + assert!(matches!(first, RoaCacheLookupResult::Hit(_))); + assert_eq!(first_metrics.crl_gate_reused_crls, 0); + assert_eq!(gate_set.gates_by_uri.len(), 1); + + let (second, second_metrics) = view.lookup_with_metrics( + &file, + &mut crl_cache, + issuer_der, + validation_time, + Some(&mut gate_set), + ); + assert!(matches!(second, RoaCacheLookupResult::Hit(_))); + assert_eq!(second_metrics.crl_gate_reused_crls, 1); + assert_eq!(second_metrics.crl_gate_verified_crls, 0); + assert_eq!(gate_set.gates_by_uri.len(), 1); + } + + #[test] + fn cached_verified_crl_reports_hash_and_validity_window() { + let crl_bytes = fixture_bytes( + "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.crl", + ); + let crl = crate::data_model::crl::RpkixCrl::decode_der(&crl_bytes).expect("decode CRL"); + let verified = Arc::new(VerifiedIssuerCrl { + crl, + revoked_serials: std::collections::HashSet::new(), + sha256_hex: sha256_hex(&crl_bytes), + }); + let mut cached = CachedIssuerCrl::Ok(Arc::clone(&verified)); + + assert_eq!(cached.current_sha256_hex(), verified.sha256_hex); + assert!(crl_valid_at_time( + &verified.crl, + fixed_time("2026-01-21T00:00:00Z") + )); + assert!(!crl_valid_at_time( + &verified.crl, + fixed_time("2026-01-22T00:00:00Z") + )); + } + + #[test] + fn roa_validation_cache_view_from_projection_preserves_metadata() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_bytes = b"current-crl".to_vec(); + let crl_hash = sha256_32(&crl_bytes); + let roa_hash = [0x11; 32]; + let vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + let projection = sample_roa_cache_projection(&vcir, roa_hash); + assert_eq!( + projection.ca_validation_context_digest, + Some(TEST_CA_VALIDATION_CONTEXT) + ); + assert_eq!(projection.policy_fingerprint, Some(TEST_POLICY_FINGERPRINT)); + assert_eq!( + projection.entries[0].outputs_effective_until_unix, + fixed_time("2026-06-07T00:00:00Z").unix_timestamp() + ); + assert_eq!( + projection.entries[0].ee_serial.as_deref(), + Some(&[0x01][..]) + ); + assert_eq!(projection.entries[0].crl_uri.as_deref(), Some(TEST_CRL_URI)); + + let view = RoaValidationCacheView::from_projection(&projection, validation_time); + let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); + let mut crl_cache = sample_crl_cache(crl_bytes); + let hit = view.lookup(&file, &mut crl_cache, issuer_der, validation_time); + let RoaCacheLookupResult::Hit(ok) = hit else { + panic!("expected projection cache hit, got {hit:?}"); + }; + assert_eq!(ok.local_outputs[0].source_object_uri, TEST_ROA_URI); + } + + #[test] + fn roa_validation_cache_view_blocks_on_context_hash_and_expiry_gates() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_bytes = b"current-crl".to_vec(); + let crl_hash = sha256_32(&crl_bytes); + let roa_hash = [0x11; 32]; + let vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); + + let mut projection = sample_roa_cache_projection(&vcir, roa_hash); + projection.issuer_ca_sha256_hex = Some("00".repeat(32)); + let issuer_changed = RoaValidationCacheView::from_projection(&projection, validation_time); + assert!(!issuer_changed.matches_current_context( + issuer_der, + Some(TEST_CA_VALIDATION_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); + + let projection = sample_roa_cache_projection(&vcir, roa_hash); + let parent_changed = RoaValidationCacheView::from_projection(&projection, validation_time); + assert!(!parent_changed.matches_current_context( + issuer_der, + Some([0x99; 32]), + Some(TEST_POLICY_FINGERPRINT) + )); + assert!(!parent_changed.matches_current_context( + issuer_der, + Some(TEST_CA_VALIDATION_CONTEXT), + Some([0x98; 32]) + )); + + let mut projection = sample_roa_cache_projection(&vcir, roa_hash); + projection.entries[0].source_object_hash = [0xff; 32]; + let roa_changed = RoaValidationCacheView::from_projection(&projection, validation_time); + let mut crl_cache = sample_crl_cache(crl_bytes); + assert!(matches!( + roa_changed.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::HashBlocked + )); + + let expired_vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-04T00:00:00Z"), + ); + let expired_projection = sample_roa_cache_projection(&expired_vcir, roa_hash); + let expired_view = + RoaValidationCacheView::from_projection(&expired_projection, validation_time); + assert!(!expired_view.matches_current_context( + issuer_der, + Some(TEST_CA_VALIDATION_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); + let mut crl_cache = sample_crl_cache(b"current-crl".to_vec()); + assert!(matches!( + expired_view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::ExpiredBlocked + )); + } + + #[test] + fn active_roa_cache_view_records_context_block() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_hash = [0x22; 32]; + let roa_hash = [0x11; 32]; + let vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + let projection = sample_roa_cache_projection(&vcir, roa_hash); + let view = RoaValidationCacheView::from_projection(&projection, validation_time); + let input = RoaValidationCacheInput::enabled_with_context( + Some(&view), + [0x99; 32], + TEST_POLICY_FINGERPRINT, + ); + let mut stats = RoaValidationCacheStats::for_input(input, 1); + + assert!(active_roa_cache_view(input, issuer_der, &mut stats, 1).is_none()); + assert_eq!(stats.blocked_roas, 1); + assert_eq!(stats.context_blocked_roas, 1); + assert_eq!(stats.fresh_roas, 1); + } + + #[test] + fn roa_validation_cache_view_blocks_expired_output() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_bytes = b"current-crl".to_vec(); + let crl_hash = sha256_32(&crl_bytes); + let roa_hash = [0x11; 32]; + let vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-04T01:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + let projection = sample_roa_cache_projection(&vcir, roa_hash); + let view = RoaValidationCacheView::from_projection(&projection, validation_time); + let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); + let mut crl_cache = sample_crl_cache(crl_bytes); + + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::ExpiredBlocked + )); + } + + #[test] + fn roa_validation_cache_view_blocks_before_safe_reuse_time() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_bytes = b"current-crl".to_vec(); + let crl_hash = sha256_32(&crl_bytes); + let roa_hash = [0x11; 32]; + let vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + let mut projection = sample_roa_cache_projection(&vcir, roa_hash); + projection.entries[0].earliest_safe_reuse_time_unix = + Some(fixed_time("2026-06-06T00:00:00Z").unix_timestamp()); + let view = RoaValidationCacheView::from_projection(&projection, validation_time); + let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); + let mut crl_cache = sample_crl_cache(crl_bytes); + + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::ExpiredBlocked + )); + } + + #[test] + fn roa_validation_cache_view_treats_legacy_entry_without_lower_bound_as_miss() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_hash = sha256_32(b"current-crl"); + let roa_hash = [0x11; 32]; + let vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + let mut projection = sample_roa_cache_projection(&vcir, roa_hash); + projection.entries[0].earliest_safe_reuse_time_unix = None; + let view = RoaValidationCacheView::from_projection(&projection, validation_time); + let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); + let mut crl_cache = sample_crl_cache(b"current-crl".to_vec()); + + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::Miss + )); + } + + #[test] + fn roa_validation_cache_stats_records_vcir_miss_to_timing() { + let stats = RoaValidationCacheStats::for_input(RoaValidationCacheInput::enabled(None), 3); + assert_eq!(stats.enabled_publication_points, 1); + assert_eq!(stats.vcir_miss_publication_points, 1); + assert_eq!(stats.miss_roas, 3); + assert_eq!(stats.fresh_roas, 3); + + let timing = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339: "2026-06-05T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-06-05T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + stats.record_to_timing(Some(&timing)); + let dir = tempfile::tempdir().expect("timing dir"); + let path = dir.path().join("timing.json"); + timing.write_json(&path, 10).expect("write timing"); + let report: serde_json::Value = + serde_json::from_slice(&std::fs::read(path).expect("read timing")) + .expect("parse timing"); + + assert_eq!( + report["counts"]["roa_validation_cache_enabled_publication_points"], + 1 + ); + assert_eq!( + report["counts"]["roa_validation_cache_vcir_miss_publication_points"], + 1 + ); + assert_eq!(report["counts"]["roa_validation_cache_miss_roas"], 3); + assert_eq!(report["counts"]["roa_validation_cache_fresh_roas"], 3); + } + + #[test] + fn roa_validation_cache_stats_records_block_breakdown_to_timing() { + let mut stats = RoaValidationCacheStats::default(); + record_roa_cache_block(&mut stats, &RoaCacheLookupResult::HashBlocked); + record_roa_cache_block(&mut stats, &RoaCacheLookupResult::ExpiredBlocked); + record_roa_cache_block(&mut stats, &RoaCacheLookupResult::RevokedBlocked); + record_roa_cache_block(&mut stats, &RoaCacheLookupResult::MetadataBlocked); + stats.crl_recheck_hit_roas = 2; + stats.context_blocked_roas = 3; + stats.context_gate_nanos = 4; + stats.lookup_nanos = 5; + stats.lookup_entry_gate_nanos = 6; + stats.lookup_crl_gate_nanos = 7; + stats.lookup_materialize_nanos = 8; + stats.lookup_crl_gate_verified_crls = 9; + stats.lookup_crl_gate_reused_crls = 10; + + let timing = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339: "2026-06-05T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-06-05T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + stats.record_to_timing(Some(&timing)); + let dir = tempfile::tempdir().expect("timing dir"); + let path = dir.path().join("timing.json"); + timing.write_json(&path, 10).expect("write timing"); + let report: serde_json::Value = + serde_json::from_slice(&std::fs::read(path).expect("read timing")) + .expect("parse timing"); + + assert_eq!(stats.blocked_roas, 4); + assert_eq!(stats.fresh_roas, 4); + assert_eq!(report["counts"]["roa_validation_cache_blocked_roas"], 4); + assert_eq!( + report["counts"]["roa_validation_cache_hash_blocked_roas"], + 1 + ); + assert_eq!( + report["counts"]["roa_validation_cache_expired_blocked_roas"], + 1 + ); + assert_eq!( + report["counts"]["roa_validation_cache_revoked_blocked_roas"], + 1 + ); + assert_eq!( + report["counts"]["roa_validation_cache_metadata_blocked_roas"], + 1 + ); + assert_eq!( + report["counts"]["roa_validation_cache_crl_recheck_hit_roas"], + 2 + ); + assert_eq!( + report["counts"]["roa_validation_cache_context_blocked_roas"], + 3 + ); + assert_eq!( + report["counts"]["roa_validation_cache_context_gate_nanos"], + 4 + ); + assert_eq!(report["counts"]["roa_validation_cache_lookup_nanos"], 5); + assert_eq!( + report["counts"]["roa_validation_cache_lookup_entry_gate_nanos"], + 6 + ); + assert_eq!( + report["counts"]["roa_validation_cache_lookup_crl_gate_nanos"], + 7 + ); + assert_eq!( + report["counts"]["roa_validation_cache_lookup_materialize_nanos"], + 8 + ); + assert_eq!( + report["counts"]["roa_validation_cache_lookup_crl_gate_verified_crls"], + 9 + ); + assert_eq!( + report["counts"]["roa_validation_cache_lookup_crl_gate_reused_crls"], + 10 + ); + } + + #[test] + fn roa_validation_cache_view_ignores_rejected_artifacts_and_non_roa_outputs() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_bytes = b"current-crl".to_vec(); + let crl_hash = sha256_32(&crl_bytes); + let roa_hash = [0x11; 32]; + let mut vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + vcir.related_artifacts.insert( + 0, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::IssuerCert, + artifact_kind: VcirArtifactKind::Cer, + uri: Some("rsync://example.test/repo/rejected.cer".to_string()), + sha256: "00".repeat(32), + object_type: Some("cer".to_string()), + validation_status: VcirArtifactValidationStatus::Rejected, + reject_reason: None, + }, + ); + vcir.local_outputs.push(VcirLocalOutput { + output_type: VcirOutputType::Aspa, + item_effective_until: PackTime::from_utc_offset_datetime(fixed_time( + "2026-06-07T00:00:00Z", + )), + source_object_uri: "rsync://example.test/repo/a.asa".to_string(), + source_object_type: VcirSourceObjectType::Aspa, + source_object_hash: [0x44; 32], + source_ee_cert_hash: [0x55; 32], + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: 64500, + provider_as_ids: vec![64501], + }, + rule_hash: [0x66; 32], + }); + let projection = sample_roa_cache_projection(&vcir, roa_hash); + let view = RoaValidationCacheView::from_projection(&projection, validation_time); + let mut crl_cache = sample_crl_cache(crl_bytes); + + assert!(view.matches_current_context( + issuer_der, + Some(TEST_CA_VALIDATION_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); + assert!(matches!( + view.lookup( + &PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash), + &mut crl_cache, + issuer_der, + validation_time + ), + RoaCacheLookupResult::Hit(_) + )); + assert!(matches!( + view.lookup( + &PackFile::from_bytes_with_sha256( + "rsync://example.test/repo/a.asa", + vec![0x03], + [0x44; 32], + ), + &mut crl_cache, + issuer_der, + validation_time + ), + RoaCacheLookupResult::Miss + )); + } + + #[test] + fn roa_validation_cache_context_requires_issuer_parent_and_policy() { + let mut view = RoaValidationCacheView { + entries_by_uri: HashMap::new(), + issuer_ca_sha256_hex: None, + ca_validation_context_digest: Some(TEST_CA_VALIDATION_CONTEXT), + policy_fingerprint: Some(TEST_POLICY_FINGERPRINT), + crl_sha256_by_uri: HashMap::new(), + blocked: false, + }; + assert!(!view.matches_current_context( + b"issuer-ca", + Some(TEST_CA_VALIDATION_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); + + view.issuer_ca_sha256_hex = Some("00".repeat(32)); + assert!(!view.matches_current_context( + b"issuer-ca", + Some(TEST_CA_VALIDATION_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); + + view.issuer_ca_sha256_hex = Some(sha256_hex(b"issuer-ca")); + assert!(view.matches_current_context( + b"issuer-ca", + Some(TEST_CA_VALIDATION_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); + assert!(!view.matches_current_context(b"issuer-ca", None, Some(TEST_POLICY_FINGERPRINT))); + + view.ca_validation_context_digest = None; + assert!(!view.matches_current_context( + b"issuer-ca", + Some(TEST_CA_VALIDATION_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); + view.ca_validation_context_digest = Some(TEST_CA_VALIDATION_CONTEXT); + view.policy_fingerprint = None; + assert!(!view.matches_current_context( + b"issuer-ca", + Some(TEST_CA_VALIDATION_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); + } + + #[test] + fn roa_validation_cache_lookup_classifies_hash_empty_payload_and_missing_uri() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_bytes = b"current-crl".to_vec(); + let crl_hash_hex = sha256_hex(&crl_bytes); + let roa_hash = [0x11; 32]; + let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); + let mut output = sample_roa_cache_vcir( + issuer_der, + sha256_32(&crl_bytes), + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ) + .local_outputs + .remove(0); + let outputs_effective_until_unix = output + .item_effective_until + .parse() + .expect("parse output effective until") + .unix_timestamp(); + let mut view = RoaValidationCacheView { + entries_by_uri: HashMap::new(), + issuer_ca_sha256_hex: Some(sha256_hex(issuer_der)), + ca_validation_context_digest: Some(TEST_CA_VALIDATION_CONTEXT), + policy_fingerprint: Some(TEST_POLICY_FINGERPRINT), + crl_sha256_by_uri: HashMap::from([(TEST_CRL_URI.to_string(), crl_hash_hex.clone())]), + blocked: false, + }; + let mut crl_cache = sample_crl_cache(crl_bytes.clone()); + + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::Miss + )); + + view.entries_by_uri.insert( + file.rsync_uri.clone(), + CachedRoaValidationResult { + source_object_hash: roa_hash, + ee_serial: Some(vec![0x01]), + crl_uri: None, + earliest_safe_reuse_time_unix: validation_time.unix_timestamp(), + outputs_effective_until_unix, + outputs: vec![output.clone()], + }, + ); + let mut crl_cache = sample_crl_cache(crl_bytes.clone()); + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::MetadataBlocked + )); + + view.entries_by_uri.insert( + file.rsync_uri.clone(), + CachedRoaValidationResult { + source_object_hash: roa_hash, + ee_serial: None, + crl_uri: Some(TEST_CRL_URI.to_string()), + earliest_safe_reuse_time_unix: validation_time.unix_timestamp(), + outputs_effective_until_unix, + outputs: vec![output.clone()], + }, + ); + let mut crl_cache = sample_crl_cache(crl_bytes.clone()); + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::MetadataBlocked + )); + + view.entries_by_uri.insert( + file.rsync_uri.clone(), + CachedRoaValidationResult { + source_object_hash: roa_hash, + ee_serial: Some(vec![0x01]), + crl_uri: Some(TEST_CRL_URI.to_string()), + earliest_safe_reuse_time_unix: validation_time.unix_timestamp(), + outputs_effective_until_unix, + outputs: vec![output.clone()], + }, + ); + let mut crl_cache = HashMap::new(); + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::MetadataBlocked + )); + + view.crl_sha256_by_uri + .insert(TEST_CRL_URI.to_string(), "00".repeat(32)); + let mut crl_cache = sample_crl_cache(crl_bytes.clone()); + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::MetadataBlocked + )); + view.crl_sha256_by_uri + .insert(TEST_CRL_URI.to_string(), crl_hash_hex); + + view.entries_by_uri.insert( + file.rsync_uri.clone(), + CachedRoaValidationResult { + source_object_hash: [0xff; 32], + ee_serial: Some(vec![0x01]), + crl_uri: Some(TEST_CRL_URI.to_string()), + earliest_safe_reuse_time_unix: validation_time.unix_timestamp(), + outputs_effective_until_unix, + outputs: vec![output.clone()], + }, + ); + let mut crl_cache = sample_crl_cache(crl_bytes.clone()); + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::HashBlocked + )); + + view.entries_by_uri.insert( + file.rsync_uri.clone(), + CachedRoaValidationResult { + source_object_hash: roa_hash, + ee_serial: Some(vec![0x01]), + crl_uri: Some(TEST_CRL_URI.to_string()), + earliest_safe_reuse_time_unix: validation_time.unix_timestamp(), + outputs_effective_until_unix, + outputs: Vec::new(), + }, + ); + let mut crl_cache = sample_crl_cache(crl_bytes.clone()); + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::Miss + )); + + output.payload = VcirLocalOutputPayload::Aspa { + customer_as_id: 64500, + provider_as_ids: vec![64501], + }; + view.entries_by_uri.insert( + file.rsync_uri.clone(), + CachedRoaValidationResult { + source_object_hash: roa_hash, + ee_serial: Some(vec![0x01]), + crl_uri: Some(TEST_CRL_URI.to_string()), + earliest_safe_reuse_time_unix: validation_time.unix_timestamp(), + outputs_effective_until_unix, + outputs: vec![output], + }, + ); + let mut crl_cache = sample_crl_cache(crl_bytes); + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::MetadataBlocked + )); + } diff --git a/crates/panda-rpki-validator/src/validation/objects/tests/output.rs b/crates/panda-rpki-validator/src/validation/objects/tests/output.rs new file mode 100644 index 0000000..4e14d75 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/objects/tests/output.rs @@ -0,0 +1,117 @@ + #[test] + fn roa_output_helpers_cover_vrps_and_afi_strings() { + let roa_der = + fixture_bytes("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"); + let roa = RoaObject::decode_der(&roa_der).expect("decode roa"); + let vrps = roa_to_vrps(&roa); + assert!(!vrps.is_empty()); + assert!(vrps.iter().all(|vrp| vrp.asn == roa.roa.as_id)); + assert_eq!(roa_afi_to_string(RoaAfi::Ipv4), "ipv4"); + assert_eq!(roa_afi_to_string(RoaAfi::Ipv6), "ipv6"); + } + + #[test] + fn parallel_stage_roa_tasks_share_stage_owned_payloads() { + let ta_constraints = Arc::new( + crate::ta_constraints::TaConstraints::from_file( + &std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/ta_constraints/afrinic-current-ipv4-deny.constraints"), + ) + .expect("load constraints fixture"), + ); + let stage = ParallelObjectsStage { + publication_point_id: 7, + shared: Arc::new(RoaTaskShared { + locked_files: Arc::<[PackFile]>::from(vec![ + PackFile::from_bytes_with_sha256( + "rsync://example.test/repo/a.roa", + vec![1, 2, 3], + [1u8; 32], + ), + PackFile::from_bytes_with_sha256( + "rsync://example.test/repo/b.roa", + vec![4, 5, 6], + [2u8; 32], + ), + ]), + manifest_rsync_uri: Arc::::from("rsync://example.test/repo/manifest.mft"), + issuer_ca_der: Arc::from([0x01u8].as_slice()), + issuer_ca: Arc::new( + ResourceCertificate::decode_der(&fixture_bytes( + "tests/fixtures/ta/apnic-ta.cer", + )) + .expect("decode fixture CA certificate"), + ), + issuer_spki_der: Arc::from([0x02u8].as_slice()), + issuer_ca_rsync_uri: Some(Arc::::from("rsync://example.test/repo/ca.cer")), + crl_cache: Arc::new(Mutex::new(HashMap::new())), + issuer_resources_index: Arc::new(IssuerResourcesIndex::default()), + issuer_effective_ip: None, + issuer_effective_as: None, + resource_validation_mode: ResourceValidationMode::default(), + ta_constraints: Some(Arc::clone(&ta_constraints)), + }), + validation_time: OffsetDateTime::now_utc(), + collect_vcir_local_outputs: false, + strict_cms_der: false, + strict_name: false, + resource_validation_mode: ResourceValidationMode::default(), + roa_task_indices: vec![0, 1], + cached_roa_results: Vec::new(), + roa_cache_stats: RoaValidationCacheStats::default(), + warnings: Vec::new(), + stats: ObjectsStats { + roa_total: 2, + ..ObjectsStats::default() + }, + audit: Vec::new(), + }; + let tasks = stage.build_roa_tasks(); + assert_eq!(tasks.len(), 2); + assert!(Arc::ptr_eq(&tasks[0].shared, &tasks[1].shared)); + assert!(Arc::ptr_eq( + tasks[0] + .shared + .ta_constraints + .as_ref() + .expect("task constraints snapshot"), + &ta_constraints + )); + } + + #[test] + fn strict_name_manifest_decode_failure_drops_publication_point() { + let publication_point = PublicationPointSnapshot { + format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, + manifest_rsync_uri: "rsync://example.test/repo/manifest.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), + manifest_number_be: vec![0x01], + this_update: PackTime::from_utc_offset_datetime(OffsetDateTime::now_utc()), + next_update: PackTime::from_utc_offset_datetime(OffsetDateTime::now_utc()), + verified_at: PackTime::from_utc_offset_datetime(OffsetDateTime::now_utc()), + manifest_bytes: vec![0x01, 0x02, 0x03], + files: vec![], + }; + let policy = Policy::default(); + let output = process_publication_point_for_issuer_with_options( + &publication_point, + &policy, + &[], + None, + None, + None, + OffsetDateTime::now_utc(), + None, + false, + ); + assert!(output.stats.publication_point_dropped); + assert!(output.vrps.is_empty()); + assert!( + output + .warnings + .iter() + .any(|warning| warning.message.contains("manifest decode failed")), + "{:?}", + output.warnings + ); + } diff --git a/crates/panda-rpki-validator/src/validation/objects/tests/resource_helpers.rs b/crates/panda-rpki-validator/src/validation/objects/tests/resource_helpers.rs new file mode 100644 index 0000000..a28342e --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/objects/tests/resource_helpers.rs @@ -0,0 +1,654 @@ + #[test] + fn parallel_roa_cache_blocked_falls_back_to_single_fresh_task() { + let manifest_bytes = fixture_bytes( + "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft", + ); + let issuer_ca_der = fixture_bytes( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ); + let crl_bytes = fixture_bytes( + "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.crl", + ); + let crl_hash = sha256_32(&crl_bytes); + let actual_roa_hash = [0x11; 32]; + let cached_roa_hash = [0x33; 32]; + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let publication_point = PublicationPointSnapshot { + format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, + manifest_rsync_uri: + "rsync://rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft" + .to_string(), + publication_point_rsync_uri: "rsync://rpki.cernet.net/repo/cernet/0/".to_string(), + manifest_number_be: vec![1], + this_update: PackTime::from_utc_offset_datetime(validation_time), + next_update: PackTime::from_utc_offset_datetime( + validation_time + time::Duration::days(1), + ), + verified_at: PackTime::from_utc_offset_datetime(validation_time), + manifest_bytes, + files: vec![ + PackFile::from_bytes_with_sha256(TEST_CRL_URI, crl_bytes, crl_hash), + PackFile::from_bytes_with_sha256( + "rsync://example.test/repo/a.roa", + vec![0x01], + actual_roa_hash, + ), + ], + }; + let vcir = sample_roa_cache_vcir( + &issuer_ca_der, + crl_hash, + cached_roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + let projection = sample_roa_cache_projection(&vcir, cached_roa_hash); + let view = RoaValidationCacheView::from_projection(&projection, validation_time); + + let stage = match prepare_publication_point_for_parallel_roa_with_cache( + 7, + &publication_point, + &Policy::default(), + &issuer_ca_der, + None, + None, + None, + validation_time, + false, + RoaValidationCacheInput::enabled_with_context( + Some(&view), + TEST_CA_VALIDATION_CONTEXT, + TEST_POLICY_FINGERPRINT, + ), + ) { + ParallelObjectsPrepare::Staged(stage) => stage, + ParallelObjectsPrepare::Complete(_) => panic!("expected staged ROA fallback"), + }; + + assert_eq!(stage.roa_cache_stats.blocked_roas, 1); + assert_eq!(stage.roa_cache_stats.fresh_roas, 1); + assert_eq!(stage.roa_task_indices, vec![1]); + assert_eq!(stage.roa_task_count(), 1); + } + + #[test] + fn merge_as_intervals_merges_overlapping_and_adjacent() { + let v = vec![(1, 2), (3, 5), (10, 10), (11, 12)]; + let merged = merge_as_intervals(&v); + assert_eq!(merged, vec![(1, 5), (10, 12)]); + } + + #[test] + fn as_choice_subset_rejects_inherit() { + let child = Some(&AsIdentifierChoice::Inherit); + let parent = Some(&AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { min: 1, max: 10 }, + ])); + assert!(!as_choice_subset(child, parent)); + } + + #[test] + fn as_choice_subset_checks_ranges() { + let child = Some(&AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Id(5), + AsIdOrRange::Range { min: 7, max: 9 }, + ])); + let parent = Some(&AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { min: 1, max: 10 }, + ])); + assert!(as_choice_subset(child, parent)); + } + + #[test] + fn ip_resources_is_subset_accepts_prefixes_and_ranges() { + let parent = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![ + IpAddressOrRange::Prefix(IpPrefix { + afi: Afi::Ipv4, + prefix_len: 8, + addr: vec![10, 0, 0, 0], + }), + IpAddressOrRange::Range(IpAddressRange { + min: vec![192, 0, 2, 0], + max: vec![192, 0, 2, 255], + }), + ]), + }], + }; + let child = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![ + IpAddressOrRange::Prefix(IpPrefix { + afi: Afi::Ipv4, + prefix_len: 16, + addr: vec![10, 1, 0, 0], + }), + IpAddressOrRange::Range(IpAddressRange { + min: vec![192, 0, 2, 10], + max: vec![192, 0, 2, 20], + }), + ]), + }], + }; + + assert!(ip_resources_is_subset(&child, &parent)); + } + + #[test] + fn ip_resources_is_subset_rejects_inherit_in_child() { + let parent = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv6, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( + IpPrefix { + afi: Afi::Ipv6, + prefix_len: 32, + addr: vec![0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + }, + )]), + }], + }; + let child = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv6, + choice: IpAddressChoice::Inherit, + }], + }; + assert!(!ip_resources_is_subset(&child, &parent)); + } + + #[test] + fn increment_bytes_wraps_all_ff_to_zero() { + assert_eq!(increment_bytes(&[0xFF, 0xFF]), vec![0x00, 0x00]); + } + + #[test] + fn merge_ip_intervals_merges_contiguous() { + let mut v = vec![ + (vec![0, 0, 0, 0], vec![0, 0, 0, 10]), + (vec![0, 0, 0, 11], vec![0, 0, 0, 20]), + ]; + merge_ip_intervals_in_place(&mut v); + assert_eq!(v, vec![(vec![0, 0, 0, 0], vec![0, 0, 0, 20])]); + } + + #[test] + fn choose_crl_for_certificate_reports_missing_crl_in_snapshot() { + let roa_der = + fixture_bytes("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"); + let roa = RoaObject::decode_der(&roa_der).expect("decode roa"); + let ee_crldp_uris = roa.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .extensions + .crl_distribution_points_uris + .as_ref(); + let crl_cache: HashMap = HashMap::new(); + let err = choose_crl_uri_for_certificate(ee_crldp_uris, &crl_cache).unwrap_err(); + assert!(matches!(err, ObjectValidateError::MissingCrlInPack)); + } + + #[test] + fn choose_crl_for_certificate_reports_missing_crldp_uris() { + let mut crl_cache: HashMap = HashMap::new(); + crl_cache.insert( + "rsync://example.test/a.crl".to_string(), + CachedIssuerCrl::Pending { + bytes: vec![0x01], + sha256_hex: None, + }, + ); + let err = choose_crl_uri_for_certificate(None, &crl_cache).unwrap_err(); + assert!(matches!(err, ObjectValidateError::MissingCrlDpUris)); + } + + #[test] + fn choose_crl_for_certificate_prefers_matching_crldp_uri_in_order() { + let roa_der = + fixture_bytes("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"); + let roa = RoaObject::decode_der(&roa_der).expect("decode roa"); + let ee_crldp_uris = roa.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .extensions + .crl_distribution_points_uris + .as_ref() + .expect("fixture ee has crldp"); + + let matching_uri = ee_crldp_uris[0].as_str().to_string(); + let mut crl_cache: HashMap = HashMap::new(); + crl_cache.insert( + "rsync://example.test/other.crl".to_string(), + CachedIssuerCrl::Pending { + bytes: vec![0x00], + sha256_hex: None, + }, + ); + crl_cache.insert( + matching_uri.clone(), + CachedIssuerCrl::Pending { + bytes: vec![0x01], + sha256_hex: None, + }, + ); + + let uri = choose_crl_uri_for_certificate(Some(ee_crldp_uris), &crl_cache).unwrap(); + assert_eq!(uri, matching_uri); + } + + #[test] + fn choose_crl_for_certificate_reports_not_found_when_crldp_does_not_match_snapshot() { + let roa_der = + fixture_bytes("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"); + let roa = RoaObject::decode_der(&roa_der).expect("decode roa"); + let ee_crldp_uris = roa.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .extensions + .crl_distribution_points_uris + .as_ref(); + + let mut crl_cache: HashMap = HashMap::new(); + crl_cache.insert( + "rsync://example.test/other.crl".to_string(), + CachedIssuerCrl::Pending { + bytes: vec![0x01], + sha256_hex: None, + }, + ); + let err = choose_crl_uri_for_certificate(ee_crldp_uris, &crl_cache).unwrap_err(); + assert!(matches!(err, ObjectValidateError::CrlNotFound(_))); + } + + #[test] + fn validate_ee_resources_subset_reports_missing_issuer_effective_ip() { + let roa_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"); + let roa_der = std::fs::read(roa_path).expect("read roa"); + let roa = RoaObject::decode_der(&roa_der).expect("decode roa"); + let ee = &roa.signed_object.signed_data.certificates[0].resource_cert; + + let idx = IssuerResourcesIndex::default(); + let err = validate_ee_resources_subset(ee, None, None, &idx).unwrap_err(); + assert!(matches!(err, ObjectValidateError::MissingIssuerEffectiveIp)); + } + + #[test] + fn validate_ee_resources_subset_reports_missing_issuer_effective_as() { + let aspa_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/5m80fwYws_3FiFD7JiQjAqZ1RYQ.asa", + ); + let aspa_der = std::fs::read(aspa_path).expect("read aspa"); + let aspa = AspaObject::decode_der(&aspa_der).expect("decode aspa"); + let ee = &aspa.signed_object.signed_data.certificates[0].resource_cert; + + let issuer_ip = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv6, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( + IpPrefix { + afi: Afi::Ipv6, + prefix_len: 32, + addr: vec![0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + }, + )]), + }], + }; + + let idx = build_issuer_resources_index(Some(&issuer_ip), None); + let err = validate_ee_resources_subset(ee, Some(&issuer_ip), None, &idx).unwrap_err(); + assert!(matches!(err, ObjectValidateError::MissingIssuerEffectiveAs)); + } + + #[test] + fn validate_ee_resources_subset_reports_not_subset() { + let roa_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"); + let roa_der = std::fs::read(roa_path).expect("read roa"); + let roa = RoaObject::decode_der(&roa_der).expect("decode roa"); + let ee = &roa.signed_object.signed_data.certificates[0].resource_cert; + + // Unrelated parent resources. + let issuer_ip = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv6, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( + IpPrefix { + afi: Afi::Ipv6, + prefix_len: 32, + addr: vec![0x26, 0x20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + }, + )]), + }], + }; + + let idx = build_issuer_resources_index(Some(&issuer_ip), None); + let err = validate_ee_resources_subset(ee, Some(&issuer_ip), None, &idx).unwrap_err(); + assert!(matches!(err, ObjectValidateError::EeResourcesNotSubset)); + } + + #[test] + fn validation_update_03_ee_resources_reduce_to_vrs_for_roa_checks() { + let roa_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"); + let roa_der = std::fs::read(roa_path).expect("read roa"); + let roa = RoaObject::decode_der(&roa_der).expect("decode roa"); + let ee = &roa.signed_object.signed_data.certificates[0].resource_cert; + let ee_ip = ee + .tbs + .extensions + .ip_resources + .as_ref() + .expect("fixture EE has IP resources"); + let first_family = ee_ip.families.first().expect("family"); + let first_item = match &first_family.choice { + IpAddressChoice::AddressesOrRanges(items) => items.first().expect("item").clone(), + IpAddressChoice::Inherit => panic!("fixture should not inherit"), + }; + let issuer_ip = IpResourceSet { + families: vec![IpAddressFamily { + afi: first_family.afi, + choice: IpAddressChoice::AddressesOrRanges(vec![first_item]), + }], + }; + let idx = build_issuer_resources_index(Some(&issuer_ip), None); + + let strict_err = validate_ee_resources_for_mode( + ee, + Some(&issuer_ip), + None, + &idx, + ResourceValidationMode::Rfc6487, + ) + .unwrap_err(); + assert!(matches!( + strict_err, + ObjectValidateError::EeResourcesNotSubset + )); + + let vrs = validate_ee_resources_for_mode( + ee, + Some(&issuer_ip), + None, + &idx, + ResourceValidationMode::ValidationUpdate03, + ) + .expect("vrs resource resolution"); + assert_eq!(vrs.ip.expect("vrs ip").families.len(), 1); + } + + #[test] + fn validation_update_03_ee_as_vrs_filters_aspa_customer_resources() { + let issuer_as = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { + min: 64500, + max: 64510, + }, + ])), + rdi: None, + }; + let child_as = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { + min: 64505, + max: 64520, + }, + ])), + rdi: None, + }; + let idx = build_issuer_resources_index(None, Some(&issuer_as)); + let vrs = intersect_ee_as_resources_vrs(&child_as, Some(&issuer_as), &idx) + .expect("intersect AS resources"); + assert!(as_resource_set_contains_asn(&vrs, 64505)); + assert!(as_resource_set_contains_asn(&vrs, 64510)); + assert!(!as_resource_set_contains_asn(&vrs, 64511)); + } + + #[test] + fn validation_update_03_empty_ee_vrs_rejects_roa_and_aspa_outputs() { + let roa_der = + fixture_bytes("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"); + let roa = RoaObject::decode_der(&roa_der).expect("decode roa"); + let empty_ip_vrs = IpResourceSet { + families: Vec::new(), + }; + let roa_err = roa_to_vrps_with_vrs(&roa, Some(&empty_ip_vrs)).unwrap_err(); + assert!(matches!(roa_err, ObjectValidateError::EeResourcesNotSubset)); + let missing_ip_err = roa_to_vrps_with_vrs(&roa, None).unwrap_err(); + assert!(matches!( + missing_ip_err, + ObjectValidateError::EeResourcesNotSubset + )); + + let aspa_der = fixture_bytes( + "tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/5m80fwYws_3FiFD7JiQjAqZ1RYQ.asa", + ); + let aspa = AspaObject::decode_der(&aspa_der).expect("decode aspa"); + let empty_as_vrs = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(Vec::new())), + rdi: None, + }; + let aspa_err = validate_aspa_customer_in_vrs(&aspa, Some(&empty_as_vrs)).unwrap_err(); + assert!(matches!( + aspa_err, + ObjectValidateError::EeResourcesNotSubset + )); + let missing_as_err = validate_aspa_customer_in_vrs(&aspa, None).unwrap_err(); + assert!(matches!( + missing_as_err, + ObjectValidateError::EeResourcesNotSubset + )); + } + + #[test] + fn extra_rfc_refs_for_crl_selection_distinguishes_crl_errors() { + assert_eq!( + extra_rfc_refs_for_crl_selection(&ObjectValidateError::MissingCrlDpUris), + RFC_CRLDP + ); + assert_eq!( + extra_rfc_refs_for_crl_selection(&ObjectValidateError::CrlNotFound( + "rsync://example.test/x.crl".to_string(), + )), + RFC_CRLDP_AND_LOCKED_PACK + ); + assert!( + extra_rfc_refs_for_crl_selection(&ObjectValidateError::MissingCrlInPack).is_empty() + ); + } + + #[test] + fn as_subset_helpers_cover_success_and_failure_paths() { + let child = AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Id(5), + AsIdOrRange::Range { min: 7, max: 9 }, + ]); + let parent = + AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Range { min: 1, max: 10 }]); + let parent_intervals = [(1, 10)]; + + assert!(as_choice_subset(None, Some(&parent))); + assert!(!as_choice_subset(Some(&child), None)); + assert!(!as_choice_subset( + Some(&AsIdentifierChoice::Inherit), + Some(&parent) + )); + assert!(!as_choice_subset( + Some(&child), + Some(&AsIdentifierChoice::Inherit) + )); + assert!(as_choice_subset(Some(&child), Some(&parent))); + + assert!(as_choice_subset_indexed(None, Some(&parent_intervals))); + assert!(!as_choice_subset_indexed(Some(&child), None)); + assert!(!as_choice_subset_indexed( + Some(&AsIdentifierChoice::Inherit), + Some(&parent_intervals), + )); + assert!(as_choice_subset_indexed( + Some(&child), + Some(&parent_intervals) + )); + assert!(!as_choice_subset_indexed( + Some(&AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { min: 11, max: 12 } + ])), + Some(&parent_intervals), + )); + + let child_set = AsResourceSet { + asnum: Some(child.clone()), + rdi: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id(42)])), + }; + let parent_set = AsResourceSet { + asnum: Some(parent.clone()), + rdi: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { min: 40, max: 50 }, + ])), + }; + assert!(as_resources_is_subset(&child_set, &parent_set)); + assert!(as_resources_is_subset_indexed( + &child_set, + &parent_set, + &IssuerResourcesIndex { + asnum: Some(vec![(1, 10)]), + rdi: Some(vec![(40, 50)]), + ..IssuerResourcesIndex::default() + }, + )); + } + + #[test] + fn ip_subset_helpers_cover_strict_and_indexed_paths() { + let parent = IpResourceSet { + families: vec![ + IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![ + IpAddressOrRange::Prefix(IpPrefix { + afi: Afi::Ipv4, + prefix_len: 8, + addr: vec![10, 0, 0, 0], + }), + IpAddressOrRange::Range(IpAddressRange { + min: vec![192, 0, 2, 0], + max: vec![192, 0, 2, 255], + }), + ]), + }, + IpAddressFamily { + afi: Afi::Ipv6, + choice: IpAddressChoice::Inherit, + }, + ], + }; + let child = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![ + IpAddressOrRange::Prefix(IpPrefix { + afi: Afi::Ipv4, + prefix_len: 16, + addr: vec![10, 1, 0, 0], + }), + IpAddressOrRange::Range(IpAddressRange { + min: vec![192, 0, 2, 10], + max: vec![192, 0, 2, 20], + }), + ]), + }], + }; + let strict_bad = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::Inherit, + }], + }; + + assert!(ip_resources_is_subset(&child, &parent)); + assert!(ip_resources_to_merged_intervals(&parent).contains_key(&AfiKey::V4)); + assert!(ip_resources_to_merged_intervals_strict(&child).is_ok()); + assert!(ip_resources_to_merged_intervals_strict(&strict_bad).is_err()); + + let idx = build_issuer_resources_index( + Some(&parent), + Some(&AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { + min: 64500, + max: 64510, + }, + ])), + rdi: Some(AsIdentifierChoice::Inherit), + }), + ); + assert!(idx.ip_v4.is_some()); + assert!(idx.ip_v6.is_none()); + assert!(idx.asnum.is_some()); + assert!(idx.rdi.is_none()); + assert!(ip_resources_is_subset_indexed(&child, &parent, &idx)); + assert!(!ip_resources_is_subset_indexed(&strict_bad, &parent, &idx)); + } + + #[test] + fn interval_and_byte_helpers_cover_edge_cases() { + let parent = vec![(vec![0, 0, 0, 0], vec![0, 0, 0, 10])]; + assert!(interval_is_covered(&parent, &[0, 0, 0, 1], &[0, 0, 0, 2])); + assert!(!interval_is_covered( + &parent, + &[0, 0, 0, 11], + &[0, 0, 0, 12] + )); + assert!(intervals_are_covered( + &parent, + &[(vec![0, 0, 0, 1], vec![0, 0, 0, 2])] + )); + assert!(!intervals_are_covered( + &parent, + &[(vec![0, 0, 0, 9], vec![0, 0, 0, 11])], + )); + + let prefix = RcIpPrefix { + afi: Afi::Ipv4, + prefix_len: 24, + addr: vec![203, 0, 113, 7], + }; + assert_eq!( + prefix_to_range(&prefix), + (vec![203, 0, 113, 0], vec![203, 0, 113, 255]) + ); + assert_eq!(increment_bytes(&[0, 0, 0, 255]), vec![0, 0, 1, 0]); + assert!(bytes_is_next(&[0, 0, 1, 0], &[0, 0, 0, 255])); + assert!(!bytes_is_next(&[1, 2], &[1])); + } + + #[test] + fn merged_interval_helpers_cover_empty_and_break_paths() { + let mut empty: Vec<(Vec, Vec)> = Vec::new(); + merge_ip_intervals_in_place(&mut empty); + assert!(empty.is_empty()); + + let mut v = vec![ + (vec![0, 0, 0, 20], vec![0, 0, 0, 30]), + (vec![0, 0, 0, 0], vec![0, 0, 0, 10]), + (vec![0, 0, 0, 11], vec![0, 0, 0, 19]), + ]; + v.sort_by(|(a, _), (b, _)| a.cmp(b)); + merge_ip_intervals_in_place(&mut v); + assert_eq!(v, vec![(vec![0, 0, 0, 0], vec![0, 0, 0, 30])]); + + assert_eq!( + as_choice_to_merged_intervals(&AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Id(1), + AsIdOrRange::Range { min: 2, max: 3 }, + AsIdOrRange::Range { min: 7, max: 9 }, + ])), + vec![(1, 3), (7, 9)] + ); + assert!(as_interval_is_covered(&[(1, 3), (7, 9)], 2, 3)); + assert!(!as_interval_is_covered(&[(7, 9)], 2, 3)); + } diff --git a/crates/panda-rpki-validator/src/validation/run_tree_from_tal.rs b/crates/panda-rpki-validator/src/validation/run_tree_from_tal.rs index 1cf2605..c37cd97 100644 --- a/crates/panda-rpki-validator/src/validation/run_tree_from_tal.rs +++ b/crates/panda-rpki-validator/src/validation/run_tree_from_tal.rs @@ -45,3001 +45,10 @@ use crate::validation::tree_runner::Rpkiv1PublicationPointRunner; use std::collections::HashMap; use std::sync::{Arc, Mutex}; -fn tal_id_from_url_like(s: &str) -> Option { - let url = Url::parse(s).ok()?; - if let Some(last) = url - .path_segments() - .and_then(|segments| segments.filter(|seg| !seg.is_empty()).next_back()) - { - let stem = last.rsplit_once('.').map(|(stem, _)| stem).unwrap_or(last); - let trimmed = stem.trim(); - if !trimmed.is_empty() { - return Some(trimmed.to_string()); - } - } - url.host_str().map(|host| host.to_string()) -} - -fn derive_tal_id(discovery: &DiscoveredRootCaInstance) -> String { - discovery - .tal_url - .as_deref() - .and_then(tal_id_from_url_like) - .or_else(|| { - discovery - .trust_anchor - .resolved_ta_uri - .as_ref() - .and_then(|uri| tal_id_from_url_like(uri.as_str())) - }) - .or_else(|| { - discovery - .trust_anchor - .tal - .ta_uris - .first() - .and_then(|uri| tal_id_from_url_like(uri.as_str())) - }) - .unwrap_or_else(|| "unknown-tal".to_string()) -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RunTreeFromTalOutput { - pub discovery: DiscoveredRootCaInstance, - pub tree: TreeRunOutput, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct RunTreeFromTalAuditOutput { - pub discovery: DiscoveredRootCaInstance, - pub discoveries: Vec, - pub successful_tal_inputs: Vec, - pub tree: TreeRunOutput, - pub publication_points: Vec, - pub roa_cache_stats: crate::validation::objects::RoaValidationCacheStats, - pub cir_input: crate::cir::CirInputSnapshot, - pub downloads: Vec, - pub download_stats: crate::audit::AuditDownloadStats, - pub current_repo_objects: Vec, - pub ccr_accumulator: Option, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct TalRootDiscovery { - pub tal_input: TalInputSpec, - pub discovery: DiscoveredRootCaInstance, - pub root_handle: CaInstanceHandle, -} - -fn snapshot_current_repo_objects( - current_repo_index: Option<&CurrentRepoIndexHandle>, - collect: bool, -) -> Vec { - if !collect { - return Vec::new(); - } - current_repo_index - .and_then(|handle| handle.read().ok().map(|idx| idx.snapshot_objects())) - .unwrap_or_default() -} - -fn make_live_runner<'a>( - store: &'a crate::storage::RocksStore, - policy: &'a crate::policy::Policy, - http_fetcher: &'a dyn Fetcher, - rsync_fetcher: &'a dyn crate::fetch::rsync::RsyncFetcher, - validation_time: time::OffsetDateTime, - timing: Option, - download_log: Option, - current_repo_index: Option, - repo_sync_runtime: Option>, - parallel_phase2_config: Option, - ccr_accumulator: Option, - persist_vcir: bool, - enable_roa_validation_cache: bool, - enable_child_certificate_validation_cache: bool, - publication_point_cache_observe_only: bool, - enable_publication_point_validation_cache: bool, -) -> Rpkiv1PublicationPointRunner<'a> { - let parallel_roa_worker_pool = parallel_phase2_config - .as_ref() - .and_then(|config| ParallelRoaWorkerPool::new(config).ok()); - Rpkiv1PublicationPointRunner { - store, - policy, - http_fetcher, - rsync_fetcher, - validation_time, - timing, - download_log, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: true, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: true, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index, - repo_sync_runtime, - parallel_phase2_config, - parallel_roa_worker_pool, - ccr_accumulator: ccr_accumulator.map(Mutex::new), - persist_vcir, - enable_roa_validation_cache, - enable_child_certificate_validation_cache, - publication_point_cache_observe_only, - enable_publication_point_validation_cache, - } -} - -fn build_phase1_repo_sync_runtime( - store: Arc, - policy: &crate::policy::Policy, - http_fetcher: &H, - rsync_fetcher: &R, - parallel_config: ParallelPhase1Config, - timing: Option, - download_log: Option, - tal_inputs: Vec, - record_transport_prefetch_requests: bool, -) -> Result<(Arc, CurrentRepoIndexHandle), RunTreeFromTalError> -where - H: Fetcher + Clone + 'static, - R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, -{ - let coordinator = GlobalRunCoordinator::new(parallel_config.clone(), tal_inputs); - let current_repo_index = coordinator.current_repo_index_handle(); - let rsync_fetcher_arc = Arc::new(rsync_fetcher.clone()); - let executor = LiveRepoTransportExecutor::new( - Arc::clone(&store), - current_repo_index.clone(), - Arc::new(http_fetcher.clone()), - Arc::clone(&rsync_fetcher_arc), - timing.clone(), - download_log, - ); - let pool = RepoTransportWorkerPool::new(RepoWorkerPoolConfig::from(¶llel_config), executor) - .map_err(RunTreeFromTalError::Replay)?; - let resolver_rsync_fetcher_arc = Arc::clone(&rsync_fetcher_arc); - let resolver: Arc String + Send + Sync> = - Arc::new(move |base: &str| resolver_rsync_fetcher_arc.dedup_key(base)); - let failure_rsync_fetcher_arc = Arc::clone(&rsync_fetcher_arc); - let failure_resolver: Arc Option + Send + Sync> = - Arc::new(move |base: &str| failure_rsync_fetcher_arc.failure_dedup_key(base)); - let _ = policy; // policy reserved for later runtime-level decisions - let runtime = Arc::new( - Phase1RepoSyncRuntime::new_with_failure_scope_and_prefetch_recording( - coordinator, - pool, - resolver, - failure_resolver, - policy.sync_preference, - record_transport_prefetch_requests, - ), - ); - Ok((runtime, current_repo_index)) -} - -fn record_transport_prefetch_count(timing: Option<&TimingHandle>, key: &'static str, value: u64) { - if value == 0 { - return; - } - if let Some(timing) = timing { - timing.record_count(key, value); - } -} - -fn apply_transport_request_prefetch( - store: &crate::storage::RocksStore, - runtime: &Arc, - sync_preference: SyncPreference, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, - timing: Option<&TimingHandle>, -) -> Result<(), RunTreeFromTalError> { - if !config.enable_transport_request_prefetch { - return Ok(()); - } - - let snapshot = match store.get_transport_prefetch_snapshot() { - Ok(Some(snapshot)) => snapshot, - Ok(None) => { - crate::progress_log::emit( - "phase1_repo_prefetch_missing_snapshot", - serde_json::json!({ "status": "missing" }), - ); - return Ok(()); - } - Err(err) => { - record_transport_prefetch_count(timing, "transport_prefetch_load_errors", 1); - crate::progress_log::emit( - "phase1_repo_prefetch_load_error", - serde_json::json!({ "error": err.to_string() }), - ); - return Ok(()); - } - }; - - if !snapshot.is_compatible_with(sync_preference) { - record_transport_prefetch_count( - timing, - "transport_prefetch_skipped_incompatible_snapshots", - 1, - ); - crate::progress_log::emit( - "phase1_repo_prefetch_skipped", - serde_json::json!({ - "reason": "incompatible_snapshot", - "schema_version": snapshot.schema_version, - "request_count": snapshot.requests.len(), - }), - ); - return Ok(()); - } - - let stats = runtime - .prefetch_transport_requests(&snapshot, validation_time) - .map_err(TreeRunError::Runner)?; - record_transport_prefetch_count( - timing.clone(), - "transport_prefetch_loaded_requests", - stats.loaded_requests, - ); - record_transport_prefetch_count( - timing.clone(), - "transport_prefetch_enqueued_tasks", - stats.enqueued_tasks, - ); - record_transport_prefetch_count( - timing, - "transport_prefetch_waiting_requests", - stats.waiting_requests, - ); - record_transport_prefetch_count( - timing, - "transport_prefetch_reused_results", - stats.reused_results, - ); - record_transport_prefetch_count( - timing, - "transport_prefetch_skipped_incompatible_requests", - stats.skipped_incompatible, - ); - crate::progress_log::emit( - "phase1_repo_prefetch_applied", - serde_json::json!({ - "loaded_requests": stats.loaded_requests, - "enqueued_tasks": stats.enqueued_tasks, - "waiting_requests": stats.waiting_requests, - "reused_results": stats.reused_results, - "skipped_incompatible": stats.skipped_incompatible, - }), - ); - Ok(()) -} - -fn persist_transport_request_prefetch_snapshot( - store: &crate::storage::RocksStore, - runtime: &Arc, - config: &TreeRunConfig, - timing: Option<&TimingHandle>, -) -> Result<(), RunTreeFromTalError> { - if !config.enable_transport_request_prefetch { - return Ok(()); - } - let snapshot = runtime.transport_prefetch_snapshot(); - let recorded = snapshot.requests.len() as u64; - let persist_started = std::time::Instant::now(); - store - .put_transport_prefetch_snapshot(&snapshot) - .map_err(|err| TreeRunError::Runner(err.to_string()))?; - let persist_ms = persist_started.elapsed().as_millis() as u64; - record_transport_prefetch_count(timing, "transport_prefetch_recorded_requests", recorded); - crate::progress_log::emit( - "phase1_repo_prefetch_persisted", - serde_json::json!({ - "recorded_requests": recorded, - "schema_version": snapshot.schema_version, - "persist_ms": persist_ms, - }), - ); - Ok(()) -} - -/// Persist the dead-repo blacklist working copy at run end (#141). Failures -/// degrade to a progress event; a broken blacklist file must never fail a run. -fn persist_dead_repo_blacklist( - runtime: &Arc, - timing: Option<&TimingHandle>, -) -> Result<(), RunTreeFromTalError> { - let Some((blacklist_config, blacklist)) = runtime.dead_repo_blacklist_state() else { - return Ok(()); - }; - let persist_started = std::time::Instant::now(); - let now_unix = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|duration| duration.as_secs()) - .unwrap_or(0); - match blacklist.store_atomic(&blacklist_config.path, now_unix) { - Ok(()) => { - let persist_ms = persist_started.elapsed().as_millis() as u64; - record_transport_prefetch_count( - timing, - "dead_repo_blacklist_entries", - blacklist.len() as u64, - ); - record_transport_prefetch_count( - timing, - "dead_repo_blacklist_blacklisted", - blacklist.blacklisted_len() as u64, - ); - crate::progress_log::emit( - "dead_repo_blacklist_persisted", - serde_json::json!({ - "path": blacklist_config.path.display().to_string(), - "entries": blacklist.len(), - "blacklisted": blacklist.blacklisted_len(), - "persist_ms": persist_ms, - }), - ); - } - Err(err) => { - crate::progress_log::emit( - "dead_repo_blacklist_persist_error", - serde_json::json!({ - "path": blacklist_config.path.display().to_string(), - "error": err.to_string(), - }), - ); - } - } - Ok(()) -} - -fn root_discovery_from_tal_input( - tal_input: &TalInputSpec, - http_fetcher: &dyn Fetcher, - rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, - strict_name: bool, -) -> Result { - match &tal_input.source { - TalSource::Url(url) => { - if strict_name { - discover_root_ca_instance_from_tal_url_with_fetchers_strict_name( - http_fetcher, - rsync_fetcher, - url, - ) - } else { - discover_root_ca_instance_from_tal_url_with_fetchers( - http_fetcher, - rsync_fetcher, - url, - ) - } - } - TalSource::DerBytes { - tal_bytes, ta_der, .. - } => { - if strict_name { - discover_root_ca_instance_from_tal_and_ta_der_with_strict_name( - tal_bytes, ta_der, None, - ) - } else { - discover_root_ca_instance_from_tal_and_ta_der(tal_bytes, ta_der, None) - } - } - TalSource::FilePath(path) => { - let tal_bytes = std::fs::read(path).map_err(|e| { - FromTalError::TalFetch(format!("read TAL file failed: {}: {e}", path.display())) - })?; - let tal = crate::data_model::tal::Tal::decode_bytes(&tal_bytes) - .map_err(FromTalError::from)?; - if strict_name { - discover_root_ca_instance_from_tal_with_fetchers_strict_name( - http_fetcher, - rsync_fetcher, - tal, - None, - ) - } else { - discover_root_ca_instance_from_tal_with_fetchers( - http_fetcher, - rsync_fetcher, - tal, - None, - ) - } - } - TalSource::FilePathWithTa { tal_path, ta_path } => { - let tal_bytes = std::fs::read(tal_path).map_err(|e| { - FromTalError::TalFetch(format!("read TAL file failed: {}: {e}", tal_path.display())) - })?; - let ta_der = std::fs::read(ta_path).map_err(|e| { - FromTalError::TaFetch(format!("read TA file failed: {}: {e}", ta_path.display())) - })?; - let resolved_ta_uri = canonical_tal_rsync_uri_from_bytes(&tal_bytes)?; - if strict_name { - discover_root_ca_instance_from_tal_and_ta_der_with_strict_name( - &tal_bytes, - &ta_der, - Some(&resolved_ta_uri), - ) - } else { - discover_root_ca_instance_from_tal_and_ta_der( - &tal_bytes, - &ta_der, - Some(&resolved_ta_uri), - ) - } - } - } -} - -fn discover_root_ca_instance_from_tal_url_with_policy( - policy: &crate::policy::Policy, - http_fetcher: &dyn Fetcher, - rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, - tal_url: &str, -) -> Result { - if policy.strict.name { - discover_root_ca_instance_from_tal_url_with_fetchers_strict_name( - http_fetcher, - rsync_fetcher, - tal_url, - ) - } else { - discover_root_ca_instance_from_tal_url_with_fetchers(http_fetcher, rsync_fetcher, tal_url) - } -} - -fn discover_root_ca_instance_from_tal_and_ta_der_with_policy( - policy: &crate::policy::Policy, - tal_bytes: &[u8], - ta_der: &[u8], - resolved_ta_uri: Option<&Url>, -) -> Result { - if policy.strict.name { - discover_root_ca_instance_from_tal_and_ta_der_with_strict_name( - tal_bytes, - ta_der, - resolved_ta_uri, - ) - } else { - discover_root_ca_instance_from_tal_and_ta_der(tal_bytes, ta_der, resolved_ta_uri) - } -} - -fn discover_multiple_roots_from_tal_inputs( - tal_inputs: &[TalInputSpec], - http_fetcher: &dyn Fetcher, - rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, - strict_name: bool, -) -> Result, RunTreeFromTalError> { - let mut roots = Vec::with_capacity(tal_inputs.len()); - for tal_input in tal_inputs { - let discovery = match root_discovery_from_tal_input( - tal_input, - http_fetcher, - rsync_fetcher, - strict_name, - ) { - Ok(discovery) => discovery, - Err(error) - if should_isolate_multi_tal_strict_name_failure( - tal_inputs.len(), - strict_name, - &error, - ) => - { - eprintln!( - "warning: skipping TAL '{}' because strict name validation failed during trust anchor discovery: {error}", - tal_input.tal_id - ); - continue; - } - Err(error) => return Err(error.into()), - }; - let root_handle = root_handle_from_trust_anchor( - &discovery.trust_anchor, - tal_input.tal_id.clone(), - None, - &discovery.ca_instance, - ); - roots.push(TalRootDiscovery { - tal_input: tal_input.clone(), - discovery, - root_handle, - }); - } - if roots.is_empty() { - return Err(RunTreeFromTalError::Replay( - "multi-TAL root discovery returned no usable roots after strict name filtering" - .to_string(), - )); - } - Ok(roots) -} - -fn should_isolate_multi_tal_strict_name_failure( - tal_input_count: usize, - strict_name: bool, - error: &FromTalError, -) -> bool { - strict_name - && tal_input_count > 1 - && error.to_string().contains("Name strict validation failed") -} - -#[derive(Debug, thiserror::Error)] -pub enum RunTreeFromTalError { - #[error("{0}")] - FromTal(#[from] FromTalError), - - #[error("payload replay setup failed: {0}")] - Replay(String), - - #[error("{0}")] - Tree(#[from] TreeRunError), -} - -pub fn root_handle_from_trust_anchor( - trust_anchor: &TrustAnchor, - tal_id: String, - ca_certificate_rsync_uri: Option, - ca_instance: &crate::validation::ca_instance::CaInstanceUris, -) -> CaInstanceHandle { - let ta_rc = trust_anchor.ta_certificate.rc_ca.clone(); - CaInstanceHandle { - depth: 0, - tal_id, - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(trust_anchor.ta_certificate.raw_der.clone()), - ca_certificate_rsync_uri, - effective_ip_resources: ta_rc.tbs.extensions.ip_resources.clone(), - effective_as_resources: ta_rc.tbs.extensions.as_resources.clone(), - rsync_base_uri: ca_instance.rsync_base_uri.clone(), - manifest_rsync_uri: ca_instance.manifest_rsync_uri.clone(), - publication_point_rsync_uri: ca_instance.publication_point_rsync_uri.clone(), - rrdp_notification_uri: ca_instance.rrdp_notification_uri.clone(), - } -} - -pub fn run_tree_from_tal_url_serial( - store: &crate::storage::RocksStore, - policy: &crate::policy::Policy, - tal_url: &str, - http_fetcher: &dyn Fetcher, - rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, -) -> Result { - let discovery = discover_root_ca_instance_from_tal_url_with_policy( - policy, - http_fetcher, - rsync_fetcher, - tal_url, - )?; - - let runner = make_live_runner( - store, - policy, - http_fetcher, - rsync_fetcher, - validation_time, - None, - None, - None, - None, - None, - None, - config.persist_vcir, - config.enable_roa_validation_cache, - config.enable_child_certificate_validation_cache, - config.publication_point_cache_observe_only, - config.enable_publication_point_validation_cache, - ); - - let root = root_handle_from_trust_anchor( - &discovery.trust_anchor, - derive_tal_id(&discovery), - None, - &discovery.ca_instance, - ); - let tree = run_tree_serial(root, &runner, config)?; - - Ok(RunTreeFromTalOutput { discovery, tree }) -} - -pub fn run_tree_from_tal_url_serial_audit( - store: &crate::storage::RocksStore, - policy: &crate::policy::Policy, - tal_url: &str, - http_fetcher: &dyn Fetcher, - rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, -) -> Result { - let discovery = discover_root_ca_instance_from_tal_url_with_policy( - policy, - http_fetcher, - rsync_fetcher, - tal_url, - )?; - - let download_log = DownloadLogHandle::new(); - let runner = make_live_runner( - store, - policy, - http_fetcher, - rsync_fetcher, - validation_time, - None, - Some(download_log.clone()), - None, - None, - None, - None, - config.persist_vcir, - config.enable_roa_validation_cache, - config.enable_child_certificate_validation_cache, - config.publication_point_cache_observe_only, - config.enable_publication_point_validation_cache, - ); - - let root = root_handle_from_trust_anchor( - &discovery.trust_anchor, - derive_tal_id(&discovery), - None, - &discovery.ca_instance, - ); - let TreeRunAuditOutput { - tree, - publication_points, - roa_cache_stats, - cir_input, - } = run_tree_serial_audit(root, &runner, config)?; - - let downloads = download_log.snapshot_events(); - let download_stats = DownloadLogHandle::stats_from_events(&downloads); - Ok(RunTreeFromTalAuditOutput { - discovery: discovery.clone(), - discoveries: vec![discovery], - successful_tal_inputs: Vec::new(), - tree, - publication_points, - roa_cache_stats, - cir_input, - downloads, - download_stats, - current_repo_objects: Vec::new(), - ccr_accumulator: None, - }) -} - -pub fn run_tree_from_tal_url_serial_audit_with_timing( - store: &crate::storage::RocksStore, - policy: &crate::policy::Policy, - tal_url: &str, - http_fetcher: &dyn Fetcher, - rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, - timing: &TimingHandle, -) -> Result { - let _tal = timing.span_phase("tal_bootstrap"); - let discovery = discover_root_ca_instance_from_tal_url_with_policy( - policy, - http_fetcher, - rsync_fetcher, - tal_url, - )?; - drop(_tal); - - let download_log = DownloadLogHandle::new(); - let runner = make_live_runner( - store, - policy, - http_fetcher, - rsync_fetcher, - validation_time, - Some(timing.clone()), - Some(download_log.clone()), - None, - None, - None, - None, - config.persist_vcir, - config.enable_roa_validation_cache, - config.enable_child_certificate_validation_cache, - config.publication_point_cache_observe_only, - config.enable_publication_point_validation_cache, - ); - - let root = root_handle_from_trust_anchor( - &discovery.trust_anchor, - derive_tal_id(&discovery), - None, - &discovery.ca_instance, - ); - let _tree = timing.span_phase("tree_run_total"); - let TreeRunAuditOutput { - tree, - publication_points, - roa_cache_stats, - cir_input, - } = run_tree_serial_audit(root, &runner, config)?; - - let downloads = download_log.snapshot_events(); - let download_stats = DownloadLogHandle::stats_from_events(&downloads); - Ok(RunTreeFromTalAuditOutput { - discovery: discovery.clone(), - discoveries: vec![discovery], - successful_tal_inputs: Vec::new(), - tree, - publication_points, - roa_cache_stats, - cir_input, - downloads, - download_stats, - current_repo_objects: Vec::new(), - ccr_accumulator: None, - }) -} - -fn run_single_root_parallel_audit_inner( - store: Arc, - policy: &crate::policy::Policy, - discovery: DiscoveredRootCaInstance, - tal_inputs: Vec, - http_fetcher: &H, - rsync_fetcher: &R, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, - parallel_config: ParallelPhase1Config, - phase2_config: Option, - collect_current_repo_objects: bool, - timing: Option, -) -> Result -where - H: Fetcher + Clone + 'static, - R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, -{ - let phase2_enabled = phase2_config.is_some(); - let download_log = DownloadLogHandle::new(); - let (runtime, current_repo_index) = build_phase1_repo_sync_runtime( - Arc::clone(&store), - policy, - http_fetcher, - rsync_fetcher, - parallel_config, - timing.clone(), - Some(download_log.clone()), - tal_inputs, - config.enable_transport_request_prefetch, - )?; - apply_transport_request_prefetch( - store.as_ref(), - &runtime, - policy.sync_preference, - validation_time, - config, - timing.as_ref(), - )?; - let current_repo_index_for_output = current_repo_index.clone(); - let runner = make_live_runner( - store.as_ref(), - policy, - http_fetcher, - rsync_fetcher, - validation_time, - timing.clone(), - Some(download_log.clone()), - Some(current_repo_index), - Some(Arc::clone(&runtime)), - phase2_config, - (phase2_enabled && config.build_ccr_accumulator) - .then(|| CcrAccumulator::new(vec![discovery.trust_anchor.clone()])), - config.persist_vcir, - config.enable_roa_validation_cache, - config.enable_child_certificate_validation_cache, - config.publication_point_cache_observe_only, - config.enable_publication_point_validation_cache, - ); - - let root = root_handle_from_trust_anchor( - &discovery.trust_anchor, - derive_tal_id(&discovery), - None, - &discovery.ca_instance, - ); - let TreeRunAuditOutput { - tree, - publication_points, - roa_cache_stats, - cir_input, - } = if phase2_enabled { - run_tree_parallel_phase2_audit(root, &runner, config)? - } else { - run_tree_serial_audit(root, &runner, config)? - }; - persist_transport_request_prefetch_snapshot(store.as_ref(), &runtime, config, timing.as_ref())?; - persist_dead_repo_blacklist(&runtime, timing.as_ref())?; - let downloads = download_log.snapshot_events(); - let download_stats = DownloadLogHandle::stats_from_events(&downloads); - Ok(RunTreeFromTalAuditOutput { - discovery: discovery.clone(), - discoveries: vec![discovery], - successful_tal_inputs: Vec::new(), - tree, - publication_points, - roa_cache_stats, - cir_input, - downloads, - download_stats, - current_repo_objects: snapshot_current_repo_objects( - Some(¤t_repo_index_for_output), - collect_current_repo_objects, - ), - ccr_accumulator: runner.ccr_accumulator_snapshot(), - }) -} - -fn run_multi_root_parallel_audit_inner( - store: Arc, - policy: &crate::policy::Policy, - tal_inputs: Vec, - http_fetcher: &H, - rsync_fetcher: &R, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, - parallel_config: ParallelPhase1Config, - phase2_config: Option, - collect_current_repo_objects: bool, - timing: Option, -) -> Result -where - H: Fetcher + Clone + 'static, - R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, -{ - // Constraints are an immutable per-run policy snapshot. The phase-2 - // ready-stage binds the snapshot to each CA's TAL and moves an Arc into - // ROA/ASPA worker state, so constrained multi-TAL runs retain the same - // parallel scheduler as unconstrained runs. - let phase2_enabled = phase2_config.is_some(); - if tal_inputs.is_empty() { - return Err(RunTreeFromTalError::Replay( - "multi-TAL run requires at least one TAL input".to_string(), - )); - } - let roots = discover_multiple_roots_from_tal_inputs( - &tal_inputs, - http_fetcher, - rsync_fetcher, - policy.strict.name, - )?; - let primary = roots.first().cloned().ok_or_else(|| { - RunTreeFromTalError::Replay("multi-TAL root discovery returned no roots".to_string()) - })?; - let discoveries = roots - .iter() - .map(|item| item.discovery.clone()) - .collect::>(); - let successful_tal_inputs = roots - .iter() - .map(|item| item.tal_input.clone()) - .collect::>(); - let root_handles = roots - .iter() - .map(|item| item.root_handle.clone()) - .collect::>(); - - let download_log = DownloadLogHandle::new(); - let (runtime, current_repo_index) = build_phase1_repo_sync_runtime( - Arc::clone(&store), - policy, - http_fetcher, - rsync_fetcher, - parallel_config, - timing.clone(), - Some(download_log.clone()), - successful_tal_inputs.clone(), - config.enable_transport_request_prefetch, - )?; - apply_transport_request_prefetch( - store.as_ref(), - &runtime, - policy.sync_preference, - validation_time, - config, - timing.as_ref(), - )?; - let current_repo_index_for_output = current_repo_index.clone(); - let runner = make_live_runner( - store.as_ref(), - policy, - http_fetcher, - rsync_fetcher, - validation_time, - timing.clone(), - Some(download_log.clone()), - Some(current_repo_index), - Some(Arc::clone(&runtime)), - phase2_config, - (phase2_enabled && config.build_ccr_accumulator).then(|| { - CcrAccumulator::new( - discoveries - .iter() - .map(|item| item.trust_anchor.clone()) - .collect::>(), - ) - }), - config.persist_vcir, - config.enable_roa_validation_cache, - config.enable_child_certificate_validation_cache, - config.publication_point_cache_observe_only, - config.enable_publication_point_validation_cache, - ); - - let TreeRunAuditOutput { - tree, - publication_points, - roa_cache_stats, - cir_input, - } = if phase2_enabled { - run_tree_parallel_phase2_audit_multi_root(root_handles, &runner, config)? - } else { - run_tree_serial_audit_multi_root(root_handles, &runner, config)? - }; - persist_transport_request_prefetch_snapshot(store.as_ref(), &runtime, config, timing.as_ref())?; - persist_dead_repo_blacklist(&runtime, timing.as_ref())?; - let downloads = download_log.snapshot_events(); - let download_stats = DownloadLogHandle::stats_from_events(&downloads); - Ok(RunTreeFromTalAuditOutput { - discovery: primary.discovery.clone(), - discoveries, - successful_tal_inputs, - tree, - publication_points, - roa_cache_stats, - cir_input, - downloads, - download_stats, - current_repo_objects: snapshot_current_repo_objects( - Some(¤t_repo_index_for_output), - collect_current_repo_objects, - ), - ccr_accumulator: runner.ccr_accumulator_snapshot(), - }) -} - -pub fn run_tree_from_tal_url_parallel_phase1_audit( - store: Arc, - policy: &crate::policy::Policy, - tal_url: &str, - http_fetcher: &H, - rsync_fetcher: &R, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, - parallel_config: ParallelPhase1Config, - collect_current_repo_objects: bool, -) -> Result -where - H: Fetcher + Clone + 'static, - R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, -{ - let discovery = discover_root_ca_instance_from_tal_url_with_policy( - policy, - http_fetcher, - rsync_fetcher, - tal_url, - )?; - run_single_root_parallel_audit_inner( - store, - policy, - discovery, - vec![TalInputSpec::from_url(tal_url.to_string())], - http_fetcher, - rsync_fetcher, - validation_time, - config, - parallel_config, - None, - collect_current_repo_objects, - None, - ) -} - -pub fn run_tree_from_tal_and_ta_der_parallel_phase1_audit( - store: Arc, - policy: &crate::policy::Policy, - tal_bytes: &[u8], - ta_der: &[u8], - resolved_ta_uri: Option<&url::Url>, - http_fetcher: &H, - rsync_fetcher: &R, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, - parallel_config: ParallelPhase1Config, - collect_current_repo_objects: bool, -) -> Result -where - H: Fetcher + Clone + 'static, - R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, -{ - let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( - policy, - tal_bytes, - ta_der, - resolved_ta_uri, - )?; - let derived_tal_id = derive_tal_id(&discovery); - let tal_inputs = vec![TalInputSpec { - tal_id: derived_tal_id.clone(), - rir_id: derived_tal_id, - source: TalSource::DerBytes { - tal_url: discovery - .tal_url - .clone() - .unwrap_or_else(|| "embedded-tal".to_string()), - tal_bytes: tal_bytes.to_vec(), - ta_der: ta_der.to_vec(), - }, - }]; - run_single_root_parallel_audit_inner( - store, - policy, - discovery, - tal_inputs, - http_fetcher, - rsync_fetcher, - validation_time, - config, - parallel_config, - None, - collect_current_repo_objects, - None, - ) -} - -pub fn run_tree_from_multiple_tals_parallel_phase1_audit( - store: Arc, - policy: &crate::policy::Policy, - tal_inputs: Vec, - http_fetcher: &H, - rsync_fetcher: &R, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, - parallel_config: ParallelPhase1Config, - collect_current_repo_objects: bool, -) -> Result -where - H: Fetcher + Clone + 'static, - R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, -{ - run_multi_root_parallel_audit_inner( - store, - policy, - tal_inputs, - http_fetcher, - rsync_fetcher, - validation_time, - config, - parallel_config, - None, - collect_current_repo_objects, - None, - ) -} - -pub fn run_tree_from_tal_url_parallel_phase2_audit( - store: Arc, - policy: &crate::policy::Policy, - tal_url: &str, - http_fetcher: &H, - rsync_fetcher: &R, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, - parallel_config: ParallelPhase1Config, - phase2_config: ParallelPhase2Config, - collect_current_repo_objects: bool, -) -> Result -where - H: Fetcher + Clone + 'static, - R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, -{ - let discovery = discover_root_ca_instance_from_tal_url_with_policy( - policy, - http_fetcher, - rsync_fetcher, - tal_url, - )?; - run_single_root_parallel_audit_inner( - store, - policy, - discovery, - vec![TalInputSpec::from_url(tal_url.to_string())], - http_fetcher, - rsync_fetcher, - validation_time, - config, - parallel_config, - Some(phase2_config), - collect_current_repo_objects, - None, - ) -} - -pub fn run_tree_from_tal_url_parallel_phase2_audit_with_timing( - store: Arc, - policy: &crate::policy::Policy, - tal_url: &str, - http_fetcher: &H, - rsync_fetcher: &R, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, - parallel_config: ParallelPhase1Config, - phase2_config: ParallelPhase2Config, - collect_current_repo_objects: bool, - timing: &TimingHandle, -) -> Result -where - H: Fetcher + Clone + 'static, - R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, -{ - let discovery = discover_root_ca_instance_from_tal_url_with_policy( - policy, - http_fetcher, - rsync_fetcher, - tal_url, - )?; - run_single_root_parallel_audit_inner( - store, - policy, - discovery, - vec![TalInputSpec::from_url(tal_url.to_string())], - http_fetcher, - rsync_fetcher, - validation_time, - config, - parallel_config, - Some(phase2_config), - collect_current_repo_objects, - Some(timing.clone()), - ) -} - -pub fn run_tree_from_tal_and_ta_der_parallel_phase2_audit( - store: Arc, - policy: &crate::policy::Policy, - tal_bytes: &[u8], - ta_der: &[u8], - resolved_ta_uri: Option<&url::Url>, - http_fetcher: &H, - rsync_fetcher: &R, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, - parallel_config: ParallelPhase1Config, - phase2_config: ParallelPhase2Config, - collect_current_repo_objects: bool, -) -> Result -where - H: Fetcher + Clone + 'static, - R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, -{ - let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( - policy, - tal_bytes, - ta_der, - resolved_ta_uri, - )?; - let derived_tal_id = derive_tal_id(&discovery); - let tal_inputs = vec![TalInputSpec { - tal_id: derived_tal_id.clone(), - rir_id: derived_tal_id, - source: TalSource::DerBytes { - tal_url: discovery - .tal_url - .clone() - .unwrap_or_else(|| "embedded-tal".to_string()), - tal_bytes: tal_bytes.to_vec(), - ta_der: ta_der.to_vec(), - }, - }]; - run_single_root_parallel_audit_inner( - store, - policy, - discovery, - tal_inputs, - http_fetcher, - rsync_fetcher, - validation_time, - config, - parallel_config, - Some(phase2_config), - collect_current_repo_objects, - None, - ) -} - -pub fn run_tree_from_tal_and_ta_der_parallel_phase2_audit_with_timing( - store: Arc, - policy: &crate::policy::Policy, - tal_bytes: &[u8], - ta_der: &[u8], - resolved_ta_uri: Option<&url::Url>, - http_fetcher: &H, - rsync_fetcher: &R, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, - parallel_config: ParallelPhase1Config, - phase2_config: ParallelPhase2Config, - collect_current_repo_objects: bool, - timing: &TimingHandle, -) -> Result -where - H: Fetcher + Clone + 'static, - R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, -{ - let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( - policy, - tal_bytes, - ta_der, - resolved_ta_uri, - )?; - let derived_tal_id = derive_tal_id(&discovery); - let tal_inputs = vec![TalInputSpec { - tal_id: derived_tal_id.clone(), - rir_id: derived_tal_id, - source: TalSource::DerBytes { - tal_url: discovery - .tal_url - .clone() - .unwrap_or_else(|| "embedded-tal".to_string()), - tal_bytes: tal_bytes.to_vec(), - ta_der: ta_der.to_vec(), - }, - }]; - run_single_root_parallel_audit_inner( - store, - policy, - discovery, - tal_inputs, - http_fetcher, - rsync_fetcher, - validation_time, - config, - parallel_config, - Some(phase2_config), - collect_current_repo_objects, - Some(timing.clone()), - ) -} - -pub fn run_tree_from_multiple_tals_parallel_phase2_audit( - store: Arc, - policy: &crate::policy::Policy, - tal_inputs: Vec, - http_fetcher: &H, - rsync_fetcher: &R, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, - parallel_config: ParallelPhase1Config, - phase2_config: ParallelPhase2Config, - collect_current_repo_objects: bool, -) -> Result -where - H: Fetcher + Clone + 'static, - R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, -{ - run_multi_root_parallel_audit_inner( - store, - policy, - tal_inputs, - http_fetcher, - rsync_fetcher, - validation_time, - config, - parallel_config, - Some(phase2_config), - collect_current_repo_objects, - None, - ) -} - -pub fn run_tree_from_multiple_tals_parallel_phase2_audit_with_timing( - store: Arc, - policy: &crate::policy::Policy, - tal_inputs: Vec, - http_fetcher: &H, - rsync_fetcher: &R, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, - parallel_config: ParallelPhase1Config, - phase2_config: ParallelPhase2Config, - collect_current_repo_objects: bool, - timing: &TimingHandle, -) -> Result -where - H: Fetcher + Clone + 'static, - R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, -{ - run_multi_root_parallel_audit_inner( - store, - policy, - tal_inputs, - http_fetcher, - rsync_fetcher, - validation_time, - config, - parallel_config, - Some(phase2_config), - collect_current_repo_objects, - Some(timing.clone()), - ) -} - -pub fn run_tree_from_tal_and_ta_der_serial( - store: &crate::storage::RocksStore, - policy: &crate::policy::Policy, - tal_bytes: &[u8], - ta_der: &[u8], - resolved_ta_uri: Option<&Url>, - http_fetcher: &dyn Fetcher, - rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, -) -> Result { - let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( - policy, - tal_bytes, - ta_der, - resolved_ta_uri, - )?; - - let runner = Rpkiv1PublicationPointRunner { - store, - policy, - http_fetcher, - rsync_fetcher, - validation_time, - timing: None, - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: true, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: true, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: config.enable_roa_validation_cache, - enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache, - publication_point_cache_observe_only: config.publication_point_cache_observe_only, - enable_publication_point_validation_cache: config.enable_publication_point_validation_cache, - }; - - let root = root_handle_from_trust_anchor( - &discovery.trust_anchor, - derive_tal_id(&discovery), - None, - &discovery.ca_instance, - ); - let tree = run_tree_serial(root, &runner, config)?; - - Ok(RunTreeFromTalOutput { discovery, tree }) -} - -pub fn run_tree_from_tal_bytes_serial_audit( - store: &crate::storage::RocksStore, - policy: &crate::policy::Policy, - tal_bytes: &[u8], - tal_uri: Option, - http_fetcher: &dyn Fetcher, - rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, -) -> Result { - let tal = crate::data_model::tal::Tal::decode_bytes(tal_bytes).map_err(FromTalError::from)?; - let discovery = discover_root_ca_instance_from_tal_with_fetchers( - http_fetcher, - rsync_fetcher, - tal, - tal_uri, - )?; - - let download_log = DownloadLogHandle::new(); - let runner = Rpkiv1PublicationPointRunner { - store, - policy, - http_fetcher, - rsync_fetcher, - validation_time, - timing: None, - download_log: Some(download_log.clone()), - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: true, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: true, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: config.enable_roa_validation_cache, - enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache, - publication_point_cache_observe_only: config.publication_point_cache_observe_only, - enable_publication_point_validation_cache: config.enable_publication_point_validation_cache, - }; - - let root = root_handle_from_trust_anchor( - &discovery.trust_anchor, - derive_tal_id(&discovery), - None, - &discovery.ca_instance, - ); - let TreeRunAuditOutput { - tree, - publication_points, - roa_cache_stats, - cir_input, - } = run_tree_serial_audit(root, &runner, config)?; - - let downloads = download_log.snapshot_events(); - let download_stats = DownloadLogHandle::stats_from_events(&downloads); - Ok(RunTreeFromTalAuditOutput { - discovery: discovery.clone(), - discoveries: vec![discovery], - successful_tal_inputs: Vec::new(), - tree, - publication_points, - roa_cache_stats, - cir_input, - downloads, - download_stats, - current_repo_objects: Vec::new(), - ccr_accumulator: None, - }) -} - -pub fn run_tree_from_tal_bytes_serial_audit_with_timing( - store: &crate::storage::RocksStore, - policy: &crate::policy::Policy, - tal_bytes: &[u8], - tal_uri: Option, - http_fetcher: &dyn Fetcher, - rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, - timing: &TimingHandle, -) -> Result { - let _tal = timing.span_phase("tal_bootstrap"); - let tal = crate::data_model::tal::Tal::decode_bytes(tal_bytes).map_err(FromTalError::from)?; - let discovery = discover_root_ca_instance_from_tal_with_fetchers( - http_fetcher, - rsync_fetcher, - tal, - tal_uri, - )?; - drop(_tal); - - let download_log = DownloadLogHandle::new(); - let runner = Rpkiv1PublicationPointRunner { - store, - policy, - http_fetcher, - rsync_fetcher, - validation_time, - timing: Some(timing.clone()), - download_log: Some(download_log.clone()), - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: true, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: true, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: config.enable_roa_validation_cache, - enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache, - publication_point_cache_observe_only: config.publication_point_cache_observe_only, - enable_publication_point_validation_cache: config.enable_publication_point_validation_cache, - }; - - let root = root_handle_from_trust_anchor( - &discovery.trust_anchor, - derive_tal_id(&discovery), - None, - &discovery.ca_instance, - ); - let _tree = timing.span_phase("tree_run"); - let TreeRunAuditOutput { - tree, - publication_points, - roa_cache_stats, - cir_input, - } = run_tree_serial_audit(root, &runner, config)?; - drop(_tree); - - let downloads = download_log.snapshot_events(); - let download_stats = DownloadLogHandle::stats_from_events(&downloads); - Ok(RunTreeFromTalAuditOutput { - discovery: discovery.clone(), - discoveries: vec![discovery], - successful_tal_inputs: Vec::new(), - tree, - publication_points, - roa_cache_stats, - cir_input, - downloads, - download_stats, - current_repo_objects: Vec::new(), - ccr_accumulator: None, - }) -} - -pub fn run_tree_from_tal_and_ta_der_serial_audit( - store: &crate::storage::RocksStore, - policy: &crate::policy::Policy, - tal_bytes: &[u8], - ta_der: &[u8], - resolved_ta_uri: Option<&Url>, - http_fetcher: &dyn Fetcher, - rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, -) -> Result { - let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( - policy, - tal_bytes, - ta_der, - resolved_ta_uri, - )?; - - let download_log = DownloadLogHandle::new(); - let runner = Rpkiv1PublicationPointRunner { - store, - policy, - http_fetcher, - rsync_fetcher, - validation_time, - timing: None, - download_log: Some(download_log.clone()), - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: true, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: true, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: config.enable_roa_validation_cache, - enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache, - publication_point_cache_observe_only: config.publication_point_cache_observe_only, - enable_publication_point_validation_cache: config.enable_publication_point_validation_cache, - }; - - let root = root_handle_from_trust_anchor( - &discovery.trust_anchor, - derive_tal_id(&discovery), - None, - &discovery.ca_instance, - ); - let TreeRunAuditOutput { - tree, - publication_points, - roa_cache_stats, - cir_input, - } = run_tree_serial_audit(root, &runner, config)?; - - let downloads = download_log.snapshot_events(); - let download_stats = DownloadLogHandle::stats_from_events(&downloads); - Ok(RunTreeFromTalAuditOutput { - discovery: discovery.clone(), - discoveries: vec![discovery], - successful_tal_inputs: Vec::new(), - tree, - publication_points, - roa_cache_stats, - cir_input, - downloads, - download_stats, - current_repo_objects: Vec::new(), - ccr_accumulator: None, - }) -} - -pub fn run_tree_from_tal_and_ta_der_serial_audit_with_timing( - store: &crate::storage::RocksStore, - policy: &crate::policy::Policy, - tal_bytes: &[u8], - ta_der: &[u8], - resolved_ta_uri: Option<&Url>, - http_fetcher: &dyn Fetcher, - rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, - timing: &TimingHandle, -) -> Result { - let _tal = timing.span_phase("tal_bootstrap"); - let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( - policy, - tal_bytes, - ta_der, - resolved_ta_uri, - )?; - drop(_tal); - - let download_log = DownloadLogHandle::new(); - let runner = Rpkiv1PublicationPointRunner { - store, - policy, - http_fetcher, - rsync_fetcher, - validation_time, - timing: Some(timing.clone()), - download_log: Some(download_log.clone()), - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: true, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: true, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: config.enable_roa_validation_cache, - enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache, - publication_point_cache_observe_only: config.publication_point_cache_observe_only, - enable_publication_point_validation_cache: config.enable_publication_point_validation_cache, - }; - - let root = root_handle_from_trust_anchor( - &discovery.trust_anchor, - derive_tal_id(&discovery), - None, - &discovery.ca_instance, - ); - let _tree = timing.span_phase("tree_run_total"); - let TreeRunAuditOutput { - tree, - publication_points, - roa_cache_stats, - cir_input, - } = run_tree_serial_audit(root, &runner, config)?; - - let downloads = download_log.snapshot_events(); - let download_stats = DownloadLogHandle::stats_from_events(&downloads); - Ok(RunTreeFromTalAuditOutput { - discovery: discovery.clone(), - discoveries: vec![discovery], - successful_tal_inputs: Vec::new(), - tree, - publication_points, - roa_cache_stats, - cir_input, - downloads, - download_stats, - current_repo_objects: Vec::new(), - ccr_accumulator: None, - }) -} - -pub fn run_tree_from_tal_and_ta_der_payload_replay_serial( - store: &crate::storage::RocksStore, - policy: &crate::policy::Policy, - tal_bytes: &[u8], - ta_der: &[u8], - resolved_ta_uri: Option<&Url>, - payload_archive_root: &std::path::Path, - payload_locks_path: &std::path::Path, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, -) -> Result { - let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( - policy, - tal_bytes, - ta_der, - resolved_ta_uri, - )?; - let replay_index = Arc::new( - ReplayArchiveIndex::load_allow_missing_rsync_modules( - payload_archive_root, - payload_locks_path, - ) - .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?, - ); - let http_fetcher = PayloadReplayHttpFetcher::new(replay_index.clone()) - .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?; - let rsync_fetcher = PayloadReplayRsyncFetcher::new(replay_index.clone()); - - let runner = Rpkiv1PublicationPointRunner { - store, - policy, - http_fetcher: &http_fetcher, - rsync_fetcher: &rsync_fetcher, - validation_time, - timing: None, - download_log: None, - replay_archive_index: Some(replay_index), - replay_delta_index: None, - rrdp_dedup: true, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: true, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: config.enable_roa_validation_cache, - enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache, - publication_point_cache_observe_only: config.publication_point_cache_observe_only, - enable_publication_point_validation_cache: config.enable_publication_point_validation_cache, - }; - - let root = root_handle_from_trust_anchor( - &discovery.trust_anchor, - derive_tal_id(&discovery), - None, - &discovery.ca_instance, - ); - let tree = run_tree_serial(root, &runner, config)?; - - Ok(RunTreeFromTalOutput { discovery, tree }) -} - -pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit( - store: &crate::storage::RocksStore, - policy: &crate::policy::Policy, - tal_bytes: &[u8], - ta_der: &[u8], - resolved_ta_uri: Option<&Url>, - payload_archive_root: &std::path::Path, - payload_locks_path: &std::path::Path, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, -) -> Result { - let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( - policy, - tal_bytes, - ta_der, - resolved_ta_uri, - )?; - let replay_index = Arc::new( - ReplayArchiveIndex::load_allow_missing_rsync_modules( - payload_archive_root, - payload_locks_path, - ) - .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?, - ); - let http_fetcher = PayloadReplayHttpFetcher::new(replay_index.clone()) - .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?; - let rsync_fetcher = PayloadReplayRsyncFetcher::new(replay_index.clone()); - let download_log = DownloadLogHandle::new(); - - let runner = Rpkiv1PublicationPointRunner { - store, - policy, - http_fetcher: &http_fetcher, - rsync_fetcher: &rsync_fetcher, - validation_time, - timing: None, - download_log: Some(download_log.clone()), - replay_archive_index: Some(replay_index), - replay_delta_index: None, - rrdp_dedup: true, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: true, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: config.enable_roa_validation_cache, - enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache, - publication_point_cache_observe_only: config.publication_point_cache_observe_only, - enable_publication_point_validation_cache: config.enable_publication_point_validation_cache, - }; - - let root = root_handle_from_trust_anchor( - &discovery.trust_anchor, - derive_tal_id(&discovery), - None, - &discovery.ca_instance, - ); - let TreeRunAuditOutput { - tree, - publication_points, - roa_cache_stats, - cir_input, - } = run_tree_serial_audit(root, &runner, config)?; - - let downloads = download_log.snapshot_events(); - let download_stats = DownloadLogHandle::stats_from_events(&downloads); - Ok(RunTreeFromTalAuditOutput { - discovery: discovery.clone(), - discoveries: vec![discovery], - successful_tal_inputs: Vec::new(), - tree, - publication_points, - roa_cache_stats, - cir_input, - downloads, - download_stats, - current_repo_objects: Vec::new(), - ccr_accumulator: None, - }) -} - -pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit_with_timing( - store: &crate::storage::RocksStore, - policy: &crate::policy::Policy, - tal_bytes: &[u8], - ta_der: &[u8], - resolved_ta_uri: Option<&Url>, - payload_archive_root: &std::path::Path, - payload_locks_path: &std::path::Path, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, - timing: &TimingHandle, -) -> Result { - let _tal = timing.span_phase("tal_bootstrap"); - let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( - policy, - tal_bytes, - ta_der, - resolved_ta_uri, - )?; - drop(_tal); - let replay_index = Arc::new( - ReplayArchiveIndex::load_allow_missing_rsync_modules( - payload_archive_root, - payload_locks_path, - ) - .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?, - ); - let http_fetcher = PayloadReplayHttpFetcher::new(replay_index.clone()) - .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?; - let rsync_fetcher = PayloadReplayRsyncFetcher::new(replay_index.clone()); - let download_log = DownloadLogHandle::new(); - - let runner = Rpkiv1PublicationPointRunner { - store, - policy, - http_fetcher: &http_fetcher, - rsync_fetcher: &rsync_fetcher, - validation_time, - timing: Some(timing.clone()), - download_log: Some(download_log.clone()), - replay_archive_index: Some(replay_index), - replay_delta_index: None, - rrdp_dedup: true, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: true, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: config.enable_roa_validation_cache, - enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache, - publication_point_cache_observe_only: config.publication_point_cache_observe_only, - enable_publication_point_validation_cache: config.enable_publication_point_validation_cache, - }; - - let root = root_handle_from_trust_anchor( - &discovery.trust_anchor, - derive_tal_id(&discovery), - None, - &discovery.ca_instance, - ); - let _tree = timing.span_phase("tree_run_total"); - let TreeRunAuditOutput { - tree, - publication_points, - roa_cache_stats, - cir_input, - } = run_tree_serial_audit(root, &runner, config)?; - - let downloads = download_log.snapshot_events(); - let download_stats = DownloadLogHandle::stats_from_events(&downloads); - Ok(RunTreeFromTalAuditOutput { - discovery: discovery.clone(), - discoveries: vec![discovery], - successful_tal_inputs: Vec::new(), - tree, - publication_points, - roa_cache_stats, - cir_input, - downloads, - download_stats, - current_repo_objects: Vec::new(), - ccr_accumulator: None, - }) -} - -fn build_payload_replay_runner<'a>( - store: &'a crate::storage::RocksStore, - policy: &'a crate::policy::Policy, - replay_index: Arc, - http_fetcher: &'a PayloadReplayHttpFetcher, - rsync_fetcher: &'a PayloadReplayRsyncFetcher, - validation_time: time::OffsetDateTime, - timing: Option, - download_log: Option, - enable_roa_validation_cache: bool, -) -> Rpkiv1PublicationPointRunner<'a> { - Rpkiv1PublicationPointRunner { - store, - policy, - http_fetcher, - rsync_fetcher, - validation_time, - timing, - download_log, - replay_archive_index: Some(replay_index), - replay_delta_index: None, - rrdp_dedup: true, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: true, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - } -} - -fn build_payload_delta_replay_runner<'a>( - store: &'a crate::storage::RocksStore, - policy: &'a crate::policy::Policy, - delta_index: Arc, - http_fetcher: &'a PayloadDeltaReplayHttpFetcher, - rsync_fetcher: &'a PayloadDeltaReplayRsyncFetcher, - validation_time: time::OffsetDateTime, - timing: Option, - download_log: Option, - enable_roa_validation_cache: bool, -) -> Rpkiv1PublicationPointRunner<'a> { - Rpkiv1PublicationPointRunner { - store, - policy, - http_fetcher, - rsync_fetcher, - validation_time, - timing, - download_log, - replay_archive_index: None, - replay_delta_index: Some(delta_index), - rrdp_dedup: true, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: true, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - } -} - -fn build_payload_delta_replay_current_store_runner<'a>( - store: &'a crate::storage::RocksStore, - policy: &'a crate::policy::Policy, - delta_index: Arc, - http_fetcher: &'a PayloadDeltaReplayHttpFetcher, - rsync_fetcher: &'a PayloadDeltaReplayCurrentStoreRsyncFetcher<'a>, - validation_time: time::OffsetDateTime, - timing: Option, - download_log: Option, - enable_roa_validation_cache: bool, -) -> Rpkiv1PublicationPointRunner<'a> { - Rpkiv1PublicationPointRunner { - store, - policy, - http_fetcher, - rsync_fetcher, - validation_time, - timing, - download_log, - replay_archive_index: None, - replay_delta_index: Some(delta_index), - rrdp_dedup: true, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: true, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - } -} - -fn run_payload_delta_replay_audit_inner( - store: &crate::storage::RocksStore, - policy: &crate::policy::Policy, - discovery: DiscoveredRootCaInstance, - base_payload_archive_root: &std::path::Path, - base_locks_path: &std::path::Path, - delta_payload_archive_root: &std::path::Path, - delta_locks_path: &std::path::Path, - base_validation_time: time::OffsetDateTime, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, - timing: Option, -) -> Result { - let base_index = Arc::new( - ReplayArchiveIndex::load_allow_missing_rsync_modules( - base_payload_archive_root, - base_locks_path, - ) - .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?, - ); - let delta_index = Arc::new( - ReplayDeltaArchiveIndex::load(delta_payload_archive_root, delta_locks_path) - .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?, - ); - delta_index - .validate_base_locks_sha256_file(base_locks_path) - .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?; - - let root = root_handle_from_trust_anchor( - &discovery.trust_anchor, - derive_tal_id(&discovery), - None, - &discovery.ca_instance, - ); - - let base_http_fetcher = PayloadReplayHttpFetcher::new(base_index.clone()) - .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?; - let base_rsync_fetcher = PayloadReplayRsyncFetcher::new(base_index.clone()); - if let Some(t) = timing.as_ref() { - let _phase = t.span_phase("payload_delta_replay_base_total"); - let base_runner = build_payload_replay_runner( - store, - policy, - base_index.clone(), - &base_http_fetcher, - &base_rsync_fetcher, - base_validation_time, - Some(t.clone()), - None, - config.enable_roa_validation_cache, - ); - let _base = run_tree_serial(root.clone(), &base_runner, config)?; - } else { - let base_runner = build_payload_replay_runner( - store, - policy, - base_index.clone(), - &base_http_fetcher, - &base_rsync_fetcher, - base_validation_time, - None, - None, - config.enable_roa_validation_cache, - ); - let _base = run_tree_serial(root.clone(), &base_runner, config)?; - } - - let delta_http_fetcher = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) - .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?; - let delta_rsync_fetcher = PayloadDeltaReplayRsyncFetcher::new(base_index, delta_index.clone()); - let download_log = DownloadLogHandle::new(); - let (tree, publication_points, roa_cache_stats, cir_input) = if let Some(t) = timing.as_ref() { - let _phase = t.span_phase("payload_delta_replay_target_total"); - let delta_runner = build_payload_delta_replay_runner( - store, - policy, - delta_index, - &delta_http_fetcher, - &delta_rsync_fetcher, - base_validation_time, - Some(t.clone()), - Some(download_log.clone()), - config.enable_roa_validation_cache, - ); - let TreeRunAuditOutput { - tree, - publication_points, - roa_cache_stats, - cir_input, - } = run_tree_serial_audit(root, &delta_runner, config)?; - (tree, publication_points, roa_cache_stats, cir_input) - } else { - let delta_runner = build_payload_delta_replay_runner( - store, - policy, - delta_index, - &delta_http_fetcher, - &delta_rsync_fetcher, - validation_time, - None, - Some(download_log.clone()), - config.enable_roa_validation_cache, - ); - let TreeRunAuditOutput { - tree, - publication_points, - roa_cache_stats, - cir_input, - } = run_tree_serial_audit(root, &delta_runner, config)?; - (tree, publication_points, roa_cache_stats, cir_input) - }; - let downloads = download_log.snapshot_events(); - let download_stats = DownloadLogHandle::stats_from_events(&downloads); - Ok(RunTreeFromTalAuditOutput { - discovery: discovery.clone(), - discoveries: vec![discovery], - successful_tal_inputs: Vec::new(), - tree, - publication_points, - roa_cache_stats, - cir_input, - downloads, - download_stats, - current_repo_objects: Vec::new(), - ccr_accumulator: None, - }) -} - -pub fn run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit( - store: &crate::storage::RocksStore, - policy: &crate::policy::Policy, - tal_bytes: &[u8], - ta_der: &[u8], - resolved_ta_uri: Option<&Url>, - base_payload_archive_root: &std::path::Path, - base_locks_path: &std::path::Path, - delta_payload_archive_root: &std::path::Path, - delta_locks_path: &std::path::Path, - base_validation_time: time::OffsetDateTime, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, -) -> Result { - let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( - policy, - tal_bytes, - ta_der, - resolved_ta_uri, - )?; - run_payload_delta_replay_audit_inner( - store, - policy, - discovery, - base_payload_archive_root, - base_locks_path, - delta_payload_archive_root, - delta_locks_path, - base_validation_time, - validation_time, - config, - None, - ) -} - -pub fn run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit_with_timing( - store: &crate::storage::RocksStore, - policy: &crate::policy::Policy, - tal_bytes: &[u8], - ta_der: &[u8], - resolved_ta_uri: Option<&Url>, - base_payload_archive_root: &std::path::Path, - base_locks_path: &std::path::Path, - delta_payload_archive_root: &std::path::Path, - delta_locks_path: &std::path::Path, - base_validation_time: time::OffsetDateTime, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, - timing: &TimingHandle, -) -> Result { - let _tal = timing.span_phase("tal_bootstrap"); - let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( - policy, - tal_bytes, - ta_der, - resolved_ta_uri, - )?; - drop(_tal); - run_payload_delta_replay_audit_inner( - store, - policy, - discovery, - base_payload_archive_root, - base_locks_path, - delta_payload_archive_root, - delta_locks_path, - base_validation_time, - validation_time, - config, - Some(timing.clone()), - ) -} - -fn run_payload_delta_replay_step_audit_inner( - store: &crate::storage::RocksStore, - policy: &crate::policy::Policy, - discovery: DiscoveredRootCaInstance, - delta_payload_archive_root: &std::path::Path, - previous_locks_path: &std::path::Path, - delta_locks_path: &std::path::Path, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, - timing: Option, -) -> Result { - let delta_index = Arc::new( - ReplayDeltaArchiveIndex::load(delta_payload_archive_root, delta_locks_path) - .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?, - ); - delta_index - .validate_base_locks_sha256_file(previous_locks_path) - .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?; - - let root = root_handle_from_trust_anchor( - &discovery.trust_anchor, - derive_tal_id(&discovery), - None, - &discovery.ca_instance, - ); - - let delta_http_fetcher = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) - .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?; - let delta_rsync_fetcher = - PayloadDeltaReplayCurrentStoreRsyncFetcher::new(store, delta_index.clone()); - let download_log = DownloadLogHandle::new(); - - let (tree, publication_points, roa_cache_stats, cir_input) = if let Some(t) = timing.as_ref() { - let _phase = t.span_phase("payload_delta_replay_step_total"); - let delta_runner = build_payload_delta_replay_current_store_runner( - store, - policy, - delta_index, - &delta_http_fetcher, - &delta_rsync_fetcher, - validation_time, - Some(t.clone()), - Some(download_log.clone()), - config.enable_roa_validation_cache, - ); - let TreeRunAuditOutput { - tree, - publication_points, - roa_cache_stats, - cir_input, - } = run_tree_serial_audit(root, &delta_runner, config)?; - (tree, publication_points, roa_cache_stats, cir_input) - } else { - let delta_runner = build_payload_delta_replay_current_store_runner( - store, - policy, - delta_index, - &delta_http_fetcher, - &delta_rsync_fetcher, - validation_time, - None, - Some(download_log.clone()), - config.enable_roa_validation_cache, - ); - let TreeRunAuditOutput { - tree, - publication_points, - roa_cache_stats, - cir_input, - } = run_tree_serial_audit(root, &delta_runner, config)?; - (tree, publication_points, roa_cache_stats, cir_input) - }; - let downloads = download_log.snapshot_events(); - let download_stats = DownloadLogHandle::stats_from_events(&downloads); - Ok(RunTreeFromTalAuditOutput { - discovery: discovery.clone(), - discoveries: vec![discovery], - successful_tal_inputs: Vec::new(), - tree, - publication_points, - roa_cache_stats, - cir_input, - downloads, - download_stats, - current_repo_objects: Vec::new(), - ccr_accumulator: None, - }) -} - -pub fn run_tree_from_tal_and_ta_der_payload_delta_replay_step_serial_audit( - store: &crate::storage::RocksStore, - policy: &crate::policy::Policy, - tal_bytes: &[u8], - ta_der: &[u8], - resolved_ta_uri: Option<&Url>, - delta_payload_archive_root: &std::path::Path, - previous_locks_path: &std::path::Path, - delta_locks_path: &std::path::Path, - validation_time: time::OffsetDateTime, - config: &TreeRunConfig, -) -> Result { - let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( - policy, - tal_bytes, - ta_der, - resolved_ta_uri, - )?; - run_payload_delta_replay_step_audit_inner( - store, - policy, - discovery, - delta_payload_archive_root, - previous_locks_path, - delta_locks_path, - validation_time, - config, - None, - ) -} - -#[cfg(test)] -mod multi_tal_tests { - use super::*; - use crate::current_repo_index::CurrentRepoIndex; - use crate::storage::{RepositoryViewEntry, RepositoryViewState}; - - struct RejectingHttpFetcher; - - impl Fetcher for RejectingHttpFetcher { - fn fetch(&self, uri: &str) -> Result, String> { - Err(format!("unexpected http fetch: {uri}")) - } - } - - struct RejectingRsyncFetcher; - - impl crate::fetch::rsync::RsyncFetcher for RejectingRsyncFetcher { - fn fetch_objects( - &self, - _rsync_base_uri: &str, - ) -> crate::fetch::rsync::RsyncFetchResult)>> { - Err(crate::fetch::rsync::RsyncFetchError::Fetch( - "unexpected rsync fetch".to_string(), - )) - } - - fn dedup_key(&self, base_uri: &str) -> String { - base_uri.to_string() - } - } - - #[test] - fn snapshot_current_repo_objects_is_on_demand() { - let handle = CurrentRepoIndex::shared(); - handle - .write() - .expect("write-lock index") - .apply_repository_view_entries(&[RepositoryViewEntry { - rsync_uri: "rsync://example.test/repo/a.roa".to_string(), - current_hash: Some("11".repeat(32)), - repository_source: Some("rsync://example.test/repo/".to_string()), - object_type: Some("roa".to_string()), - state: RepositoryViewState::Present, - }]) - .expect("apply present entry"); - - assert!( - snapshot_current_repo_objects(Some(&handle), false).is_empty(), - "collection should be skipped when disabled" - ); - - let collected = snapshot_current_repo_objects(Some(&handle), true); - assert_eq!(collected.len(), 1); - assert_eq!(collected[0].rsync_uri, "rsync://example.test/repo/a.roa"); - } - - #[test] - fn discover_multiple_roots_from_tal_inputs_builds_multiple_root_handles() { - let apnic_tal = - std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal").expect("read apnic tal"); - let apnic_ta = std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta"); - let arin_tal = std::fs::read("tests/fixtures/tal/arin.tal").expect("read arin tal"); - let arin_ta = std::fs::read("tests/fixtures/ta/arin-ta.cer").expect("read arin ta"); - - let tal_inputs = vec![ - TalInputSpec::from_ta_der("https://example.test/apnic.tal", apnic_tal, apnic_ta), - TalInputSpec::from_ta_der("https://example.test/arin.tal", arin_tal, arin_ta), - ]; - - let roots = discover_multiple_roots_from_tal_inputs( - &tal_inputs, - &RejectingHttpFetcher, - &RejectingRsyncFetcher, - false, - ) - .expect("discover roots"); - - assert_eq!(roots.len(), 2); - assert_eq!(roots[0].tal_input.tal_id, "apnic"); - assert_eq!(roots[1].tal_input.tal_id, "arin"); - assert_eq!(roots[0].root_handle.tal_id, "apnic"); - assert_eq!(roots[1].root_handle.tal_id, "arin"); - assert_ne!( - roots[0].root_handle.manifest_rsync_uri, - roots[1].root_handle.manifest_rsync_uri - ); - } - - #[test] - fn discover_multiple_roots_isolates_strict_name_failure() { - let apnic_tal = - std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal").expect("read apnic tal"); - let apnic_ta = std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta"); - let arin_tal = std::fs::read("tests/fixtures/tal/arin.tal").expect("read arin tal"); - let arin_ta = std::fs::read("tests/fixtures/ta/arin-ta.cer").expect("read arin ta"); - - let tal_inputs = vec![ - TalInputSpec::from_ta_der("https://example.test/apnic.tal", apnic_tal, apnic_ta), - TalInputSpec::from_ta_der("https://example.test/arin.tal", arin_tal, arin_ta), - ]; - - let roots = discover_multiple_roots_from_tal_inputs( - &tal_inputs, - &RejectingHttpFetcher, - &RejectingRsyncFetcher, - true, - ) - .expect("strict discovery should keep usable roots"); - - assert_eq!(roots.len(), 1); - assert_eq!(roots[0].tal_input.tal_id, "arin"); - assert_eq!(roots[0].root_handle.tal_id, "arin"); - } - - #[test] - fn discover_single_root_keeps_strict_name_failure_fatal() { - let apnic_tal = - std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal").expect("read apnic tal"); - let apnic_ta = std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta"); - let tal_inputs = vec![TalInputSpec::from_ta_der( - "https://example.test/apnic.tal", - apnic_tal, - apnic_ta, - )]; - - let error = discover_multiple_roots_from_tal_inputs( - &tal_inputs, - &RejectingHttpFetcher, - &RejectingRsyncFetcher, - true, - ) - .expect_err("single-TAL strict failure should remain fatal"); - - assert!(error.to_string().contains("Name strict validation failed")); - } -} - -#[cfg(test)] -mod replay_api_tests { - use super::*; - use crate::analysis::timing::{TimingHandle, TimingMeta}; - use time::format_description::well_known::Rfc3339; - - fn apnic_replay_inputs() -> ( - Vec, - Vec, - std::path::PathBuf, - std::path::PathBuf, - time::OffsetDateTime, - ) { - let tal_bytes = std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal") - .expect("read apnic tal fixture"); - let ta_der = - std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta fixture"); - let archive_root = std::path::PathBuf::from("target/live/payload_replay/payload-archive"); - let locks_path = std::path::PathBuf::from("target/live/payload_replay/locks.json"); - let validation_time = time::OffsetDateTime::parse("2026-03-13T02:30:00Z", &Rfc3339) - .expect("parse validation time"); - (tal_bytes, ta_der, archive_root, locks_path, validation_time) - } - - fn apnic_multi_rir_replay_inputs() -> ( - Vec, - Vec, - std::path::PathBuf, - std::path::PathBuf, - time::OffsetDateTime, - ) { - let tal_bytes = std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal") - .expect("read apnic tal fixture"); - let ta_der = - std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta fixture"); - let archive_root = std::path::PathBuf::from( - "../../rpki/target/live/20260316-112341-multi-final3/apnic/base-payload-archive", - ); - let locks_path = std::path::PathBuf::from( - "../../rpki/target/live/20260316-112341-multi-final3/apnic/base-locks.json", - ); - let validation_time = time::OffsetDateTime::parse("2026-03-16T11:49:48+08:00", &Rfc3339) - .expect("parse validation time"); - (tal_bytes, ta_der, archive_root, locks_path, validation_time) - } - - fn apnic_delta_replay_inputs() -> ( - Vec, - Vec, - std::path::PathBuf, - std::path::PathBuf, - std::path::PathBuf, - std::path::PathBuf, - time::OffsetDateTime, - ) { - let tal_bytes = std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal") - .expect("read apnic tal fixture"); - let ta_der = - std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta fixture"); - let root = - std::path::PathBuf::from("target/live/apnic_delta_demo/20260315-170223-autoplay"); - let base_archive = root.join("base-payload-archive"); - let base_locks = root.join("base-locks.json"); - let delta_archive = root.join("payload-delta-archive"); - let delta_locks = root.join("locks-delta.json"); - let validation_time = time::OffsetDateTime::parse("2026-03-15T10:00:00Z", &Rfc3339) - .expect("parse validation time"); - ( - tal_bytes, - ta_der, - base_archive, - base_locks, - delta_archive, - delta_locks, - validation_time, - ) - } - - #[test] - fn payload_replay_api_reports_setup_error_for_missing_archive() { - let temp = tempfile::tempdir().expect("tempdir"); - let store = crate::storage::RocksStore::open(&temp.path().join("db")).expect("open db"); - let tal_bytes = std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal") - .expect("read apnic tal fixture"); - let ta_der = - std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta fixture"); - let err = run_tree_from_tal_and_ta_der_payload_replay_serial_audit( - &store, - &crate::policy::Policy::default(), - &tal_bytes, - &ta_der, - None, - std::path::Path::new("tests/fixtures/missing-payload-archive"), - std::path::Path::new("tests/fixtures/missing-locks.json"), - time::OffsetDateTime::now_utc(), - &TreeRunConfig { - max_depth: Some(0), - max_instances: Some(1), - compact_audit: false, - persist_vcir: true, - build_ccr_accumulator: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - enable_transport_request_prefetch: false, - }, - ) - .unwrap_err(); - assert!(matches!(err, RunTreeFromTalError::Replay(_)), "{err}"); - } - - #[test] - fn payload_replay_api_root_only_apnic_archive_runs() { - let temp = tempfile::tempdir().expect("tempdir"); - let store = crate::storage::RocksStore::open(&temp.path().join("db")).expect("open db"); - let (tal_bytes, ta_der, archive_root, locks_path, validation_time) = apnic_replay_inputs(); - if !archive_root.is_dir() || !locks_path.is_file() { - eprintln!( - "skipping payload replay api test; missing fixtures: archive={} locks={}", - archive_root.display(), - locks_path.display() - ); - return; - } - - let out = run_tree_from_tal_and_ta_der_payload_replay_serial_audit( - &store, - &crate::policy::Policy::default(), - &tal_bytes, - &ta_der, - None, - &archive_root, - &locks_path, - validation_time, - &TreeRunConfig { - max_depth: Some(0), - max_instances: Some(1), - compact_audit: false, - persist_vcir: true, - build_ccr_accumulator: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - enable_transport_request_prefetch: false, - }, - ) - .expect("run replay root-only audit"); - - assert_eq!(out.tree.instances_processed, 1); - assert_eq!(out.tree.instances_failed, 0); - assert_eq!(out.publication_points.len(), 1); - assert_eq!(out.discovery.trust_anchor.resolved_ta_uri, None); - assert!(!out.downloads.is_empty()); - assert!( - out.downloads.iter().all(|d| d.success), - "expected successful replay downloads" - ); - } - - #[test] - fn payload_replay_api_root_only_apnic_multi_rir_bundle_runs_with_lenient_rsync_modules() { - let temp = tempfile::tempdir().expect("tempdir"); - let store = crate::storage::RocksStore::open(&temp.path().join("db")).expect("open db"); - let (tal_bytes, ta_der, archive_root, locks_path, validation_time) = - apnic_multi_rir_replay_inputs(); - if !archive_root.is_dir() || !locks_path.is_file() { - eprintln!( - "skipping multi-rir payload replay api test; missing fixtures: archive={} locks={}", - archive_root.display(), - locks_path.display() - ); - return; - } - - let out = run_tree_from_tal_and_ta_der_payload_replay_serial_audit( - &store, - &crate::policy::Policy::default(), - &tal_bytes, - &ta_der, - None, - &archive_root, - &locks_path, - validation_time, - &TreeRunConfig { - max_depth: Some(0), - max_instances: Some(1), - compact_audit: false, - persist_vcir: true, - build_ccr_accumulator: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - enable_transport_request_prefetch: false, - }, - ) - .expect("run replay root-only audit"); - - assert_eq!(out.tree.instances_processed, 1); - assert_eq!(out.tree.instances_failed, 0); - assert_eq!(out.publication_points.len(), 1); - } - - #[test] - fn payload_replay_api_root_only_apnic_archive_runs_with_timing() { - let temp = tempfile::tempdir().expect("tempdir"); - let db_path = temp.path().join("db"); - let store = crate::storage::RocksStore::open(&db_path).expect("open db"); - let (tal_bytes, ta_der, archive_root, locks_path, validation_time) = apnic_replay_inputs(); - if !archive_root.is_dir() || !locks_path.is_file() { - eprintln!( - "skipping payload replay api timing test; missing fixtures: archive={} locks={}", - archive_root.display(), - locks_path.display() - ); - return; - } - - let timing = TimingHandle::new(TimingMeta { - recorded_at_utc_rfc3339: "2026-03-13T03:00:00Z".to_string(), - validation_time_utc_rfc3339: "2026-03-13T02:30:00Z".to_string(), - tal_url: None, - db_path: Some(db_path.to_string_lossy().into_owned()), - }); - - let out = run_tree_from_tal_and_ta_der_payload_replay_serial_audit_with_timing( - &store, - &crate::policy::Policy::default(), - &tal_bytes, - &ta_der, - None, - &archive_root, - &locks_path, - validation_time, - &TreeRunConfig { - max_depth: Some(0), - max_instances: Some(1), - compact_audit: false, - persist_vcir: true, - build_ccr_accumulator: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - enable_transport_request_prefetch: false, - }, - &timing, - ) - .expect("run replay root-only audit with timing"); - - assert_eq!(out.tree.instances_processed, 1); - let timing_json = temp.path().join("timing_replay.json"); - timing - .write_json(&timing_json, 20) - .expect("write timing json"); - let json: serde_json::Value = - serde_json::from_slice(&std::fs::read(&timing_json).expect("read timing json")) - .expect("parse timing json"); - let counts = json.get("counts").expect("counts"); - assert!( - counts - .get("repo_sync_rrdp_ok_total") - .and_then(|v| v.as_u64()) - .unwrap_or(0) - >= 1 - ); - } - - #[test] - fn payload_delta_replay_api_rejects_base_locks_sha_mismatch() { - let temp = tempfile::tempdir().expect("tempdir"); - let store = crate::storage::RocksStore::open(&temp.path().join("db")).expect("open db"); - let ( - tal_bytes, - ta_der, - base_archive, - _base_locks, - delta_archive, - delta_locks, - validation_time, - ) = apnic_delta_replay_inputs(); - let wrong_base_locks = temp.path().join("wrong-base-locks.json"); - std::fs::write(&wrong_base_locks, b"wrong-base-locks").expect("write wrong base locks"); - let err = run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit( - &store, - &crate::policy::Policy::default(), - &tal_bytes, - &ta_der, - None, - &base_archive, - &wrong_base_locks, - &delta_archive, - &delta_locks, - validation_time, - validation_time, - &TreeRunConfig { - max_depth: Some(0), - max_instances: Some(1), - compact_audit: false, - persist_vcir: true, - build_ccr_accumulator: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - enable_transport_request_prefetch: false, - }, - ) - .unwrap_err(); - assert!(matches!(err, RunTreeFromTalError::Replay(_)), "{err}"); - } - - #[test] - fn payload_delta_replay_api_reports_setup_error_for_missing_inputs() { - let temp = tempfile::tempdir().expect("tempdir"); - let store = crate::storage::RocksStore::open(&temp.path().join("db")).expect("open db"); - let tal_bytes = std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal") - .expect("read apnic tal fixture"); - let ta_der = - std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta fixture"); - let err = run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit( - &store, - &crate::policy::Policy::default(), - &tal_bytes, - &ta_der, - None, - std::path::Path::new("tests/fixtures/missing-base-archive"), - std::path::Path::new("tests/fixtures/missing-base-locks.json"), - std::path::Path::new("tests/fixtures/missing-delta-archive"), - std::path::Path::new("tests/fixtures/missing-delta-locks.json"), - time::OffsetDateTime::now_utc(), - time::OffsetDateTime::now_utc(), - &TreeRunConfig { - max_depth: Some(0), - max_instances: Some(1), - compact_audit: false, - persist_vcir: true, - build_ccr_accumulator: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - enable_transport_request_prefetch: false, - }, - ) - .unwrap_err(); - assert!(matches!(err, RunTreeFromTalError::Replay(_)), "{err}"); - } - - #[test] - fn payload_delta_replay_api_root_only_apnic_bundle_runs() { - let temp = tempfile::tempdir().expect("tempdir"); - let store = crate::storage::RocksStore::open(&temp.path().join("db")).expect("open db"); - let ( - tal_bytes, - ta_der, - base_archive, - base_locks, - delta_archive, - delta_locks, - validation_time, - ) = apnic_delta_replay_inputs(); - if !base_archive.is_dir() - || !base_locks.is_file() - || !delta_archive.is_dir() - || !delta_locks.is_file() - { - eprintln!( - "skipping payload delta replay api test; missing fixtures: base_archive={} base_locks={} delta_archive={} delta_locks={}", - base_archive.display(), - base_locks.display(), - delta_archive.display(), - delta_locks.display() - ); - return; - } - - let out = run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit( - &store, - &crate::policy::Policy::default(), - &tal_bytes, - &ta_der, - None, - &base_archive, - &base_locks, - &delta_archive, - &delta_locks, - validation_time, - validation_time, - &TreeRunConfig { - max_depth: Some(0), - max_instances: Some(1), - compact_audit: false, - persist_vcir: true, - build_ccr_accumulator: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - enable_transport_request_prefetch: false, - }, - ) - .expect("run delta replay root-only audit"); - - assert_eq!(out.tree.instances_processed, 1); - assert_eq!(out.tree.instances_failed, 0); - assert_eq!(out.publication_points.len(), 1); - } - - #[test] - fn payload_delta_replay_api_root_only_apnic_bundle_runs_with_timing() { - let temp = tempfile::tempdir().expect("tempdir"); - let db_path = temp.path().join("db"); - let store = crate::storage::RocksStore::open(&db_path).expect("open db"); - let ( - tal_bytes, - ta_der, - base_archive, - base_locks, - delta_archive, - delta_locks, - validation_time, - ) = apnic_delta_replay_inputs(); - if !base_archive.is_dir() - || !base_locks.is_file() - || !delta_archive.is_dir() - || !delta_locks.is_file() - { - eprintln!( - "skipping payload delta replay timing test; missing fixtures: base_archive={} base_locks={} delta_archive={} delta_locks={}", - base_archive.display(), - base_locks.display(), - delta_archive.display(), - delta_locks.display() - ); - return; - } - let timing = TimingHandle::new(TimingMeta { - recorded_at_utc_rfc3339: "2026-03-16T00:00:00Z".to_string(), - validation_time_utc_rfc3339: "2026-03-15T10:00:00Z".to_string(), - tal_url: None, - db_path: Some(db_path.to_string_lossy().into_owned()), - }); - let out = run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit_with_timing( - &store, - &crate::policy::Policy::default(), - &tal_bytes, - &ta_der, - None, - &base_archive, - &base_locks, - &delta_archive, - &delta_locks, - validation_time, - validation_time, - &TreeRunConfig { - max_depth: Some(0), - max_instances: Some(1), - compact_audit: false, - persist_vcir: true, - build_ccr_accumulator: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - enable_transport_request_prefetch: false, - }, - &timing, - ) - .expect("run delta replay root-only audit with timing"); - assert_eq!(out.tree.instances_processed, 1); - let timing_json = temp.path().join("timing_delta_replay.json"); - timing - .write_json(&timing_json, 20) - .expect("write timing json"); - let json: serde_json::Value = - serde_json::from_slice(&std::fs::read(&timing_json).expect("read timing json")) - .expect("parse timing json"); - assert_eq!( - json["phases"]["payload_delta_replay_base_total"]["count"].as_u64(), - Some(1) - ); - assert_eq!( - json["phases"]["payload_delta_replay_target_total"]["count"].as_u64(), - Some(1) - ); - } -} +include!("run_tree_from_tal/discovery.rs"); +include!("run_tree_from_tal/serial.rs"); +include!("run_tree_from_tal/phase1.rs"); +include!("run_tree_from_tal/serial_replay.rs"); +include!("run_tree_from_tal/replay_setup.rs"); +include!("run_tree_from_tal/replay_delta.rs"); +include!("run_tree_from_tal/tests.rs"); diff --git a/crates/panda-rpki-validator/src/validation/run_tree_from_tal/discovery.rs b/crates/panda-rpki-validator/src/validation/run_tree_from_tal/discovery.rs new file mode 100644 index 0000000..5105819 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/run_tree_from_tal/discovery.rs @@ -0,0 +1,558 @@ +fn tal_id_from_url_like(s: &str) -> Option { + let url = Url::parse(s).ok()?; + if let Some(last) = url + .path_segments() + .and_then(|segments| segments.filter(|seg| !seg.is_empty()).next_back()) + { + let stem = last.rsplit_once('.').map(|(stem, _)| stem).unwrap_or(last); + let trimmed = stem.trim(); + if !trimmed.is_empty() { + return Some(trimmed.to_string()); + } + } + url.host_str().map(|host| host.to_string()) +} + +fn derive_tal_id(discovery: &DiscoveredRootCaInstance) -> String { + discovery + .tal_url + .as_deref() + .and_then(tal_id_from_url_like) + .or_else(|| { + discovery + .trust_anchor + .resolved_ta_uri + .as_ref() + .and_then(|uri| tal_id_from_url_like(uri.as_str())) + }) + .or_else(|| { + discovery + .trust_anchor + .tal + .ta_uris + .first() + .and_then(|uri| tal_id_from_url_like(uri.as_str())) + }) + .unwrap_or_else(|| "unknown-tal".to_string()) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RunTreeFromTalOutput { + pub discovery: DiscoveredRootCaInstance, + pub tree: TreeRunOutput, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RunTreeFromTalAuditOutput { + pub discovery: DiscoveredRootCaInstance, + pub discoveries: Vec, + pub successful_tal_inputs: Vec, + pub tree: TreeRunOutput, + pub publication_points: Vec, + pub roa_cache_stats: crate::validation::objects::RoaValidationCacheStats, + pub cir_input: crate::cir::CirInputSnapshot, + pub downloads: Vec, + pub download_stats: crate::audit::AuditDownloadStats, + pub current_repo_objects: Vec, + pub ccr_accumulator: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TalRootDiscovery { + pub tal_input: TalInputSpec, + pub discovery: DiscoveredRootCaInstance, + pub root_handle: CaInstanceHandle, +} + +fn snapshot_current_repo_objects( + current_repo_index: Option<&CurrentRepoIndexHandle>, + collect: bool, +) -> Vec { + if !collect { + return Vec::new(); + } + current_repo_index + .and_then(|handle| handle.read().ok().map(|idx| idx.snapshot_objects())) + .unwrap_or_default() +} + +fn make_live_runner<'a>( + store: &'a crate::storage::RocksStore, + policy: &'a crate::policy::Policy, + http_fetcher: &'a dyn Fetcher, + rsync_fetcher: &'a dyn crate::fetch::rsync::RsyncFetcher, + validation_time: time::OffsetDateTime, + timing: Option, + download_log: Option, + current_repo_index: Option, + repo_sync_runtime: Option>, + parallel_phase2_config: Option, + ccr_accumulator: Option, + persist_vcir: bool, + enable_roa_validation_cache: bool, + enable_child_certificate_validation_cache: bool, + publication_point_cache_observe_only: bool, + enable_publication_point_validation_cache: bool, +) -> Rpkiv1PublicationPointRunner<'a> { + let parallel_roa_worker_pool = parallel_phase2_config + .as_ref() + .and_then(|config| ParallelRoaWorkerPool::new(config).ok()); + Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher, + rsync_fetcher, + validation_time, + timing, + download_log, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index, + repo_sync_runtime, + parallel_phase2_config, + parallel_roa_worker_pool, + ccr_accumulator: ccr_accumulator.map(Mutex::new), + persist_vcir, + enable_roa_validation_cache, + enable_child_certificate_validation_cache, + publication_point_cache_observe_only, + enable_publication_point_validation_cache, + } +} + +fn build_phase1_repo_sync_runtime( + store: Arc, + policy: &crate::policy::Policy, + http_fetcher: &H, + rsync_fetcher: &R, + parallel_config: ParallelPhase1Config, + timing: Option, + download_log: Option, + tal_inputs: Vec, + record_transport_prefetch_requests: bool, +) -> Result<(Arc, CurrentRepoIndexHandle), RunTreeFromTalError> +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + let coordinator = GlobalRunCoordinator::new(parallel_config.clone(), tal_inputs); + let current_repo_index = coordinator.current_repo_index_handle(); + let rsync_fetcher_arc = Arc::new(rsync_fetcher.clone()); + let executor = LiveRepoTransportExecutor::new( + Arc::clone(&store), + current_repo_index.clone(), + Arc::new(http_fetcher.clone()), + Arc::clone(&rsync_fetcher_arc), + timing.clone(), + download_log, + ); + let pool = RepoTransportWorkerPool::new(RepoWorkerPoolConfig::from(¶llel_config), executor) + .map_err(RunTreeFromTalError::Replay)?; + let resolver_rsync_fetcher_arc = Arc::clone(&rsync_fetcher_arc); + let resolver: Arc String + Send + Sync> = + Arc::new(move |base: &str| resolver_rsync_fetcher_arc.dedup_key(base)); + let failure_rsync_fetcher_arc = Arc::clone(&rsync_fetcher_arc); + let failure_resolver: Arc Option + Send + Sync> = + Arc::new(move |base: &str| failure_rsync_fetcher_arc.failure_dedup_key(base)); + let _ = policy; // policy reserved for later runtime-level decisions + let runtime = Arc::new( + Phase1RepoSyncRuntime::new_with_failure_scope_and_prefetch_recording( + coordinator, + pool, + resolver, + failure_resolver, + policy.sync_preference, + record_transport_prefetch_requests, + ), + ); + Ok((runtime, current_repo_index)) +} + +fn record_transport_prefetch_count(timing: Option<&TimingHandle>, key: &'static str, value: u64) { + if value == 0 { + return; + } + if let Some(timing) = timing { + timing.record_count(key, value); + } +} + +fn apply_transport_request_prefetch( + store: &crate::storage::RocksStore, + runtime: &Arc, + sync_preference: SyncPreference, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + timing: Option<&TimingHandle>, +) -> Result<(), RunTreeFromTalError> { + if !config.enable_transport_request_prefetch { + return Ok(()); + } + + let snapshot = match store.get_transport_prefetch_snapshot() { + Ok(Some(snapshot)) => snapshot, + Ok(None) => { + crate::progress_log::emit( + "phase1_repo_prefetch_missing_snapshot", + serde_json::json!({ "status": "missing" }), + ); + return Ok(()); + } + Err(err) => { + record_transport_prefetch_count(timing, "transport_prefetch_load_errors", 1); + crate::progress_log::emit( + "phase1_repo_prefetch_load_error", + serde_json::json!({ "error": err.to_string() }), + ); + return Ok(()); + } + }; + + if !snapshot.is_compatible_with(sync_preference) { + record_transport_prefetch_count( + timing, + "transport_prefetch_skipped_incompatible_snapshots", + 1, + ); + crate::progress_log::emit( + "phase1_repo_prefetch_skipped", + serde_json::json!({ + "reason": "incompatible_snapshot", + "schema_version": snapshot.schema_version, + "request_count": snapshot.requests.len(), + }), + ); + return Ok(()); + } + + let stats = runtime + .prefetch_transport_requests(&snapshot, validation_time) + .map_err(TreeRunError::Runner)?; + record_transport_prefetch_count( + timing.clone(), + "transport_prefetch_loaded_requests", + stats.loaded_requests, + ); + record_transport_prefetch_count( + timing.clone(), + "transport_prefetch_enqueued_tasks", + stats.enqueued_tasks, + ); + record_transport_prefetch_count( + timing, + "transport_prefetch_waiting_requests", + stats.waiting_requests, + ); + record_transport_prefetch_count( + timing, + "transport_prefetch_reused_results", + stats.reused_results, + ); + record_transport_prefetch_count( + timing, + "transport_prefetch_skipped_incompatible_requests", + stats.skipped_incompatible, + ); + crate::progress_log::emit( + "phase1_repo_prefetch_applied", + serde_json::json!({ + "loaded_requests": stats.loaded_requests, + "enqueued_tasks": stats.enqueued_tasks, + "waiting_requests": stats.waiting_requests, + "reused_results": stats.reused_results, + "skipped_incompatible": stats.skipped_incompatible, + }), + ); + Ok(()) +} + +fn persist_transport_request_prefetch_snapshot( + store: &crate::storage::RocksStore, + runtime: &Arc, + config: &TreeRunConfig, + timing: Option<&TimingHandle>, +) -> Result<(), RunTreeFromTalError> { + if !config.enable_transport_request_prefetch { + return Ok(()); + } + let snapshot = runtime.transport_prefetch_snapshot(); + let recorded = snapshot.requests.len() as u64; + let persist_started = std::time::Instant::now(); + store + .put_transport_prefetch_snapshot(&snapshot) + .map_err(|err| TreeRunError::Runner(err.to_string()))?; + let persist_ms = persist_started.elapsed().as_millis() as u64; + record_transport_prefetch_count(timing, "transport_prefetch_recorded_requests", recorded); + crate::progress_log::emit( + "phase1_repo_prefetch_persisted", + serde_json::json!({ + "recorded_requests": recorded, + "schema_version": snapshot.schema_version, + "persist_ms": persist_ms, + }), + ); + Ok(()) +} + +/// Persist the dead-repository blacklist working copy at run end. Failures +/// degrade to a progress event; a broken blacklist file must never fail a run. +fn persist_dead_repo_blacklist( + runtime: &Arc, + timing: Option<&TimingHandle>, +) -> Result<(), RunTreeFromTalError> { + let Some((blacklist_config, blacklist)) = runtime.dead_repo_blacklist_state() else { + return Ok(()); + }; + let persist_started = std::time::Instant::now(); + let now_unix = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_secs()) + .unwrap_or(0); + match blacklist.store_atomic(&blacklist_config.path, now_unix) { + Ok(()) => { + let persist_ms = persist_started.elapsed().as_millis() as u64; + record_transport_prefetch_count( + timing, + "dead_repo_blacklist_entries", + blacklist.len() as u64, + ); + record_transport_prefetch_count( + timing, + "dead_repo_blacklist_blacklisted", + blacklist.blacklisted_len() as u64, + ); + crate::progress_log::emit( + "dead_repo_blacklist_persisted", + serde_json::json!({ + "path": blacklist_config.path.display().to_string(), + "entries": blacklist.len(), + "blacklisted": blacklist.blacklisted_len(), + "persist_ms": persist_ms, + }), + ); + } + Err(err) => { + crate::progress_log::emit( + "dead_repo_blacklist_persist_error", + serde_json::json!({ + "path": blacklist_config.path.display().to_string(), + "error": err.to_string(), + }), + ); + } + } + Ok(()) +} + +fn root_discovery_from_tal_input( + tal_input: &TalInputSpec, + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, + strict_name: bool, +) -> Result { + match &tal_input.source { + TalSource::Url(url) => { + if strict_name { + discover_root_ca_instance_from_tal_url_with_fetchers_strict_name( + http_fetcher, + rsync_fetcher, + url, + ) + } else { + discover_root_ca_instance_from_tal_url_with_fetchers( + http_fetcher, + rsync_fetcher, + url, + ) + } + } + TalSource::DerBytes { + tal_bytes, ta_der, .. + } => { + if strict_name { + discover_root_ca_instance_from_tal_and_ta_der_with_strict_name( + tal_bytes, ta_der, None, + ) + } else { + discover_root_ca_instance_from_tal_and_ta_der(tal_bytes, ta_der, None) + } + } + TalSource::FilePath(path) => { + let tal_bytes = std::fs::read(path).map_err(|e| { + FromTalError::TalFetch(format!("read TAL file failed: {}: {e}", path.display())) + })?; + let tal = crate::data_model::tal::Tal::decode_bytes(&tal_bytes) + .map_err(FromTalError::from)?; + if strict_name { + discover_root_ca_instance_from_tal_with_fetchers_strict_name( + http_fetcher, + rsync_fetcher, + tal, + None, + ) + } else { + discover_root_ca_instance_from_tal_with_fetchers( + http_fetcher, + rsync_fetcher, + tal, + None, + ) + } + } + TalSource::FilePathWithTa { tal_path, ta_path } => { + let tal_bytes = std::fs::read(tal_path).map_err(|e| { + FromTalError::TalFetch(format!("read TAL file failed: {}: {e}", tal_path.display())) + })?; + let ta_der = std::fs::read(ta_path).map_err(|e| { + FromTalError::TaFetch(format!("read TA file failed: {}: {e}", ta_path.display())) + })?; + let resolved_ta_uri = canonical_tal_rsync_uri_from_bytes(&tal_bytes)?; + if strict_name { + discover_root_ca_instance_from_tal_and_ta_der_with_strict_name( + &tal_bytes, + &ta_der, + Some(&resolved_ta_uri), + ) + } else { + discover_root_ca_instance_from_tal_and_ta_der( + &tal_bytes, + &ta_der, + Some(&resolved_ta_uri), + ) + } + } + } +} + +fn discover_root_ca_instance_from_tal_url_with_policy( + policy: &crate::policy::Policy, + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, + tal_url: &str, +) -> Result { + if policy.strict.name { + discover_root_ca_instance_from_tal_url_with_fetchers_strict_name( + http_fetcher, + rsync_fetcher, + tal_url, + ) + } else { + discover_root_ca_instance_from_tal_url_with_fetchers(http_fetcher, rsync_fetcher, tal_url) + } +} + +fn discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&Url>, +) -> Result { + if policy.strict.name { + discover_root_ca_instance_from_tal_and_ta_der_with_strict_name( + tal_bytes, + ta_der, + resolved_ta_uri, + ) + } else { + discover_root_ca_instance_from_tal_and_ta_der(tal_bytes, ta_der, resolved_ta_uri) + } +} + +fn discover_multiple_roots_from_tal_inputs( + tal_inputs: &[TalInputSpec], + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, + strict_name: bool, +) -> Result, RunTreeFromTalError> { + let mut roots = Vec::with_capacity(tal_inputs.len()); + for tal_input in tal_inputs { + let discovery = match root_discovery_from_tal_input( + tal_input, + http_fetcher, + rsync_fetcher, + strict_name, + ) { + Ok(discovery) => discovery, + Err(error) + if should_isolate_multi_tal_strict_name_failure( + tal_inputs.len(), + strict_name, + &error, + ) => + { + eprintln!( + "warning: skipping TAL '{}' because strict name validation failed during trust anchor discovery: {error}", + tal_input.tal_id + ); + continue; + } + Err(error) => return Err(error.into()), + }; + let root_handle = root_handle_from_trust_anchor( + &discovery.trust_anchor, + tal_input.tal_id.clone(), + None, + &discovery.ca_instance, + ); + roots.push(TalRootDiscovery { + tal_input: tal_input.clone(), + discovery, + root_handle, + }); + } + if roots.is_empty() { + return Err(RunTreeFromTalError::Replay( + "multi-TAL root discovery returned no usable roots after strict name filtering" + .to_string(), + )); + } + Ok(roots) +} + +fn should_isolate_multi_tal_strict_name_failure( + tal_input_count: usize, + strict_name: bool, + error: &FromTalError, +) -> bool { + strict_name + && tal_input_count > 1 + && error.to_string().contains("Name strict validation failed") +} + +#[derive(Debug, thiserror::Error)] +pub enum RunTreeFromTalError { + #[error("{0}")] + FromTal(#[from] FromTalError), + + #[error("payload replay setup failed: {0}")] + Replay(String), + + #[error("{0}")] + Tree(#[from] TreeRunError), +} + +pub fn root_handle_from_trust_anchor( + trust_anchor: &TrustAnchor, + tal_id: String, + ca_certificate_rsync_uri: Option, + ca_instance: &crate::validation::ca_instance::CaInstanceUris, +) -> CaInstanceHandle { + let ta_rc = trust_anchor.ta_certificate.rc_ca.clone(); + CaInstanceHandle { + depth: 0, + tal_id, + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(trust_anchor.ta_certificate.raw_der.clone()), + ca_certificate_rsync_uri, + effective_ip_resources: ta_rc.tbs.extensions.ip_resources.clone(), + effective_as_resources: ta_rc.tbs.extensions.as_resources.clone(), + rsync_base_uri: ca_instance.rsync_base_uri.clone(), + manifest_rsync_uri: ca_instance.manifest_rsync_uri.clone(), + publication_point_rsync_uri: ca_instance.publication_point_rsync_uri.clone(), + rrdp_notification_uri: ca_instance.rrdp_notification_uri.clone(), + } +} diff --git a/crates/panda-rpki-validator/src/validation/run_tree_from_tal/phase1.rs b/crates/panda-rpki-validator/src/validation/run_tree_from_tal/phase1.rs new file mode 100644 index 0000000..f2cfc5e --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/run_tree_from_tal/phase1.rs @@ -0,0 +1,365 @@ +pub fn run_tree_from_tal_url_parallel_phase1_audit( + store: Arc, + policy: &crate::policy::Policy, + tal_url: &str, + http_fetcher: &H, + rsync_fetcher: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + parallel_config: ParallelPhase1Config, + collect_current_repo_objects: bool, +) -> Result +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + let discovery = discover_root_ca_instance_from_tal_url_with_policy( + policy, + http_fetcher, + rsync_fetcher, + tal_url, + )?; + run_single_root_parallel_audit_inner( + store, + policy, + discovery, + vec![TalInputSpec::from_url(tal_url.to_string())], + http_fetcher, + rsync_fetcher, + validation_time, + config, + parallel_config, + None, + collect_current_repo_objects, + None, + ) +} + +pub fn run_tree_from_tal_and_ta_der_parallel_phase1_audit( + store: Arc, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&url::Url>, + http_fetcher: &H, + rsync_fetcher: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + parallel_config: ParallelPhase1Config, + collect_current_repo_objects: bool, +) -> Result +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy, + tal_bytes, + ta_der, + resolved_ta_uri, + )?; + let derived_tal_id = derive_tal_id(&discovery); + let tal_inputs = vec![TalInputSpec { + tal_id: derived_tal_id.clone(), + rir_id: derived_tal_id, + source: TalSource::DerBytes { + tal_url: discovery + .tal_url + .clone() + .unwrap_or_else(|| "embedded-tal".to_string()), + tal_bytes: tal_bytes.to_vec(), + ta_der: ta_der.to_vec(), + }, + }]; + run_single_root_parallel_audit_inner( + store, + policy, + discovery, + tal_inputs, + http_fetcher, + rsync_fetcher, + validation_time, + config, + parallel_config, + None, + collect_current_repo_objects, + None, + ) +} + +pub fn run_tree_from_multiple_tals_parallel_phase1_audit( + store: Arc, + policy: &crate::policy::Policy, + tal_inputs: Vec, + http_fetcher: &H, + rsync_fetcher: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + parallel_config: ParallelPhase1Config, + collect_current_repo_objects: bool, +) -> Result +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + run_multi_root_parallel_audit_inner( + store, + policy, + tal_inputs, + http_fetcher, + rsync_fetcher, + validation_time, + config, + parallel_config, + None, + collect_current_repo_objects, + None, + ) +} + +pub fn run_tree_from_tal_url_parallel_phase2_audit( + store: Arc, + policy: &crate::policy::Policy, + tal_url: &str, + http_fetcher: &H, + rsync_fetcher: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + parallel_config: ParallelPhase1Config, + phase2_config: ParallelPhase2Config, + collect_current_repo_objects: bool, +) -> Result +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + let discovery = discover_root_ca_instance_from_tal_url_with_policy( + policy, + http_fetcher, + rsync_fetcher, + tal_url, + )?; + run_single_root_parallel_audit_inner( + store, + policy, + discovery, + vec![TalInputSpec::from_url(tal_url.to_string())], + http_fetcher, + rsync_fetcher, + validation_time, + config, + parallel_config, + Some(phase2_config), + collect_current_repo_objects, + None, + ) +} + +pub fn run_tree_from_tal_url_parallel_phase2_audit_with_timing( + store: Arc, + policy: &crate::policy::Policy, + tal_url: &str, + http_fetcher: &H, + rsync_fetcher: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + parallel_config: ParallelPhase1Config, + phase2_config: ParallelPhase2Config, + collect_current_repo_objects: bool, + timing: &TimingHandle, +) -> Result +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + let discovery = discover_root_ca_instance_from_tal_url_with_policy( + policy, + http_fetcher, + rsync_fetcher, + tal_url, + )?; + run_single_root_parallel_audit_inner( + store, + policy, + discovery, + vec![TalInputSpec::from_url(tal_url.to_string())], + http_fetcher, + rsync_fetcher, + validation_time, + config, + parallel_config, + Some(phase2_config), + collect_current_repo_objects, + Some(timing.clone()), + ) +} + +pub fn run_tree_from_tal_and_ta_der_parallel_phase2_audit( + store: Arc, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&url::Url>, + http_fetcher: &H, + rsync_fetcher: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + parallel_config: ParallelPhase1Config, + phase2_config: ParallelPhase2Config, + collect_current_repo_objects: bool, +) -> Result +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy, + tal_bytes, + ta_der, + resolved_ta_uri, + )?; + let derived_tal_id = derive_tal_id(&discovery); + let tal_inputs = vec![TalInputSpec { + tal_id: derived_tal_id.clone(), + rir_id: derived_tal_id, + source: TalSource::DerBytes { + tal_url: discovery + .tal_url + .clone() + .unwrap_or_else(|| "embedded-tal".to_string()), + tal_bytes: tal_bytes.to_vec(), + ta_der: ta_der.to_vec(), + }, + }]; + run_single_root_parallel_audit_inner( + store, + policy, + discovery, + tal_inputs, + http_fetcher, + rsync_fetcher, + validation_time, + config, + parallel_config, + Some(phase2_config), + collect_current_repo_objects, + None, + ) +} + +pub fn run_tree_from_tal_and_ta_der_parallel_phase2_audit_with_timing( + store: Arc, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&url::Url>, + http_fetcher: &H, + rsync_fetcher: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + parallel_config: ParallelPhase1Config, + phase2_config: ParallelPhase2Config, + collect_current_repo_objects: bool, + timing: &TimingHandle, +) -> Result +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy, + tal_bytes, + ta_der, + resolved_ta_uri, + )?; + let derived_tal_id = derive_tal_id(&discovery); + let tal_inputs = vec![TalInputSpec { + tal_id: derived_tal_id.clone(), + rir_id: derived_tal_id, + source: TalSource::DerBytes { + tal_url: discovery + .tal_url + .clone() + .unwrap_or_else(|| "embedded-tal".to_string()), + tal_bytes: tal_bytes.to_vec(), + ta_der: ta_der.to_vec(), + }, + }]; + run_single_root_parallel_audit_inner( + store, + policy, + discovery, + tal_inputs, + http_fetcher, + rsync_fetcher, + validation_time, + config, + parallel_config, + Some(phase2_config), + collect_current_repo_objects, + Some(timing.clone()), + ) +} + +pub fn run_tree_from_multiple_tals_parallel_phase2_audit( + store: Arc, + policy: &crate::policy::Policy, + tal_inputs: Vec, + http_fetcher: &H, + rsync_fetcher: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + parallel_config: ParallelPhase1Config, + phase2_config: ParallelPhase2Config, + collect_current_repo_objects: bool, +) -> Result +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + run_multi_root_parallel_audit_inner( + store, + policy, + tal_inputs, + http_fetcher, + rsync_fetcher, + validation_time, + config, + parallel_config, + Some(phase2_config), + collect_current_repo_objects, + None, + ) +} + +pub fn run_tree_from_multiple_tals_parallel_phase2_audit_with_timing( + store: Arc, + policy: &crate::policy::Policy, + tal_inputs: Vec, + http_fetcher: &H, + rsync_fetcher: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + parallel_config: ParallelPhase1Config, + phase2_config: ParallelPhase2Config, + collect_current_repo_objects: bool, + timing: &TimingHandle, +) -> Result +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + run_multi_root_parallel_audit_inner( + store, + policy, + tal_inputs, + http_fetcher, + rsync_fetcher, + validation_time, + config, + parallel_config, + Some(phase2_config), + collect_current_repo_objects, + Some(timing.clone()), + ) +} diff --git a/crates/panda-rpki-validator/src/validation/run_tree_from_tal/replay_delta.rs b/crates/panda-rpki-validator/src/validation/run_tree_from_tal/replay_delta.rs new file mode 100644 index 0000000..42c9ee1 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/run_tree_from_tal/replay_delta.rs @@ -0,0 +1,119 @@ +fn run_payload_delta_replay_step_audit_inner( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + discovery: DiscoveredRootCaInstance, + delta_payload_archive_root: &std::path::Path, + previous_locks_path: &std::path::Path, + delta_locks_path: &std::path::Path, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + timing: Option, +) -> Result { + let delta_index = Arc::new( + ReplayDeltaArchiveIndex::load(delta_payload_archive_root, delta_locks_path) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?, + ); + delta_index + .validate_base_locks_sha256_file(previous_locks_path) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?; + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + + let delta_http_fetcher = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?; + let delta_rsync_fetcher = + PayloadDeltaReplayCurrentStoreRsyncFetcher::new(store, delta_index.clone()); + let download_log = DownloadLogHandle::new(); + + let (tree, publication_points, roa_cache_stats, cir_input) = if let Some(t) = timing.as_ref() { + let _phase = t.span_phase("payload_delta_replay_step_total"); + let delta_runner = build_payload_delta_replay_current_store_runner( + store, + policy, + delta_index, + &delta_http_fetcher, + &delta_rsync_fetcher, + validation_time, + Some(t.clone()), + Some(download_log.clone()), + config.enable_roa_validation_cache, + ); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = run_tree_serial_audit(root, &delta_runner, config)?; + (tree, publication_points, roa_cache_stats, cir_input) + } else { + let delta_runner = build_payload_delta_replay_current_store_runner( + store, + policy, + delta_index, + &delta_http_fetcher, + &delta_rsync_fetcher, + validation_time, + None, + Some(download_log.clone()), + config.enable_roa_validation_cache, + ); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = run_tree_serial_audit(root, &delta_runner, config)?; + (tree, publication_points, roa_cache_stats, cir_input) + }; + let downloads = download_log.snapshot_events(); + let download_stats = DownloadLogHandle::stats_from_events(&downloads); + Ok(RunTreeFromTalAuditOutput { + discovery: discovery.clone(), + discoveries: vec![discovery], + successful_tal_inputs: Vec::new(), + tree, + publication_points, + roa_cache_stats, + cir_input, + downloads, + download_stats, + current_repo_objects: Vec::new(), + ccr_accumulator: None, + }) +} + +pub fn run_tree_from_tal_and_ta_der_payload_delta_replay_step_serial_audit( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&Url>, + delta_payload_archive_root: &std::path::Path, + previous_locks_path: &std::path::Path, + delta_locks_path: &std::path::Path, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, +) -> Result { + let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy, + tal_bytes, + ta_der, + resolved_ta_uri, + )?; + run_payload_delta_replay_step_audit_inner( + store, + policy, + discovery, + delta_payload_archive_root, + previous_locks_path, + delta_locks_path, + validation_time, + config, + None, + ) +} diff --git a/crates/panda-rpki-validator/src/validation/run_tree_from_tal/replay_setup.rs b/crates/panda-rpki-validator/src/validation/run_tree_from_tal/replay_setup.rs new file mode 100644 index 0000000..e44fa6d --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/run_tree_from_tal/replay_setup.rs @@ -0,0 +1,314 @@ +fn build_payload_replay_runner<'a>( + store: &'a crate::storage::RocksStore, + policy: &'a crate::policy::Policy, + replay_index: Arc, + http_fetcher: &'a PayloadReplayHttpFetcher, + rsync_fetcher: &'a PayloadReplayRsyncFetcher, + validation_time: time::OffsetDateTime, + timing: Option, + download_log: Option, + enable_roa_validation_cache: bool, +) -> Rpkiv1PublicationPointRunner<'a> { + Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher, + rsync_fetcher, + validation_time, + timing, + download_log, + replay_archive_index: Some(replay_index), + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + } +} + +fn build_payload_delta_replay_runner<'a>( + store: &'a crate::storage::RocksStore, + policy: &'a crate::policy::Policy, + delta_index: Arc, + http_fetcher: &'a PayloadDeltaReplayHttpFetcher, + rsync_fetcher: &'a PayloadDeltaReplayRsyncFetcher, + validation_time: time::OffsetDateTime, + timing: Option, + download_log: Option, + enable_roa_validation_cache: bool, +) -> Rpkiv1PublicationPointRunner<'a> { + Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher, + rsync_fetcher, + validation_time, + timing, + download_log, + replay_archive_index: None, + replay_delta_index: Some(delta_index), + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + } +} + +fn build_payload_delta_replay_current_store_runner<'a>( + store: &'a crate::storage::RocksStore, + policy: &'a crate::policy::Policy, + delta_index: Arc, + http_fetcher: &'a PayloadDeltaReplayHttpFetcher, + rsync_fetcher: &'a PayloadDeltaReplayCurrentStoreRsyncFetcher<'a>, + validation_time: time::OffsetDateTime, + timing: Option, + download_log: Option, + enable_roa_validation_cache: bool, +) -> Rpkiv1PublicationPointRunner<'a> { + Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher, + rsync_fetcher, + validation_time, + timing, + download_log, + replay_archive_index: None, + replay_delta_index: Some(delta_index), + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + } +} + +fn run_payload_delta_replay_audit_inner( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + discovery: DiscoveredRootCaInstance, + base_payload_archive_root: &std::path::Path, + base_locks_path: &std::path::Path, + delta_payload_archive_root: &std::path::Path, + delta_locks_path: &std::path::Path, + base_validation_time: time::OffsetDateTime, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + timing: Option, +) -> Result { + let base_index = Arc::new( + ReplayArchiveIndex::load_allow_missing_rsync_modules( + base_payload_archive_root, + base_locks_path, + ) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?, + ); + let delta_index = Arc::new( + ReplayDeltaArchiveIndex::load(delta_payload_archive_root, delta_locks_path) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?, + ); + delta_index + .validate_base_locks_sha256_file(base_locks_path) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?; + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + + let base_http_fetcher = PayloadReplayHttpFetcher::new(base_index.clone()) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?; + let base_rsync_fetcher = PayloadReplayRsyncFetcher::new(base_index.clone()); + if let Some(t) = timing.as_ref() { + let _phase = t.span_phase("payload_delta_replay_base_total"); + let base_runner = build_payload_replay_runner( + store, + policy, + base_index.clone(), + &base_http_fetcher, + &base_rsync_fetcher, + base_validation_time, + Some(t.clone()), + None, + config.enable_roa_validation_cache, + ); + let _base = run_tree_serial(root.clone(), &base_runner, config)?; + } else { + let base_runner = build_payload_replay_runner( + store, + policy, + base_index.clone(), + &base_http_fetcher, + &base_rsync_fetcher, + base_validation_time, + None, + None, + config.enable_roa_validation_cache, + ); + let _base = run_tree_serial(root.clone(), &base_runner, config)?; + } + + let delta_http_fetcher = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?; + let delta_rsync_fetcher = PayloadDeltaReplayRsyncFetcher::new(base_index, delta_index.clone()); + let download_log = DownloadLogHandle::new(); + let (tree, publication_points, roa_cache_stats, cir_input) = if let Some(t) = timing.as_ref() { + let _phase = t.span_phase("payload_delta_replay_target_total"); + let delta_runner = build_payload_delta_replay_runner( + store, + policy, + delta_index, + &delta_http_fetcher, + &delta_rsync_fetcher, + base_validation_time, + Some(t.clone()), + Some(download_log.clone()), + config.enable_roa_validation_cache, + ); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = run_tree_serial_audit(root, &delta_runner, config)?; + (tree, publication_points, roa_cache_stats, cir_input) + } else { + let delta_runner = build_payload_delta_replay_runner( + store, + policy, + delta_index, + &delta_http_fetcher, + &delta_rsync_fetcher, + validation_time, + None, + Some(download_log.clone()), + config.enable_roa_validation_cache, + ); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = run_tree_serial_audit(root, &delta_runner, config)?; + (tree, publication_points, roa_cache_stats, cir_input) + }; + let downloads = download_log.snapshot_events(); + let download_stats = DownloadLogHandle::stats_from_events(&downloads); + Ok(RunTreeFromTalAuditOutput { + discovery: discovery.clone(), + discoveries: vec![discovery], + successful_tal_inputs: Vec::new(), + tree, + publication_points, + roa_cache_stats, + cir_input, + downloads, + download_stats, + current_repo_objects: Vec::new(), + ccr_accumulator: None, + }) +} + +pub fn run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&Url>, + base_payload_archive_root: &std::path::Path, + base_locks_path: &std::path::Path, + delta_payload_archive_root: &std::path::Path, + delta_locks_path: &std::path::Path, + base_validation_time: time::OffsetDateTime, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, +) -> Result { + let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy, + tal_bytes, + ta_der, + resolved_ta_uri, + )?; + run_payload_delta_replay_audit_inner( + store, + policy, + discovery, + base_payload_archive_root, + base_locks_path, + delta_payload_archive_root, + delta_locks_path, + base_validation_time, + validation_time, + config, + None, + ) +} + +pub fn run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit_with_timing( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&Url>, + base_payload_archive_root: &std::path::Path, + base_locks_path: &std::path::Path, + delta_payload_archive_root: &std::path::Path, + delta_locks_path: &std::path::Path, + base_validation_time: time::OffsetDateTime, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + timing: &TimingHandle, +) -> Result { + let _tal = timing.span_phase("tal_bootstrap"); + let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy, + tal_bytes, + ta_der, + resolved_ta_uri, + )?; + drop(_tal); + run_payload_delta_replay_audit_inner( + store, + policy, + discovery, + base_payload_archive_root, + base_locks_path, + delta_payload_archive_root, + delta_locks_path, + base_validation_time, + validation_time, + config, + Some(timing.clone()), + ) +} diff --git a/crates/panda-rpki-validator/src/validation/run_tree_from_tal/serial.rs b/crates/panda-rpki-validator/src/validation/run_tree_from_tal/serial.rs new file mode 100644 index 0000000..0201509 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/run_tree_from_tal/serial.rs @@ -0,0 +1,407 @@ +pub fn run_tree_from_tal_url_serial( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_url: &str, + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, +) -> Result { + let discovery = discover_root_ca_instance_from_tal_url_with_policy( + policy, + http_fetcher, + rsync_fetcher, + tal_url, + )?; + + let runner = make_live_runner( + store, + policy, + http_fetcher, + rsync_fetcher, + validation_time, + None, + None, + None, + None, + None, + None, + config.persist_vcir, + config.enable_roa_validation_cache, + config.enable_child_certificate_validation_cache, + config.publication_point_cache_observe_only, + config.enable_publication_point_validation_cache, + ); + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + let tree = run_tree_serial(root, &runner, config)?; + + Ok(RunTreeFromTalOutput { discovery, tree }) +} + +pub fn run_tree_from_tal_url_serial_audit( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_url: &str, + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, +) -> Result { + let discovery = discover_root_ca_instance_from_tal_url_with_policy( + policy, + http_fetcher, + rsync_fetcher, + tal_url, + )?; + + let download_log = DownloadLogHandle::new(); + let runner = make_live_runner( + store, + policy, + http_fetcher, + rsync_fetcher, + validation_time, + None, + Some(download_log.clone()), + None, + None, + None, + None, + config.persist_vcir, + config.enable_roa_validation_cache, + config.enable_child_certificate_validation_cache, + config.publication_point_cache_observe_only, + config.enable_publication_point_validation_cache, + ); + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = run_tree_serial_audit(root, &runner, config)?; + + let downloads = download_log.snapshot_events(); + let download_stats = DownloadLogHandle::stats_from_events(&downloads); + Ok(RunTreeFromTalAuditOutput { + discovery: discovery.clone(), + discoveries: vec![discovery], + successful_tal_inputs: Vec::new(), + tree, + publication_points, + roa_cache_stats, + cir_input, + downloads, + download_stats, + current_repo_objects: Vec::new(), + ccr_accumulator: None, + }) +} + +pub fn run_tree_from_tal_url_serial_audit_with_timing( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_url: &str, + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + timing: &TimingHandle, +) -> Result { + let _tal = timing.span_phase("tal_bootstrap"); + let discovery = discover_root_ca_instance_from_tal_url_with_policy( + policy, + http_fetcher, + rsync_fetcher, + tal_url, + )?; + drop(_tal); + + let download_log = DownloadLogHandle::new(); + let runner = make_live_runner( + store, + policy, + http_fetcher, + rsync_fetcher, + validation_time, + Some(timing.clone()), + Some(download_log.clone()), + None, + None, + None, + None, + config.persist_vcir, + config.enable_roa_validation_cache, + config.enable_child_certificate_validation_cache, + config.publication_point_cache_observe_only, + config.enable_publication_point_validation_cache, + ); + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + let _tree = timing.span_phase("tree_run_total"); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = run_tree_serial_audit(root, &runner, config)?; + + let downloads = download_log.snapshot_events(); + let download_stats = DownloadLogHandle::stats_from_events(&downloads); + Ok(RunTreeFromTalAuditOutput { + discovery: discovery.clone(), + discoveries: vec![discovery], + successful_tal_inputs: Vec::new(), + tree, + publication_points, + roa_cache_stats, + cir_input, + downloads, + download_stats, + current_repo_objects: Vec::new(), + ccr_accumulator: None, + }) +} + +fn run_single_root_parallel_audit_inner( + store: Arc, + policy: &crate::policy::Policy, + discovery: DiscoveredRootCaInstance, + tal_inputs: Vec, + http_fetcher: &H, + rsync_fetcher: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + parallel_config: ParallelPhase1Config, + phase2_config: Option, + collect_current_repo_objects: bool, + timing: Option, +) -> Result +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + let phase2_enabled = phase2_config.is_some(); + let download_log = DownloadLogHandle::new(); + let (runtime, current_repo_index) = build_phase1_repo_sync_runtime( + Arc::clone(&store), + policy, + http_fetcher, + rsync_fetcher, + parallel_config, + timing.clone(), + Some(download_log.clone()), + tal_inputs, + config.enable_transport_request_prefetch, + )?; + apply_transport_request_prefetch( + store.as_ref(), + &runtime, + policy.sync_preference, + validation_time, + config, + timing.as_ref(), + )?; + let current_repo_index_for_output = current_repo_index.clone(); + let runner = make_live_runner( + store.as_ref(), + policy, + http_fetcher, + rsync_fetcher, + validation_time, + timing.clone(), + Some(download_log.clone()), + Some(current_repo_index), + Some(Arc::clone(&runtime)), + phase2_config, + (phase2_enabled && config.build_ccr_accumulator) + .then(|| CcrAccumulator::new(vec![discovery.trust_anchor.clone()])), + config.persist_vcir, + config.enable_roa_validation_cache, + config.enable_child_certificate_validation_cache, + config.publication_point_cache_observe_only, + config.enable_publication_point_validation_cache, + ); + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = if phase2_enabled { + run_tree_parallel_phase2_audit(root, &runner, config)? + } else { + run_tree_serial_audit(root, &runner, config)? + }; + persist_transport_request_prefetch_snapshot(store.as_ref(), &runtime, config, timing.as_ref())?; + persist_dead_repo_blacklist(&runtime, timing.as_ref())?; + let downloads = download_log.snapshot_events(); + let download_stats = DownloadLogHandle::stats_from_events(&downloads); + Ok(RunTreeFromTalAuditOutput { + discovery: discovery.clone(), + discoveries: vec![discovery], + successful_tal_inputs: Vec::new(), + tree, + publication_points, + roa_cache_stats, + cir_input, + downloads, + download_stats, + current_repo_objects: snapshot_current_repo_objects( + Some(¤t_repo_index_for_output), + collect_current_repo_objects, + ), + ccr_accumulator: runner.ccr_accumulator_snapshot(), + }) +} + +fn run_multi_root_parallel_audit_inner( + store: Arc, + policy: &crate::policy::Policy, + tal_inputs: Vec, + http_fetcher: &H, + rsync_fetcher: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + parallel_config: ParallelPhase1Config, + phase2_config: Option, + collect_current_repo_objects: bool, + timing: Option, +) -> Result +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + // Constraints are an immutable per-run policy snapshot. The phase-2 + // ready-stage binds the snapshot to each CA's TAL and moves an Arc into + // ROA/ASPA worker state, so constrained multi-TAL runs retain the same + // parallel scheduler as unconstrained runs. + let phase2_enabled = phase2_config.is_some(); + if tal_inputs.is_empty() { + return Err(RunTreeFromTalError::Replay( + "multi-TAL run requires at least one TAL input".to_string(), + )); + } + let roots = discover_multiple_roots_from_tal_inputs( + &tal_inputs, + http_fetcher, + rsync_fetcher, + policy.strict.name, + )?; + let primary = roots.first().cloned().ok_or_else(|| { + RunTreeFromTalError::Replay("multi-TAL root discovery returned no roots".to_string()) + })?; + let discoveries = roots + .iter() + .map(|item| item.discovery.clone()) + .collect::>(); + let successful_tal_inputs = roots + .iter() + .map(|item| item.tal_input.clone()) + .collect::>(); + let root_handles = roots + .iter() + .map(|item| item.root_handle.clone()) + .collect::>(); + + let download_log = DownloadLogHandle::new(); + let (runtime, current_repo_index) = build_phase1_repo_sync_runtime( + Arc::clone(&store), + policy, + http_fetcher, + rsync_fetcher, + parallel_config, + timing.clone(), + Some(download_log.clone()), + successful_tal_inputs.clone(), + config.enable_transport_request_prefetch, + )?; + apply_transport_request_prefetch( + store.as_ref(), + &runtime, + policy.sync_preference, + validation_time, + config, + timing.as_ref(), + )?; + let current_repo_index_for_output = current_repo_index.clone(); + let runner = make_live_runner( + store.as_ref(), + policy, + http_fetcher, + rsync_fetcher, + validation_time, + timing.clone(), + Some(download_log.clone()), + Some(current_repo_index), + Some(Arc::clone(&runtime)), + phase2_config, + (phase2_enabled && config.build_ccr_accumulator).then(|| { + CcrAccumulator::new( + discoveries + .iter() + .map(|item| item.trust_anchor.clone()) + .collect::>(), + ) + }), + config.persist_vcir, + config.enable_roa_validation_cache, + config.enable_child_certificate_validation_cache, + config.publication_point_cache_observe_only, + config.enable_publication_point_validation_cache, + ); + + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = if phase2_enabled { + run_tree_parallel_phase2_audit_multi_root(root_handles, &runner, config)? + } else { + run_tree_serial_audit_multi_root(root_handles, &runner, config)? + }; + persist_transport_request_prefetch_snapshot(store.as_ref(), &runtime, config, timing.as_ref())?; + persist_dead_repo_blacklist(&runtime, timing.as_ref())?; + let downloads = download_log.snapshot_events(); + let download_stats = DownloadLogHandle::stats_from_events(&downloads); + Ok(RunTreeFromTalAuditOutput { + discovery: primary.discovery.clone(), + discoveries, + successful_tal_inputs, + tree, + publication_points, + roa_cache_stats, + cir_input, + downloads, + download_stats, + current_repo_objects: snapshot_current_repo_objects( + Some(¤t_repo_index_for_output), + collect_current_repo_objects, + ), + ccr_accumulator: runner.ccr_accumulator_snapshot(), + }) +} diff --git a/crates/panda-rpki-validator/src/validation/run_tree_from_tal/serial_replay.rs b/crates/panda-rpki-validator/src/validation/run_tree_from_tal/serial_replay.rs new file mode 100644 index 0000000..7a47821 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/run_tree_from_tal/serial_replay.rs @@ -0,0 +1,602 @@ +pub fn run_tree_from_tal_and_ta_der_serial( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&Url>, + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, +) -> Result { + let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy, + tal_bytes, + ta_der, + resolved_ta_uri, + )?; + + let runner = Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher, + rsync_fetcher, + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: config.enable_roa_validation_cache, + enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache, + publication_point_cache_observe_only: config.publication_point_cache_observe_only, + enable_publication_point_validation_cache: config.enable_publication_point_validation_cache, + }; + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + let tree = run_tree_serial(root, &runner, config)?; + + Ok(RunTreeFromTalOutput { discovery, tree }) +} + +pub fn run_tree_from_tal_bytes_serial_audit( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + tal_uri: Option, + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, +) -> Result { + let tal = crate::data_model::tal::Tal::decode_bytes(tal_bytes).map_err(FromTalError::from)?; + let discovery = discover_root_ca_instance_from_tal_with_fetchers( + http_fetcher, + rsync_fetcher, + tal, + tal_uri, + )?; + + let download_log = DownloadLogHandle::new(); + let runner = Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher, + rsync_fetcher, + validation_time, + timing: None, + download_log: Some(download_log.clone()), + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: config.enable_roa_validation_cache, + enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache, + publication_point_cache_observe_only: config.publication_point_cache_observe_only, + enable_publication_point_validation_cache: config.enable_publication_point_validation_cache, + }; + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = run_tree_serial_audit(root, &runner, config)?; + + let downloads = download_log.snapshot_events(); + let download_stats = DownloadLogHandle::stats_from_events(&downloads); + Ok(RunTreeFromTalAuditOutput { + discovery: discovery.clone(), + discoveries: vec![discovery], + successful_tal_inputs: Vec::new(), + tree, + publication_points, + roa_cache_stats, + cir_input, + downloads, + download_stats, + current_repo_objects: Vec::new(), + ccr_accumulator: None, + }) +} + +pub fn run_tree_from_tal_bytes_serial_audit_with_timing( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + tal_uri: Option, + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + timing: &TimingHandle, +) -> Result { + let _tal = timing.span_phase("tal_bootstrap"); + let tal = crate::data_model::tal::Tal::decode_bytes(tal_bytes).map_err(FromTalError::from)?; + let discovery = discover_root_ca_instance_from_tal_with_fetchers( + http_fetcher, + rsync_fetcher, + tal, + tal_uri, + )?; + drop(_tal); + + let download_log = DownloadLogHandle::new(); + let runner = Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher, + rsync_fetcher, + validation_time, + timing: Some(timing.clone()), + download_log: Some(download_log.clone()), + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: config.enable_roa_validation_cache, + enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache, + publication_point_cache_observe_only: config.publication_point_cache_observe_only, + enable_publication_point_validation_cache: config.enable_publication_point_validation_cache, + }; + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + let _tree = timing.span_phase("tree_run"); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = run_tree_serial_audit(root, &runner, config)?; + drop(_tree); + + let downloads = download_log.snapshot_events(); + let download_stats = DownloadLogHandle::stats_from_events(&downloads); + Ok(RunTreeFromTalAuditOutput { + discovery: discovery.clone(), + discoveries: vec![discovery], + successful_tal_inputs: Vec::new(), + tree, + publication_points, + roa_cache_stats, + cir_input, + downloads, + download_stats, + current_repo_objects: Vec::new(), + ccr_accumulator: None, + }) +} + +pub fn run_tree_from_tal_and_ta_der_serial_audit( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&Url>, + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, +) -> Result { + let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy, + tal_bytes, + ta_der, + resolved_ta_uri, + )?; + + let download_log = DownloadLogHandle::new(); + let runner = Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher, + rsync_fetcher, + validation_time, + timing: None, + download_log: Some(download_log.clone()), + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: config.enable_roa_validation_cache, + enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache, + publication_point_cache_observe_only: config.publication_point_cache_observe_only, + enable_publication_point_validation_cache: config.enable_publication_point_validation_cache, + }; + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = run_tree_serial_audit(root, &runner, config)?; + + let downloads = download_log.snapshot_events(); + let download_stats = DownloadLogHandle::stats_from_events(&downloads); + Ok(RunTreeFromTalAuditOutput { + discovery: discovery.clone(), + discoveries: vec![discovery], + successful_tal_inputs: Vec::new(), + tree, + publication_points, + roa_cache_stats, + cir_input, + downloads, + download_stats, + current_repo_objects: Vec::new(), + ccr_accumulator: None, + }) +} + +pub fn run_tree_from_tal_and_ta_der_serial_audit_with_timing( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&Url>, + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + timing: &TimingHandle, +) -> Result { + let _tal = timing.span_phase("tal_bootstrap"); + let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy, + tal_bytes, + ta_der, + resolved_ta_uri, + )?; + drop(_tal); + + let download_log = DownloadLogHandle::new(); + let runner = Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher, + rsync_fetcher, + validation_time, + timing: Some(timing.clone()), + download_log: Some(download_log.clone()), + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: config.enable_roa_validation_cache, + enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache, + publication_point_cache_observe_only: config.publication_point_cache_observe_only, + enable_publication_point_validation_cache: config.enable_publication_point_validation_cache, + }; + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + let _tree = timing.span_phase("tree_run_total"); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = run_tree_serial_audit(root, &runner, config)?; + + let downloads = download_log.snapshot_events(); + let download_stats = DownloadLogHandle::stats_from_events(&downloads); + Ok(RunTreeFromTalAuditOutput { + discovery: discovery.clone(), + discoveries: vec![discovery], + successful_tal_inputs: Vec::new(), + tree, + publication_points, + roa_cache_stats, + cir_input, + downloads, + download_stats, + current_repo_objects: Vec::new(), + ccr_accumulator: None, + }) +} + +pub fn run_tree_from_tal_and_ta_der_payload_replay_serial( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&Url>, + payload_archive_root: &std::path::Path, + payload_locks_path: &std::path::Path, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, +) -> Result { + let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy, + tal_bytes, + ta_der, + resolved_ta_uri, + )?; + let replay_index = Arc::new( + ReplayArchiveIndex::load_allow_missing_rsync_modules( + payload_archive_root, + payload_locks_path, + ) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?, + ); + let http_fetcher = PayloadReplayHttpFetcher::new(replay_index.clone()) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?; + let rsync_fetcher = PayloadReplayRsyncFetcher::new(replay_index.clone()); + + let runner = Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher: &http_fetcher, + rsync_fetcher: &rsync_fetcher, + validation_time, + timing: None, + download_log: None, + replay_archive_index: Some(replay_index), + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: config.enable_roa_validation_cache, + enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache, + publication_point_cache_observe_only: config.publication_point_cache_observe_only, + enable_publication_point_validation_cache: config.enable_publication_point_validation_cache, + }; + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + let tree = run_tree_serial(root, &runner, config)?; + + Ok(RunTreeFromTalOutput { discovery, tree }) +} + +pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&Url>, + payload_archive_root: &std::path::Path, + payload_locks_path: &std::path::Path, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, +) -> Result { + let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy, + tal_bytes, + ta_der, + resolved_ta_uri, + )?; + let replay_index = Arc::new( + ReplayArchiveIndex::load_allow_missing_rsync_modules( + payload_archive_root, + payload_locks_path, + ) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?, + ); + let http_fetcher = PayloadReplayHttpFetcher::new(replay_index.clone()) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?; + let rsync_fetcher = PayloadReplayRsyncFetcher::new(replay_index.clone()); + let download_log = DownloadLogHandle::new(); + + let runner = Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher: &http_fetcher, + rsync_fetcher: &rsync_fetcher, + validation_time, + timing: None, + download_log: Some(download_log.clone()), + replay_archive_index: Some(replay_index), + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: config.enable_roa_validation_cache, + enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache, + publication_point_cache_observe_only: config.publication_point_cache_observe_only, + enable_publication_point_validation_cache: config.enable_publication_point_validation_cache, + }; + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = run_tree_serial_audit(root, &runner, config)?; + + let downloads = download_log.snapshot_events(); + let download_stats = DownloadLogHandle::stats_from_events(&downloads); + Ok(RunTreeFromTalAuditOutput { + discovery: discovery.clone(), + discoveries: vec![discovery], + successful_tal_inputs: Vec::new(), + tree, + publication_points, + roa_cache_stats, + cir_input, + downloads, + download_stats, + current_repo_objects: Vec::new(), + ccr_accumulator: None, + }) +} + +pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit_with_timing( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&Url>, + payload_archive_root: &std::path::Path, + payload_locks_path: &std::path::Path, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + timing: &TimingHandle, +) -> Result { + let _tal = timing.span_phase("tal_bootstrap"); + let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy, + tal_bytes, + ta_der, + resolved_ta_uri, + )?; + drop(_tal); + let replay_index = Arc::new( + ReplayArchiveIndex::load_allow_missing_rsync_modules( + payload_archive_root, + payload_locks_path, + ) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?, + ); + let http_fetcher = PayloadReplayHttpFetcher::new(replay_index.clone()) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?; + let rsync_fetcher = PayloadReplayRsyncFetcher::new(replay_index.clone()); + let download_log = DownloadLogHandle::new(); + + let runner = Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher: &http_fetcher, + rsync_fetcher: &rsync_fetcher, + validation_time, + timing: Some(timing.clone()), + download_log: Some(download_log.clone()), + replay_archive_index: Some(replay_index), + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: config.enable_roa_validation_cache, + enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache, + publication_point_cache_observe_only: config.publication_point_cache_observe_only, + enable_publication_point_validation_cache: config.enable_publication_point_validation_cache, + }; + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + let _tree = timing.span_phase("tree_run_total"); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = run_tree_serial_audit(root, &runner, config)?; + + let downloads = download_log.snapshot_events(); + let download_stats = DownloadLogHandle::stats_from_events(&downloads); + Ok(RunTreeFromTalAuditOutput { + discovery: discovery.clone(), + discoveries: vec![discovery], + successful_tal_inputs: Vec::new(), + tree, + publication_points, + roa_cache_stats, + cir_input, + downloads, + download_stats, + current_repo_objects: Vec::new(), + ccr_accumulator: None, + }) +} diff --git a/crates/panda-rpki-validator/src/validation/run_tree_from_tal/tests.rs b/crates/panda-rpki-validator/src/validation/run_tree_from_tal/tests.rs new file mode 100644 index 0000000..a961b5d --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/run_tree_from_tal/tests.rs @@ -0,0 +1,627 @@ +#[cfg(test)] +mod multi_tal_tests { + use super::*; + use crate::current_repo_index::CurrentRepoIndex; + use crate::storage::{RepositoryViewEntry, RepositoryViewState}; + + struct RejectingHttpFetcher; + + impl Fetcher for RejectingHttpFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + Err(format!("unexpected http fetch: {uri}")) + } + } + + struct RejectingRsyncFetcher; + + impl crate::fetch::rsync::RsyncFetcher for RejectingRsyncFetcher { + fn fetch_objects( + &self, + _rsync_base_uri: &str, + ) -> crate::fetch::rsync::RsyncFetchResult)>> { + Err(crate::fetch::rsync::RsyncFetchError::Fetch( + "unexpected rsync fetch".to_string(), + )) + } + + fn dedup_key(&self, base_uri: &str) -> String { + base_uri.to_string() + } + } + + #[test] + fn snapshot_current_repo_objects_is_on_demand() { + let handle = CurrentRepoIndex::shared(); + handle + .write() + .expect("write-lock index") + .apply_repository_view_entries(&[RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/a.roa".to_string(), + current_hash: Some("11".repeat(32)), + repository_source: Some("rsync://example.test/repo/".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }]) + .expect("apply present entry"); + + assert!( + snapshot_current_repo_objects(Some(&handle), false).is_empty(), + "collection should be skipped when disabled" + ); + + let collected = snapshot_current_repo_objects(Some(&handle), true); + assert_eq!(collected.len(), 1); + assert_eq!(collected[0].rsync_uri, "rsync://example.test/repo/a.roa"); + } + + #[test] + fn discover_multiple_roots_from_tal_inputs_builds_multiple_root_handles() { + let apnic_tal = + std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal").expect("read apnic tal"); + let apnic_ta = std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta"); + let arin_tal = std::fs::read("tests/fixtures/tal/arin.tal").expect("read arin tal"); + let arin_ta = std::fs::read("tests/fixtures/ta/arin-ta.cer").expect("read arin ta"); + + let tal_inputs = vec![ + TalInputSpec::from_ta_der("https://example.test/apnic.tal", apnic_tal, apnic_ta), + TalInputSpec::from_ta_der("https://example.test/arin.tal", arin_tal, arin_ta), + ]; + + let roots = discover_multiple_roots_from_tal_inputs( + &tal_inputs, + &RejectingHttpFetcher, + &RejectingRsyncFetcher, + false, + ) + .expect("discover roots"); + + assert_eq!(roots.len(), 2); + assert_eq!(roots[0].tal_input.tal_id, "apnic"); + assert_eq!(roots[1].tal_input.tal_id, "arin"); + assert_eq!(roots[0].root_handle.tal_id, "apnic"); + assert_eq!(roots[1].root_handle.tal_id, "arin"); + assert_ne!( + roots[0].root_handle.manifest_rsync_uri, + roots[1].root_handle.manifest_rsync_uri + ); + } + + #[test] + fn discover_multiple_roots_isolates_strict_name_failure() { + let apnic_tal = + std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal").expect("read apnic tal"); + let apnic_ta = std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta"); + let arin_tal = std::fs::read("tests/fixtures/tal/arin.tal").expect("read arin tal"); + let arin_ta = std::fs::read("tests/fixtures/ta/arin-ta.cer").expect("read arin ta"); + + let tal_inputs = vec![ + TalInputSpec::from_ta_der("https://example.test/apnic.tal", apnic_tal, apnic_ta), + TalInputSpec::from_ta_der("https://example.test/arin.tal", arin_tal, arin_ta), + ]; + + let roots = discover_multiple_roots_from_tal_inputs( + &tal_inputs, + &RejectingHttpFetcher, + &RejectingRsyncFetcher, + true, + ) + .expect("strict discovery should keep usable roots"); + + assert_eq!(roots.len(), 1); + assert_eq!(roots[0].tal_input.tal_id, "arin"); + assert_eq!(roots[0].root_handle.tal_id, "arin"); + } + + #[test] + fn discover_single_root_keeps_strict_name_failure_fatal() { + let apnic_tal = + std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal").expect("read apnic tal"); + let apnic_ta = std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta"); + let tal_inputs = vec![TalInputSpec::from_ta_der( + "https://example.test/apnic.tal", + apnic_tal, + apnic_ta, + )]; + + let error = discover_multiple_roots_from_tal_inputs( + &tal_inputs, + &RejectingHttpFetcher, + &RejectingRsyncFetcher, + true, + ) + .expect_err("single-TAL strict failure should remain fatal"); + + assert!(error.to_string().contains("Name strict validation failed")); + } +} + +#[cfg(test)] +mod replay_api_tests { + use super::*; + use crate::analysis::timing::{TimingHandle, TimingMeta}; + use time::format_description::well_known::Rfc3339; + + fn apnic_replay_inputs() -> ( + Vec, + Vec, + std::path::PathBuf, + std::path::PathBuf, + time::OffsetDateTime, + ) { + let tal_bytes = std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal") + .expect("read apnic tal fixture"); + let ta_der = + std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta fixture"); + let archive_root = std::path::PathBuf::from("target/live/payload_replay/payload-archive"); + let locks_path = std::path::PathBuf::from("target/live/payload_replay/locks.json"); + let validation_time = time::OffsetDateTime::parse("2026-03-13T02:30:00Z", &Rfc3339) + .expect("parse validation time"); + (tal_bytes, ta_der, archive_root, locks_path, validation_time) + } + + fn apnic_multi_rir_replay_inputs() -> ( + Vec, + Vec, + std::path::PathBuf, + std::path::PathBuf, + time::OffsetDateTime, + ) { + let tal_bytes = std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal") + .expect("read apnic tal fixture"); + let ta_der = + std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta fixture"); + let archive_root = std::path::PathBuf::from( + "../../rpki/target/live/20260316-112341-multi-final3/apnic/base-payload-archive", + ); + let locks_path = std::path::PathBuf::from( + "../../rpki/target/live/20260316-112341-multi-final3/apnic/base-locks.json", + ); + let validation_time = time::OffsetDateTime::parse("2026-03-16T11:49:48+08:00", &Rfc3339) + .expect("parse validation time"); + (tal_bytes, ta_der, archive_root, locks_path, validation_time) + } + + fn apnic_delta_replay_inputs() -> ( + Vec, + Vec, + std::path::PathBuf, + std::path::PathBuf, + std::path::PathBuf, + std::path::PathBuf, + time::OffsetDateTime, + ) { + let tal_bytes = std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal") + .expect("read apnic tal fixture"); + let ta_der = + std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta fixture"); + let root = + std::path::PathBuf::from("target/live/apnic_delta_demo/20260315-170223-autoplay"); + let base_archive = root.join("base-payload-archive"); + let base_locks = root.join("base-locks.json"); + let delta_archive = root.join("payload-delta-archive"); + let delta_locks = root.join("locks-delta.json"); + let validation_time = time::OffsetDateTime::parse("2026-03-15T10:00:00Z", &Rfc3339) + .expect("parse validation time"); + ( + tal_bytes, + ta_der, + base_archive, + base_locks, + delta_archive, + delta_locks, + validation_time, + ) + } + + #[test] + fn payload_replay_api_reports_setup_error_for_missing_archive() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = crate::storage::RocksStore::open(&temp.path().join("db")).expect("open db"); + let tal_bytes = std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal") + .expect("read apnic tal fixture"); + let ta_der = + std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta fixture"); + let err = run_tree_from_tal_and_ta_der_payload_replay_serial_audit( + &store, + &crate::policy::Policy::default(), + &tal_bytes, + &ta_der, + None, + std::path::Path::new("tests/fixtures/missing-payload-archive"), + std::path::Path::new("tests/fixtures/missing-locks.json"), + time::OffsetDateTime::now_utc(), + &TreeRunConfig { + max_depth: Some(0), + max_instances: Some(1), + compact_audit: false, + persist_vcir: true, + build_ccr_accumulator: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + enable_transport_request_prefetch: false, + }, + ) + .unwrap_err(); + assert!(matches!(err, RunTreeFromTalError::Replay(_)), "{err}"); + } + + #[test] + fn payload_replay_api_root_only_apnic_archive_runs() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = crate::storage::RocksStore::open(&temp.path().join("db")).expect("open db"); + let (tal_bytes, ta_der, archive_root, locks_path, validation_time) = apnic_replay_inputs(); + if !archive_root.is_dir() || !locks_path.is_file() { + eprintln!( + "skipping payload replay api test; missing fixtures: archive={} locks={}", + archive_root.display(), + locks_path.display() + ); + return; + } + + let out = run_tree_from_tal_and_ta_der_payload_replay_serial_audit( + &store, + &crate::policy::Policy::default(), + &tal_bytes, + &ta_der, + None, + &archive_root, + &locks_path, + validation_time, + &TreeRunConfig { + max_depth: Some(0), + max_instances: Some(1), + compact_audit: false, + persist_vcir: true, + build_ccr_accumulator: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + enable_transport_request_prefetch: false, + }, + ) + .expect("run replay root-only audit"); + + assert_eq!(out.tree.instances_processed, 1); + assert_eq!(out.tree.instances_failed, 0); + assert_eq!(out.publication_points.len(), 1); + assert_eq!(out.discovery.trust_anchor.resolved_ta_uri, None); + assert!(!out.downloads.is_empty()); + assert!( + out.downloads.iter().all(|d| d.success), + "expected successful replay downloads" + ); + } + + #[test] + fn payload_replay_api_root_only_apnic_multi_rir_bundle_runs_with_lenient_rsync_modules() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = crate::storage::RocksStore::open(&temp.path().join("db")).expect("open db"); + let (tal_bytes, ta_der, archive_root, locks_path, validation_time) = + apnic_multi_rir_replay_inputs(); + if !archive_root.is_dir() || !locks_path.is_file() { + eprintln!( + "skipping multi-rir payload replay api test; missing fixtures: archive={} locks={}", + archive_root.display(), + locks_path.display() + ); + return; + } + + let out = run_tree_from_tal_and_ta_der_payload_replay_serial_audit( + &store, + &crate::policy::Policy::default(), + &tal_bytes, + &ta_der, + None, + &archive_root, + &locks_path, + validation_time, + &TreeRunConfig { + max_depth: Some(0), + max_instances: Some(1), + compact_audit: false, + persist_vcir: true, + build_ccr_accumulator: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + enable_transport_request_prefetch: false, + }, + ) + .expect("run replay root-only audit"); + + assert_eq!(out.tree.instances_processed, 1); + assert_eq!(out.tree.instances_failed, 0); + assert_eq!(out.publication_points.len(), 1); + } + + #[test] + fn payload_replay_api_root_only_apnic_archive_runs_with_timing() { + let temp = tempfile::tempdir().expect("tempdir"); + let db_path = temp.path().join("db"); + let store = crate::storage::RocksStore::open(&db_path).expect("open db"); + let (tal_bytes, ta_der, archive_root, locks_path, validation_time) = apnic_replay_inputs(); + if !archive_root.is_dir() || !locks_path.is_file() { + eprintln!( + "skipping payload replay api timing test; missing fixtures: archive={} locks={}", + archive_root.display(), + locks_path.display() + ); + return; + } + + let timing = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339: "2026-03-13T03:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-03-13T02:30:00Z".to_string(), + tal_url: None, + db_path: Some(db_path.to_string_lossy().into_owned()), + }); + + let out = run_tree_from_tal_and_ta_der_payload_replay_serial_audit_with_timing( + &store, + &crate::policy::Policy::default(), + &tal_bytes, + &ta_der, + None, + &archive_root, + &locks_path, + validation_time, + &TreeRunConfig { + max_depth: Some(0), + max_instances: Some(1), + compact_audit: false, + persist_vcir: true, + build_ccr_accumulator: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + enable_transport_request_prefetch: false, + }, + &timing, + ) + .expect("run replay root-only audit with timing"); + + assert_eq!(out.tree.instances_processed, 1); + let timing_json = temp.path().join("timing_replay.json"); + timing + .write_json(&timing_json, 20) + .expect("write timing json"); + let json: serde_json::Value = + serde_json::from_slice(&std::fs::read(&timing_json).expect("read timing json")) + .expect("parse timing json"); + let counts = json.get("counts").expect("counts"); + assert!( + counts + .get("repo_sync_rrdp_ok_total") + .and_then(|v| v.as_u64()) + .unwrap_or(0) + >= 1 + ); + } + + #[test] + fn payload_delta_replay_api_rejects_base_locks_sha_mismatch() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = crate::storage::RocksStore::open(&temp.path().join("db")).expect("open db"); + let ( + tal_bytes, + ta_der, + base_archive, + _base_locks, + delta_archive, + delta_locks, + validation_time, + ) = apnic_delta_replay_inputs(); + let wrong_base_locks = temp.path().join("wrong-base-locks.json"); + std::fs::write(&wrong_base_locks, b"wrong-base-locks").expect("write wrong base locks"); + let err = run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit( + &store, + &crate::policy::Policy::default(), + &tal_bytes, + &ta_der, + None, + &base_archive, + &wrong_base_locks, + &delta_archive, + &delta_locks, + validation_time, + validation_time, + &TreeRunConfig { + max_depth: Some(0), + max_instances: Some(1), + compact_audit: false, + persist_vcir: true, + build_ccr_accumulator: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + enable_transport_request_prefetch: false, + }, + ) + .unwrap_err(); + assert!(matches!(err, RunTreeFromTalError::Replay(_)), "{err}"); + } + + #[test] + fn payload_delta_replay_api_reports_setup_error_for_missing_inputs() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = crate::storage::RocksStore::open(&temp.path().join("db")).expect("open db"); + let tal_bytes = std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal") + .expect("read apnic tal fixture"); + let ta_der = + std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta fixture"); + let err = run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit( + &store, + &crate::policy::Policy::default(), + &tal_bytes, + &ta_der, + None, + std::path::Path::new("tests/fixtures/missing-base-archive"), + std::path::Path::new("tests/fixtures/missing-base-locks.json"), + std::path::Path::new("tests/fixtures/missing-delta-archive"), + std::path::Path::new("tests/fixtures/missing-delta-locks.json"), + time::OffsetDateTime::now_utc(), + time::OffsetDateTime::now_utc(), + &TreeRunConfig { + max_depth: Some(0), + max_instances: Some(1), + compact_audit: false, + persist_vcir: true, + build_ccr_accumulator: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + enable_transport_request_prefetch: false, + }, + ) + .unwrap_err(); + assert!(matches!(err, RunTreeFromTalError::Replay(_)), "{err}"); + } + + #[test] + fn payload_delta_replay_api_root_only_apnic_bundle_runs() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = crate::storage::RocksStore::open(&temp.path().join("db")).expect("open db"); + let ( + tal_bytes, + ta_der, + base_archive, + base_locks, + delta_archive, + delta_locks, + validation_time, + ) = apnic_delta_replay_inputs(); + if !base_archive.is_dir() + || !base_locks.is_file() + || !delta_archive.is_dir() + || !delta_locks.is_file() + { + eprintln!( + "skipping payload delta replay api test; missing fixtures: base_archive={} base_locks={} delta_archive={} delta_locks={}", + base_archive.display(), + base_locks.display(), + delta_archive.display(), + delta_locks.display() + ); + return; + } + + let out = run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit( + &store, + &crate::policy::Policy::default(), + &tal_bytes, + &ta_der, + None, + &base_archive, + &base_locks, + &delta_archive, + &delta_locks, + validation_time, + validation_time, + &TreeRunConfig { + max_depth: Some(0), + max_instances: Some(1), + compact_audit: false, + persist_vcir: true, + build_ccr_accumulator: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + enable_transport_request_prefetch: false, + }, + ) + .expect("run delta replay root-only audit"); + + assert_eq!(out.tree.instances_processed, 1); + assert_eq!(out.tree.instances_failed, 0); + assert_eq!(out.publication_points.len(), 1); + } + + #[test] + fn payload_delta_replay_api_root_only_apnic_bundle_runs_with_timing() { + let temp = tempfile::tempdir().expect("tempdir"); + let db_path = temp.path().join("db"); + let store = crate::storage::RocksStore::open(&db_path).expect("open db"); + let ( + tal_bytes, + ta_der, + base_archive, + base_locks, + delta_archive, + delta_locks, + validation_time, + ) = apnic_delta_replay_inputs(); + if !base_archive.is_dir() + || !base_locks.is_file() + || !delta_archive.is_dir() + || !delta_locks.is_file() + { + eprintln!( + "skipping payload delta replay timing test; missing fixtures: base_archive={} base_locks={} delta_archive={} delta_locks={}", + base_archive.display(), + base_locks.display(), + delta_archive.display(), + delta_locks.display() + ); + return; + } + let timing = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339: "2026-03-16T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-03-15T10:00:00Z".to_string(), + tal_url: None, + db_path: Some(db_path.to_string_lossy().into_owned()), + }); + let out = run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit_with_timing( + &store, + &crate::policy::Policy::default(), + &tal_bytes, + &ta_der, + None, + &base_archive, + &base_locks, + &delta_archive, + &delta_locks, + validation_time, + validation_time, + &TreeRunConfig { + max_depth: Some(0), + max_instances: Some(1), + compact_audit: false, + persist_vcir: true, + build_ccr_accumulator: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + enable_transport_request_prefetch: false, + }, + &timing, + ) + .expect("run delta replay root-only audit with timing"); + assert_eq!(out.tree.instances_processed, 1); + let timing_json = temp.path().join("timing_delta_replay.json"); + timing + .write_json(&timing_json, 20) + .expect("write timing json"); + let json: serde_json::Value = + serde_json::from_slice(&std::fs::read(&timing_json).expect("read timing json")) + .expect("parse timing json"); + assert_eq!( + json["phases"]["payload_delta_replay_base_total"]["count"].as_u64(), + Some(1) + ); + assert_eq!( + json["phases"]["payload_delta_replay_target_total"]["count"].as_u64(), + Some(1) + ); + } +} diff --git a/crates/panda-rpki-validator/src/validation/tree_parallel.rs b/crates/panda-rpki-validator/src/validation/tree_parallel.rs index f7858f6..f3e6243 100644 --- a/crates/panda-rpki-validator/src/validation/tree_parallel.rs +++ b/crates/panda-rpki-validator/src/validation/tree_parallel.rs @@ -29,3815 +29,15 @@ use crate::validation::tree_runner::{ FreshPublicationPointFinalizeOutput, FreshPublicationPointStage, Rpkiv1PublicationPointRunner, }; -#[derive(Clone, Debug)] -struct QueuedCaInstance { - id: u64, - handle: CaInstanceHandle, - parent_id: Option, - discovered_from: Option, -} - -#[derive(Clone, Debug)] -struct ReadyCaInstance { - node: QueuedCaInstance, - repo_outcome: RepoSyncRuntimeOutcome, - ready_enqueued_at: Instant, -} - -struct InflightPublicationPoint { - node: QueuedCaInstance, - fresh_stage: FreshPublicationPointStage, - objects_prepare: ParallelObjectsPrepare, - repo_outcome: RepoSyncRuntimeOutcome, - warnings: Vec, - started_at: Instant, - objects_started_at: Instant, - task_count: usize, - tasks_submitted: usize, - first_task_submitted_at: Option, - last_task_submitted_at: Option, - first_result_at: Option, - last_result_at: Option, - worker_ms_total: u64, - worker_ms_max: u64, - queue_wait_ms_total: u64, - queue_wait_ms_max: u64, - finalize_enqueued_at: Option, - results: Vec, -} - -struct FinishedPublicationPoint { - node: FinishedPublicationPointNode, - result: FinishedPublicationPointResult, -} - -#[derive(Clone, Debug)] -struct FinishedPublicationPointNode { - id: u64, - parent_id: Option, - discovered_from: Option, - manifest_rsync_uri: String, -} - -impl FinishedPublicationPointNode { - fn from_queued(node: QueuedCaInstance) -> Self { - Self { - id: node.id, - parent_id: node.parent_id, - discovered_from: node.discovered_from, - manifest_rsync_uri: node.handle.manifest_rsync_uri, - } - } -} - -#[derive(Debug)] -enum FinishedPublicationPointResult { - Ok { - source: PublicationPointSource, - warnings: Vec, - objects: ObjectsOutput, - audit: PublicationPointAudit, - cir_fresh_objects: Vec, - cir_cached_objects: Vec, - }, - Err(String), -} - -struct FinalizeTask { - state: InflightPublicationPoint, -} - -/// Outcome of the pure compute phase for one ready publication point. -/// -/// `compute_ready_publication_point_stage` only performs read-only validation -/// work (publication point cache lookup, fresh snapshot staging, ROA prepare) -/// and returns this enum. `apply_ready_publication_point_stage` then performs -/// every write to control-loop state (`ca_queue`/`next_id`, `finished`, -/// `pending_roa_dispatch`, `pending_finalization`, `inflight_publication_points`) -/// in the same per-publication-point order the monolithic staging function did. -/// Each variant payload is boxed so the enum itself stays small. -enum StageOutcome { - /// Publication point cache hit: the run result is fully built in memory and - /// only needs child enqueueing plus a `finished` entry. - CacheHit(Box), - /// Fresh staging failed: apply runs the existing blocking - /// `run_publication_point` fallback inline on the control thread. - FreshError(Box), - /// Fresh staging succeeded and ROA prepare returned complete objects (no - /// ROA tasks to dispatch): apply hands the publication point to the - /// finalize worker through `pending_finalization` instead of running the - /// finalize synchronously on the control thread. - Complete(Box), - /// Fresh staging succeeded with a staged objects plan that contains zero - /// ROA tasks: apply queues the finalize task directly. - ZeroTask(Box), - /// Fresh staging succeeded with ROA tasks to dispatch: apply appends the - /// tasks to `pending_roa_dispatch` and registers the inflight publication - /// point. - Fresh(Box), -} - -struct CacheHitOutcome { - ready: ReadyCaInstance, - publication_point_started: Instant, - result: PublicationPointRunResult, -} - -struct FreshErrorOutcome { - ready: ReadyCaInstance, - publication_point_started: Instant, -} - -struct CompleteOutcome { - ready: ReadyCaInstance, - publication_point_started: Instant, - fresh_stage: FreshPublicationPointStage, - warnings: Vec, - objects: ObjectsOutput, -} - -struct StagedOutcome { - ready: ReadyCaInstance, - publication_point_started: Instant, - fresh_stage: FreshPublicationPointStage, - warnings: Vec, - objects_stage: ParallelObjectsStage, -} - -struct FinalizeWorkerResult { - finished: FinishedPublicationPoint, - metrics: FinalizePublicationPointMetrics, -} - -/// Task submitted to the experimental ready-stage worker pool: everything -/// `compute_ready_publication_point_stage` needs for one ready publication -/// point. -struct ReadyStageTask { - ready: ReadyCaInstance, - ready_queue_len_after_pop: usize, - submitted_at: Instant, -} - -/// Result drained from the ready-stage worker pool: the compute outcome and -/// its metrics plus per-task pool timing for the `phase2_stage_pool_stats` -/// observability event. -struct ReadyStageWorkerResult { - outcome: StageOutcome, - metrics: ReadyStageMetrics, - queue_wait_ms: u64, - worker_ms: u64, -} - -/// Executor borrowing the publication point runner so stage workers can run -/// the read-only compute phase off the control thread. The runner is shared -/// with the finalize worker and the ROA pool in the same way; the scoped pool -/// guarantees all borrows end before the enclosing `std::thread::scope`. -struct ReadyStageTaskExecutor<'a> { - runner: &'a Rpkiv1PublicationPointRunner<'a>, -} - -impl<'a> ObjectTaskExecutor for ReadyStageTaskExecutor<'a> { - fn execute(&self, _worker_index: usize, task: ReadyStageTask) -> ReadyStageWorkerResult { - let worker_started = Instant::now(); - let queue_wait_ms = worker_started - .saturating_duration_since(task.submitted_at) - .as_millis() as u64; - let (outcome, metrics) = compute_ready_publication_point_stage( - self.runner, - task.ready, - task.ready_queue_len_after_pop, - ); - ReadyStageWorkerResult { - outcome, - metrics, - queue_wait_ms, - worker_ms: elapsed_ms(worker_started), - } - } -} - -type ReadyStagePool<'scope, 'env> = ScopedObjectWorkerPool< - 'scope, - 'env, - ReadyStageTask, - ReadyStageWorkerResult, - ReadyStageTaskExecutor<'env>, ->; - -#[derive(Default)] -struct StageDispatchMetrics { - submitted: usize, - queue_full: bool, - duration_ms: u64, -} - -#[derive(Default)] -struct StageDrainMetrics { - results_drained: usize, - queue_wait_ms_total: u64, - queue_wait_ms_max: u64, - worker_ms_total: u64, - worker_ms_max: u64, - duration_ms: u64, -} - -#[derive(Default)] -struct ReadyStageMetrics { - manifest_rsync_uri: Option, - publication_point_rsync_uri: Option, - ready_count: usize, - fallback_count: usize, - complete_count: usize, - staged_count: usize, - zero_task_count: usize, - error_count: usize, - discovered_children: usize, - locked_files: usize, - roa_tasks: usize, - aspa_objects: usize, - stage_fresh_ms: u64, - snapshot_prepare_ms: u64, - snapshot_current_index_lock_ms: u64, - snapshot_manifest_load_ms: u64, - snapshot_manifest_index_lookup_ms: u64, - snapshot_manifest_blob_load_ms: u64, - snapshot_manifest_decode_ms: u64, - snapshot_replay_guard_ms: u64, - replay_meta_hit_count: usize, - replay_meta_miss_count: usize, - snapshot_manifest_entries_ms: u64, - snapshot_pack_files_ms: u64, - snapshot_pack_files_index_lookup_ms: u64, - snapshot_pack_files_blob_load_ms: u64, - snapshot_ee_path_validate_ms: u64, - snapshot_manifest_file_count: usize, - child_discovery_ms: u64, - child_enqueue_ms: u64, - ready_queue_wait_ms: u64, - ready_queue_len_after_pop: usize, - roa_presence_scan_ms: u64, - roa_cache_view_ms: u64, - direct_finalize_ms: u64, - fallback_full_run_ms: u64, - prepare_ms: u64, - build_roa_tasks_ms: u64, - total_ms: u64, -} - -#[derive(Default)] -struct ReadyStageBatchMetrics { - ready_count: usize, - fallback_count: usize, - complete_count: usize, - staged_count: usize, - zero_task_count: usize, - error_count: usize, - discovered_children: usize, - locked_files: usize, - roa_tasks: usize, - aspa_objects: usize, - stage_fresh_ms_total: u64, - stage_fresh_ms_max: u64, - stage_fresh_ms_max_manifest_rsync_uri: Option, - stage_fresh_ms_max_publication_point_rsync_uri: Option, - snapshot_prepare_ms_total: u64, - snapshot_prepare_ms_max: u64, - snapshot_current_index_lock_ms_total: u64, - snapshot_current_index_lock_ms_max: u64, - snapshot_manifest_load_ms_total: u64, - snapshot_manifest_load_ms_max: u64, - snapshot_manifest_index_lookup_ms_total: u64, - snapshot_manifest_index_lookup_ms_max: u64, - snapshot_manifest_blob_load_ms_total: u64, - snapshot_manifest_blob_load_ms_max: u64, - snapshot_manifest_decode_ms_total: u64, - snapshot_manifest_decode_ms_max: u64, - snapshot_replay_guard_ms_total: u64, - snapshot_replay_guard_ms_max: u64, - replay_meta_hit_count: usize, - replay_meta_miss_count: usize, - snapshot_manifest_entries_ms_total: u64, - snapshot_manifest_entries_ms_max: u64, - snapshot_pack_files_ms_total: u64, - snapshot_pack_files_ms_max: u64, - snapshot_pack_files_index_lookup_ms_total: u64, - snapshot_pack_files_index_lookup_ms_max: u64, - snapshot_pack_files_blob_load_ms_total: u64, - snapshot_pack_files_blob_load_ms_max: u64, - snapshot_ee_path_validate_ms_total: u64, - snapshot_ee_path_validate_ms_max: u64, - snapshot_manifest_file_count_total: usize, - snapshot_manifest_file_count_max: usize, - child_discovery_ms_total: u64, - child_discovery_ms_max: u64, - child_enqueue_ms_total: u64, - child_enqueue_ms_max: u64, - ready_queue_wait_ms_total: u64, - ready_queue_wait_ms_max: u64, - roa_presence_scan_ms_total: u64, - roa_presence_scan_ms_max: u64, - roa_cache_view_ms_total: u64, - roa_cache_view_ms_max: u64, - direct_finalize_ms_total: u64, - direct_finalize_ms_max: u64, - fallback_full_run_ms_total: u64, - fallback_full_run_ms_max: u64, - prepare_ms_total: u64, - prepare_ms_max: u64, - build_roa_tasks_ms_total: u64, - build_roa_tasks_ms_max: u64, - total_ms: u64, -} - -impl ReadyStageBatchMetrics { - fn record(&mut self, metrics: ReadyStageMetrics) { - self.ready_count += metrics.ready_count; - self.fallback_count += metrics.fallback_count; - self.complete_count += metrics.complete_count; - self.staged_count += metrics.staged_count; - self.zero_task_count += metrics.zero_task_count; - self.error_count += metrics.error_count; - self.discovered_children += metrics.discovered_children; - self.locked_files += metrics.locked_files; - self.roa_tasks += metrics.roa_tasks; - self.aspa_objects += metrics.aspa_objects; - if metrics.stage_fresh_ms >= self.stage_fresh_ms_max { - self.stage_fresh_ms_max_manifest_rsync_uri = metrics.manifest_rsync_uri.clone(); - self.stage_fresh_ms_max_publication_point_rsync_uri = - metrics.publication_point_rsync_uri.clone(); - } - self.stage_fresh_ms_total += metrics.stage_fresh_ms; - self.stage_fresh_ms_max = self.stage_fresh_ms_max.max(metrics.stage_fresh_ms); - self.snapshot_prepare_ms_total += metrics.snapshot_prepare_ms; - self.snapshot_prepare_ms_max = self - .snapshot_prepare_ms_max - .max(metrics.snapshot_prepare_ms); - self.snapshot_current_index_lock_ms_total += metrics.snapshot_current_index_lock_ms; - self.snapshot_current_index_lock_ms_max = self - .snapshot_current_index_lock_ms_max - .max(metrics.snapshot_current_index_lock_ms); - self.snapshot_manifest_load_ms_total += metrics.snapshot_manifest_load_ms; - self.snapshot_manifest_load_ms_max = self - .snapshot_manifest_load_ms_max - .max(metrics.snapshot_manifest_load_ms); - self.snapshot_manifest_index_lookup_ms_total += metrics.snapshot_manifest_index_lookup_ms; - self.snapshot_manifest_index_lookup_ms_max = self - .snapshot_manifest_index_lookup_ms_max - .max(metrics.snapshot_manifest_index_lookup_ms); - self.snapshot_manifest_blob_load_ms_total += metrics.snapshot_manifest_blob_load_ms; - self.snapshot_manifest_blob_load_ms_max = self - .snapshot_manifest_blob_load_ms_max - .max(metrics.snapshot_manifest_blob_load_ms); - self.snapshot_manifest_decode_ms_total += metrics.snapshot_manifest_decode_ms; - self.snapshot_manifest_decode_ms_max = self - .snapshot_manifest_decode_ms_max - .max(metrics.snapshot_manifest_decode_ms); - self.snapshot_replay_guard_ms_total += metrics.snapshot_replay_guard_ms; - self.snapshot_replay_guard_ms_max = self - .snapshot_replay_guard_ms_max - .max(metrics.snapshot_replay_guard_ms); - self.replay_meta_hit_count += metrics.replay_meta_hit_count; - self.replay_meta_miss_count += metrics.replay_meta_miss_count; - self.snapshot_manifest_entries_ms_total += metrics.snapshot_manifest_entries_ms; - self.snapshot_manifest_entries_ms_max = self - .snapshot_manifest_entries_ms_max - .max(metrics.snapshot_manifest_entries_ms); - self.snapshot_pack_files_ms_total += metrics.snapshot_pack_files_ms; - self.snapshot_pack_files_ms_max = self - .snapshot_pack_files_ms_max - .max(metrics.snapshot_pack_files_ms); - self.snapshot_pack_files_index_lookup_ms_total += - metrics.snapshot_pack_files_index_lookup_ms; - self.snapshot_pack_files_index_lookup_ms_max = self - .snapshot_pack_files_index_lookup_ms_max - .max(metrics.snapshot_pack_files_index_lookup_ms); - self.snapshot_pack_files_blob_load_ms_total += metrics.snapshot_pack_files_blob_load_ms; - self.snapshot_pack_files_blob_load_ms_max = self - .snapshot_pack_files_blob_load_ms_max - .max(metrics.snapshot_pack_files_blob_load_ms); - self.snapshot_ee_path_validate_ms_total += metrics.snapshot_ee_path_validate_ms; - self.snapshot_ee_path_validate_ms_max = self - .snapshot_ee_path_validate_ms_max - .max(metrics.snapshot_ee_path_validate_ms); - self.snapshot_manifest_file_count_total += metrics.snapshot_manifest_file_count; - self.snapshot_manifest_file_count_max = self - .snapshot_manifest_file_count_max - .max(metrics.snapshot_manifest_file_count); - self.child_discovery_ms_total += metrics.child_discovery_ms; - self.child_discovery_ms_max = self.child_discovery_ms_max.max(metrics.child_discovery_ms); - self.child_enqueue_ms_total += metrics.child_enqueue_ms; - self.child_enqueue_ms_max = self.child_enqueue_ms_max.max(metrics.child_enqueue_ms); - self.ready_queue_wait_ms_total += metrics.ready_queue_wait_ms; - self.ready_queue_wait_ms_max = self - .ready_queue_wait_ms_max - .max(metrics.ready_queue_wait_ms); - self.roa_presence_scan_ms_total += metrics.roa_presence_scan_ms; - self.roa_presence_scan_ms_max = self - .roa_presence_scan_ms_max - .max(metrics.roa_presence_scan_ms); - self.roa_cache_view_ms_total += metrics.roa_cache_view_ms; - self.roa_cache_view_ms_max = self.roa_cache_view_ms_max.max(metrics.roa_cache_view_ms); - self.direct_finalize_ms_total += metrics.direct_finalize_ms; - self.direct_finalize_ms_max = self.direct_finalize_ms_max.max(metrics.direct_finalize_ms); - self.fallback_full_run_ms_total += metrics.fallback_full_run_ms; - self.fallback_full_run_ms_max = self - .fallback_full_run_ms_max - .max(metrics.fallback_full_run_ms); - self.prepare_ms_total += metrics.prepare_ms; - self.prepare_ms_max = self.prepare_ms_max.max(metrics.prepare_ms); - self.build_roa_tasks_ms_total += metrics.build_roa_tasks_ms; - self.build_roa_tasks_ms_max = self.build_roa_tasks_ms_max.max(metrics.build_roa_tasks_ms); - self.total_ms += metrics.total_ms; - } -} - -#[derive(Default)] -struct RoaDispatchMetrics { - attempted: usize, - submitted: usize, - queue_full: bool, - pending_remaining: usize, - duration_ms: u64, -} - -#[derive(Default)] -struct ObjectDrainMetrics { - results_drained: usize, - publication_points_completed: usize, - worker_ms_total: u64, - worker_ms_max: u64, - queue_wait_ms_total: u64, - queue_wait_ms_max: u64, - result_budget_exhausted: bool, - duration_ms: u64, -} - -#[derive(Default)] -struct FinalizeSubmitMetrics { - submitted: usize, - queue_full: bool, - duration_ms: u64, -} - -#[derive(Default)] -struct FinalizePublicationPointMetrics { - reduce_ms: u64, - finalize_ms: u64, - finalize_queue_wait_ms: Option, - finalize_worker_ms: u64, - snapshot_pack_ms: u64, - persist_vcir_ms: u64, - persist_build_vcir_ms: u64, - persist_replace_vcir_ms: u64, - persist_replace_breakdown: VcirReplaceTimingBreakdown, - ccr_projection_build_ms: u64, - ccr_append_ms: u64, - audit_build_ms: u64, - locked_files: usize, - child_count: usize, - warning_count: usize, - vrp_count: usize, - vap_count: usize, - router_key_count: usize, - audit_object_count: usize, -} - -#[derive(Default)] -struct FinalizeResultsDrainMetrics { - results_drained: usize, - reduce_ms_total: u64, - reduce_ms_max: u64, - finalize_ms_total: u64, - finalize_ms_max: u64, - finalize_queue_wait_ms_max: u64, - finalize_worker_ms_total: u64, - finalize_worker_ms_max: u64, - snapshot_pack_ms_total: u64, - snapshot_pack_ms_max: u64, - persist_vcir_ms_total: u64, - persist_vcir_ms_max: u64, - persist_build_vcir_ms_total: u64, - persist_build_vcir_ms_max: u64, - persist_replace_vcir_ms_total: u64, - persist_replace_vcir_ms_max: u64, - ccr_projection_build_ms_total: u64, - ccr_projection_build_ms_max: u64, - ccr_append_ms_total: u64, - ccr_append_ms_max: u64, - audit_build_ms_total: u64, - audit_build_ms_max: u64, - duration_ms: u64, -} - -#[derive(Default)] -struct RepoDrainMetrics { - event_count: usize, - completions: usize, - ready_enqueued: usize, - duration_ms: u64, -} - -const REPO_RESULT_DRAIN_MAX_EVENTS: usize = 64; - -fn elapsed_ms(started: Instant) -> u64 { - started.elapsed().as_millis() as u64 -} - -fn emit_control_loop_slow( - duration_ms: u64, - repo_poll_timeout: Duration, - repo_metrics: &RepoDrainMetrics, - ready_batch_metrics: &ReadyStageBatchMetrics, - ca_queue_len: usize, - ready_queue_len: usize, - ca_waiting_repo_identities: usize, - pending_roa_dispatch_len: usize, - inflight_publication_points_len: usize, - pending_finalization_len: usize, - finalize_inflight: usize, -) { - let threshold_ms = crate::progress_log::control_loop_slow_threshold_ms(); - if duration_ms < threshold_ms { - return; - } - crate::progress_log::emit( - "phase2_control_loop_slow", - serde_json::json!({ - "duration_ms": duration_ms, - "slow_threshold_ms": threshold_ms, - "repo_poll_timeout_ms": repo_poll_timeout.as_millis() as u64, - "repo_event_count": repo_metrics.event_count, - "repo_completions": repo_metrics.completions, - "repo_ready_enqueued": repo_metrics.ready_enqueued, - "repo_drain_duration_ms": repo_metrics.duration_ms, - "ready_count": ready_batch_metrics.ready_count, - "ready_batch_duration_ms": ready_batch_metrics.total_ms, - "ready_batch_stage_fresh_ms_total": ready_batch_metrics.stage_fresh_ms_total, - "ready_batch_stage_fresh_ms_max": ready_batch_metrics.stage_fresh_ms_max, - "ready_batch_stage_fresh_ms_max_manifest_rsync_uri": ready_batch_metrics.stage_fresh_ms_max_manifest_rsync_uri, - "ready_batch_child_discovery_ms_total": ready_batch_metrics.child_discovery_ms_total, - "ready_batch_child_discovery_ms_max": ready_batch_metrics.child_discovery_ms_max, - "ready_batch_prepare_ms_total": ready_batch_metrics.prepare_ms_total, - "ready_batch_prepare_ms_max": ready_batch_metrics.prepare_ms_max, - "ready_batch_direct_finalize_ms_total": ready_batch_metrics.direct_finalize_ms_total, - "ready_batch_direct_finalize_ms_max": ready_batch_metrics.direct_finalize_ms_max, - "ca_queue_len": ca_queue_len, - "ready_queue_len": ready_queue_len, - "ca_waiting_repo_identities": ca_waiting_repo_identities, - "pending_roa_dispatch_len": pending_roa_dispatch_len, - "inflight_publication_points_len": inflight_publication_points_len, - "pending_finalization_len": pending_finalization_len, - "finalize_inflight": finalize_inflight, - }), - ); -} - -fn compact_phase2_finished_result( - mut result: PublicationPointRunResult, - compact_audit: bool, -) -> FinishedPublicationPointResult { - result.objects.audit.clear(); - result.objects.local_outputs_cache.clear(); - let cir_fresh_objects = if compact_audit && result.source == PublicationPointSource::Fresh { - result.audit.objects.clone() - } else { - result.cir_fresh_objects - }; - if compact_audit { - result.audit.objects.clear(); - result.audit.warnings.clear(); - } - FinishedPublicationPointResult::Ok { - source: result.source, - warnings: result.warnings, - objects: result.objects, - audit: result.audit, - cir_fresh_objects, - cir_cached_objects: result.cir_cached_objects, - } -} - -fn compact_phase2_finished_result_result( - result: Result, - compact_audit: bool, -) -> FinishedPublicationPointResult { - match result { - Ok(result) => compact_phase2_finished_result(result, compact_audit), - Err(err) => FinishedPublicationPointResult::Err(err), - } -} - -pub fn run_tree_parallel_phase2_audit_multi_root( - roots: Vec, - runner: &Rpkiv1PublicationPointRunner<'_>, - config: &TreeRunConfig, -) -> Result { - if runner.policy.signed_object_failure_policy == SignedObjectFailurePolicy::DropPublicationPoint - { - return run_tree_serial_audit_multi_root(roots, runner, config); - } - - let Some(repo_runtime) = runner.repo_sync_runtime.as_ref() else { - return run_tree_serial_audit_multi_root(roots, runner, config); - }; - if runner.parallel_roa_worker_pool.is_none() { - return run_tree_serial_audit_multi_root(roots, runner, config); - } - - let mut next_id: u64 = 0; - let mut ca_queue: VecDeque = VecDeque::new(); - for root in roots { - ca_queue.push_back(QueuedCaInstance { - id: next_id, - handle: root, - parent_id: None, - discovered_from: None, - }); - next_id += 1; - } - - let mut visited_manifest_uris: HashSet = HashSet::new(); - let mut ca_waiting_repo_by_identity: HashMap> = - HashMap::new(); - let mut ready_queue: VecDeque = VecDeque::new(); - let mut inflight_publication_points: HashMap = HashMap::new(); - let mut pending_finalization: VecDeque = VecDeque::new(); - let mut pending_roa_dispatch: VecDeque = VecDeque::new(); - let mut finished: Vec = Vec::new(); - let mut instances_started = 0usize; - let phase2_config = runner.parallel_phase2_config.as_ref(); - let ready_batch_size = phase2_config - .map(|cfg| cfg.ready_batch_size) - .unwrap_or(256) - .max(1); - let ready_batch_wall_time_budget_ms = phase2_config - .map(|cfg| cfg.ready_batch_wall_time_budget_ms) - .unwrap_or(100) - .max(1); - let ready_batch_wall_time_budget = Duration::from_millis(ready_batch_wall_time_budget_ms); - let object_result_drain_batch_size = phase2_config - .map(|cfg| cfg.object_result_drain_batch_size) - .unwrap_or(2048) - .max(1); - let publication_point_finalize_queue_capacity = phase2_config - .map(|cfg| cfg.publication_point_finalize_queue_capacity) - .unwrap_or(32768) - .max(1); - // Experimental ready-stage pool: `stage_workers == 0` keeps the inline - // compute+apply staging path byte-for-byte; any positive value moves the - // compute phase to a scoped worker pool. - let stage_worker_count = phase2_config.map(|cfg| cfg.stage_workers).unwrap_or(0); - let stage_queue_capacity = phase2_config - .map(|cfg| cfg.worker_queue_capacity) - .unwrap_or(256) - .max(1); - - let (finalize_task_tx, finalize_task_rx) = - mpsc::sync_channel::(publication_point_finalize_queue_capacity); - let (finalize_result_tx, finalize_result_rx) = mpsc::channel::(); - let mut finalize_inflight = 0usize; - - return std::thread::scope(|scope| { - let finalize_worker = scope.spawn(move || { - run_finalize_worker( - runner, - finalize_task_rx, - finalize_result_tx, - config.compact_audit, - ) - }); - - // The stage pool borrows the runner like the finalize worker does; it - // lives inside this scope and is dropped before the scope joins. - let mut stage_pool = if stage_worker_count > 0 { - Some( - ReadyStagePool::new( - scope, - stage_worker_count, - stage_queue_capacity, - ReadyStageTaskExecutor { runner }, - ) - .map_err(TreeRunError::Runner)?, - ) - } else { - None - }; - // Submitted-but-not-yet-drained stage tasks. The drain loop collects - // every available result each turn, so `staging_inflight == 0` also - // implies the stage result channel is empty. - let mut staging_inflight = 0usize; - - let run_result: Result<(), TreeRunError> = (|| { - loop { - let control_loop_started = Instant::now(); - // With the stage pool enabled the batch wall clock covers the - // whole turn (turn-head drain + dispatch + apply); the inline - // path keeps its historical start point at the ready batch. - let turn_stage_started = Instant::now(); - let mut ready_batch_metrics = ReadyStageBatchMetrics::default(); - let mut stage_drain_metrics = StageDrainMetrics::default(); - drain_finalize_results_with_progress( - &finalize_result_rx, - &mut finished, - &mut finalize_inflight, - pending_finalization.len(), - pending_roa_dispatch.len(), - inflight_publication_points.len(), - )?; - flush_pending_roa_dispatch_with_progress( - runner, - &mut pending_roa_dispatch, - &mut inflight_publication_points, - &pending_finalization, - )?; - drain_object_results_with_progress( - runner, - &mut inflight_publication_points, - &mut pending_finalization, - pending_roa_dispatch.len(), - object_result_drain_batch_size, - )?; - if let Some(pool) = stage_pool.as_ref() { - drain_stage_results( - pool, - runner, - &mut next_id, - &mut ca_queue, - &mut pending_roa_dispatch, - &mut inflight_publication_points, - &mut pending_finalization, - &mut finished, - &mut staging_inflight, - &mut ready_batch_metrics, - &mut stage_drain_metrics, - config, - )?; - } - submit_pending_finalization_with_progress( - &finalize_task_tx, - &mut pending_finalization, - &mut finalize_inflight, - publication_point_finalize_queue_capacity, - pending_roa_dispatch.len(), - inflight_publication_points.len(), - )?; - - start_queued_ca_instances( - repo_runtime.as_ref(), - &mut ca_queue, - &mut ready_queue, - &mut ca_waiting_repo_by_identity, - &mut finished, - &mut visited_manifest_uris, - &mut instances_started, - config, - ); - - let repo_poll_timeout = event_poll_timeout( - &ca_queue, - &ready_queue, - &pending_roa_dispatch, - &inflight_publication_points, - &pending_finalization, - finalize_inflight, - staging_inflight, - instances_started, - config, - ); - let repo_metrics = drain_repo_events( - repo_runtime.as_ref(), - &mut ca_waiting_repo_by_identity, - &mut ready_queue, - repo_poll_timeout, - )?; - if repo_metrics.event_count > 0 { - crate::progress_log::emit( - "phase2_repo_events_drain", - serde_json::json!({ - "event_count": repo_metrics.event_count, - "completions": repo_metrics.completions, - "ready_enqueued": repo_metrics.ready_enqueued, - "duration_ms": repo_metrics.duration_ms, - "ready_queue_len": ready_queue.len(), - "ca_waiting_repo_identities": ca_waiting_repo_by_identity.len(), - }), - ); - } - - let ready_batch_started = Instant::now(); - let mut ready_time_budget_exhausted = false; - let mut stage_dispatch_metrics = StageDispatchMetrics::default(); - if let Some(pool) = stage_pool.as_mut() { - // Pool path: the ready batch becomes a dispatch loop that - // submits compute tasks and applies whatever results are - // already available; backpressure requeues for next turn. - stage_dispatch_metrics = submit_ready_batch_to_stage_pool( - pool, - &mut ready_queue, - &mut staging_inflight, - ready_batch_size, - ready_batch_wall_time_budget, - )?; - ready_time_budget_exhausted = stage_dispatch_metrics.queue_full - || (!ready_queue.is_empty() - && ready_batch_started.elapsed() >= ready_batch_wall_time_budget); - drain_stage_results( - pool, - runner, - &mut next_id, - &mut ca_queue, - &mut pending_roa_dispatch, - &mut inflight_publication_points, - &mut pending_finalization, - &mut finished, - &mut staging_inflight, - &mut ready_batch_metrics, - &mut stage_drain_metrics, - config, - )?; - } else { - while ready_batch_metrics.ready_count < ready_batch_size { - let Some(ready) = ready_queue.pop_front() else { - break; - }; - let ready_queue_len_after_pop = ready_queue.len(); - let (outcome, metrics) = compute_ready_publication_point_stage( - runner, - ready, - ready_queue_len_after_pop, - ); - let metrics = apply_ready_publication_point_stage( - runner, - &mut next_id, - &mut ca_queue, - &mut pending_roa_dispatch, - &mut inflight_publication_points, - &mut pending_finalization, - &mut finished, - outcome, - metrics, - config, - config.compact_audit, - ); - ready_batch_metrics.record(metrics); - if ready_batch_metrics.ready_count > 0 - && ready_batch_started.elapsed() >= ready_batch_wall_time_budget - { - ready_time_budget_exhausted = !ready_queue.is_empty(); - break; - } - } - } - if ready_batch_metrics.ready_count > 0 { - ready_batch_metrics.total_ms = if stage_pool.is_some() { - elapsed_ms(turn_stage_started) - } else { - elapsed_ms(ready_batch_started) - }; - let ready_count_budget_exhausted = ready_batch_metrics.ready_count - >= ready_batch_size - && !ready_queue.is_empty(); - ready_time_budget_exhausted = ready_time_budget_exhausted - || (!ready_queue.is_empty() - && ready_batch_metrics.total_ms >= ready_batch_wall_time_budget_ms); - emit_ready_queue_batch_progress( - &ready_batch_metrics, - ready_batch_size, - ready_batch_wall_time_budget_ms, - ready_queue.len(), - ready_count_budget_exhausted, - ready_time_budget_exhausted, - ca_queue.len(), - pending_roa_dispatch.len(), - inflight_publication_points.len(), - pending_finalization.len(), - finalize_inflight, - ); - crate::progress_log::emit( - "phase2_ready_queue_stage_fresh_breakdown", - serde_json::json!({ - "ready_count": ready_batch_metrics.ready_count, - "stage_fresh_ms_total": ready_batch_metrics.stage_fresh_ms_total, - "stage_fresh_ms_max": ready_batch_metrics.stage_fresh_ms_max, - "stage_fresh_ms_max_manifest_rsync_uri": ready_batch_metrics.stage_fresh_ms_max_manifest_rsync_uri, - "stage_fresh_ms_max_publication_point_rsync_uri": ready_batch_metrics.stage_fresh_ms_max_publication_point_rsync_uri, - "snapshot_prepare_ms_total": ready_batch_metrics.snapshot_prepare_ms_total, - "snapshot_prepare_ms_max": ready_batch_metrics.snapshot_prepare_ms_max, - "snapshot_current_index_lock_ms_total": ready_batch_metrics.snapshot_current_index_lock_ms_total, - "snapshot_current_index_lock_ms_max": ready_batch_metrics.snapshot_current_index_lock_ms_max, - "snapshot_manifest_load_ms_total": ready_batch_metrics.snapshot_manifest_load_ms_total, - "snapshot_manifest_load_ms_max": ready_batch_metrics.snapshot_manifest_load_ms_max, - "snapshot_manifest_index_lookup_ms_total": ready_batch_metrics.snapshot_manifest_index_lookup_ms_total, - "snapshot_manifest_index_lookup_ms_max": ready_batch_metrics.snapshot_manifest_index_lookup_ms_max, - "snapshot_manifest_blob_load_ms_total": ready_batch_metrics.snapshot_manifest_blob_load_ms_total, - "snapshot_manifest_blob_load_ms_max": ready_batch_metrics.snapshot_manifest_blob_load_ms_max, - "snapshot_manifest_decode_ms_total": ready_batch_metrics.snapshot_manifest_decode_ms_total, - "snapshot_manifest_decode_ms_max": ready_batch_metrics.snapshot_manifest_decode_ms_max, - "snapshot_replay_guard_ms_total": ready_batch_metrics.snapshot_replay_guard_ms_total, - "snapshot_replay_guard_ms_max": ready_batch_metrics.snapshot_replay_guard_ms_max, - "replay_meta_hit_count": ready_batch_metrics.replay_meta_hit_count, - "replay_meta_miss_count": ready_batch_metrics.replay_meta_miss_count, - "snapshot_manifest_entries_ms_total": ready_batch_metrics.snapshot_manifest_entries_ms_total, - "snapshot_manifest_entries_ms_max": ready_batch_metrics.snapshot_manifest_entries_ms_max, - "snapshot_pack_files_ms_total": ready_batch_metrics.snapshot_pack_files_ms_total, - "snapshot_pack_files_ms_max": ready_batch_metrics.snapshot_pack_files_ms_max, - "snapshot_pack_files_index_lookup_ms_total": ready_batch_metrics.snapshot_pack_files_index_lookup_ms_total, - "snapshot_pack_files_index_lookup_ms_max": ready_batch_metrics.snapshot_pack_files_index_lookup_ms_max, - "snapshot_pack_files_blob_load_ms_total": ready_batch_metrics.snapshot_pack_files_blob_load_ms_total, - "snapshot_pack_files_blob_load_ms_max": ready_batch_metrics.snapshot_pack_files_blob_load_ms_max, - "snapshot_ee_path_validate_ms_total": ready_batch_metrics.snapshot_ee_path_validate_ms_total, - "snapshot_ee_path_validate_ms_max": ready_batch_metrics.snapshot_ee_path_validate_ms_max, - "snapshot_manifest_file_count_total": ready_batch_metrics.snapshot_manifest_file_count_total, - "snapshot_manifest_file_count_max": ready_batch_metrics.snapshot_manifest_file_count_max, - "child_discovery_ms_total": ready_batch_metrics.child_discovery_ms_total, - "child_discovery_ms_max": ready_batch_metrics.child_discovery_ms_max, - "batch_duration_ms": ready_batch_metrics.total_ms, - }), - ); - } - if stage_pool.is_some() - && (stage_dispatch_metrics.submitted > 0 - || stage_drain_metrics.results_drained > 0 - || stage_dispatch_metrics.queue_full) - { - crate::progress_log::emit( - "phase2_stage_pool_stats", - serde_json::json!({ - "stage_workers": stage_worker_count, - "submitted": stage_dispatch_metrics.submitted, - "results_drained": stage_drain_metrics.results_drained, - "queue_full": stage_dispatch_metrics.queue_full, - "staging_inflight": staging_inflight, - "ready_queue_len": ready_queue.len(), - "queue_wait_ms_total": stage_drain_metrics.queue_wait_ms_total, - "queue_wait_ms_max": stage_drain_metrics.queue_wait_ms_max, - "worker_ms_total": stage_drain_metrics.worker_ms_total, - "worker_ms_max": stage_drain_metrics.worker_ms_max, - "dispatch_duration_ms": stage_dispatch_metrics.duration_ms, - "drain_duration_ms": stage_drain_metrics.duration_ms, - }), - ); - } - - flush_pending_roa_dispatch_with_progress( - runner, - &mut pending_roa_dispatch, - &mut inflight_publication_points, - &pending_finalization, - )?; - drain_object_results_with_progress( - runner, - &mut inflight_publication_points, - &mut pending_finalization, - pending_roa_dispatch.len(), - object_result_drain_batch_size, - )?; - submit_pending_finalization_with_progress( - &finalize_task_tx, - &mut pending_finalization, - &mut finalize_inflight, - publication_point_finalize_queue_capacity, - pending_roa_dispatch.len(), - inflight_publication_points.len(), - )?; - drain_finalize_results_with_progress( - &finalize_result_rx, - &mut finished, - &mut finalize_inflight, - pending_finalization.len(), - pending_roa_dispatch.len(), - inflight_publication_points.len(), - )?; - - emit_control_loop_slow( - elapsed_ms(control_loop_started), - repo_poll_timeout, - &repo_metrics, - &ready_batch_metrics, - ca_queue.len(), - ready_queue.len(), - ca_waiting_repo_by_identity.len(), - pending_roa_dispatch.len(), - inflight_publication_points.len(), - pending_finalization.len(), - finalize_inflight, - ); - - if is_complete( - &ca_queue, - &ready_queue, - &ca_waiting_repo_by_identity, - &pending_roa_dispatch, - &inflight_publication_points, - &pending_finalization, - finalize_inflight, - staging_inflight, - instances_started, - config, - ) { - break; - } - } - - repo_runtime - .reset_run_state() - .map_err(TreeRunError::Runner)?; - Ok(()) - })(); - - drop(finalize_task_tx); - // Dropping the pool closes the stage task queues so the scoped stage - // workers exit before the scope joins them below. - drop(stage_pool); - let worker_result = finalize_worker - .join() - .map_err(|_| TreeRunError::Runner("phase2 finalize worker panicked".to_string()))?; - run_result?; - worker_result?; - drain_finalize_results_with_progress( - &finalize_result_rx, - &mut finished, - &mut finalize_inflight, - pending_finalization.len(), - pending_roa_dispatch.len(), - inflight_publication_points.len(), - )?; - if finalize_inflight != 0 || !pending_finalization.is_empty() { - return Err(TreeRunError::Runner(format!( - "phase2 finalize worker stopped with pending work: queued={} inflight={}", - pending_finalization.len(), - finalize_inflight - ))); - } - Ok(build_tree_output(finished)) - }); -} - -fn emit_ready_queue_batch_progress( - metrics: &ReadyStageBatchMetrics, - ready_batch_size: usize, - ready_batch_wall_time_budget_ms: u64, - ready_queue_len_after_batch: usize, - ready_count_budget_exhausted: bool, - ready_time_budget_exhausted: bool, - ca_queue_len_after_batch: usize, - pending_roa_dispatch_len_after_batch: usize, - inflight_publication_points_after_batch: usize, - pending_finalization_len_after_batch: usize, - finalize_inflight_after_batch: usize, -) { - crate::progress_log::emit( - "phase2_ready_queue_batch", - serde_json::json!({ - "ready_count": metrics.ready_count, - "fallback_count": metrics.fallback_count, - "complete_count": metrics.complete_count, - "staged_count": metrics.staged_count, - "zero_task_count": metrics.zero_task_count, - "error_count": metrics.error_count, - "discovered_children": metrics.discovered_children, - "locked_files": metrics.locked_files, - "roa_tasks": metrics.roa_tasks, - "aspa_objects": metrics.aspa_objects, - "stage_fresh_ms_total": metrics.stage_fresh_ms_total, - "stage_fresh_ms_max": metrics.stage_fresh_ms_max, - "stage_fresh_ms_max_manifest_rsync_uri": metrics.stage_fresh_ms_max_manifest_rsync_uri, - "stage_fresh_ms_max_publication_point_rsync_uri": metrics.stage_fresh_ms_max_publication_point_rsync_uri, - "prepare_ms_total": metrics.prepare_ms_total, - "prepare_ms_max": metrics.prepare_ms_max, - "build_roa_tasks_ms_total": metrics.build_roa_tasks_ms_total, - "build_roa_tasks_ms_max": metrics.build_roa_tasks_ms_max, - "batch_duration_ms": metrics.total_ms, - "ready_batch_size": ready_batch_size, - "ready_batch_wall_time_budget_ms": ready_batch_wall_time_budget_ms, - "ready_queue_len_after_batch": ready_queue_len_after_batch, - "ready_queue_budget_exhausted": ready_queue_len_after_batch > 0, - "ready_count_budget_exhausted": ready_count_budget_exhausted, - "ready_time_budget_exhausted": ready_time_budget_exhausted, - "ca_queue_len_after_batch": ca_queue_len_after_batch, - "pending_roa_dispatch_len_after_batch": pending_roa_dispatch_len_after_batch, - "inflight_publication_points_after_batch": inflight_publication_points_after_batch, - "pending_finalization_len_after_batch": pending_finalization_len_after_batch, - "finalize_inflight_after_batch": finalize_inflight_after_batch, - }), - ); - crate::progress_log::emit( - "phase2_ready_queue_control_breakdown", - serde_json::json!({ - "ready_count": metrics.ready_count, - "ready_queue_wait_ms_total": metrics.ready_queue_wait_ms_total, - "ready_queue_wait_ms_max": metrics.ready_queue_wait_ms_max, - "child_enqueue_ms_total": metrics.child_enqueue_ms_total, - "child_enqueue_ms_max": metrics.child_enqueue_ms_max, - "roa_presence_scan_ms_total": metrics.roa_presence_scan_ms_total, - "roa_presence_scan_ms_max": metrics.roa_presence_scan_ms_max, - "roa_cache_view_ms_total": metrics.roa_cache_view_ms_total, - "roa_cache_view_ms_max": metrics.roa_cache_view_ms_max, - "direct_finalize_ms_total": metrics.direct_finalize_ms_total, - "direct_finalize_ms_max": metrics.direct_finalize_ms_max, - "fallback_full_run_ms_total": metrics.fallback_full_run_ms_total, - "fallback_full_run_ms_max": metrics.fallback_full_run_ms_max, - "batch_duration_ms": metrics.total_ms, - }), - ); -} - -fn can_start_more(instances_started: usize, config: &TreeRunConfig) -> bool { - config - .max_instances - .map(|max| instances_started < max) - .unwrap_or(true) -} - -fn start_queued_ca_instances( - repo_runtime: &dyn crate::parallel::repo_runtime::RepoSyncRuntime, - ca_queue: &mut VecDeque, - ready_queue: &mut VecDeque, - ca_waiting_repo_by_identity: &mut HashMap>, - finished: &mut Vec, - visited_manifest_uris: &mut HashSet, - instances_started: &mut usize, - config: &TreeRunConfig, -) { - while can_start_more(*instances_started, config) { - let Some(node) = ca_queue.pop_front() else { - break; - }; - if !visited_manifest_uris.insert(node.handle.manifest_rsync_uri.clone()) { - continue; - } - if !ca_depth_is_allowed(config, node.handle.depth) { - continue; - } - *instances_started += 1; - match repo_runtime.request_publication_point_repo(&node.handle, 0) { - Ok(RepoSyncRequestStatus::Ready { mut outcome, .. }) => { - // Ready here means this CA is reusing repo work that has already completed - // (often due to child prefetch). Do not add the transport duration again. - outcome.repo_sync_duration_ms = 0; - ready_queue.push_back(ReadyCaInstance { - node, - repo_outcome: outcome, - ready_enqueued_at: Instant::now(), - }); - } - Ok(RepoSyncRequestStatus::Pending { identity, .. }) => { - ca_waiting_repo_by_identity - .entry(identity) - .or_default() - .push(node); - } - Err(err) => { - finished.push(FinishedPublicationPoint { - node: FinishedPublicationPointNode::from_queued(node), - result: FinishedPublicationPointResult::Err(err), - }); - } - } - } -} - -fn compute_ready_publication_point_stage( - runner: &Rpkiv1PublicationPointRunner<'_>, - ready: ReadyCaInstance, - ready_queue_len_after_pop: usize, -) -> (StageOutcome, ReadyStageMetrics) { - let publication_point_started = Instant::now(); - let ready_queue_wait_ms = publication_point_started - .saturating_duration_since(ready.ready_enqueued_at) - .as_millis() as u64; - let mut metrics = ReadyStageMetrics { - ready_count: 1, - manifest_rsync_uri: Some(ready.node.handle.manifest_rsync_uri.clone()), - publication_point_rsync_uri: Some(ready.node.handle.publication_point_rsync_uri.clone()), - ready_queue_wait_ms, - ready_queue_len_after_pop, - ..ReadyStageMetrics::default() - }; - let mut warnings = ready.repo_outcome.warnings.clone(); - let repo_outcome = ready.repo_outcome.clone(); - if let Some(result) = runner.observe_or_reuse_publication_point_cache( - &ready.node.handle, - repo_outcome.repo_sync_source.as_deref(), - repo_outcome.repo_sync_phase.as_deref(), - repo_outcome.repo_sync_duration_ms, - repo_outcome.repo_sync_err.as_deref(), - &warnings, - ) { - metrics.complete_count = 1; - metrics.discovered_children = result.discovered_children.len(); - return ( - StageOutcome::CacheHit(Box::new(CacheHitOutcome { - ready, - publication_point_started, - result, - })), - metrics, - ); - } - - let stage_fresh_started = Instant::now(); - let stage = runner.stage_fresh_publication_point_after_repo_ready( - &ready.node.handle, - repo_outcome.repo_sync_ok, - repo_outcome.repo_sync_err.as_deref(), - ); - metrics.stage_fresh_ms = elapsed_ms(stage_fresh_started); - - let fresh_stage = match stage { - Ok(stage) => stage, - Err(err) => { - if metrics.stage_fresh_ms >= crate::progress_log::stage_fresh_slow_threshold_ms() { - crate::progress_log::emit( - "phase2_stage_fresh_slow", - serde_json::json!({ - "manifest_rsync_uri": ready.node.handle.manifest_rsync_uri.as_str(), - "publication_point_rsync_uri": ready.node.handle.publication_point_rsync_uri.as_str(), - "status": "error", - "error": err.error.to_string(), - "stage_fresh_ms": metrics.stage_fresh_ms, - "snapshot_prepare_ms": err.snapshot_prepare_ms, - "repo_sync_source": repo_outcome.repo_sync_source.as_deref(), - "repo_sync_phase": repo_outcome.repo_sync_phase.as_deref(), - "repo_sync_duration_ms": repo_outcome.repo_sync_duration_ms, - }), - ); - } - // The blocking `run_publication_point` fallback stays on the control - // thread; it is executed by the apply phase for this outcome. - return ( - StageOutcome::FreshError(Box::new(FreshErrorOutcome { - ready, - publication_point_started, - })), - metrics, - ); - } - }; - metrics.snapshot_prepare_ms = fresh_stage.snapshot_prepare_ms; - metrics.snapshot_current_index_lock_ms = - fresh_stage.snapshot_prepare_timing.current_index_lock_ms; - metrics.snapshot_manifest_load_ms = fresh_stage.snapshot_prepare_timing.manifest_load_ms; - metrics.snapshot_manifest_index_lookup_ms = - fresh_stage.snapshot_prepare_timing.manifest_index_lookup_ms; - metrics.snapshot_manifest_blob_load_ms = - fresh_stage.snapshot_prepare_timing.manifest_blob_load_ms; - metrics.snapshot_manifest_decode_ms = fresh_stage.snapshot_prepare_timing.manifest_decode_ms; - metrics.snapshot_replay_guard_ms = fresh_stage.snapshot_prepare_timing.replay_guard_ms; - metrics.replay_meta_hit_count = fresh_stage.snapshot_prepare_timing.replay_meta_hit as usize; - metrics.replay_meta_miss_count = fresh_stage.snapshot_prepare_timing.replay_meta_miss as usize; - metrics.snapshot_manifest_entries_ms = fresh_stage.snapshot_prepare_timing.manifest_entries_ms; - metrics.snapshot_pack_files_ms = fresh_stage.snapshot_prepare_timing.pack_files_ms; - metrics.snapshot_pack_files_index_lookup_ms = fresh_stage - .snapshot_prepare_timing - .pack_files_index_lookup_ms; - metrics.snapshot_pack_files_blob_load_ms = - fresh_stage.snapshot_prepare_timing.pack_files_blob_load_ms; - metrics.snapshot_ee_path_validate_ms = fresh_stage.snapshot_prepare_timing.ee_path_validate_ms; - metrics.snapshot_manifest_file_count = fresh_stage.snapshot_prepare_timing.manifest_file_count; - metrics.child_discovery_ms = fresh_stage.child_discovery_ms; - if metrics.stage_fresh_ms >= crate::progress_log::stage_fresh_slow_threshold_ms() { - crate::progress_log::emit( - "phase2_stage_fresh_slow", - serde_json::json!({ - "manifest_rsync_uri": ready.node.handle.manifest_rsync_uri.as_str(), - "publication_point_rsync_uri": ready.node.handle.publication_point_rsync_uri.as_str(), - "status": "ok", - "stage_fresh_ms": metrics.stage_fresh_ms, - "snapshot_prepare_ms": fresh_stage.snapshot_prepare_ms, - "snapshot_current_index_lock_ms": fresh_stage.snapshot_prepare_timing.current_index_lock_ms, - "snapshot_manifest_load_ms": fresh_stage.snapshot_prepare_timing.manifest_load_ms, - "snapshot_manifest_index_lookup_ms": fresh_stage.snapshot_prepare_timing.manifest_index_lookup_ms, - "snapshot_manifest_blob_load_ms": fresh_stage.snapshot_prepare_timing.manifest_blob_load_ms, - "snapshot_manifest_decode_ms": fresh_stage.snapshot_prepare_timing.manifest_decode_ms, - "snapshot_replay_guard_ms": fresh_stage.snapshot_prepare_timing.replay_guard_ms, - "replay_meta_hit": fresh_stage.snapshot_prepare_timing.replay_meta_hit, - "replay_meta_miss": fresh_stage.snapshot_prepare_timing.replay_meta_miss, - "snapshot_manifest_entries_ms": fresh_stage.snapshot_prepare_timing.manifest_entries_ms, - "snapshot_pack_files_ms": fresh_stage.snapshot_prepare_timing.pack_files_ms, - "snapshot_pack_files_index_lookup_ms": fresh_stage.snapshot_prepare_timing.pack_files_index_lookup_ms, - "snapshot_pack_files_blob_load_ms": fresh_stage.snapshot_prepare_timing.pack_files_blob_load_ms, - "snapshot_ee_path_validate_ms": fresh_stage.snapshot_prepare_timing.ee_path_validate_ms, - "snapshot_manifest_file_count": fresh_stage.snapshot_prepare_timing.manifest_file_count, - "child_discovery_ms": fresh_stage.child_discovery_ms, - "child_count": fresh_stage.discovered_children.len(), - "repo_sync_source": repo_outcome.repo_sync_source.as_deref(), - "repo_sync_phase": repo_outcome.repo_sync_phase.as_deref(), - "repo_sync_duration_ms": repo_outcome.repo_sync_duration_ms, - }), - ); - } - warnings.extend(fresh_stage.warnings.clone()); - - metrics.discovered_children = fresh_stage.discovered_children.len(); - - let prepare_started = Instant::now(); - let roa_presence_scan_started = Instant::now(); - let has_roa = fresh_stage - .fresh_point - .files() - .iter() - .any(|file| file.rsync_uri.ends_with(".roa")); - metrics.roa_presence_scan_ms = elapsed_ms(roa_presence_scan_started); - if runner.enable_roa_validation_cache { - if let Some(timing) = runner.timing.as_ref() { - if has_roa { - timing.record_count("roa_validation_cache_roa_candidate_publication_points", 1); - } else { - timing.record_count("roa_validation_cache_skipped_no_roa_publication_points", 1); - } - } - } - let roa_cache_view = if has_roa { - let roa_cache_view_started = Instant::now(); - let view = runner - .roa_validation_cache_view_for_fresh_point(&fresh_stage.fresh_point.manifest_rsync_uri); - metrics.roa_cache_view_ms = elapsed_ms(roa_cache_view_started); - view - } else { - None - }; - let roa_cache = if runner.enable_roa_validation_cache && has_roa { - RoaValidationCacheInput::enabled_with_context( - roa_cache_view.as_ref(), - crate::validation::tree_runner::ca_validation_context_digest_for_ca(&ready.node.handle), - crate::validation::tree_runner::publication_point_cache_policy_fingerprint( - runner.policy, - ), - ) - } else { - RoaValidationCacheInput::disabled() - }; - let ta_constraints = runner - .policy - .ta_constraints - .shared_for_tal(&ready.node.handle.tal_id); - if ta_constraints.is_some() { - if let Some(timing) = runner.timing.as_ref() { - timing.record_count("ta_constraints_parallel_publication_points", 1); - } - } - match prepare_publication_point_for_parallel_roa_with_cache_and_ta_constraints( - ready.node.id, - &fresh_stage.fresh_point, - runner.policy, - fresh_stage.issuer_ca_der.as_ref(), - ready.node.handle.ca_certificate_rsync_uri.as_deref(), - ready.node.handle.effective_ip_resources.as_ref(), - ready.node.handle.effective_as_resources.as_ref(), - runner.validation_time, - runner.persist_vcir, - roa_cache, - ta_constraints, - ) { - ParallelObjectsPrepare::Complete(mut objects) => { - metrics.prepare_ms = elapsed_ms(prepare_started); - runner.record_publication_point_step_ms( - &ready.node.handle.manifest_rsync_uri, - "fresh_objects_prepare", - metrics.prepare_ms, - ); - metrics.complete_count = 1; - metrics.roa_tasks = objects.stats.roa_total; - metrics.aspa_objects = objects.stats.aspa_total; - objects - .router_keys - .extend(fresh_stage.discovered_router_keys.clone()); - objects.local_outputs_cache.extend( - crate::validation::tree_runner::build_router_key_local_outputs( - &ready.node.handle, - &objects.router_keys, - ), - ); - ( - StageOutcome::Complete(Box::new(CompleteOutcome { - ready, - publication_point_started, - fresh_stage, - warnings, - objects, - })), - metrics, - ) - } - ParallelObjectsPrepare::Staged(objects_stage) => { - metrics.prepare_ms = elapsed_ms(prepare_started); - runner.record_publication_point_step_ms( - &ready.node.handle.manifest_rsync_uri, - "fresh_objects_prepare", - metrics.prepare_ms, - ); - metrics.staged_count = 1; - metrics.locked_files = objects_stage.locked_file_count(); - metrics.aspa_objects = objects_stage.aspa_task_count(); - let task_count = objects_stage.roa_task_count(); - metrics.roa_tasks = task_count; - let outcome = StagedOutcome { - ready, - publication_point_started, - fresh_stage, - warnings, - objects_stage, - }; - if task_count == 0 { - metrics.zero_task_count = 1; - (StageOutcome::ZeroTask(Box::new(outcome)), metrics) - } else { - (StageOutcome::Fresh(Box::new(outcome)), metrics) - } - } - } -} - -fn apply_ready_publication_point_stage( - runner: &Rpkiv1PublicationPointRunner<'_>, - next_id: &mut u64, - ca_queue: &mut VecDeque, - pending_roa_dispatch: &mut VecDeque, - inflight_publication_points: &mut HashMap, - pending_finalization: &mut VecDeque, - finished: &mut Vec, - outcome: StageOutcome, - mut metrics: ReadyStageMetrics, - config: &TreeRunConfig, - compact_audit: bool, -) -> ReadyStageMetrics { - match outcome { - StageOutcome::CacheHit(outcome) => { - let CacheHitOutcome { - ready, - publication_point_started, - result, - } = *outcome; - let repo_outcome = ready.repo_outcome.clone(); - let child_enqueue_started = Instant::now(); - enqueue_discovered_children( - runner, - next_id, - ca_queue, - &ready.node, - config, - result.discovered_children.clone(), - ); - metrics.child_enqueue_ms = elapsed_ms(child_enqueue_started); - runner.record_publication_point_step_ms( - metrics.manifest_rsync_uri.as_deref().unwrap_or_default(), - "publication_point_cache_child_enqueue", - metrics.child_enqueue_ms, - ); - finished.push(FinishedPublicationPoint { - node: FinishedPublicationPointNode::from_queued(ready.node), - result: compact_phase2_finished_result(result, compact_audit), - }); - metrics.total_ms = elapsed_ms(publication_point_started); - emit_ready_publication_point_control_slow( - metrics.manifest_rsync_uri.as_deref().unwrap_or_default(), - metrics - .publication_point_rsync_uri - .as_deref() - .unwrap_or_default(), - &repo_outcome, - &metrics, - "publication_point_cache", - false, - ); - metrics - } - StageOutcome::FreshError(outcome) => { - let FreshErrorOutcome { - ready, - publication_point_started, - } = *outcome; - let repo_outcome = ready.repo_outcome.clone(); - metrics.fallback_count = 1; - let fallback_started = Instant::now(); - let fallback = runner.run_publication_point(&ready.node.handle); - metrics.fallback_full_run_ms = elapsed_ms(fallback_started); - if let Ok(result) = fallback.as_ref() { - metrics.discovered_children = result.discovered_children.len(); - let child_enqueue_started = Instant::now(); - enqueue_discovered_children( - runner, - next_id, - ca_queue, - &ready.node, - config, - result.discovered_children.clone(), - ); - metrics.child_enqueue_ms = elapsed_ms(child_enqueue_started); - } - finished.push(FinishedPublicationPoint { - node: FinishedPublicationPointNode::from_queued(ready.node), - result: compact_phase2_finished_result_result(fallback, compact_audit), - }); - metrics.total_ms = elapsed_ms(publication_point_started); - emit_ready_publication_point_control_slow( - metrics.manifest_rsync_uri.as_deref().unwrap_or_default(), - metrics - .publication_point_rsync_uri - .as_deref() - .unwrap_or_default(), - &repo_outcome, - &metrics, - "fallback", - true, - ); - metrics - } - StageOutcome::Complete(outcome) => { - let CompleteOutcome { - ready, - publication_point_started, - fresh_stage, - warnings, - objects, - } = *outcome; - let repo_outcome = ready.repo_outcome.clone(); - let child_enqueue_started = Instant::now(); - enqueue_discovered_children( - runner, - next_id, - ca_queue, - &ready.node, - config, - fresh_stage.discovered_children.clone(), - ); - metrics.child_enqueue_ms = elapsed_ms(child_enqueue_started); - // The finalize no longer runs synchronously here: the publication - // point is queued for the shared finalize worker through the same - // pending_finalization path (queue capacity backpressure and - // finalize_inflight accounting included) as zero-task staging. The - // per-publication-point total timing is recorded by the finalize - // worker, exactly like the zero-task and staged paths. - let direct_finalize_started = Instant::now(); - pending_finalization.push_back(FinalizeTask { - state: InflightPublicationPoint { - node: ready.node, - fresh_stage, - objects_prepare: ParallelObjectsPrepare::Complete(objects), - repo_outcome: repo_outcome.clone(), - warnings, - started_at: publication_point_started, - objects_started_at: Instant::now(), - task_count: 0, - tasks_submitted: 0, - first_task_submitted_at: None, - last_task_submitted_at: None, - first_result_at: None, - last_result_at: None, - worker_ms_total: 0, - worker_ms_max: 0, - queue_wait_ms_total: 0, - queue_wait_ms_max: 0, - finalize_enqueued_at: Some(Instant::now()), - results: Vec::new(), - }, - }); - metrics.direct_finalize_ms = elapsed_ms(direct_finalize_started); - runner.record_publication_point_step_ms( - &metrics.manifest_rsync_uri.clone().unwrap_or_default(), - "fresh_direct_finalize", - metrics.direct_finalize_ms, - ); - metrics.total_ms = elapsed_ms(publication_point_started); - emit_ready_publication_point_control_slow( - metrics.manifest_rsync_uri.as_deref().unwrap_or_default(), - metrics - .publication_point_rsync_uri - .as_deref() - .unwrap_or_default(), - &repo_outcome, - &metrics, - "complete", - false, - ); - metrics - } - StageOutcome::ZeroTask(outcome) => { - let StagedOutcome { - ready, - publication_point_started, - fresh_stage, - warnings, - objects_stage, - } = *outcome; - let repo_outcome = ready.repo_outcome.clone(); - let child_enqueue_started = Instant::now(); - enqueue_discovered_children( - runner, - next_id, - ca_queue, - &ready.node, - config, - fresh_stage.discovered_children.clone(), - ); - metrics.child_enqueue_ms = elapsed_ms(child_enqueue_started); - let build_tasks_started = Instant::now(); - objects_stage.append_roa_tasks_to(pending_roa_dispatch); - metrics.build_roa_tasks_ms = elapsed_ms(build_tasks_started); - runner.record_publication_point_step_ms( - &ready.node.handle.manifest_rsync_uri, - "fresh_build_roa_tasks", - metrics.build_roa_tasks_ms, - ); - let task_count = objects_stage.roa_task_count(); - pending_finalization.push_back(FinalizeTask { - state: InflightPublicationPoint { - node: ready.node, - fresh_stage, - objects_prepare: ParallelObjectsPrepare::Staged(objects_stage), - repo_outcome: repo_outcome.clone(), - warnings, - started_at: publication_point_started, - objects_started_at: Instant::now(), - task_count, - tasks_submitted: 0, - first_task_submitted_at: None, - last_task_submitted_at: None, - first_result_at: None, - last_result_at: None, - worker_ms_total: 0, - worker_ms_max: 0, - queue_wait_ms_total: 0, - queue_wait_ms_max: 0, - finalize_enqueued_at: Some(Instant::now()), - results: Vec::new(), - }, - }); - metrics.total_ms = elapsed_ms(publication_point_started); - emit_ready_publication_point_control_slow( - metrics.manifest_rsync_uri.as_deref().unwrap_or_default(), - metrics - .publication_point_rsync_uri - .as_deref() - .unwrap_or_default(), - &repo_outcome, - &metrics, - "zero_task", - false, - ); - metrics - } - StageOutcome::Fresh(outcome) => { - let StagedOutcome { - ready, - publication_point_started, - fresh_stage, - warnings, - objects_stage, - } = *outcome; - let repo_outcome = ready.repo_outcome.clone(); - let child_enqueue_started = Instant::now(); - enqueue_discovered_children( - runner, - next_id, - ca_queue, - &ready.node, - config, - fresh_stage.discovered_children.clone(), - ); - metrics.child_enqueue_ms = elapsed_ms(child_enqueue_started); - let build_tasks_started = Instant::now(); - objects_stage.append_roa_tasks_to(pending_roa_dispatch); - metrics.build_roa_tasks_ms = elapsed_ms(build_tasks_started); - runner.record_publication_point_step_ms( - &ready.node.handle.manifest_rsync_uri, - "fresh_build_roa_tasks", - metrics.build_roa_tasks_ms, - ); - let task_count = objects_stage.roa_task_count(); - inflight_publication_points.insert( - ready.node.id, - InflightPublicationPoint { - node: ready.node, - fresh_stage, - objects_prepare: ParallelObjectsPrepare::Staged(objects_stage), - repo_outcome: repo_outcome.clone(), - warnings, - started_at: publication_point_started, - objects_started_at: Instant::now(), - task_count, - tasks_submitted: 0, - first_task_submitted_at: None, - last_task_submitted_at: None, - first_result_at: None, - last_result_at: None, - worker_ms_total: 0, - worker_ms_max: 0, - queue_wait_ms_total: 0, - queue_wait_ms_max: 0, - finalize_enqueued_at: None, - results: Vec::with_capacity(task_count), - }, - ); - metrics.total_ms = elapsed_ms(publication_point_started); - emit_ready_publication_point_control_slow( - metrics.manifest_rsync_uri.as_deref().unwrap_or_default(), - metrics - .publication_point_rsync_uri - .as_deref() - .unwrap_or_default(), - &repo_outcome, - &metrics, - "staged", - false, - ); - metrics - } - } -} - -fn emit_ready_publication_point_control_slow( - manifest_rsync_uri: &str, - publication_point_rsync_uri: &str, - repo_outcome: &RepoSyncRuntimeOutcome, - metrics: &ReadyStageMetrics, - status: &str, - force_error_path: bool, -) { - let threshold_ms = crate::progress_log::pp_control_slow_threshold_ms(); - if !force_error_path && metrics.total_ms < threshold_ms { - return; - } - crate::progress_log::emit( - "phase2_ready_publication_point_control_slow", - serde_json::json!({ - "manifest_rsync_uri": manifest_rsync_uri, - "publication_point_rsync_uri": publication_point_rsync_uri, - "status": status, - "repo_sync_source": repo_outcome.repo_sync_source.as_deref(), - "repo_sync_phase": repo_outcome.repo_sync_phase.as_deref(), - "repo_sync_duration_ms": repo_outcome.repo_sync_duration_ms, - "repo_sync_ok": repo_outcome.repo_sync_ok, - "repo_sync_err": repo_outcome.repo_sync_err.as_deref(), - "ready_queue_wait_ms": metrics.ready_queue_wait_ms, - "ready_queue_len_after_pop": metrics.ready_queue_len_after_pop, - "stage_fresh_ms": metrics.stage_fresh_ms, - "child_discovery_ms": metrics.child_discovery_ms, - "child_enqueue_ms": metrics.child_enqueue_ms, - "discovered_children": metrics.discovered_children, - "roa_presence_scan_ms": metrics.roa_presence_scan_ms, - "roa_cache_view_ms": metrics.roa_cache_view_ms, - "prepare_ms": metrics.prepare_ms, - "build_roa_tasks_ms": metrics.build_roa_tasks_ms, - "direct_finalize_ms": metrics.direct_finalize_ms, - "fallback_full_run_ms": metrics.fallback_full_run_ms, - "locked_files": metrics.locked_files, - "roa_tasks": metrics.roa_tasks, - "aspa_objects": metrics.aspa_objects, - "complete_count": metrics.complete_count, - "staged_count": metrics.staged_count, - "zero_task_count": metrics.zero_task_count, - "fallback_count": metrics.fallback_count, - "total_ms": metrics.total_ms, - "slow_threshold_ms": threshold_ms, - }), - ); - crate::progress_log::emit( - "phase2_ready_publication_point_control_snapshot_breakdown", - serde_json::json!({ - "manifest_rsync_uri": manifest_rsync_uri, - "publication_point_rsync_uri": publication_point_rsync_uri, - "status": status, - "snapshot_prepare_ms": metrics.snapshot_prepare_ms, - "snapshot_current_index_lock_ms": metrics.snapshot_current_index_lock_ms, - "snapshot_manifest_load_ms": metrics.snapshot_manifest_load_ms, - "snapshot_manifest_index_lookup_ms": metrics.snapshot_manifest_index_lookup_ms, - "snapshot_manifest_blob_load_ms": metrics.snapshot_manifest_blob_load_ms, - "snapshot_manifest_decode_ms": metrics.snapshot_manifest_decode_ms, - "snapshot_replay_guard_ms": metrics.snapshot_replay_guard_ms, - "replay_meta_hit_count": metrics.replay_meta_hit_count, - "replay_meta_miss_count": metrics.replay_meta_miss_count, - "snapshot_manifest_entries_ms": metrics.snapshot_manifest_entries_ms, - "snapshot_pack_files_ms": metrics.snapshot_pack_files_ms, - "snapshot_pack_files_index_lookup_ms": metrics.snapshot_pack_files_index_lookup_ms, - "snapshot_pack_files_blob_load_ms": metrics.snapshot_pack_files_blob_load_ms, - "snapshot_ee_path_validate_ms": metrics.snapshot_ee_path_validate_ms, - "snapshot_manifest_file_count": metrics.snapshot_manifest_file_count, - "total_ms": metrics.total_ms, - "slow_threshold_ms": threshold_ms, - }), - ); -} - -fn enqueue_discovered_children( - runner: &Rpkiv1PublicationPointRunner<'_>, - next_id: &mut u64, - ca_queue: &mut VecDeque, - parent: &QueuedCaInstance, - config: &TreeRunConfig, - mut children: Vec, -) { - let Some(child_depth) = next_allowed_ca_depth(config, parent.handle.depth) else { - return; - }; - - children.sort_by(|a, b| { - a.handle - .manifest_rsync_uri - .cmp(&b.handle.manifest_rsync_uri) - .then_with(|| { - a.discovered_from - .child_ca_certificate_rsync_uri - .cmp(&b.discovered_from.child_ca_certificate_rsync_uri) - }) - }); - if let Some(runtime) = runner.repo_sync_runtime.as_ref() { - let _ = runtime.prefetch_discovered_children(&children); - } - for child in children { - let mut handle = child.handle.with_depth(child_depth); - handle.parent_manifest_rsync_uri = Some(parent.handle.manifest_rsync_uri.clone()); - ca_queue.push_back(QueuedCaInstance { - id: *next_id, - handle, - parent_id: Some(parent.id), - discovered_from: Some(child.discovered_from), - }); - *next_id += 1; - } -} - -fn finalize_metrics_from_output( - output: &FreshPublicationPointFinalizeOutput, - reduce_ms: u64, - finalize_ms: u64, - finalize_queue_wait_ms: Option, - finalize_worker_ms: u64, - locked_files: usize, -) -> FinalizePublicationPointMetrics { - FinalizePublicationPointMetrics { - reduce_ms, - finalize_ms, - finalize_queue_wait_ms, - finalize_worker_ms, - snapshot_pack_ms: output.snapshot_pack_ms, - persist_vcir_ms: output.persist_vcir_ms, - persist_build_vcir_ms: output.persist_vcir_timing.build_vcir_ms, - persist_replace_vcir_ms: output.persist_vcir_timing.replace_vcir_ms, - persist_replace_breakdown: output.persist_vcir_timing.replace_vcir.clone(), - ccr_projection_build_ms: output.ccr_projection_build_ms, - ccr_append_ms: output.ccr_append_ms, - audit_build_ms: output.audit_build_ms, - locked_files, - child_count: output.result.discovered_children.len(), - warning_count: output.result.warnings.len(), - vrp_count: output.result.objects.vrps.len(), - vap_count: output.result.objects.aspas.len(), - router_key_count: output.result.objects.router_keys.len(), - audit_object_count: output.result.audit.objects.len(), - } -} - -fn emit_finalize_breakdown( - event_name: &str, - manifest_rsync_uri: &str, - publication_point_rsync_uri: &str, - metrics: &FinalizePublicationPointMetrics, -) { - crate::progress_log::emit( - event_name, - serde_json::json!({ - "manifest_rsync_uri": manifest_rsync_uri, - "publication_point_rsync_uri": publication_point_rsync_uri, - "reduce_ms": metrics.reduce_ms, - "finalize_ms": metrics.finalize_ms, - "finalize_queue_wait_ms": metrics.finalize_queue_wait_ms, - "finalize_worker_ms": metrics.finalize_worker_ms, - "snapshot_pack_ms": metrics.snapshot_pack_ms, - "persist_vcir_ms": metrics.persist_vcir_ms, - "persist_build_vcir_ms": metrics.persist_build_vcir_ms, - "persist_replace_vcir_ms": metrics.persist_replace_vcir_ms, - "persist_replace_breakdown": &metrics.persist_replace_breakdown, - "ccr_projection_build_ms": metrics.ccr_projection_build_ms, - "ccr_append_ms": metrics.ccr_append_ms, - "audit_build_ms": metrics.audit_build_ms, - "locked_files": metrics.locked_files, - "child_count": metrics.child_count, - "warning_count": metrics.warning_count, - "vrp_count": metrics.vrp_count, - "vap_count": metrics.vap_count, - "router_key_count": metrics.router_key_count, - "audit_object_count": metrics.audit_object_count, - }), - ); -} - -fn flush_pending_roa_dispatch( - runner: &Rpkiv1PublicationPointRunner<'_>, - pending_roa_dispatch: &mut VecDeque, - inflight_publication_points: &mut HashMap, -) -> Result { - let started = Instant::now(); - let mut metrics = RoaDispatchMetrics::default(); - let Some(pool) = runner.parallel_roa_worker_pool.as_ref() else { - return Ok(metrics); - }; - while let Some(mut task) = pending_roa_dispatch.pop_front() { - metrics.attempted += 1; - let pp_id = task.publication_point_id; - task.submitted_at = Some(Instant::now()); - match pool.try_submit_round_robin(task) { - Ok(_) => { - metrics.submitted += 1; - if let Some(state) = inflight_publication_points.get_mut(&pp_id) { - let now = Instant::now(); - state.tasks_submitted += 1; - if state.first_task_submitted_at.is_none() { - state.first_task_submitted_at = Some(now); - } - state.last_task_submitted_at = Some(now); - } - } - Err(ObjectWorkerSubmitError::QueueFull { task, .. }) => { - pending_roa_dispatch.push_front(task); - metrics.queue_full = true; - break; - } - Err(ObjectWorkerSubmitError::Disconnected { .. }) => { - return Err(TreeRunError::Runner( - "parallel ROA worker queue disconnected".to_string(), - )); - } - } - } - metrics.pending_remaining = pending_roa_dispatch.len(); - metrics.duration_ms = elapsed_ms(started); - Ok(metrics) -} - -fn flush_pending_roa_dispatch_with_progress( - runner: &Rpkiv1PublicationPointRunner<'_>, - pending_roa_dispatch: &mut VecDeque, - inflight_publication_points: &mut HashMap, - pending_finalization: &VecDeque, -) -> Result<(), TreeRunError> { - let dispatch_metrics = - flush_pending_roa_dispatch(runner, pending_roa_dispatch, inflight_publication_points)?; - if dispatch_metrics.attempted > 0 || dispatch_metrics.queue_full { - crate::progress_log::emit( - "phase2_roa_dispatch_batch", - serde_json::json!({ - "attempted": dispatch_metrics.attempted, - "submitted": dispatch_metrics.submitted, - "queue_full": dispatch_metrics.queue_full, - "pending_remaining": dispatch_metrics.pending_remaining, - "duration_ms": dispatch_metrics.duration_ms, - "inflight_publication_points": inflight_publication_points.len(), - "pending_finalization_len": pending_finalization.len(), - }), - ); - } - Ok(()) -} - -fn drain_object_results( - runner: &Rpkiv1PublicationPointRunner<'_>, - inflight_publication_points: &mut HashMap, - pending_finalization: &mut VecDeque, - result_budget: usize, -) -> Result { - let started = Instant::now(); - let mut metrics = ObjectDrainMetrics::default(); - let Some(pool) = runner.parallel_roa_worker_pool.as_ref() else { - return Ok(metrics); - }; - let result_budget = result_budget.max(1); - while metrics.results_drained < result_budget { - let Some(result) = pool - .recv_result_timeout(Duration::from_millis(0)) - .map_err(TreeRunError::Runner)? - else { - break; - }; - metrics.results_drained += 1; - let pp_id = result.publication_point_id; - let _worker_index = result.worker_index; - metrics.worker_ms_total += result.worker_ms; - metrics.worker_ms_max = metrics.worker_ms_max.max(result.worker_ms); - metrics.queue_wait_ms_total += result.queue_wait_ms; - metrics.queue_wait_ms_max = metrics.queue_wait_ms_max.max(result.queue_wait_ms); - let should_finalize = if let Some(state) = inflight_publication_points.get_mut(&pp_id) { - let now = Instant::now(); - if state.first_result_at.is_none() { - state.first_result_at = Some(now); - } - state.last_result_at = Some(now); - state.worker_ms_total += result.worker_ms; - state.worker_ms_max = state.worker_ms_max.max(result.worker_ms); - state.queue_wait_ms_total += result.queue_wait_ms; - state.queue_wait_ms_max = state.queue_wait_ms_max.max(result.queue_wait_ms); - state.results.push(result); - state.results.len() == state.task_count - } else { - false - }; - if should_finalize { - let mut state = inflight_publication_points - .remove(&pp_id) - .expect("inflight publication point must exist"); - state.finalize_enqueued_at = Some(Instant::now()); - metrics.publication_points_completed += 1; - pending_finalization.push_back(FinalizeTask { state }); - } - } - metrics.result_budget_exhausted = metrics.results_drained == result_budget; - metrics.duration_ms = elapsed_ms(started); - Ok(metrics) -} - -fn drain_object_results_with_progress( - runner: &Rpkiv1PublicationPointRunner<'_>, - inflight_publication_points: &mut HashMap, - pending_finalization: &mut VecDeque, - pending_roa_dispatch_len: usize, - result_budget: usize, -) -> Result<(), TreeRunError> { - let drain_metrics = drain_object_results( - runner, - inflight_publication_points, - pending_finalization, - result_budget, - )?; - if drain_metrics.results_drained > 0 || drain_metrics.result_budget_exhausted { - crate::progress_log::emit( - "phase2_object_results_drain", - serde_json::json!({ - "results_drained": drain_metrics.results_drained, - "publication_points_completed": drain_metrics.publication_points_completed, - "result_budget_exhausted": drain_metrics.result_budget_exhausted, - "result_drain_batch_size": result_budget, - "worker_ms_total": drain_metrics.worker_ms_total, - "worker_ms_max": drain_metrics.worker_ms_max, - "queue_wait_ms_total": drain_metrics.queue_wait_ms_total, - "queue_wait_ms_max": drain_metrics.queue_wait_ms_max, - "duration_ms": drain_metrics.duration_ms, - "pending_roa_dispatch_len": pending_roa_dispatch_len, - "inflight_publication_points": inflight_publication_points.len(), - "pending_finalization_len": pending_finalization.len(), - }), - ); - } - Ok(()) -} - -/// Submission endpoint of the stage pool, abstracted so the dispatch loop can -/// be tested with a deterministic backpressure source. -trait ReadyStageSubmitter { - // The error must hand the task back so the dispatch loop can requeue it; - // boxing it away would only shuffle the same bytes around. - #[allow(clippy::result_large_err)] - fn try_submit_ready_stage( - &mut self, - task: ReadyStageTask, - ) -> Result<(), ObjectWorkerSubmitError>; -} - -impl ReadyStageSubmitter for ReadyStagePool<'_, '_> { - fn try_submit_ready_stage( - &mut self, - task: ReadyStageTask, - ) -> Result<(), ObjectWorkerSubmitError> { - self.try_submit_round_robin(task).map(|_| ()) - } -} - -/// Dispatch loop of the pool-enabled ready batch: pop ready publication -/// points and submit their compute tasks to the stage pool, following the -/// `flush_pending_roa_dispatch` backpressure pattern — on a full worker queue -/// the publication point goes back to the head of the ready queue so the next -/// turn retries it first (nothing lost, nothing duplicated). -fn submit_ready_batch_to_stage_pool( - stage_pool: &mut impl ReadyStageSubmitter, - ready_queue: &mut VecDeque, - staging_inflight: &mut usize, - ready_batch_size: usize, - ready_batch_wall_time_budget: Duration, -) -> Result { - let started = Instant::now(); - let mut metrics = StageDispatchMetrics::default(); - while metrics.submitted < ready_batch_size { - let Some(ready) = ready_queue.pop_front() else { - break; - }; - let task = ReadyStageTask { - ready_queue_len_after_pop: ready_queue.len(), - ready, - submitted_at: Instant::now(), - }; - match stage_pool.try_submit_ready_stage(task) { - Ok(_) => { - *staging_inflight += 1; - metrics.submitted += 1; - } - Err(ObjectWorkerSubmitError::QueueFull { task, .. }) => { - ready_queue.push_front(task.ready); - metrics.queue_full = true; - break; - } - Err(ObjectWorkerSubmitError::Disconnected { .. }) => { - return Err(TreeRunError::Runner( - "ready stage worker queue disconnected".to_string(), - )); - } - } - if metrics.submitted > 0 && started.elapsed() >= ready_batch_wall_time_budget { - break; - } - } - metrics.duration_ms = elapsed_ms(started); - Ok(metrics) -} - -/// Collect every available stage result without blocking and run the apply -/// phase for each on the control thread, exactly like the inline path would -/// have done right after compute. Drained metrics aggregate into the same -/// `ReadyStageBatchMetrics`, keeping the `phase2_ready_queue_*` events -/// unchanged. -#[allow(clippy::too_many_arguments)] -fn drain_stage_results( - stage_pool: &ReadyStagePool<'_, '_>, - runner: &Rpkiv1PublicationPointRunner<'_>, - next_id: &mut u64, - ca_queue: &mut VecDeque, - pending_roa_dispatch: &mut VecDeque, - inflight_publication_points: &mut HashMap, - pending_finalization: &mut VecDeque, - finished: &mut Vec, - staging_inflight: &mut usize, - batch_metrics: &mut ReadyStageBatchMetrics, - pool_metrics: &mut StageDrainMetrics, - config: &TreeRunConfig, -) -> Result<(), TreeRunError> { - let started = Instant::now(); - loop { - let Some(result) = stage_pool - .recv_result_timeout(Duration::from_millis(0)) - .map_err(TreeRunError::Runner)? - else { - break; - }; - pool_metrics.results_drained += 1; - pool_metrics.queue_wait_ms_total += result.queue_wait_ms; - pool_metrics.queue_wait_ms_max = pool_metrics.queue_wait_ms_max.max(result.queue_wait_ms); - pool_metrics.worker_ms_total += result.worker_ms; - pool_metrics.worker_ms_max = pool_metrics.worker_ms_max.max(result.worker_ms); - *staging_inflight = staging_inflight.saturating_sub(1); - let metrics = apply_ready_publication_point_stage( - runner, - next_id, - ca_queue, - pending_roa_dispatch, - inflight_publication_points, - pending_finalization, - finished, - result.outcome, - result.metrics, - config, - config.compact_audit, - ); - batch_metrics.record(metrics); - } - pool_metrics.duration_ms += elapsed_ms(started); - Ok(()) -} - -fn submit_pending_finalization( - finalize_task_tx: &SyncSender, - pending_finalization: &mut VecDeque, - finalize_inflight: &mut usize, -) -> Result { - let started = Instant::now(); - let mut metrics = FinalizeSubmitMetrics::default(); - while let Some(task) = pending_finalization.pop_front() { - match finalize_task_tx.try_send(task) { - Ok(()) => { - metrics.submitted += 1; - *finalize_inflight += 1; - } - Err(TrySendError::Full(task)) => { - pending_finalization.push_front(task); - metrics.queue_full = true; - break; - } - Err(TrySendError::Disconnected(_task)) => { - return Err(TreeRunError::Runner( - "phase2 finalize worker queue disconnected".to_string(), - )); - } - } - } - metrics.duration_ms = elapsed_ms(started); - Ok(metrics) -} - -fn submit_pending_finalization_with_progress( - finalize_task_tx: &SyncSender, - pending_finalization: &mut VecDeque, - finalize_inflight: &mut usize, - finalize_queue_capacity: usize, - pending_roa_dispatch_len: usize, - inflight_publication_points_len: usize, -) -> Result<(), TreeRunError> { - let submit_metrics = - submit_pending_finalization(finalize_task_tx, pending_finalization, finalize_inflight)?; - if submit_metrics.submitted > 0 || submit_metrics.queue_full { - crate::progress_log::emit( - "phase2_finalize_task_submit", - serde_json::json!({ - "submitted": submit_metrics.submitted, - "queue_full": submit_metrics.queue_full, - "duration_ms": submit_metrics.duration_ms, - "finalize_queue_capacity": finalize_queue_capacity, - "pending_finalization_len": pending_finalization.len(), - "finalize_inflight": *finalize_inflight, - "pending_roa_dispatch_len": pending_roa_dispatch_len, - "inflight_publication_points": inflight_publication_points_len, - }), - ); - } - Ok(()) -} - -fn drain_finalize_results( - finalize_result_rx: &Receiver, - finished: &mut Vec, - finalize_inflight: &mut usize, -) -> Result { - let started = Instant::now(); - let mut metrics = FinalizeResultsDrainMetrics::default(); - loop { - match finalize_result_rx.try_recv() { - Ok(result) => { - metrics.results_drained += 1; - metrics.reduce_ms_total += result.metrics.reduce_ms; - metrics.reduce_ms_max = metrics.reduce_ms_max.max(result.metrics.reduce_ms); - metrics.finalize_ms_total += result.metrics.finalize_ms; - metrics.finalize_ms_max = metrics.finalize_ms_max.max(result.metrics.finalize_ms); - metrics.finalize_queue_wait_ms_max = metrics - .finalize_queue_wait_ms_max - .max(result.metrics.finalize_queue_wait_ms.unwrap_or(0)); - metrics.finalize_worker_ms_total += result.metrics.finalize_worker_ms; - metrics.finalize_worker_ms_max = metrics - .finalize_worker_ms_max - .max(result.metrics.finalize_worker_ms); - metrics.snapshot_pack_ms_total += result.metrics.snapshot_pack_ms; - metrics.snapshot_pack_ms_max = metrics - .snapshot_pack_ms_max - .max(result.metrics.snapshot_pack_ms); - metrics.persist_vcir_ms_total += result.metrics.persist_vcir_ms; - metrics.persist_vcir_ms_max = metrics - .persist_vcir_ms_max - .max(result.metrics.persist_vcir_ms); - metrics.persist_build_vcir_ms_total += result.metrics.persist_build_vcir_ms; - metrics.persist_build_vcir_ms_max = metrics - .persist_build_vcir_ms_max - .max(result.metrics.persist_build_vcir_ms); - metrics.persist_replace_vcir_ms_total += result.metrics.persist_replace_vcir_ms; - metrics.persist_replace_vcir_ms_max = metrics - .persist_replace_vcir_ms_max - .max(result.metrics.persist_replace_vcir_ms); - metrics.ccr_projection_build_ms_total += result.metrics.ccr_projection_build_ms; - metrics.ccr_projection_build_ms_max = metrics - .ccr_projection_build_ms_max - .max(result.metrics.ccr_projection_build_ms); - metrics.ccr_append_ms_total += result.metrics.ccr_append_ms; - metrics.ccr_append_ms_max = - metrics.ccr_append_ms_max.max(result.metrics.ccr_append_ms); - metrics.audit_build_ms_total += result.metrics.audit_build_ms; - metrics.audit_build_ms_max = metrics - .audit_build_ms_max - .max(result.metrics.audit_build_ms); - *finalize_inflight = finalize_inflight.saturating_sub(1); - finished.push(result.finished); - } - Err(TryRecvError::Empty) => break, - Err(TryRecvError::Disconnected) => { - if *finalize_inflight == 0 { - break; - } - return Err(TreeRunError::Runner( - "phase2 finalize result channel disconnected".to_string(), - )); - } - } - } - metrics.duration_ms = elapsed_ms(started); - Ok(metrics) -} - -fn drain_finalize_results_with_progress( - finalize_result_rx: &Receiver, - finished: &mut Vec, - finalize_inflight: &mut usize, - pending_finalization_len: usize, - pending_roa_dispatch_len: usize, - inflight_publication_points_len: usize, -) -> Result<(), TreeRunError> { - let drain_metrics = drain_finalize_results(finalize_result_rx, finished, finalize_inflight)?; - if drain_metrics.results_drained >= 64 - || (drain_metrics.results_drained > 0 - && (*finalize_inflight == 0 || pending_finalization_len > 0)) - { - crate::progress_log::emit( - "phase2_finalize_results_drain", - serde_json::json!({ - "results_drained": drain_metrics.results_drained, - "reduce_ms_total": drain_metrics.reduce_ms_total, - "reduce_ms_max": drain_metrics.reduce_ms_max, - "finalize_ms_total": drain_metrics.finalize_ms_total, - "finalize_ms_max": drain_metrics.finalize_ms_max, - "finalize_queue_wait_ms_max": drain_metrics.finalize_queue_wait_ms_max, - "finalize_worker_ms_total": drain_metrics.finalize_worker_ms_total, - "finalize_worker_ms_max": drain_metrics.finalize_worker_ms_max, - "snapshot_pack_ms_total": drain_metrics.snapshot_pack_ms_total, - "snapshot_pack_ms_max": drain_metrics.snapshot_pack_ms_max, - "persist_vcir_ms_total": drain_metrics.persist_vcir_ms_total, - "persist_vcir_ms_max": drain_metrics.persist_vcir_ms_max, - "persist_build_vcir_ms_total": drain_metrics.persist_build_vcir_ms_total, - "persist_build_vcir_ms_max": drain_metrics.persist_build_vcir_ms_max, - "persist_replace_vcir_ms_total": drain_metrics.persist_replace_vcir_ms_total, - "persist_replace_vcir_ms_max": drain_metrics.persist_replace_vcir_ms_max, - "ccr_projection_build_ms_total": drain_metrics.ccr_projection_build_ms_total, - "ccr_projection_build_ms_max": drain_metrics.ccr_projection_build_ms_max, - "ccr_append_ms_total": drain_metrics.ccr_append_ms_total, - "ccr_append_ms_max": drain_metrics.ccr_append_ms_max, - "audit_build_ms_total": drain_metrics.audit_build_ms_total, - "audit_build_ms_max": drain_metrics.audit_build_ms_max, - "duration_ms": drain_metrics.duration_ms, - "pending_finalization_len": pending_finalization_len, - "finalize_inflight": *finalize_inflight, - "pending_roa_dispatch_len": pending_roa_dispatch_len, - "inflight_publication_points": inflight_publication_points_len, - }), - ); - } - Ok(()) -} - -fn run_finalize_worker( - runner: &Rpkiv1PublicationPointRunner<'_>, - finalize_task_rx: Receiver, - finalize_result_tx: mpsc::Sender, - compact_audit: bool, -) -> Result<(), TreeRunError> { - while let Ok(task) = finalize_task_rx.recv() { - let result = finalize_publication_point_state(runner, task.state, compact_audit); - if finalize_result_tx.send(result).is_err() { - return Err(TreeRunError::Runner( - "phase2 finalize result receiver disconnected".to_string(), - )); - } - } - Ok(()) -} - -fn finalize_publication_point_state( - runner: &Rpkiv1PublicationPointRunner<'_>, - state: InflightPublicationPoint, - compact_audit: bool, -) -> FinalizeWorkerResult { - let finalize_worker_started = Instant::now(); - let InflightPublicationPoint { - node, - fresh_stage, - objects_prepare, - repo_outcome, - warnings, - started_at, - objects_started_at, - task_count, - tasks_submitted, - first_task_submitted_at, - last_task_submitted_at, - first_result_at, - last_result_at, - worker_ms_total, - worker_ms_max, - queue_wait_ms_total, - queue_wait_ms_max, - finalize_enqueued_at, - results, - } = state; - let finalize_queue_wait_ms = finalize_enqueued_at.or(last_result_at).map(|ready_at| { - Instant::now() - .saturating_duration_since(ready_at) - .as_millis() as u64 - }); - let objects_processing_ms = objects_started_at.elapsed().as_millis() as u64; - runner.record_publication_point_step_ms( - &node.handle.manifest_rsync_uri, - "fresh_objects_processing_lifetime", - objects_processing_ms, - ); - - let (result, mut metrics, reduce_ms) = match objects_prepare { - ParallelObjectsPrepare::Staged(objects_stage) => { - let reduce_started = Instant::now(); - let locked_files = objects_stage.locked_file_count(); - let reduce_result = - reduce_parallel_roa_stage(objects_stage, results, runner.timing.as_ref()); - let reduce_ms = elapsed_ms(reduce_started); - runner.record_publication_point_step_ms( - &node.handle.manifest_rsync_uri, - "fresh_roa_reduce", - reduce_ms, - ); - - let (result, metrics) = match reduce_result { - Ok(mut objects) => { - let finalize_started = Instant::now(); - objects - .router_keys - .extend(fresh_stage.discovered_router_keys.clone()); - objects.local_outputs_cache.extend( - crate::validation::tree_runner::build_router_key_local_outputs( - &node.handle, - &objects.router_keys, - ), - ); - let finalized = runner.finalize_fresh_publication_point_from_reducer( - &node.handle, - &fresh_stage.fresh_point, - warnings, - objects, - fresh_stage.child_audits, - fresh_stage.discovered_children, - repo_outcome.repo_sync_source.as_deref(), - repo_outcome.repo_sync_phase.as_deref(), - repo_outcome.repo_sync_duration_ms, - repo_outcome.repo_sync_err.as_deref(), - ); - let finalize_ms = elapsed_ms(finalize_started); - match finalized { - Ok(output) => { - let metrics = finalize_metrics_from_output( - &output, - reduce_ms, - finalize_ms, - finalize_queue_wait_ms, - 0, - locked_files, - ); - ( - compact_phase2_finished_result(output.result, compact_audit), - metrics, - ) - } - Err(err) => ( - FinishedPublicationPointResult::Err(err), - FinalizePublicationPointMetrics { - reduce_ms, - finalize_ms, - finalize_queue_wait_ms, - locked_files, - ..FinalizePublicationPointMetrics::default() - }, - ), - } - } - Err(err) => ( - FinishedPublicationPointResult::Err(err), - FinalizePublicationPointMetrics { - reduce_ms, - finalize_queue_wait_ms, - locked_files, - ..FinalizePublicationPointMetrics::default() - }, - ), - }; - (result, metrics, reduce_ms) - } - ParallelObjectsPrepare::Complete(objects) => { - // ROA prepare already produced complete objects for this publication - // point, so there is nothing to reduce; finalize directly. This is - // the former control-thread "direct finalize", now running on the - // finalize worker through the regular task queue. - let locked_files = fresh_stage.fresh_point.files().len(); - let finalize_started = Instant::now(); - let finalized = runner.finalize_fresh_publication_point_from_reducer( - &node.handle, - &fresh_stage.fresh_point, - warnings, - objects, - fresh_stage.child_audits, - fresh_stage.discovered_children, - repo_outcome.repo_sync_source.as_deref(), - repo_outcome.repo_sync_phase.as_deref(), - repo_outcome.repo_sync_duration_ms, - repo_outcome.repo_sync_err.as_deref(), - ); - let finalize_ms = elapsed_ms(finalize_started); - let (result, metrics) = match finalized { - Ok(output) => { - let metrics = finalize_metrics_from_output( - &output, - 0, - finalize_ms, - finalize_queue_wait_ms, - 0, - locked_files, - ); - ( - compact_phase2_finished_result(output.result, compact_audit), - metrics, - ) - } - Err(err) => ( - FinishedPublicationPointResult::Err(err), - FinalizePublicationPointMetrics { - finalize_ms, - finalize_queue_wait_ms, - locked_files, - ..FinalizePublicationPointMetrics::default() - }, - ), - }; - (result, metrics, 0) - } - }; - let finalize_worker_ms = elapsed_ms(finalize_worker_started); - metrics.finalize_worker_ms = finalize_worker_ms; - runner.record_publication_point_step_ms( - &node.handle.manifest_rsync_uri, - "fresh_finalize_worker", - finalize_worker_ms, - ); - runner.record_publication_point_step_ms( - &node.handle.manifest_rsync_uri, - "fresh_finalize_queue_wait", - finalize_queue_wait_ms.unwrap_or(0), - ); - runner.record_publication_point_step_ms( - &node.handle.manifest_rsync_uri, - "fresh_finalize", - metrics.finalize_ms, - ); - runner.record_publication_point_total_ms( - &node.handle.manifest_rsync_uri, - started_at.elapsed().as_millis() as u64, - ); - emit_finalize_breakdown( - "phase2_finalize_worker_breakdown", - node.handle.manifest_rsync_uri.as_str(), - node.handle.publication_point_rsync_uri.as_str(), - &metrics, - ); - crate::progress_log::emit( - "phase2_publication_point_reduced", - serde_json::json!({ - "manifest_rsync_uri": node.handle.manifest_rsync_uri.as_str(), - "publication_point_rsync_uri": node.handle.publication_point_rsync_uri.as_str(), - "objects_processing_ms": objects_processing_ms, - "task_count": task_count, - "tasks_submitted": tasks_submitted, - "first_task_submitted_ms": first_task_submitted_at.map(|t| t.saturating_duration_since(objects_started_at).as_millis() as u64), - "last_task_submitted_ms": last_task_submitted_at.map(|t| t.saturating_duration_since(objects_started_at).as_millis() as u64), - "task_submit_span_ms": match (first_task_submitted_at, last_task_submitted_at) { - (Some(first), Some(last)) => Some(last.saturating_duration_since(first).as_millis() as u64), - _ => None, - }, - "first_result_ms": first_result_at.map(|t| t.saturating_duration_since(objects_started_at).as_millis() as u64), - "last_result_ms": last_result_at.map(|t| t.saturating_duration_since(objects_started_at).as_millis() as u64), - "all_results_ready_ms": last_result_at.map(|t| t.saturating_duration_since(objects_started_at).as_millis() as u64), - "finalize_queue_wait_ms": finalize_queue_wait_ms, - "result_span_ms": match (first_result_at, last_result_at) { - (Some(first), Some(last)) => Some(last.saturating_duration_since(first).as_millis() as u64), - _ => None, - }, - "worker_ms_total": worker_ms_total, - "worker_ms_max": worker_ms_max, - "worker_ms_avg": if task_count > 0 { worker_ms_total / task_count as u64 } else { 0 }, - "queue_wait_ms_total": queue_wait_ms_total, - "queue_wait_ms_max": queue_wait_ms_max, - "queue_wait_ms_avg": if task_count > 0 { queue_wait_ms_total / task_count as u64 } else { 0 }, - "reduce_ms": reduce_ms, - "finalize_ms": metrics.finalize_ms, - "finalize_worker_ms": finalize_worker_ms, - "snapshot_pack_ms": metrics.snapshot_pack_ms, - "persist_vcir_ms": metrics.persist_vcir_ms, - "persist_build_vcir_ms": metrics.persist_build_vcir_ms, - "persist_replace_vcir_ms": metrics.persist_replace_vcir_ms, - "ccr_projection_build_ms": metrics.ccr_projection_build_ms, - "ccr_append_ms": metrics.ccr_append_ms, - "audit_build_ms": metrics.audit_build_ms, - "locked_files": metrics.locked_files, - "child_count": metrics.child_count, - "warning_count": metrics.warning_count, - "vrp_count": metrics.vrp_count, - "vap_count": metrics.vap_count, - "router_key_count": metrics.router_key_count, - "audit_object_count": metrics.audit_object_count, - "total_duration_ms": started_at.elapsed().as_millis() as u64, - }), - ); - FinalizeWorkerResult { - finished: FinishedPublicationPoint { - node: FinishedPublicationPointNode::from_queued(node), - result, - }, - metrics, - } -} - -fn drain_repo_events( - repo_runtime: &dyn crate::parallel::repo_runtime::RepoSyncRuntime, - ca_waiting_repo_by_identity: &mut HashMap>, - ready_queue: &mut VecDeque, - timeout: Duration, -) -> Result { - let started = Instant::now(); - let mut metrics = RepoDrainMetrics::default(); - let events = repo_runtime - .drain_repo_results_timeout(timeout, REPO_RESULT_DRAIN_MAX_EVENTS) - .map_err(TreeRunError::Runner)?; - for event in events { - metrics.event_count += 1; - metrics.completions += event.completions.len(); - for completion in event.completions { - let mut outcome = completion.outcome; - if completion.identity != event.transport_identity { - // Shared RRDP/rsync transports release many publication points, but the transport - // wall time should only be counted once in per-PP stage timing aggregation. - outcome.repo_sync_duration_ms = 0; - } - if let Some(waiters) = ca_waiting_repo_by_identity.remove(&completion.identity) { - metrics.ready_enqueued += waiters.len(); - for node in waiters { - ready_queue.push_back(ReadyCaInstance { - node, - repo_outcome: outcome.clone(), - ready_enqueued_at: Instant::now(), - }); - } - } - } - } - metrics.duration_ms = elapsed_ms(started); - Ok(metrics) -} - -fn event_poll_timeout( - ca_queue: &VecDeque, - ready_queue: &VecDeque, - pending_roa_dispatch: &VecDeque, - inflight_publication_points: &HashMap, - pending_finalization: &VecDeque, - finalize_inflight: usize, - staging_inflight: usize, - instances_started: usize, - config: &TreeRunConfig, -) -> Duration { - // Stage results pending collection must not let the loop sleep: with the - // stage pool enabled a 50ms nap per turn would collapse throughput. - if !ready_queue.is_empty() - || !pending_roa_dispatch.is_empty() - || !inflight_publication_points.is_empty() - || !pending_finalization.is_empty() - || staging_inflight > 0 - || (!ca_queue.is_empty() && can_start_more(instances_started, config)) - { - Duration::from_millis(0) - } else if finalize_inflight > 0 { - Duration::from_millis(10) - } else { - Duration::from_millis(50) - } -} - -fn is_complete( - ca_queue: &VecDeque, - ready_queue: &VecDeque, - ca_waiting_repo_by_identity: &HashMap>, - pending_roa_dispatch: &VecDeque, - inflight_publication_points: &HashMap, - pending_finalization: &VecDeque, - finalize_inflight: usize, - staging_inflight: usize, - instances_started: usize, - config: &TreeRunConfig, -) -> bool { - // `staging_inflight == 0` additionally implies the stage result channel is - // drained empty: the drain loop collects every available result each turn - // and only decrements the counter while collecting. - let ca_queue_done = ca_queue.is_empty() || !can_start_more(instances_started, config); - ca_queue_done - && ready_queue.is_empty() - && ca_waiting_repo_by_identity.is_empty() - && pending_roa_dispatch.is_empty() - && inflight_publication_points.is_empty() - && pending_finalization.is_empty() - && finalize_inflight == 0 - && staging_inflight == 0 -} - -/// Minimum publication points per reduction shard; smaller runs stay -/// single-threaded to avoid thread-spawn overhead dominating the reduction. -const TREE_OUTPUT_MIN_SHARD_LEN: usize = 4096; -/// Hard cap on reduction shards regardless of core count. -const TREE_OUTPUT_MAX_SHARDS: usize = 8; - -fn tree_output_shard_count(len: usize) -> usize { - if len < TREE_OUTPUT_MIN_SHARD_LEN * 2 { - return 1; - } - let parallel = std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(1); - parallel - .min(TREE_OUTPUT_MAX_SHARDS) - .min(len / TREE_OUTPUT_MIN_SHARD_LEN) - .max(1) -} - -/// Per-shard reduction result of the phase2 output merge. Shards cover -/// contiguous ranges of the id-sorted finished list, so concatenating shard -/// outputs in shard order reproduces the sequential single-threaded order -/// exactly. -#[derive(Default)] -struct TreeOutputShardReduction { - instances_processed: usize, - instances_failed: usize, - warnings: Vec, - vrps: Vec, - aspas: Vec, - router_keys: Vec, - publication_points: Vec, - roa_cache_stats: crate::validation::objects::RoaValidationCacheStats, - cir_input: CirInputAccumulator, -} - -fn reduce_finished_shard(items: Vec) -> TreeOutputShardReduction { - let mut reduction = TreeOutputShardReduction::default(); - for item in items { - match item.result { - FinishedPublicationPointResult::Ok { - source, - warnings: result_warnings, - objects, - audit, - cir_fresh_objects, - cir_cached_objects, - } => { - reduction.instances_processed += 1; - reduction.warnings.extend(result_warnings); - reduction.warnings.extend(objects.warnings); - reduction - .roa_cache_stats - .add_assign(&objects.roa_cache_stats); - reduction.vrps.extend(objects.vrps); - reduction.aspas.extend(objects.aspas); - reduction.router_keys.extend(objects.router_keys); - - let mut audit: PublicationPointAudit = audit; - audit.node_id = Some(item.node.id); - audit.parent_node_id = item.node.parent_id; - audit.discovered_from = item.node.discovered_from; - crate::validation::tree::submit_publication_point_cir_input( - &mut reduction.cir_input, - source, - &audit, - &cir_fresh_objects, - &cir_cached_objects, - ) - .expect("CIR input collection from validated audit must not fail"); - reduction.publication_points.push(audit); - } - FinishedPublicationPointResult::Err(err) => { - reduction.instances_failed += 1; - reduction.warnings.push( - Warning::new(format!("publication point failed: {err}")) - .with_context(&item.node.manifest_rsync_uri), - ); - } - } - } - reduction -} - -fn merge_shard_reductions(reductions: Vec) -> TreeOutputShardReduction { - let mut iter = reductions.into_iter(); - let mut merged = iter.next().unwrap_or_default(); - for mut other in iter { - merged.instances_processed += other.instances_processed; - merged.instances_failed += other.instances_failed; - merged.warnings.append(&mut other.warnings); - merged.vrps.append(&mut other.vrps); - merged.aspas.append(&mut other.aspas); - merged.router_keys.append(&mut other.router_keys); - merged - .publication_points - .append(&mut other.publication_points); - merged.roa_cache_stats.add_assign(&other.roa_cache_stats); - merged - .cir_input - .merge(std::mem::take(&mut other.cir_input)) - .expect("CIR input merge from validated audits must not fail"); - } - merged -} - -fn build_tree_output(mut finished: Vec) -> TreeRunAuditOutput { - let total_started = Instant::now(); - finished.sort_by_key(|item| item.node.id); - let sort_ms = total_started.elapsed().as_millis() as u64; - let shard_count = tree_output_shard_count(finished.len()); - - let reduce_started = Instant::now(); - let reductions = if shard_count <= 1 { - vec![reduce_finished_shard(finished)] - } else { - let chunk_len = finished.len().div_ceil(shard_count); - let mut shards: Vec> = Vec::new(); - let mut rest = finished; - while !rest.is_empty() { - let split_at = chunk_len.min(rest.len()); - let tail = rest.split_off(split_at); - shards.push(std::mem::replace(&mut rest, tail)); - } - std::thread::scope(|scope| { - let handles: Vec<_> = shards - .into_iter() - .map(|shard| scope.spawn(move || reduce_finished_shard(shard))) - .collect(); - handles - .into_iter() - .map(|handle| handle.join().expect("tree output reduction shard panicked")) - .collect() - }) - }; - let reduce_ms = reduce_started.elapsed().as_millis() as u64; - - let merge_started = Instant::now(); - let merged = merge_shard_reductions(reductions); - let merge_ms = merge_started.elapsed().as_millis() as u64; - - let finalize_started = Instant::now(); - let output = TreeRunAuditOutput { - tree: TreeRunOutput { - instances_processed: merged.instances_processed, - instances_failed: merged.instances_failed, - warnings: merged.warnings, - vrps: merged.vrps, - aspas: merged.aspas, - router_keys: merged.router_keys, - }, - publication_points: merged.publication_points, - roa_cache_stats: merged.roa_cache_stats, - cir_input: merged.cir_input.finalize(), - }; - let finalize_ms = finalize_started.elapsed().as_millis() as u64; - - crate::progress_log::emit( - "phase2_build_tree_output", - serde_json::json!({ - "sort_ms": sort_ms, - "shard_count": shard_count, - "reduce_ms": reduce_ms, - "merge_ms": merge_ms, - "finalize_ms": finalize_ms, - "total_ms": total_started.elapsed().as_millis() as u64, - "publication_points": output.publication_points.len(), - "instances_processed": output.tree.instances_processed, - "instances_failed": output.tree.instances_failed, - }), - ); - output -} - -pub fn run_tree_parallel_phase2_audit( - root: CaInstanceHandle, - runner: &Rpkiv1PublicationPointRunner<'_>, - config: &TreeRunConfig, -) -> Result { - run_tree_parallel_phase2_audit_multi_root(vec![root], runner, config) -} +include!("tree_parallel/state.rs"); +include!("tree_parallel/phase2.rs"); +include!("tree_parallel/ready_stage.rs"); +include!("tree_parallel/dispatch.rs"); +include!("tree_parallel/finalize.rs"); #[cfg(test)] mod tests { - use super::{ - CacheHitOutcome, CompleteOutcome, FinishedPublicationPoint, FinishedPublicationPointNode, - FinishedPublicationPointResult, InflightPublicationPoint, QueuedCaInstance, - ReadyCaInstance, ReadyStageMetrics, ReadyStageSubmitter, StageOutcome, - TREE_OUTPUT_MAX_SHARDS, TREE_OUTPUT_MIN_SHARD_LEN, apply_ready_publication_point_stage, - build_tree_output, compact_phase2_finished_result, compact_phase2_finished_result_result, - compute_ready_publication_point_stage, event_poll_timeout, finalize_metrics_from_output, - finalize_publication_point_state, is_complete, merge_shard_reductions, - reduce_finished_shard, submit_ready_batch_to_stage_pool, tree_output_shard_count, - }; - use crate::audit::{ - AuditObjectKind, AuditObjectResult, DiscoveredFrom, ObjectAuditEntry, PublicationPointAudit, - }; - use crate::fetch::rsync::{RsyncFetchError, RsyncFetcher}; - use crate::parallel::repo_runtime::RepoSyncRuntimeOutcome; - use crate::policy::{CaFailedFetchPolicy, Policy, SyncPreference}; - use crate::storage::{PackTime, RocksStore}; - use crate::sync::rrdp::Fetcher; - use crate::validation::manifest::{ - FreshPublicationPointTimingBreakdown, FreshValidatedPublicationPoint, - PublicationPointSource, - }; - use crate::validation::objects::{ObjectsOutput, ObjectsStats, ParallelObjectsPrepare}; - use crate::validation::publication_point::PublicationPointSnapshot; - use crate::validation::tree::{ - CaCertificateRef, CaInstanceHandle, DiscoveredChildCaInstance, PublicationPointRunResult, - TreeRunConfig, - }; - use crate::validation::tree_runner::{ - BuildVcirTimingBreakdown, FreshPublicationPointFinalizeOutput, FreshPublicationPointStage, - PersistVcirTimingBreakdown, Rpkiv1PublicationPointRunner, - }; - use std::collections::{HashMap, VecDeque}; - use std::sync::Mutex; - use std::time::{Duration, Instant}; - - fn sample_snapshot() -> PublicationPointSnapshot { - PublicationPointSnapshot { - format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, - manifest_rsync_uri: "rsync://example.test/repo/example.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), - manifest_number_be: vec![1], - this_update: PackTime { - rfc3339_utc: "2026-04-21T00:00:00Z".to_string(), - }, - next_update: PackTime { - rfc3339_utc: "2026-04-22T00:00:00Z".to_string(), - }, - verified_at: PackTime { - rfc3339_utc: "2026-04-21T00:00:01Z".to_string(), - }, - manifest_bytes: vec![1, 2, 3], - files: Vec::new(), - } - } - - fn sample_result() -> PublicationPointRunResult { - PublicationPointRunResult { - source: PublicationPointSource::Fresh, - snapshot: Some(sample_snapshot()), - warnings: Vec::new(), - objects: ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings: Vec::new(), - stats: ObjectsStats::default(), - audit: Vec::new(), - roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), - roa_cache_object_meta: Vec::new(), - }, - audit: PublicationPointAudit::default(), - cir_fresh_objects: Vec::new(), - cir_cached_objects: Vec::new(), - discovered_children: Vec::new(), - } - } - - #[test] - fn compact_phase2_finished_result_drops_snapshot() { - let result = compact_phase2_finished_result(sample_result(), false); - match result { - FinishedPublicationPointResult::Ok { warnings, .. } => { - assert!(warnings.is_empty()); - } - FinishedPublicationPointResult::Err(err) => panic!("unexpected error: {err}"), - } - } - - #[test] - fn compact_phase2_finished_result_can_drop_audit_payload() { - let mut sample = sample_result(); - sample.audit.objects.push(crate::audit::ObjectAuditEntry { - rsync_uri: "rsync://example.test/repo/a.roa".to_string(), - sha256_hex: "11".repeat(32), - kind: crate::audit::AuditObjectKind::Roa, - result: crate::audit::AuditObjectResult::Ok, - detail: None, - }); - sample.audit.warnings.push(crate::audit::AuditWarning { - message: "warning".to_string(), - category: "unclassified".to_string(), - rfc_refs: Vec::new(), - context: None, - }); - sample.objects.audit.push(crate::audit::ObjectAuditEntry { - rsync_uri: "rsync://example.test/repo/b.roa".to_string(), - sha256_hex: "22".repeat(32), - kind: crate::audit::AuditObjectKind::Roa, - result: crate::audit::AuditObjectResult::Ok, - detail: None, - }); - let result = compact_phase2_finished_result(sample, true); - match result { - FinishedPublicationPointResult::Ok { - objects, - audit, - cir_fresh_objects, - .. - } => { - assert!(audit.objects.is_empty()); - assert!(audit.warnings.is_empty()); - assert!(objects.audit.is_empty()); - assert_eq!(cir_fresh_objects.len(), 1); - } - FinishedPublicationPointResult::Err(err) => panic!("unexpected error: {err}"), - } - } - - #[test] - fn compact_phase2_finished_result_result_preserves_err() { - match compact_phase2_finished_result_result(Err("boom".to_string()), false) { - FinishedPublicationPointResult::Err(err) => assert_eq!(err, "boom"), - FinishedPublicationPointResult::Ok { .. } => panic!("error should be preserved"), - } - } - - fn finished_ok_item(id: u64) -> FinishedPublicationPoint { - let mut result = sample_result(); - result.warnings.push(crate::report::Warning::new(format!( - "result-warning-{id:05}" - ))); - result - .objects - .warnings - .push(crate::report::Warning::new(format!( - "objects-warning-{id:05}" - ))); - result.objects.vrps.push(crate::validation::objects::Vrp { - asn: 64496 + id as u32, - prefix: crate::data_model::roa::IpPrefix { - afi: crate::data_model::roa::RoaAfi::Ipv4, - addr: [192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - prefix_len: 24, - }, - max_length: 24, - }); - result.audit.objects.push(crate::audit::ObjectAuditEntry { - rsync_uri: format!("rsync://example.test/repo/{id:05}.roa"), - sha256_hex: format!("{id:064x}"), - kind: crate::audit::AuditObjectKind::Roa, - result: crate::audit::AuditObjectResult::Ok, - detail: None, - }); - FinishedPublicationPoint { - node: FinishedPublicationPointNode { - id, - parent_id: None, - discovered_from: None, - manifest_rsync_uri: format!("rsync://example.test/repo/{id:05}.mft"), - }, - result: compact_phase2_finished_result(result, false), - } - } - - fn finished_err_item(id: u64) -> FinishedPublicationPoint { - FinishedPublicationPoint { - node: FinishedPublicationPointNode { - id, - parent_id: None, - discovered_from: None, - manifest_rsync_uri: format!("rsync://example.test/repo/{id:05}.mft"), - }, - result: FinishedPublicationPointResult::Err(format!("boom-{id:05}")), - } - } - - fn mixed_finished_items(n: u64) -> Vec { - (0..n) - .map(|id| { - if id % 97 == 0 { - finished_err_item(id) - } else { - finished_ok_item(id) - } - }) - .collect() - } - - #[test] - fn tree_output_shard_count_respects_thresholds() { - assert_eq!(tree_output_shard_count(0), 1); - assert_eq!( - tree_output_shard_count(TREE_OUTPUT_MIN_SHARD_LEN * 2 - 1), - 1 - ); - assert!(tree_output_shard_count(TREE_OUTPUT_MIN_SHARD_LEN * 2) >= 1); - let parallel = std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(1); - assert_eq!( - tree_output_shard_count(1_000_000), - parallel.min(TREE_OUTPUT_MAX_SHARDS) - ); - } - - #[test] - fn tree_output_sharded_merge_matches_single_reduction() { - let single = merge_shard_reductions(vec![reduce_finished_shard(mixed_finished_items(300))]); - - let mut chunks: Vec> = Vec::new(); - let mut rest = mixed_finished_items(300); - while !rest.is_empty() { - let split_at = 100.min(rest.len()); - let tail = rest.split_off(split_at); - chunks.push(std::mem::replace(&mut rest, tail)); - } - let merged = - merge_shard_reductions(chunks.into_iter().map(reduce_finished_shard).collect()); - - assert_eq!(single.instances_processed, merged.instances_processed); - assert_eq!(single.instances_failed, merged.instances_failed); - assert_eq!( - format!("{:?}", single.warnings), - format!("{:?}", merged.warnings) - ); - assert_eq!(format!("{:?}", single.vrps), format!("{:?}", merged.vrps)); - assert_eq!( - format!("{:?}", single.publication_points), - format!("{:?}", merged.publication_points) - ); - assert_eq!( - format!("{:?}", single.roa_cache_stats), - format!("{:?}", merged.roa_cache_stats) - ); - assert_eq!(single.cir_input.finalize(), merged.cir_input.finalize()); - } - - #[test] - fn build_tree_output_orders_results_by_node_id() { - let mut items = mixed_finished_items(50); - items.reverse(); - let output = build_tree_output(items); - let got: Vec = output - .publication_points - .iter() - .map(|pp| pp.node_id.expect("node id set")) - .collect(); - let want: Vec = (1..50).filter(|id| id % 97 != 0).collect(); - assert_eq!(got, want); - assert_eq!(output.tree.instances_failed, 1); - let first_warning = format!("{:?}", output.tree.warnings[0]); - assert!( - first_warning.contains("publication point failed: boom-00000"), - "failed publication point warning keeps id order: {first_warning}" - ); - assert_eq!(output.cir_input.fresh_validated_objects.len(), 49); - assert_eq!( - output.cir_input.fresh_validated_objects[0].rsync_uri, - "rsync://example.test/repo/00001.roa" - ); - } - - #[test] - fn finalize_metrics_from_output_captures_breakdown_and_counts() { - let mut result = sample_result(); - result.objects.vrps.push(crate::validation::objects::Vrp { - asn: 64496, - prefix: crate::data_model::roa::IpPrefix { - afi: crate::data_model::roa::RoaAfi::Ipv4, - addr: [192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - prefix_len: 24, - }, - max_length: 24, - }); - result - .objects - .aspas - .push(crate::validation::objects::AspaAttestation { - customer_as_id: 64497, - provider_as_ids: vec![64498], - }); - result.audit.objects.push(ObjectAuditEntry { - rsync_uri: "rsync://example.test/repo/a.roa".to_string(), - sha256_hex: "11".repeat(32), - kind: AuditObjectKind::Roa, - result: AuditObjectResult::Ok, - detail: None, - }); - result - .discovered_children - .push(crate::validation::tree::DiscoveredChildCaInstance { - handle: crate::validation::tree::CaInstanceHandle { - tal_id: "test".to_string(), - ca_certificate: crate::validation::tree::CaCertificateRef::inline_der(vec![1]), - ca_certificate_rsync_uri: Some( - "rsync://example.test/repo/child.cer".to_string(), - ), - effective_ip_resources: None, - effective_as_resources: None, - manifest_rsync_uri: "rsync://example.test/repo/child.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), - rsync_base_uri: "rsync://example.test/repo/".to_string(), - rrdp_notification_uri: None, - parent_manifest_rsync_uri: None, - depth: 1, - }, - discovered_from: crate::audit::DiscoveredFrom { - parent_manifest_rsync_uri: "rsync://example.test/repo/example.mft".to_string(), - child_ca_certificate_rsync_uri: "rsync://example.test/repo/child.cer" - .to_string(), - child_ca_certificate_sha256_hex: "55".repeat(32), - }, - child_entry_projection: None, - }); - let output = FreshPublicationPointFinalizeOutput { - result, - snapshot_pack_ms: 1, - persist_vcir_ms: 2, - persist_vcir_timing: PersistVcirTimingBreakdown { - build_vcir_ms: 3, - replace_vcir_ms: 4, - build_vcir: BuildVcirTimingBreakdown { - related_artifacts_ms: 5, - ..BuildVcirTimingBreakdown::default() - }, - ..PersistVcirTimingBreakdown::default() - }, - ccr_projection_build_ms: 6, - ccr_append_ms: 7, - audit_build_ms: 8, - }; - - let metrics = finalize_metrics_from_output(&output, 9, 10, Some(11), 12, 13); - - assert_eq!(metrics.snapshot_pack_ms, 1); - assert_eq!(metrics.persist_vcir_ms, 2); - assert_eq!(metrics.persist_build_vcir_ms, 3); - assert_eq!(metrics.persist_replace_vcir_ms, 4); - assert_eq!( - metrics.persist_replace_breakdown, - crate::storage::VcirReplaceTimingBreakdown::default() - ); - assert_eq!(metrics.ccr_projection_build_ms, 6); - assert_eq!(metrics.ccr_append_ms, 7); - assert_eq!(metrics.audit_build_ms, 8); - assert_eq!(metrics.reduce_ms, 9); - assert_eq!(metrics.finalize_ms, 10); - assert_eq!(metrics.finalize_queue_wait_ms, Some(11)); - assert_eq!(metrics.finalize_worker_ms, 12); - assert_eq!(metrics.locked_files, 13); - assert_eq!(metrics.child_count, 1); - assert_eq!(metrics.vrp_count, 1); - assert_eq!(metrics.vap_count, 1); - assert_eq!(metrics.audit_object_count, 1); - } - - struct NeverHttpFetcher; - impl Fetcher for NeverHttpFetcher { - fn fetch(&self, _uri: &str) -> Result, String> { - Err("http fetch disabled in test".to_string()) - } - } - - struct FailingRsyncFetcher; - impl RsyncFetcher for FailingRsyncFetcher { - fn fetch_objects( - &self, - _rsync_base_uri: &str, - ) -> Result)>, RsyncFetchError> { - Err(RsyncFetchError::Fetch("rsync disabled in test".to_string())) - } - } - - fn stage_test_runner<'a>( - store: &'a RocksStore, - policy: &'a Policy, - ) -> Rpkiv1PublicationPointRunner<'a> { - Rpkiv1PublicationPointRunner { - store, - policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &FailingRsyncFetcher, - validation_time: time::OffsetDateTime::now_utc(), - timing: None, - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: false, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - } - } - - fn stage_test_handle(manifest_rsync_uri: &str) -> CaInstanceHandle { - CaInstanceHandle { - tal_id: "test".to_string(), - ca_certificate: CaCertificateRef::inline_der(vec![1]), - ca_certificate_rsync_uri: Some("rsync://example.test/repo/ca.cer".to_string()), - effective_ip_resources: None, - effective_as_resources: None, - manifest_rsync_uri: manifest_rsync_uri.to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), - rsync_base_uri: "rsync://example.test/repo/".to_string(), - rrdp_notification_uri: None, - parent_manifest_rsync_uri: None, - depth: 0, - } - } - - fn stage_test_ready(id: u64) -> ReadyCaInstance { - ReadyCaInstance { - node: QueuedCaInstance { - id, - handle: stage_test_handle("rsync://example.test/repo/example.mft"), - parent_id: None, - discovered_from: None, - }, - repo_outcome: stage_test_repo_outcome(), - ready_enqueued_at: Instant::now(), - } - } - - fn stage_test_repo_outcome() -> RepoSyncRuntimeOutcome { - RepoSyncRuntimeOutcome { - repo_sync_ok: true, - repo_sync_err: None, - repo_sync_source: Some("rsync".to_string()), - repo_sync_phase: Some("rsync".to_string()), - repo_sync_duration_ms: 0, - warnings: Vec::new(), - } - } - - fn stage_test_child( - manifest_rsync_uri: &str, - ca_certificate_rsync_uri: &str, - ) -> DiscoveredChildCaInstance { - DiscoveredChildCaInstance { - handle: CaInstanceHandle { - ca_certificate_rsync_uri: Some(ca_certificate_rsync_uri.to_string()), - ..stage_test_handle(manifest_rsync_uri) - }, - discovered_from: DiscoveredFrom { - parent_manifest_rsync_uri: "rsync://example.test/repo/example.mft".to_string(), - child_ca_certificate_rsync_uri: ca_certificate_rsync_uri.to_string(), - child_ca_certificate_sha256_hex: "55".repeat(32), - }, - child_entry_projection: None, - } - } - - fn stage_test_fresh_stage() -> FreshPublicationPointStage { - FreshPublicationPointStage { - fresh_point: FreshValidatedPublicationPoint { - manifest_rsync_uri: "rsync://example.test/repo/example.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), - manifest_number_be: vec![1], - this_update: PackTime { - rfc3339_utc: "2026-04-21T00:00:00Z".to_string(), - }, - next_update: PackTime { - rfc3339_utc: "2026-04-22T00:00:00Z".to_string(), - }, - verified_at: PackTime { - rfc3339_utc: "2026-04-21T00:00:01Z".to_string(), - }, - manifest_bytes: vec![1, 2, 3], - files: Vec::new(), - }, - issuer_ca_der: vec![1u8].into(), - snapshot_prepare_timing: FreshPublicationPointTimingBreakdown::default(), - snapshot_prepare_ms: 0, - discovered_children: Vec::new(), - child_audits: Vec::new(), - discovered_router_keys: Vec::new(), - child_discovery_ms: 0, - warnings: Vec::new(), - } - } - - fn stage_test_metrics() -> ReadyStageMetrics { - ReadyStageMetrics { - ready_count: 1, - manifest_rsync_uri: Some("rsync://example.test/repo/example.mft".to_string()), - publication_point_rsync_uri: Some("rsync://example.test/repo/".to_string()), - ..ReadyStageMetrics::default() - } - } - - #[test] - fn compute_apply_fresh_error_runs_inline_fallback_and_finishes_err() { - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy { - sync_preference: SyncPreference::RsyncOnly, - ca_failed_fetch_policy: CaFailedFetchPolicy::StopAllOutput, - ..Policy::default() - }; - let runner = stage_test_runner(&store, &policy); - let config = TreeRunConfig::default(); - let mut next_id = 1u64; - let mut ca_queue = VecDeque::new(); - let mut pending_roa_dispatch = VecDeque::new(); - let mut inflight_publication_points = HashMap::new(); - let mut pending_finalization = VecDeque::new(); - let mut finished = Vec::new(); - - let (outcome, metrics) = - compute_ready_publication_point_stage(&runner, stage_test_ready(0), 0); - assert!( - matches!(outcome, StageOutcome::FreshError(_)), - "fresh staging against an empty store must fail" - ); - let metrics = apply_ready_publication_point_stage( - &runner, - &mut next_id, - &mut ca_queue, - &mut pending_roa_dispatch, - &mut inflight_publication_points, - &mut pending_finalization, - &mut finished, - outcome, - metrics, - &config, - false, - ); - - assert_eq!(metrics.ready_count, 1); - assert_eq!(metrics.fallback_count, 1); - assert_eq!(metrics.complete_count, 0); - assert_eq!(metrics.staged_count, 0); - assert_eq!(metrics.zero_task_count, 0); - assert_eq!(finished.len(), 1); - match &finished[0].result { - FinishedPublicationPointResult::Err(_) => {} - FinishedPublicationPointResult::Ok { .. } => { - panic!("fallback without repository data must fail") - } - } - assert!(ca_queue.is_empty()); - assert!(pending_roa_dispatch.is_empty()); - assert!(pending_finalization.is_empty()); - assert!(inflight_publication_points.is_empty()); - assert_eq!(next_id, 1); - } - - #[test] - fn apply_cache_hit_enqueues_children_sorted_and_finishes() { - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy::default(); - let runner = stage_test_runner(&store, &policy); - let config = TreeRunConfig::default(); - let mut next_id = 42u64; - let mut ca_queue = VecDeque::new(); - let mut pending_roa_dispatch = VecDeque::new(); - let mut inflight_publication_points = HashMap::new(); - let mut pending_finalization = VecDeque::new(); - let mut finished = Vec::new(); - - let mut result = sample_result(); - result.discovered_children = vec![ - stage_test_child( - "rsync://example.test/repo/z.mft", - "rsync://example.test/repo/z.cer", - ), - stage_test_child( - "rsync://example.test/repo/a.mft", - "rsync://example.test/repo/a.cer", - ), - ]; - let outcome = StageOutcome::CacheHit(Box::new(CacheHitOutcome { - ready: stage_test_ready(7), - publication_point_started: Instant::now(), - result, - })); - let mut metrics = stage_test_metrics(); - metrics.complete_count = 1; - metrics.discovered_children = 2; - let metrics = apply_ready_publication_point_stage( - &runner, - &mut next_id, - &mut ca_queue, - &mut pending_roa_dispatch, - &mut inflight_publication_points, - &mut pending_finalization, - &mut finished, - outcome, - metrics, - &config, - false, - ); - - assert_eq!(finished.len(), 1); - match &finished[0].result { - FinishedPublicationPointResult::Ok { .. } => {} - FinishedPublicationPointResult::Err(err) => { - panic!("cache hit must finish ok: {err}") - } - } - assert_eq!(finished[0].node.id, 7); - // Children are enqueued sorted by manifest URI with sequential ids - // taken from next_id, exactly like the monolithic staging did. - assert_eq!(ca_queue.len(), 2); - assert_eq!( - ca_queue[0].handle.manifest_rsync_uri, - "rsync://example.test/repo/a.mft" - ); - assert_eq!( - ca_queue[1].handle.manifest_rsync_uri, - "rsync://example.test/repo/z.mft" - ); - assert_eq!(ca_queue[0].id, 42); - assert_eq!(ca_queue[1].id, 43); - assert_eq!(ca_queue[0].parent_id, Some(7)); - assert_eq!(ca_queue[1].parent_id, Some(7)); - assert_eq!(next_id, 44); - assert!(pending_roa_dispatch.is_empty()); - assert!(pending_finalization.is_empty()); - assert!(inflight_publication_points.is_empty()); - assert_eq!(metrics.complete_count, 1); - assert_eq!(metrics.discovered_children, 2); - } - - #[test] - fn apply_complete_enqueues_finalize_task_instead_of_finishing() { - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy::default(); - let runner = stage_test_runner(&store, &policy); - let config = TreeRunConfig::default(); - let mut next_id = 1u64; - let mut ca_queue = VecDeque::new(); - let mut pending_roa_dispatch = VecDeque::new(); - let mut inflight_publication_points = HashMap::new(); - let mut pending_finalization = VecDeque::new(); - let mut finished = Vec::new(); - - let outcome = StageOutcome::Complete(Box::new(CompleteOutcome { - ready: stage_test_ready(9), - publication_point_started: Instant::now(), - fresh_stage: stage_test_fresh_stage(), - warnings: Vec::new(), - objects: sample_result().objects, - })); - let mut metrics = stage_test_metrics(); - metrics.complete_count = 1; - let metrics = apply_ready_publication_point_stage( - &runner, - &mut next_id, - &mut ca_queue, - &mut pending_roa_dispatch, - &mut inflight_publication_points, - &mut pending_finalization, - &mut finished, - outcome, - metrics, - &config, - false, - ); - - assert!( - finished.is_empty(), - "complete staging must defer the finalize to the worker queue" - ); - assert_eq!(pending_finalization.len(), 1); - let task = pending_finalization.pop_front().expect("finalize task"); - let state = task.state; - assert_eq!(state.node.id, 9); - assert_eq!(state.task_count, 0); - assert!(state.finalize_enqueued_at.is_some()); - assert!(state.results.is_empty()); - assert!(matches!( - state.objects_prepare, - ParallelObjectsPrepare::Complete(_) - )); - assert!(inflight_publication_points.is_empty()); - assert!(pending_roa_dispatch.is_empty()); - assert!(ca_queue.is_empty()); - assert_eq!(next_id, 1); - assert_eq!(metrics.complete_count, 1); - } - - #[test] - fn finalize_worker_complete_arm_finalizes_without_reduce() { - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy::default(); - let mut runner = stage_test_runner(&store, &policy); - runner.persist_vcir = false; - - let state = InflightPublicationPoint { - node: QueuedCaInstance { - id: 3, - handle: stage_test_handle("rsync://example.test/repo/example.mft"), - parent_id: None, - discovered_from: None, - }, - fresh_stage: stage_test_fresh_stage(), - objects_prepare: ParallelObjectsPrepare::Complete(sample_result().objects), - repo_outcome: stage_test_repo_outcome(), - warnings: Vec::new(), - started_at: Instant::now(), - objects_started_at: Instant::now(), - task_count: 0, - tasks_submitted: 0, - first_task_submitted_at: None, - last_task_submitted_at: None, - first_result_at: None, - last_result_at: None, - worker_ms_total: 0, - worker_ms_max: 0, - queue_wait_ms_total: 0, - queue_wait_ms_max: 0, - finalize_enqueued_at: Some(Instant::now()), - results: Vec::new(), - }; - let result = finalize_publication_point_state(&runner, state, false); - - match result.finished.result { - FinishedPublicationPointResult::Ok { source, .. } => { - assert_eq!(source, PublicationPointSource::Fresh); - } - FinishedPublicationPointResult::Err(err) => { - panic!("complete finalize must succeed: {err}") - } - } - assert_eq!(result.finished.node.id, 3); - assert_eq!(result.metrics.reduce_ms, 0); - assert_eq!(result.metrics.locked_files, 0); - } - - #[test] - fn is_complete_waits_for_staging_inflight() { - let ca_queue = VecDeque::new(); - let ready_queue = VecDeque::new(); - let ca_waiting_repo_by_identity = HashMap::new(); - let pending_roa_dispatch = VecDeque::new(); - let inflight_publication_points = HashMap::new(); - let pending_finalization = VecDeque::new(); - let config = TreeRunConfig::default(); - - // Everything drained and no staging in flight: the loop may exit. - assert!(is_complete( - &ca_queue, - &ready_queue, - &ca_waiting_repo_by_identity, - &pending_roa_dispatch, - &inflight_publication_points, - &pending_finalization, - 0, - 0, - 0, - &config, - )); - // A submitted stage task whose result has not been collected yet must - // keep the loop alive, otherwise its publication point would be lost. - assert!(!is_complete( - &ca_queue, - &ready_queue, - &ca_waiting_repo_by_identity, - &pending_roa_dispatch, - &inflight_publication_points, - &pending_finalization, - 0, - 1, - 0, - &config, - )); - } - - #[test] - fn event_poll_timeout_stays_awake_while_staging() { - let ca_queue = VecDeque::new(); - let ready_queue = VecDeque::new(); - let pending_roa_dispatch = VecDeque::new(); - let inflight_publication_points = HashMap::new(); - let pending_finalization = VecDeque::new(); - let config = TreeRunConfig::default(); - - // Staging in flight: poll must not sleep, or stage results pile up. - assert_eq!( - event_poll_timeout( - &ca_queue, - &ready_queue, - &pending_roa_dispatch, - &inflight_publication_points, - &pending_finalization, - 0, - 1, - 0, - &config, - ), - Duration::from_millis(0) - ); - // Without staging the historical tiers are unchanged. - assert_eq!( - event_poll_timeout( - &ca_queue, - &ready_queue, - &pending_roa_dispatch, - &inflight_publication_points, - &pending_finalization, - 1, - 0, - 0, - &config, - ), - Duration::from_millis(10) - ); - assert_eq!( - event_poll_timeout( - &ca_queue, - &ready_queue, - &pending_roa_dispatch, - &inflight_publication_points, - &pending_finalization, - 0, - 0, - 0, - &config, - ), - Duration::from_millis(50) - ); - } - - struct MockStageSubmitter { - capacity: usize, - submitted_ids: Vec, - } - - impl ReadyStageSubmitter for MockStageSubmitter { - fn try_submit_ready_stage( - &mut self, - task: super::ReadyStageTask, - ) -> Result< - (), - crate::parallel::object_worker::ObjectWorkerSubmitError, - > { - if self.submitted_ids.len() < self.capacity { - self.submitted_ids.push(task.ready.node.id); - Ok(()) - } else { - Err( - crate::parallel::object_worker::ObjectWorkerSubmitError::QueueFull { - worker_index: 0, - task, - }, - ) - } - } - } - - #[test] - fn submit_ready_batch_requeues_on_backpressure_without_loss_or_duplication() { - let mut ready_queue = VecDeque::new(); - for id in [10, 11, 12] { - ready_queue.push_back(stage_test_ready(id)); - } - let mut staging_inflight = 0usize; - - // First turn: only two tasks fit, the third must be returned to the - // head of the ready queue. - let mut submitter = MockStageSubmitter { - capacity: 2, - submitted_ids: Vec::new(), - }; - let metrics = submit_ready_batch_to_stage_pool( - &mut submitter, - &mut ready_queue, - &mut staging_inflight, - 256, - Duration::from_secs(60), - ) - .expect("dispatch"); - assert_eq!(metrics.submitted, 2); - assert!(metrics.queue_full); - assert_eq!(staging_inflight, 2); - assert_eq!(submitter.submitted_ids, vec![10, 11]); - assert_eq!(ready_queue.len(), 1); - assert_eq!(ready_queue[0].node.id, 12); - - // Next turn retries the requeued publication point; across both turns - // every id is submitted exactly once. - let mut submitter = MockStageSubmitter { - capacity: 8, - submitted_ids: Vec::new(), - }; - let metrics = submit_ready_batch_to_stage_pool( - &mut submitter, - &mut ready_queue, - &mut staging_inflight, - 256, - Duration::from_secs(60), - ) - .expect("dispatch"); - assert_eq!(metrics.submitted, 1); - assert!(!metrics.queue_full); - assert_eq!(staging_inflight, 3); - assert_eq!(submitter.submitted_ids, vec![12]); - assert!(ready_queue.is_empty()); - } + include!("tree_parallel/tests/control_loop.rs"); + include!("tree_parallel/tests/stage.rs"); + include!("tree_parallel/tests/backpressure.rs"); } diff --git a/crates/panda-rpki-validator/src/validation/tree_parallel/dispatch.rs b/crates/panda-rpki-validator/src/validation/tree_parallel/dispatch.rs new file mode 100644 index 0000000..32c3162 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_parallel/dispatch.rs @@ -0,0 +1,468 @@ +fn flush_pending_roa_dispatch( + runner: &Rpkiv1PublicationPointRunner<'_>, + pending_roa_dispatch: &mut VecDeque, + inflight_publication_points: &mut HashMap, +) -> Result { + let started = Instant::now(); + let mut metrics = RoaDispatchMetrics::default(); + let Some(pool) = runner.parallel_roa_worker_pool.as_ref() else { + return Ok(metrics); + }; + while let Some(mut task) = pending_roa_dispatch.pop_front() { + metrics.attempted += 1; + let pp_id = task.publication_point_id; + task.submitted_at = Some(Instant::now()); + match pool.try_submit_round_robin(task) { + Ok(_) => { + metrics.submitted += 1; + if let Some(state) = inflight_publication_points.get_mut(&pp_id) { + let now = Instant::now(); + state.tasks_submitted += 1; + if state.first_task_submitted_at.is_none() { + state.first_task_submitted_at = Some(now); + } + state.last_task_submitted_at = Some(now); + } + } + Err(ObjectWorkerSubmitError::QueueFull { task, .. }) => { + pending_roa_dispatch.push_front(task); + metrics.queue_full = true; + break; + } + Err(ObjectWorkerSubmitError::Disconnected { .. }) => { + return Err(TreeRunError::Runner( + "parallel ROA worker queue disconnected".to_string(), + )); + } + } + } + metrics.pending_remaining = pending_roa_dispatch.len(); + metrics.duration_ms = elapsed_ms(started); + Ok(metrics) +} + +fn flush_pending_roa_dispatch_with_progress( + runner: &Rpkiv1PublicationPointRunner<'_>, + pending_roa_dispatch: &mut VecDeque, + inflight_publication_points: &mut HashMap, + pending_finalization: &VecDeque, +) -> Result<(), TreeRunError> { + let dispatch_metrics = + flush_pending_roa_dispatch(runner, pending_roa_dispatch, inflight_publication_points)?; + if dispatch_metrics.attempted > 0 || dispatch_metrics.queue_full { + crate::progress_log::emit( + "phase2_roa_dispatch_batch", + serde_json::json!({ + "attempted": dispatch_metrics.attempted, + "submitted": dispatch_metrics.submitted, + "queue_full": dispatch_metrics.queue_full, + "pending_remaining": dispatch_metrics.pending_remaining, + "duration_ms": dispatch_metrics.duration_ms, + "inflight_publication_points": inflight_publication_points.len(), + "pending_finalization_len": pending_finalization.len(), + }), + ); + } + Ok(()) +} + +fn drain_object_results( + runner: &Rpkiv1PublicationPointRunner<'_>, + inflight_publication_points: &mut HashMap, + pending_finalization: &mut VecDeque, + result_budget: usize, +) -> Result { + let started = Instant::now(); + let mut metrics = ObjectDrainMetrics::default(); + let Some(pool) = runner.parallel_roa_worker_pool.as_ref() else { + return Ok(metrics); + }; + let result_budget = result_budget.max(1); + while metrics.results_drained < result_budget { + let Some(result) = pool + .recv_result_timeout(Duration::from_millis(0)) + .map_err(TreeRunError::Runner)? + else { + break; + }; + metrics.results_drained += 1; + let pp_id = result.publication_point_id; + let _worker_index = result.worker_index; + metrics.worker_ms_total += result.worker_ms; + metrics.worker_ms_max = metrics.worker_ms_max.max(result.worker_ms); + metrics.queue_wait_ms_total += result.queue_wait_ms; + metrics.queue_wait_ms_max = metrics.queue_wait_ms_max.max(result.queue_wait_ms); + let should_finalize = if let Some(state) = inflight_publication_points.get_mut(&pp_id) { + let now = Instant::now(); + if state.first_result_at.is_none() { + state.first_result_at = Some(now); + } + state.last_result_at = Some(now); + state.worker_ms_total += result.worker_ms; + state.worker_ms_max = state.worker_ms_max.max(result.worker_ms); + state.queue_wait_ms_total += result.queue_wait_ms; + state.queue_wait_ms_max = state.queue_wait_ms_max.max(result.queue_wait_ms); + state.results.push(result); + state.results.len() == state.task_count + } else { + false + }; + if should_finalize { + let mut state = inflight_publication_points + .remove(&pp_id) + .expect("inflight publication point must exist"); + state.finalize_enqueued_at = Some(Instant::now()); + metrics.publication_points_completed += 1; + pending_finalization.push_back(FinalizeTask { state }); + } + } + metrics.result_budget_exhausted = metrics.results_drained == result_budget; + metrics.duration_ms = elapsed_ms(started); + Ok(metrics) +} + +fn drain_object_results_with_progress( + runner: &Rpkiv1PublicationPointRunner<'_>, + inflight_publication_points: &mut HashMap, + pending_finalization: &mut VecDeque, + pending_roa_dispatch_len: usize, + result_budget: usize, +) -> Result<(), TreeRunError> { + let drain_metrics = drain_object_results( + runner, + inflight_publication_points, + pending_finalization, + result_budget, + )?; + if drain_metrics.results_drained > 0 || drain_metrics.result_budget_exhausted { + crate::progress_log::emit( + "phase2_object_results_drain", + serde_json::json!({ + "results_drained": drain_metrics.results_drained, + "publication_points_completed": drain_metrics.publication_points_completed, + "result_budget_exhausted": drain_metrics.result_budget_exhausted, + "result_drain_batch_size": result_budget, + "worker_ms_total": drain_metrics.worker_ms_total, + "worker_ms_max": drain_metrics.worker_ms_max, + "queue_wait_ms_total": drain_metrics.queue_wait_ms_total, + "queue_wait_ms_max": drain_metrics.queue_wait_ms_max, + "duration_ms": drain_metrics.duration_ms, + "pending_roa_dispatch_len": pending_roa_dispatch_len, + "inflight_publication_points": inflight_publication_points.len(), + "pending_finalization_len": pending_finalization.len(), + }), + ); + } + Ok(()) +} + +/// Submission endpoint of the stage pool, abstracted so the dispatch loop can +/// be tested with a deterministic backpressure source. +trait ReadyStageSubmitter { + // The error must hand the task back so the dispatch loop can requeue it; + // boxing it away would only shuffle the same bytes around. + #[allow(clippy::result_large_err)] + fn try_submit_ready_stage( + &mut self, + task: ReadyStageTask, + ) -> Result<(), ObjectWorkerSubmitError>; +} + +impl ReadyStageSubmitter for ReadyStagePool<'_, '_> { + fn try_submit_ready_stage( + &mut self, + task: ReadyStageTask, + ) -> Result<(), ObjectWorkerSubmitError> { + self.try_submit_round_robin(task).map(|_| ()) + } +} + +/// Dispatch loop of the pool-enabled ready batch: pop ready publication +/// points and submit their compute tasks to the stage pool, following the +/// `flush_pending_roa_dispatch` backpressure pattern — on a full worker queue +/// the publication point goes back to the head of the ready queue so the next +/// turn retries it first (nothing lost, nothing duplicated). +fn submit_ready_batch_to_stage_pool( + stage_pool: &mut impl ReadyStageSubmitter, + ready_queue: &mut VecDeque, + staging_inflight: &mut usize, + ready_batch_size: usize, + ready_batch_wall_time_budget: Duration, +) -> Result { + let started = Instant::now(); + let mut metrics = StageDispatchMetrics::default(); + while metrics.submitted < ready_batch_size { + let Some(ready) = ready_queue.pop_front() else { + break; + }; + let task = ReadyStageTask { + ready_queue_len_after_pop: ready_queue.len(), + ready, + submitted_at: Instant::now(), + }; + match stage_pool.try_submit_ready_stage(task) { + Ok(_) => { + *staging_inflight += 1; + metrics.submitted += 1; + } + Err(ObjectWorkerSubmitError::QueueFull { task, .. }) => { + ready_queue.push_front(task.ready); + metrics.queue_full = true; + break; + } + Err(ObjectWorkerSubmitError::Disconnected { .. }) => { + return Err(TreeRunError::Runner( + "ready stage worker queue disconnected".to_string(), + )); + } + } + if metrics.submitted > 0 && started.elapsed() >= ready_batch_wall_time_budget { + break; + } + } + metrics.duration_ms = elapsed_ms(started); + Ok(metrics) +} + +/// Collect every available stage result without blocking and run the apply +/// phase for each on the control thread, exactly like the inline path would +/// have done right after compute. Drained metrics aggregate into the same +/// `ReadyStageBatchMetrics`, keeping the `phase2_ready_queue_*` events +/// unchanged. +#[allow(clippy::too_many_arguments)] +fn drain_stage_results( + stage_pool: &ReadyStagePool<'_, '_>, + runner: &Rpkiv1PublicationPointRunner<'_>, + next_id: &mut u64, + ca_queue: &mut VecDeque, + pending_roa_dispatch: &mut VecDeque, + inflight_publication_points: &mut HashMap, + pending_finalization: &mut VecDeque, + finished: &mut Vec, + staging_inflight: &mut usize, + batch_metrics: &mut ReadyStageBatchMetrics, + pool_metrics: &mut StageDrainMetrics, + config: &TreeRunConfig, +) -> Result<(), TreeRunError> { + let started = Instant::now(); + loop { + let Some(result) = stage_pool + .recv_result_timeout(Duration::from_millis(0)) + .map_err(TreeRunError::Runner)? + else { + break; + }; + pool_metrics.results_drained += 1; + pool_metrics.queue_wait_ms_total += result.queue_wait_ms; + pool_metrics.queue_wait_ms_max = pool_metrics.queue_wait_ms_max.max(result.queue_wait_ms); + pool_metrics.worker_ms_total += result.worker_ms; + pool_metrics.worker_ms_max = pool_metrics.worker_ms_max.max(result.worker_ms); + *staging_inflight = staging_inflight.saturating_sub(1); + let metrics = apply_ready_publication_point_stage( + runner, + next_id, + ca_queue, + pending_roa_dispatch, + inflight_publication_points, + pending_finalization, + finished, + result.outcome, + result.metrics, + config, + config.compact_audit, + ); + batch_metrics.record(metrics); + } + pool_metrics.duration_ms += elapsed_ms(started); + Ok(()) +} + +fn submit_pending_finalization( + finalize_task_tx: &SyncSender, + pending_finalization: &mut VecDeque, + finalize_inflight: &mut usize, +) -> Result { + let started = Instant::now(); + let mut metrics = FinalizeSubmitMetrics::default(); + while let Some(task) = pending_finalization.pop_front() { + match finalize_task_tx.try_send(task) { + Ok(()) => { + metrics.submitted += 1; + *finalize_inflight += 1; + } + Err(TrySendError::Full(task)) => { + pending_finalization.push_front(task); + metrics.queue_full = true; + break; + } + Err(TrySendError::Disconnected(_task)) => { + return Err(TreeRunError::Runner( + "phase2 finalize worker queue disconnected".to_string(), + )); + } + } + } + metrics.duration_ms = elapsed_ms(started); + Ok(metrics) +} + +fn submit_pending_finalization_with_progress( + finalize_task_tx: &SyncSender, + pending_finalization: &mut VecDeque, + finalize_inflight: &mut usize, + finalize_queue_capacity: usize, + pending_roa_dispatch_len: usize, + inflight_publication_points_len: usize, +) -> Result<(), TreeRunError> { + let submit_metrics = + submit_pending_finalization(finalize_task_tx, pending_finalization, finalize_inflight)?; + if submit_metrics.submitted > 0 || submit_metrics.queue_full { + crate::progress_log::emit( + "phase2_finalize_task_submit", + serde_json::json!({ + "submitted": submit_metrics.submitted, + "queue_full": submit_metrics.queue_full, + "duration_ms": submit_metrics.duration_ms, + "finalize_queue_capacity": finalize_queue_capacity, + "pending_finalization_len": pending_finalization.len(), + "finalize_inflight": *finalize_inflight, + "pending_roa_dispatch_len": pending_roa_dispatch_len, + "inflight_publication_points": inflight_publication_points_len, + }), + ); + } + Ok(()) +} + +fn drain_finalize_results( + finalize_result_rx: &Receiver, + finished: &mut Vec, + finalize_inflight: &mut usize, +) -> Result { + let started = Instant::now(); + let mut metrics = FinalizeResultsDrainMetrics::default(); + loop { + match finalize_result_rx.try_recv() { + Ok(result) => { + metrics.results_drained += 1; + metrics.reduce_ms_total += result.metrics.reduce_ms; + metrics.reduce_ms_max = metrics.reduce_ms_max.max(result.metrics.reduce_ms); + metrics.finalize_ms_total += result.metrics.finalize_ms; + metrics.finalize_ms_max = metrics.finalize_ms_max.max(result.metrics.finalize_ms); + metrics.finalize_queue_wait_ms_max = metrics + .finalize_queue_wait_ms_max + .max(result.metrics.finalize_queue_wait_ms.unwrap_or(0)); + metrics.finalize_worker_ms_total += result.metrics.finalize_worker_ms; + metrics.finalize_worker_ms_max = metrics + .finalize_worker_ms_max + .max(result.metrics.finalize_worker_ms); + metrics.snapshot_pack_ms_total += result.metrics.snapshot_pack_ms; + metrics.snapshot_pack_ms_max = metrics + .snapshot_pack_ms_max + .max(result.metrics.snapshot_pack_ms); + metrics.persist_vcir_ms_total += result.metrics.persist_vcir_ms; + metrics.persist_vcir_ms_max = metrics + .persist_vcir_ms_max + .max(result.metrics.persist_vcir_ms); + metrics.persist_build_vcir_ms_total += result.metrics.persist_build_vcir_ms; + metrics.persist_build_vcir_ms_max = metrics + .persist_build_vcir_ms_max + .max(result.metrics.persist_build_vcir_ms); + metrics.persist_replace_vcir_ms_total += result.metrics.persist_replace_vcir_ms; + metrics.persist_replace_vcir_ms_max = metrics + .persist_replace_vcir_ms_max + .max(result.metrics.persist_replace_vcir_ms); + metrics.ccr_projection_build_ms_total += result.metrics.ccr_projection_build_ms; + metrics.ccr_projection_build_ms_max = metrics + .ccr_projection_build_ms_max + .max(result.metrics.ccr_projection_build_ms); + metrics.ccr_append_ms_total += result.metrics.ccr_append_ms; + metrics.ccr_append_ms_max = + metrics.ccr_append_ms_max.max(result.metrics.ccr_append_ms); + metrics.audit_build_ms_total += result.metrics.audit_build_ms; + metrics.audit_build_ms_max = metrics + .audit_build_ms_max + .max(result.metrics.audit_build_ms); + *finalize_inflight = finalize_inflight.saturating_sub(1); + finished.push(result.finished); + } + Err(TryRecvError::Empty) => break, + Err(TryRecvError::Disconnected) => { + if *finalize_inflight == 0 { + break; + } + return Err(TreeRunError::Runner( + "phase2 finalize result channel disconnected".to_string(), + )); + } + } + } + metrics.duration_ms = elapsed_ms(started); + Ok(metrics) +} + +fn drain_finalize_results_with_progress( + finalize_result_rx: &Receiver, + finished: &mut Vec, + finalize_inflight: &mut usize, + pending_finalization_len: usize, + pending_roa_dispatch_len: usize, + inflight_publication_points_len: usize, +) -> Result<(), TreeRunError> { + let drain_metrics = drain_finalize_results(finalize_result_rx, finished, finalize_inflight)?; + if drain_metrics.results_drained >= 64 + || (drain_metrics.results_drained > 0 + && (*finalize_inflight == 0 || pending_finalization_len > 0)) + { + crate::progress_log::emit( + "phase2_finalize_results_drain", + serde_json::json!({ + "results_drained": drain_metrics.results_drained, + "reduce_ms_total": drain_metrics.reduce_ms_total, + "reduce_ms_max": drain_metrics.reduce_ms_max, + "finalize_ms_total": drain_metrics.finalize_ms_total, + "finalize_ms_max": drain_metrics.finalize_ms_max, + "finalize_queue_wait_ms_max": drain_metrics.finalize_queue_wait_ms_max, + "finalize_worker_ms_total": drain_metrics.finalize_worker_ms_total, + "finalize_worker_ms_max": drain_metrics.finalize_worker_ms_max, + "snapshot_pack_ms_total": drain_metrics.snapshot_pack_ms_total, + "snapshot_pack_ms_max": drain_metrics.snapshot_pack_ms_max, + "persist_vcir_ms_total": drain_metrics.persist_vcir_ms_total, + "persist_vcir_ms_max": drain_metrics.persist_vcir_ms_max, + "persist_build_vcir_ms_total": drain_metrics.persist_build_vcir_ms_total, + "persist_build_vcir_ms_max": drain_metrics.persist_build_vcir_ms_max, + "persist_replace_vcir_ms_total": drain_metrics.persist_replace_vcir_ms_total, + "persist_replace_vcir_ms_max": drain_metrics.persist_replace_vcir_ms_max, + "ccr_projection_build_ms_total": drain_metrics.ccr_projection_build_ms_total, + "ccr_projection_build_ms_max": drain_metrics.ccr_projection_build_ms_max, + "ccr_append_ms_total": drain_metrics.ccr_append_ms_total, + "ccr_append_ms_max": drain_metrics.ccr_append_ms_max, + "audit_build_ms_total": drain_metrics.audit_build_ms_total, + "audit_build_ms_max": drain_metrics.audit_build_ms_max, + "duration_ms": drain_metrics.duration_ms, + "pending_finalization_len": pending_finalization_len, + "finalize_inflight": *finalize_inflight, + "pending_roa_dispatch_len": pending_roa_dispatch_len, + "inflight_publication_points": inflight_publication_points_len, + }), + ); + } + Ok(()) +} + +fn run_finalize_worker( + runner: &Rpkiv1PublicationPointRunner<'_>, + finalize_task_rx: Receiver, + finalize_result_tx: mpsc::Sender, + compact_audit: bool, +) -> Result<(), TreeRunError> { + while let Ok(task) = finalize_task_rx.recv() { + let result = finalize_publication_point_state(runner, task.state, compact_audit); + if finalize_result_tx.send(result).is_err() { + return Err(TreeRunError::Runner( + "phase2 finalize result receiver disconnected".to_string(), + )); + } + } + Ok(()) +} diff --git a/crates/panda-rpki-validator/src/validation/tree_parallel/finalize.rs b/crates/panda-rpki-validator/src/validation/tree_parallel/finalize.rs new file mode 100644 index 0000000..074c969 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_parallel/finalize.rs @@ -0,0 +1,520 @@ +fn finalize_publication_point_state( + runner: &Rpkiv1PublicationPointRunner<'_>, + state: InflightPublicationPoint, + compact_audit: bool, +) -> FinalizeWorkerResult { + let finalize_worker_started = Instant::now(); + let InflightPublicationPoint { + node, + fresh_stage, + objects_prepare, + repo_outcome, + warnings, + started_at, + objects_started_at, + task_count, + tasks_submitted, + first_task_submitted_at, + last_task_submitted_at, + first_result_at, + last_result_at, + worker_ms_total, + worker_ms_max, + queue_wait_ms_total, + queue_wait_ms_max, + finalize_enqueued_at, + results, + } = state; + let finalize_queue_wait_ms = finalize_enqueued_at.or(last_result_at).map(|ready_at| { + Instant::now() + .saturating_duration_since(ready_at) + .as_millis() as u64 + }); + let objects_processing_ms = objects_started_at.elapsed().as_millis() as u64; + runner.record_publication_point_step_ms( + &node.handle.manifest_rsync_uri, + "fresh_objects_processing_lifetime", + objects_processing_ms, + ); + + let (result, mut metrics, reduce_ms) = match objects_prepare { + ParallelObjectsPrepare::Staged(objects_stage) => { + let reduce_started = Instant::now(); + let locked_files = objects_stage.locked_file_count(); + let reduce_result = + reduce_parallel_roa_stage(objects_stage, results, runner.timing.as_ref()); + let reduce_ms = elapsed_ms(reduce_started); + runner.record_publication_point_step_ms( + &node.handle.manifest_rsync_uri, + "fresh_roa_reduce", + reduce_ms, + ); + + let (result, metrics) = match reduce_result { + Ok(mut objects) => { + let finalize_started = Instant::now(); + objects + .router_keys + .extend(fresh_stage.discovered_router_keys.clone()); + objects.local_outputs_cache.extend( + crate::validation::tree_runner::build_router_key_local_outputs( + &node.handle, + &objects.router_keys, + ), + ); + let finalized = runner.finalize_fresh_publication_point_from_reducer( + &node.handle, + &fresh_stage.fresh_point, + warnings, + objects, + fresh_stage.child_audits, + fresh_stage.discovered_children, + repo_outcome.repo_sync_source.as_deref(), + repo_outcome.repo_sync_phase.as_deref(), + repo_outcome.repo_sync_duration_ms, + repo_outcome.repo_sync_err.as_deref(), + ); + let finalize_ms = elapsed_ms(finalize_started); + match finalized { + Ok(output) => { + let metrics = finalize_metrics_from_output( + &output, + reduce_ms, + finalize_ms, + finalize_queue_wait_ms, + 0, + locked_files, + ); + ( + compact_phase2_finished_result(output.result, compact_audit), + metrics, + ) + } + Err(err) => ( + FinishedPublicationPointResult::Err(err), + FinalizePublicationPointMetrics { + reduce_ms, + finalize_ms, + finalize_queue_wait_ms, + locked_files, + ..FinalizePublicationPointMetrics::default() + }, + ), + } + } + Err(err) => ( + FinishedPublicationPointResult::Err(err), + FinalizePublicationPointMetrics { + reduce_ms, + finalize_queue_wait_ms, + locked_files, + ..FinalizePublicationPointMetrics::default() + }, + ), + }; + (result, metrics, reduce_ms) + } + ParallelObjectsPrepare::Complete(objects) => { + // ROA prepare already produced complete objects for this publication + // point, so there is nothing to reduce; finalize directly. This is + // the former control-thread "direct finalize", now running on the + // finalize worker through the regular task queue. + let locked_files = fresh_stage.fresh_point.files().len(); + let finalize_started = Instant::now(); + let finalized = runner.finalize_fresh_publication_point_from_reducer( + &node.handle, + &fresh_stage.fresh_point, + warnings, + objects, + fresh_stage.child_audits, + fresh_stage.discovered_children, + repo_outcome.repo_sync_source.as_deref(), + repo_outcome.repo_sync_phase.as_deref(), + repo_outcome.repo_sync_duration_ms, + repo_outcome.repo_sync_err.as_deref(), + ); + let finalize_ms = elapsed_ms(finalize_started); + let (result, metrics) = match finalized { + Ok(output) => { + let metrics = finalize_metrics_from_output( + &output, + 0, + finalize_ms, + finalize_queue_wait_ms, + 0, + locked_files, + ); + ( + compact_phase2_finished_result(output.result, compact_audit), + metrics, + ) + } + Err(err) => ( + FinishedPublicationPointResult::Err(err), + FinalizePublicationPointMetrics { + finalize_ms, + finalize_queue_wait_ms, + locked_files, + ..FinalizePublicationPointMetrics::default() + }, + ), + }; + (result, metrics, 0) + } + }; + let finalize_worker_ms = elapsed_ms(finalize_worker_started); + metrics.finalize_worker_ms = finalize_worker_ms; + runner.record_publication_point_step_ms( + &node.handle.manifest_rsync_uri, + "fresh_finalize_worker", + finalize_worker_ms, + ); + runner.record_publication_point_step_ms( + &node.handle.manifest_rsync_uri, + "fresh_finalize_queue_wait", + finalize_queue_wait_ms.unwrap_or(0), + ); + runner.record_publication_point_step_ms( + &node.handle.manifest_rsync_uri, + "fresh_finalize", + metrics.finalize_ms, + ); + runner.record_publication_point_total_ms( + &node.handle.manifest_rsync_uri, + started_at.elapsed().as_millis() as u64, + ); + emit_finalize_breakdown( + "phase2_finalize_worker_breakdown", + node.handle.manifest_rsync_uri.as_str(), + node.handle.publication_point_rsync_uri.as_str(), + &metrics, + ); + crate::progress_log::emit( + "phase2_publication_point_reduced", + serde_json::json!({ + "manifest_rsync_uri": node.handle.manifest_rsync_uri.as_str(), + "publication_point_rsync_uri": node.handle.publication_point_rsync_uri.as_str(), + "objects_processing_ms": objects_processing_ms, + "task_count": task_count, + "tasks_submitted": tasks_submitted, + "first_task_submitted_ms": first_task_submitted_at.map(|t| t.saturating_duration_since(objects_started_at).as_millis() as u64), + "last_task_submitted_ms": last_task_submitted_at.map(|t| t.saturating_duration_since(objects_started_at).as_millis() as u64), + "task_submit_span_ms": match (first_task_submitted_at, last_task_submitted_at) { + (Some(first), Some(last)) => Some(last.saturating_duration_since(first).as_millis() as u64), + _ => None, + }, + "first_result_ms": first_result_at.map(|t| t.saturating_duration_since(objects_started_at).as_millis() as u64), + "last_result_ms": last_result_at.map(|t| t.saturating_duration_since(objects_started_at).as_millis() as u64), + "all_results_ready_ms": last_result_at.map(|t| t.saturating_duration_since(objects_started_at).as_millis() as u64), + "finalize_queue_wait_ms": finalize_queue_wait_ms, + "result_span_ms": match (first_result_at, last_result_at) { + (Some(first), Some(last)) => Some(last.saturating_duration_since(first).as_millis() as u64), + _ => None, + }, + "worker_ms_total": worker_ms_total, + "worker_ms_max": worker_ms_max, + "worker_ms_avg": if task_count > 0 { worker_ms_total / task_count as u64 } else { 0 }, + "queue_wait_ms_total": queue_wait_ms_total, + "queue_wait_ms_max": queue_wait_ms_max, + "queue_wait_ms_avg": if task_count > 0 { queue_wait_ms_total / task_count as u64 } else { 0 }, + "reduce_ms": reduce_ms, + "finalize_ms": metrics.finalize_ms, + "finalize_worker_ms": finalize_worker_ms, + "snapshot_pack_ms": metrics.snapshot_pack_ms, + "persist_vcir_ms": metrics.persist_vcir_ms, + "persist_build_vcir_ms": metrics.persist_build_vcir_ms, + "persist_replace_vcir_ms": metrics.persist_replace_vcir_ms, + "ccr_projection_build_ms": metrics.ccr_projection_build_ms, + "ccr_append_ms": metrics.ccr_append_ms, + "audit_build_ms": metrics.audit_build_ms, + "locked_files": metrics.locked_files, + "child_count": metrics.child_count, + "warning_count": metrics.warning_count, + "vrp_count": metrics.vrp_count, + "vap_count": metrics.vap_count, + "router_key_count": metrics.router_key_count, + "audit_object_count": metrics.audit_object_count, + "total_duration_ms": started_at.elapsed().as_millis() as u64, + }), + ); + FinalizeWorkerResult { + finished: FinishedPublicationPoint { + node: FinishedPublicationPointNode::from_queued(node), + result, + }, + metrics, + } +} + +fn drain_repo_events( + repo_runtime: &dyn crate::parallel::repo_runtime::RepoSyncRuntime, + ca_waiting_repo_by_identity: &mut HashMap>, + ready_queue: &mut VecDeque, + timeout: Duration, +) -> Result { + let started = Instant::now(); + let mut metrics = RepoDrainMetrics::default(); + let events = repo_runtime + .drain_repo_results_timeout(timeout, REPO_RESULT_DRAIN_MAX_EVENTS) + .map_err(TreeRunError::Runner)?; + for event in events { + metrics.event_count += 1; + metrics.completions += event.completions.len(); + for completion in event.completions { + let mut outcome = completion.outcome; + if completion.identity != event.transport_identity { + // Shared RRDP/rsync transports release many publication points, but the transport + // wall time should only be counted once in per-PP stage timing aggregation. + outcome.repo_sync_duration_ms = 0; + } + if let Some(waiters) = ca_waiting_repo_by_identity.remove(&completion.identity) { + metrics.ready_enqueued += waiters.len(); + for node in waiters { + ready_queue.push_back(ReadyCaInstance { + node, + repo_outcome: outcome.clone(), + ready_enqueued_at: Instant::now(), + }); + } + } + } + } + metrics.duration_ms = elapsed_ms(started); + Ok(metrics) +} + +fn event_poll_timeout( + ca_queue: &VecDeque, + ready_queue: &VecDeque, + pending_roa_dispatch: &VecDeque, + inflight_publication_points: &HashMap, + pending_finalization: &VecDeque, + finalize_inflight: usize, + staging_inflight: usize, + instances_started: usize, + config: &TreeRunConfig, +) -> Duration { + // Stage results pending collection must not let the loop sleep: with the + // stage pool enabled a 50ms nap per turn would collapse throughput. + if !ready_queue.is_empty() + || !pending_roa_dispatch.is_empty() + || !inflight_publication_points.is_empty() + || !pending_finalization.is_empty() + || staging_inflight > 0 + || (!ca_queue.is_empty() && can_start_more(instances_started, config)) + { + Duration::from_millis(0) + } else if finalize_inflight > 0 { + Duration::from_millis(10) + } else { + Duration::from_millis(50) + } +} + +fn is_complete( + ca_queue: &VecDeque, + ready_queue: &VecDeque, + ca_waiting_repo_by_identity: &HashMap>, + pending_roa_dispatch: &VecDeque, + inflight_publication_points: &HashMap, + pending_finalization: &VecDeque, + finalize_inflight: usize, + staging_inflight: usize, + instances_started: usize, + config: &TreeRunConfig, +) -> bool { + // `staging_inflight == 0` additionally implies the stage result channel is + // drained empty: the drain loop collects every available result each turn + // and only decrements the counter while collecting. + let ca_queue_done = ca_queue.is_empty() || !can_start_more(instances_started, config); + ca_queue_done + && ready_queue.is_empty() + && ca_waiting_repo_by_identity.is_empty() + && pending_roa_dispatch.is_empty() + && inflight_publication_points.is_empty() + && pending_finalization.is_empty() + && finalize_inflight == 0 + && staging_inflight == 0 +} + +/// Minimum publication points per reduction shard; smaller runs stay +/// single-threaded to avoid thread-spawn overhead dominating the reduction. +const TREE_OUTPUT_MIN_SHARD_LEN: usize = 4096; +/// Hard cap on reduction shards regardless of core count. +const TREE_OUTPUT_MAX_SHARDS: usize = 8; + +fn tree_output_shard_count(len: usize) -> usize { + if len < TREE_OUTPUT_MIN_SHARD_LEN * 2 { + return 1; + } + let parallel = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1); + parallel + .min(TREE_OUTPUT_MAX_SHARDS) + .min(len / TREE_OUTPUT_MIN_SHARD_LEN) + .max(1) +} + +/// Per-shard reduction result of the phase2 output merge. Shards cover +/// contiguous ranges of the id-sorted finished list, so concatenating shard +/// outputs in shard order reproduces the sequential single-threaded order +/// exactly. +#[derive(Default)] +struct TreeOutputShardReduction { + instances_processed: usize, + instances_failed: usize, + warnings: Vec, + vrps: Vec, + aspas: Vec, + router_keys: Vec, + publication_points: Vec, + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats, + cir_input: CirInputAccumulator, +} + +fn reduce_finished_shard(items: Vec) -> TreeOutputShardReduction { + let mut reduction = TreeOutputShardReduction::default(); + for item in items { + match item.result { + FinishedPublicationPointResult::Ok { + source, + warnings: result_warnings, + objects, + audit, + cir_fresh_objects, + cir_cached_objects, + } => { + reduction.instances_processed += 1; + reduction.warnings.extend(result_warnings); + reduction.warnings.extend(objects.warnings); + reduction + .roa_cache_stats + .add_assign(&objects.roa_cache_stats); + reduction.vrps.extend(objects.vrps); + reduction.aspas.extend(objects.aspas); + reduction.router_keys.extend(objects.router_keys); + + let mut audit: PublicationPointAudit = audit; + audit.node_id = Some(item.node.id); + audit.parent_node_id = item.node.parent_id; + audit.discovered_from = item.node.discovered_from; + crate::validation::tree::submit_publication_point_cir_input( + &mut reduction.cir_input, + source, + &audit, + &cir_fresh_objects, + &cir_cached_objects, + ) + .expect("CIR input collection from validated audit must not fail"); + reduction.publication_points.push(audit); + } + FinishedPublicationPointResult::Err(err) => { + reduction.instances_failed += 1; + reduction.warnings.push( + Warning::new(format!("publication point failed: {err}")) + .with_context(&item.node.manifest_rsync_uri), + ); + } + } + } + reduction +} + +fn merge_shard_reductions(reductions: Vec) -> TreeOutputShardReduction { + let mut iter = reductions.into_iter(); + let mut merged = iter.next().unwrap_or_default(); + for mut other in iter { + merged.instances_processed += other.instances_processed; + merged.instances_failed += other.instances_failed; + merged.warnings.append(&mut other.warnings); + merged.vrps.append(&mut other.vrps); + merged.aspas.append(&mut other.aspas); + merged.router_keys.append(&mut other.router_keys); + merged + .publication_points + .append(&mut other.publication_points); + merged.roa_cache_stats.add_assign(&other.roa_cache_stats); + merged + .cir_input + .merge(std::mem::take(&mut other.cir_input)) + .expect("CIR input merge from validated audits must not fail"); + } + merged +} + +fn build_tree_output(mut finished: Vec) -> TreeRunAuditOutput { + let total_started = Instant::now(); + finished.sort_by_key(|item| item.node.id); + let sort_ms = total_started.elapsed().as_millis() as u64; + let shard_count = tree_output_shard_count(finished.len()); + + let reduce_started = Instant::now(); + let reductions = if shard_count <= 1 { + vec![reduce_finished_shard(finished)] + } else { + let chunk_len = finished.len().div_ceil(shard_count); + let mut shards: Vec> = Vec::new(); + let mut rest = finished; + while !rest.is_empty() { + let split_at = chunk_len.min(rest.len()); + let tail = rest.split_off(split_at); + shards.push(std::mem::replace(&mut rest, tail)); + } + std::thread::scope(|scope| { + let handles: Vec<_> = shards + .into_iter() + .map(|shard| scope.spawn(move || reduce_finished_shard(shard))) + .collect(); + handles + .into_iter() + .map(|handle| handle.join().expect("tree output reduction shard panicked")) + .collect() + }) + }; + let reduce_ms = reduce_started.elapsed().as_millis() as u64; + + let merge_started = Instant::now(); + let merged = merge_shard_reductions(reductions); + let merge_ms = merge_started.elapsed().as_millis() as u64; + + let finalize_started = Instant::now(); + let output = TreeRunAuditOutput { + tree: TreeRunOutput { + instances_processed: merged.instances_processed, + instances_failed: merged.instances_failed, + warnings: merged.warnings, + vrps: merged.vrps, + aspas: merged.aspas, + router_keys: merged.router_keys, + }, + publication_points: merged.publication_points, + roa_cache_stats: merged.roa_cache_stats, + cir_input: merged.cir_input.finalize(), + }; + let finalize_ms = finalize_started.elapsed().as_millis() as u64; + + crate::progress_log::emit( + "phase2_build_tree_output", + serde_json::json!({ + "sort_ms": sort_ms, + "shard_count": shard_count, + "reduce_ms": reduce_ms, + "merge_ms": merge_ms, + "finalize_ms": finalize_ms, + "total_ms": total_started.elapsed().as_millis() as u64, + "publication_points": output.publication_points.len(), + "instances_processed": output.tree.instances_processed, + "instances_failed": output.tree.instances_failed, + }), + ); + output +} + +pub fn run_tree_parallel_phase2_audit( + root: CaInstanceHandle, + runner: &Rpkiv1PublicationPointRunner<'_>, + config: &TreeRunConfig, +) -> Result { + run_tree_parallel_phase2_audit_multi_root(vec![root], runner, config) +} diff --git a/crates/panda-rpki-validator/src/validation/tree_parallel/phase2.rs b/crates/panda-rpki-validator/src/validation/tree_parallel/phase2.rs new file mode 100644 index 0000000..98672fc --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_parallel/phase2.rs @@ -0,0 +1,445 @@ +pub fn run_tree_parallel_phase2_audit_multi_root( + roots: Vec, + runner: &Rpkiv1PublicationPointRunner<'_>, + config: &TreeRunConfig, +) -> Result { + if runner.policy.signed_object_failure_policy == SignedObjectFailurePolicy::DropPublicationPoint + { + return run_tree_serial_audit_multi_root(roots, runner, config); + } + + let Some(repo_runtime) = runner.repo_sync_runtime.as_ref() else { + return run_tree_serial_audit_multi_root(roots, runner, config); + }; + if runner.parallel_roa_worker_pool.is_none() { + return run_tree_serial_audit_multi_root(roots, runner, config); + } + + let mut next_id: u64 = 0; + let mut ca_queue: VecDeque = VecDeque::new(); + for root in roots { + ca_queue.push_back(QueuedCaInstance { + id: next_id, + handle: root, + parent_id: None, + discovered_from: None, + }); + next_id += 1; + } + + let mut visited_manifest_uris: HashSet = HashSet::new(); + let mut ca_waiting_repo_by_identity: HashMap> = + HashMap::new(); + let mut ready_queue: VecDeque = VecDeque::new(); + let mut inflight_publication_points: HashMap = HashMap::new(); + let mut pending_finalization: VecDeque = VecDeque::new(); + let mut pending_roa_dispatch: VecDeque = VecDeque::new(); + let mut finished: Vec = Vec::new(); + let mut instances_started = 0usize; + let phase2_config = runner.parallel_phase2_config.as_ref(); + let ready_batch_size = phase2_config + .map(|cfg| cfg.ready_batch_size) + .unwrap_or(256) + .max(1); + let ready_batch_wall_time_budget_ms = phase2_config + .map(|cfg| cfg.ready_batch_wall_time_budget_ms) + .unwrap_or(100) + .max(1); + let ready_batch_wall_time_budget = Duration::from_millis(ready_batch_wall_time_budget_ms); + let object_result_drain_batch_size = phase2_config + .map(|cfg| cfg.object_result_drain_batch_size) + .unwrap_or(2048) + .max(1); + let publication_point_finalize_queue_capacity = phase2_config + .map(|cfg| cfg.publication_point_finalize_queue_capacity) + .unwrap_or(32768) + .max(1); + // Experimental ready-stage pool: `stage_workers == 0` keeps the inline + // compute+apply staging path byte-for-byte; any positive value moves the + // compute phase to a scoped worker pool. + let stage_worker_count = phase2_config.map(|cfg| cfg.stage_workers).unwrap_or(0); + let stage_queue_capacity = phase2_config + .map(|cfg| cfg.worker_queue_capacity) + .unwrap_or(256) + .max(1); + + let (finalize_task_tx, finalize_task_rx) = + mpsc::sync_channel::(publication_point_finalize_queue_capacity); + let (finalize_result_tx, finalize_result_rx) = mpsc::channel::(); + let mut finalize_inflight = 0usize; + + return std::thread::scope(|scope| { + let finalize_worker = scope.spawn(move || { + run_finalize_worker( + runner, + finalize_task_rx, + finalize_result_tx, + config.compact_audit, + ) + }); + + // The stage pool borrows the runner like the finalize worker does; it + // lives inside this scope and is dropped before the scope joins. + let mut stage_pool = if stage_worker_count > 0 { + Some( + ReadyStagePool::new( + scope, + stage_worker_count, + stage_queue_capacity, + ReadyStageTaskExecutor { runner }, + ) + .map_err(TreeRunError::Runner)?, + ) + } else { + None + }; + // Submitted-but-not-yet-drained stage tasks. The drain loop collects + // every available result each turn, so `staging_inflight == 0` also + // implies the stage result channel is empty. + let mut staging_inflight = 0usize; + + let run_result: Result<(), TreeRunError> = (|| { + loop { + let control_loop_started = Instant::now(); + // With the stage pool enabled the batch wall clock covers the + // whole turn (turn-head drain + dispatch + apply); the inline + // path keeps its historical start point at the ready batch. + let turn_stage_started = Instant::now(); + let mut ready_batch_metrics = ReadyStageBatchMetrics::default(); + let mut stage_drain_metrics = StageDrainMetrics::default(); + drain_finalize_results_with_progress( + &finalize_result_rx, + &mut finished, + &mut finalize_inflight, + pending_finalization.len(), + pending_roa_dispatch.len(), + inflight_publication_points.len(), + )?; + flush_pending_roa_dispatch_with_progress( + runner, + &mut pending_roa_dispatch, + &mut inflight_publication_points, + &pending_finalization, + )?; + drain_object_results_with_progress( + runner, + &mut inflight_publication_points, + &mut pending_finalization, + pending_roa_dispatch.len(), + object_result_drain_batch_size, + )?; + if let Some(pool) = stage_pool.as_ref() { + drain_stage_results( + pool, + runner, + &mut next_id, + &mut ca_queue, + &mut pending_roa_dispatch, + &mut inflight_publication_points, + &mut pending_finalization, + &mut finished, + &mut staging_inflight, + &mut ready_batch_metrics, + &mut stage_drain_metrics, + config, + )?; + } + submit_pending_finalization_with_progress( + &finalize_task_tx, + &mut pending_finalization, + &mut finalize_inflight, + publication_point_finalize_queue_capacity, + pending_roa_dispatch.len(), + inflight_publication_points.len(), + )?; + + start_queued_ca_instances( + repo_runtime.as_ref(), + &mut ca_queue, + &mut ready_queue, + &mut ca_waiting_repo_by_identity, + &mut finished, + &mut visited_manifest_uris, + &mut instances_started, + config, + ); + + let repo_poll_timeout = event_poll_timeout( + &ca_queue, + &ready_queue, + &pending_roa_dispatch, + &inflight_publication_points, + &pending_finalization, + finalize_inflight, + staging_inflight, + instances_started, + config, + ); + let repo_metrics = drain_repo_events( + repo_runtime.as_ref(), + &mut ca_waiting_repo_by_identity, + &mut ready_queue, + repo_poll_timeout, + )?; + if repo_metrics.event_count > 0 { + crate::progress_log::emit( + "phase2_repo_events_drain", + serde_json::json!({ + "event_count": repo_metrics.event_count, + "completions": repo_metrics.completions, + "ready_enqueued": repo_metrics.ready_enqueued, + "duration_ms": repo_metrics.duration_ms, + "ready_queue_len": ready_queue.len(), + "ca_waiting_repo_identities": ca_waiting_repo_by_identity.len(), + }), + ); + } + + let ready_batch_started = Instant::now(); + let mut ready_time_budget_exhausted = false; + let mut stage_dispatch_metrics = StageDispatchMetrics::default(); + if let Some(pool) = stage_pool.as_mut() { + // Pool path: the ready batch becomes a dispatch loop that + // submits compute tasks and applies whatever results are + // already available; backpressure requeues for next turn. + stage_dispatch_metrics = submit_ready_batch_to_stage_pool( + pool, + &mut ready_queue, + &mut staging_inflight, + ready_batch_size, + ready_batch_wall_time_budget, + )?; + ready_time_budget_exhausted = stage_dispatch_metrics.queue_full + || (!ready_queue.is_empty() + && ready_batch_started.elapsed() >= ready_batch_wall_time_budget); + drain_stage_results( + pool, + runner, + &mut next_id, + &mut ca_queue, + &mut pending_roa_dispatch, + &mut inflight_publication_points, + &mut pending_finalization, + &mut finished, + &mut staging_inflight, + &mut ready_batch_metrics, + &mut stage_drain_metrics, + config, + )?; + } else { + while ready_batch_metrics.ready_count < ready_batch_size { + let Some(ready) = ready_queue.pop_front() else { + break; + }; + let ready_queue_len_after_pop = ready_queue.len(); + let (outcome, metrics) = compute_ready_publication_point_stage( + runner, + ready, + ready_queue_len_after_pop, + ); + let metrics = apply_ready_publication_point_stage( + runner, + &mut next_id, + &mut ca_queue, + &mut pending_roa_dispatch, + &mut inflight_publication_points, + &mut pending_finalization, + &mut finished, + outcome, + metrics, + config, + config.compact_audit, + ); + ready_batch_metrics.record(metrics); + if ready_batch_metrics.ready_count > 0 + && ready_batch_started.elapsed() >= ready_batch_wall_time_budget + { + ready_time_budget_exhausted = !ready_queue.is_empty(); + break; + } + } + } + if ready_batch_metrics.ready_count > 0 { + ready_batch_metrics.total_ms = if stage_pool.is_some() { + elapsed_ms(turn_stage_started) + } else { + elapsed_ms(ready_batch_started) + }; + let ready_count_budget_exhausted = ready_batch_metrics.ready_count + >= ready_batch_size + && !ready_queue.is_empty(); + ready_time_budget_exhausted = ready_time_budget_exhausted + || (!ready_queue.is_empty() + && ready_batch_metrics.total_ms >= ready_batch_wall_time_budget_ms); + emit_ready_queue_batch_progress( + &ready_batch_metrics, + ready_batch_size, + ready_batch_wall_time_budget_ms, + ready_queue.len(), + ready_count_budget_exhausted, + ready_time_budget_exhausted, + ca_queue.len(), + pending_roa_dispatch.len(), + inflight_publication_points.len(), + pending_finalization.len(), + finalize_inflight, + ); + crate::progress_log::emit( + "phase2_ready_queue_stage_fresh_breakdown", + serde_json::json!({ + "ready_count": ready_batch_metrics.ready_count, + "stage_fresh_ms_total": ready_batch_metrics.stage_fresh_ms_total, + "stage_fresh_ms_max": ready_batch_metrics.stage_fresh_ms_max, + "stage_fresh_ms_max_manifest_rsync_uri": ready_batch_metrics.stage_fresh_ms_max_manifest_rsync_uri, + "stage_fresh_ms_max_publication_point_rsync_uri": ready_batch_metrics.stage_fresh_ms_max_publication_point_rsync_uri, + "snapshot_prepare_ms_total": ready_batch_metrics.snapshot_prepare_ms_total, + "snapshot_prepare_ms_max": ready_batch_metrics.snapshot_prepare_ms_max, + "snapshot_current_index_lock_ms_total": ready_batch_metrics.snapshot_current_index_lock_ms_total, + "snapshot_current_index_lock_ms_max": ready_batch_metrics.snapshot_current_index_lock_ms_max, + "snapshot_manifest_load_ms_total": ready_batch_metrics.snapshot_manifest_load_ms_total, + "snapshot_manifest_load_ms_max": ready_batch_metrics.snapshot_manifest_load_ms_max, + "snapshot_manifest_index_lookup_ms_total": ready_batch_metrics.snapshot_manifest_index_lookup_ms_total, + "snapshot_manifest_index_lookup_ms_max": ready_batch_metrics.snapshot_manifest_index_lookup_ms_max, + "snapshot_manifest_blob_load_ms_total": ready_batch_metrics.snapshot_manifest_blob_load_ms_total, + "snapshot_manifest_blob_load_ms_max": ready_batch_metrics.snapshot_manifest_blob_load_ms_max, + "snapshot_manifest_decode_ms_total": ready_batch_metrics.snapshot_manifest_decode_ms_total, + "snapshot_manifest_decode_ms_max": ready_batch_metrics.snapshot_manifest_decode_ms_max, + "snapshot_replay_guard_ms_total": ready_batch_metrics.snapshot_replay_guard_ms_total, + "snapshot_replay_guard_ms_max": ready_batch_metrics.snapshot_replay_guard_ms_max, + "replay_meta_hit_count": ready_batch_metrics.replay_meta_hit_count, + "replay_meta_miss_count": ready_batch_metrics.replay_meta_miss_count, + "snapshot_manifest_entries_ms_total": ready_batch_metrics.snapshot_manifest_entries_ms_total, + "snapshot_manifest_entries_ms_max": ready_batch_metrics.snapshot_manifest_entries_ms_max, + "snapshot_pack_files_ms_total": ready_batch_metrics.snapshot_pack_files_ms_total, + "snapshot_pack_files_ms_max": ready_batch_metrics.snapshot_pack_files_ms_max, + "snapshot_pack_files_index_lookup_ms_total": ready_batch_metrics.snapshot_pack_files_index_lookup_ms_total, + "snapshot_pack_files_index_lookup_ms_max": ready_batch_metrics.snapshot_pack_files_index_lookup_ms_max, + "snapshot_pack_files_blob_load_ms_total": ready_batch_metrics.snapshot_pack_files_blob_load_ms_total, + "snapshot_pack_files_blob_load_ms_max": ready_batch_metrics.snapshot_pack_files_blob_load_ms_max, + "snapshot_ee_path_validate_ms_total": ready_batch_metrics.snapshot_ee_path_validate_ms_total, + "snapshot_ee_path_validate_ms_max": ready_batch_metrics.snapshot_ee_path_validate_ms_max, + "snapshot_manifest_file_count_total": ready_batch_metrics.snapshot_manifest_file_count_total, + "snapshot_manifest_file_count_max": ready_batch_metrics.snapshot_manifest_file_count_max, + "child_discovery_ms_total": ready_batch_metrics.child_discovery_ms_total, + "child_discovery_ms_max": ready_batch_metrics.child_discovery_ms_max, + "batch_duration_ms": ready_batch_metrics.total_ms, + }), + ); + } + if stage_pool.is_some() + && (stage_dispatch_metrics.submitted > 0 + || stage_drain_metrics.results_drained > 0 + || stage_dispatch_metrics.queue_full) + { + crate::progress_log::emit( + "phase2_stage_pool_stats", + serde_json::json!({ + "stage_workers": stage_worker_count, + "submitted": stage_dispatch_metrics.submitted, + "results_drained": stage_drain_metrics.results_drained, + "queue_full": stage_dispatch_metrics.queue_full, + "staging_inflight": staging_inflight, + "ready_queue_len": ready_queue.len(), + "queue_wait_ms_total": stage_drain_metrics.queue_wait_ms_total, + "queue_wait_ms_max": stage_drain_metrics.queue_wait_ms_max, + "worker_ms_total": stage_drain_metrics.worker_ms_total, + "worker_ms_max": stage_drain_metrics.worker_ms_max, + "dispatch_duration_ms": stage_dispatch_metrics.duration_ms, + "drain_duration_ms": stage_drain_metrics.duration_ms, + }), + ); + } + + flush_pending_roa_dispatch_with_progress( + runner, + &mut pending_roa_dispatch, + &mut inflight_publication_points, + &pending_finalization, + )?; + drain_object_results_with_progress( + runner, + &mut inflight_publication_points, + &mut pending_finalization, + pending_roa_dispatch.len(), + object_result_drain_batch_size, + )?; + submit_pending_finalization_with_progress( + &finalize_task_tx, + &mut pending_finalization, + &mut finalize_inflight, + publication_point_finalize_queue_capacity, + pending_roa_dispatch.len(), + inflight_publication_points.len(), + )?; + drain_finalize_results_with_progress( + &finalize_result_rx, + &mut finished, + &mut finalize_inflight, + pending_finalization.len(), + pending_roa_dispatch.len(), + inflight_publication_points.len(), + )?; + + emit_control_loop_slow( + elapsed_ms(control_loop_started), + repo_poll_timeout, + &repo_metrics, + &ready_batch_metrics, + ca_queue.len(), + ready_queue.len(), + ca_waiting_repo_by_identity.len(), + pending_roa_dispatch.len(), + inflight_publication_points.len(), + pending_finalization.len(), + finalize_inflight, + ); + + if is_complete( + &ca_queue, + &ready_queue, + &ca_waiting_repo_by_identity, + &pending_roa_dispatch, + &inflight_publication_points, + &pending_finalization, + finalize_inflight, + staging_inflight, + instances_started, + config, + ) { + break; + } + } + + repo_runtime + .reset_run_state() + .map_err(TreeRunError::Runner)?; + Ok(()) + })(); + + drop(finalize_task_tx); + // Dropping the pool closes the stage task queues so the scoped stage + // workers exit before the scope joins them below. + drop(stage_pool); + let worker_result = finalize_worker + .join() + .map_err(|_| TreeRunError::Runner("phase2 finalize worker panicked".to_string()))?; + run_result?; + worker_result?; + drain_finalize_results_with_progress( + &finalize_result_rx, + &mut finished, + &mut finalize_inflight, + pending_finalization.len(), + pending_roa_dispatch.len(), + inflight_publication_points.len(), + )?; + if finalize_inflight != 0 || !pending_finalization.is_empty() { + return Err(TreeRunError::Runner(format!( + "phase2 finalize worker stopped with pending work: queued={} inflight={}", + pending_finalization.len(), + finalize_inflight + ))); + } + Ok(build_tree_output(finished)) + }); +} diff --git a/crates/panda-rpki-validator/src/validation/tree_parallel/ready_stage.rs b/crates/panda-rpki-validator/src/validation/tree_parallel/ready_stage.rs new file mode 100644 index 0000000..ec9b29d --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_parallel/ready_stage.rs @@ -0,0 +1,849 @@ +fn emit_ready_queue_batch_progress( + metrics: &ReadyStageBatchMetrics, + ready_batch_size: usize, + ready_batch_wall_time_budget_ms: u64, + ready_queue_len_after_batch: usize, + ready_count_budget_exhausted: bool, + ready_time_budget_exhausted: bool, + ca_queue_len_after_batch: usize, + pending_roa_dispatch_len_after_batch: usize, + inflight_publication_points_after_batch: usize, + pending_finalization_len_after_batch: usize, + finalize_inflight_after_batch: usize, +) { + crate::progress_log::emit( + "phase2_ready_queue_batch", + serde_json::json!({ + "ready_count": metrics.ready_count, + "fallback_count": metrics.fallback_count, + "complete_count": metrics.complete_count, + "staged_count": metrics.staged_count, + "zero_task_count": metrics.zero_task_count, + "error_count": metrics.error_count, + "discovered_children": metrics.discovered_children, + "locked_files": metrics.locked_files, + "roa_tasks": metrics.roa_tasks, + "aspa_objects": metrics.aspa_objects, + "stage_fresh_ms_total": metrics.stage_fresh_ms_total, + "stage_fresh_ms_max": metrics.stage_fresh_ms_max, + "stage_fresh_ms_max_manifest_rsync_uri": metrics.stage_fresh_ms_max_manifest_rsync_uri, + "stage_fresh_ms_max_publication_point_rsync_uri": metrics.stage_fresh_ms_max_publication_point_rsync_uri, + "prepare_ms_total": metrics.prepare_ms_total, + "prepare_ms_max": metrics.prepare_ms_max, + "build_roa_tasks_ms_total": metrics.build_roa_tasks_ms_total, + "build_roa_tasks_ms_max": metrics.build_roa_tasks_ms_max, + "batch_duration_ms": metrics.total_ms, + "ready_batch_size": ready_batch_size, + "ready_batch_wall_time_budget_ms": ready_batch_wall_time_budget_ms, + "ready_queue_len_after_batch": ready_queue_len_after_batch, + "ready_queue_budget_exhausted": ready_queue_len_after_batch > 0, + "ready_count_budget_exhausted": ready_count_budget_exhausted, + "ready_time_budget_exhausted": ready_time_budget_exhausted, + "ca_queue_len_after_batch": ca_queue_len_after_batch, + "pending_roa_dispatch_len_after_batch": pending_roa_dispatch_len_after_batch, + "inflight_publication_points_after_batch": inflight_publication_points_after_batch, + "pending_finalization_len_after_batch": pending_finalization_len_after_batch, + "finalize_inflight_after_batch": finalize_inflight_after_batch, + }), + ); + crate::progress_log::emit( + "phase2_ready_queue_control_breakdown", + serde_json::json!({ + "ready_count": metrics.ready_count, + "ready_queue_wait_ms_total": metrics.ready_queue_wait_ms_total, + "ready_queue_wait_ms_max": metrics.ready_queue_wait_ms_max, + "child_enqueue_ms_total": metrics.child_enqueue_ms_total, + "child_enqueue_ms_max": metrics.child_enqueue_ms_max, + "roa_presence_scan_ms_total": metrics.roa_presence_scan_ms_total, + "roa_presence_scan_ms_max": metrics.roa_presence_scan_ms_max, + "roa_cache_view_ms_total": metrics.roa_cache_view_ms_total, + "roa_cache_view_ms_max": metrics.roa_cache_view_ms_max, + "direct_finalize_ms_total": metrics.direct_finalize_ms_total, + "direct_finalize_ms_max": metrics.direct_finalize_ms_max, + "fallback_full_run_ms_total": metrics.fallback_full_run_ms_total, + "fallback_full_run_ms_max": metrics.fallback_full_run_ms_max, + "batch_duration_ms": metrics.total_ms, + }), + ); +} + +fn can_start_more(instances_started: usize, config: &TreeRunConfig) -> bool { + config + .max_instances + .map(|max| instances_started < max) + .unwrap_or(true) +} + +fn start_queued_ca_instances( + repo_runtime: &dyn crate::parallel::repo_runtime::RepoSyncRuntime, + ca_queue: &mut VecDeque, + ready_queue: &mut VecDeque, + ca_waiting_repo_by_identity: &mut HashMap>, + finished: &mut Vec, + visited_manifest_uris: &mut HashSet, + instances_started: &mut usize, + config: &TreeRunConfig, +) { + while can_start_more(*instances_started, config) { + let Some(node) = ca_queue.pop_front() else { + break; + }; + if !visited_manifest_uris.insert(node.handle.manifest_rsync_uri.clone()) { + continue; + } + if !ca_depth_is_allowed(config, node.handle.depth) { + continue; + } + *instances_started += 1; + match repo_runtime.request_publication_point_repo(&node.handle, 0) { + Ok(RepoSyncRequestStatus::Ready { mut outcome, .. }) => { + // Ready here means this CA is reusing repo work that has already completed + // (often due to child prefetch). Do not add the transport duration again. + outcome.repo_sync_duration_ms = 0; + ready_queue.push_back(ReadyCaInstance { + node, + repo_outcome: outcome, + ready_enqueued_at: Instant::now(), + }); + } + Ok(RepoSyncRequestStatus::Pending { identity, .. }) => { + ca_waiting_repo_by_identity + .entry(identity) + .or_default() + .push(node); + } + Err(err) => { + finished.push(FinishedPublicationPoint { + node: FinishedPublicationPointNode::from_queued(node), + result: FinishedPublicationPointResult::Err(err), + }); + } + } + } +} + +fn compute_ready_publication_point_stage( + runner: &Rpkiv1PublicationPointRunner<'_>, + ready: ReadyCaInstance, + ready_queue_len_after_pop: usize, +) -> (StageOutcome, ReadyStageMetrics) { + let publication_point_started = Instant::now(); + let ready_queue_wait_ms = publication_point_started + .saturating_duration_since(ready.ready_enqueued_at) + .as_millis() as u64; + let mut metrics = ReadyStageMetrics { + ready_count: 1, + manifest_rsync_uri: Some(ready.node.handle.manifest_rsync_uri.clone()), + publication_point_rsync_uri: Some(ready.node.handle.publication_point_rsync_uri.clone()), + ready_queue_wait_ms, + ready_queue_len_after_pop, + ..ReadyStageMetrics::default() + }; + let mut warnings = ready.repo_outcome.warnings.clone(); + let repo_outcome = ready.repo_outcome.clone(); + if let Some(result) = runner.observe_or_reuse_publication_point_cache( + &ready.node.handle, + repo_outcome.repo_sync_source.as_deref(), + repo_outcome.repo_sync_phase.as_deref(), + repo_outcome.repo_sync_duration_ms, + repo_outcome.repo_sync_err.as_deref(), + &warnings, + ) { + metrics.complete_count = 1; + metrics.discovered_children = result.discovered_children.len(); + return ( + StageOutcome::CacheHit(Box::new(CacheHitOutcome { + ready, + publication_point_started, + result, + })), + metrics, + ); + } + + let stage_fresh_started = Instant::now(); + let stage = runner.stage_fresh_publication_point_after_repo_ready( + &ready.node.handle, + repo_outcome.repo_sync_ok, + repo_outcome.repo_sync_err.as_deref(), + ); + metrics.stage_fresh_ms = elapsed_ms(stage_fresh_started); + + let fresh_stage = match stage { + Ok(stage) => stage, + Err(err) => { + if metrics.stage_fresh_ms >= crate::progress_log::stage_fresh_slow_threshold_ms() { + crate::progress_log::emit( + "phase2_stage_fresh_slow", + serde_json::json!({ + "manifest_rsync_uri": ready.node.handle.manifest_rsync_uri.as_str(), + "publication_point_rsync_uri": ready.node.handle.publication_point_rsync_uri.as_str(), + "status": "error", + "error": err.error.to_string(), + "stage_fresh_ms": metrics.stage_fresh_ms, + "snapshot_prepare_ms": err.snapshot_prepare_ms, + "repo_sync_source": repo_outcome.repo_sync_source.as_deref(), + "repo_sync_phase": repo_outcome.repo_sync_phase.as_deref(), + "repo_sync_duration_ms": repo_outcome.repo_sync_duration_ms, + }), + ); + } + // The blocking `run_publication_point` fallback stays on the control + // thread; it is executed by the apply phase for this outcome. + return ( + StageOutcome::FreshError(Box::new(FreshErrorOutcome { + ready, + publication_point_started, + })), + metrics, + ); + } + }; + metrics.snapshot_prepare_ms = fresh_stage.snapshot_prepare_ms; + metrics.snapshot_current_index_lock_ms = + fresh_stage.snapshot_prepare_timing.current_index_lock_ms; + metrics.snapshot_manifest_load_ms = fresh_stage.snapshot_prepare_timing.manifest_load_ms; + metrics.snapshot_manifest_index_lookup_ms = + fresh_stage.snapshot_prepare_timing.manifest_index_lookup_ms; + metrics.snapshot_manifest_blob_load_ms = + fresh_stage.snapshot_prepare_timing.manifest_blob_load_ms; + metrics.snapshot_manifest_decode_ms = fresh_stage.snapshot_prepare_timing.manifest_decode_ms; + metrics.snapshot_replay_guard_ms = fresh_stage.snapshot_prepare_timing.replay_guard_ms; + metrics.replay_meta_hit_count = fresh_stage.snapshot_prepare_timing.replay_meta_hit as usize; + metrics.replay_meta_miss_count = fresh_stage.snapshot_prepare_timing.replay_meta_miss as usize; + metrics.snapshot_manifest_entries_ms = fresh_stage.snapshot_prepare_timing.manifest_entries_ms; + metrics.snapshot_pack_files_ms = fresh_stage.snapshot_prepare_timing.pack_files_ms; + metrics.snapshot_pack_files_index_lookup_ms = fresh_stage + .snapshot_prepare_timing + .pack_files_index_lookup_ms; + metrics.snapshot_pack_files_blob_load_ms = + fresh_stage.snapshot_prepare_timing.pack_files_blob_load_ms; + metrics.snapshot_ee_path_validate_ms = fresh_stage.snapshot_prepare_timing.ee_path_validate_ms; + metrics.snapshot_manifest_file_count = fresh_stage.snapshot_prepare_timing.manifest_file_count; + metrics.child_discovery_ms = fresh_stage.child_discovery_ms; + if metrics.stage_fresh_ms >= crate::progress_log::stage_fresh_slow_threshold_ms() { + crate::progress_log::emit( + "phase2_stage_fresh_slow", + serde_json::json!({ + "manifest_rsync_uri": ready.node.handle.manifest_rsync_uri.as_str(), + "publication_point_rsync_uri": ready.node.handle.publication_point_rsync_uri.as_str(), + "status": "ok", + "stage_fresh_ms": metrics.stage_fresh_ms, + "snapshot_prepare_ms": fresh_stage.snapshot_prepare_ms, + "snapshot_current_index_lock_ms": fresh_stage.snapshot_prepare_timing.current_index_lock_ms, + "snapshot_manifest_load_ms": fresh_stage.snapshot_prepare_timing.manifest_load_ms, + "snapshot_manifest_index_lookup_ms": fresh_stage.snapshot_prepare_timing.manifest_index_lookup_ms, + "snapshot_manifest_blob_load_ms": fresh_stage.snapshot_prepare_timing.manifest_blob_load_ms, + "snapshot_manifest_decode_ms": fresh_stage.snapshot_prepare_timing.manifest_decode_ms, + "snapshot_replay_guard_ms": fresh_stage.snapshot_prepare_timing.replay_guard_ms, + "replay_meta_hit": fresh_stage.snapshot_prepare_timing.replay_meta_hit, + "replay_meta_miss": fresh_stage.snapshot_prepare_timing.replay_meta_miss, + "snapshot_manifest_entries_ms": fresh_stage.snapshot_prepare_timing.manifest_entries_ms, + "snapshot_pack_files_ms": fresh_stage.snapshot_prepare_timing.pack_files_ms, + "snapshot_pack_files_index_lookup_ms": fresh_stage.snapshot_prepare_timing.pack_files_index_lookup_ms, + "snapshot_pack_files_blob_load_ms": fresh_stage.snapshot_prepare_timing.pack_files_blob_load_ms, + "snapshot_ee_path_validate_ms": fresh_stage.snapshot_prepare_timing.ee_path_validate_ms, + "snapshot_manifest_file_count": fresh_stage.snapshot_prepare_timing.manifest_file_count, + "child_discovery_ms": fresh_stage.child_discovery_ms, + "child_count": fresh_stage.discovered_children.len(), + "repo_sync_source": repo_outcome.repo_sync_source.as_deref(), + "repo_sync_phase": repo_outcome.repo_sync_phase.as_deref(), + "repo_sync_duration_ms": repo_outcome.repo_sync_duration_ms, + }), + ); + } + warnings.extend(fresh_stage.warnings.clone()); + + metrics.discovered_children = fresh_stage.discovered_children.len(); + + let prepare_started = Instant::now(); + let roa_presence_scan_started = Instant::now(); + let has_roa = fresh_stage + .fresh_point + .files() + .iter() + .any(|file| file.rsync_uri.ends_with(".roa")); + metrics.roa_presence_scan_ms = elapsed_ms(roa_presence_scan_started); + if runner.enable_roa_validation_cache { + if let Some(timing) = runner.timing.as_ref() { + if has_roa { + timing.record_count("roa_validation_cache_roa_candidate_publication_points", 1); + } else { + timing.record_count("roa_validation_cache_skipped_no_roa_publication_points", 1); + } + } + } + let roa_cache_view = if has_roa { + let roa_cache_view_started = Instant::now(); + let view = runner + .roa_validation_cache_view_for_fresh_point(&fresh_stage.fresh_point.manifest_rsync_uri); + metrics.roa_cache_view_ms = elapsed_ms(roa_cache_view_started); + view + } else { + None + }; + let roa_cache = if runner.enable_roa_validation_cache && has_roa { + RoaValidationCacheInput::enabled_with_context( + roa_cache_view.as_ref(), + crate::validation::tree_runner::ca_validation_context_digest_for_ca(&ready.node.handle), + crate::validation::tree_runner::publication_point_cache_policy_fingerprint( + runner.policy, + ), + ) + } else { + RoaValidationCacheInput::disabled() + }; + let ta_constraints = runner + .policy + .ta_constraints + .shared_for_tal(&ready.node.handle.tal_id); + if ta_constraints.is_some() { + if let Some(timing) = runner.timing.as_ref() { + timing.record_count("ta_constraints_parallel_publication_points", 1); + } + } + match prepare_publication_point_for_parallel_roa_with_cache_and_ta_constraints( + ready.node.id, + &fresh_stage.fresh_point, + runner.policy, + fresh_stage.issuer_ca_der.as_ref(), + ready.node.handle.ca_certificate_rsync_uri.as_deref(), + ready.node.handle.effective_ip_resources.as_ref(), + ready.node.handle.effective_as_resources.as_ref(), + runner.validation_time, + runner.persist_vcir, + roa_cache, + ta_constraints, + ) { + ParallelObjectsPrepare::Complete(mut objects) => { + metrics.prepare_ms = elapsed_ms(prepare_started); + runner.record_publication_point_step_ms( + &ready.node.handle.manifest_rsync_uri, + "fresh_objects_prepare", + metrics.prepare_ms, + ); + metrics.complete_count = 1; + metrics.roa_tasks = objects.stats.roa_total; + metrics.aspa_objects = objects.stats.aspa_total; + objects + .router_keys + .extend(fresh_stage.discovered_router_keys.clone()); + objects.local_outputs_cache.extend( + crate::validation::tree_runner::build_router_key_local_outputs( + &ready.node.handle, + &objects.router_keys, + ), + ); + ( + StageOutcome::Complete(Box::new(CompleteOutcome { + ready, + publication_point_started, + fresh_stage, + warnings, + objects, + })), + metrics, + ) + } + ParallelObjectsPrepare::Staged(objects_stage) => { + metrics.prepare_ms = elapsed_ms(prepare_started); + runner.record_publication_point_step_ms( + &ready.node.handle.manifest_rsync_uri, + "fresh_objects_prepare", + metrics.prepare_ms, + ); + metrics.staged_count = 1; + metrics.locked_files = objects_stage.locked_file_count(); + metrics.aspa_objects = objects_stage.aspa_task_count(); + let task_count = objects_stage.roa_task_count(); + metrics.roa_tasks = task_count; + let outcome = StagedOutcome { + ready, + publication_point_started, + fresh_stage, + warnings, + objects_stage, + }; + if task_count == 0 { + metrics.zero_task_count = 1; + (StageOutcome::ZeroTask(Box::new(outcome)), metrics) + } else { + (StageOutcome::Fresh(Box::new(outcome)), metrics) + } + } + } +} + +fn apply_ready_publication_point_stage( + runner: &Rpkiv1PublicationPointRunner<'_>, + next_id: &mut u64, + ca_queue: &mut VecDeque, + pending_roa_dispatch: &mut VecDeque, + inflight_publication_points: &mut HashMap, + pending_finalization: &mut VecDeque, + finished: &mut Vec, + outcome: StageOutcome, + mut metrics: ReadyStageMetrics, + config: &TreeRunConfig, + compact_audit: bool, +) -> ReadyStageMetrics { + match outcome { + StageOutcome::CacheHit(outcome) => { + let CacheHitOutcome { + ready, + publication_point_started, + result, + } = *outcome; + let repo_outcome = ready.repo_outcome.clone(); + let child_enqueue_started = Instant::now(); + enqueue_discovered_children( + runner, + next_id, + ca_queue, + &ready.node, + config, + result.discovered_children.clone(), + ); + metrics.child_enqueue_ms = elapsed_ms(child_enqueue_started); + runner.record_publication_point_step_ms( + metrics.manifest_rsync_uri.as_deref().unwrap_or_default(), + "publication_point_cache_child_enqueue", + metrics.child_enqueue_ms, + ); + finished.push(FinishedPublicationPoint { + node: FinishedPublicationPointNode::from_queued(ready.node), + result: compact_phase2_finished_result(result, compact_audit), + }); + metrics.total_ms = elapsed_ms(publication_point_started); + emit_ready_publication_point_control_slow( + metrics.manifest_rsync_uri.as_deref().unwrap_or_default(), + metrics + .publication_point_rsync_uri + .as_deref() + .unwrap_or_default(), + &repo_outcome, + &metrics, + "publication_point_cache", + false, + ); + metrics + } + StageOutcome::FreshError(outcome) => { + let FreshErrorOutcome { + ready, + publication_point_started, + } = *outcome; + let repo_outcome = ready.repo_outcome.clone(); + metrics.fallback_count = 1; + let fallback_started = Instant::now(); + let fallback = runner.run_publication_point(&ready.node.handle); + metrics.fallback_full_run_ms = elapsed_ms(fallback_started); + if let Ok(result) = fallback.as_ref() { + metrics.discovered_children = result.discovered_children.len(); + let child_enqueue_started = Instant::now(); + enqueue_discovered_children( + runner, + next_id, + ca_queue, + &ready.node, + config, + result.discovered_children.clone(), + ); + metrics.child_enqueue_ms = elapsed_ms(child_enqueue_started); + } + finished.push(FinishedPublicationPoint { + node: FinishedPublicationPointNode::from_queued(ready.node), + result: compact_phase2_finished_result_result(fallback, compact_audit), + }); + metrics.total_ms = elapsed_ms(publication_point_started); + emit_ready_publication_point_control_slow( + metrics.manifest_rsync_uri.as_deref().unwrap_or_default(), + metrics + .publication_point_rsync_uri + .as_deref() + .unwrap_or_default(), + &repo_outcome, + &metrics, + "fallback", + true, + ); + metrics + } + StageOutcome::Complete(outcome) => { + let CompleteOutcome { + ready, + publication_point_started, + fresh_stage, + warnings, + objects, + } = *outcome; + let repo_outcome = ready.repo_outcome.clone(); + let child_enqueue_started = Instant::now(); + enqueue_discovered_children( + runner, + next_id, + ca_queue, + &ready.node, + config, + fresh_stage.discovered_children.clone(), + ); + metrics.child_enqueue_ms = elapsed_ms(child_enqueue_started); + // The finalize no longer runs synchronously here: the publication + // point is queued for the shared finalize worker through the same + // pending_finalization path (queue capacity backpressure and + // finalize_inflight accounting included) as zero-task staging. The + // per-publication-point total timing is recorded by the finalize + // worker, exactly like the zero-task and staged paths. + let direct_finalize_started = Instant::now(); + pending_finalization.push_back(FinalizeTask { + state: InflightPublicationPoint { + node: ready.node, + fresh_stage, + objects_prepare: ParallelObjectsPrepare::Complete(objects), + repo_outcome: repo_outcome.clone(), + warnings, + started_at: publication_point_started, + objects_started_at: Instant::now(), + task_count: 0, + tasks_submitted: 0, + first_task_submitted_at: None, + last_task_submitted_at: None, + first_result_at: None, + last_result_at: None, + worker_ms_total: 0, + worker_ms_max: 0, + queue_wait_ms_total: 0, + queue_wait_ms_max: 0, + finalize_enqueued_at: Some(Instant::now()), + results: Vec::new(), + }, + }); + metrics.direct_finalize_ms = elapsed_ms(direct_finalize_started); + runner.record_publication_point_step_ms( + &metrics.manifest_rsync_uri.clone().unwrap_or_default(), + "fresh_direct_finalize", + metrics.direct_finalize_ms, + ); + metrics.total_ms = elapsed_ms(publication_point_started); + emit_ready_publication_point_control_slow( + metrics.manifest_rsync_uri.as_deref().unwrap_or_default(), + metrics + .publication_point_rsync_uri + .as_deref() + .unwrap_or_default(), + &repo_outcome, + &metrics, + "complete", + false, + ); + metrics + } + StageOutcome::ZeroTask(outcome) => { + let StagedOutcome { + ready, + publication_point_started, + fresh_stage, + warnings, + objects_stage, + } = *outcome; + let repo_outcome = ready.repo_outcome.clone(); + let child_enqueue_started = Instant::now(); + enqueue_discovered_children( + runner, + next_id, + ca_queue, + &ready.node, + config, + fresh_stage.discovered_children.clone(), + ); + metrics.child_enqueue_ms = elapsed_ms(child_enqueue_started); + let build_tasks_started = Instant::now(); + objects_stage.append_roa_tasks_to(pending_roa_dispatch); + metrics.build_roa_tasks_ms = elapsed_ms(build_tasks_started); + runner.record_publication_point_step_ms( + &ready.node.handle.manifest_rsync_uri, + "fresh_build_roa_tasks", + metrics.build_roa_tasks_ms, + ); + let task_count = objects_stage.roa_task_count(); + pending_finalization.push_back(FinalizeTask { + state: InflightPublicationPoint { + node: ready.node, + fresh_stage, + objects_prepare: ParallelObjectsPrepare::Staged(objects_stage), + repo_outcome: repo_outcome.clone(), + warnings, + started_at: publication_point_started, + objects_started_at: Instant::now(), + task_count, + tasks_submitted: 0, + first_task_submitted_at: None, + last_task_submitted_at: None, + first_result_at: None, + last_result_at: None, + worker_ms_total: 0, + worker_ms_max: 0, + queue_wait_ms_total: 0, + queue_wait_ms_max: 0, + finalize_enqueued_at: Some(Instant::now()), + results: Vec::new(), + }, + }); + metrics.total_ms = elapsed_ms(publication_point_started); + emit_ready_publication_point_control_slow( + metrics.manifest_rsync_uri.as_deref().unwrap_or_default(), + metrics + .publication_point_rsync_uri + .as_deref() + .unwrap_or_default(), + &repo_outcome, + &metrics, + "zero_task", + false, + ); + metrics + } + StageOutcome::Fresh(outcome) => { + let StagedOutcome { + ready, + publication_point_started, + fresh_stage, + warnings, + objects_stage, + } = *outcome; + let repo_outcome = ready.repo_outcome.clone(); + let child_enqueue_started = Instant::now(); + enqueue_discovered_children( + runner, + next_id, + ca_queue, + &ready.node, + config, + fresh_stage.discovered_children.clone(), + ); + metrics.child_enqueue_ms = elapsed_ms(child_enqueue_started); + let build_tasks_started = Instant::now(); + objects_stage.append_roa_tasks_to(pending_roa_dispatch); + metrics.build_roa_tasks_ms = elapsed_ms(build_tasks_started); + runner.record_publication_point_step_ms( + &ready.node.handle.manifest_rsync_uri, + "fresh_build_roa_tasks", + metrics.build_roa_tasks_ms, + ); + let task_count = objects_stage.roa_task_count(); + inflight_publication_points.insert( + ready.node.id, + InflightPublicationPoint { + node: ready.node, + fresh_stage, + objects_prepare: ParallelObjectsPrepare::Staged(objects_stage), + repo_outcome: repo_outcome.clone(), + warnings, + started_at: publication_point_started, + objects_started_at: Instant::now(), + task_count, + tasks_submitted: 0, + first_task_submitted_at: None, + last_task_submitted_at: None, + first_result_at: None, + last_result_at: None, + worker_ms_total: 0, + worker_ms_max: 0, + queue_wait_ms_total: 0, + queue_wait_ms_max: 0, + finalize_enqueued_at: None, + results: Vec::with_capacity(task_count), + }, + ); + metrics.total_ms = elapsed_ms(publication_point_started); + emit_ready_publication_point_control_slow( + metrics.manifest_rsync_uri.as_deref().unwrap_or_default(), + metrics + .publication_point_rsync_uri + .as_deref() + .unwrap_or_default(), + &repo_outcome, + &metrics, + "staged", + false, + ); + metrics + } + } +} + +fn emit_ready_publication_point_control_slow( + manifest_rsync_uri: &str, + publication_point_rsync_uri: &str, + repo_outcome: &RepoSyncRuntimeOutcome, + metrics: &ReadyStageMetrics, + status: &str, + force_error_path: bool, +) { + let threshold_ms = crate::progress_log::pp_control_slow_threshold_ms(); + if !force_error_path && metrics.total_ms < threshold_ms { + return; + } + crate::progress_log::emit( + "phase2_ready_publication_point_control_slow", + serde_json::json!({ + "manifest_rsync_uri": manifest_rsync_uri, + "publication_point_rsync_uri": publication_point_rsync_uri, + "status": status, + "repo_sync_source": repo_outcome.repo_sync_source.as_deref(), + "repo_sync_phase": repo_outcome.repo_sync_phase.as_deref(), + "repo_sync_duration_ms": repo_outcome.repo_sync_duration_ms, + "repo_sync_ok": repo_outcome.repo_sync_ok, + "repo_sync_err": repo_outcome.repo_sync_err.as_deref(), + "ready_queue_wait_ms": metrics.ready_queue_wait_ms, + "ready_queue_len_after_pop": metrics.ready_queue_len_after_pop, + "stage_fresh_ms": metrics.stage_fresh_ms, + "child_discovery_ms": metrics.child_discovery_ms, + "child_enqueue_ms": metrics.child_enqueue_ms, + "discovered_children": metrics.discovered_children, + "roa_presence_scan_ms": metrics.roa_presence_scan_ms, + "roa_cache_view_ms": metrics.roa_cache_view_ms, + "prepare_ms": metrics.prepare_ms, + "build_roa_tasks_ms": metrics.build_roa_tasks_ms, + "direct_finalize_ms": metrics.direct_finalize_ms, + "fallback_full_run_ms": metrics.fallback_full_run_ms, + "locked_files": metrics.locked_files, + "roa_tasks": metrics.roa_tasks, + "aspa_objects": metrics.aspa_objects, + "complete_count": metrics.complete_count, + "staged_count": metrics.staged_count, + "zero_task_count": metrics.zero_task_count, + "fallback_count": metrics.fallback_count, + "total_ms": metrics.total_ms, + "slow_threshold_ms": threshold_ms, + }), + ); + crate::progress_log::emit( + "phase2_ready_publication_point_control_snapshot_breakdown", + serde_json::json!({ + "manifest_rsync_uri": manifest_rsync_uri, + "publication_point_rsync_uri": publication_point_rsync_uri, + "status": status, + "snapshot_prepare_ms": metrics.snapshot_prepare_ms, + "snapshot_current_index_lock_ms": metrics.snapshot_current_index_lock_ms, + "snapshot_manifest_load_ms": metrics.snapshot_manifest_load_ms, + "snapshot_manifest_index_lookup_ms": metrics.snapshot_manifest_index_lookup_ms, + "snapshot_manifest_blob_load_ms": metrics.snapshot_manifest_blob_load_ms, + "snapshot_manifest_decode_ms": metrics.snapshot_manifest_decode_ms, + "snapshot_replay_guard_ms": metrics.snapshot_replay_guard_ms, + "replay_meta_hit_count": metrics.replay_meta_hit_count, + "replay_meta_miss_count": metrics.replay_meta_miss_count, + "snapshot_manifest_entries_ms": metrics.snapshot_manifest_entries_ms, + "snapshot_pack_files_ms": metrics.snapshot_pack_files_ms, + "snapshot_pack_files_index_lookup_ms": metrics.snapshot_pack_files_index_lookup_ms, + "snapshot_pack_files_blob_load_ms": metrics.snapshot_pack_files_blob_load_ms, + "snapshot_ee_path_validate_ms": metrics.snapshot_ee_path_validate_ms, + "snapshot_manifest_file_count": metrics.snapshot_manifest_file_count, + "total_ms": metrics.total_ms, + "slow_threshold_ms": threshold_ms, + }), + ); +} + +fn enqueue_discovered_children( + runner: &Rpkiv1PublicationPointRunner<'_>, + next_id: &mut u64, + ca_queue: &mut VecDeque, + parent: &QueuedCaInstance, + config: &TreeRunConfig, + mut children: Vec, +) { + let Some(child_depth) = next_allowed_ca_depth(config, parent.handle.depth) else { + return; + }; + + children.sort_by(|a, b| { + a.handle + .manifest_rsync_uri + .cmp(&b.handle.manifest_rsync_uri) + .then_with(|| { + a.discovered_from + .child_ca_certificate_rsync_uri + .cmp(&b.discovered_from.child_ca_certificate_rsync_uri) + }) + }); + if let Some(runtime) = runner.repo_sync_runtime.as_ref() { + let _ = runtime.prefetch_discovered_children(&children); + } + for child in children { + let mut handle = child.handle.with_depth(child_depth); + handle.parent_manifest_rsync_uri = Some(parent.handle.manifest_rsync_uri.clone()); + ca_queue.push_back(QueuedCaInstance { + id: *next_id, + handle, + parent_id: Some(parent.id), + discovered_from: Some(child.discovered_from), + }); + *next_id += 1; + } +} + +fn finalize_metrics_from_output( + output: &FreshPublicationPointFinalizeOutput, + reduce_ms: u64, + finalize_ms: u64, + finalize_queue_wait_ms: Option, + finalize_worker_ms: u64, + locked_files: usize, +) -> FinalizePublicationPointMetrics { + FinalizePublicationPointMetrics { + reduce_ms, + finalize_ms, + finalize_queue_wait_ms, + finalize_worker_ms, + snapshot_pack_ms: output.snapshot_pack_ms, + persist_vcir_ms: output.persist_vcir_ms, + persist_build_vcir_ms: output.persist_vcir_timing.build_vcir_ms, + persist_replace_vcir_ms: output.persist_vcir_timing.replace_vcir_ms, + persist_replace_breakdown: output.persist_vcir_timing.replace_vcir.clone(), + ccr_projection_build_ms: output.ccr_projection_build_ms, + ccr_append_ms: output.ccr_append_ms, + audit_build_ms: output.audit_build_ms, + locked_files, + child_count: output.result.discovered_children.len(), + warning_count: output.result.warnings.len(), + vrp_count: output.result.objects.vrps.len(), + vap_count: output.result.objects.aspas.len(), + router_key_count: output.result.objects.router_keys.len(), + audit_object_count: output.result.audit.objects.len(), + } +} + +fn emit_finalize_breakdown( + event_name: &str, + manifest_rsync_uri: &str, + publication_point_rsync_uri: &str, + metrics: &FinalizePublicationPointMetrics, +) { + crate::progress_log::emit( + event_name, + serde_json::json!({ + "manifest_rsync_uri": manifest_rsync_uri, + "publication_point_rsync_uri": publication_point_rsync_uri, + "reduce_ms": metrics.reduce_ms, + "finalize_ms": metrics.finalize_ms, + "finalize_queue_wait_ms": metrics.finalize_queue_wait_ms, + "finalize_worker_ms": metrics.finalize_worker_ms, + "snapshot_pack_ms": metrics.snapshot_pack_ms, + "persist_vcir_ms": metrics.persist_vcir_ms, + "persist_build_vcir_ms": metrics.persist_build_vcir_ms, + "persist_replace_vcir_ms": metrics.persist_replace_vcir_ms, + "persist_replace_breakdown": &metrics.persist_replace_breakdown, + "ccr_projection_build_ms": metrics.ccr_projection_build_ms, + "ccr_append_ms": metrics.ccr_append_ms, + "audit_build_ms": metrics.audit_build_ms, + "locked_files": metrics.locked_files, + "child_count": metrics.child_count, + "warning_count": metrics.warning_count, + "vrp_count": metrics.vrp_count, + "vap_count": metrics.vap_count, + "router_key_count": metrics.router_key_count, + "audit_object_count": metrics.audit_object_count, + }), + ); +} diff --git a/crates/panda-rpki-validator/src/validation/tree_parallel/state.rs b/crates/panda-rpki-validator/src/validation/tree_parallel/state.rs new file mode 100644 index 0000000..a815d9c --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_parallel/state.rs @@ -0,0 +1,597 @@ +#[derive(Clone, Debug)] +struct QueuedCaInstance { + id: u64, + handle: CaInstanceHandle, + parent_id: Option, + discovered_from: Option, +} + +#[derive(Clone, Debug)] +struct ReadyCaInstance { + node: QueuedCaInstance, + repo_outcome: RepoSyncRuntimeOutcome, + ready_enqueued_at: Instant, +} + +struct InflightPublicationPoint { + node: QueuedCaInstance, + fresh_stage: FreshPublicationPointStage, + objects_prepare: ParallelObjectsPrepare, + repo_outcome: RepoSyncRuntimeOutcome, + warnings: Vec, + started_at: Instant, + objects_started_at: Instant, + task_count: usize, + tasks_submitted: usize, + first_task_submitted_at: Option, + last_task_submitted_at: Option, + first_result_at: Option, + last_result_at: Option, + worker_ms_total: u64, + worker_ms_max: u64, + queue_wait_ms_total: u64, + queue_wait_ms_max: u64, + finalize_enqueued_at: Option, + results: Vec, +} + +struct FinishedPublicationPoint { + node: FinishedPublicationPointNode, + result: FinishedPublicationPointResult, +} + +#[derive(Clone, Debug)] +struct FinishedPublicationPointNode { + id: u64, + parent_id: Option, + discovered_from: Option, + manifest_rsync_uri: String, +} + +impl FinishedPublicationPointNode { + fn from_queued(node: QueuedCaInstance) -> Self { + Self { + id: node.id, + parent_id: node.parent_id, + discovered_from: node.discovered_from, + manifest_rsync_uri: node.handle.manifest_rsync_uri, + } + } +} + +#[derive(Debug)] +enum FinishedPublicationPointResult { + Ok { + source: PublicationPointSource, + warnings: Vec, + objects: ObjectsOutput, + audit: PublicationPointAudit, + cir_fresh_objects: Vec, + cir_cached_objects: Vec, + }, + Err(String), +} + +struct FinalizeTask { + state: InflightPublicationPoint, +} + +/// Outcome of the pure compute phase for one ready publication point. +/// +/// `compute_ready_publication_point_stage` only performs read-only validation +/// work (publication point cache lookup, fresh snapshot staging, ROA prepare) +/// and returns this enum. `apply_ready_publication_point_stage` then performs +/// every write to control-loop state (`ca_queue`/`next_id`, `finished`, +/// `pending_roa_dispatch`, `pending_finalization`, `inflight_publication_points`) +/// in the same per-publication-point order the monolithic staging function did. +/// Each variant payload is boxed so the enum itself stays small. +enum StageOutcome { + /// Publication point cache hit: the run result is fully built in memory and + /// only needs child enqueueing plus a `finished` entry. + CacheHit(Box), + /// Fresh staging failed: apply runs the existing blocking + /// `run_publication_point` fallback inline on the control thread. + FreshError(Box), + /// Fresh staging succeeded and ROA prepare returned complete objects (no + /// ROA tasks to dispatch): apply hands the publication point to the + /// finalize worker through `pending_finalization` instead of running the + /// finalize synchronously on the control thread. + Complete(Box), + /// Fresh staging succeeded with a staged objects plan that contains zero + /// ROA tasks: apply queues the finalize task directly. + ZeroTask(Box), + /// Fresh staging succeeded with ROA tasks to dispatch: apply appends the + /// tasks to `pending_roa_dispatch` and registers the inflight publication + /// point. + Fresh(Box), +} + +struct CacheHitOutcome { + ready: ReadyCaInstance, + publication_point_started: Instant, + result: PublicationPointRunResult, +} + +struct FreshErrorOutcome { + ready: ReadyCaInstance, + publication_point_started: Instant, +} + +struct CompleteOutcome { + ready: ReadyCaInstance, + publication_point_started: Instant, + fresh_stage: FreshPublicationPointStage, + warnings: Vec, + objects: ObjectsOutput, +} + +struct StagedOutcome { + ready: ReadyCaInstance, + publication_point_started: Instant, + fresh_stage: FreshPublicationPointStage, + warnings: Vec, + objects_stage: ParallelObjectsStage, +} + +struct FinalizeWorkerResult { + finished: FinishedPublicationPoint, + metrics: FinalizePublicationPointMetrics, +} + +/// Task submitted to the experimental ready-stage worker pool: everything +/// `compute_ready_publication_point_stage` needs for one ready publication +/// point. +struct ReadyStageTask { + ready: ReadyCaInstance, + ready_queue_len_after_pop: usize, + submitted_at: Instant, +} + +/// Result drained from the ready-stage worker pool: the compute outcome and +/// its metrics plus per-task pool timing for the `phase2_stage_pool_stats` +/// observability event. +struct ReadyStageWorkerResult { + outcome: StageOutcome, + metrics: ReadyStageMetrics, + queue_wait_ms: u64, + worker_ms: u64, +} + +/// Executor borrowing the publication point runner so stage workers can run +/// the read-only compute phase off the control thread. The runner is shared +/// with the finalize worker and the ROA pool in the same way; the scoped pool +/// guarantees all borrows end before the enclosing `std::thread::scope`. +struct ReadyStageTaskExecutor<'a> { + runner: &'a Rpkiv1PublicationPointRunner<'a>, +} + +impl<'a> ObjectTaskExecutor for ReadyStageTaskExecutor<'a> { + fn execute(&self, _worker_index: usize, task: ReadyStageTask) -> ReadyStageWorkerResult { + let worker_started = Instant::now(); + let queue_wait_ms = worker_started + .saturating_duration_since(task.submitted_at) + .as_millis() as u64; + let (outcome, metrics) = compute_ready_publication_point_stage( + self.runner, + task.ready, + task.ready_queue_len_after_pop, + ); + ReadyStageWorkerResult { + outcome, + metrics, + queue_wait_ms, + worker_ms: elapsed_ms(worker_started), + } + } +} + +type ReadyStagePool<'scope, 'env> = ScopedObjectWorkerPool< + 'scope, + 'env, + ReadyStageTask, + ReadyStageWorkerResult, + ReadyStageTaskExecutor<'env>, +>; + +#[derive(Default)] +struct StageDispatchMetrics { + submitted: usize, + queue_full: bool, + duration_ms: u64, +} + +#[derive(Default)] +struct StageDrainMetrics { + results_drained: usize, + queue_wait_ms_total: u64, + queue_wait_ms_max: u64, + worker_ms_total: u64, + worker_ms_max: u64, + duration_ms: u64, +} + +#[derive(Default)] +struct ReadyStageMetrics { + manifest_rsync_uri: Option, + publication_point_rsync_uri: Option, + ready_count: usize, + fallback_count: usize, + complete_count: usize, + staged_count: usize, + zero_task_count: usize, + error_count: usize, + discovered_children: usize, + locked_files: usize, + roa_tasks: usize, + aspa_objects: usize, + stage_fresh_ms: u64, + snapshot_prepare_ms: u64, + snapshot_current_index_lock_ms: u64, + snapshot_manifest_load_ms: u64, + snapshot_manifest_index_lookup_ms: u64, + snapshot_manifest_blob_load_ms: u64, + snapshot_manifest_decode_ms: u64, + snapshot_replay_guard_ms: u64, + replay_meta_hit_count: usize, + replay_meta_miss_count: usize, + snapshot_manifest_entries_ms: u64, + snapshot_pack_files_ms: u64, + snapshot_pack_files_index_lookup_ms: u64, + snapshot_pack_files_blob_load_ms: u64, + snapshot_ee_path_validate_ms: u64, + snapshot_manifest_file_count: usize, + child_discovery_ms: u64, + child_enqueue_ms: u64, + ready_queue_wait_ms: u64, + ready_queue_len_after_pop: usize, + roa_presence_scan_ms: u64, + roa_cache_view_ms: u64, + direct_finalize_ms: u64, + fallback_full_run_ms: u64, + prepare_ms: u64, + build_roa_tasks_ms: u64, + total_ms: u64, +} + +#[derive(Default)] +struct ReadyStageBatchMetrics { + ready_count: usize, + fallback_count: usize, + complete_count: usize, + staged_count: usize, + zero_task_count: usize, + error_count: usize, + discovered_children: usize, + locked_files: usize, + roa_tasks: usize, + aspa_objects: usize, + stage_fresh_ms_total: u64, + stage_fresh_ms_max: u64, + stage_fresh_ms_max_manifest_rsync_uri: Option, + stage_fresh_ms_max_publication_point_rsync_uri: Option, + snapshot_prepare_ms_total: u64, + snapshot_prepare_ms_max: u64, + snapshot_current_index_lock_ms_total: u64, + snapshot_current_index_lock_ms_max: u64, + snapshot_manifest_load_ms_total: u64, + snapshot_manifest_load_ms_max: u64, + snapshot_manifest_index_lookup_ms_total: u64, + snapshot_manifest_index_lookup_ms_max: u64, + snapshot_manifest_blob_load_ms_total: u64, + snapshot_manifest_blob_load_ms_max: u64, + snapshot_manifest_decode_ms_total: u64, + snapshot_manifest_decode_ms_max: u64, + snapshot_replay_guard_ms_total: u64, + snapshot_replay_guard_ms_max: u64, + replay_meta_hit_count: usize, + replay_meta_miss_count: usize, + snapshot_manifest_entries_ms_total: u64, + snapshot_manifest_entries_ms_max: u64, + snapshot_pack_files_ms_total: u64, + snapshot_pack_files_ms_max: u64, + snapshot_pack_files_index_lookup_ms_total: u64, + snapshot_pack_files_index_lookup_ms_max: u64, + snapshot_pack_files_blob_load_ms_total: u64, + snapshot_pack_files_blob_load_ms_max: u64, + snapshot_ee_path_validate_ms_total: u64, + snapshot_ee_path_validate_ms_max: u64, + snapshot_manifest_file_count_total: usize, + snapshot_manifest_file_count_max: usize, + child_discovery_ms_total: u64, + child_discovery_ms_max: u64, + child_enqueue_ms_total: u64, + child_enqueue_ms_max: u64, + ready_queue_wait_ms_total: u64, + ready_queue_wait_ms_max: u64, + roa_presence_scan_ms_total: u64, + roa_presence_scan_ms_max: u64, + roa_cache_view_ms_total: u64, + roa_cache_view_ms_max: u64, + direct_finalize_ms_total: u64, + direct_finalize_ms_max: u64, + fallback_full_run_ms_total: u64, + fallback_full_run_ms_max: u64, + prepare_ms_total: u64, + prepare_ms_max: u64, + build_roa_tasks_ms_total: u64, + build_roa_tasks_ms_max: u64, + total_ms: u64, +} + +impl ReadyStageBatchMetrics { + fn record(&mut self, metrics: ReadyStageMetrics) { + self.ready_count += metrics.ready_count; + self.fallback_count += metrics.fallback_count; + self.complete_count += metrics.complete_count; + self.staged_count += metrics.staged_count; + self.zero_task_count += metrics.zero_task_count; + self.error_count += metrics.error_count; + self.discovered_children += metrics.discovered_children; + self.locked_files += metrics.locked_files; + self.roa_tasks += metrics.roa_tasks; + self.aspa_objects += metrics.aspa_objects; + if metrics.stage_fresh_ms >= self.stage_fresh_ms_max { + self.stage_fresh_ms_max_manifest_rsync_uri = metrics.manifest_rsync_uri.clone(); + self.stage_fresh_ms_max_publication_point_rsync_uri = + metrics.publication_point_rsync_uri.clone(); + } + self.stage_fresh_ms_total += metrics.stage_fresh_ms; + self.stage_fresh_ms_max = self.stage_fresh_ms_max.max(metrics.stage_fresh_ms); + self.snapshot_prepare_ms_total += metrics.snapshot_prepare_ms; + self.snapshot_prepare_ms_max = self + .snapshot_prepare_ms_max + .max(metrics.snapshot_prepare_ms); + self.snapshot_current_index_lock_ms_total += metrics.snapshot_current_index_lock_ms; + self.snapshot_current_index_lock_ms_max = self + .snapshot_current_index_lock_ms_max + .max(metrics.snapshot_current_index_lock_ms); + self.snapshot_manifest_load_ms_total += metrics.snapshot_manifest_load_ms; + self.snapshot_manifest_load_ms_max = self + .snapshot_manifest_load_ms_max + .max(metrics.snapshot_manifest_load_ms); + self.snapshot_manifest_index_lookup_ms_total += metrics.snapshot_manifest_index_lookup_ms; + self.snapshot_manifest_index_lookup_ms_max = self + .snapshot_manifest_index_lookup_ms_max + .max(metrics.snapshot_manifest_index_lookup_ms); + self.snapshot_manifest_blob_load_ms_total += metrics.snapshot_manifest_blob_load_ms; + self.snapshot_manifest_blob_load_ms_max = self + .snapshot_manifest_blob_load_ms_max + .max(metrics.snapshot_manifest_blob_load_ms); + self.snapshot_manifest_decode_ms_total += metrics.snapshot_manifest_decode_ms; + self.snapshot_manifest_decode_ms_max = self + .snapshot_manifest_decode_ms_max + .max(metrics.snapshot_manifest_decode_ms); + self.snapshot_replay_guard_ms_total += metrics.snapshot_replay_guard_ms; + self.snapshot_replay_guard_ms_max = self + .snapshot_replay_guard_ms_max + .max(metrics.snapshot_replay_guard_ms); + self.replay_meta_hit_count += metrics.replay_meta_hit_count; + self.replay_meta_miss_count += metrics.replay_meta_miss_count; + self.snapshot_manifest_entries_ms_total += metrics.snapshot_manifest_entries_ms; + self.snapshot_manifest_entries_ms_max = self + .snapshot_manifest_entries_ms_max + .max(metrics.snapshot_manifest_entries_ms); + self.snapshot_pack_files_ms_total += metrics.snapshot_pack_files_ms; + self.snapshot_pack_files_ms_max = self + .snapshot_pack_files_ms_max + .max(metrics.snapshot_pack_files_ms); + self.snapshot_pack_files_index_lookup_ms_total += + metrics.snapshot_pack_files_index_lookup_ms; + self.snapshot_pack_files_index_lookup_ms_max = self + .snapshot_pack_files_index_lookup_ms_max + .max(metrics.snapshot_pack_files_index_lookup_ms); + self.snapshot_pack_files_blob_load_ms_total += metrics.snapshot_pack_files_blob_load_ms; + self.snapshot_pack_files_blob_load_ms_max = self + .snapshot_pack_files_blob_load_ms_max + .max(metrics.snapshot_pack_files_blob_load_ms); + self.snapshot_ee_path_validate_ms_total += metrics.snapshot_ee_path_validate_ms; + self.snapshot_ee_path_validate_ms_max = self + .snapshot_ee_path_validate_ms_max + .max(metrics.snapshot_ee_path_validate_ms); + self.snapshot_manifest_file_count_total += metrics.snapshot_manifest_file_count; + self.snapshot_manifest_file_count_max = self + .snapshot_manifest_file_count_max + .max(metrics.snapshot_manifest_file_count); + self.child_discovery_ms_total += metrics.child_discovery_ms; + self.child_discovery_ms_max = self.child_discovery_ms_max.max(metrics.child_discovery_ms); + self.child_enqueue_ms_total += metrics.child_enqueue_ms; + self.child_enqueue_ms_max = self.child_enqueue_ms_max.max(metrics.child_enqueue_ms); + self.ready_queue_wait_ms_total += metrics.ready_queue_wait_ms; + self.ready_queue_wait_ms_max = self + .ready_queue_wait_ms_max + .max(metrics.ready_queue_wait_ms); + self.roa_presence_scan_ms_total += metrics.roa_presence_scan_ms; + self.roa_presence_scan_ms_max = self + .roa_presence_scan_ms_max + .max(metrics.roa_presence_scan_ms); + self.roa_cache_view_ms_total += metrics.roa_cache_view_ms; + self.roa_cache_view_ms_max = self.roa_cache_view_ms_max.max(metrics.roa_cache_view_ms); + self.direct_finalize_ms_total += metrics.direct_finalize_ms; + self.direct_finalize_ms_max = self.direct_finalize_ms_max.max(metrics.direct_finalize_ms); + self.fallback_full_run_ms_total += metrics.fallback_full_run_ms; + self.fallback_full_run_ms_max = self + .fallback_full_run_ms_max + .max(metrics.fallback_full_run_ms); + self.prepare_ms_total += metrics.prepare_ms; + self.prepare_ms_max = self.prepare_ms_max.max(metrics.prepare_ms); + self.build_roa_tasks_ms_total += metrics.build_roa_tasks_ms; + self.build_roa_tasks_ms_max = self.build_roa_tasks_ms_max.max(metrics.build_roa_tasks_ms); + self.total_ms += metrics.total_ms; + } +} + +#[derive(Default)] +struct RoaDispatchMetrics { + attempted: usize, + submitted: usize, + queue_full: bool, + pending_remaining: usize, + duration_ms: u64, +} + +#[derive(Default)] +struct ObjectDrainMetrics { + results_drained: usize, + publication_points_completed: usize, + worker_ms_total: u64, + worker_ms_max: u64, + queue_wait_ms_total: u64, + queue_wait_ms_max: u64, + result_budget_exhausted: bool, + duration_ms: u64, +} + +#[derive(Default)] +struct FinalizeSubmitMetrics { + submitted: usize, + queue_full: bool, + duration_ms: u64, +} + +#[derive(Default)] +struct FinalizePublicationPointMetrics { + reduce_ms: u64, + finalize_ms: u64, + finalize_queue_wait_ms: Option, + finalize_worker_ms: u64, + snapshot_pack_ms: u64, + persist_vcir_ms: u64, + persist_build_vcir_ms: u64, + persist_replace_vcir_ms: u64, + persist_replace_breakdown: VcirReplaceTimingBreakdown, + ccr_projection_build_ms: u64, + ccr_append_ms: u64, + audit_build_ms: u64, + locked_files: usize, + child_count: usize, + warning_count: usize, + vrp_count: usize, + vap_count: usize, + router_key_count: usize, + audit_object_count: usize, +} + +#[derive(Default)] +struct FinalizeResultsDrainMetrics { + results_drained: usize, + reduce_ms_total: u64, + reduce_ms_max: u64, + finalize_ms_total: u64, + finalize_ms_max: u64, + finalize_queue_wait_ms_max: u64, + finalize_worker_ms_total: u64, + finalize_worker_ms_max: u64, + snapshot_pack_ms_total: u64, + snapshot_pack_ms_max: u64, + persist_vcir_ms_total: u64, + persist_vcir_ms_max: u64, + persist_build_vcir_ms_total: u64, + persist_build_vcir_ms_max: u64, + persist_replace_vcir_ms_total: u64, + persist_replace_vcir_ms_max: u64, + ccr_projection_build_ms_total: u64, + ccr_projection_build_ms_max: u64, + ccr_append_ms_total: u64, + ccr_append_ms_max: u64, + audit_build_ms_total: u64, + audit_build_ms_max: u64, + duration_ms: u64, +} + +#[derive(Default)] +struct RepoDrainMetrics { + event_count: usize, + completions: usize, + ready_enqueued: usize, + duration_ms: u64, +} + +const REPO_RESULT_DRAIN_MAX_EVENTS: usize = 64; + +fn elapsed_ms(started: Instant) -> u64 { + started.elapsed().as_millis() as u64 +} + +fn emit_control_loop_slow( + duration_ms: u64, + repo_poll_timeout: Duration, + repo_metrics: &RepoDrainMetrics, + ready_batch_metrics: &ReadyStageBatchMetrics, + ca_queue_len: usize, + ready_queue_len: usize, + ca_waiting_repo_identities: usize, + pending_roa_dispatch_len: usize, + inflight_publication_points_len: usize, + pending_finalization_len: usize, + finalize_inflight: usize, +) { + let threshold_ms = crate::progress_log::control_loop_slow_threshold_ms(); + if duration_ms < threshold_ms { + return; + } + crate::progress_log::emit( + "phase2_control_loop_slow", + serde_json::json!({ + "duration_ms": duration_ms, + "slow_threshold_ms": threshold_ms, + "repo_poll_timeout_ms": repo_poll_timeout.as_millis() as u64, + "repo_event_count": repo_metrics.event_count, + "repo_completions": repo_metrics.completions, + "repo_ready_enqueued": repo_metrics.ready_enqueued, + "repo_drain_duration_ms": repo_metrics.duration_ms, + "ready_count": ready_batch_metrics.ready_count, + "ready_batch_duration_ms": ready_batch_metrics.total_ms, + "ready_batch_stage_fresh_ms_total": ready_batch_metrics.stage_fresh_ms_total, + "ready_batch_stage_fresh_ms_max": ready_batch_metrics.stage_fresh_ms_max, + "ready_batch_stage_fresh_ms_max_manifest_rsync_uri": ready_batch_metrics.stage_fresh_ms_max_manifest_rsync_uri, + "ready_batch_child_discovery_ms_total": ready_batch_metrics.child_discovery_ms_total, + "ready_batch_child_discovery_ms_max": ready_batch_metrics.child_discovery_ms_max, + "ready_batch_prepare_ms_total": ready_batch_metrics.prepare_ms_total, + "ready_batch_prepare_ms_max": ready_batch_metrics.prepare_ms_max, + "ready_batch_direct_finalize_ms_total": ready_batch_metrics.direct_finalize_ms_total, + "ready_batch_direct_finalize_ms_max": ready_batch_metrics.direct_finalize_ms_max, + "ca_queue_len": ca_queue_len, + "ready_queue_len": ready_queue_len, + "ca_waiting_repo_identities": ca_waiting_repo_identities, + "pending_roa_dispatch_len": pending_roa_dispatch_len, + "inflight_publication_points_len": inflight_publication_points_len, + "pending_finalization_len": pending_finalization_len, + "finalize_inflight": finalize_inflight, + }), + ); +} + +fn compact_phase2_finished_result( + mut result: PublicationPointRunResult, + compact_audit: bool, +) -> FinishedPublicationPointResult { + result.objects.audit.clear(); + result.objects.local_outputs_cache.clear(); + let cir_fresh_objects = if compact_audit && result.source == PublicationPointSource::Fresh { + result.audit.objects.clone() + } else { + result.cir_fresh_objects + }; + if compact_audit { + result.audit.objects.clear(); + result.audit.warnings.clear(); + } + FinishedPublicationPointResult::Ok { + source: result.source, + warnings: result.warnings, + objects: result.objects, + audit: result.audit, + cir_fresh_objects, + cir_cached_objects: result.cir_cached_objects, + } +} + +fn compact_phase2_finished_result_result( + result: Result, + compact_audit: bool, +) -> FinishedPublicationPointResult { + match result { + Ok(result) => compact_phase2_finished_result(result, compact_audit), + Err(err) => FinishedPublicationPointResult::Err(err), + } +} diff --git a/crates/panda-rpki-validator/src/validation/tree_parallel/tests/backpressure.rs b/crates/panda-rpki-validator/src/validation/tree_parallel/tests/backpressure.rs new file mode 100644 index 0000000..da3f821 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_parallel/tests/backpressure.rs @@ -0,0 +1,76 @@ + struct MockStageSubmitter { + capacity: usize, + submitted_ids: Vec, + } + + impl ReadyStageSubmitter for MockStageSubmitter { + fn try_submit_ready_stage( + &mut self, + task: super::ReadyStageTask, + ) -> Result< + (), + crate::parallel::object_worker::ObjectWorkerSubmitError, + > { + if self.submitted_ids.len() < self.capacity { + self.submitted_ids.push(task.ready.node.id); + Ok(()) + } else { + Err( + crate::parallel::object_worker::ObjectWorkerSubmitError::QueueFull { + worker_index: 0, + task, + }, + ) + } + } + } + + #[test] + fn submit_ready_batch_requeues_on_backpressure_without_loss_or_duplication() { + let mut ready_queue = VecDeque::new(); + for id in [10, 11, 12] { + ready_queue.push_back(stage_test_ready(id)); + } + let mut staging_inflight = 0usize; + + // First turn: only two tasks fit, the third must be returned to the + // head of the ready queue. + let mut submitter = MockStageSubmitter { + capacity: 2, + submitted_ids: Vec::new(), + }; + let metrics = submit_ready_batch_to_stage_pool( + &mut submitter, + &mut ready_queue, + &mut staging_inflight, + 256, + Duration::from_secs(60), + ) + .expect("dispatch"); + assert_eq!(metrics.submitted, 2); + assert!(metrics.queue_full); + assert_eq!(staging_inflight, 2); + assert_eq!(submitter.submitted_ids, vec![10, 11]); + assert_eq!(ready_queue.len(), 1); + assert_eq!(ready_queue[0].node.id, 12); + + // Next turn retries the requeued publication point; across both turns + // every id is submitted exactly once. + let mut submitter = MockStageSubmitter { + capacity: 8, + submitted_ids: Vec::new(), + }; + let metrics = submit_ready_batch_to_stage_pool( + &mut submitter, + &mut ready_queue, + &mut staging_inflight, + 256, + Duration::from_secs(60), + ) + .expect("dispatch"); + assert_eq!(metrics.submitted, 1); + assert!(!metrics.queue_full); + assert_eq!(staging_inflight, 3); + assert_eq!(submitter.submitted_ids, vec![12]); + assert!(ready_queue.is_empty()); + } diff --git a/crates/panda-rpki-validator/src/validation/tree_parallel/tests/control_loop.rs b/crates/panda-rpki-validator/src/validation/tree_parallel/tests/control_loop.rs new file mode 100644 index 0000000..d2a9caf --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_parallel/tests/control_loop.rs @@ -0,0 +1,502 @@ + use super::{ + CacheHitOutcome, CompleteOutcome, FinishedPublicationPoint, FinishedPublicationPointNode, + FinishedPublicationPointResult, InflightPublicationPoint, QueuedCaInstance, + ReadyCaInstance, ReadyStageMetrics, ReadyStageSubmitter, StageOutcome, + TREE_OUTPUT_MAX_SHARDS, TREE_OUTPUT_MIN_SHARD_LEN, apply_ready_publication_point_stage, + build_tree_output, compact_phase2_finished_result, compact_phase2_finished_result_result, + compute_ready_publication_point_stage, event_poll_timeout, finalize_metrics_from_output, + finalize_publication_point_state, is_complete, merge_shard_reductions, + reduce_finished_shard, submit_ready_batch_to_stage_pool, tree_output_shard_count, + }; + use crate::audit::{ + AuditObjectKind, AuditObjectResult, DiscoveredFrom, ObjectAuditEntry, PublicationPointAudit, + }; + use crate::fetch::rsync::{RsyncFetchError, RsyncFetcher}; + use crate::parallel::repo_runtime::RepoSyncRuntimeOutcome; + use crate::policy::{CaFailedFetchPolicy, Policy, SyncPreference}; + use crate::storage::{PackTime, RocksStore}; + use crate::sync::rrdp::Fetcher; + use crate::validation::manifest::{ + FreshPublicationPointTimingBreakdown, FreshValidatedPublicationPoint, + PublicationPointSource, + }; + use crate::validation::objects::{ObjectsOutput, ObjectsStats, ParallelObjectsPrepare}; + use crate::validation::publication_point::PublicationPointSnapshot; + use crate::validation::tree::{ + CaCertificateRef, CaInstanceHandle, DiscoveredChildCaInstance, PublicationPointRunResult, + TreeRunConfig, + }; + use crate::validation::tree_runner::{ + BuildVcirTimingBreakdown, FreshPublicationPointFinalizeOutput, FreshPublicationPointStage, + PersistVcirTimingBreakdown, Rpkiv1PublicationPointRunner, + }; + use std::collections::{HashMap, VecDeque}; + use std::sync::Mutex; + use std::time::{Duration, Instant}; + + fn sample_snapshot() -> PublicationPointSnapshot { + PublicationPointSnapshot { + format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, + manifest_rsync_uri: "rsync://example.test/repo/example.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), + manifest_number_be: vec![1], + this_update: PackTime { + rfc3339_utc: "2026-04-21T00:00:00Z".to_string(), + }, + next_update: PackTime { + rfc3339_utc: "2026-04-22T00:00:00Z".to_string(), + }, + verified_at: PackTime { + rfc3339_utc: "2026-04-21T00:00:01Z".to_string(), + }, + manifest_bytes: vec![1, 2, 3], + files: Vec::new(), + } + } + + fn sample_result() -> PublicationPointRunResult { + PublicationPointRunResult { + source: PublicationPointSource::Fresh, + snapshot: Some(sample_snapshot()), + warnings: Vec::new(), + objects: ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: Vec::new(), + stats: ObjectsStats::default(), + audit: Vec::new(), + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + }, + audit: PublicationPointAudit::default(), + cir_fresh_objects: Vec::new(), + cir_cached_objects: Vec::new(), + discovered_children: Vec::new(), + } + } + + #[test] + fn compact_phase2_finished_result_drops_snapshot() { + let result = compact_phase2_finished_result(sample_result(), false); + match result { + FinishedPublicationPointResult::Ok { warnings, .. } => { + assert!(warnings.is_empty()); + } + FinishedPublicationPointResult::Err(err) => panic!("unexpected error: {err}"), + } + } + + #[test] + fn compact_phase2_finished_result_can_drop_audit_payload() { + let mut sample = sample_result(); + sample.audit.objects.push(crate::audit::ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/a.roa".to_string(), + sha256_hex: "11".repeat(32), + kind: crate::audit::AuditObjectKind::Roa, + result: crate::audit::AuditObjectResult::Ok, + detail: None, + }); + sample.audit.warnings.push(crate::audit::AuditWarning { + message: "warning".to_string(), + category: "unclassified".to_string(), + rfc_refs: Vec::new(), + context: None, + }); + sample.objects.audit.push(crate::audit::ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/b.roa".to_string(), + sha256_hex: "22".repeat(32), + kind: crate::audit::AuditObjectKind::Roa, + result: crate::audit::AuditObjectResult::Ok, + detail: None, + }); + let result = compact_phase2_finished_result(sample, true); + match result { + FinishedPublicationPointResult::Ok { + objects, + audit, + cir_fresh_objects, + .. + } => { + assert!(audit.objects.is_empty()); + assert!(audit.warnings.is_empty()); + assert!(objects.audit.is_empty()); + assert_eq!(cir_fresh_objects.len(), 1); + } + FinishedPublicationPointResult::Err(err) => panic!("unexpected error: {err}"), + } + } + + #[test] + fn compact_phase2_finished_result_result_preserves_err() { + match compact_phase2_finished_result_result(Err("boom".to_string()), false) { + FinishedPublicationPointResult::Err(err) => assert_eq!(err, "boom"), + FinishedPublicationPointResult::Ok { .. } => panic!("error should be preserved"), + } + } + + fn finished_ok_item(id: u64) -> FinishedPublicationPoint { + let mut result = sample_result(); + result.warnings.push(crate::report::Warning::new(format!( + "result-warning-{id:05}" + ))); + result + .objects + .warnings + .push(crate::report::Warning::new(format!( + "objects-warning-{id:05}" + ))); + result.objects.vrps.push(crate::validation::objects::Vrp { + asn: 64496 + id as u32, + prefix: crate::data_model::roa::IpPrefix { + afi: crate::data_model::roa::RoaAfi::Ipv4, + addr: [192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + prefix_len: 24, + }, + max_length: 24, + }); + result.audit.objects.push(crate::audit::ObjectAuditEntry { + rsync_uri: format!("rsync://example.test/repo/{id:05}.roa"), + sha256_hex: format!("{id:064x}"), + kind: crate::audit::AuditObjectKind::Roa, + result: crate::audit::AuditObjectResult::Ok, + detail: None, + }); + FinishedPublicationPoint { + node: FinishedPublicationPointNode { + id, + parent_id: None, + discovered_from: None, + manifest_rsync_uri: format!("rsync://example.test/repo/{id:05}.mft"), + }, + result: compact_phase2_finished_result(result, false), + } + } + + fn finished_err_item(id: u64) -> FinishedPublicationPoint { + FinishedPublicationPoint { + node: FinishedPublicationPointNode { + id, + parent_id: None, + discovered_from: None, + manifest_rsync_uri: format!("rsync://example.test/repo/{id:05}.mft"), + }, + result: FinishedPublicationPointResult::Err(format!("boom-{id:05}")), + } + } + + fn mixed_finished_items(n: u64) -> Vec { + (0..n) + .map(|id| { + if id % 97 == 0 { + finished_err_item(id) + } else { + finished_ok_item(id) + } + }) + .collect() + } + + #[test] + fn tree_output_shard_count_respects_thresholds() { + assert_eq!(tree_output_shard_count(0), 1); + assert_eq!( + tree_output_shard_count(TREE_OUTPUT_MIN_SHARD_LEN * 2 - 1), + 1 + ); + assert!(tree_output_shard_count(TREE_OUTPUT_MIN_SHARD_LEN * 2) >= 1); + let parallel = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1); + assert_eq!( + tree_output_shard_count(1_000_000), + parallel.min(TREE_OUTPUT_MAX_SHARDS) + ); + } + + #[test] + fn tree_output_sharded_merge_matches_single_reduction() { + let single = merge_shard_reductions(vec![reduce_finished_shard(mixed_finished_items(300))]); + + let mut chunks: Vec> = Vec::new(); + let mut rest = mixed_finished_items(300); + while !rest.is_empty() { + let split_at = 100.min(rest.len()); + let tail = rest.split_off(split_at); + chunks.push(std::mem::replace(&mut rest, tail)); + } + let merged = + merge_shard_reductions(chunks.into_iter().map(reduce_finished_shard).collect()); + + assert_eq!(single.instances_processed, merged.instances_processed); + assert_eq!(single.instances_failed, merged.instances_failed); + assert_eq!( + format!("{:?}", single.warnings), + format!("{:?}", merged.warnings) + ); + assert_eq!(format!("{:?}", single.vrps), format!("{:?}", merged.vrps)); + assert_eq!( + format!("{:?}", single.publication_points), + format!("{:?}", merged.publication_points) + ); + assert_eq!( + format!("{:?}", single.roa_cache_stats), + format!("{:?}", merged.roa_cache_stats) + ); + assert_eq!(single.cir_input.finalize(), merged.cir_input.finalize()); + } + + #[test] + fn build_tree_output_orders_results_by_node_id() { + let mut items = mixed_finished_items(50); + items.reverse(); + let output = build_tree_output(items); + let got: Vec = output + .publication_points + .iter() + .map(|pp| pp.node_id.expect("node id set")) + .collect(); + let want: Vec = (1..50).filter(|id| id % 97 != 0).collect(); + assert_eq!(got, want); + assert_eq!(output.tree.instances_failed, 1); + let first_warning = format!("{:?}", output.tree.warnings[0]); + assert!( + first_warning.contains("publication point failed: boom-00000"), + "failed publication point warning keeps id order: {first_warning}" + ); + assert_eq!(output.cir_input.fresh_validated_objects.len(), 49); + assert_eq!( + output.cir_input.fresh_validated_objects[0].rsync_uri, + "rsync://example.test/repo/00001.roa" + ); + } + + #[test] + fn finalize_metrics_from_output_captures_breakdown_and_counts() { + let mut result = sample_result(); + result.objects.vrps.push(crate::validation::objects::Vrp { + asn: 64496, + prefix: crate::data_model::roa::IpPrefix { + afi: crate::data_model::roa::RoaAfi::Ipv4, + addr: [192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + prefix_len: 24, + }, + max_length: 24, + }); + result + .objects + .aspas + .push(crate::validation::objects::AspaAttestation { + customer_as_id: 64497, + provider_as_ids: vec![64498], + }); + result.audit.objects.push(ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/a.roa".to_string(), + sha256_hex: "11".repeat(32), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Ok, + detail: None, + }); + result + .discovered_children + .push(crate::validation::tree::DiscoveredChildCaInstance { + handle: crate::validation::tree::CaInstanceHandle { + tal_id: "test".to_string(), + ca_certificate: crate::validation::tree::CaCertificateRef::inline_der(vec![1]), + ca_certificate_rsync_uri: Some( + "rsync://example.test/repo/child.cer".to_string(), + ), + effective_ip_resources: None, + effective_as_resources: None, + manifest_rsync_uri: "rsync://example.test/repo/child.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), + rsync_base_uri: "rsync://example.test/repo/".to_string(), + rrdp_notification_uri: None, + parent_manifest_rsync_uri: None, + depth: 1, + }, + discovered_from: crate::audit::DiscoveredFrom { + parent_manifest_rsync_uri: "rsync://example.test/repo/example.mft".to_string(), + child_ca_certificate_rsync_uri: "rsync://example.test/repo/child.cer" + .to_string(), + child_ca_certificate_sha256_hex: "55".repeat(32), + }, + child_entry_projection: None, + }); + let output = FreshPublicationPointFinalizeOutput { + result, + snapshot_pack_ms: 1, + persist_vcir_ms: 2, + persist_vcir_timing: PersistVcirTimingBreakdown { + build_vcir_ms: 3, + replace_vcir_ms: 4, + build_vcir: BuildVcirTimingBreakdown { + related_artifacts_ms: 5, + ..BuildVcirTimingBreakdown::default() + }, + ..PersistVcirTimingBreakdown::default() + }, + ccr_projection_build_ms: 6, + ccr_append_ms: 7, + audit_build_ms: 8, + }; + + let metrics = finalize_metrics_from_output(&output, 9, 10, Some(11), 12, 13); + + assert_eq!(metrics.snapshot_pack_ms, 1); + assert_eq!(metrics.persist_vcir_ms, 2); + assert_eq!(metrics.persist_build_vcir_ms, 3); + assert_eq!(metrics.persist_replace_vcir_ms, 4); + assert_eq!( + metrics.persist_replace_breakdown, + crate::storage::VcirReplaceTimingBreakdown::default() + ); + assert_eq!(metrics.ccr_projection_build_ms, 6); + assert_eq!(metrics.ccr_append_ms, 7); + assert_eq!(metrics.audit_build_ms, 8); + assert_eq!(metrics.reduce_ms, 9); + assert_eq!(metrics.finalize_ms, 10); + assert_eq!(metrics.finalize_queue_wait_ms, Some(11)); + assert_eq!(metrics.finalize_worker_ms, 12); + assert_eq!(metrics.locked_files, 13); + assert_eq!(metrics.child_count, 1); + assert_eq!(metrics.vrp_count, 1); + assert_eq!(metrics.vap_count, 1); + assert_eq!(metrics.audit_object_count, 1); + } + + struct NeverHttpFetcher; + impl Fetcher for NeverHttpFetcher { + fn fetch(&self, _uri: &str) -> Result, String> { + Err("http fetch disabled in test".to_string()) + } + } + + struct FailingRsyncFetcher; + impl RsyncFetcher for FailingRsyncFetcher { + fn fetch_objects( + &self, + _rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + Err(RsyncFetchError::Fetch("rsync disabled in test".to_string())) + } + } + + fn stage_test_runner<'a>( + store: &'a RocksStore, + policy: &'a Policy, + ) -> Rpkiv1PublicationPointRunner<'a> { + Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &FailingRsyncFetcher, + validation_time: time::OffsetDateTime::now_utc(), + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + } + } + + fn stage_test_handle(manifest_rsync_uri: &str) -> CaInstanceHandle { + CaInstanceHandle { + tal_id: "test".to_string(), + ca_certificate: CaCertificateRef::inline_der(vec![1]), + ca_certificate_rsync_uri: Some("rsync://example.test/repo/ca.cer".to_string()), + effective_ip_resources: None, + effective_as_resources: None, + manifest_rsync_uri: manifest_rsync_uri.to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), + rsync_base_uri: "rsync://example.test/repo/".to_string(), + rrdp_notification_uri: None, + parent_manifest_rsync_uri: None, + depth: 0, + } + } + + fn stage_test_ready(id: u64) -> ReadyCaInstance { + ReadyCaInstance { + node: QueuedCaInstance { + id, + handle: stage_test_handle("rsync://example.test/repo/example.mft"), + parent_id: None, + discovered_from: None, + }, + repo_outcome: stage_test_repo_outcome(), + ready_enqueued_at: Instant::now(), + } + } + + fn stage_test_repo_outcome() -> RepoSyncRuntimeOutcome { + RepoSyncRuntimeOutcome { + repo_sync_ok: true, + repo_sync_err: None, + repo_sync_source: Some("rsync".to_string()), + repo_sync_phase: Some("rsync".to_string()), + repo_sync_duration_ms: 0, + warnings: Vec::new(), + } + } + + fn stage_test_child( + manifest_rsync_uri: &str, + ca_certificate_rsync_uri: &str, + ) -> DiscoveredChildCaInstance { + DiscoveredChildCaInstance { + handle: CaInstanceHandle { + ca_certificate_rsync_uri: Some(ca_certificate_rsync_uri.to_string()), + ..stage_test_handle(manifest_rsync_uri) + }, + discovered_from: DiscoveredFrom { + parent_manifest_rsync_uri: "rsync://example.test/repo/example.mft".to_string(), + child_ca_certificate_rsync_uri: ca_certificate_rsync_uri.to_string(), + child_ca_certificate_sha256_hex: "55".repeat(32), + }, + child_entry_projection: None, + } + } + + fn stage_test_fresh_stage() -> FreshPublicationPointStage { + FreshPublicationPointStage { + fresh_point: FreshValidatedPublicationPoint { + manifest_rsync_uri: "rsync://example.test/repo/example.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), + manifest_number_be: vec![1], + this_update: PackTime { + rfc3339_utc: "2026-04-21T00:00:00Z".to_string(), + }, + next_update: PackTime { + rfc3339_utc: "2026-04-22T00:00:00Z".to_string(), + }, + verified_at: PackTime { + rfc3339_utc: "2026-04-21T00:00:01Z".to_string(), + }, + manifest_bytes: vec![1, 2, 3], + files: Vec::new(), + }, + issuer_ca_der: vec![1u8].into(), + snapshot_prepare_timing: FreshPublicationPointTimingBreakdown::default(), + snapshot_prepare_ms: 0, + discovered_children: Vec::new(), + child_audits: Vec::new(), + discovered_router_keys: Vec::new(), + child_discovery_ms: 0, + warnings: Vec::new(), + } + } diff --git a/crates/panda-rpki-validator/src/validation/tree_parallel/tests/stage.rs b/crates/panda-rpki-validator/src/validation/tree_parallel/tests/stage.rs new file mode 100644 index 0000000..3ceb242 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_parallel/tests/stage.rs @@ -0,0 +1,346 @@ + + fn stage_test_metrics() -> ReadyStageMetrics { + ReadyStageMetrics { + ready_count: 1, + manifest_rsync_uri: Some("rsync://example.test/repo/example.mft".to_string()), + publication_point_rsync_uri: Some("rsync://example.test/repo/".to_string()), + ..ReadyStageMetrics::default() + } + } + + #[test] + fn compute_apply_fresh_error_runs_inline_fallback_and_finishes_err() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy { + sync_preference: SyncPreference::RsyncOnly, + ca_failed_fetch_policy: CaFailedFetchPolicy::StopAllOutput, + ..Policy::default() + }; + let runner = stage_test_runner(&store, &policy); + let config = TreeRunConfig::default(); + let mut next_id = 1u64; + let mut ca_queue = VecDeque::new(); + let mut pending_roa_dispatch = VecDeque::new(); + let mut inflight_publication_points = HashMap::new(); + let mut pending_finalization = VecDeque::new(); + let mut finished = Vec::new(); + + let (outcome, metrics) = + compute_ready_publication_point_stage(&runner, stage_test_ready(0), 0); + assert!( + matches!(outcome, StageOutcome::FreshError(_)), + "fresh staging against an empty store must fail" + ); + let metrics = apply_ready_publication_point_stage( + &runner, + &mut next_id, + &mut ca_queue, + &mut pending_roa_dispatch, + &mut inflight_publication_points, + &mut pending_finalization, + &mut finished, + outcome, + metrics, + &config, + false, + ); + + assert_eq!(metrics.ready_count, 1); + assert_eq!(metrics.fallback_count, 1); + assert_eq!(metrics.complete_count, 0); + assert_eq!(metrics.staged_count, 0); + assert_eq!(metrics.zero_task_count, 0); + assert_eq!(finished.len(), 1); + match &finished[0].result { + FinishedPublicationPointResult::Err(_) => {} + FinishedPublicationPointResult::Ok { .. } => { + panic!("fallback without repository data must fail") + } + } + assert!(ca_queue.is_empty()); + assert!(pending_roa_dispatch.is_empty()); + assert!(pending_finalization.is_empty()); + assert!(inflight_publication_points.is_empty()); + assert_eq!(next_id, 1); + } + + #[test] + fn apply_cache_hit_enqueues_children_sorted_and_finishes() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let runner = stage_test_runner(&store, &policy); + let config = TreeRunConfig::default(); + let mut next_id = 42u64; + let mut ca_queue = VecDeque::new(); + let mut pending_roa_dispatch = VecDeque::new(); + let mut inflight_publication_points = HashMap::new(); + let mut pending_finalization = VecDeque::new(); + let mut finished = Vec::new(); + + let mut result = sample_result(); + result.discovered_children = vec![ + stage_test_child( + "rsync://example.test/repo/z.mft", + "rsync://example.test/repo/z.cer", + ), + stage_test_child( + "rsync://example.test/repo/a.mft", + "rsync://example.test/repo/a.cer", + ), + ]; + let outcome = StageOutcome::CacheHit(Box::new(CacheHitOutcome { + ready: stage_test_ready(7), + publication_point_started: Instant::now(), + result, + })); + let mut metrics = stage_test_metrics(); + metrics.complete_count = 1; + metrics.discovered_children = 2; + let metrics = apply_ready_publication_point_stage( + &runner, + &mut next_id, + &mut ca_queue, + &mut pending_roa_dispatch, + &mut inflight_publication_points, + &mut pending_finalization, + &mut finished, + outcome, + metrics, + &config, + false, + ); + + assert_eq!(finished.len(), 1); + match &finished[0].result { + FinishedPublicationPointResult::Ok { .. } => {} + FinishedPublicationPointResult::Err(err) => { + panic!("cache hit must finish ok: {err}") + } + } + assert_eq!(finished[0].node.id, 7); + // Children are enqueued sorted by manifest URI with sequential ids + // taken from next_id, exactly like the monolithic staging did. + assert_eq!(ca_queue.len(), 2); + assert_eq!( + ca_queue[0].handle.manifest_rsync_uri, + "rsync://example.test/repo/a.mft" + ); + assert_eq!( + ca_queue[1].handle.manifest_rsync_uri, + "rsync://example.test/repo/z.mft" + ); + assert_eq!(ca_queue[0].id, 42); + assert_eq!(ca_queue[1].id, 43); + assert_eq!(ca_queue[0].parent_id, Some(7)); + assert_eq!(ca_queue[1].parent_id, Some(7)); + assert_eq!(next_id, 44); + assert!(pending_roa_dispatch.is_empty()); + assert!(pending_finalization.is_empty()); + assert!(inflight_publication_points.is_empty()); + assert_eq!(metrics.complete_count, 1); + assert_eq!(metrics.discovered_children, 2); + } + + #[test] + fn apply_complete_enqueues_finalize_task_instead_of_finishing() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let runner = stage_test_runner(&store, &policy); + let config = TreeRunConfig::default(); + let mut next_id = 1u64; + let mut ca_queue = VecDeque::new(); + let mut pending_roa_dispatch = VecDeque::new(); + let mut inflight_publication_points = HashMap::new(); + let mut pending_finalization = VecDeque::new(); + let mut finished = Vec::new(); + + let outcome = StageOutcome::Complete(Box::new(CompleteOutcome { + ready: stage_test_ready(9), + publication_point_started: Instant::now(), + fresh_stage: stage_test_fresh_stage(), + warnings: Vec::new(), + objects: sample_result().objects, + })); + let mut metrics = stage_test_metrics(); + metrics.complete_count = 1; + let metrics = apply_ready_publication_point_stage( + &runner, + &mut next_id, + &mut ca_queue, + &mut pending_roa_dispatch, + &mut inflight_publication_points, + &mut pending_finalization, + &mut finished, + outcome, + metrics, + &config, + false, + ); + + assert!( + finished.is_empty(), + "complete staging must defer the finalize to the worker queue" + ); + assert_eq!(pending_finalization.len(), 1); + let task = pending_finalization.pop_front().expect("finalize task"); + let state = task.state; + assert_eq!(state.node.id, 9); + assert_eq!(state.task_count, 0); + assert!(state.finalize_enqueued_at.is_some()); + assert!(state.results.is_empty()); + assert!(matches!( + state.objects_prepare, + ParallelObjectsPrepare::Complete(_) + )); + assert!(inflight_publication_points.is_empty()); + assert!(pending_roa_dispatch.is_empty()); + assert!(ca_queue.is_empty()); + assert_eq!(next_id, 1); + assert_eq!(metrics.complete_count, 1); + } + + #[test] + fn finalize_worker_complete_arm_finalizes_without_reduce() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let mut runner = stage_test_runner(&store, &policy); + runner.persist_vcir = false; + + let state = InflightPublicationPoint { + node: QueuedCaInstance { + id: 3, + handle: stage_test_handle("rsync://example.test/repo/example.mft"), + parent_id: None, + discovered_from: None, + }, + fresh_stage: stage_test_fresh_stage(), + objects_prepare: ParallelObjectsPrepare::Complete(sample_result().objects), + repo_outcome: stage_test_repo_outcome(), + warnings: Vec::new(), + started_at: Instant::now(), + objects_started_at: Instant::now(), + task_count: 0, + tasks_submitted: 0, + first_task_submitted_at: None, + last_task_submitted_at: None, + first_result_at: None, + last_result_at: None, + worker_ms_total: 0, + worker_ms_max: 0, + queue_wait_ms_total: 0, + queue_wait_ms_max: 0, + finalize_enqueued_at: Some(Instant::now()), + results: Vec::new(), + }; + let result = finalize_publication_point_state(&runner, state, false); + + match result.finished.result { + FinishedPublicationPointResult::Ok { source, .. } => { + assert_eq!(source, PublicationPointSource::Fresh); + } + FinishedPublicationPointResult::Err(err) => { + panic!("complete finalize must succeed: {err}") + } + } + assert_eq!(result.finished.node.id, 3); + assert_eq!(result.metrics.reduce_ms, 0); + assert_eq!(result.metrics.locked_files, 0); + } + + #[test] + fn is_complete_waits_for_staging_inflight() { + let ca_queue = VecDeque::new(); + let ready_queue = VecDeque::new(); + let ca_waiting_repo_by_identity = HashMap::new(); + let pending_roa_dispatch = VecDeque::new(); + let inflight_publication_points = HashMap::new(); + let pending_finalization = VecDeque::new(); + let config = TreeRunConfig::default(); + + // Everything drained and no staging in flight: the loop may exit. + assert!(is_complete( + &ca_queue, + &ready_queue, + &ca_waiting_repo_by_identity, + &pending_roa_dispatch, + &inflight_publication_points, + &pending_finalization, + 0, + 0, + 0, + &config, + )); + // A submitted stage task whose result has not been collected yet must + // keep the loop alive, otherwise its publication point would be lost. + assert!(!is_complete( + &ca_queue, + &ready_queue, + &ca_waiting_repo_by_identity, + &pending_roa_dispatch, + &inflight_publication_points, + &pending_finalization, + 0, + 1, + 0, + &config, + )); + } + + #[test] + fn event_poll_timeout_stays_awake_while_staging() { + let ca_queue = VecDeque::new(); + let ready_queue = VecDeque::new(); + let pending_roa_dispatch = VecDeque::new(); + let inflight_publication_points = HashMap::new(); + let pending_finalization = VecDeque::new(); + let config = TreeRunConfig::default(); + + // Staging in flight: poll must not sleep, or stage results pile up. + assert_eq!( + event_poll_timeout( + &ca_queue, + &ready_queue, + &pending_roa_dispatch, + &inflight_publication_points, + &pending_finalization, + 0, + 1, + 0, + &config, + ), + Duration::from_millis(0) + ); + // Without staging the historical tiers are unchanged. + assert_eq!( + event_poll_timeout( + &ca_queue, + &ready_queue, + &pending_roa_dispatch, + &inflight_publication_points, + &pending_finalization, + 1, + 0, + 0, + &config, + ), + Duration::from_millis(10) + ); + assert_eq!( + event_poll_timeout( + &ca_queue, + &ready_queue, + &pending_roa_dispatch, + &inflight_publication_points, + &pending_finalization, + 0, + 0, + 0, + &config, + ), + Duration::from_millis(50) + ); + } diff --git a/crates/panda-rpki-validator/src/validation/tree_runner.rs b/crates/panda-rpki-validator/src/validation/tree_runner.rs index ef2d16c..03f4a1d 100644 --- a/crates/panda-rpki-validator/src/validation/tree_runner.rs +++ b/crates/panda-rpki-validator/src/validation/tree_runner.rs @@ -1,3 +1,5 @@ +mod labels; + use crate::analysis::timing::TimingHandle; use crate::audit::{ AuditObjectKind, AuditObjectResult, AuditWarning, ObjectAuditEntry, PublicationPointAudit, @@ -60,6 +62,11 @@ use crate::validation::tree::{ CaCertificateRef, CaInstanceHandle, DiscoveredChildCaInstance, DiscoveredChildEntryProjection, PublicationPointRunResult, PublicationPointRunner, }; +use labels::{ + audit_result_from_vcir_status, effective_repo_sync_duration_ms, kind_from_rsync_uri, + kind_from_vcir_artifact_kind, repo_sync_failure_phase_label, repo_sync_phase_label, + repo_sync_source_label, source_label, terminal_state_label, +}; use sha2::Digest; use std::collections::{HashMap, HashSet}; use std::sync::{Arc, Mutex}; @@ -70,6125 +77,19 @@ use x509_parser::x509::SubjectPublicKeyInfo; use crate::ccr::manifest_location::select_manifest_signed_object_location; -const PUBLICATION_POINT_CACHE_CHILD_RESTORE_PARALLEL_MIN_CHILDREN: usize = 256; -const PUBLICATION_POINT_CACHE_CHILD_RESTORE_MAX_WORKERS: usize = 16; -const CHILD_CERTIFICATE_CACHE_MMAP_MIN_CER_COUNT: usize = 2048; - -fn sha256_hex_to_32(hex_value: &str) -> [u8; 32] { - let mut out = [0u8; 32]; - hex::decode_to_slice(hex_value, &mut out).expect("internal sha256 hex should decode"); - out -} - -#[derive(Clone, Debug, Default)] -pub(crate) struct BuildVcirTimingBreakdown { - pub(crate) select_crl_ms: u64, - pub(crate) current_ca_decode_ms: u64, - pub(crate) local_outputs_ms: u64, - pub(crate) child_entries_ms: u64, - pub(crate) related_artifacts_ms: u64, - pub(crate) struct_build_ms: u64, -} - -#[derive(Clone, Debug, Default)] -pub(crate) struct PersistVcirTimingBreakdown { - pub(crate) embedded_collect_ms: u64, - pub(crate) embedded_store_ms: u64, - pub(crate) build_vcir_ms: u64, - pub(crate) replace_vcir_ms: u64, - pub(crate) publication_point_cache_future_notbefore_guarded: bool, - pub(crate) build_vcir: BuildVcirTimingBreakdown, - pub(crate) replace_vcir: VcirReplaceTimingBreakdown, -} - -#[derive(Clone, Debug)] -pub(crate) struct FreshPublicationPointStage { - pub(crate) fresh_point: FreshValidatedPublicationPoint, - pub(crate) issuer_ca_der: Arc<[u8]>, - pub(crate) snapshot_prepare_timing: FreshPublicationPointTimingBreakdown, - pub(crate) snapshot_prepare_ms: u64, - pub(crate) discovered_children: Vec, - pub(crate) child_audits: Vec, - pub(crate) discovered_router_keys: Vec, - pub(crate) child_discovery_ms: u64, - pub(crate) warnings: Vec, -} - -#[derive(Debug)] -pub(crate) struct FreshPublicationPointStageError { - pub(crate) error: ManifestFreshError, - pub(crate) snapshot_prepare_ms: u64, -} - -#[derive(Clone, Debug)] -pub(crate) struct FreshPublicationPointFinalizeOutput { - pub(crate) result: PublicationPointRunResult, - pub(crate) snapshot_pack_ms: u64, - pub(crate) persist_vcir_ms: u64, - pub(crate) persist_vcir_timing: PersistVcirTimingBreakdown, - pub(crate) ccr_projection_build_ms: u64, - pub(crate) ccr_append_ms: u64, - pub(crate) audit_build_ms: u64, -} - -pub struct Rpkiv1PublicationPointRunner<'a> { - pub store: &'a RocksStore, - pub policy: &'a Policy, - pub http_fetcher: &'a dyn Fetcher, - pub rsync_fetcher: &'a dyn RsyncFetcher, - pub validation_time: time::OffsetDateTime, - pub timing: Option, - pub download_log: Option, - pub replay_archive_index: Option>, - pub replay_delta_index: Option>, - /// In-run RRDP dedup: when RRDP is enabled, only sync each `rrdp_notification_uri` once per run. - /// - /// - If RRDP succeeded for a repo, later publication points referencing that same RRDP repo - /// skip network fetches and reuse the already-populated current repository view. - /// - If RRDP failed for a repo, later publication points skip RRDP attempts and go straight - /// to rsync for their own `rsync_base_uri` (still per-publication-point). - pub rrdp_dedup: bool, - pub rrdp_repo_cache: Mutex>, // notification_uri -> rrdp_ok - - /// In-run rsync dedup: when rsync is used, only sync each `rsync_base_uri` once per run. - /// - /// This reduces duplicate rsync network fetches when multiple publication points share the - /// same `rsync_base_uri` (observed in APNIC full sync timing reports). - pub rsync_dedup: bool, - pub rsync_repo_cache: Mutex>, // rsync_base_uri -> rsync_ok - pub current_repo_index: Option, - pub repo_sync_runtime: Option>, - pub parallel_phase2_config: Option, - pub parallel_roa_worker_pool: Option, - pub ccr_accumulator: Option>, - /// When false, skip VCIR persistence and per-output VCIR projection building. - /// - /// This is intended for replay/compare-only runs where the caller does not need - /// the resulting DB to be reused by a later delta run. - pub persist_vcir: bool, - pub enable_roa_validation_cache: bool, - pub enable_child_certificate_validation_cache: bool, - pub publication_point_cache_observe_only: bool, - pub enable_publication_point_validation_cache: bool, -} - -impl<'a> Rpkiv1PublicationPointRunner<'a> { - pub(crate) fn roa_validation_cache_view_for_fresh_point( - &self, - manifest_rsync_uri: &str, - ) -> Option { - if !self.enable_roa_validation_cache { - return None; - } - let load_started = std::time::Instant::now(); - let loaded_projection = self.store.get_roa_cache_projection(manifest_rsync_uri); - if let Some(timing) = self.timing.as_ref() { - timing.record_phase_nanos( - "roa_validation_cache_projection_load_total", - load_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64, - ); - } - match loaded_projection { - Ok(Some(projection)) => { - if let Some(timing) = self.timing.as_ref() { - timing - .record_count("roa_validation_cache_projection_hit_publication_points", 1); - } - let view_started = std::time::Instant::now(); - let view = - RoaValidationCacheView::from_projection(&projection, self.validation_time); - if let Some(timing) = self.timing.as_ref() { - timing.record_phase_nanos( - "roa_validation_cache_projection_build_total", - view_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64, - ); - } - Some(view) - } - Ok(None) => { - if let Some(timing) = self.timing.as_ref() { - timing.record_count( - "roa_validation_cache_projection_missing_publication_points", - 1, - ); - } - None - } - Err(err) => { - if let Some(timing) = self.timing.as_ref() { - timing.record_count("roa_validation_cache_projection_load_errors", 1); - } - crate::progress_log::emit( - "roa_validation_cache_projection_load_error", - serde_json::json!({ - "manifest_rsync_uri": manifest_rsync_uri, - "error": err.to_string(), - }), - ); - None - } - } - } - - pub(crate) fn observe_or_reuse_publication_point_cache( - &self, - ca: &CaInstanceHandle, - repo_sync_source: Option<&str>, - repo_sync_phase: Option<&str>, - repo_sync_duration_ms: u64, - repo_sync_err: Option<&str>, - warnings: &[Warning], - ) -> Option { - if !self.publication_point_cache_observe_only - && !self.enable_publication_point_validation_cache - { - return None; - } - - let lookup_started = std::time::Instant::now(); - if let Some(timing) = self.timing.as_ref() { - timing.record_count("publication_point_cache_lookup_total", 1); - } - let projection = match self - .store - .get_publication_point_cache_projection_cached(&ca.manifest_rsync_uri) - { - Ok(Some(projection)) => projection, - Ok(None) => { - self.finish_publication_point_cache_miss(ca, "missing_projection", lookup_started); - return None; - } - Err(e) => { - self.finish_publication_point_cache_miss( - ca, - "projection_load_error", - lookup_started, - ); - crate::progress_log::emit( - "publication_point_cache_lookup_error", - serde_json::json!({ - "manifest_rsync_uri": ca.manifest_rsync_uri, - "error": e.to_string(), - }), - ); - return None; - } - }; - - let current_identity = match self.current_publication_point_cache_identity(ca) { - Ok(identity) => identity, - Err(reason) => { - self.finish_publication_point_cache_miss(ca, reason.as_str(), lookup_started); - return None; - } - }; - - if projection.ca_cert_uri != ca.ca_certificate_rsync_uri { - self.finish_publication_point_cache_miss(ca, "ca_uri_mismatch", lookup_started); - return None; - } - if projection.ca_cert_sha256 != current_identity.ca_cert_sha256 { - self.finish_publication_point_cache_miss(ca, "ca_hash_mismatch", lookup_started); - return None; - } - if projection.manifest_sha256 != current_identity.manifest_sha256 { - self.finish_publication_point_cache_miss(ca, "manifest_hash_mismatch", lookup_started); - return None; - } - if projection.tal_id != ca.tal_id { - self.finish_publication_point_cache_miss(ca, "tal_mismatch", lookup_started); - return None; - } - if projection.ta_context_digest != current_identity.ta_context_digest { - self.finish_publication_point_cache_miss(ca, "ta_context_mismatch", lookup_started); - return None; - } - if projection.ca_validation_context_digest != current_identity.ca_validation_context_digest - { - self.finish_publication_point_cache_miss(ca, "parent_context_mismatch", lookup_started); - return None; - } - if projection.validation_policy_fingerprint != current_identity.policy_fingerprint { - self.finish_publication_point_cache_miss(ca, "policy_mismatch", lookup_started); - return None; - } - - let instance_not_before = - match parse_snapshot_time_value(&projection.instance_effective_not_before) { - Ok(value) => value, - Err(_) => { - self.finish_publication_point_cache_miss( - ca, - "instance_not_before_invalid", - lookup_started, - ); - return None; - } - }; - let instance_until = match parse_snapshot_time_value(&projection.instance_effective_until) { - Ok(value) => value, - Err(_) => { - self.finish_publication_point_cache_miss( - ca, - "instance_until_invalid", - lookup_started, - ); - return None; - } - }; - if self.validation_time < instance_not_before || self.validation_time >= instance_until { - self.finish_publication_point_cache_miss(ca, "instance_time_gate_miss", lookup_started); - return None; - } - if let Err(reason) = - publication_point_cache_projection_items_valid(&projection, self.validation_time) - { - self.finish_publication_point_cache_miss(ca, reason, lookup_started); - return None; - } - - if let Some(timing) = self.timing.as_ref() { - let lookup_nanos = lookup_started - .elapsed() - .as_nanos() - .min(u128::from(u64::MAX)) as u64; - timing.record_count("publication_point_cache_theoretical_hits", 1); - timing.record_phase_nanos("publication_point_cache_lookup_total", lookup_nanos); - timing.record_phase_nanos("publication_point_cache_lookup_hit_total", lookup_nanos); - timing.record_phase_nanos( - "publication_point_cache_lookup_duration_total", - lookup_nanos, - ); - } - if self.publication_point_cache_observe_only { - return None; - } - - match self.build_publication_point_cache_result( - ca, - projection, - repo_sync_source, - repo_sync_phase, - repo_sync_duration_ms, - repo_sync_err, - warnings, - ) { - Ok(result) => { - if let Some(timing) = self.timing.as_ref() { - timing.record_count("publication_point_cache_reuse_hits", 1); - } - Some(result) - } - Err(e) => { - self.record_publication_point_cache_miss("reuse_build_error"); - crate::progress_log::emit( - "publication_point_cache_reuse_error", - serde_json::json!({ - "manifest_rsync_uri": ca.manifest_rsync_uri, - "error": e, - }), - ); - None - } - } - } - - fn finish_publication_point_cache_miss( - &self, - ca: &CaInstanceHandle, - reason: &str, - lookup_started: std::time::Instant, - ) { - self.record_publication_point_cache_miss(reason); - let lookup_nanos = lookup_started - .elapsed() - .as_nanos() - .min(u128::from(u64::MAX)) as u64; - if let Some(timing) = self.timing.as_ref() { - timing.record_phase_nanos("publication_point_cache_lookup_miss_total", lookup_nanos); - timing.record_phase_nanos( - "publication_point_cache_lookup_duration_total", - lookup_nanos, - ); - } - let elapsed_ms = lookup_nanos / 1_000_000; - if elapsed_ms >= crate::progress_log::pp_cache_slow_threshold_ms() { - crate::progress_log::emit( - "publication_point_cache_miss_slow", - serde_json::json!({ - "manifest_rsync_uri": ca.manifest_rsync_uri.as_str(), - "publication_point_rsync_uri": ca.publication_point_rsync_uri.as_str(), - "ca_certificate_rsync_uri": ca.ca_certificate_rsync_uri.as_deref(), - "reason": reason, - "elapsed_ms": elapsed_ms, - "slow_threshold_ms": crate::progress_log::pp_cache_slow_threshold_ms(), - }), - ); - } - } - - fn record_publication_point_cache_miss(&self, reason: &str) { - if let Some(timing) = self.timing.as_ref() { - timing.record_count("publication_point_cache_miss_total", 1); - match reason { - "missing_projection" => { - timing.record_count("publication_point_cache_miss_missing_projection", 1) - } - "projection_load_error" => { - timing.record_count("publication_point_cache_miss_projection_load_error", 1) - } - "current_manifest_missing" => { - timing.record_count("publication_point_cache_miss_current_manifest_missing", 1) - } - "ca_uri_mismatch" => { - timing.record_count("publication_point_cache_miss_ca_uri_mismatch", 1) - } - "ca_hash_mismatch" => { - timing.record_count("publication_point_cache_miss_ca_hash_mismatch", 1) - } - "manifest_hash_mismatch" => { - timing.record_count("publication_point_cache_miss_manifest_hash_mismatch", 1) - } - "tal_mismatch" => { - timing.record_count("publication_point_cache_miss_tal_mismatch", 1) - } - "ta_context_mismatch" => { - timing.record_count("publication_point_cache_miss_ta_context_mismatch", 1) - } - "parent_context_mismatch" => { - timing.record_count("publication_point_cache_miss_parent_context_mismatch", 1) - } - "policy_mismatch" => { - timing.record_count("publication_point_cache_miss_policy_mismatch", 1) - } - "instance_not_before_invalid" => timing.record_count( - "publication_point_cache_miss_instance_not_before_invalid", - 1, - ), - "instance_until_invalid" => { - timing.record_count("publication_point_cache_miss_instance_until_invalid", 1) - } - "instance_time_gate_miss" => { - timing.record_count("publication_point_cache_miss_instance_time_gate", 1) - } - "output_time_gate_miss" => { - timing.record_count("publication_point_cache_miss_output_time_gate", 1) - } - "child_time_gate_miss" => { - timing.record_count("publication_point_cache_miss_child_time_gate", 1) - } - "reuse_build_error" => { - timing.record_count("publication_point_cache_miss_reuse_build_error", 1) - } - _ => timing.record_count("publication_point_cache_miss_other", 1), - } - } - } - - fn current_publication_point_cache_identity( - &self, - ca: &CaInstanceHandle, - ) -> Result { - let ca_cert_sha256 = match ca.ca_certificate_rsync_uri.as_deref() { - Some(uri) => self - .current_hash_for_uri(uri) - .or_else(|| ca.ca_certificate_sha256_32()) - .ok_or_else(|| "current_ca_certificate_hash_missing".to_string())?, - None => ca - .ca_certificate_sha256_32() - .ok_or_else(|| "current_ca_certificate_hash_missing".to_string())?, - }; - let manifest_sha256 = self - .current_hash_for_uri(&ca.manifest_rsync_uri) - .ok_or_else(|| "current_manifest_missing".to_string())?; - Ok(PublicationPointCacheIdentity { - ca_cert_sha256, - manifest_sha256, - ta_context_digest: ta_context_digest_for_ca(ca), - ca_validation_context_digest: ca_validation_context_digest_for_ca(ca), - policy_fingerprint: publication_point_cache_policy_fingerprint(self.policy), - }) - } - - fn current_hash_for_uri(&self, uri: &str) -> Option<[u8; 32]> { - if let Some(index) = self.current_repo_index.as_ref() { - if let Ok(index) = index.read() { - if let Some(entry) = index.get_by_uri(uri) { - return Some(entry.current_hash); - } - } - } - self.store - .load_current_object_with_hash_by_uri(uri) - .ok() - .flatten() - .map(|entry| entry.current_hash) - } - - fn build_publication_point_cache_result( - &self, - ca: &CaInstanceHandle, - projection: PublicationPointCacheProjection, - repo_sync_source: Option<&str>, - repo_sync_phase: Option<&str>, - repo_sync_duration_ms: u64, - repo_sync_err: Option<&str>, - warnings: &[Warning], - ) -> Result { - let build_started = std::time::Instant::now(); - let mut warnings = warnings.to_vec(); - let output_reuse_count = projection.outputs.len() as u64; - let child_reuse_count = projection.children.len() as u64; - let related_object_reuse_count = projection.related_objects.len() as u64; - let build_objects_started = std::time::Instant::now(); - let mut objects = build_objects_output_from_publication_point_cache_projection( - &projection, - self.validation_time, - &mut warnings, - ); - let build_objects_ms = self.record_publication_point_cache_phase_ms( - "publication_point_cache_build_objects_total", - build_objects_started, - ); - self.record_publication_point_step_ms( - &ca.manifest_rsync_uri, - "publication_point_cache_build_objects", - build_objects_ms, - ); - let restore_children_started = std::time::Instant::now(); - let child_restore_workers = self.publication_point_cache_child_restore_worker_count(); - let (discovered_children, child_audits) = restore_children_from_publication_point_cache( - self.store, - ca, - &projection, - self.validation_time, - &mut warnings, - child_restore_workers, - self.timing.as_ref(), - ); - let restore_children_ms = self.record_publication_point_cache_phase_ms( - "publication_point_cache_restore_children_total", - restore_children_started, - ); - self.record_publication_point_step_ms( - &ca.manifest_rsync_uri, - "publication_point_cache_restore_children", - restore_children_ms, - ); - let ccr_projection = projection.ccr_manifest_projection.clone(); - let ccr_append_started = std::time::Instant::now(); - self.append_ccr_manifest_projection(&ccr_projection)?; - let ccr_append_ms = self.record_publication_point_cache_phase_ms( - "publication_point_cache_ccr_append_total", - ccr_append_started, - ); - self.record_publication_point_step_ms( - &ca.manifest_rsync_uri, - "publication_point_cache_ccr_append", - ccr_append_ms, - ); - let audit_build_started = std::time::Instant::now(); - let audit = build_publication_point_audit_from_publication_point_cache_projection( - ca, - PublicationPointSource::PublicationPointCache, - repo_sync_source, - repo_sync_phase, - Some(repo_sync_duration_ms), - repo_sync_err, - &projection, - self.validation_time, - &warnings, - &objects, - &child_audits, - ); - let audit_build_ms = self.record_publication_point_cache_phase_ms( - "publication_point_cache_audit_build_total", - audit_build_started, - ); - self.record_publication_point_step_ms( - &ca.manifest_rsync_uri, - "publication_point_cache_audit_build", - audit_build_ms, - ); - let audit_object_count = audit.objects.len() as u64; - let cir_cached_objects = audit.objects.clone(); - let cir_cached_objects_count = cir_cached_objects.len() as u64; - objects.local_outputs_cache.clear(); - let total_ms = self.record_publication_point_cache_phase_ms( - "publication_point_cache_reuse_build_total", - build_started, - ); - self.record_publication_point_step_ms( - &ca.manifest_rsync_uri, - "publication_point_cache_reuse_build", - total_ms, - ); - if let Some(timing) = self.timing.as_ref() { - timing.record_count("publication_point_cache_outputs_reused", output_reuse_count); - timing.record_count("publication_point_cache_children_reused", child_reuse_count); - timing.record_count( - "publication_point_cache_related_objects_reused", - related_object_reuse_count, - ); - timing.record_count( - "publication_point_cache_audit_objects_reused", - audit_object_count, - ); - if total_ms > 0 { - timing.record_count("publication_point_cache_reuse_nonzero_ms_hits", 1); - timing.record_count("publication_point_cache_reuse_nonzero_ms_total", total_ms); - } - } - if total_ms >= crate::progress_log::pp_cache_slow_threshold_ms() { - crate::progress_log::emit( - "publication_point_cache_reuse_slow", - serde_json::json!({ - "manifest_rsync_uri": ca.manifest_rsync_uri.as_str(), - "publication_point_rsync_uri": ca.publication_point_rsync_uri.as_str(), - "repo_sync_source": repo_sync_source, - "repo_sync_phase": repo_sync_phase, - "repo_sync_duration_ms": repo_sync_duration_ms, - "repo_sync_err": repo_sync_err, - "outputs_reused": output_reuse_count, - "children_reused": child_reuse_count, - "related_objects_reused": related_object_reuse_count, - "audit_objects_reused": audit_object_count, - "build_objects_ms": build_objects_ms, - "restore_children_ms": restore_children_ms, - "restore_children_workers": child_restore_workers, - "ccr_append_ms": ccr_append_ms, - "audit_build_ms": audit_build_ms, - "total_ms": total_ms, - "cir_cached_objects": cir_cached_objects_count, - "slow_threshold_ms": crate::progress_log::pp_cache_slow_threshold_ms(), - }), - ); - } - Ok(PublicationPointRunResult { - source: PublicationPointSource::PublicationPointCache, - snapshot: None, - warnings, - objects, - audit, - cir_fresh_objects: Vec::new(), - cir_cached_objects, - discovered_children, - }) - } - - fn record_publication_point_cache_phase_ms( - &self, - phase: &'static str, - started: std::time::Instant, - ) -> u64 { - let nanos = started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64; - if let Some(timing) = self.timing.as_ref() { - timing.record_phase_nanos(phase, nanos); - } - nanos / 1_000_000 - } - - pub(crate) fn record_publication_point_total_ms(&self, manifest_rsync_uri: &str, ms: u64) { - if let Some(timing) = self.timing.as_ref() { - timing.record_publication_point_nanos(manifest_rsync_uri, ms.saturating_mul(1_000_000)); - } - } - - pub(crate) fn record_publication_point_step_ms( - &self, - manifest_rsync_uri: &str, - step: &'static str, - ms: u64, - ) { - if let Some(timing) = self.timing.as_ref() { - timing.record_publication_point_step_nanos( - manifest_rsync_uri, - step, - ms.saturating_mul(1_000_000), - ); - } - } - - fn publication_point_cache_child_restore_worker_count(&self) -> usize { - self.parallel_phase2_config - .as_ref() - .map(|config| config.object_workers) - .unwrap_or(1) - .clamp(1, PUBLICATION_POINT_CACHE_CHILD_RESTORE_MAX_WORKERS) - } - - pub(crate) fn ccr_accumulator_snapshot(&self) -> Option { - self.ccr_accumulator - .as_ref() - .and_then(|accumulator| accumulator.lock().ok().map(|guard| guard.clone())) - } - - pub(crate) fn append_ccr_manifest_projection( - &self, - projection: &VcirCcrManifestProjection, - ) -> Result<(), String> { - if let Some(accumulator) = self.ccr_accumulator.as_ref() { - accumulator - .lock() - .map_err(|_| "lock CCR accumulator failed".to_string())? - .append_manifest_projection(projection)?; - } - Ok(()) - } - - fn append_ccr_manifest_projection_from_reuse( - &self, - projection: &VcirReuseProjection, - ) -> Result<(), String> { - match projection.source { - PublicationPointSource::Fresh => Err( - "invalid reuse projection source: fresh does not belong to failed-fetch reuse" - .to_string(), - ), - PublicationPointSource::PublicationPointCache => self.append_ccr_manifest_projection( - projection.ccr_manifest_projection.as_ref().ok_or_else(|| { - "publication-point cache reuse is missing CCR manifest projection".to_string() - })?, - ), - PublicationPointSource::VcirCurrentInstance => self.append_ccr_manifest_projection( - projection.ccr_manifest_projection.as_ref().ok_or_else(|| { - "vcir current-instance reuse is missing CCR manifest projection".to_string() - })?, - ), - PublicationPointSource::FailedFetchNoCache => Ok(()), - } - } - - fn current_manifest_hash_hex_for_audit(&self, ca: &CaInstanceHandle) -> Option { - if let Some(index_handle) = self.current_repo_index.as_ref() - && let Ok(index) = index_handle.read() - && let Some(entry) = index.get_by_uri(&ca.manifest_rsync_uri) - { - return Some(entry.current_hash_hex.clone()); - } - - self.store - .load_current_object_with_hash_by_uri(&ca.manifest_rsync_uri) - .ok() - .flatten() - .map(|current| current.current_hash_hex) - } - - fn rejected_manifest_audit_entry_for_failed_fetch( - &self, - ca: &CaInstanceHandle, - fresh_err: &ManifestFreshError, - ) -> Option { - let sha256_hex = self.current_manifest_hash_hex_for_audit(ca)?; - Some(ObjectAuditEntry { - rsync_uri: ca.manifest_rsync_uri.clone(), - sha256_hex, - kind: AuditObjectKind::Manifest, - result: AuditObjectResult::Error, - detail: Some(fresh_err.to_string()), - }) - } - - fn fresh_failure_audit_entries_for_cir( - &self, - ca: &CaInstanceHandle, - fresh_err: &ManifestFreshError, - ) -> Vec { - if !fresh_err.should_warn_when_current_instance_reused() { - return Vec::new(); - } - self.rejected_manifest_audit_entry_for_failed_fetch(ca, fresh_err) - .into_iter() - .collect() - } - - pub(crate) fn stage_fresh_publication_point_after_repo_ready( - &self, - ca: &CaInstanceHandle, - repo_sync_ok: bool, - repo_sync_err: Option<&str>, - ) -> Result { - let snapshot_prepare_started = std::time::Instant::now(); - let issuer_ca_der = ca_certificate_der_for_validation(ca, self.store, self.timing.as_ref()) - .map_err(|detail| FreshPublicationPointStageError { - error: ManifestFreshError::IssuerCaLoadFailed { detail }, - snapshot_prepare_ms: snapshot_prepare_started.elapsed().as_millis() as u64, - })?; - let issuer_ca_der: Arc<[u8]> = Arc::from(issuer_ca_der.as_ref()); - let fresh_publication_point = { - let _manifest_total = self - .timing - .as_ref() - .map(|t| t.span_phase("manifest_processing_total")); - process_manifest_publication_point_fresh_after_repo_sync_with_timing( - self.store, - &ca.manifest_rsync_uri, - &ca.publication_point_rsync_uri, - self.current_repo_index.as_ref(), - issuer_ca_der.as_ref(), - ca.ca_certificate_rsync_uri.as_deref(), - self.validation_time, - repo_sync_ok, - repo_sync_err, - ) - }; - let snapshot_prepare_ms = snapshot_prepare_started.elapsed().as_millis() as u64; - let (fresh_point, snapshot_prepare_timing) = - fresh_publication_point.map_err(|error| FreshPublicationPointStageError { - error, - snapshot_prepare_ms, - })?; - if let Some(timing) = self.timing.as_ref() { - timing.record_phase_nanos( - "fresh_snapshot_prepare_total", - snapshot_prepare_ms.saturating_mul(1_000_000), - ); - timing.record_phase_nanos( - "fresh_snapshot_manifest_load_total", - snapshot_prepare_timing - .manifest_load_ms - .saturating_mul(1_000_000), - ); - timing.record_phase_nanos( - "fresh_snapshot_manifest_decode_total", - snapshot_prepare_timing - .manifest_decode_ms - .saturating_mul(1_000_000), - ); - timing.record_phase_nanos( - "fresh_snapshot_replay_guard_total", - snapshot_prepare_timing - .replay_guard_ms - .saturating_mul(1_000_000), - ); - timing.record_phase_nanos( - "fresh_snapshot_manifest_entries_total", - snapshot_prepare_timing - .manifest_entries_ms - .saturating_mul(1_000_000), - ); - timing.record_phase_nanos( - "fresh_snapshot_pack_files_total", - snapshot_prepare_timing - .pack_files_ms - .saturating_mul(1_000_000), - ); - timing.record_phase_nanos( - "fresh_snapshot_ee_path_validate_total", - snapshot_prepare_timing - .ee_path_validate_ms - .saturating_mul(1_000_000), - ); - timing.record_count("fresh_publication_points", 1); - timing.record_count( - "fresh_manifest_files_total", - snapshot_prepare_timing.manifest_file_count as u64, - ); - } - self.record_publication_point_step_ms( - &ca.manifest_rsync_uri, - "fresh_snapshot_prepare", - snapshot_prepare_ms, - ); - self.record_publication_point_step_ms( - &ca.manifest_rsync_uri, - "fresh_snapshot_manifest_load", - snapshot_prepare_timing.manifest_load_ms, - ); - self.record_publication_point_step_ms( - &ca.manifest_rsync_uri, - "fresh_snapshot_manifest_decode", - snapshot_prepare_timing.manifest_decode_ms, - ); - self.record_publication_point_step_ms( - &ca.manifest_rsync_uri, - "fresh_snapshot_replay_guard", - snapshot_prepare_timing.replay_guard_ms, - ); - self.record_publication_point_step_ms( - &ca.manifest_rsync_uri, - "fresh_snapshot_manifest_entries", - snapshot_prepare_timing.manifest_entries_ms, - ); - self.record_publication_point_step_ms( - &ca.manifest_rsync_uri, - "fresh_snapshot_pack_files", - snapshot_prepare_timing.pack_files_ms, - ); - self.record_publication_point_step_ms( - &ca.manifest_rsync_uri, - "fresh_snapshot_ee_path_validate", - snapshot_prepare_timing.ee_path_validate_ms, - ); - - let child_discovery_started = std::time::Instant::now(); - let out = { - let _child_disc_total = self - .timing - .as_ref() - .map(|t| t.span_phase("child_discovery_total")); - discover_children_from_fresh_snapshot_with_audit_cached_with_issuer_der( - ca, - issuer_ca_der.as_ref(), - &fresh_point, - self.validation_time, - self.timing.as_ref(), - self.policy, - if self.enable_child_certificate_validation_cache { - Some(ChildCertificateValidationCacheContext { - store: self.store, - issuer_ca_sha256: sha256_digest_32(issuer_ca_der.as_ref()), - ca_validation_context_digest: ca_validation_context_digest_for_ca(ca), - policy_fingerprint: publication_point_cache_policy_fingerprint(self.policy), - }) - } else { - None - }, - ) - }; - let (discovered_children, child_audits, discovered_router_keys, warnings) = match out { - Ok(out) => (out.children, out.audits, out.router_keys, Vec::new()), - Err(e) => ( - Vec::new(), - Vec::new(), - Vec::new(), - vec![ - Warning::new(format!("child CA discovery failed: {e}")) - .with_rfc_refs(&[RfcRef("RFC 6487 §7.2")]) - .with_context(&ca.manifest_rsync_uri), - ], - ), - }; - let child_discovery_ms = child_discovery_started.elapsed().as_millis() as u64; - if let Some(timing) = self.timing.as_ref() { - timing.record_phase_nanos( - "fresh_child_discovery_total", - child_discovery_ms.saturating_mul(1_000_000), - ); - timing.record_count( - "fresh_children_discovered", - discovered_children.len() as u64, - ); - timing.record_count("fresh_child_audits", child_audits.len() as u64); - timing.record_count( - "fresh_router_keys_discovered", - discovered_router_keys.len() as u64, - ); - } - self.record_publication_point_step_ms( - &ca.manifest_rsync_uri, - "fresh_child_discovery", - child_discovery_ms, - ); - - Ok(FreshPublicationPointStage { - fresh_point, - issuer_ca_der, - snapshot_prepare_timing, - snapshot_prepare_ms, - discovered_children, - child_audits, - discovered_router_keys, - child_discovery_ms, - warnings, - }) - } - - pub(crate) fn finalize_fresh_publication_point_from_reducer( - &self, - ca: &CaInstanceHandle, - fresh_point: &FreshValidatedPublicationPoint, - warnings: Vec, - mut objects: crate::validation::objects::ObjectsOutput, - child_audits: Vec, - discovered_children: Vec, - repo_sync_source: Option<&str>, - repo_sync_phase: Option<&str>, - repo_sync_duration_ms: u64, - repo_sync_err: Option<&str>, - ) -> Result { - let snapshot_pack_started = std::time::Instant::now(); - let pack = fresh_point.to_publication_point_snapshot(); - let snapshot_pack_ms = snapshot_pack_started.elapsed().as_millis() as u64; - if let Some(timing) = self.timing.as_ref() { - timing.record_phase_nanos( - "fresh_snapshot_pack_total", - snapshot_pack_ms.saturating_mul(1_000_000), - ); - } - self.record_publication_point_step_ms( - &ca.manifest_rsync_uri, - "fresh_snapshot_pack", - snapshot_pack_ms, - ); - - let persist_vcir_started = std::time::Instant::now(); - let persist_vcir_timing = if self.persist_vcir { - persist_vcir_for_fresh_result_with_timing( - self.store, - self.policy, - ca, - &pack, - &mut objects, - &warnings, - &child_audits, - &discovered_children, - self.validation_time, - self.publication_point_cache_observe_only - || self.enable_publication_point_validation_cache, - ) - .map_err(|e| format!("persist VCIR failed: {e}"))? - } else { - PersistVcirTimingBreakdown::default() - }; - let persist_vcir_ms = persist_vcir_started.elapsed().as_millis() as u64; - if let Some(timing) = self.timing.as_ref() { - timing.record_phase_nanos( - "fresh_persist_vcir_total", - persist_vcir_ms.saturating_mul(1_000_000), - ); - timing.record_phase_nanos( - "fresh_persist_embedded_store_total", - persist_vcir_timing - .embedded_store_ms - .saturating_mul(1_000_000), - ); - timing.record_phase_nanos( - "fresh_persist_build_vcir_total", - persist_vcir_timing.build_vcir_ms.saturating_mul(1_000_000), - ); - timing.record_phase_nanos( - "fresh_persist_replace_vcir_total", - persist_vcir_timing - .replace_vcir_ms - .saturating_mul(1_000_000), - ); - timing.record_phase_nanos( - "fresh_persist_local_outputs_total", - persist_vcir_timing - .build_vcir - .local_outputs_ms - .saturating_mul(1_000_000), - ); - timing.record_phase_nanos( - "fresh_persist_child_entries_total", - persist_vcir_timing - .build_vcir - .child_entries_ms - .saturating_mul(1_000_000), - ); - timing.record_phase_nanos( - "fresh_persist_related_artifacts_total", - persist_vcir_timing - .build_vcir - .related_artifacts_ms - .saturating_mul(1_000_000), - ); - if persist_vcir_timing.publication_point_cache_future_notbefore_guarded { - timing.record_count("publication_point_cache_future_notbefore_guarded", 1); - } - } - self.record_publication_point_step_ms( - &ca.manifest_rsync_uri, - "fresh_persist_vcir", - persist_vcir_ms, - ); - self.record_publication_point_step_ms( - &ca.manifest_rsync_uri, - "fresh_persist_build_vcir", - persist_vcir_timing.build_vcir_ms, - ); - self.record_publication_point_step_ms( - &ca.manifest_rsync_uri, - "fresh_persist_replace_vcir", - persist_vcir_timing.replace_vcir_ms, - ); - - // local_outputs_cache only exists to build/persist VCIR. Release it before the - // publication point result is retained for the rest of the run. - let _released_local_outputs = std::mem::take(&mut objects.local_outputs_cache); - let _released_roa_cache_object_meta = std::mem::take(&mut objects.roa_cache_object_meta); - - let mut ccr_projection_build_ms = 0; - let mut ccr_append_ms = 0; - if self.ccr_accumulator.is_some() { - let ccr_projection_build_started = std::time::Instant::now(); - let child_entries = - build_vcir_child_entries(self.store, &discovered_children, self.validation_time)?; - let ccr_manifest_projection = - build_vcir_ccr_manifest_projection_from_fresh(ca, &pack, &child_entries)?; - ccr_projection_build_ms = ccr_projection_build_started.elapsed().as_millis() as u64; - let ccr_append_started = std::time::Instant::now(); - self.append_ccr_manifest_projection(&ccr_manifest_projection)?; - ccr_append_ms = ccr_append_started.elapsed().as_millis() as u64; - } - if let Some(timing) = self.timing.as_ref() { - timing.record_phase_nanos( - "fresh_ccr_projection_build_total", - ccr_projection_build_ms.saturating_mul(1_000_000), - ); - timing.record_phase_nanos( - "fresh_ccr_append_total", - ccr_append_ms.saturating_mul(1_000_000), - ); - } - self.record_publication_point_step_ms( - &ca.manifest_rsync_uri, - "fresh_ccr_projection_build", - ccr_projection_build_ms, - ); - self.record_publication_point_step_ms( - &ca.manifest_rsync_uri, - "fresh_ccr_append", - ccr_append_ms, - ); - - let audit_build_started = std::time::Instant::now(); - let audit = build_publication_point_audit_from_snapshot( - ca, - PublicationPointSource::Fresh, - repo_sync_source, - repo_sync_phase, - Some(repo_sync_duration_ms), - repo_sync_err, - &pack, - &warnings, - &objects, - &child_audits, - ); - let audit_build_ms = audit_build_started.elapsed().as_millis() as u64; - if let Some(timing) = self.timing.as_ref() { - timing.record_phase_nanos( - "fresh_audit_build_total", - audit_build_ms.saturating_mul(1_000_000), - ); - } - self.record_publication_point_step_ms( - &ca.manifest_rsync_uri, - "fresh_audit_build", - audit_build_ms, - ); - - Ok(FreshPublicationPointFinalizeOutput { - result: PublicationPointRunResult { - source: PublicationPointSource::Fresh, - snapshot: Some(pack), - warnings, - objects, - audit, - cir_fresh_objects: Vec::new(), - cir_cached_objects: Vec::new(), - discovered_children, - }, - snapshot_pack_ms, - persist_vcir_ms, - persist_vcir_timing, - ccr_projection_build_ms, - ccr_append_ms, - audit_build_ms, - }) - } -} - -impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> { - fn prefetch_discovered_children( - &self, - children: &[DiscoveredChildCaInstance], - ) -> Result<(), String> { - if let Some(runtime) = self.repo_sync_runtime.as_ref() { - runtime.prefetch_discovered_children(children)?; - } - Ok(()) - } - - fn run_publication_point( - &self, - ca: &CaInstanceHandle, - ) -> Result { - let publication_point_started = std::time::Instant::now(); - let _pp_total = self - .timing - .as_ref() - .map(|t| t.span_publication_point(&ca.manifest_rsync_uri)); - if let Some(t) = self.timing.as_ref() { - t.record_count("publication_points_seen", 1); - if ca.rrdp_notification_uri.is_some() { - t.record_count("publication_points_rrdp_notify_present_total", 1); - } else { - t.record_count("publication_points_rrdp_notify_missing_total", 1); - } - } - - let mut warnings: Vec = Vec::new(); - crate::progress_log::emit( - "publication_point_start", - serde_json::json!({ - "manifest_rsync_uri": ca.manifest_rsync_uri, - "publication_point_rsync_uri": ca.publication_point_rsync_uri, - "rsync_base_uri": ca.rsync_base_uri, - "rrdp_notification_uri": ca.rrdp_notification_uri, - }), - ); - - let attempted_rrdp = - self.policy.sync_preference == crate::policy::SyncPreference::RrdpThenRsync; - let original_notification_uri = ca.rrdp_notification_uri.as_deref(); - let mut effective_notification_uri = if attempted_rrdp { - original_notification_uri - } else { - None - }; - let mut skip_sync_due_to_dedup = false; - - if attempted_rrdp && self.rrdp_dedup { - if let Some(notification_uri) = original_notification_uri { - if let Some(rrdp_ok) = self - .rrdp_repo_cache - .lock() - .expect("rrdp_repo_cache lock") - .get(notification_uri) - .copied() - { - if let Some(t) = self.timing.as_ref() { - t.record_count("rrdp_repo_dedup_hits", 1); - } - if rrdp_ok { - if let Some(t) = self.timing.as_ref() { - t.record_count("rrdp_repo_dedup_rrdp_ok_skip", 1); - } - skip_sync_due_to_dedup = true; - } else { - if let Some(t) = self.timing.as_ref() { - t.record_count("rrdp_repo_dedup_rrdp_failed_skip", 1); - } - effective_notification_uri = None; - } - } else if let Some(t) = self.timing.as_ref() { - t.record_count("rrdp_repo_dedup_misses", 1); - } - } - } - - if !skip_sync_due_to_dedup && effective_notification_uri.is_none() && self.rsync_dedup { - let base = self.rsync_fetcher.dedup_key(&ca.rsync_base_uri); - let hit_ok = self - .rsync_repo_cache - .lock() - .expect("rsync_repo_cache lock") - .get(&base) - .copied() - .unwrap_or(false); - if hit_ok { - if let Some(t) = self.timing.as_ref() { - t.record_count("rsync_repo_dedup_hits", 1); - t.record_count("rsync_repo_dedup_skipped_sync", 1); - } - skip_sync_due_to_dedup = true; - } else if let Some(t) = self.timing.as_ref() { - t.record_count("rsync_repo_dedup_misses", 1); - } - } - - let repo_sync_started = std::time::Instant::now(); - let mut runtime_repo_sync_duration_ms = None; - let (repo_sync_ok, repo_sync_err, repo_sync_source, repo_sync_phase): ( - bool, - Option, - Option, - Option, - ) = if let Some(runtime) = self.repo_sync_runtime.as_ref() { - let RepoSyncRuntimeOutcome { - repo_sync_ok, - repo_sync_err, - repo_sync_source, - repo_sync_phase, - repo_sync_duration_ms, - warnings: repo_warnings, - } = runtime.sync_publication_point_repo(ca)?; - runtime_repo_sync_duration_ms = Some(repo_sync_duration_ms); - warnings.extend(repo_warnings); - ( - repo_sync_ok, - repo_sync_err, - repo_sync_source, - repo_sync_phase, - ) - } else if skip_sync_due_to_dedup { - let source = if effective_notification_uri.is_some() { - Some("rrdp_dedup_skip".to_string()) - } else { - Some("rsync_dedup_skip".to_string()) - }; - let phase = source.clone(); - (true, None, source, phase) - } else { - let repo_key = effective_notification_uri.unwrap_or_else(|| ca.rsync_base_uri.as_str()); - let _repo_total = self - .timing - .as_ref() - .map(|t| t.span_phase("repo_sync_total")); - let _repo_span = self.timing.as_ref().map(|t| t.span_rrdp_repo(repo_key)); - - match if let Some(delta_index) = self.replay_delta_index.as_ref() { - sync_publication_point_replay_delta( - self.store, - delta_index, - effective_notification_uri, - &ca.rsync_base_uri, - self.http_fetcher, - self.rsync_fetcher, - self.timing.as_ref(), - self.download_log.as_ref(), - ) - } else if let Some(replay_index) = self.replay_archive_index.as_ref() { - sync_publication_point_replay( - self.store, - replay_index, - effective_notification_uri, - &ca.rsync_base_uri, - self.http_fetcher, - self.rsync_fetcher, - self.timing.as_ref(), - self.download_log.as_ref(), - ) - } else { - sync_publication_point( - self.store, - self.policy, - effective_notification_uri, - &ca.rsync_base_uri, - self.http_fetcher, - self.rsync_fetcher, - self.timing.as_ref(), - self.download_log.as_ref(), - ) - } { - Ok(res) => { - if self.rsync_dedup && res.source == crate::sync::repo::RepoSyncSource::Rsync { - let base = self.rsync_fetcher.dedup_key(&ca.rsync_base_uri); - self.rsync_repo_cache - .lock() - .expect("rsync_repo_cache lock") - .insert(base, true); - if let Some(t) = self.timing.as_ref() { - t.record_count("rsync_repo_dedup_mark_ok", 1); - } - } - - if attempted_rrdp && self.rrdp_dedup { - if let Some(notification_uri) = original_notification_uri { - if effective_notification_uri.is_some() { - let rrdp_ok = res.source == crate::sync::repo::RepoSyncSource::Rrdp; - self.rrdp_repo_cache - .lock() - .expect("rrdp_repo_cache lock") - .insert(notification_uri.to_string(), rrdp_ok); - if let Some(t) = self.timing.as_ref() { - if rrdp_ok { - t.record_count("rrdp_repo_dedup_mark_ok", 1); - } else { - t.record_count("rrdp_repo_dedup_mark_failed", 1); - } - } - } - } - } - - warnings.extend(res.warnings); - ( - true, - None, - Some(repo_sync_source_label(res.source).to_string()), - Some(repo_sync_phase_label(res.phase).to_string()), - ) - } - Err(e) => { - if attempted_rrdp && self.rrdp_dedup { - if let Some(notification_uri) = original_notification_uri { - if effective_notification_uri.is_some() { - self.rrdp_repo_cache - .lock() - .expect("rrdp_repo_cache lock") - .insert(notification_uri.to_string(), false); - } - } - } - warnings.push( - Warning::new(format!("repo sync failed (fresh processing stopped): {e}")) - .with_rfc_refs(&[RfcRef("RFC 8182 §3.4.5"), RfcRef("RFC 9286 §6.6")]) - .with_context(&ca.rsync_base_uri), - ); - ( - false, - Some(e.to_string()), - None, - Some( - repo_sync_failure_phase_label( - attempted_rrdp, - original_notification_uri, - effective_notification_uri, - ) - .to_string(), - ), - ) - } - } - }; - let repo_sync_duration_ms = effective_repo_sync_duration_ms( - repo_sync_started.elapsed().as_millis() as u64, - runtime_repo_sync_duration_ms, - repo_sync_ok, - ); - crate::progress_log::emit( - "publication_point_repo_sync_done", - serde_json::json!({ - "manifest_rsync_uri": ca.manifest_rsync_uri, - "publication_point_rsync_uri": ca.publication_point_rsync_uri, - "repo_sync_ok": repo_sync_ok, - "repo_sync_source": repo_sync_source, - "repo_sync_phase": repo_sync_phase, - "repo_sync_error": repo_sync_err, - "repo_sync_duration_ms": repo_sync_duration_ms, - }), - ); - - if let Some(result) = self.observe_or_reuse_publication_point_cache( - ca, - repo_sync_source.as_deref(), - repo_sync_phase.as_deref(), - repo_sync_duration_ms, - repo_sync_err.as_deref(), - &warnings, - ) { - let total_duration_ms = publication_point_started.elapsed().as_millis() as u64; - crate::progress_log::emit( - "publication_point_finish", - serde_json::json!({ - "manifest_rsync_uri": ca.manifest_rsync_uri, - "publication_point_rsync_uri": ca.publication_point_rsync_uri, - "source": source_label(result.source), - "repo_sync_source": repo_sync_source, - "repo_sync_phase": repo_sync_phase, - "repo_sync_duration_ms": repo_sync_duration_ms, - "total_duration_ms": total_duration_ms, - "post_repo_duration_ms": total_duration_ms.saturating_sub(repo_sync_duration_ms), - "warning_count": result.warnings.len(), - "vrp_count": result.objects.vrps.len(), - "vap_count": result.objects.aspas.len(), - "router_key_count": result.objects.router_keys.len(), - "child_count": result.discovered_children.len(), - }), - ); - return Ok(result); - } - - let fresh_stage = self.stage_fresh_publication_point_after_repo_ready( - ca, - repo_sync_ok, - repo_sync_err.as_deref(), - ); - - match fresh_stage { - Ok(stage) => { - let FreshPublicationPointStage { - fresh_point, - issuer_ca_der, - snapshot_prepare_timing, - snapshot_prepare_ms, - discovered_children, - child_audits, - discovered_router_keys, - child_discovery_ms, - warnings: stage_warnings, - } = stage; - warnings.extend(stage_warnings); - - let has_roa = fresh_point - .files() - .iter() - .any(|file| file.rsync_uri.ends_with(".roa")); - if self.enable_roa_validation_cache { - if let Some(timing) = self.timing.as_ref() { - if has_roa { - timing.record_count( - "roa_validation_cache_roa_candidate_publication_points", - 1, - ); - } else { - timing.record_count( - "roa_validation_cache_skipped_no_roa_publication_points", - 1, - ); - } - } - } - let roa_cache_view = if has_roa { - self.roa_validation_cache_view_for_fresh_point(fresh_point.manifest_rsync_uri()) - } else { - None - }; - let roa_cache = if self.enable_roa_validation_cache && has_roa { - RoaValidationCacheInput::enabled_with_context( - roa_cache_view.as_ref(), - ca_validation_context_digest_for_ca(ca), - publication_point_cache_policy_fingerprint(self.policy), - ) - } else { - RoaValidationCacheInput::disabled() - }; - let objects_processing_started = std::time::Instant::now(); - let ta_constraints = self.policy.ta_constraints.for_tal(&ca.tal_id); - let mut objects = { - let _objects_total = self - .timing - .as_ref() - .map(|t| t.span_phase("objects_processing_total")); - if let Some(ta_constraints) = ta_constraints { - // This method is the serial/fallback publication-point path. The - // phase-2 scheduler uses the stage-specific parallel prepare path, - // which carries the same immutable per-TAL snapshot into workers. - process_publication_point_for_issuer_with_cache_options_and_ta_constraints( - &fresh_point, - self.policy, - issuer_ca_der.as_ref(), - ca.ca_certificate_rsync_uri.as_deref(), - ca.effective_ip_resources.as_ref(), - ca.effective_as_resources.as_ref(), - self.validation_time, - self.timing.as_ref(), - false, - roa_cache, - Some(ta_constraints), - ) - } else if let Some(phase2_pool) = self.parallel_roa_worker_pool.as_ref() { - process_publication_point_for_issuer_parallel_roa_with_pool_cache_options( - &fresh_point, - self.policy, - issuer_ca_der.as_ref(), - ca.ca_certificate_rsync_uri.as_deref(), - ca.effective_ip_resources.as_ref(), - ca.effective_as_resources.as_ref(), - self.validation_time, - self.timing.as_ref(), - phase2_pool, - false, - roa_cache, - ) - } else if let Some(phase2_config) = self.parallel_phase2_config.as_ref() { - process_publication_point_for_issuer_parallel_roa_with_cache_options( - &fresh_point, - self.policy, - issuer_ca_der.as_ref(), - ca.ca_certificate_rsync_uri.as_deref(), - ca.effective_ip_resources.as_ref(), - ca.effective_as_resources.as_ref(), - self.validation_time, - self.timing.as_ref(), - phase2_config, - false, - roa_cache, - ) - } else { - crate::validation::objects::process_publication_point_for_issuer_with_cache_options( - &fresh_point, - self.policy, - issuer_ca_der.as_ref(), - ca.ca_certificate_rsync_uri.as_deref(), - ca.effective_ip_resources.as_ref(), - ca.effective_as_resources.as_ref(), - self.validation_time, - self.timing.as_ref(), - false, - roa_cache, - ) - } - }; - let objects_processing_ms = objects_processing_started.elapsed().as_millis() as u64; - self.record_publication_point_step_ms( - &ca.manifest_rsync_uri, - "fresh_objects_processing", - objects_processing_ms, - ); - - objects.router_keys.extend(discovered_router_keys); - objects - .local_outputs_cache - .extend(build_router_key_local_outputs(ca, &objects.router_keys)); - - let finalized = self.finalize_fresh_publication_point_from_reducer( - ca, - &fresh_point, - warnings, - objects, - child_audits, - discovered_children, - repo_sync_source.as_deref(), - repo_sync_phase.as_deref(), - repo_sync_duration_ms, - repo_sync_err.as_deref(), - )?; - let FreshPublicationPointFinalizeOutput { - result, - snapshot_pack_ms, - persist_vcir_ms, - persist_vcir_timing, - ccr_projection_build_ms, - ccr_append_ms, - audit_build_ms, - } = finalized; - let total_duration_ms = publication_point_started.elapsed().as_millis() as u64; - crate::progress_log::emit( - "publication_point_finish", - serde_json::json!({ - "manifest_rsync_uri": ca.manifest_rsync_uri, - "publication_point_rsync_uri": ca.publication_point_rsync_uri, - "source": "fresh", - "repo_sync_source": repo_sync_source, - "repo_sync_phase": repo_sync_phase, - "repo_sync_duration_ms": repo_sync_duration_ms, - "total_duration_ms": total_duration_ms, - "post_repo_duration_ms": total_duration_ms.saturating_sub(repo_sync_duration_ms), - "snapshot_prepare_ms": snapshot_prepare_ms, - "snapshot_manifest_load_ms": snapshot_prepare_timing.manifest_load_ms, - "snapshot_manifest_decode_ms": snapshot_prepare_timing.manifest_decode_ms, - "snapshot_replay_guard_ms": snapshot_prepare_timing.replay_guard_ms, - "snapshot_manifest_entries_ms": snapshot_prepare_timing.manifest_entries_ms, - "snapshot_pack_files_ms": snapshot_prepare_timing.pack_files_ms, - "snapshot_ee_path_validate_ms": snapshot_prepare_timing.ee_path_validate_ms, - "objects_processing_ms": objects_processing_ms, - "child_discovery_ms": child_discovery_ms, - "snapshot_pack_ms": snapshot_pack_ms, - "persist_vcir_ms": persist_vcir_ms, - "persist_embedded_collect_ms": persist_vcir_timing.embedded_collect_ms, - "persist_embedded_store_ms": persist_vcir_timing.embedded_store_ms, - "persist_build_vcir_ms": persist_vcir_timing.build_vcir_ms, - "persist_replace_vcir_ms": persist_vcir_timing.replace_vcir_ms, - "persist_select_crl_ms": persist_vcir_timing.build_vcir.select_crl_ms, - "persist_current_ca_decode_ms": persist_vcir_timing.build_vcir.current_ca_decode_ms, - "persist_local_outputs_ms": persist_vcir_timing.build_vcir.local_outputs_ms, - "persist_child_entries_ms": persist_vcir_timing.build_vcir.child_entries_ms, - "persist_related_artifacts_ms": persist_vcir_timing.build_vcir.related_artifacts_ms, - "persist_vcir_struct_ms": persist_vcir_timing.build_vcir.struct_build_ms, - "persist_replace_breakdown": &persist_vcir_timing.replace_vcir, - "publication_point_cache_future_notbefore_guarded": persist_vcir_timing.publication_point_cache_future_notbefore_guarded, - "ccr_projection_build_ms": ccr_projection_build_ms, - "ccr_append_ms": ccr_append_ms, - "audit_build_ms": audit_build_ms, - "warning_count": result.warnings.len(), - "vrp_count": result.objects.vrps.len(), - "vap_count": result.objects.aspas.len(), - "router_key_count": result.objects.router_keys.len(), - "child_count": result.discovered_children.len(), - }), - ); - if (total_duration_ms as f64) / 1000.0 >= crate::progress_log::slow_threshold_secs() - { - crate::progress_log::emit( - "publication_point_slow", - serde_json::json!({ - "manifest_rsync_uri": ca.manifest_rsync_uri, - "publication_point_rsync_uri": ca.publication_point_rsync_uri, - "source": "fresh", - "repo_sync_source": repo_sync_source, - "repo_sync_phase": repo_sync_phase, - "repo_sync_duration_ms": repo_sync_duration_ms, - "total_duration_ms": total_duration_ms, - "post_repo_duration_ms": total_duration_ms.saturating_sub(repo_sync_duration_ms), - "snapshot_prepare_ms": snapshot_prepare_ms, - "snapshot_manifest_load_ms": snapshot_prepare_timing.manifest_load_ms, - "snapshot_manifest_decode_ms": snapshot_prepare_timing.manifest_decode_ms, - "snapshot_replay_guard_ms": snapshot_prepare_timing.replay_guard_ms, - "snapshot_manifest_entries_ms": snapshot_prepare_timing.manifest_entries_ms, - "snapshot_pack_files_ms": snapshot_prepare_timing.pack_files_ms, - "snapshot_ee_path_validate_ms": snapshot_prepare_timing.ee_path_validate_ms, - "objects_processing_ms": objects_processing_ms, - "child_discovery_ms": child_discovery_ms, - "snapshot_pack_ms": snapshot_pack_ms, - "persist_vcir_ms": persist_vcir_ms, - "persist_embedded_collect_ms": persist_vcir_timing.embedded_collect_ms, - "persist_embedded_store_ms": persist_vcir_timing.embedded_store_ms, - "persist_build_vcir_ms": persist_vcir_timing.build_vcir_ms, - "persist_replace_vcir_ms": persist_vcir_timing.replace_vcir_ms, - "persist_select_crl_ms": persist_vcir_timing.build_vcir.select_crl_ms, - "persist_current_ca_decode_ms": persist_vcir_timing.build_vcir.current_ca_decode_ms, - "persist_local_outputs_ms": persist_vcir_timing.build_vcir.local_outputs_ms, - "persist_child_entries_ms": persist_vcir_timing.build_vcir.child_entries_ms, - "persist_related_artifacts_ms": persist_vcir_timing.build_vcir.related_artifacts_ms, - "persist_vcir_struct_ms": persist_vcir_timing.build_vcir.struct_build_ms, - "persist_replace_breakdown": &persist_vcir_timing.replace_vcir, - "publication_point_cache_future_notbefore_guarded": persist_vcir_timing.publication_point_cache_future_notbefore_guarded, - "ccr_projection_build_ms": ccr_projection_build_ms, - "ccr_append_ms": ccr_append_ms, - "audit_build_ms": audit_build_ms, - }), - ); - } - Ok(result) - } - Err(stage_err) => { - let snapshot_prepare_ms = stage_err.snapshot_prepare_ms; - let fresh_err = stage_err.error; - match self.policy.ca_failed_fetch_policy { - crate::policy::CaFailedFetchPolicy::StopAllOutput => { - let total_duration_ms = - publication_point_started.elapsed().as_millis() as u64; - crate::progress_log::emit( - "publication_point_finish", - serde_json::json!({ - "manifest_rsync_uri": ca.manifest_rsync_uri, - "publication_point_rsync_uri": ca.publication_point_rsync_uri, - "source": "error", - "repo_sync_source": repo_sync_source, - "repo_sync_phase": repo_sync_phase, - "repo_sync_duration_ms": repo_sync_duration_ms, - "total_duration_ms": total_duration_ms, - "post_repo_duration_ms": total_duration_ms.saturating_sub(repo_sync_duration_ms), - "snapshot_prepare_ms": snapshot_prepare_ms, - "projection_ms": 0, - "audit_build_ms": 0, - "error": fresh_err.to_string(), - }), - ); - crate::progress_log::emit( - "repo_terminal_failure", - serde_json::json!({ - "manifest_rsync_uri": ca.manifest_rsync_uri, - "publication_point_rsync_uri": ca.publication_point_rsync_uri, - "repo_sync_source": repo_sync_source, - "repo_sync_phase": repo_sync_phase, - "repo_sync_error": repo_sync_err, - "repo_sync_duration_ms": repo_sync_duration_ms, - "terminal_state": "stop_all_output", - "error": fresh_err.to_string(), - }), - ); - Err(format!("{fresh_err}")) - } - crate::policy::CaFailedFetchPolicy::ReuseCurrentInstanceVcir => { - let projection_started = std::time::Instant::now(); - let projection = project_current_instance_vcir_on_failed_fetch( - self.store, - ca, - &fresh_err, - self.policy, - self.validation_time, - ) - .map_err(|e| format!("failed fetch VCIR projection failed: {e}"))?; - let fresh_failure_audits = - self.fresh_failure_audit_entries_for_cir(ca, &fresh_err); - self.append_ccr_manifest_projection_from_reuse(&projection)?; - let projection_ms = projection_started.elapsed().as_millis() as u64; - warnings.extend(projection.warnings.clone()); - let audit_build_started = std::time::Instant::now(); - let audit = build_publication_point_audit_from_vcir( - ca, - projection.source, - repo_sync_source.as_deref(), - repo_sync_phase.as_deref(), - Some(repo_sync_duration_ms), - repo_sync_err.as_deref(), - projection.vcir.as_ref(), - projection.snapshot.as_ref(), - &warnings, - &projection.objects, - &projection.child_audits, - &fresh_failure_audits, - ); - let audit_build_ms = audit_build_started.elapsed().as_millis() as u64; - let cir_cached_objects = - if projection.source == PublicationPointSource::VcirCurrentInstance { - audit - .objects - .iter() - .filter(|entry| { - !fresh_failure_audits.iter().any(|fresh| fresh == *entry) - }) - .cloned() - .collect() - } else { - Vec::new() - }; - let result = PublicationPointRunResult { - source: projection.source, - snapshot: projection.snapshot, - warnings, - objects: projection.objects, - audit, - cir_fresh_objects: if projection.source - == PublicationPointSource::VcirCurrentInstance - { - fresh_failure_audits - } else { - Vec::new() - }, - cir_cached_objects, - discovered_children: projection.discovered_children, - }; - let total_duration_ms = - publication_point_started.elapsed().as_millis() as u64; - crate::progress_log::emit( - "publication_point_finish", - serde_json::json!({ - "manifest_rsync_uri": ca.manifest_rsync_uri, - "publication_point_rsync_uri": ca.publication_point_rsync_uri, - "source": source_label(result.source), - "repo_sync_source": repo_sync_source, - "repo_sync_phase": repo_sync_phase, - "repo_sync_duration_ms": repo_sync_duration_ms, - "total_duration_ms": total_duration_ms, - "post_repo_duration_ms": total_duration_ms.saturating_sub(repo_sync_duration_ms), - "snapshot_prepare_ms": snapshot_prepare_ms, - "projection_ms": projection_ms, - "audit_build_ms": audit_build_ms, - "warning_count": result.warnings.len(), - "vrp_count": result.objects.vrps.len(), - "vap_count": result.objects.aspas.len(), - "router_key_count": result.objects.router_keys.len(), - "child_count": result.discovered_children.len(), - }), - ); - match result.source { - PublicationPointSource::VcirCurrentInstance if !repo_sync_ok => { - crate::progress_log::emit( - "rsync_failed_fallback_current_instance", - serde_json::json!({ - "manifest_rsync_uri": ca.manifest_rsync_uri, - "publication_point_rsync_uri": ca.publication_point_rsync_uri, - "repo_sync_source": repo_sync_source, - "repo_sync_phase": repo_sync_phase, - "repo_sync_error": repo_sync_err, - "repo_sync_duration_ms": repo_sync_duration_ms, - "terminal_state": "fallback_current_instance", - }), - ); - } - PublicationPointSource::FailedFetchNoCache => { - if !repo_sync_ok { - crate::progress_log::emit( - "rsync_failed_no_cache", - serde_json::json!({ - "manifest_rsync_uri": ca.manifest_rsync_uri, - "publication_point_rsync_uri": ca.publication_point_rsync_uri, - "repo_sync_source": repo_sync_source, - "repo_sync_phase": repo_sync_phase, - "repo_sync_error": repo_sync_err, - "repo_sync_duration_ms": repo_sync_duration_ms, - "terminal_state": "failed_no_cache", - }), - ); - } - crate::progress_log::emit( - "repo_terminal_failure", - serde_json::json!({ - "manifest_rsync_uri": ca.manifest_rsync_uri, - "publication_point_rsync_uri": ca.publication_point_rsync_uri, - "repo_sync_source": repo_sync_source, - "repo_sync_phase": repo_sync_phase, - "repo_sync_error": repo_sync_err, - "repo_sync_duration_ms": repo_sync_duration_ms, - "terminal_state": "failed_no_cache", - }), - ); - } - PublicationPointSource::Fresh => {} - PublicationPointSource::PublicationPointCache => {} - PublicationPointSource::VcirCurrentInstance => {} - } - if (total_duration_ms as f64) / 1000.0 - >= crate::progress_log::slow_threshold_secs() - { - crate::progress_log::emit( - "publication_point_slow", - serde_json::json!({ - "manifest_rsync_uri": ca.manifest_rsync_uri, - "publication_point_rsync_uri": ca.publication_point_rsync_uri, - "source": source_label(result.source), - "repo_sync_source": repo_sync_source, - "repo_sync_phase": repo_sync_phase, - "repo_sync_duration_ms": repo_sync_duration_ms, - "total_duration_ms": total_duration_ms, - "post_repo_duration_ms": total_duration_ms.saturating_sub(repo_sync_duration_ms), - "snapshot_prepare_ms": snapshot_prepare_ms, - "projection_ms": projection_ms, - "audit_build_ms": audit_build_ms, - }), - ); - } - Ok(result) - } - } - } - } - } -} - -struct ChildDiscoveryOutput { - children: Vec, - audits: Vec, - router_keys: Vec, -} - -#[derive(Clone, Debug)] -struct VerifiedIssuerCrl { - crl: crate::data_model::crl::RpkixCrl, - revoked_serials: std::collections::HashSet>, - sha256_hex: String, -} - -#[derive(Clone, Debug)] -enum CachedIssuerCrl { - Pending { - bytes: Vec, - sha256_hex: Option, - }, - Ok(VerifiedIssuerCrl), -} - -impl CachedIssuerCrl { - fn current_sha256_hex(&mut self) -> &str { - match self { - CachedIssuerCrl::Pending { bytes, sha256_hex } => { - if sha256_hex.is_none() { - *sha256_hex = Some(crate::audit::sha256_hex(bytes)); - } - sha256_hex - .as_deref() - .expect("pending CRL sha256 must be populated") - } - CachedIssuerCrl::Ok(verified) => verified.sha256_hex.as_str(), - } - } -} - -struct PublicationPointCacheIdentity { - ca_cert_sha256: [u8; 32], - manifest_sha256: [u8; 32], - ta_context_digest: [u8; 32], - ca_validation_context_digest: [u8; 32], - policy_fingerprint: [u8; 32], -} - -fn sha256_digest_32(bytes: impl AsRef<[u8]>) -> [u8; 32] { - let digest = sha2::Sha256::digest(bytes.as_ref()); - let mut out = [0u8; 32]; - out.copy_from_slice(&digest); - out -} - -fn hash_serialized_parts(parts: &[(&str, &[u8])]) -> [u8; 32] { - let mut hasher = sha2::Sha256::new(); - for (label, value) in parts { - hasher.update((label.len() as u64).to_be_bytes()); - hasher.update(label.as_bytes()); - hasher.update((value.len() as u64).to_be_bytes()); - hasher.update(*value); - } - let digest = hasher.finalize(); - let mut out = [0u8; 32]; - out.copy_from_slice(&digest); - out -} - -fn cbor_or_debug_bytes(value: &T) -> Vec { - serde_cbor::to_vec(value).unwrap_or_else(|_| format!("{value:?}").into_bytes()) -} - -fn ta_context_digest_for_ca(ca: &CaInstanceHandle) -> [u8; 32] { - hash_serialized_parts(&[ - ("version", b"publication-point-cache-ta-v1"), - ("tal_id", ca.tal_id.as_bytes()), - ]) -} - -pub(crate) fn ca_validation_context_digest_for_ca(ca: &CaInstanceHandle) -> [u8; 32] { - let parent_manifest = ca - .parent_manifest_rsync_uri - .as_deref() - .unwrap_or("") - .as_bytes(); - let effective_ip = cbor_or_debug_bytes(&ca.effective_ip_resources); - let effective_as = cbor_or_debug_bytes(&ca.effective_as_resources); - hash_serialized_parts(&[ - ("version", b"publication-point-cache-parent-context-v1"), - ("tal_id", ca.tal_id.as_bytes()), - ("parent_manifest", parent_manifest), - ("effective_ip", effective_ip.as_slice()), - ("effective_as", effective_as.as_slice()), - ]) -} - -pub(crate) fn publication_point_cache_policy_fingerprint(policy: &Policy) -> [u8; 32] { - let policy_bytes = cbor_or_debug_bytes(policy); - hash_serialized_parts(&[ - ("version", b"publication-point-cache-policy-v3"), - ("policy", policy_bytes.as_slice()), - ("ta_constraints", policy.ta_constraints.fingerprint_bytes()), - ]) -} - -fn publication_point_cache_projection_items_valid( - projection: &PublicationPointCacheProjection, - validation_time: time::OffsetDateTime, -) -> Result<(), &'static str> { - for output in &projection.outputs { - if !pack_time_window_contains( - &output.item_effective_not_before, - &output.item_effective_until, - validation_time, - ) { - return Err("output_time_gate_miss"); - } - } - for child in &projection.children { - if !pack_time_window_contains( - &child.child_effective_not_before, - &child.child_effective_until, - validation_time, - ) { - return Err("child_time_gate_miss"); - } - } - Ok(()) -} - -fn pack_time_window_contains( - not_before: &PackTime, - until: &PackTime, - validation_time: time::OffsetDateTime, -) -> bool { - let Ok(not_before) = parse_snapshot_time_value(not_before) else { - return false; - }; - let Ok(until) = parse_snapshot_time_value(until) else { - return false; - }; - validation_time >= not_before && validation_time < until -} - -#[derive(Clone, Copy)] -struct ChildCertificateValidationCacheContext<'a> { - store: &'a RocksStore, - issuer_ca_sha256: [u8; 32], - ca_validation_context_digest: [u8; 32], - policy_fingerprint: [u8; 32], -} - -fn child_certificate_cache_key_sha256_hex( - child_cert_uri: &str, - child_cert_sha256: &[u8; 32], - issuer_ca_sha256: &[u8; 32], - ca_validation_context_digest: &[u8; 32], - policy_fingerprint: &[u8; 32], -) -> String { - let digest = hash_serialized_parts(&[ - ("version", b"child-certificate-cache-key-v1"), - ("child_cert_uri", child_cert_uri.as_bytes()), - ("child_cert_sha256", child_cert_sha256), - ("issuer_ca_sha256", issuer_ca_sha256), - // Keep the persisted cache-key label stable; this change only renames Rust identifiers. - ("parent_context_digest", ca_validation_context_digest), - ("policy_fingerprint", policy_fingerprint), - ]); - sha256_hex_from_32(&digest) -} - -#[derive(Clone, Debug)] -struct ChildCertificateCacheCandidate { - projection: Option, -} - -fn remember_child_certificate_cache_dirty_projection( - dirty_projections: &mut Option>, - projection: &ChildCertificateCacheProjection, -) { - if let Some(dirty_projections) = dirty_projections { - dirty_projections.insert(projection.cache_key_sha256_hex.clone(), projection.clone()); - } -} - -fn load_child_certificate_der_for_discovery<'a>( - file: &'a PackFile, - elapsed_nanos: &mut u64, - count: &mut u64, -) -> Result<&'a [u8], String> { - let started = std::time::Instant::now(); - let bytes = file - .bytes() - .map_err(|e| format!("child certificate bytes load failed: {e}"))?; - *elapsed_nanos = - elapsed_nanos.saturating_add(started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); - *count = count.saturating_add(1); - Ok(bytes) -} - -fn ca_certificate_der_for_validation<'a>( - ca: &'a CaInstanceHandle, - store: &RocksStore, - timing: Option<&TimingHandle>, -) -> Result, String> { - let started = std::time::Instant::now(); - let was_lazy = ca.ca_certificate_sha256_hex().is_some(); - let der = ca.ca_certificate_der(store)?; - if was_lazy { - let elapsed = started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64; - if let Some(timing) = timing { - timing.record_count("ca_certificate_lazy_load_count", 1); - timing.record_count("ca_certificate_lazy_load_bytes", der.len() as u64); - timing.record_phase_nanos("ca_certificate_lazy_load_total", elapsed); - } - } - Ok(der) -} - -fn child_certificate_cache_certificate_window( - child_not_before: time::OffsetDateTime, - child_not_after: time::OffsetDateTime, - issuer_not_before: time::OffsetDateTime, - issuer_not_after: time::OffsetDateTime, -) -> (PackTime, PackTime) { - let effective_not_before = child_not_before.max(issuer_not_before); - let effective_until = child_not_after.min(issuer_not_after); - ( - PackTime::from_utc_offset_datetime(effective_not_before), - PackTime::from_utc_offset_datetime(effective_until), - ) -} - -fn get_current_crl_sha256_hex( - crl_rsync_uri: &str, - crl_cache: &mut std::collections::HashMap, -) -> Option { - crl_cache - .get_mut(crl_rsync_uri) - .map(|entry| entry.current_sha256_hex().to_string()) -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum ChildCertificateCacheCrlGate { - Unchanged, - ChangedValid, - Expired, - Invalid, - Missing, -} - -#[derive(Default)] -struct ChildCertificateCacheCrlGateSet { - gates_by_uri_and_expected_hash: - std::collections::HashMap<(String, String), ChildCertificateCacheCrlGate>, -} - -impl ChildCertificateCacheCrlGateSet { - fn evaluate( - &mut self, - projection: &ChildCertificateCacheProjection, - crl_cache: &mut std::collections::HashMap, - issuer_ca_der: &[u8], - validation_time: time::OffsetDateTime, - ) -> (ChildCertificateCacheCrlGate, bool) { - let key = ( - projection.issuer_crl_uri.clone(), - projection.issuer_crl_sha256_hex.clone(), - ); - if let Some(gate) = self.gates_by_uri_and_expected_hash.get(&key) { - return (*gate, true); - } - - let gate = evaluate_child_certificate_cache_crl_gate( - projection, - crl_cache, - issuer_ca_der, - validation_time, - ); - self.gates_by_uri_and_expected_hash.insert(key, gate); - (gate, false) - } -} - -fn evaluate_child_certificate_cache_crl_gate( - projection: &ChildCertificateCacheProjection, - crl_cache: &mut std::collections::HashMap, - issuer_ca_der: &[u8], - validation_time: time::OffsetDateTime, -) -> ChildCertificateCacheCrlGate { - let Some(current_crl_hash) = get_current_crl_sha256_hex(&projection.issuer_crl_uri, crl_cache) - else { - return ChildCertificateCacheCrlGate::Missing; - }; - if current_crl_hash == projection.issuer_crl_sha256_hex { - let verified_crl = match ensure_issuer_crl_verified( - &projection.issuer_crl_uri, - crl_cache, - issuer_ca_der, - ) { - Ok(verified_crl) => verified_crl, - Err(_) => return ChildCertificateCacheCrlGate::Invalid, - }; - if !crl_valid_at_time_for_cache(&verified_crl.crl, validation_time) { - return ChildCertificateCacheCrlGate::Expired; - } - return ChildCertificateCacheCrlGate::Unchanged; - } - - let verified_crl = - match ensure_issuer_crl_verified(&projection.issuer_crl_uri, crl_cache, issuer_ca_der) { - Ok(verified_crl) => verified_crl, - Err(_) => return ChildCertificateCacheCrlGate::Invalid, - }; - if !crl_valid_at_time_for_cache(&verified_crl.crl, validation_time) { - return ChildCertificateCacheCrlGate::Expired; - } - ChildCertificateCacheCrlGate::ChangedValid -} - -fn crl_valid_at_time_for_cache( - crl: &crate::data_model::crl::RpkixCrl, - validation_time: time::OffsetDateTime, -) -> bool { - let this_update = crl.this_update.utc.to_offset(time::UtcOffset::UTC); - let next_update = crl.next_update.utc.to_offset(time::UtcOffset::UTC); - validation_time >= this_update && validation_time < next_update -} - -#[derive(Clone, Debug)] -struct VcirReuseProjection { - source: PublicationPointSource, - vcir: Option, - ccr_manifest_projection: Option, - snapshot: Option, - objects: crate::validation::objects::ObjectsOutput, - child_audits: Vec, - discovered_children: Vec, - warnings: Vec, -} - -fn discover_children_from_fresh_snapshot_with_audit( - issuer: &CaInstanceHandle, - publication_point: &P, - validation_time: time::OffsetDateTime, - timing: Option<&TimingHandle>, -) -> Result { - let issuer_ca_der = match &issuer.ca_certificate { - CaCertificateRef::InlineDer(bytes) => bytes.as_slice(), - CaCertificateRef::RepoBytes { .. } => { - return Err("lazy CA certificate requires store-backed child discovery".to_string()); - } - }; - let default_policy = Policy::default(); - discover_children_from_fresh_snapshot_with_audit_cached_with_issuer_der( - issuer, - issuer_ca_der, - publication_point, - validation_time, - timing, - &default_policy, - None, - ) -} - -fn discover_children_from_fresh_snapshot_with_audit_cached( - issuer: &CaInstanceHandle, - publication_point: &P, - validation_time: time::OffsetDateTime, - timing: Option<&TimingHandle>, - cache_context: Option>, -) -> Result { - let issuer_ca_der = match &issuer.ca_certificate { - CaCertificateRef::InlineDer(bytes) => bytes.as_slice(), - CaCertificateRef::RepoBytes { .. } => { - return Err("lazy CA certificate requires store-backed child discovery".to_string()); - } - }; - let default_policy = Policy::default(); - discover_children_from_fresh_snapshot_with_audit_cached_with_issuer_der( - issuer, - issuer_ca_der, - publication_point, - validation_time, - timing, - &default_policy, - cache_context, - ) -} - -fn discover_children_from_fresh_snapshot_with_audit_cached_with_issuer_der< - P: PublicationPointData, ->( - issuer: &CaInstanceHandle, - issuer_ca_der: &[u8], - publication_point: &P, - validation_time: time::OffsetDateTime, - timing: Option<&TimingHandle>, - policy: &Policy, - cache_context: Option>, -) -> Result { - let locked_files = publication_point.files(); - // Issuer CA is only required when we actually attempt to validate a subordinate CA. For some - // audit-only error paths (e.g., missing CRL in the snapshot), we still want discovery to succeed. - let issuer_ca_decode_error: Option; - let issuer_ca = match crate::data_model::rc::ResourceCertificate::decode_der(issuer_ca_der) { - Ok(v) => { - match v.validate_rfc6487_profile(crate::data_model::rc::ResourceCertificateRole::Ca) { - Ok(()) => { - issuer_ca_decode_error = None; - Some(v) - } - Err(e) => { - issuer_ca_decode_error = Some(format!( - "issuer CA profile validation failed: {e} (RFC 6487 §4.8)" - )); - None - } - } - } - Err(e) => { - issuer_ca_decode_error = Some(format!( - "issuer CA decode failed: {e} (RFC 5280 §4.1; RFC 6487 §4)" - )); - None - } - }; - - let issuer_spki_error: Option; - let issuer_spki: Option> = if let Some(ca) = issuer_ca.as_ref() { - match SubjectPublicKeyInfo::from_der(&ca.tbs.subject_public_key_info) { - Ok((rem, spki)) if rem.is_empty() => { - issuer_spki_error = None; - Some(spki) - } - Ok((rem, _)) => { - issuer_spki_error = Some(format!( - "trailing bytes after issuer SubjectPublicKeyInfo DER: {} bytes (DER; RFC 5280 §4.1.2.7)", - rem.len() - )); - None - } - Err(e) => { - issuer_spki_error = Some(format!( - "issuer SubjectPublicKeyInfo parse error: {e} (RFC 5280 §4.1.2.7)" - )); - None - } - } - } else { - issuer_spki_error = issuer_ca_decode_error.clone(); - None - }; - - let mut crl_cache: std::collections::HashMap = locked_files - .iter() - .filter(|f| f.rsync_uri.ends_with(".crl")) - .map(|f| -> Result<(String, CachedIssuerCrl), String> { - let bytes = f - .bytes_cloned() - .map_err(|e| format!("snapshot CRL bytes load failed: {e}"))?; - Ok(( - f.rsync_uri.clone(), - CachedIssuerCrl::Pending { - bytes, - sha256_hex: Some(sha256_hex_from_32(&f.sha256)), - }, - )) - }) - .collect::>()?; - - let mut out: Vec = Vec::new(); - let mut audits: Vec = Vec::new(); - let mut router_keys: Vec = Vec::new(); - let issuer_resources_index = IssuerEffectiveResourcesIndex::from_effective_resources( - issuer.effective_ip_resources.as_ref(), - issuer.effective_as_resources.as_ref(), - ) - .map_err(|e| format!("build issuer effective resources index failed: {e}"))?; - - let mut cer_seen: u64 = 0; - let mut ca_skipped_not_ca: u64 = 0; - let mut ca_ok: u64 = 0; - let mut ca_error: u64 = 0; - let mut router_ok: u64 = 0; - let mut router_error: u64 = 0; - let mut router_skipped_non_router: u64 = 0; - let mut crl_select_error: u64 = 0; - let mut uri_discovery_error: u64 = 0; - let mut child_cert_cache_lookup: u64 = 0; - let mut child_cert_cache_hit: u64 = 0; - let mut child_cert_cache_hit_ca: u64 = 0; - let mut child_cert_cache_hit_router: u64 = 0; - let mut child_cert_cache_miss_not_found: u64 = 0; - let mut child_cert_cache_miss_time_gate: u64 = 0; - let mut child_cert_cache_miss_crl_missing: u64 = 0; - let mut child_cert_cache_miss_crl_invalid: u64 = 0; - let mut child_cert_cache_miss_crl_expired: u64 = 0; - let mut child_cert_cache_miss_revoked: u64 = 0; - let mut child_cert_cache_crl_recheck_hit: u64 = 0; - let mut child_cert_cache_crl_gate_reused: u64 = 0; - let mut child_cert_cache_load_error: u64 = 0; - let mut child_cert_cache_write_ok: u64 = 0; - let mut child_cert_cache_write_error: u64 = 0; - - let mut select_crl_nanos: u64 = 0; - let mut child_decode_nanos: u64 = 0; - let mut validate_sub_ca_nanos: u64 = 0; - let mut validate_router_nanos: u64 = 0; - let mut uri_discovery_nanos: u64 = 0; - let mut enqueue_nanos: u64 = 0; - let mut child_cert_cache_lookup_nanos: u64 = 0; - let mut child_cert_cache_write_nanos: u64 = 0; - let child_cert_der_load_cache_hit_nanos: u64 = 0; - let mut child_cert_der_load_fresh_nanos: u64 = 0; - let child_cert_der_load_cache_hit_count: u64 = 0; - let mut child_cert_der_load_fresh_count: u64 = 0; - - let mut eff_ip_items_bucket_le_10: u64 = 0; - let mut eff_ip_items_bucket_le_100: u64 = 0; - let mut eff_ip_items_bucket_gt_100: u64 = 0; - let mut eff_as_items_bucket_le_10: u64 = 0; - let mut eff_as_items_bucket_le_100: u64 = 0; - let mut eff_as_items_bucket_gt_100: u64 = 0; - - let mut child_cache_candidates: HashMap = - HashMap::new(); - let mut child_cache_segment_keys = Vec::::new(); - let mut child_cache_segment_dirty_projections: Option< - HashMap, - > = None; - let mut child_cert_cache_batch_lookup_publication_points: u64 = 0; - let mut child_cert_cache_batch_lookup_entries: u64 = 0; - let mut child_cert_cache_batch_lookup_errors: u64 = 0; - let mut child_cert_cache_batch_lookup_nanos: u64 = 0; - let mut child_cert_cache_mmap_lookup_publication_points: u64 = 0; - let mut child_cert_cache_mmap_lookup_entries: u64 = 0; - let mut child_cert_cache_mmap_lookup_hits: u64 = 0; - let mut child_cert_cache_mmap_lookup_misses: u64 = 0; - let mut child_cert_cache_mmap_lookup_missing_segments: u64 = 0; - let mut child_cert_cache_mmap_lookup_errors: u64 = 0; - let mut child_cert_cache_mmap_lookup_file_bytes: u64 = 0; - let mut child_cert_cache_mmap_lookup_nanos: u64 = 0; - let mut child_cert_cache_mmap_write_entries: u64 = 0; - let mut child_cert_cache_mmap_write_errors: u64 = 0; - let mut child_cert_cache_mmap_write_file_bytes: u64 = 0; - let mut child_cert_cache_mmap_write_nanos: u64 = 0; - - if let Some(cache) = cache_context { - let mut uris = Vec::new(); - let mut keys = Vec::new(); - for f in locked_files - .iter() - .filter(|file| file.rsync_uri.ends_with(".cer")) - { - uris.push(f.rsync_uri.clone()); - keys.push(child_certificate_cache_key_sha256_hex( - &f.rsync_uri, - &f.sha256, - &cache.issuer_ca_sha256, - &cache.ca_validation_context_digest, - &cache.policy_fingerprint, - )); - } - let use_mmap_segment = keys.len() >= CHILD_CERTIFICATE_CACHE_MMAP_MIN_CER_COUNT; - if use_mmap_segment { - child_cache_segment_keys = keys.clone(); - child_cache_segment_dirty_projections = Some(HashMap::new()); - } - - let mut db_lookup_indices: Vec = Vec::new(); - if !keys.is_empty() { - if use_mmap_segment { - let mmap_lookup_started = std::time::Instant::now(); - child_cert_cache_mmap_lookup_publication_points = 1; - child_cert_cache_mmap_lookup_entries = keys.len() as u64; - match cache - .store - .get_child_certificate_cache_projections_mmap_segment( - publication_point.manifest_rsync_uri(), - &keys, - ) { - Ok(Some(lookup)) => { - child_cert_cache_mmap_lookup_hits = lookup.hits as u64; - child_cert_cache_mmap_lookup_misses = lookup.misses as u64; - child_cert_cache_mmap_lookup_file_bytes = lookup.file_bytes; - for (idx, ((uri, _key), projection)) in uris - .iter() - .zip(keys.iter()) - .zip(lookup.projections.into_iter()) - .enumerate() - { - if let Some(projection) = projection { - child_cache_candidates.insert( - uri.clone(), - ChildCertificateCacheCandidate { - projection: Some(projection), - }, - ); - } else { - db_lookup_indices.push(idx); - } - } - } - Ok(None) => { - child_cert_cache_mmap_lookup_missing_segments = 1; - db_lookup_indices.extend(0..keys.len()); - } - Err(_) => { - child_cert_cache_mmap_lookup_errors = - child_cert_cache_mmap_lookup_errors.saturating_add(keys.len() as u64); - db_lookup_indices.extend(0..keys.len()); - } - } - child_cert_cache_mmap_lookup_nanos = mmap_lookup_started - .elapsed() - .as_nanos() - .min(u128::from(u64::MAX)) - as u64; - } else { - db_lookup_indices.extend(0..keys.len()); - } - - if !db_lookup_indices.is_empty() { - let batch_lookup_started = std::time::Instant::now(); - child_cert_cache_batch_lookup_publication_points = 1; - child_cert_cache_batch_lookup_entries = db_lookup_indices.len() as u64; - let db_keys = db_lookup_indices - .iter() - .map(|idx| keys[*idx].clone()) - .collect::>(); - match cache - .store - .get_child_certificate_cache_projections_batch(&db_keys) - { - Ok(projections) => { - for (idx, projection) in db_lookup_indices - .iter() - .copied() - .zip(projections.into_iter()) - { - if let Some(projection) = projection { - remember_child_certificate_cache_dirty_projection( - &mut child_cache_segment_dirty_projections, - &projection, - ); - child_cache_candidates.insert( - uris[idx].clone(), - ChildCertificateCacheCandidate { - projection: Some(projection), - }, - ); - } - } - } - Err(_) => { - child_cert_cache_batch_lookup_errors = child_cert_cache_batch_lookup_errors - .saturating_add(child_cert_cache_batch_lookup_entries); - } - } - child_cert_cache_batch_lookup_nanos = child_cert_cache_batch_lookup_nanos - .saturating_add( - batch_lookup_started - .elapsed() - .as_nanos() - .min(u128::from(u64::MAX)) as u64, - ); - } - } - } - - fn bucketize(v: usize) -> u8 { - if v <= 10 { - 0 - } else if v <= 100 { - 1 - } else { - 2 - } - } - - fn ip_item_count(ip: Option<&crate::data_model::rc::IpResourceSet>) -> usize { - let Some(ip) = ip else { return 0 }; - ip.families - .iter() - .map(|f| match &f.choice { - crate::data_model::rc::IpAddressChoice::Inherit => 0usize, - crate::data_model::rc::IpAddressChoice::AddressesOrRanges(items) => items.len(), - }) - .sum() - } - - fn as_item_count(asr: Option<&crate::data_model::rc::AsResourceSet>) -> usize { - let Some(asr) = asr else { return 0 }; - let mut n = 0usize; - if let Some(c) = asr.asnum.as_ref() { - if let crate::data_model::rc::AsIdentifierChoice::AsIdsOrRanges(items) = c { - n = n.saturating_add(items.len()); - } - } - if let Some(c) = asr.rdi.as_ref() { - if let crate::data_model::rc::AsIdentifierChoice::AsIdsOrRanges(items) = c { - n = n.saturating_add(items.len()); - } - } - n - } - - let mut child_cert_crl_gate_set = ChildCertificateCacheCrlGateSet::default(); - - for f in locked_files { - if !f.rsync_uri.ends_with(".cer") { - continue; - } - cer_seen = cer_seen.saturating_add(1); - let child_cert_sha256_hex = sha256_hex_from_32(&f.sha256); - - if cache_context.is_some() { - let lookup_started = std::time::Instant::now(); - child_cert_cache_lookup = child_cert_cache_lookup.saturating_add(1); - let cache_lookup_failed = child_cert_cache_batch_lookup_errors > 0 - && !child_cache_candidates.contains_key(&f.rsync_uri); - if cache_lookup_failed { - child_cert_cache_load_error = child_cert_cache_load_error.saturating_add(1); - } else { - match child_cache_candidates - .get(&f.rsync_uri) - .and_then(|candidate| candidate.projection.as_ref()) - { - Some(projection) => { - let time_gate_ok = pack_time_window_contains( - &projection.effective_not_before, - &projection.effective_until, - validation_time, - ); - if !time_gate_ok { - child_cert_cache_miss_time_gate = - child_cert_cache_miss_time_gate.saturating_add(1); - } else { - let (crl_gate, gate_reused) = child_cert_crl_gate_set.evaluate( - projection, - &mut crl_cache, - issuer_ca_der, - validation_time, - ); - if gate_reused { - child_cert_cache_crl_gate_reused = - child_cert_cache_crl_gate_reused.saturating_add(1); - } - let mut crl_gate_allows_reuse = false; - match crl_gate { - ChildCertificateCacheCrlGate::Unchanged => { - crl_gate_allows_reuse = true; - } - ChildCertificateCacheCrlGate::ChangedValid => { - match ensure_issuer_crl_verified( - &projection.issuer_crl_uri, - &mut crl_cache, - issuer_ca_der, - ) { - Ok(verified_crl) => { - if verified_crl - .revoked_serials - .contains(&projection.child_cert_serial) - { - child_cert_cache_miss_revoked = - child_cert_cache_miss_revoked.saturating_add(1); - } else { - child_cert_cache_crl_recheck_hit = - child_cert_cache_crl_recheck_hit - .saturating_add(1); - crl_gate_allows_reuse = true; - } - } - Err(_) => { - child_cert_cache_miss_crl_invalid = - child_cert_cache_miss_crl_invalid.saturating_add(1); - } - } - } - ChildCertificateCacheCrlGate::Expired => { - child_cert_cache_miss_crl_expired = - child_cert_cache_miss_crl_expired.saturating_add(1); - } - ChildCertificateCacheCrlGate::Invalid => { - child_cert_cache_miss_crl_invalid = - child_cert_cache_miss_crl_invalid.saturating_add(1); - } - ChildCertificateCacheCrlGate::Missing => { - child_cert_cache_miss_crl_missing = - child_cert_cache_miss_crl_missing.saturating_add(1); - } - } - - if crl_gate_allows_reuse { - match &projection.payload { - ChildCertificateCachePayload::ChildCa { - child_manifest_rsync_uri, - child_ski, - child_rsync_base_uri, - child_publication_point_rsync_uri, - child_rrdp_notification_uri, - child_effective_ip_resources, - child_effective_as_resources, - .. - } => { - child_cert_cache_lookup_nanos = - child_cert_cache_lookup_nanos.saturating_add( - lookup_started - .elapsed() - .as_nanos() - .min(u128::from(u64::MAX)) - as u64, - ); - out.push(DiscoveredChildCaInstance { - handle: CaInstanceHandle { - depth: 0, - tal_id: issuer.tal_id.clone(), - parent_manifest_rsync_uri: Some( - issuer.manifest_rsync_uri.clone(), - ), - ca_certificate: CaCertificateRef::repo_bytes( - child_cert_sha256_hex.clone(), - ), - ca_certificate_rsync_uri: Some(f.rsync_uri.clone()), - effective_ip_resources: - child_effective_ip_resources.clone(), - effective_as_resources: - child_effective_as_resources.clone(), - rsync_base_uri: child_rsync_base_uri.clone(), - manifest_rsync_uri: child_manifest_rsync_uri - .clone(), - publication_point_rsync_uri: - child_publication_point_rsync_uri.clone(), - rrdp_notification_uri: child_rrdp_notification_uri - .clone(), - }, - discovered_from: crate::audit::DiscoveredFrom { - parent_manifest_rsync_uri: issuer - .manifest_rsync_uri - .clone(), - child_ca_certificate_rsync_uri: f.rsync_uri.clone(), - child_ca_certificate_sha256_hex: - child_cert_sha256_hex.clone(), - }, - child_entry_projection: Some( - DiscoveredChildEntryProjection { - child_ski: child_ski.clone(), - }, - ), - }); - ca_ok = ca_ok.saturating_add(1); - child_cert_cache_hit = - child_cert_cache_hit.saturating_add(1); - child_cert_cache_hit_ca = - child_cert_cache_hit_ca.saturating_add(1); - audits.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: child_cert_sha256_hex.clone(), - kind: AuditObjectKind::Certificate, - result: AuditObjectResult::Ok, - detail: Some( - "restored subordinate CA discovery from child certificate validation cache" - .to_string(), - ), - }); - continue; - } - ChildCertificateCachePayload::Router { - router_keys: cached_keys, - } => { - for cached_key in cached_keys { - router_keys.push(RouterKeyPayload { - as_id: cached_key.as_id, - ski: cached_key.ski.clone(), - spki_der: cached_key.spki_der.clone(), - source_object_uri: f.rsync_uri.clone(), - source_object_hash: child_cert_sha256_hex.clone(), - source_ee_cert_hash: child_cert_sha256_hex.clone(), - item_effective_until: cached_key - .item_effective_until - .clone(), - }); - } - router_ok = router_ok.saturating_add(1); - child_cert_cache_hit = - child_cert_cache_hit.saturating_add(1); - child_cert_cache_hit_router = - child_cert_cache_hit_router.saturating_add(1); - audits.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: child_cert_sha256_hex.clone(), - kind: AuditObjectKind::RouterCertificate, - result: AuditObjectResult::Ok, - detail: Some( - "restored BGPsec router certificate from child certificate validation cache" - .to_string(), - ), - }); - child_cert_cache_lookup_nanos = - child_cert_cache_lookup_nanos.saturating_add( - lookup_started - .elapsed() - .as_nanos() - .min(u128::from(u64::MAX)) - as u64, - ); - continue; - } - } - } - } - } - None => { - child_cert_cache_miss_not_found = - child_cert_cache_miss_not_found.saturating_add(1); - } - } - } - child_cert_cache_lookup_nanos = child_cert_cache_lookup_nanos.saturating_add( - lookup_started - .elapsed() - .as_nanos() - .min(u128::from(u64::MAX)) as u64, - ); - } - - let child_der = load_child_certificate_der_for_discovery( - f, - &mut child_cert_der_load_fresh_nanos, - &mut child_cert_der_load_fresh_count, - )?; - - let tdecode = std::time::Instant::now(); - let child_cert = match crate::data_model::rc::ResourceCertificate::decode_der(child_der) { - Ok(v) => v, - Err(e) => { - ca_error = ca_error.saturating_add(1); - audits.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::Certificate, - result: AuditObjectResult::Error, - detail: Some(format!("child certificate decode failed: {e}")), - }); - continue; - } - }; - child_decode_nanos = child_decode_nanos - .saturating_add(tdecode.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); - - let t0 = std::time::Instant::now(); - let issuer_crl_uri = match select_issuer_crl_uri_for_child(&child_cert, &crl_cache) { - Ok(v) => v.to_string(), - Err(e) => { - crl_select_error = crl_select_error.saturating_add(1); - audits.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::Certificate, - result: AuditObjectResult::Error, - detail: Some(format!( - "cannot select issuer CRL for child certificate: {e}" - )), - }); - continue; - } - }; - select_crl_nanos = select_crl_nanos - .saturating_add(t0.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); - - let t1 = std::time::Instant::now(); - let Some(issuer_ca_ref) = issuer_ca.as_ref() else { - ca_error = ca_error.saturating_add(1); - audits.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::Certificate, - result: AuditObjectResult::Error, - detail: Some( - issuer_ca_decode_error - .clone() - .unwrap_or_else(|| "issuer CA decode failed".to_string()), - ), - }); - continue; - }; - let Some(issuer_spki_ref) = issuer_spki.as_ref() else { - ca_error = ca_error.saturating_add(1); - audits.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::Certificate, - result: AuditObjectResult::Error, - detail: Some( - issuer_spki_error - .clone() - .unwrap_or_else(|| "issuer SubjectPublicKeyInfo unavailable".to_string()), - ), - }); - continue; - }; - let validated = match validate_subordinate_ca_cert_with_cached_issuer( - child_der, - child_cert, - issuer_ca_der, - issuer_ca_ref, - issuer_spki_ref, - issuer_crl_uri.as_str(), - &mut crl_cache, - issuer.ca_certificate_rsync_uri.as_deref(), - issuer.effective_ip_resources.as_ref(), - issuer.effective_as_resources.as_ref(), - &issuer_resources_index, - validation_time, - policy.resource_validation_mode, - ) { - Ok(v) => v, - Err(CaPathError::ChildNotCa) => { - let tr = std::time::Instant::now(); - let router_result = match ensure_issuer_crl_verified( - issuer_crl_uri.as_str(), - &mut crl_cache, - issuer_ca_der, - ) { - Ok(verified_crl) => { - BgpsecRouterCertificate::validate_path_with_prevalidated_issuer( - child_der, - issuer_ca_ref, - issuer_spki_ref, - &verified_crl.crl, - &verified_crl.revoked_serials, - issuer.ca_certificate_rsync_uri.as_deref(), - Some(issuer_crl_uri.as_str()), - validation_time, - ) - } - Err(err) => { - validate_router_nanos = validate_router_nanos.saturating_add( - tr.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64, - ); - router_error = router_error.saturating_add(1); - audits.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::RouterCertificate, - result: AuditObjectResult::Error, - detail: Some(format!( - "router certificate issuer CRL validation failed: {err}" - )), - }); - continue; - } - }; - validate_router_nanos = validate_router_nanos - .saturating_add(tr.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); - - match router_result { - Ok(router) => { - if let Some(ta_constraints) = policy.ta_constraints.for_tal(&issuer.tal_id) - { - if let Err(error) = - ta_constraints.validate_ee_certificate(&router.resource_cert) - { - router_error = router_error.saturating_add(1); - audits.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::RouterCertificate, - result: AuditObjectResult::Error, - detail: Some(format!( - "router certificate violates TA constraints: {error}" - )), - }); - continue; - } - } - let router_asns = match router_asns_for_resource_mode( - &router.asns, - issuer.effective_as_resources.as_ref(), - policy.resource_validation_mode, - ) { - Ok(v) => v, - Err(detail) => { - router_error = router_error.saturating_add(1); - audits.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::RouterCertificate, - result: AuditObjectResult::Error, - detail: Some(format!( - "router certificate AS resource validation failed: {detail}" - )), - }); - continue; - } - }; - router_ok = router_ok.saturating_add(1); - let source_object_hash = sha256_hex_from_32(&f.sha256); - let item_effective_until = PackTime::from_utc_offset_datetime( - router.resource_cert.tbs.validity_not_after, - ); - for as_id in &router_asns { - router_keys.push(RouterKeyPayload { - as_id: *as_id, - ski: router.subject_key_identifier.clone(), - spki_der: router.spki_der.clone(), - source_object_uri: f.rsync_uri.clone(), - source_object_hash: source_object_hash.clone(), - source_ee_cert_hash: source_object_hash.clone(), - item_effective_until: item_effective_until.clone(), - }); - } - if let Some(cache) = cache_context { - let write_started = std::time::Instant::now(); - if let Ok(verified_crl) = ensure_issuer_crl_verified( - issuer_crl_uri.as_str(), - &mut crl_cache, - issuer_ca_der, - ) { - let (effective_not_before, effective_until) = - child_certificate_cache_certificate_window( - router.resource_cert.tbs.validity_not_before, - router.resource_cert.tbs.validity_not_after, - issuer_ca_ref.tbs.validity_not_before, - issuer_ca_ref.tbs.validity_not_after, - ); - let cache_key_sha256_hex = child_certificate_cache_key_sha256_hex( - &f.rsync_uri, - &f.sha256, - &cache.issuer_ca_sha256, - &cache.ca_validation_context_digest, - &cache.policy_fingerprint, - ); - let projection = ChildCertificateCacheProjection { - schema_version: CHILD_CERTIFICATE_CACHE_SCHEMA_VERSION, - algorithm_version: CHILD_CERTIFICATE_CACHE_ALGORITHM_VERSION, - cache_key_sha256_hex, - child_cert_uri: f.rsync_uri.clone(), - child_cert_sha256_hex: child_cert_sha256_hex.clone(), - child_cert_serial: BigUnsigned::from_biguint( - &router.resource_cert.tbs.serial_number, - ) - .bytes_be, - issuer_ca_sha256_hex: sha256_hex_from_32( - &cache.issuer_ca_sha256, - ), - issuer_crl_uri: issuer_crl_uri.clone(), - issuer_crl_sha256_hex: verified_crl.sha256_hex.clone(), - ca_validation_context_digest: cache - .ca_validation_context_digest, - validation_policy_fingerprint: cache.policy_fingerprint, - effective_not_before, - effective_until, - payload: ChildCertificateCachePayload::Router { - router_keys: router_asns - .iter() - .map(|as_id| ChildCertificateCacheRouterKeyProjection { - as_id: *as_id, - ski: router.subject_key_identifier.clone(), - spki_der: router.spki_der.clone(), - item_effective_until: item_effective_until.clone(), - }) - .collect(), - }, - }; - if cache - .store - .put_child_certificate_cache_projection(&projection) - .is_ok() - { - remember_child_certificate_cache_dirty_projection( - &mut child_cache_segment_dirty_projections, - &projection, - ); - child_cert_cache_write_ok = - child_cert_cache_write_ok.saturating_add(1); - } else { - child_cert_cache_write_error = - child_cert_cache_write_error.saturating_add(1); - } - } else { - child_cert_cache_write_error = - child_cert_cache_write_error.saturating_add(1); - } - child_cert_cache_write_nanos = child_cert_cache_write_nanos - .saturating_add( - write_started.elapsed().as_nanos().min(u128::from(u64::MAX)) - as u64, - ); - } - audits.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::RouterCertificate, - result: AuditObjectResult::Ok, - detail: Some( - "validated BGPsec router certificate (RFC 8209); no child CA instance enqueued" - .to_string(), - ), - }); - } - Err(err) if is_non_router_certificate(&err) => { - ca_skipped_not_ca = ca_skipped_not_ca.saturating_add(1); - router_skipped_non_router = router_skipped_non_router.saturating_add(1); - audits.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::Certificate, - result: AuditObjectResult::Skipped, - detail: Some( - "skipped: not a CA resource certificate or BGPsec router certificate" - .to_string(), - ), - }); - } - Err(err) => { - router_error = router_error.saturating_add(1); - audits.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::RouterCertificate, - result: AuditObjectResult::Error, - detail: Some(format!("router certificate validation failed: {err}")), - }); - } - } - continue; - } - Err(e) => { - ca_error = ca_error.saturating_add(1); - audits.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::Certificate, - result: AuditObjectResult::Error, - detail: Some(format!("child CA validation failed: {e}")), - }); - continue; - } - }; - validate_sub_ca_nanos = validate_sub_ca_nanos - .saturating_add(t1.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); - - if !validated.resource_warnings.is_empty() { - audits.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::Certificate, - result: AuditObjectResult::Ok, - detail: Some(format!( - "resource validation warning ({:?}): {}", - policy.resource_validation_mode, - validated.resource_warnings.summary() - )), - }); - } - - let eff_ip_items = ip_item_count(validated.effective_ip_resources.as_ref()); - match bucketize(eff_ip_items) { - 0 => eff_ip_items_bucket_le_10 = eff_ip_items_bucket_le_10.saturating_add(1), - 1 => eff_ip_items_bucket_le_100 = eff_ip_items_bucket_le_100.saturating_add(1), - _ => eff_ip_items_bucket_gt_100 = eff_ip_items_bucket_gt_100.saturating_add(1), - } - let eff_as_items = as_item_count(validated.effective_as_resources.as_ref()); - match bucketize(eff_as_items) { - 0 => eff_as_items_bucket_le_10 = eff_as_items_bucket_le_10.saturating_add(1), - 1 => eff_as_items_bucket_le_100 = eff_as_items_bucket_le_100.saturating_add(1), - _ => eff_as_items_bucket_gt_100 = eff_as_items_bucket_gt_100.saturating_add(1), - } - - let t2 = std::time::Instant::now(); - let uris = match ca_instance_uris_from_ca_certificate(&validated.child_ca) { - Ok(v) => v, - Err(e) => { - uri_discovery_error = uri_discovery_error.saturating_add(1); - audits.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::Certificate, - result: AuditObjectResult::Error, - detail: Some(format!("CA instance URI discovery failed: {e}")), - }); - continue; - } - }; - uri_discovery_nanos = uri_discovery_nanos - .saturating_add(t2.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); - - let t3 = std::time::Instant::now(); - let child_rsync_base_uri = uris.rsync_base_uri.clone(); - let child_manifest_rsync_uri = uris.manifest_rsync_uri.clone(); - let child_publication_point_rsync_uri = uris.publication_point_rsync_uri.clone(); - let child_rrdp_notification_uri = uris.rrdp_notification_uri.clone(); - out.push(DiscoveredChildCaInstance { - handle: CaInstanceHandle { - depth: 0, - tal_id: issuer.tal_id.clone(), - parent_manifest_rsync_uri: Some(issuer.manifest_rsync_uri.clone()), - ca_certificate: CaCertificateRef::inline_der(child_der.to_vec()), - ca_certificate_rsync_uri: Some(f.rsync_uri.clone()), - effective_ip_resources: validated.effective_ip_resources.clone(), - effective_as_resources: validated.effective_as_resources.clone(), - rsync_base_uri: child_rsync_base_uri.clone(), - manifest_rsync_uri: child_manifest_rsync_uri.clone(), - publication_point_rsync_uri: child_publication_point_rsync_uri.clone(), - rrdp_notification_uri: child_rrdp_notification_uri.clone(), - }, - discovered_from: crate::audit::DiscoveredFrom { - parent_manifest_rsync_uri: issuer.manifest_rsync_uri.clone(), - child_ca_certificate_rsync_uri: f.rsync_uri.clone(), - child_ca_certificate_sha256_hex: child_cert_sha256_hex.clone(), - }, - child_entry_projection: validated - .child_ca - .tbs - .extensions - .subject_key_identifier - .as_ref() - .map(|child_ski| DiscoveredChildEntryProjection { - child_ski: hex::encode(child_ski), - }), - }); - enqueue_nanos = - enqueue_nanos.saturating_add(t3.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); - - if let Some(cache) = cache_context { - let write_started = std::time::Instant::now(); - if let Ok(verified_crl) = - ensure_issuer_crl_verified(issuer_crl_uri.as_str(), &mut crl_cache, issuer_ca_der) - { - if let Some(child_ski) = validated - .child_ca - .tbs - .extensions - .subject_key_identifier - .as_ref() - { - let (effective_not_before, effective_until) = - child_certificate_cache_certificate_window( - validated.child_ca.tbs.validity_not_before, - validated.child_ca.tbs.validity_not_after, - issuer_ca_ref.tbs.validity_not_before, - issuer_ca_ref.tbs.validity_not_after, - ); - let cache_key_sha256_hex = child_certificate_cache_key_sha256_hex( - &f.rsync_uri, - &f.sha256, - &cache.issuer_ca_sha256, - &cache.ca_validation_context_digest, - &cache.policy_fingerprint, - ); - let projection = ChildCertificateCacheProjection { - schema_version: CHILD_CERTIFICATE_CACHE_SCHEMA_VERSION, - algorithm_version: CHILD_CERTIFICATE_CACHE_ALGORITHM_VERSION, - cache_key_sha256_hex, - child_cert_uri: f.rsync_uri.clone(), - child_cert_sha256_hex: child_cert_sha256_hex.clone(), - child_cert_serial: BigUnsigned::from_biguint( - &validated.child_ca.tbs.serial_number, - ) - .bytes_be, - issuer_ca_sha256_hex: sha256_hex_from_32(&cache.issuer_ca_sha256), - issuer_crl_uri: issuer_crl_uri.clone(), - issuer_crl_sha256_hex: verified_crl.sha256_hex.clone(), - ca_validation_context_digest: cache.ca_validation_context_digest, - validation_policy_fingerprint: cache.policy_fingerprint, - effective_not_before, - effective_until, - payload: ChildCertificateCachePayload::ChildCa { - child_manifest_rsync_uri, - child_ski: hex::encode(child_ski), - child_rsync_base_uri, - child_publication_point_rsync_uri, - child_rrdp_notification_uri, - child_effective_ip_resources: validated.effective_ip_resources.clone(), - child_effective_as_resources: validated.effective_as_resources.clone(), - }, - }; - if cache - .store - .put_child_certificate_cache_projection(&projection) - .is_ok() - { - remember_child_certificate_cache_dirty_projection( - &mut child_cache_segment_dirty_projections, - &projection, - ); - child_cert_cache_write_ok = child_cert_cache_write_ok.saturating_add(1); - } else { - child_cert_cache_write_error = - child_cert_cache_write_error.saturating_add(1); - } - } else { - child_cert_cache_write_error = child_cert_cache_write_error.saturating_add(1); - } - } else { - child_cert_cache_write_error = child_cert_cache_write_error.saturating_add(1); - } - child_cert_cache_write_nanos = - child_cert_cache_write_nanos.saturating_add( - write_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64, - ); - } - - ca_ok = ca_ok.saturating_add(1); - audits.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: child_cert_sha256_hex.clone(), - kind: AuditObjectKind::Certificate, - result: AuditObjectResult::Ok, - detail: Some("validated subordinate CA certificate; enqueued CA instance".to_string()), - }); - } - - if let (Some(cache), Some(dirty_projections)) = - (cache_context, child_cache_segment_dirty_projections.take()) - { - if !dirty_projections.is_empty() { - let write_started = std::time::Instant::now(); - let projections = dirty_projections.into_values().collect::>(); - child_cert_cache_mmap_write_entries = projections.len() as u64; - match cache - .store - .write_child_certificate_cache_mmap_segment_overlay( - publication_point.manifest_rsync_uri(), - &child_cache_segment_keys, - &projections, - ) { - Ok(stats) => { - child_cert_cache_mmap_write_file_bytes = stats.file_bytes; - } - Err(_) => { - child_cert_cache_mmap_write_errors = - child_cert_cache_mmap_write_errors.saturating_add(1); - } - } - child_cert_cache_mmap_write_nanos = - write_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64; - } - } - - if let Some(t) = timing { - t.record_count("child_cer_seen", cer_seen); - t.record_count("child_ca_ok", ca_ok); - t.record_count("child_ca_error", ca_error); - t.record_count("child_ca_skipped_not_ca", ca_skipped_not_ca); - t.record_count("child_router_ok", router_ok); - t.record_count("child_router_error", router_error); - t.record_count("child_router_skipped_non_router", router_skipped_non_router); - t.record_count("child_crl_select_error", crl_select_error); - t.record_count("child_uri_discovery_error", uri_discovery_error); - t.record_count("child_certificate_cache_lookup", child_cert_cache_lookup); - t.record_count("child_certificate_cache_hit", child_cert_cache_hit); - t.record_count("child_certificate_cache_hit_ca", child_cert_cache_hit_ca); - t.record_count( - "child_certificate_cache_hit_router", - child_cert_cache_hit_router, - ); - t.record_count( - "child_certificate_cache_miss_not_found", - child_cert_cache_miss_not_found, - ); - t.record_count( - "child_certificate_cache_miss_time_gate", - child_cert_cache_miss_time_gate, - ); - t.record_count( - "child_certificate_cache_miss_crl_missing", - child_cert_cache_miss_crl_missing, - ); - t.record_count( - "child_certificate_cache_miss_crl_invalid", - child_cert_cache_miss_crl_invalid, - ); - t.record_count( - "child_certificate_cache_miss_crl_expired", - child_cert_cache_miss_crl_expired, - ); - t.record_count( - "child_certificate_cache_miss_revoked", - child_cert_cache_miss_revoked, - ); - t.record_count( - "child_certificate_cache_crl_recheck_hit", - child_cert_cache_crl_recheck_hit, - ); - t.record_count( - "child_certificate_cache_crl_gate_reused", - child_cert_cache_crl_gate_reused, - ); - t.record_count( - "child_certificate_cache_load_error", - child_cert_cache_load_error, - ); - t.record_count( - "child_certificate_cache_write_ok", - child_cert_cache_write_ok, - ); - t.record_count( - "child_certificate_cache_write_error", - child_cert_cache_write_error, - ); - t.record_count( - "child_certificate_cache_batch_lookup_publication_points", - child_cert_cache_batch_lookup_publication_points, - ); - t.record_count( - "child_certificate_cache_batch_lookup_entries", - child_cert_cache_batch_lookup_entries, - ); - t.record_count( - "child_certificate_cache_batch_lookup_errors", - child_cert_cache_batch_lookup_errors, - ); - t.record_count( - "child_certificate_cache_mmap_lookup_publication_points", - child_cert_cache_mmap_lookup_publication_points, - ); - t.record_count( - "child_certificate_cache_mmap_lookup_entries", - child_cert_cache_mmap_lookup_entries, - ); - t.record_count( - "child_certificate_cache_mmap_lookup_hits", - child_cert_cache_mmap_lookup_hits, - ); - t.record_count( - "child_certificate_cache_mmap_lookup_misses", - child_cert_cache_mmap_lookup_misses, - ); - t.record_count( - "child_certificate_cache_mmap_lookup_missing_segments", - child_cert_cache_mmap_lookup_missing_segments, - ); - t.record_count( - "child_certificate_cache_mmap_lookup_errors", - child_cert_cache_mmap_lookup_errors, - ); - t.record_count( - "child_certificate_cache_mmap_lookup_file_bytes", - child_cert_cache_mmap_lookup_file_bytes, - ); - t.record_count( - "child_certificate_cache_mmap_write_entries", - child_cert_cache_mmap_write_entries, - ); - t.record_count( - "child_certificate_cache_mmap_write_errors", - child_cert_cache_mmap_write_errors, - ); - t.record_count( - "child_certificate_cache_mmap_write_file_bytes", - child_cert_cache_mmap_write_file_bytes, - ); - t.record_count( - "child_certificate_der_load_cache_hit_count", - child_cert_der_load_cache_hit_count, - ); - t.record_count( - "child_certificate_der_load_fresh_count", - child_cert_der_load_fresh_count, - ); - - t.record_count("child_effective_ip_items_le_10", eff_ip_items_bucket_le_10); - t.record_count( - "child_effective_ip_items_le_100", - eff_ip_items_bucket_le_100, - ); - t.record_count( - "child_effective_ip_items_gt_100", - eff_ip_items_bucket_gt_100, - ); - t.record_count("child_effective_as_items_le_10", eff_as_items_bucket_le_10); - t.record_count( - "child_effective_as_items_le_100", - eff_as_items_bucket_le_100, - ); - t.record_count( - "child_effective_as_items_gt_100", - eff_as_items_bucket_gt_100, - ); - - t.record_phase_nanos("child_select_issuer_crl_total", select_crl_nanos); - t.record_phase_nanos("child_decode_certificate_total", child_decode_nanos); - t.record_phase_nanos("child_validate_subordinate_total", validate_sub_ca_nanos); - t.record_phase_nanos( - "child_validate_router_certificate_total", - validate_router_nanos, - ); - t.record_phase_nanos("child_ca_instance_uri_discovery_total", uri_discovery_nanos); - t.record_phase_nanos("child_enqueue_total", enqueue_nanos); - t.record_phase_nanos( - "child_certificate_cache_lookup_total", - child_cert_cache_lookup_nanos, - ); - t.record_phase_nanos( - "child_certificate_cache_write_total", - child_cert_cache_write_nanos, - ); - t.record_phase_nanos( - "child_certificate_cache_batch_lookup_total", - child_cert_cache_batch_lookup_nanos, - ); - t.record_phase_nanos( - "child_certificate_cache_mmap_lookup_total", - child_cert_cache_mmap_lookup_nanos, - ); - t.record_phase_nanos( - "child_certificate_cache_mmap_write_total", - child_cert_cache_mmap_write_nanos, - ); - t.record_phase_nanos( - "child_certificate_der_load_cache_hit_total", - child_cert_der_load_cache_hit_nanos, - ); - t.record_phase_nanos( - "child_certificate_der_load_fresh_total", - child_cert_der_load_fresh_nanos, - ); - t.record_phase_nanos( - "child_certificate_der_load_total", - child_cert_der_load_cache_hit_nanos.saturating_add(child_cert_der_load_fresh_nanos), - ); - } - - Ok(ChildDiscoveryOutput { - children: out, - audits, - router_keys, - }) -} - -fn is_non_router_certificate(err: &BgpsecRouterCertificatePathError) -> bool { - matches!( - err, - BgpsecRouterCertificatePathError::Decode(BgpsecRouterCertificateDecodeError::Validate( - BgpsecRouterCertificateProfileError::NotEe - | BgpsecRouterCertificateProfileError::MissingExtendedKeyUsage - | BgpsecRouterCertificateProfileError::MissingBgpsecRouterEku - )) - ) -} - -fn router_asns_for_resource_mode( - router_asns: &[u32], - issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, - mode: ResourceValidationMode, -) -> Result, String> { - let Some(issuer_effective_as) = issuer_effective_as else { - return Err("issuer has no effective AS resources".to_string()); - }; - - match mode { - ResourceValidationMode::Rfc6487 => { - let outside: Vec = router_asns - .iter() - .copied() - .filter(|asn| !as_resource_set_contains_asn(issuer_effective_as, *asn)) - .collect(); - if outside.is_empty() { - Ok(router_asns.to_vec()) - } else { - Err(format!( - "router AS resources are not a subset of issuer effective AS resources: {outside:?}" - )) - } - } - ResourceValidationMode::ValidationUpdate03 => { - let filtered: Vec = router_asns - .iter() - .copied() - .filter(|asn| as_resource_set_contains_asn(issuer_effective_as, *asn)) - .collect(); - if filtered.is_empty() { - Err("router AS resources have empty validated resource set".to_string()) - } else { - Ok(filtered) - } - } - } -} - -fn as_resource_set_contains_asn( - resources: &crate::data_model::rc::AsResourceSet, - asn: u32, -) -> bool { - let Some(choice) = resources.asnum.as_ref() else { - return false; - }; - match choice { - crate::data_model::rc::AsIdentifierChoice::Inherit => false, - crate::data_model::rc::AsIdentifierChoice::AsIdsOrRanges(items) => { - items.iter().any(|item| match item { - crate::data_model::rc::AsIdOrRange::Id(id) => *id == asn, - crate::data_model::rc::AsIdOrRange::Range { min, max } => { - *min <= asn && asn <= *max - } - }) - } - } -} - -fn select_issuer_crl_uri_for_child<'a>( - child: &'a crate::data_model::rc::ResourceCertificate, - crl_cache: &std::collections::HashMap, -) -> Result<&'a str, String> { - if crl_cache.is_empty() { - return Err( - "no CRL available in publication point snapshot (cannot validate certificates) (RFC 9286 §7; RFC 6487 §4.8.6)" - .to_string(), - ); - } - let Some(crldp_uris) = child.tbs.extensions.crl_distribution_points_uris.as_ref() else { - return Err( - "child certificate CRLDistributionPoints missing (RFC 6487 §4.8.6)".to_string(), - ); - }; - - for u in crldp_uris { - let s = u.as_str(); - if crl_cache.contains_key(s) { - return Ok(s); - } - } - - Err(format!( - "CRL referenced by child certificate CRLDistributionPoints not found in publication point snapshot: {} (RFC 6487 §4.8.6; RFC 9286 §4.2.1)", - crldp_uris - .iter() - .map(|u| u.as_str()) - .collect::>() - .join(", ") - )) -} - -fn ensure_issuer_crl_verified<'a>( - crl_rsync_uri: &str, - crl_cache: &'a mut std::collections::HashMap, - issuer_ca_der: &[u8], -) -> Result<&'a VerifiedIssuerCrl, CaPathError> { - let entry = crl_cache - .get_mut(crl_rsync_uri) - .expect("CRL must exist in cache"); - match entry { - CachedIssuerCrl::Ok(v) => Ok(v), - CachedIssuerCrl::Pending { - bytes, - sha256_hex: cached_sha256_hex, - } => { - let der = std::mem::take(bytes); - let crl = crate::data_model::crl::RpkixCrl::decode_der(&der)?; - crl.verify_signature_with_issuer_certificate_der(issuer_ca_der)?; - let sha256_hex = cached_sha256_hex - .take() - .unwrap_or_else(|| crate::audit::sha256_hex(&der)); - - let mut revoked_serials: std::collections::HashSet> = - std::collections::HashSet::with_capacity(crl.revoked_certs.len()); - for rc in &crl.revoked_certs { - revoked_serials.insert(rc.serial_number.bytes_be.clone()); - } - - *entry = CachedIssuerCrl::Ok(VerifiedIssuerCrl { - crl, - revoked_serials, - sha256_hex, - }); - match entry { - CachedIssuerCrl::Ok(v) => Ok(v), - _ => unreachable!(), - } - } - } -} - -fn validate_subordinate_ca_cert_with_cached_issuer( - child_ca_der: &[u8], - child_ca: crate::data_model::rc::ResourceCertificate, - issuer_ca_der: &[u8], - issuer_ca: &crate::data_model::rc::ResourceCertificate, - issuer_spki: &SubjectPublicKeyInfo<'_>, - issuer_crl_rsync_uri: &str, - crl_cache: &mut std::collections::HashMap, - issuer_ca_rsync_uri: Option<&str>, - issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, - issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, - issuer_resources_index: &IssuerEffectiveResourcesIndex, - validation_time: time::OffsetDateTime, - resource_validation_mode: crate::policy::ResourceValidationMode, -) -> Result { - let verified_crl = ensure_issuer_crl_verified(issuer_crl_rsync_uri, crl_cache, issuer_ca_der)?; - - validate_subordinate_ca_cert_with_prevalidated_issuer_and_resources( - child_ca_der, - child_ca, - issuer_ca, - issuer_spki, - &verified_crl.crl, - &verified_crl.revoked_serials, - issuer_ca_rsync_uri, - issuer_crl_rsync_uri, - issuer_effective_ip, - issuer_effective_as, - issuer_resources_index, - validation_time, - resource_validation_mode, - ) -} - -#[cfg(test)] -fn select_issuer_crl_from_snapshot<'a>( - child_cert_der: &[u8], - pack: &'a PublicationPointSnapshot, -) -> Result<(&'a str, &'a [u8]), String> { - let child = crate::data_model::rc::ResourceCertificate::decode_der(child_cert_der) - .map_err(|e| format!("child certificate decode failed: {e}"))?; - let Some(crldp_uris) = child.tbs.extensions.crl_distribution_points_uris.as_ref() else { - return Err( - "child certificate CRLDistributionPoints missing (RFC 6487 §4.8.6)".to_string(), - ); - }; - - for u in crldp_uris { - let s = u.as_str(); - if let Some(f) = pack.files.iter().find(|f| f.rsync_uri == s) { - let bytes = f - .bytes() - .map_err(|e| format!("snapshot CRL bytes load failed: {e}"))?; - return Ok((f.rsync_uri.as_str(), bytes)); - } - } - - Err(format!( - "CRL referenced by child certificate CRLDistributionPoints not found in publication point snapshot: {} (RFC 6487 §4.8.6; RFC 9286 §4.2.1)", - crldp_uris - .iter() - .map(|u| u.as_str()) - .collect::>() - .join(", ") - )) -} - -fn kind_from_rsync_uri(uri: &str) -> AuditObjectKind { - if uri.ends_with(".crl") { - AuditObjectKind::Crl - } else if uri.ends_with(".cer") { - AuditObjectKind::Certificate - } else if uri.ends_with(".roa") { - AuditObjectKind::Roa - } else if uri.ends_with(".asa") { - AuditObjectKind::Aspa - } else { - AuditObjectKind::Other - } -} - -fn source_label(source: PublicationPointSource) -> String { - match source { - PublicationPointSource::Fresh => "fresh".to_string(), - PublicationPointSource::PublicationPointCache => "publication_point_cache".to_string(), - PublicationPointSource::VcirCurrentInstance => "vcir_current_instance".to_string(), - PublicationPointSource::FailedFetchNoCache => "failed_fetch_no_cache".to_string(), - } -} - -fn repo_sync_phase_label(phase: crate::sync::repo::RepoSyncPhase) -> &'static str { - match phase { - crate::sync::repo::RepoSyncPhase::RrdpOk => "rrdp_ok", - crate::sync::repo::RepoSyncPhase::RrdpFailedRsyncOk => "rrdp_failed_rsync_ok", - crate::sync::repo::RepoSyncPhase::RsyncOnlyOk => "rsync_only_ok", - crate::sync::repo::RepoSyncPhase::ReplayRrdpOk => "replay_rrdp_ok", - crate::sync::repo::RepoSyncPhase::ReplayRsyncOk => "replay_rsync_ok", - crate::sync::repo::RepoSyncPhase::ReplayNoopRrdp => "replay_noop_rrdp", - crate::sync::repo::RepoSyncPhase::ReplayNoopRsync => "replay_noop_rsync", - } -} - -fn repo_sync_failure_phase_label( - attempted_rrdp: bool, - original_notification_uri: Option<&str>, - effective_notification_uri: Option<&str>, -) -> &'static str { - if attempted_rrdp && original_notification_uri.is_some() && effective_notification_uri.is_some() - { - "rrdp_failed_rsync_failed" - } else if attempted_rrdp - && original_notification_uri.is_some() - && effective_notification_uri.is_none() - { - "rsync_only_failed_after_rrdp_dedup" - } else { - "rsync_only_failed" - } -} - -fn terminal_state_label(source: PublicationPointSource) -> &'static str { - match source { - PublicationPointSource::Fresh => "fresh", - PublicationPointSource::PublicationPointCache => "publication_point_cache", - PublicationPointSource::VcirCurrentInstance => "fallback_current_instance", - PublicationPointSource::FailedFetchNoCache => "failed_no_cache", - } -} - -fn repo_sync_source_label(source: crate::sync::repo::RepoSyncSource) -> &'static str { - match source { - crate::sync::repo::RepoSyncSource::Rrdp => "rrdp", - crate::sync::repo::RepoSyncSource::Rsync => "rsync", - } -} - -fn effective_repo_sync_duration_ms( - elapsed_ms: u64, - runtime_reported_duration_ms: Option, - repo_sync_ok: bool, -) -> u64 { - if repo_sync_ok { - return elapsed_ms; - } - runtime_reported_duration_ms - .map(|runtime_ms| elapsed_ms.max(runtime_ms)) - .unwrap_or(elapsed_ms) -} - -fn kind_from_vcir_artifact_kind(kind: VcirArtifactKind) -> AuditObjectKind { - match kind { - VcirArtifactKind::Mft => AuditObjectKind::Manifest, - VcirArtifactKind::Crl => AuditObjectKind::Crl, - VcirArtifactKind::Cer => AuditObjectKind::Certificate, - VcirArtifactKind::Roa => AuditObjectKind::Roa, - VcirArtifactKind::Aspa => AuditObjectKind::Aspa, - VcirArtifactKind::Gbr | VcirArtifactKind::Tal | VcirArtifactKind::Other => { - AuditObjectKind::Other - } - } -} - -fn audit_result_from_vcir_status(status: VcirArtifactValidationStatus) -> AuditObjectResult { - match status { - VcirArtifactValidationStatus::Accepted => AuditObjectResult::Ok, - VcirArtifactValidationStatus::Rejected => AuditObjectResult::Error, - VcirArtifactValidationStatus::WarningOnly => AuditObjectResult::Skipped, - } -} - -/// Fallback detail for cached artifacts rejected by an earlier run whose cache -/// entry predates reject-reason recording (field added 2026-07-27). -const CACHED_REJECT_REASON_NOT_RECORDED: &str = - "rejected in an earlier validation run (reject reason not recorded in cache)"; - -/// Detail for an audit entry rebuilt from a cached artifact: the real reject -/// reason when the cache recorded one, an explicit fallback for legacy cache -/// entries, and `None` for non-rejected artifacts. -fn audit_detail_from_vcir_status( - status: VcirArtifactValidationStatus, - reject_reason: Option<&str>, -) -> Option { - match status { - VcirArtifactValidationStatus::Rejected => Some( - reject_reason - .map(str::to_string) - .unwrap_or_else(|| CACHED_REJECT_REASON_NOT_RECORDED.to_string()), - ), - VcirArtifactValidationStatus::Accepted | VcirArtifactValidationStatus::WarningOnly => None, - } -} - -fn build_publication_point_audit_from_snapshot( - ca: &CaInstanceHandle, - source: PublicationPointSource, - repo_sync_source: Option<&str>, - repo_sync_phase: Option<&str>, - repo_sync_duration_ms: Option, - repo_sync_error: Option<&str>, - pack: &PublicationPointSnapshot, - runner_warnings: &[Warning], - objects: &crate::validation::objects::ObjectsOutput, - child_audits: &[ObjectAuditEntry], -) -> PublicationPointAudit { - use crate::data_model::crl::RpkixCrl; - use std::collections::HashMap; - - let locked_files = &pack.files; - let mut audit_by_uri: HashMap = HashMap::new(); - for f in locked_files { - audit_by_uri.insert( - f.rsync_uri.clone(), - ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: kind_from_rsync_uri(&f.rsync_uri), - result: AuditObjectResult::Skipped, - detail: Some("skipped: not processed in stage2".to_string()), - }, - ); - } - - for f in locked_files { - if !f.rsync_uri.ends_with(".crl") { - continue; - } - let ok = f - .bytes() - .ok() - .and_then(|bytes| RpkixCrl::decode_der(bytes).ok()) - .is_some(); - audit_by_uri.insert( - f.rsync_uri.clone(), - ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: AuditObjectKind::Crl, - result: if ok { - AuditObjectResult::Ok - } else { - AuditObjectResult::Error - }, - detail: if ok { - None - } else { - Some("CRL decode failed".to_string()) - }, - }, - ); - } - - for e in child_audits { - audit_by_uri.insert(e.rsync_uri.clone(), e.clone()); - } - for e in &objects.audit { - audit_by_uri.insert(e.rsync_uri.clone(), e.clone()); - } - - let mut objects_out: Vec = Vec::with_capacity(pack.files.len() + 1); - objects_out.push(ObjectAuditEntry { - rsync_uri: pack.manifest_rsync_uri.clone(), - sha256_hex: sha256_hex(&pack.manifest_bytes), - kind: AuditObjectKind::Manifest, - result: AuditObjectResult::Ok, - detail: None, - }); - for f in locked_files { - if let Some(e) = audit_by_uri.remove(&f.rsync_uri) { - objects_out.push(e); - } else { - objects_out.push(ObjectAuditEntry { - rsync_uri: f.rsync_uri.clone(), - sha256_hex: sha256_hex_from_32(&f.sha256), - kind: kind_from_rsync_uri(&f.rsync_uri), - result: AuditObjectResult::Skipped, - detail: Some("skipped: no audit entry".to_string()), - }); - } - } - - let mut warnings = Vec::new(); - warnings.extend(runner_warnings.iter().map(AuditWarning::from)); - warnings.extend(objects.warnings.iter().map(AuditWarning::from)); - - PublicationPointAudit { - node_id: None, - parent_node_id: None, - discovered_from: None, - rsync_base_uri: ca.rsync_base_uri.clone(), - manifest_rsync_uri: ca.manifest_rsync_uri.clone(), - publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(), - rrdp_notification_uri: ca.rrdp_notification_uri.clone(), - source: source_label(source), - repo_sync_source: repo_sync_source.map(ToString::to_string), - repo_sync_phase: repo_sync_phase.map(ToString::to_string), - repo_sync_duration_ms, - repo_sync_error: repo_sync_error.map(ToString::to_string), - repo_terminal_state: terminal_state_label(source).to_string(), - this_update_rfc3339_utc: pack.this_update.rfc3339_utc.clone(), - next_update_rfc3339_utc: pack.next_update.rfc3339_utc.clone(), - verified_at_rfc3339_utc: pack.verified_at.rfc3339_utc.clone(), - warnings, - objects: objects_out, - } -} - -fn build_publication_point_audit_from_vcir( - ca: &CaInstanceHandle, - source: PublicationPointSource, - repo_sync_source: Option<&str>, - repo_sync_phase: Option<&str>, - repo_sync_duration_ms: Option, - repo_sync_error: Option<&str>, - vcir: Option<&ValidatedCaInstanceResult>, - pack: Option<&PublicationPointSnapshot>, - runner_warnings: &[Warning], - objects: &crate::validation::objects::ObjectsOutput, - child_audits: &[ObjectAuditEntry], - fresh_failure_audits: &[ObjectAuditEntry], -) -> PublicationPointAudit { - if let Some(pack) = pack { - return build_publication_point_audit_from_snapshot( - ca, - source, - repo_sync_source, - repo_sync_phase, - repo_sync_duration_ms, - repo_sync_error, - pack, - runner_warnings, - objects, - child_audits, - ); - } - - let mut warnings = Vec::new(); - warnings.extend(runner_warnings.iter().map(AuditWarning::from)); - warnings.extend(objects.warnings.iter().map(AuditWarning::from)); - - let Some(vcir) = vcir else { - return PublicationPointAudit { - node_id: None, - parent_node_id: None, - discovered_from: None, - rsync_base_uri: ca.rsync_base_uri.clone(), - manifest_rsync_uri: ca.manifest_rsync_uri.clone(), - publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(), - rrdp_notification_uri: ca.rrdp_notification_uri.clone(), - source: source_label(source), - repo_sync_source: repo_sync_source.map(ToString::to_string), - repo_sync_phase: repo_sync_phase.map(ToString::to_string), - repo_sync_duration_ms, - repo_sync_error: repo_sync_error.map(ToString::to_string), - repo_terminal_state: terminal_state_label(source).to_string(), - this_update_rfc3339_utc: String::new(), - next_update_rfc3339_utc: String::new(), - verified_at_rfc3339_utc: String::new(), - warnings, - objects: fresh_failure_audits.to_vec(), - }; - }; - - if source == PublicationPointSource::FailedFetchNoCache { - let mut objects_out = Vec::with_capacity( - objects.audit.len() + child_audits.len() + fresh_failure_audits.len(), - ); - objects_out.extend(child_audits.iter().cloned()); - objects_out.extend(objects.audit.iter().cloned()); - objects_out.extend(fresh_failure_audits.iter().cloned()); - return PublicationPointAudit { - node_id: None, - parent_node_id: None, - discovered_from: None, - rsync_base_uri: ca.rsync_base_uri.clone(), - manifest_rsync_uri: ca.manifest_rsync_uri.clone(), - publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(), - rrdp_notification_uri: ca.rrdp_notification_uri.clone(), - source: source_label(source), - repo_sync_source: repo_sync_source.map(ToString::to_string), - repo_sync_phase: repo_sync_phase.map(ToString::to_string), - repo_sync_duration_ms, - repo_sync_error: repo_sync_error.map(ToString::to_string), - repo_terminal_state: terminal_state_label(source).to_string(), - this_update_rfc3339_utc: vcir - .validated_manifest_meta - .validated_manifest_this_update - .rfc3339_utc - .clone(), - next_update_rfc3339_utc: vcir - .validated_manifest_meta - .validated_manifest_next_update - .rfc3339_utc - .clone(), - verified_at_rfc3339_utc: vcir.last_successful_validation_time.rfc3339_utc.clone(), - warnings, - objects: objects_out, - }; - } - - let mut audit_by_uri: HashMap = HashMap::new(); - for artifact in &vcir.related_artifacts { - let Some(uri) = artifact.uri.as_ref() else { - continue; - }; - audit_by_uri.insert( - uri.clone(), - ObjectAuditEntry { - rsync_uri: uri.clone(), - sha256_hex: artifact.sha256.clone(), - kind: kind_from_vcir_artifact_kind(artifact.artifact_kind), - result: audit_result_from_vcir_status(artifact.validation_status), - detail: audit_detail_from_vcir_status( - artifact.validation_status, - artifact.reject_reason.as_deref(), - ), - }, - ); - } - for e in child_audits { - audit_by_uri.insert(e.rsync_uri.clone(), e.clone()); - } - for e in &objects.audit { - audit_by_uri.insert(e.rsync_uri.clone(), e.clone()); - } - - let mut ordered_uris: Vec = vcir - .related_artifacts - .iter() - .filter_map(|artifact| artifact.uri.clone()) - .collect(); - ordered_uris.sort(); - ordered_uris.dedup(); - - let mut objects_out: Vec = Vec::new(); - if let Some(entry) = audit_by_uri.remove(&vcir.current_manifest_rsync_uri) { - objects_out.push(entry); - } else { - objects_out.push(ObjectAuditEntry { - rsync_uri: vcir.current_manifest_rsync_uri.clone(), - sha256_hex: vcir - .related_artifacts - .iter() - .find(|artifact| { - artifact.artifact_role == VcirArtifactRole::Manifest - && artifact.uri.as_deref() == Some(vcir.current_manifest_rsync_uri.as_str()) - }) - .map(|artifact| artifact.sha256.clone()) - .unwrap_or_default(), - kind: AuditObjectKind::Manifest, - result: AuditObjectResult::Ok, - detail: None, - }); - } - - for uri in ordered_uris { - if uri == vcir.current_manifest_rsync_uri { - continue; - } - if let Some(entry) = audit_by_uri.remove(&uri) { - objects_out.push(entry); - } - } - - let mut audit_objects = objects_out.clone(); - audit_objects.extend(fresh_failure_audits.iter().cloned()); - - PublicationPointAudit { - node_id: None, - parent_node_id: None, - discovered_from: None, - rsync_base_uri: ca.rsync_base_uri.clone(), - manifest_rsync_uri: ca.manifest_rsync_uri.clone(), - publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(), - rrdp_notification_uri: ca.rrdp_notification_uri.clone(), - source: source_label(source), - repo_sync_source: repo_sync_source.map(ToString::to_string), - repo_sync_phase: repo_sync_phase.map(ToString::to_string), - repo_sync_duration_ms, - repo_sync_error: repo_sync_error.map(ToString::to_string), - repo_terminal_state: terminal_state_label(source).to_string(), - this_update_rfc3339_utc: vcir - .validated_manifest_meta - .validated_manifest_this_update - .rfc3339_utc - .clone(), - next_update_rfc3339_utc: vcir - .validated_manifest_meta - .validated_manifest_next_update - .rfc3339_utc - .clone(), - verified_at_rfc3339_utc: vcir.last_successful_validation_time.rfc3339_utc.clone(), - warnings, - objects: audit_objects, - } -} - -fn build_publication_point_audit_from_publication_point_cache_projection( - ca: &CaInstanceHandle, - source: PublicationPointSource, - repo_sync_source: Option<&str>, - repo_sync_phase: Option<&str>, - repo_sync_duration_ms: Option, - repo_sync_error: Option<&str>, - projection: &PublicationPointCacheProjection, - validation_time: time::OffsetDateTime, - runner_warnings: &[Warning], - objects: &crate::validation::objects::ObjectsOutput, - child_audits: &[ObjectAuditEntry], -) -> PublicationPointAudit { - let mut warnings = Vec::new(); - warnings.extend(runner_warnings.iter().map(AuditWarning::from)); - warnings.extend(objects.warnings.iter().map(AuditWarning::from)); - - let mut audit_by_uri: HashMap = HashMap::new(); - for artifact in &projection.related_objects { - let Some(uri) = artifact.uri.as_ref() else { - continue; - }; - audit_by_uri.insert( - uri.clone(), - ObjectAuditEntry { - rsync_uri: uri.clone(), - sha256_hex: artifact.sha256.clone(), - kind: kind_from_vcir_artifact_kind(artifact.artifact_kind), - result: audit_result_from_vcir_status(artifact.validation_status), - detail: audit_detail_from_vcir_status( - artifact.validation_status, - artifact.reject_reason.as_deref(), - ), - }, - ); - } - for entry in child_audits { - audit_by_uri.insert(entry.rsync_uri.clone(), entry.clone()); - } - for entry in &objects.audit { - audit_by_uri.insert(entry.rsync_uri.clone(), entry.clone()); - } - - let mut ordered_uris: Vec = projection - .related_objects - .iter() - .filter_map(|artifact| artifact.uri.clone()) - .collect(); - ordered_uris.sort(); - ordered_uris.dedup(); - - let mut objects_out: Vec = Vec::with_capacity(ordered_uris.len().max(1)); - if let Some(entry) = audit_by_uri.remove(&projection.manifest_rsync_uri) { - objects_out.push(entry); - } else { - objects_out.push(ObjectAuditEntry { - rsync_uri: projection.manifest_rsync_uri.clone(), - sha256_hex: hex::encode(projection.manifest_sha256), - kind: AuditObjectKind::Manifest, - result: AuditObjectResult::Ok, - detail: None, - }); - } - - for uri in ordered_uris { - if uri == projection.manifest_rsync_uri { - continue; - } - if let Some(entry) = audit_by_uri.remove(&uri) { - objects_out.push(entry); - } - } - - PublicationPointAudit { - node_id: None, - parent_node_id: None, - discovered_from: None, - rsync_base_uri: ca.rsync_base_uri.clone(), - manifest_rsync_uri: ca.manifest_rsync_uri.clone(), - publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(), - rrdp_notification_uri: ca.rrdp_notification_uri.clone(), - source: source_label(source), - repo_sync_source: repo_sync_source.map(ToString::to_string), - repo_sync_phase: repo_sync_phase.map(ToString::to_string), - repo_sync_duration_ms, - repo_sync_error: repo_sync_error.map(ToString::to_string), - repo_terminal_state: terminal_state_label(source).to_string(), - this_update_rfc3339_utc: projection.manifest_this_update.rfc3339_utc.clone(), - next_update_rfc3339_utc: projection.manifest_next_update.rfc3339_utc.clone(), - verified_at_rfc3339_utc: PackTime::from_utc_offset_datetime(validation_time).rfc3339_utc, - warnings, - objects: objects_out, - } -} - -fn parse_snapshot_time_value(pack_time: &PackTime) -> Result { - time::OffsetDateTime::parse( - &pack_time.rfc3339_utc, - &time::format_description::well_known::Rfc3339, - ) - .map_err(|e| format!("invalid RFC3339 time '{}': {e}", pack_time.rfc3339_utc)) -} - -fn empty_objects_output() -> crate::validation::objects::ObjectsOutput { - crate::validation::objects::ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings: Vec::new(), - stats: crate::validation::objects::ObjectsStats::default(), - audit: Vec::new(), - roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), - roa_cache_object_meta: Vec::new(), - } -} - -fn reuse_ccr_manifest_projection_from_vcir( - ca: &CaInstanceHandle, - vcir: &ValidatedCaInstanceResult, -) -> Result { - if vcir.ccr_manifest_projection.manifest_rsync_uri != ca.manifest_rsync_uri { - return Err(format!( - "vcir CCR manifest projection URI mismatch: expected {}, got {}", - ca.manifest_rsync_uri, vcir.ccr_manifest_projection.manifest_rsync_uri - )); - } - Ok(vcir.ccr_manifest_projection.clone()) -} - -fn project_current_instance_vcir_on_failed_fetch( - store: &RocksStore, - ca: &CaInstanceHandle, - fresh_err: &ManifestFreshError, - policy: &Policy, - validation_time: time::OffsetDateTime, -) -> Result { - let mut warnings = Vec::new(); - - let Some(vcir) = store - .get_vcir(&ca.manifest_rsync_uri) - .map_err(|e| format!("load VCIR failed: {e}"))? - else { - return Ok(failed_fetch_no_cache_projection( - ca, - fresh_err, - None, - "no latest validated result for current CA instance; no cached output reused", - )); - }; - - if !vcir.audit_summary.failed_fetch_eligible { - return Ok(failed_fetch_no_cache_projection( - ca, - fresh_err, - Some(vcir), - "latest VCIR is not marked failed-fetch eligible; no cached output reused", - )); - } - - let reuse_identity = match store.get_vcir_failed_fetch_reuse_identity(&ca.manifest_rsync_uri) { - Ok(Some(identity)) => identity, - Ok(None) => { - return Ok(failed_fetch_no_cache_projection( - ca, - fresh_err, - Some(vcir), - "latest VCIR reuse identity is missing; no cached output or child reused", - )); - } - Err(error) => { - return Ok(failed_fetch_no_cache_projection( - ca, - fresh_err, - Some(vcir), - &format!( - "latest VCIR reuse identity is invalid ({error}); no cached output or child reused" - ), - )); - } - }; - if !failed_fetch_reuse_identity_matches_current( - &reuse_identity, - &vcir, - ca, - policy, - validation_time, - ) { - return Ok(failed_fetch_no_cache_projection( - ca, - fresh_err, - Some(vcir), - "latest VCIR reuse identity does not match the current CA context or time window; no cached output or child reused", - )); - } - - let ccr_manifest_projection = reuse_ccr_manifest_projection_from_vcir(ca, &vcir)?; - if fresh_err.should_warn_when_current_instance_reused() { - warnings.push( - Warning::new(format!("manifest failed fetch: {fresh_err}")) - .with_rfc_refs(&[RfcRef("RFC 9286 §6.6")]) - .with_context(&ca.manifest_rsync_uri), - ); - } - // Current-instance reuse is fully described by VCIR projections; rebuilding a - // byte-backed snapshot here only duplicates repo-byte I/O and creates warning noise. - let snapshot = None; - let objects = build_objects_output_from_vcir(&vcir, validation_time, &mut warnings); - let (discovered_children, child_audits) = - restore_children_from_vcir(store, ca, &vcir, &mut warnings); - - Ok(VcirReuseProjection { - source: PublicationPointSource::VcirCurrentInstance, - vcir: Some(vcir), - ccr_manifest_projection: Some(ccr_manifest_projection), - snapshot, - objects, - child_audits, - discovered_children, - warnings, - }) -} - -fn failed_fetch_no_cache_projection( - ca: &CaInstanceHandle, - fresh_err: &ManifestFreshError, - vcir: Option, - reason: &str, -) -> VcirReuseProjection { - let warnings = vec![ - Warning::new(format!("manifest failed fetch: {fresh_err}")) - .with_rfc_refs(&[RfcRef("RFC 9286 §6.6")]) - .with_context(&ca.manifest_rsync_uri), - Warning::new(reason) - .with_rfc_refs(&[RfcRef("RFC 9286 §6.6")]) - .with_context(&ca.manifest_rsync_uri), - ]; - VcirReuseProjection { - source: PublicationPointSource::FailedFetchNoCache, - vcir, - ccr_manifest_projection: None, - snapshot: None, - objects: empty_objects_output(), - child_audits: Vec::new(), - discovered_children: Vec::new(), - warnings, - } -} - -fn failed_fetch_reuse_identity_for_fresh_result( - ca: &CaInstanceHandle, - policy: &Policy, - validation_time: time::OffsetDateTime, - effective_until: PackTime, -) -> Result { - let current_ca_sha256 = ca.ca_certificate_sha256_32().ok_or_else(|| { - "current CA certificate hash unavailable for VCIR reuse identity".to_string() - })?; - let identity = VcirFailedFetchReuseIdentity { - current_ca_sha256, - ta_context_digest: ta_context_digest_for_ca(ca), - ca_validation_context_digest: ca_validation_context_digest_for_ca(ca), - policy_fingerprint: publication_point_cache_policy_fingerprint(policy), - effective_not_before: PackTime::from_utc_offset_datetime(validation_time), - effective_until, - }; - identity - .validate_internal() - .map_err(|error| error.to_string())?; - Ok(identity) -} - -fn failed_fetch_reuse_identity_matches_current( - cached: &VcirFailedFetchReuseIdentity, - vcir: &ValidatedCaInstanceResult, - ca: &CaInstanceHandle, - policy: &Policy, - validation_time: time::OffsetDateTime, -) -> bool { - let Some(current_ca_sha256) = ca.ca_certificate_sha256_32() else { - return false; - }; - cached.current_ca_sha256 == current_ca_sha256 - && cached.ta_context_digest == ta_context_digest_for_ca(ca) - && cached.ca_validation_context_digest == ca_validation_context_digest_for_ca(ca) - && cached.policy_fingerprint == publication_point_cache_policy_fingerprint(policy) - && cached.effective_until == vcir.instance_gate.instance_effective_until - && cached.contains_validation_time(validation_time) -} - -#[cfg(test)] -fn reconstruct_snapshot_from_vcir( - store: &RocksStore, - ca: &CaInstanceHandle, - vcir: &ValidatedCaInstanceResult, - warnings: &mut Vec, -) -> Option { - let manifest_artifact = vcir.related_artifacts.iter().find(|artifact| { - artifact.artifact_role == VcirArtifactRole::Manifest - && artifact.uri.as_deref() == Some(ca.manifest_rsync_uri.as_str()) - })?; - let manifest_bytes = match store.get_blob_bytes(&manifest_artifact.sha256) { - Ok(Some(bytes)) => bytes, - Ok(None) => { - warnings.push( - Warning::new("manifest raw bytes missing for VCIR audit reconstruction") - .with_context(&ca.manifest_rsync_uri), - ); - return None; - } - Err(e) => { - warnings.push( - Warning::new(format!( - "manifest raw bytes load failed for VCIR audit reconstruction: {e}" - )) - .with_context(&ca.manifest_rsync_uri), - ); - return None; - } - }; - - let mut seen = HashSet::new(); - let mut files = Vec::new(); - for artifact in &vcir.related_artifacts { - let Some(uri) = artifact.uri.as_ref() else { - continue; - }; - if artifact.artifact_role == VcirArtifactRole::Manifest - || artifact.artifact_role == VcirArtifactRole::IssuerCert - || artifact.artifact_role == VcirArtifactRole::TrustAnchorCert - || artifact.artifact_role == VcirArtifactRole::Tal - { - continue; - } - if !seen.insert(uri.clone()) { - continue; - } - match store.get_blob_bytes(&artifact.sha256) { - Ok(Some(bytes)) => files.push(PackFile::from_bytes_compute_sha256(uri, bytes)), - Ok(None) => warnings.push( - Warning::new("related artifact raw bytes missing for VCIR audit reconstruction") - .with_context(uri), - ), - Err(e) => warnings.push( - Warning::new(format!( - "related artifact raw bytes load failed for VCIR audit reconstruction: {e}" - )) - .with_context(uri), - ), - } - } - - Some(PublicationPointSnapshot { - format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, - publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(), - manifest_rsync_uri: ca.manifest_rsync_uri.clone(), - manifest_number_be: vcir - .validated_manifest_meta - .validated_manifest_number - .clone(), - this_update: vcir - .validated_manifest_meta - .validated_manifest_this_update - .clone(), - next_update: vcir - .validated_manifest_meta - .validated_manifest_next_update - .clone(), - verified_at: vcir.last_successful_validation_time.clone(), - manifest_bytes, - files, - }) -} - -fn audit_kind_for_vcir_output_type(output_type: VcirOutputType) -> AuditObjectKind { - match output_type { - VcirOutputType::Vrp => AuditObjectKind::Roa, - VcirOutputType::Aspa => AuditObjectKind::Aspa, - VcirOutputType::RouterKey => AuditObjectKind::RouterCertificate, - } -} - -fn build_objects_output_from_vcir( - vcir: &ValidatedCaInstanceResult, - validation_time: time::OffsetDateTime, - warnings: &mut Vec, -) -> crate::validation::objects::ObjectsOutput { - let mut output = empty_objects_output(); - let mut audit_by_uri: HashMap = HashMap::new(); - let mut roa_total: HashSet = HashSet::new(); - let mut aspa_total: HashSet = HashSet::new(); - let mut roa_ok: HashSet = HashSet::new(); - let mut aspa_ok: HashSet = HashSet::new(); - - for artifact in &vcir.related_artifacts { - if artifact.artifact_role != VcirArtifactRole::SignedObject { - continue; - } - if let Some(uri) = artifact.uri.as_ref() { - match artifact.artifact_kind { - VcirArtifactKind::Roa => { - roa_total.insert(uri.clone()); - } - VcirArtifactKind::Aspa => { - aspa_total.insert(uri.clone()); - } - _ => {} - } - } - } - - for local in &vcir.local_outputs { - let effective_until = match parse_snapshot_time_value(&local.item_effective_until) { - Ok(v) => v, - Err(e) => { - warnings.push( - Warning::new(format!( - "cached local output has invalid item_effective_until: {e}" - )) - .with_context(&local.source_object_uri), - ); - audit_by_uri.insert( - local.source_object_uri.clone(), - ObjectAuditEntry { - rsync_uri: local.source_object_uri.clone(), - sha256_hex: local.source_object_hash_hex(), - kind: audit_kind_for_vcir_output_type(local.output_type), - result: AuditObjectResult::Error, - detail: Some( - "cached local output has invalid item_effective_until".to_string(), - ), - }, - ); - continue; - } - }; - if validation_time > effective_until { - audit_by_uri - .entry(local.source_object_uri.clone()) - .or_insert_with(|| ObjectAuditEntry { - rsync_uri: local.source_object_uri.clone(), - sha256_hex: local.source_object_hash_hex(), - kind: audit_kind_for_vcir_output_type(local.output_type), - result: AuditObjectResult::Skipped, - detail: Some("skipped: cached local output expired".to_string()), - }); - continue; - } - - match local.output_type { - VcirOutputType::Vrp => match parse_vcir_vrp_output(local) { - Ok(vrp) => { - roa_ok.insert(local.source_object_uri.clone()); - output.vrps.push(vrp); - audit_by_uri.insert( - local.source_object_uri.clone(), - ObjectAuditEntry { - rsync_uri: local.source_object_uri.clone(), - sha256_hex: local.source_object_hash_hex(), - kind: AuditObjectKind::Roa, - result: AuditObjectResult::Ok, - detail: None, - }, - ); - } - Err(e) => { - warnings.push( - Warning::new(format!("cached ROA local output parse failed: {e}")) - .with_context(&local.source_object_uri), - ); - audit_by_uri.insert( - local.source_object_uri.clone(), - ObjectAuditEntry { - rsync_uri: local.source_object_uri.clone(), - sha256_hex: local.source_object_hash_hex(), - kind: AuditObjectKind::Roa, - result: AuditObjectResult::Error, - detail: Some(format!("cached ROA local output parse failed: {e}")), - }, - ); - } - }, - VcirOutputType::Aspa => match parse_vcir_aspa_output(local) { - Ok(aspa) => { - aspa_ok.insert(local.source_object_uri.clone()); - output.aspas.push(aspa); - audit_by_uri.insert( - local.source_object_uri.clone(), - ObjectAuditEntry { - rsync_uri: local.source_object_uri.clone(), - sha256_hex: local.source_object_hash_hex(), - kind: AuditObjectKind::Aspa, - result: AuditObjectResult::Ok, - detail: None, - }, - ); - } - Err(e) => { - warnings.push( - Warning::new(format!("cached ASPA local output parse failed: {e}")) - .with_context(&local.source_object_uri), - ); - audit_by_uri.insert( - local.source_object_uri.clone(), - ObjectAuditEntry { - rsync_uri: local.source_object_uri.clone(), - sha256_hex: local.source_object_hash_hex(), - kind: AuditObjectKind::Aspa, - result: AuditObjectResult::Error, - detail: Some(format!("cached ASPA local output parse failed: {e}")), - }, - ); - } - }, - VcirOutputType::RouterKey => match parse_vcir_router_key_output(local) { - Ok(router_key) => { - output.router_keys.push(router_key); - audit_by_uri.insert( - local.source_object_uri.clone(), - ObjectAuditEntry { - rsync_uri: local.source_object_uri.clone(), - sha256_hex: local.source_object_hash_hex(), - kind: AuditObjectKind::RouterCertificate, - result: AuditObjectResult::Ok, - detail: Some("cached Router Key local output restored".to_string()), - }, - ); - } - Err(e) => { - warnings.push( - Warning::new(format!("cached Router Key local output parse failed: {e}")) - .with_context(&local.source_object_uri), - ); - audit_by_uri.insert( - local.source_object_uri.clone(), - ObjectAuditEntry { - rsync_uri: local.source_object_uri.clone(), - sha256_hex: local.source_object_hash_hex(), - kind: AuditObjectKind::RouterCertificate, - result: AuditObjectResult::Error, - detail: Some(format!( - "cached Router Key local output parse failed: {e}" - )), - }, - ); - } - }, - } - } - - output.stats.roa_total = roa_total.len(); - output.stats.roa_ok = roa_ok.len(); - output.stats.aspa_total = aspa_total.len(); - output.stats.aspa_ok = aspa_ok.len(); - let mut audit: Vec<_> = audit_by_uri.into_values().collect(); - audit.sort_by(|a, b| a.rsync_uri.cmp(&b.rsync_uri)); - output.audit = audit; - output -} - -fn build_objects_output_from_publication_point_cache_projection( - projection: &PublicationPointCacheProjection, - validation_time: time::OffsetDateTime, - warnings: &mut Vec, -) -> crate::validation::objects::ObjectsOutput { - let mut output = empty_objects_output(); - let mut audit_by_uri: HashMap = HashMap::new(); - let mut roa_total: HashSet = HashSet::new(); - let mut aspa_total: HashSet = HashSet::new(); - let mut roa_ok: HashSet = HashSet::new(); - let mut aspa_ok: HashSet = HashSet::new(); - - for artifact in &projection.related_objects { - if artifact.artifact_role != VcirArtifactRole::SignedObject { - continue; - } - if let Some(uri) = artifact.uri.as_ref() { - match artifact.artifact_kind { - VcirArtifactKind::Roa => { - roa_total.insert(uri.clone()); - } - VcirArtifactKind::Aspa => { - aspa_total.insert(uri.clone()); - } - _ => {} - } - } - } - - for projected in &projection.outputs { - let effective_until = match parse_snapshot_time_value(&projected.item_effective_until) { - Ok(value) => value, - Err(err) => { - warnings.push( - Warning::new(format!( - "publication-point cached local output has invalid item_effective_until: {err}" - )) - .with_context(&projected.source_object_uri), - ); - audit_by_uri.insert( - projected.source_object_uri.clone(), - ObjectAuditEntry { - rsync_uri: projected.source_object_uri.clone(), - sha256_hex: hex::encode(projected.source_object_hash), - kind: audit_kind_for_vcir_output_type(projected.output_type), - result: AuditObjectResult::Error, - detail: Some( - "publication-point cached local output has invalid item_effective_until" - .to_string(), - ), - }, - ); - continue; - } - }; - if validation_time > effective_until { - audit_by_uri - .entry(projected.source_object_uri.clone()) - .or_insert_with(|| ObjectAuditEntry { - rsync_uri: projected.source_object_uri.clone(), - sha256_hex: hex::encode(projected.source_object_hash), - kind: audit_kind_for_vcir_output_type(projected.output_type), - result: AuditObjectResult::Skipped, - detail: Some( - "skipped: publication-point cached local output expired".to_string(), - ), - }); - continue; - } - - match projected.output_type { - VcirOutputType::Vrp => match parse_publication_point_cache_vrp_output(projected) { - Ok(vrp) => { - roa_ok.insert(projected.source_object_uri.clone()); - output.vrps.push(vrp); - audit_by_uri.insert( - projected.source_object_uri.clone(), - ObjectAuditEntry { - rsync_uri: projected.source_object_uri.clone(), - sha256_hex: hex::encode(projected.source_object_hash), - kind: AuditObjectKind::Roa, - result: AuditObjectResult::Ok, - detail: None, - }, - ); - } - Err(err) => { - warnings.push( - Warning::new(format!( - "publication-point cached ROA local output parse failed: {err}" - )) - .with_context(&projected.source_object_uri), - ); - audit_by_uri.insert( - projected.source_object_uri.clone(), - ObjectAuditEntry { - rsync_uri: projected.source_object_uri.clone(), - sha256_hex: hex::encode(projected.source_object_hash), - kind: AuditObjectKind::Roa, - result: AuditObjectResult::Error, - detail: Some(format!( - "publication-point cached ROA local output parse failed: {err}" - )), - }, - ); - } - }, - VcirOutputType::Aspa => match parse_publication_point_cache_aspa_output(projected) { - Ok(aspa) => { - aspa_ok.insert(projected.source_object_uri.clone()); - output.aspas.push(aspa); - audit_by_uri.insert( - projected.source_object_uri.clone(), - ObjectAuditEntry { - rsync_uri: projected.source_object_uri.clone(), - sha256_hex: hex::encode(projected.source_object_hash), - kind: AuditObjectKind::Aspa, - result: AuditObjectResult::Ok, - detail: None, - }, - ); - } - Err(err) => { - warnings.push( - Warning::new(format!( - "publication-point cached ASPA local output parse failed: {err}" - )) - .with_context(&projected.source_object_uri), - ); - audit_by_uri.insert( - projected.source_object_uri.clone(), - ObjectAuditEntry { - rsync_uri: projected.source_object_uri.clone(), - sha256_hex: hex::encode(projected.source_object_hash), - kind: AuditObjectKind::Aspa, - result: AuditObjectResult::Error, - detail: Some(format!( - "publication-point cached ASPA local output parse failed: {err}" - )), - }, - ); - } - }, - VcirOutputType::RouterKey => { - match parse_publication_point_cache_router_key_output(projected) { - Ok(router_key) => { - output.router_keys.push(router_key); - audit_by_uri.insert( - projected.source_object_uri.clone(), - ObjectAuditEntry { - rsync_uri: projected.source_object_uri.clone(), - sha256_hex: hex::encode(projected.source_object_hash), - kind: AuditObjectKind::RouterCertificate, - result: AuditObjectResult::Ok, - detail: Some( - "publication-point cached Router Key local output restored" - .to_string(), - ), - }, - ); - } - Err(err) => { - warnings.push( - Warning::new(format!( - "publication-point cached Router Key local output parse failed: {err}" - )) - .with_context(&projected.source_object_uri), - ); - audit_by_uri.insert( - projected.source_object_uri.clone(), - ObjectAuditEntry { - rsync_uri: projected.source_object_uri.clone(), - sha256_hex: hex::encode(projected.source_object_hash), - kind: AuditObjectKind::RouterCertificate, - result: AuditObjectResult::Error, - detail: Some(format!( - "publication-point cached Router Key local output parse failed: {err}" - )), - }, - ); - } - } - } - } - } - - output.stats.roa_total = roa_total.len(); - output.stats.roa_ok = roa_ok.len(); - output.stats.aspa_total = aspa_total.len(); - output.stats.aspa_ok = aspa_ok.len(); - let mut audit: Vec<_> = audit_by_uri.into_values().collect(); - audit.sort_by(|left, right| left.rsync_uri.cmp(&right.rsync_uri)); - output.audit = audit; - output -} - -fn parse_vcir_vrp_output(local: &VcirLocalOutput) -> Result { - match &local.payload { - VcirLocalOutputPayload::Vrp { - asn, - afi, - prefix_len, - addr, - max_length, - } => Ok(Vrp { - asn: *asn, - prefix: crate::data_model::roa::IpPrefix { - afi: *afi, - prefix_len: *prefix_len, - addr: *addr, - }, - max_length: *max_length, - }), - _ => Err("VCIR local output payload is not VRP".to_string()), - } -} - -fn parse_vcir_aspa_output(local: &VcirLocalOutput) -> Result { - match &local.payload { - VcirLocalOutputPayload::Aspa { - customer_as_id, - provider_as_ids, - } => Ok(AspaAttestation { - customer_as_id: *customer_as_id, - provider_as_ids: provider_as_ids.clone(), - }), - _ => Err("VCIR local output payload is not ASPA".to_string()), - } -} - -fn parse_vcir_router_key_output(local: &VcirLocalOutput) -> Result { - match &local.payload { - VcirLocalOutputPayload::RouterKey { - as_id, - ski, - spki_der, - } => Ok(RouterKeyPayload { - as_id: *as_id, - ski: ski.clone(), - spki_der: spki_der.clone(), - source_object_uri: local.source_object_uri.clone(), - source_object_hash: local.source_object_hash_hex(), - source_ee_cert_hash: local.source_ee_cert_hash_hex(), - item_effective_until: local.item_effective_until.clone(), - }), - _ => Err("VCIR local output payload is not Router Key".to_string()), - } -} - -fn parse_publication_point_cache_vrp_output( - projected: &PublicationPointCacheOutput, -) -> Result { - match &projected.payload { - VcirLocalOutputPayload::Vrp { - asn, - afi, - prefix_len, - addr, - max_length, - } => Ok(Vrp { - asn: *asn, - prefix: crate::data_model::roa::IpPrefix { - afi: *afi, - prefix_len: *prefix_len, - addr: *addr, - }, - max_length: *max_length, - }), - _ => Err("publication-point cache output payload is not VRP".to_string()), - } -} - -fn parse_publication_point_cache_aspa_output( - projected: &PublicationPointCacheOutput, -) -> Result { - match &projected.payload { - VcirLocalOutputPayload::Aspa { - customer_as_id, - provider_as_ids, - } => Ok(AspaAttestation { - customer_as_id: *customer_as_id, - provider_as_ids: provider_as_ids.clone(), - }), - _ => Err("publication-point cache output payload is not ASPA".to_string()), - } -} - -fn parse_publication_point_cache_router_key_output( - projected: &PublicationPointCacheOutput, -) -> Result { - match &projected.payload { - VcirLocalOutputPayload::RouterKey { - as_id, - ski, - spki_der, - } => Ok(RouterKeyPayload { - as_id: *as_id, - ski: ski.clone(), - spki_der: spki_der.clone(), - source_object_uri: projected.source_object_uri.clone(), - source_object_hash: hex::encode(projected.source_object_hash), - source_ee_cert_hash: hex::encode(projected.source_ee_cert_hash), - item_effective_until: projected.item_effective_until.clone(), - }), - _ => Err("publication-point cache output payload is not Router Key".to_string()), - } -} - -fn restore_children_from_vcir( - _store: &RocksStore, - ca: &CaInstanceHandle, - vcir: &ValidatedCaInstanceResult, - _warnings: &mut Vec, -) -> (Vec, Vec) { - let mut children = Vec::new(); - let mut audits = Vec::new(); - for child in &vcir.child_entries { - children.push(DiscoveredChildCaInstance { - handle: CaInstanceHandle { - depth: 0, - tal_id: ca.tal_id.clone(), - parent_manifest_rsync_uri: Some(ca.manifest_rsync_uri.clone()), - ca_certificate: CaCertificateRef::repo_bytes(child.child_cert_hash.clone()), - ca_certificate_rsync_uri: Some(child.child_cert_rsync_uri.clone()), - effective_ip_resources: child.child_effective_ip_resources.clone(), - effective_as_resources: child.child_effective_as_resources.clone(), - rsync_base_uri: child.child_rsync_base_uri.clone(), - manifest_rsync_uri: child.child_manifest_rsync_uri.clone(), - publication_point_rsync_uri: child.child_publication_point_rsync_uri.clone(), - rrdp_notification_uri: child.child_rrdp_notification_uri.clone(), - }, - discovered_from: crate::audit::DiscoveredFrom { - parent_manifest_rsync_uri: ca.manifest_rsync_uri.clone(), - child_ca_certificate_rsync_uri: child.child_cert_rsync_uri.clone(), - child_ca_certificate_sha256_hex: child.child_cert_hash.clone(), - }, - child_entry_projection: Some(DiscoveredChildEntryProjection { - child_ski: child.child_ski.clone(), - }), - }); - audits.push(ObjectAuditEntry { - rsync_uri: child.child_cert_rsync_uri.clone(), - sha256_hex: child.child_cert_hash.clone(), - kind: AuditObjectKind::Certificate, - result: AuditObjectResult::Ok, - detail: Some("restored child CA instance from VCIR".to_string()), - }); - } - (children, audits) -} - -fn restore_children_from_publication_point_cache( - store: &RocksStore, - ca: &CaInstanceHandle, - projection: &PublicationPointCacheProjection, - validation_time: time::OffsetDateTime, - warnings: &mut Vec, - worker_count: usize, - timing: Option<&TimingHandle>, -) -> (Vec, Vec) { - let worker_count = worker_count - .clamp(1, PUBLICATION_POINT_CACHE_CHILD_RESTORE_MAX_WORKERS) - .min(projection.children.len().max(1)); - let outcomes = if worker_count > 1 - && projection.children.len() >= PUBLICATION_POINT_CACHE_CHILD_RESTORE_PARALLEL_MIN_CHILDREN - { - if let Some(timing) = timing { - timing.record_count( - "publication_point_cache_restore_children_parallel_publication_points", - 1, - ); - timing.record_count( - "publication_point_cache_restore_children_parallel_children", - projection.children.len() as u64, - ); - timing.record_count( - "publication_point_cache_restore_children_workers_total", - worker_count as u64, - ); - } - restore_publication_point_cache_children_parallel( - store, - ca, - &projection.children, - validation_time, - worker_count, - ) - } else { - if let Some(timing) = timing { - timing.record_count( - "publication_point_cache_restore_children_batch_publication_points", - 1, - ); - timing.record_count( - "publication_point_cache_restore_children_batch_children", - projection.children.len() as u64, - ); - } - restore_publication_point_cache_children_chunk( - store, - ca, - &projection.children, - validation_time, - ) - }; - collect_publication_point_cache_child_restore_outcomes(outcomes, warnings) -} - -fn restore_publication_point_cache_children_parallel( - _store: &RocksStore, - ca: &CaInstanceHandle, - children: &[PublicationPointCacheChild], - validation_time: time::OffsetDateTime, - worker_count: usize, -) -> Vec { - let chunk_size = children.len().div_ceil(worker_count).max(1); - let mut chunk_results = Vec::new(); - std::thread::scope(|scope| { - let mut handles = Vec::new(); - for chunk in children.chunks(chunk_size) { - handles.push(scope.spawn(move || { - restore_publication_point_cache_children_chunk(_store, ca, chunk, validation_time) - })); - } - for handle in handles { - chunk_results.extend( - handle - .join() - .expect("publication-point cache child restore worker panicked"), - ); - } - }); - chunk_results -} - -fn restore_publication_point_cache_children_chunk( - _store: &RocksStore, - ca: &CaInstanceHandle, - children: &[PublicationPointCacheChild], - validation_time: time::OffsetDateTime, -) -> Vec { - let mut outcomes = vec![None; children.len()]; - for (position, child) in children.iter().enumerate() { - let effective_not_before = - match parse_snapshot_time_value(&child.child_effective_not_before) { - Ok(value) => value, - Err(e) => { - outcomes[position] = Some(PublicationPointCacheChildRestoreOutcome { - child: None, - audit: None, - warning: Some( - Warning::new(format!( - "publication-point cache child has invalid effective notBefore: {e}" - )) - .with_context(&child.child_cert_rsync_uri), - ), - }); - continue; - } - }; - let effective_until = match parse_snapshot_time_value(&child.child_effective_until) { - Ok(value) => value, - Err(e) => { - outcomes[position] = Some(PublicationPointCacheChildRestoreOutcome { - child: None, - audit: None, - warning: Some( - Warning::new(format!( - "publication-point cache child has invalid effective until: {e}" - )) - .with_context(&child.child_cert_rsync_uri), - ), - }); - continue; - } - }; - if validation_time < effective_not_before || validation_time > effective_until { - outcomes[position] = Some(PublicationPointCacheChildRestoreOutcome { - child: None, - warning: None, - audit: Some(publication_point_cache_child_audit( - child, - AuditObjectResult::Skipped, - Some("skipped: publication-point cache child expired".to_string()), - )), - }); - continue; - } - outcomes[position] = Some(PublicationPointCacheChildRestoreOutcome { - child: Some(publication_point_cache_discovered_child(ca, child)), - warning: None, - audit: Some(publication_point_cache_child_audit( - child, - AuditObjectResult::Ok, - Some("restored child CA instance from publication-point cache".to_string()), - )), - }); - } - - outcomes - .into_iter() - .flatten() - .collect::>() -} - -fn collect_publication_point_cache_child_restore_outcomes( - outcomes: Vec, - warnings: &mut Vec, -) -> (Vec, Vec) { - let mut children = Vec::new(); - let mut audits = Vec::new(); - for outcome in outcomes { - if let Some(warning) = outcome.warning { - warnings.push(warning); - } - if let Some(audit) = outcome.audit { - audits.push(audit); - } - if let Some(child) = outcome.child { - children.push(child); - } - } - (children, audits) -} - -#[derive(Clone)] -struct PublicationPointCacheChildRestoreOutcome { - child: Option, - audit: Option, - warning: Option, -} - -fn publication_point_cache_discovered_child( - ca: &CaInstanceHandle, - child: &PublicationPointCacheChild, -) -> DiscoveredChildCaInstance { - DiscoveredChildCaInstance { - handle: CaInstanceHandle { - depth: 0, - tal_id: ca.tal_id.clone(), - parent_manifest_rsync_uri: Some(ca.manifest_rsync_uri.clone()), - ca_certificate: CaCertificateRef::repo_bytes(child.child_cert_hash.clone()), - ca_certificate_rsync_uri: Some(child.child_cert_rsync_uri.clone()), - effective_ip_resources: child.child_effective_ip_resources.clone(), - effective_as_resources: child.child_effective_as_resources.clone(), - rsync_base_uri: child.child_rsync_base_uri.clone(), - manifest_rsync_uri: child.child_manifest_rsync_uri.clone(), - publication_point_rsync_uri: child.child_publication_point_rsync_uri.clone(), - rrdp_notification_uri: child.child_rrdp_notification_uri.clone(), - }, - discovered_from: crate::audit::DiscoveredFrom { - parent_manifest_rsync_uri: ca.manifest_rsync_uri.clone(), - child_ca_certificate_rsync_uri: child.child_cert_rsync_uri.clone(), - child_ca_certificate_sha256_hex: child.child_cert_hash.clone(), - }, - child_entry_projection: Some(DiscoveredChildEntryProjection { - child_ski: child.child_ski.clone(), - }), - } -} - -fn publication_point_cache_child_audit( - child: &PublicationPointCacheChild, - result: AuditObjectResult, - detail: Option, -) -> ObjectAuditEntry { - ObjectAuditEntry { - rsync_uri: child.child_cert_rsync_uri.clone(), - sha256_hex: child.child_cert_hash.clone(), - kind: AuditObjectKind::Certificate, - result, - detail, - } -} - -fn persist_vcir_for_fresh_result_with_timing( - store: &RocksStore, - policy: &Policy, - ca: &CaInstanceHandle, - pack: &PublicationPointSnapshot, - objects: &mut crate::validation::objects::ObjectsOutput, - warnings: &[Warning], - child_audits: &[ObjectAuditEntry], - discovered_children: &[DiscoveredChildCaInstance], - validation_time: time::OffsetDateTime, - write_publication_point_cache_projection: bool, -) -> Result { - let mut timing = PersistVcirTimingBreakdown::default(); - - if objects.stats.publication_point_dropped { - return Ok(timing); - } - - let embedded_store_started = std::time::Instant::now(); - persist_vcir_non_repository_evidence(store, ca) - .map_err(|e| format!("store VCIR audit evidence failed: {e}"))?; - timing.embedded_store_ms = embedded_store_started.elapsed().as_millis() as u64; - - let build_vcir_started = std::time::Instant::now(); - let (vcir, build_vcir_timing) = build_vcir_from_fresh_result_with_timing( - store, - ca, - pack, - objects, - warnings, - child_audits, - discovered_children, - validation_time, - )?; - timing.build_vcir_ms = build_vcir_started.elapsed().as_millis() as u64; - timing.build_vcir = build_vcir_timing; - - let replace_vcir_started = std::time::Instant::now(); - let future_not_before_cache_guard = write_publication_point_cache_projection - && publication_point_cache_has_future_not_before_risk( - pack, - objects, - child_audits, - validation_time, - policy, - ); - timing.publication_point_cache_future_notbefore_guarded = future_not_before_cache_guard; - let publication_point_cache_projection = - if write_publication_point_cache_projection && !future_not_before_cache_guard { - Some(build_publication_point_cache_projection_from_fresh( - policy, ca, pack, &vcir, - )?) - } else { - None - }; - let publication_point_cache_projection_action = if !write_publication_point_cache_projection { - PublicationPointCacheProjectionWriteAction::Keep - } else if future_not_before_cache_guard { - PublicationPointCacheProjectionWriteAction::Delete { - manifest_rsync_uri: &vcir.manifest_rsync_uri, - } - } else { - PublicationPointCacheProjectionWriteAction::Write( - publication_point_cache_projection - .as_ref() - .expect("publication point projection must exist when guard is not active"), - ) - }; - let failed_fetch_reuse_identity = failed_fetch_reuse_identity_for_fresh_result( - ca, - policy, - validation_time, - vcir.instance_gate.instance_effective_until.clone(), - )?; - let replace_timing = store - .replace_vcir_manifest_replay_meta_and_projection_action_with_failed_fetch_reuse_identity( - &vcir, - Some(&RoaCacheProjectionContext { - ca_validation_context_digest: ca_validation_context_digest_for_ca(ca), - policy_fingerprint: publication_point_cache_policy_fingerprint(policy), - object_meta: objects.roa_cache_object_meta.clone(), - }), - publication_point_cache_projection_action, - Some(&failed_fetch_reuse_identity), - ) - .map_err(|e| format!("store VCIR and manifest replay meta failed: {e}"))?; - timing.replace_vcir_ms = replace_vcir_started.elapsed().as_millis() as u64; - timing.replace_vcir = replace_timing; - - Ok(timing) -} - -fn build_publication_point_cache_projection_from_fresh( - policy: &Policy, - ca: &CaInstanceHandle, - pack: &PublicationPointSnapshot, - vcir: &ValidatedCaInstanceResult, -) -> Result { - let ca_cert_sha256 = ca - .ca_certificate_sha256_32() - .ok_or_else(|| "current CA certificate hash unavailable".to_string())?; - let manifest_sha256 = sha256_digest_32(&pack.manifest_bytes); - PublicationPointCacheProjection::from_vcir_with_context( - vcir, - pack.publication_point_rsync_uri.clone(), - ca.ca_certificate_rsync_uri.clone(), - ca_cert_sha256, - manifest_sha256, - ta_context_digest_for_ca(ca), - ca_validation_context_digest_for_ca(ca), - publication_point_cache_policy_fingerprint(policy), - ) - .map_err(|e| e.to_string()) -} - -fn publication_point_cache_has_future_not_before_risk( - pack: &PublicationPointSnapshot, - objects: &crate::validation::objects::ObjectsOutput, - child_audits: &[ObjectAuditEntry], - validation_time: time::OffsetDateTime, - policy: &Policy, -) -> bool { - let mut files_by_uri: HashMap<&str, &PackFile> = HashMap::new(); - for file in &pack.files { - files_by_uri.insert(file.rsync_uri.as_str(), file); - } - - objects - .audit - .iter() - .chain(child_audits.iter()) - .any(|entry| { - audit_entry_has_certificate_time_error(entry) - && files_by_uri - .get(entry.rsync_uri.as_str()) - .map(|file| { - audit_entry_has_future_not_before(entry, file, validation_time, policy) - }) - .unwrap_or(true) - }) -} - -fn audit_entry_has_certificate_time_error(entry: &ObjectAuditEntry) -> bool { - entry.result == AuditObjectResult::Error - && entry - .detail - .as_deref() - .is_some_and(|detail| detail.contains("certificate not valid at validation_time")) -} - -fn audit_entry_has_future_not_before( - entry: &ObjectAuditEntry, - file: &PackFile, - validation_time: time::OffsetDateTime, - policy: &Policy, -) -> bool { - match entry.kind { - AuditObjectKind::Roa => signed_object_ee_not_before(file, policy, SignedObjectKind::Roa) - .map(|not_before| validation_time < not_before) - .unwrap_or(true), - AuditObjectKind::Aspa => signed_object_ee_not_before(file, policy, SignedObjectKind::Aspa) - .map(|not_before| validation_time < not_before) - .unwrap_or(true), - AuditObjectKind::Certificate | AuditObjectKind::RouterCertificate => file - .bytes() - .ok() - .and_then(|bytes| ResourceCertificate::decode_der(bytes).ok()) - .map(|cert| validation_time < cert.tbs.validity_not_before) - .unwrap_or(true), - _ => true, - } -} - -#[derive(Clone, Copy)] -enum SignedObjectKind { - Roa, - Aspa, -} - -fn signed_object_ee_not_before( - file: &PackFile, - policy: &Policy, - kind: SignedObjectKind, -) -> Option { - let bytes = file.bytes().ok()?; - match kind { - SignedObjectKind::Roa => { - let object = RoaObject::decode_der_with_strict_options( - bytes, - policy.strict.cms_der, - policy.strict.name, - ) - .ok()?; - Some( - object.signed_object.signed_data.certificates[0] - .resource_cert - .tbs - .validity_not_before, - ) - } - SignedObjectKind::Aspa => { - let object = AspaObject::decode_der_with_strict_options( - bytes, - policy.strict.cms_der, - policy.strict.name, - ) - .ok()?; - Some( - object.signed_object.signed_data.certificates[0] - .resource_cert - .tbs - .validity_not_before, - ) - } - } -} - -fn build_vcir_from_fresh_result_with_timing( - store: &RocksStore, - ca: &CaInstanceHandle, - pack: &PublicationPointSnapshot, - objects: &mut crate::validation::objects::ObjectsOutput, - warnings: &[Warning], - child_audits: &[ObjectAuditEntry], - discovered_children: &[DiscoveredChildCaInstance], - validation_time: time::OffsetDateTime, -) -> Result<(ValidatedCaInstanceResult, BuildVcirTimingBreakdown), String> { - let mut timing = BuildVcirTimingBreakdown::default(); - - let select_crl_started = std::time::Instant::now(); - let current_crl = select_manifest_current_crl_from_snapshot(pack)?; - timing.select_crl_ms = select_crl_started.elapsed().as_millis() as u64; - - let current_ca_decode_started = std::time::Instant::now(); - let ca_der = ca.ca_certificate_der(store)?; - let ca_cert = ResourceCertificate::decode_der(ca_der.as_ref()) - .map_err(|e| format!("decode current CA certificate failed: {e}"))?; - timing.current_ca_decode_ms = current_ca_decode_started.elapsed().as_millis() as u64; - - let local_outputs_started = std::time::Instant::now(); - let local_outputs = take_or_build_vcir_local_outputs(ca, pack, objects)?; - timing.local_outputs_ms = local_outputs_started.elapsed().as_millis() as u64; - - let child_entries_started = std::time::Instant::now(); - let child_entries = build_vcir_child_entries(store, discovered_children, validation_time)?; - timing.child_entries_ms = child_entries_started.elapsed().as_millis() as u64; - - let related_artifacts_started = std::time::Instant::now(); - let related_artifacts = build_vcir_related_artifacts( - store, - ca, - pack, - current_crl.file.rsync_uri.as_str(), - objects, - child_audits, - ); - timing.related_artifacts_ms = related_artifacts_started.elapsed().as_millis() as u64; - let ccr_manifest_projection = - build_vcir_ccr_manifest_projection_from_fresh(ca, pack, &child_entries)?; - let local_vrp_count = local_outputs - .iter() - .filter(|output| output.output_type == VcirOutputType::Vrp) - .count() as u32; - let local_aspa_count = local_outputs - .iter() - .filter(|output| output.output_type == VcirOutputType::Aspa) - .count() as u32; - let local_router_key_count = local_outputs - .iter() - .filter(|output| output.output_type == VcirOutputType::RouterKey) - .count() as u32; - let accepted_object_count = related_artifacts - .iter() - .filter(|artifact| artifact.validation_status == VcirArtifactValidationStatus::Accepted) - .count() as u32; - let rejected_object_count = related_artifacts - .iter() - .filter(|artifact| artifact.validation_status == VcirArtifactValidationStatus::Rejected) - .count() as u32; - let ca_ski = hex::encode( - ca_cert - .tbs - .extensions - .subject_key_identifier - .as_ref() - .ok_or_else(|| "current CA certificate missing SubjectKeyIdentifier".to_string())?, - ); - let issuer_ski = hex::encode( - ca_cert - .tbs - .extensions - .authority_key_identifier - .as_ref() - .or(ca_cert.tbs.extensions.subject_key_identifier.as_ref()) - .ok_or_else(|| "current CA certificate missing AuthorityKeyIdentifier".to_string())?, - ); - - let struct_build_started = std::time::Instant::now(); - let vcir = ValidatedCaInstanceResult { - manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - parent_manifest_rsync_uri: ca.parent_manifest_rsync_uri.clone(), - tal_id: ca.tal_id.clone(), - ca_subject_name: ca_cert.tbs.subject_name.to_string(), - ca_ski, - issuer_ski, - last_successful_validation_time: PackTime::from_utc_offset_datetime(validation_time), - current_manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - current_crl_rsync_uri: current_crl.file.rsync_uri.clone(), - validated_manifest_meta: crate::storage::ValidatedManifestMeta { - validated_manifest_number: pack.manifest_number_be.clone(), - validated_manifest_this_update: pack.this_update.clone(), - validated_manifest_next_update: pack.next_update.clone(), - }, - ccr_manifest_projection, - instance_gate: VcirInstanceGate { - manifest_next_update: pack.next_update.clone(), - current_crl_next_update: PackTime::from_utc_offset_datetime( - current_crl.crl.next_update.utc, - ), - self_ca_not_after: PackTime::from_utc_offset_datetime(ca_cert.tbs.validity_not_after), - instance_effective_until: PackTime::from_utc_offset_datetime( - pack.next_update - .parse() - .map_err(|e| format!("parse snapshot next_update failed: {e}"))? - .min(current_crl.crl.next_update.utc) - .min(ca_cert.tbs.validity_not_after), - ), - }, - child_entries, - local_outputs, - related_artifacts, - summary: VcirSummary { - local_vrp_count, - local_aspa_count, - local_router_key_count, - child_count: discovered_children.len() as u32, - accepted_object_count, - rejected_object_count, - }, - audit_summary: VcirAuditSummary { - failed_fetch_eligible: true, - last_failed_fetch_reason: None, - warning_count: (warnings.len() + objects.warnings.len()) as u32, - audit_flags: Vec::new(), - }, - }; - vcir.validate_internal().map_err(|e| e.to_string())?; - timing.struct_build_ms = struct_build_started.elapsed().as_millis() as u64; - Ok((vcir, timing)) -} - -fn take_or_build_vcir_local_outputs( - ca: &CaInstanceHandle, - pack: &PublicationPointSnapshot, - objects: &mut crate::validation::objects::ObjectsOutput, -) -> Result, String> { - let mut cached_outputs = std::mem::take(&mut objects.local_outputs_cache); - if cached_outputs.is_empty() { - return build_vcir_local_outputs(ca, pack, objects); - } - - let covered_roa_uris: HashSet = cached_outputs - .iter() - .filter(|output| output.source_object_type == VcirSourceObjectType::Roa) - .map(|output| output.source_object_uri.clone()) - .collect(); - let covered_aspa_uris: HashSet = cached_outputs - .iter() - .filter(|output| output.source_object_type == VcirSourceObjectType::Aspa) - .map(|output| output.source_object_uri.clone()) - .collect(); - cached_outputs.extend(build_vcir_local_outputs_excluding( - ca, - pack, - objects, - &covered_roa_uris, - &covered_aspa_uris, - )?); - Ok(cached_outputs) -} - -fn build_vcir_ccr_manifest_projection_from_fresh( - ca: &CaInstanceHandle, - pack: &PublicationPointSnapshot, - child_entries: &[VcirChildEntry], -) -> Result { - let manifest = ManifestObject::decode_der(&pack.manifest_bytes) - .map_err(|e| format!("decode manifest for VCIR CCR projection failed: {e}"))?; - let ee = &manifest.signed_object.signed_data.certificates[0].resource_cert; - let manifest_ee_aki = ee - .tbs - .extensions - .authority_key_identifier - .clone() - .ok_or_else(|| "manifest EE certificate missing AuthorityKeyIdentifier".to_string())?; - let manifest_sia_locations_der = match ee - .tbs - .extensions - .subject_info_access - .as_ref() - .ok_or_else(|| "manifest EE certificate missing Subject Information Access".to_string())? - { - SubjectInfoAccess::Ee(ee_sia) => vec![select_manifest_signed_object_location( - &ca.manifest_rsync_uri, - &ee_sia.access_descriptions, - )?], - SubjectInfoAccess::Ca(_) => { - return Err( - "manifest EE certificate Subject Information Access has CA variant".to_string(), - ); - } - }; - - let mut subordinate_skis = child_entries - .iter() - .map(|child| { - hex::decode(&child.child_ski) - .map_err(|e| format!("decode child_ski for VCIR CCR projection failed: {e}")) - }) - .collect::, _>>()?; - subordinate_skis.sort(); - subordinate_skis.dedup(); - - Ok(VcirCcrManifestProjection { - manifest_rsync_uri: ca.manifest_rsync_uri.clone(), - manifest_sha256: sha2::Sha256::digest(&pack.manifest_bytes).to_vec(), - manifest_size: pack.manifest_bytes.len() as u64, - manifest_ee_aki, - manifest_number_be: pack.manifest_number_be.clone(), - manifest_this_update: pack.this_update.clone(), - manifest_sia_locations_der, - subordinate_skis, - }) -} - -struct CurrentCrlRef<'a> { - file: &'a PackFile, - crl: RpkixCrl, -} - -fn select_manifest_current_crl_from_snapshot( - pack: &PublicationPointSnapshot, -) -> Result, String> { - let manifest = ManifestObject::decode_der(&pack.manifest_bytes) - .map_err(|e| format!("decode snapshot manifest for VCIR failed: {e}"))?; - let ee = &manifest.signed_object.signed_data.certificates[0].resource_cert; - let crldp_uris = ee - .tbs - .extensions - .crl_distribution_points_uris - .as_ref() - .ok_or_else(|| "manifest EE certificate missing CRLDistributionPoints".to_string())?; - for uri in crldp_uris { - if let Some(file) = pack - .files - .iter() - .find(|candidate| candidate.rsync_uri == *uri) - { - let crl = RpkixCrl::decode_der( - file.bytes() - .map_err(|e| format!("load current CRL bytes for VCIR failed: {e}"))?, - ) - .map_err(|e| format!("decode current CRL for VCIR failed: {e}"))?; - return Ok(CurrentCrlRef { file, crl }); - } - } - Err(format!( - "manifest EE certificate CRLDistributionPoints not found in pack: {}", - crldp_uris.join(", ") - )) -} - -fn build_vcir_local_outputs( - _ca: &CaInstanceHandle, - pack: &PublicationPointSnapshot, - objects: &crate::validation::objects::ObjectsOutput, -) -> Result, String> { - build_vcir_local_outputs_excluding(_ca, pack, objects, &HashSet::new(), &HashSet::new()) -} - -fn build_vcir_local_outputs_excluding( - _ca: &CaInstanceHandle, - pack: &PublicationPointSnapshot, - objects: &crate::validation::objects::ObjectsOutput, - covered_roa_uris: &HashSet, - covered_aspa_uris: &HashSet, -) -> Result, String> { - let accepted_roa_uris: HashSet<&str> = objects - .audit - .iter() - .filter(|entry| entry.kind == AuditObjectKind::Roa && entry.result == AuditObjectResult::Ok) - .map(|entry| entry.rsync_uri.as_str()) - .collect(); - let accepted_aspa_uris: HashSet<&str> = objects - .audit - .iter() - .filter(|entry| { - entry.kind == AuditObjectKind::Aspa && entry.result == AuditObjectResult::Ok - }) - .map(|entry| entry.rsync_uri.as_str()) - .collect(); - - let mut out = Vec::new(); - for file in &pack.files { - let source_object_hash = sha256_hex_from_32(&file.sha256); - if accepted_roa_uris.contains(file.rsync_uri.as_str()) - && !covered_roa_uris.contains(file.rsync_uri.as_str()) - { - let roa = RoaObject::decode_der( - file.bytes() - .map_err(|e| format!("load accepted ROA bytes for VCIR failed: {e}"))?, - ) - .map_err(|e| format!("decode accepted ROA for VCIR failed: {e}"))?; - let ee = &roa.signed_object.signed_data.certificates[0]; - let source_ee_cert_hash = sha256_hex(ee.raw_der.as_slice()); - let item_effective_until = - PackTime::from_utc_offset_datetime(ee.resource_cert.tbs.validity_not_after); - for vrp in roa_to_vrps_for_vcir(&roa) { - let prefix = vrp_prefix_to_string(&vrp); - let rule_hash = sha256_hex( - format!( - "roa-rule:{}:{}:{}:{}", - source_object_hash, vrp.asn, prefix, vrp.max_length - ) - .as_bytes(), - ); - out.push(VcirLocalOutput { - output_type: VcirOutputType::Vrp, - item_effective_until: item_effective_until.clone(), - source_object_uri: file.rsync_uri.clone(), - source_object_type: VcirSourceObjectType::Roa, - source_object_hash: file.sha256, - source_ee_cert_hash: sha256_hex_to_32(&source_ee_cert_hash), - payload: VcirLocalOutputPayload::Vrp { - asn: vrp.asn, - afi: vrp.prefix.afi, - prefix_len: vrp.prefix.prefix_len, - addr: vrp.prefix.addr, - max_length: vrp.max_length, - }, - rule_hash: sha256_hex_to_32(&rule_hash), - }); - } - } else if accepted_aspa_uris.contains(file.rsync_uri.as_str()) - && !covered_aspa_uris.contains(file.rsync_uri.as_str()) - { - let aspa = AspaObject::decode_der( - file.bytes() - .map_err(|e| format!("load accepted ASPA bytes for VCIR failed: {e}"))?, - ) - .map_err(|e| format!("decode accepted ASPA for VCIR failed: {e}"))?; - let ee = &aspa.signed_object.signed_data.certificates[0]; - let source_ee_cert_hash = sha256_hex(ee.raw_der.as_slice()); - let item_effective_until = - PackTime::from_utc_offset_datetime(ee.resource_cert.tbs.validity_not_after); - let providers = aspa - .aspa - .provider_as_ids - .iter() - .map(u32::to_string) - .collect::>() - .join(","); - let rule_hash = sha256_hex( - format!( - "aspa-rule:{}:{}:{}", - source_object_hash, aspa.aspa.customer_as_id, providers - ) - .as_bytes(), - ); - out.push(VcirLocalOutput { - output_type: VcirOutputType::Aspa, - item_effective_until, - source_object_uri: file.rsync_uri.clone(), - source_object_type: VcirSourceObjectType::Aspa, - source_object_hash: file.sha256, - source_ee_cert_hash: sha256_hex_to_32(&source_ee_cert_hash), - payload: VcirLocalOutputPayload::Aspa { - customer_as_id: aspa.aspa.customer_as_id, - provider_as_ids: aspa.aspa.provider_as_ids.clone(), - }, - rule_hash: sha256_hex_to_32(&rule_hash), - }); - } - } - Ok(out) -} - -pub(crate) fn build_router_key_local_outputs( - _ca: &CaInstanceHandle, - router_keys: &[RouterKeyPayload], -) -> Vec { - router_keys - .iter() - .map(|router_key| { - let ski_hex = hex::encode(&router_key.ski); - let spki_der_base64 = - base64::engine::general_purpose::STANDARD.encode(&router_key.spki_der); - let rule_hash = sha256_hex( - format!( - "router-key-rule:{}:{}:{}:{}", - router_key.source_object_hash, router_key.as_id, ski_hex, spki_der_base64 - ) - .as_bytes(), - ); - VcirLocalOutput { - output_type: VcirOutputType::RouterKey, - item_effective_until: router_key.item_effective_until.clone(), - source_object_uri: router_key.source_object_uri.clone(), - source_object_type: VcirSourceObjectType::RouterKey, - source_object_hash: sha256_hex_to_32(&router_key.source_object_hash), - source_ee_cert_hash: sha256_hex_to_32(&router_key.source_ee_cert_hash), - payload: VcirLocalOutputPayload::RouterKey { - as_id: router_key.as_id, - ski: router_key.ski.clone(), - spki_der: router_key.spki_der.clone(), - }, - rule_hash: sha256_hex_to_32(&rule_hash), - } - }) - .collect() -} - -fn build_vcir_child_entries( - store: &RocksStore, - discovered_children: &[DiscoveredChildCaInstance], - validation_time: time::OffsetDateTime, -) -> Result, String> { - let mut out = Vec::with_capacity(discovered_children.len()); - for child in discovered_children { - let child_ski = match child.child_entry_projection.as_ref() { - Some(projection) => projection.child_ski.clone(), - None => { - let child_der = child.handle.ca_certificate_der(store)?; - let child_cert = ResourceCertificate::decode_der(child_der.as_ref()) - .map_err(|e| format!("decode child certificate for VCIR failed: {e}"))?; - let child_ski = child_cert - .tbs - .extensions - .subject_key_identifier - .as_ref() - .ok_or_else(|| "child certificate missing SubjectKeyIdentifier".to_string())?; - hex::encode(child_ski) - } - }; - out.push(VcirChildEntry { - child_manifest_rsync_uri: child.handle.manifest_rsync_uri.clone(), - child_cert_rsync_uri: child.discovered_from.child_ca_certificate_rsync_uri.clone(), - child_cert_hash: child - .discovered_from - .child_ca_certificate_sha256_hex - .clone(), - child_ski, - child_rsync_base_uri: child.handle.rsync_base_uri.clone(), - child_publication_point_rsync_uri: child.handle.publication_point_rsync_uri.clone(), - child_rrdp_notification_uri: child.handle.rrdp_notification_uri.clone(), - child_effective_ip_resources: child.handle.effective_ip_resources.clone(), - child_effective_as_resources: child.handle.effective_as_resources.clone(), - accepted_at_validation_time: PackTime::from_utc_offset_datetime(validation_time), - }); - } - Ok(out) -} - -fn persist_vcir_non_repository_evidence( - store: &RocksStore, - ca: &CaInstanceHandle, -) -> Result<(), String> { - let ca_der = ca.ca_certificate_der(store)?; - let current_ca_hash = sha256_hex(ca_der.as_ref()); - let mut current_ca_entry = RawByHashEntry::from_bytes(current_ca_hash, ca_der.to_vec()); - if let Some(uri) = ca.ca_certificate_rsync_uri.as_ref() { - current_ca_entry.origin_uris.push(uri.clone()); - } - current_ca_entry.object_type = Some("cer".to_string()); - current_ca_entry.encoding = Some("der".to_string()); - upsert_raw_by_hash_entry(store, current_ca_entry)?; - Ok(()) -} - -fn upsert_raw_by_hash_entry(store: &RocksStore, entry: RawByHashEntry) -> Result<(), String> { - match store.get_raw_by_hash_entry(&entry.sha256_hex) { - Ok(Some(existing)) => { - if existing.bytes != entry.bytes { - return Err(format!( - "raw_by_hash collision for sha256 {} while storing VCIR audit evidence", - entry.sha256_hex - )); - } - let mut merged = existing; - let mut changed = false; - for uri in entry.origin_uris { - if !merged - .origin_uris - .iter() - .any(|existing_uri| existing_uri == &uri) - { - merged.origin_uris.push(uri); - changed = true; - } - } - if merged.object_type.is_none() && entry.object_type.is_some() { - merged.object_type = entry.object_type; - changed = true; - } - if merged.encoding.is_none() && entry.encoding.is_some() { - merged.encoding = entry.encoding; - changed = true; - } - if changed { - store - .put_raw_by_hash_entry(&merged) - .map_err(|e| format!("update raw_by_hash entry failed: {e}"))?; - } - Ok(()) - } - Ok(None) => store - .put_raw_by_hash_entry(&entry) - .map_err(|e| format!("store raw_by_hash entry failed: {e}")), - Err(e) => Err(format!("load raw_by_hash entry failed: {e}")), - } -} - -fn build_vcir_related_artifacts( - store: &RocksStore, - ca: &CaInstanceHandle, - pack: &PublicationPointSnapshot, - current_crl_rsync_uri: &str, - objects: &crate::validation::objects::ObjectsOutput, - child_audits: &[ObjectAuditEntry], -) -> Vec { - let mut audit_by_uri: HashMap<&str, &ObjectAuditEntry> = HashMap::new(); - for entry in child_audits.iter().chain(objects.audit.iter()) { - audit_by_uri.insert(entry.rsync_uri.as_str(), entry); - } - - let mut artifacts = Vec::with_capacity(pack.files.len() + 2); - artifacts.push(VcirRelatedArtifact { - artifact_role: VcirArtifactRole::Manifest, - artifact_kind: VcirArtifactKind::Mft, - uri: Some(pack.manifest_rsync_uri.clone()), - sha256: sha256_hex(&pack.manifest_bytes), - object_type: Some("mft".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - reject_reason: None, - }); - artifacts.push(VcirRelatedArtifact { - artifact_role: if ca.parent_manifest_rsync_uri.is_none() { - VcirArtifactRole::TrustAnchorCert - } else { - VcirArtifactRole::IssuerCert - }, - artifact_kind: VcirArtifactKind::Cer, - uri: ca.ca_certificate_rsync_uri.clone(), - sha256: ca - .ca_certificate_sha256_hex() - .map(str::to_string) - .unwrap_or_else(|| { - ca.ca_certificate_der(store) - .map(|bytes| sha256_hex(bytes.as_ref())) - .unwrap_or_default() - }), - object_type: Some("cer".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - reject_reason: None, - }); - - for file in &pack.files { - let audit_entry = audit_by_uri.get(file.rsync_uri.as_str()).copied(); - let result = audit_entry - .map(|entry| entry.result.clone()) - .unwrap_or(AuditObjectResult::Ok); - let validation_status = audit_result_to_vcir_status(&result); - let reject_reason = if validation_status == VcirArtifactValidationStatus::Rejected { - audit_entry.and_then(|entry| entry.detail.clone()) - } else { - None - }; - let (artifact_role, artifact_kind) = artifact_role_and_kind(file, current_crl_rsync_uri); - artifacts.push(VcirRelatedArtifact { - artifact_role, - artifact_kind, - uri: Some(file.rsync_uri.clone()), - sha256: sha256_hex_from_32(&file.sha256), - object_type: object_type_from_uri(file.rsync_uri.as_str()), - validation_status, - reject_reason, - }); - } - - artifacts -} - -fn artifact_role_and_kind( - file: &PackFile, - current_crl_rsync_uri: &str, -) -> (VcirArtifactRole, VcirArtifactKind) { - if file.rsync_uri == current_crl_rsync_uri { - (VcirArtifactRole::CurrentCrl, VcirArtifactKind::Crl) - } else if file.rsync_uri.ends_with(".cer") { - (VcirArtifactRole::ChildCaCert, VcirArtifactKind::Cer) - } else if file.rsync_uri.ends_with(".roa") { - (VcirArtifactRole::SignedObject, VcirArtifactKind::Roa) - } else if file.rsync_uri.ends_with(".asa") { - (VcirArtifactRole::SignedObject, VcirArtifactKind::Aspa) - } else if file.rsync_uri.ends_with(".gbr") { - (VcirArtifactRole::SignedObject, VcirArtifactKind::Gbr) - } else if file.rsync_uri.ends_with(".crl") { - (VcirArtifactRole::Other, VcirArtifactKind::Crl) - } else if file.rsync_uri.ends_with(".mft") { - (VcirArtifactRole::Manifest, VcirArtifactKind::Mft) - } else { - (VcirArtifactRole::Other, VcirArtifactKind::Other) - } -} - -fn object_type_from_uri(uri: &str) -> Option { - uri.rsplit_once('.') - .map(|(_, ext)| ext.to_ascii_lowercase()) -} - -fn audit_result_to_vcir_status(result: &AuditObjectResult) -> VcirArtifactValidationStatus { - match result { - AuditObjectResult::Ok => VcirArtifactValidationStatus::Accepted, - AuditObjectResult::Error => VcirArtifactValidationStatus::Rejected, - AuditObjectResult::Skipped => VcirArtifactValidationStatus::WarningOnly, - } -} - -fn roa_to_vrps_for_vcir(roa: &RoaObject) -> Vec { - let asn = roa.roa.as_id; - let mut out = Vec::new(); - for fam in &roa.roa.ip_addr_blocks { - for entry in &fam.addresses { - let max_length = entry.max_length.unwrap_or(entry.prefix.prefix_len); - out.push(Vrp { - asn, - prefix: entry.prefix.clone(), - max_length, - }); - } - } - out -} - -fn vrp_prefix_to_string(vrp: &Vrp) -> String { - match vrp.prefix.afi { - RoaAfi::Ipv4 => { - let addr = std::net::Ipv4Addr::new( - vrp.prefix.addr[0], - vrp.prefix.addr[1], - vrp.prefix.addr[2], - vrp.prefix.addr[3], - ); - format!("{addr}/{}", vrp.prefix.prefix_len) - } - RoaAfi::Ipv6 => { - let addr = std::net::Ipv6Addr::from(vrp.prefix.addr); - format!("{addr}/{}", vrp.prefix.prefix_len) - } - } -} +include!("tree_runner/types.rs"); + +include!("tree_runner/cache_methods.rs"); +include!("tree_runner/cache_reuse_methods.rs"); + +include!("tree_runner/publication_point_runner.rs"); +include!("tree_runner/cache_types.rs"); +include!("tree_runner/discovery/wrappers.rs"); +include!("tree_runner/discovery.rs"); +include!("tree_runner/child_validation.rs"); +include!("tree_runner/audit_projection.rs"); +include!("tree_runner/vcir_outputs.rs"); +include!("tree_runner/vcir_persistence.rs"); #[cfg(test)] #[path = "tree_runner/tests.rs"] diff --git a/crates/panda-rpki-validator/src/validation/tree_runner/audit_projection.rs b/crates/panda-rpki-validator/src/validation/tree_runner/audit_projection.rs new file mode 100644 index 0000000..bc82be9 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_runner/audit_projection.rs @@ -0,0 +1,732 @@ +#[cfg(test)] +fn select_issuer_crl_from_snapshot<'a>( + child_cert_der: &[u8], + pack: &'a PublicationPointSnapshot, +) -> Result<(&'a str, &'a [u8]), String> { + let child = crate::data_model::rc::ResourceCertificate::decode_der(child_cert_der) + .map_err(|e| format!("child certificate decode failed: {e}"))?; + let Some(crldp_uris) = child.tbs.extensions.crl_distribution_points_uris.as_ref() else { + return Err( + "child certificate CRLDistributionPoints missing (RFC 6487 §4.8.6)".to_string(), + ); + }; + + for u in crldp_uris { + let s = u.as_str(); + if let Some(f) = pack.files.iter().find(|f| f.rsync_uri == s) { + let bytes = f + .bytes() + .map_err(|e| format!("snapshot CRL bytes load failed: {e}"))?; + return Ok((f.rsync_uri.as_str(), bytes)); + } + } + + Err(format!( + "CRL referenced by child certificate CRLDistributionPoints not found in publication point snapshot: {} (RFC 6487 §4.8.6; RFC 9286 §4.2.1)", + crldp_uris + .iter() + .map(|u| u.as_str()) + .collect::>() + .join(", ") + )) +} + +/// Fallback detail for cached artifacts rejected by an earlier run whose cache +/// entry predates reject-reason recording. +const CACHED_REJECT_REASON_NOT_RECORDED: &str = + "rejected in an earlier validation run (reject reason not recorded in cache)"; + +/// Detail for an audit entry rebuilt from a cached artifact: the real reject +/// reason when the cache recorded one, an explicit fallback for legacy cache +/// entries, and `None` for non-rejected artifacts. +fn audit_detail_from_vcir_status( + status: VcirArtifactValidationStatus, + reject_reason: Option<&str>, +) -> Option { + match status { + VcirArtifactValidationStatus::Rejected => Some( + reject_reason + .map(str::to_string) + .unwrap_or_else(|| CACHED_REJECT_REASON_NOT_RECORDED.to_string()), + ), + VcirArtifactValidationStatus::Accepted | VcirArtifactValidationStatus::WarningOnly => None, + } +} + +fn build_publication_point_audit_from_snapshot( + ca: &CaInstanceHandle, + source: PublicationPointSource, + repo_sync_source: Option<&str>, + repo_sync_phase: Option<&str>, + repo_sync_duration_ms: Option, + repo_sync_error: Option<&str>, + pack: &PublicationPointSnapshot, + runner_warnings: &[Warning], + objects: &crate::validation::objects::ObjectsOutput, + child_audits: &[ObjectAuditEntry], +) -> PublicationPointAudit { + use crate::data_model::crl::RpkixCrl; + use std::collections::HashMap; + + let locked_files = &pack.files; + let mut audit_by_uri: HashMap = HashMap::new(); + for f in locked_files { + audit_by_uri.insert( + f.rsync_uri.clone(), + ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: kind_from_rsync_uri(&f.rsync_uri), + result: AuditObjectResult::Skipped, + detail: Some("skipped: not processed in this run".to_string()), + }, + ); + } + + for f in locked_files { + if !f.rsync_uri.ends_with(".crl") { + continue; + } + let ok = f + .bytes() + .ok() + .and_then(|bytes| RpkixCrl::decode_der(bytes).ok()) + .is_some(); + audit_by_uri.insert( + f.rsync_uri.clone(), + ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Crl, + result: if ok { + AuditObjectResult::Ok + } else { + AuditObjectResult::Error + }, + detail: if ok { + None + } else { + Some("CRL decode failed".to_string()) + }, + }, + ); + } + + for e in child_audits { + audit_by_uri.insert(e.rsync_uri.clone(), e.clone()); + } + for e in &objects.audit { + audit_by_uri.insert(e.rsync_uri.clone(), e.clone()); + } + + let mut objects_out: Vec = Vec::with_capacity(pack.files.len() + 1); + objects_out.push(ObjectAuditEntry { + rsync_uri: pack.manifest_rsync_uri.clone(), + sha256_hex: sha256_hex(&pack.manifest_bytes), + kind: AuditObjectKind::Manifest, + result: AuditObjectResult::Ok, + detail: None, + }); + for f in locked_files { + if let Some(e) = audit_by_uri.remove(&f.rsync_uri) { + objects_out.push(e); + } else { + objects_out.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: kind_from_rsync_uri(&f.rsync_uri), + result: AuditObjectResult::Skipped, + detail: Some("skipped: no audit entry".to_string()), + }); + } + } + + let mut warnings = Vec::new(); + warnings.extend(runner_warnings.iter().map(AuditWarning::from)); + warnings.extend(objects.warnings.iter().map(AuditWarning::from)); + + PublicationPointAudit { + node_id: None, + parent_node_id: None, + discovered_from: None, + rsync_base_uri: ca.rsync_base_uri.clone(), + manifest_rsync_uri: ca.manifest_rsync_uri.clone(), + publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(), + rrdp_notification_uri: ca.rrdp_notification_uri.clone(), + source: source_label(source), + repo_sync_source: repo_sync_source.map(ToString::to_string), + repo_sync_phase: repo_sync_phase.map(ToString::to_string), + repo_sync_duration_ms, + repo_sync_error: repo_sync_error.map(ToString::to_string), + repo_terminal_state: terminal_state_label(source).to_string(), + this_update_rfc3339_utc: pack.this_update.rfc3339_utc.clone(), + next_update_rfc3339_utc: pack.next_update.rfc3339_utc.clone(), + verified_at_rfc3339_utc: pack.verified_at.rfc3339_utc.clone(), + warnings, + objects: objects_out, + } +} + +fn build_publication_point_audit_from_vcir( + ca: &CaInstanceHandle, + source: PublicationPointSource, + repo_sync_source: Option<&str>, + repo_sync_phase: Option<&str>, + repo_sync_duration_ms: Option, + repo_sync_error: Option<&str>, + vcir: Option<&ValidatedCaInstanceResult>, + pack: Option<&PublicationPointSnapshot>, + runner_warnings: &[Warning], + objects: &crate::validation::objects::ObjectsOutput, + child_audits: &[ObjectAuditEntry], + fresh_failure_audits: &[ObjectAuditEntry], +) -> PublicationPointAudit { + if let Some(pack) = pack { + return build_publication_point_audit_from_snapshot( + ca, + source, + repo_sync_source, + repo_sync_phase, + repo_sync_duration_ms, + repo_sync_error, + pack, + runner_warnings, + objects, + child_audits, + ); + } + + let mut warnings = Vec::new(); + warnings.extend(runner_warnings.iter().map(AuditWarning::from)); + warnings.extend(objects.warnings.iter().map(AuditWarning::from)); + + let Some(vcir) = vcir else { + return PublicationPointAudit { + node_id: None, + parent_node_id: None, + discovered_from: None, + rsync_base_uri: ca.rsync_base_uri.clone(), + manifest_rsync_uri: ca.manifest_rsync_uri.clone(), + publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(), + rrdp_notification_uri: ca.rrdp_notification_uri.clone(), + source: source_label(source), + repo_sync_source: repo_sync_source.map(ToString::to_string), + repo_sync_phase: repo_sync_phase.map(ToString::to_string), + repo_sync_duration_ms, + repo_sync_error: repo_sync_error.map(ToString::to_string), + repo_terminal_state: terminal_state_label(source).to_string(), + this_update_rfc3339_utc: String::new(), + next_update_rfc3339_utc: String::new(), + verified_at_rfc3339_utc: String::new(), + warnings, + objects: fresh_failure_audits.to_vec(), + }; + }; + + if source == PublicationPointSource::FailedFetchNoCache { + let mut objects_out = Vec::with_capacity( + objects.audit.len() + child_audits.len() + fresh_failure_audits.len(), + ); + objects_out.extend(child_audits.iter().cloned()); + objects_out.extend(objects.audit.iter().cloned()); + objects_out.extend(fresh_failure_audits.iter().cloned()); + return PublicationPointAudit { + node_id: None, + parent_node_id: None, + discovered_from: None, + rsync_base_uri: ca.rsync_base_uri.clone(), + manifest_rsync_uri: ca.manifest_rsync_uri.clone(), + publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(), + rrdp_notification_uri: ca.rrdp_notification_uri.clone(), + source: source_label(source), + repo_sync_source: repo_sync_source.map(ToString::to_string), + repo_sync_phase: repo_sync_phase.map(ToString::to_string), + repo_sync_duration_ms, + repo_sync_error: repo_sync_error.map(ToString::to_string), + repo_terminal_state: terminal_state_label(source).to_string(), + this_update_rfc3339_utc: vcir + .validated_manifest_meta + .validated_manifest_this_update + .rfc3339_utc + .clone(), + next_update_rfc3339_utc: vcir + .validated_manifest_meta + .validated_manifest_next_update + .rfc3339_utc + .clone(), + verified_at_rfc3339_utc: vcir.last_successful_validation_time.rfc3339_utc.clone(), + warnings, + objects: objects_out, + }; + } + + let mut audit_by_uri: HashMap = HashMap::new(); + for artifact in &vcir.related_artifacts { + let Some(uri) = artifact.uri.as_ref() else { + continue; + }; + audit_by_uri.insert( + uri.clone(), + ObjectAuditEntry { + rsync_uri: uri.clone(), + sha256_hex: artifact.sha256.clone(), + kind: kind_from_vcir_artifact_kind(artifact.artifact_kind), + result: audit_result_from_vcir_status(artifact.validation_status), + detail: audit_detail_from_vcir_status( + artifact.validation_status, + artifact.reject_reason.as_deref(), + ), + }, + ); + } + for e in child_audits { + audit_by_uri.insert(e.rsync_uri.clone(), e.clone()); + } + for e in &objects.audit { + audit_by_uri.insert(e.rsync_uri.clone(), e.clone()); + } + + let mut ordered_uris: Vec = vcir + .related_artifacts + .iter() + .filter_map(|artifact| artifact.uri.clone()) + .collect(); + ordered_uris.sort(); + ordered_uris.dedup(); + + let mut objects_out: Vec = Vec::new(); + if let Some(entry) = audit_by_uri.remove(&vcir.current_manifest_rsync_uri) { + objects_out.push(entry); + } else { + objects_out.push(ObjectAuditEntry { + rsync_uri: vcir.current_manifest_rsync_uri.clone(), + sha256_hex: vcir + .related_artifacts + .iter() + .find(|artifact| { + artifact.artifact_role == VcirArtifactRole::Manifest + && artifact.uri.as_deref() == Some(vcir.current_manifest_rsync_uri.as_str()) + }) + .map(|artifact| artifact.sha256.clone()) + .unwrap_or_default(), + kind: AuditObjectKind::Manifest, + result: AuditObjectResult::Ok, + detail: None, + }); + } + + for uri in ordered_uris { + if uri == vcir.current_manifest_rsync_uri { + continue; + } + if let Some(entry) = audit_by_uri.remove(&uri) { + objects_out.push(entry); + } + } + + let mut audit_objects = objects_out.clone(); + audit_objects.extend(fresh_failure_audits.iter().cloned()); + + PublicationPointAudit { + node_id: None, + parent_node_id: None, + discovered_from: None, + rsync_base_uri: ca.rsync_base_uri.clone(), + manifest_rsync_uri: ca.manifest_rsync_uri.clone(), + publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(), + rrdp_notification_uri: ca.rrdp_notification_uri.clone(), + source: source_label(source), + repo_sync_source: repo_sync_source.map(ToString::to_string), + repo_sync_phase: repo_sync_phase.map(ToString::to_string), + repo_sync_duration_ms, + repo_sync_error: repo_sync_error.map(ToString::to_string), + repo_terminal_state: terminal_state_label(source).to_string(), + this_update_rfc3339_utc: vcir + .validated_manifest_meta + .validated_manifest_this_update + .rfc3339_utc + .clone(), + next_update_rfc3339_utc: vcir + .validated_manifest_meta + .validated_manifest_next_update + .rfc3339_utc + .clone(), + verified_at_rfc3339_utc: vcir.last_successful_validation_time.rfc3339_utc.clone(), + warnings, + objects: audit_objects, + } +} + +fn build_publication_point_audit_from_publication_point_cache_projection( + ca: &CaInstanceHandle, + source: PublicationPointSource, + repo_sync_source: Option<&str>, + repo_sync_phase: Option<&str>, + repo_sync_duration_ms: Option, + repo_sync_error: Option<&str>, + projection: &PublicationPointCacheProjection, + validation_time: time::OffsetDateTime, + runner_warnings: &[Warning], + objects: &crate::validation::objects::ObjectsOutput, + child_audits: &[ObjectAuditEntry], +) -> PublicationPointAudit { + let mut warnings = Vec::new(); + warnings.extend(runner_warnings.iter().map(AuditWarning::from)); + warnings.extend(objects.warnings.iter().map(AuditWarning::from)); + + let mut audit_by_uri: HashMap = HashMap::new(); + for artifact in &projection.related_objects { + let Some(uri) = artifact.uri.as_ref() else { + continue; + }; + audit_by_uri.insert( + uri.clone(), + ObjectAuditEntry { + rsync_uri: uri.clone(), + sha256_hex: artifact.sha256.clone(), + kind: kind_from_vcir_artifact_kind(artifact.artifact_kind), + result: audit_result_from_vcir_status(artifact.validation_status), + detail: audit_detail_from_vcir_status( + artifact.validation_status, + artifact.reject_reason.as_deref(), + ), + }, + ); + } + for entry in child_audits { + audit_by_uri.insert(entry.rsync_uri.clone(), entry.clone()); + } + for entry in &objects.audit { + audit_by_uri.insert(entry.rsync_uri.clone(), entry.clone()); + } + + let mut ordered_uris: Vec = projection + .related_objects + .iter() + .filter_map(|artifact| artifact.uri.clone()) + .collect(); + ordered_uris.sort(); + ordered_uris.dedup(); + + let mut objects_out: Vec = Vec::with_capacity(ordered_uris.len().max(1)); + if let Some(entry) = audit_by_uri.remove(&projection.manifest_rsync_uri) { + objects_out.push(entry); + } else { + objects_out.push(ObjectAuditEntry { + rsync_uri: projection.manifest_rsync_uri.clone(), + sha256_hex: hex::encode(projection.manifest_sha256), + kind: AuditObjectKind::Manifest, + result: AuditObjectResult::Ok, + detail: None, + }); + } + + for uri in ordered_uris { + if uri == projection.manifest_rsync_uri { + continue; + } + if let Some(entry) = audit_by_uri.remove(&uri) { + objects_out.push(entry); + } + } + + PublicationPointAudit { + node_id: None, + parent_node_id: None, + discovered_from: None, + rsync_base_uri: ca.rsync_base_uri.clone(), + manifest_rsync_uri: ca.manifest_rsync_uri.clone(), + publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(), + rrdp_notification_uri: ca.rrdp_notification_uri.clone(), + source: source_label(source), + repo_sync_source: repo_sync_source.map(ToString::to_string), + repo_sync_phase: repo_sync_phase.map(ToString::to_string), + repo_sync_duration_ms, + repo_sync_error: repo_sync_error.map(ToString::to_string), + repo_terminal_state: terminal_state_label(source).to_string(), + this_update_rfc3339_utc: projection.manifest_this_update.rfc3339_utc.clone(), + next_update_rfc3339_utc: projection.manifest_next_update.rfc3339_utc.clone(), + verified_at_rfc3339_utc: PackTime::from_utc_offset_datetime(validation_time).rfc3339_utc, + warnings, + objects: objects_out, + } +} + +fn parse_snapshot_time_value(pack_time: &PackTime) -> Result { + time::OffsetDateTime::parse( + &pack_time.rfc3339_utc, + &time::format_description::well_known::Rfc3339, + ) + .map_err(|e| format!("invalid RFC3339 time '{}': {e}", pack_time.rfc3339_utc)) +} + +fn empty_objects_output() -> crate::validation::objects::ObjectsOutput { + crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: Vec::new(), + stats: crate::validation::objects::ObjectsStats::default(), + audit: Vec::new(), + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + } +} + +fn reuse_ccr_manifest_projection_from_vcir( + ca: &CaInstanceHandle, + vcir: &ValidatedCaInstanceResult, +) -> Result { + if vcir.ccr_manifest_projection.manifest_rsync_uri != ca.manifest_rsync_uri { + return Err(format!( + "vcir CCR manifest projection URI mismatch: expected {}, got {}", + ca.manifest_rsync_uri, vcir.ccr_manifest_projection.manifest_rsync_uri + )); + } + Ok(vcir.ccr_manifest_projection.clone()) +} + +fn project_current_instance_vcir_on_failed_fetch( + store: &RocksStore, + ca: &CaInstanceHandle, + fresh_err: &ManifestFreshError, + policy: &Policy, + validation_time: time::OffsetDateTime, +) -> Result { + let mut warnings = Vec::new(); + + let Some(vcir) = store + .get_vcir(&ca.manifest_rsync_uri) + .map_err(|e| format!("load VCIR failed: {e}"))? + else { + return Ok(failed_fetch_no_cache_projection( + ca, + fresh_err, + None, + "no latest validated result for current CA instance; no cached output reused", + )); + }; + + if !vcir.audit_summary.failed_fetch_eligible { + return Ok(failed_fetch_no_cache_projection( + ca, + fresh_err, + Some(vcir), + "latest VCIR is not marked failed-fetch eligible; no cached output reused", + )); + } + + let reuse_identity = match store.get_vcir_failed_fetch_reuse_identity(&ca.manifest_rsync_uri) { + Ok(Some(identity)) => identity, + Ok(None) => { + return Ok(failed_fetch_no_cache_projection( + ca, + fresh_err, + Some(vcir), + "latest VCIR reuse identity is missing; no cached output or child reused", + )); + } + Err(error) => { + return Ok(failed_fetch_no_cache_projection( + ca, + fresh_err, + Some(vcir), + &format!( + "latest VCIR reuse identity is invalid ({error}); no cached output or child reused" + ), + )); + } + }; + if !failed_fetch_reuse_identity_matches_current( + &reuse_identity, + &vcir, + ca, + policy, + validation_time, + ) { + return Ok(failed_fetch_no_cache_projection( + ca, + fresh_err, + Some(vcir), + "latest VCIR reuse identity does not match the current CA context or time window; no cached output or child reused", + )); + } + + let ccr_manifest_projection = reuse_ccr_manifest_projection_from_vcir(ca, &vcir)?; + if fresh_err.should_warn_when_current_instance_reused() { + warnings.push( + Warning::new(format!("manifest failed fetch: {fresh_err}")) + .with_rfc_refs(&[RfcRef("RFC 9286 §6.6")]) + .with_context(&ca.manifest_rsync_uri), + ); + } + // Current-instance reuse is fully described by VCIR projections; rebuilding a + // byte-backed snapshot here only duplicates repo-byte I/O and creates warning noise. + let snapshot = None; + let objects = build_objects_output_from_vcir(&vcir, validation_time, &mut warnings); + let (discovered_children, child_audits) = + restore_children_from_vcir(store, ca, &vcir, &mut warnings); + + Ok(VcirReuseProjection { + source: PublicationPointSource::VcirCurrentInstance, + vcir: Some(vcir), + ccr_manifest_projection: Some(ccr_manifest_projection), + snapshot, + objects, + child_audits, + discovered_children, + warnings, + }) +} + +fn failed_fetch_no_cache_projection( + ca: &CaInstanceHandle, + fresh_err: &ManifestFreshError, + vcir: Option, + reason: &str, +) -> VcirReuseProjection { + let warnings = vec![ + Warning::new(format!("manifest failed fetch: {fresh_err}")) + .with_rfc_refs(&[RfcRef("RFC 9286 §6.6")]) + .with_context(&ca.manifest_rsync_uri), + Warning::new(reason) + .with_rfc_refs(&[RfcRef("RFC 9286 §6.6")]) + .with_context(&ca.manifest_rsync_uri), + ]; + VcirReuseProjection { + source: PublicationPointSource::FailedFetchNoCache, + vcir, + ccr_manifest_projection: None, + snapshot: None, + objects: empty_objects_output(), + child_audits: Vec::new(), + discovered_children: Vec::new(), + warnings, + } +} + +fn failed_fetch_reuse_identity_for_fresh_result( + ca: &CaInstanceHandle, + policy: &Policy, + validation_time: time::OffsetDateTime, + effective_until: PackTime, +) -> Result { + let current_ca_sha256 = ca.ca_certificate_sha256_32().ok_or_else(|| { + "current CA certificate hash unavailable for VCIR reuse identity".to_string() + })?; + let identity = VcirFailedFetchReuseIdentity { + current_ca_sha256, + ta_context_digest: ta_context_digest_for_ca(ca), + ca_validation_context_digest: ca_validation_context_digest_for_ca(ca), + policy_fingerprint: publication_point_cache_policy_fingerprint(policy), + effective_not_before: PackTime::from_utc_offset_datetime(validation_time), + effective_until, + }; + identity + .validate_internal() + .map_err(|error| error.to_string())?; + Ok(identity) +} + +fn failed_fetch_reuse_identity_matches_current( + cached: &VcirFailedFetchReuseIdentity, + vcir: &ValidatedCaInstanceResult, + ca: &CaInstanceHandle, + policy: &Policy, + validation_time: time::OffsetDateTime, +) -> bool { + let Some(current_ca_sha256) = ca.ca_certificate_sha256_32() else { + return false; + }; + cached.current_ca_sha256 == current_ca_sha256 + && cached.ta_context_digest == ta_context_digest_for_ca(ca) + && cached.ca_validation_context_digest == ca_validation_context_digest_for_ca(ca) + && cached.policy_fingerprint == publication_point_cache_policy_fingerprint(policy) + && cached.effective_until == vcir.instance_gate.instance_effective_until + && cached.contains_validation_time(validation_time) +} + +#[cfg(test)] +fn reconstruct_snapshot_from_vcir( + store: &RocksStore, + ca: &CaInstanceHandle, + vcir: &ValidatedCaInstanceResult, + warnings: &mut Vec, +) -> Option { + let manifest_artifact = vcir.related_artifacts.iter().find(|artifact| { + artifact.artifact_role == VcirArtifactRole::Manifest + && artifact.uri.as_deref() == Some(ca.manifest_rsync_uri.as_str()) + })?; + let manifest_bytes = match store.get_blob_bytes(&manifest_artifact.sha256) { + Ok(Some(bytes)) => bytes, + Ok(None) => { + warnings.push( + Warning::new("manifest raw bytes missing for VCIR audit reconstruction") + .with_context(&ca.manifest_rsync_uri), + ); + return None; + } + Err(e) => { + warnings.push( + Warning::new(format!( + "manifest raw bytes load failed for VCIR audit reconstruction: {e}" + )) + .with_context(&ca.manifest_rsync_uri), + ); + return None; + } + }; + + let mut seen = HashSet::new(); + let mut files = Vec::new(); + for artifact in &vcir.related_artifacts { + let Some(uri) = artifact.uri.as_ref() else { + continue; + }; + if artifact.artifact_role == VcirArtifactRole::Manifest + || artifact.artifact_role == VcirArtifactRole::IssuerCert + || artifact.artifact_role == VcirArtifactRole::TrustAnchorCert + || artifact.artifact_role == VcirArtifactRole::Tal + { + continue; + } + if !seen.insert(uri.clone()) { + continue; + } + match store.get_blob_bytes(&artifact.sha256) { + Ok(Some(bytes)) => files.push(PackFile::from_bytes_compute_sha256(uri, bytes)), + Ok(None) => warnings.push( + Warning::new("related artifact raw bytes missing for VCIR audit reconstruction") + .with_context(uri), + ), + Err(e) => warnings.push( + Warning::new(format!( + "related artifact raw bytes load failed for VCIR audit reconstruction: {e}" + )) + .with_context(uri), + ), + } + } + + Some(PublicationPointSnapshot { + format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, + publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(), + manifest_rsync_uri: ca.manifest_rsync_uri.clone(), + manifest_number_be: vcir + .validated_manifest_meta + .validated_manifest_number + .clone(), + this_update: vcir + .validated_manifest_meta + .validated_manifest_this_update + .clone(), + next_update: vcir + .validated_manifest_meta + .validated_manifest_next_update + .clone(), + verified_at: vcir.last_successful_validation_time.clone(), + manifest_bytes, + files, + }) +} diff --git a/crates/panda-rpki-validator/src/validation/tree_runner/cache_methods.rs b/crates/panda-rpki-validator/src/validation/tree_runner/cache_methods.rs new file mode 100644 index 0000000..93363f9 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_runner/cache_methods.rs @@ -0,0 +1,538 @@ +impl<'a> Rpkiv1PublicationPointRunner<'a> { + pub(crate) fn roa_validation_cache_view_for_fresh_point( + &self, + manifest_rsync_uri: &str, + ) -> Option { + if !self.enable_roa_validation_cache { + return None; + } + let load_started = std::time::Instant::now(); + let loaded_projection = self.store.get_roa_cache_projection(manifest_rsync_uri); + if let Some(timing) = self.timing.as_ref() { + timing.record_phase_nanos( + "roa_validation_cache_projection_load_total", + load_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64, + ); + } + match loaded_projection { + Ok(Some(projection)) => { + if let Some(timing) = self.timing.as_ref() { + timing + .record_count("roa_validation_cache_projection_hit_publication_points", 1); + } + let view_started = std::time::Instant::now(); + let view = + RoaValidationCacheView::from_projection(&projection, self.validation_time); + if let Some(timing) = self.timing.as_ref() { + timing.record_phase_nanos( + "roa_validation_cache_projection_build_total", + view_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64, + ); + } + Some(view) + } + Ok(None) => { + if let Some(timing) = self.timing.as_ref() { + timing.record_count( + "roa_validation_cache_projection_missing_publication_points", + 1, + ); + } + None + } + Err(err) => { + if let Some(timing) = self.timing.as_ref() { + timing.record_count("roa_validation_cache_projection_load_errors", 1); + } + crate::progress_log::emit( + "roa_validation_cache_projection_load_error", + serde_json::json!({ + "manifest_rsync_uri": manifest_rsync_uri, + "error": err.to_string(), + }), + ); + None + } + } + } + + pub(crate) fn observe_or_reuse_publication_point_cache( + &self, + ca: &CaInstanceHandle, + repo_sync_source: Option<&str>, + repo_sync_phase: Option<&str>, + repo_sync_duration_ms: u64, + repo_sync_err: Option<&str>, + warnings: &[Warning], + ) -> Option { + if !self.publication_point_cache_observe_only + && !self.enable_publication_point_validation_cache + { + return None; + } + + let lookup_started = std::time::Instant::now(); + if let Some(timing) = self.timing.as_ref() { + timing.record_count("publication_point_cache_lookup_total", 1); + } + let projection = match self + .store + .get_publication_point_cache_projection_cached(&ca.manifest_rsync_uri) + { + Ok(Some(projection)) => projection, + Ok(None) => { + self.finish_publication_point_cache_miss(ca, "missing_projection", lookup_started); + return None; + } + Err(e) => { + self.finish_publication_point_cache_miss( + ca, + "projection_load_error", + lookup_started, + ); + crate::progress_log::emit( + "publication_point_cache_lookup_error", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "error": e.to_string(), + }), + ); + return None; + } + }; + + let current_identity = match self.current_publication_point_cache_identity(ca) { + Ok(identity) => identity, + Err(reason) => { + self.finish_publication_point_cache_miss(ca, reason.as_str(), lookup_started); + return None; + } + }; + + if projection.ca_cert_uri != ca.ca_certificate_rsync_uri { + self.finish_publication_point_cache_miss(ca, "ca_uri_mismatch", lookup_started); + return None; + } + if projection.ca_cert_sha256 != current_identity.ca_cert_sha256 { + self.finish_publication_point_cache_miss(ca, "ca_hash_mismatch", lookup_started); + return None; + } + if projection.manifest_sha256 != current_identity.manifest_sha256 { + self.finish_publication_point_cache_miss(ca, "manifest_hash_mismatch", lookup_started); + return None; + } + if projection.tal_id != ca.tal_id { + self.finish_publication_point_cache_miss(ca, "tal_mismatch", lookup_started); + return None; + } + if projection.ta_context_digest != current_identity.ta_context_digest { + self.finish_publication_point_cache_miss(ca, "ta_context_mismatch", lookup_started); + return None; + } + if projection.ca_validation_context_digest != current_identity.ca_validation_context_digest + { + self.finish_publication_point_cache_miss(ca, "parent_context_mismatch", lookup_started); + return None; + } + if projection.validation_policy_fingerprint != current_identity.policy_fingerprint { + self.finish_publication_point_cache_miss(ca, "policy_mismatch", lookup_started); + return None; + } + + let instance_not_before = + match parse_snapshot_time_value(&projection.instance_effective_not_before) { + Ok(value) => value, + Err(_) => { + self.finish_publication_point_cache_miss( + ca, + "instance_not_before_invalid", + lookup_started, + ); + return None; + } + }; + let instance_until = match parse_snapshot_time_value(&projection.instance_effective_until) { + Ok(value) => value, + Err(_) => { + self.finish_publication_point_cache_miss( + ca, + "instance_until_invalid", + lookup_started, + ); + return None; + } + }; + if self.validation_time < instance_not_before || self.validation_time >= instance_until { + self.finish_publication_point_cache_miss(ca, "instance_time_gate_miss", lookup_started); + return None; + } + if let Err(reason) = + publication_point_cache_projection_items_valid(&projection, self.validation_time) + { + self.finish_publication_point_cache_miss(ca, reason, lookup_started); + return None; + } + + if let Some(timing) = self.timing.as_ref() { + let lookup_nanos = lookup_started + .elapsed() + .as_nanos() + .min(u128::from(u64::MAX)) as u64; + timing.record_count("publication_point_cache_theoretical_hits", 1); + timing.record_phase_nanos("publication_point_cache_lookup_total", lookup_nanos); + timing.record_phase_nanos("publication_point_cache_lookup_hit_total", lookup_nanos); + timing.record_phase_nanos( + "publication_point_cache_lookup_duration_total", + lookup_nanos, + ); + } + if self.publication_point_cache_observe_only { + return None; + } + + match self.build_publication_point_cache_result( + ca, + projection, + repo_sync_source, + repo_sync_phase, + repo_sync_duration_ms, + repo_sync_err, + warnings, + ) { + Ok(result) => { + if let Some(timing) = self.timing.as_ref() { + timing.record_count("publication_point_cache_reuse_hits", 1); + } + Some(result) + } + Err(e) => { + self.record_publication_point_cache_miss("reuse_build_error"); + crate::progress_log::emit( + "publication_point_cache_reuse_error", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "error": e, + }), + ); + None + } + } + } + + fn finish_publication_point_cache_miss( + &self, + ca: &CaInstanceHandle, + reason: &str, + lookup_started: std::time::Instant, + ) { + self.record_publication_point_cache_miss(reason); + let lookup_nanos = lookup_started + .elapsed() + .as_nanos() + .min(u128::from(u64::MAX)) as u64; + if let Some(timing) = self.timing.as_ref() { + timing.record_phase_nanos("publication_point_cache_lookup_miss_total", lookup_nanos); + timing.record_phase_nanos( + "publication_point_cache_lookup_duration_total", + lookup_nanos, + ); + } + let elapsed_ms = lookup_nanos / 1_000_000; + if elapsed_ms >= crate::progress_log::pp_cache_slow_threshold_ms() { + crate::progress_log::emit( + "publication_point_cache_miss_slow", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri.as_str(), + "publication_point_rsync_uri": ca.publication_point_rsync_uri.as_str(), + "ca_certificate_rsync_uri": ca.ca_certificate_rsync_uri.as_deref(), + "reason": reason, + "elapsed_ms": elapsed_ms, + "slow_threshold_ms": crate::progress_log::pp_cache_slow_threshold_ms(), + }), + ); + } + } + + fn record_publication_point_cache_miss(&self, reason: &str) { + if let Some(timing) = self.timing.as_ref() { + timing.record_count("publication_point_cache_miss_total", 1); + match reason { + "missing_projection" => { + timing.record_count("publication_point_cache_miss_missing_projection", 1) + } + "projection_load_error" => { + timing.record_count("publication_point_cache_miss_projection_load_error", 1) + } + "current_manifest_missing" => { + timing.record_count("publication_point_cache_miss_current_manifest_missing", 1) + } + "ca_uri_mismatch" => { + timing.record_count("publication_point_cache_miss_ca_uri_mismatch", 1) + } + "ca_hash_mismatch" => { + timing.record_count("publication_point_cache_miss_ca_hash_mismatch", 1) + } + "manifest_hash_mismatch" => { + timing.record_count("publication_point_cache_miss_manifest_hash_mismatch", 1) + } + "tal_mismatch" => { + timing.record_count("publication_point_cache_miss_tal_mismatch", 1) + } + "ta_context_mismatch" => { + timing.record_count("publication_point_cache_miss_ta_context_mismatch", 1) + } + "parent_context_mismatch" => { + timing.record_count("publication_point_cache_miss_parent_context_mismatch", 1) + } + "policy_mismatch" => { + timing.record_count("publication_point_cache_miss_policy_mismatch", 1) + } + "instance_not_before_invalid" => timing.record_count( + "publication_point_cache_miss_instance_not_before_invalid", + 1, + ), + "instance_until_invalid" => { + timing.record_count("publication_point_cache_miss_instance_until_invalid", 1) + } + "instance_time_gate_miss" => { + timing.record_count("publication_point_cache_miss_instance_time_gate", 1) + } + "output_time_gate_miss" => { + timing.record_count("publication_point_cache_miss_output_time_gate", 1) + } + "child_time_gate_miss" => { + timing.record_count("publication_point_cache_miss_child_time_gate", 1) + } + "reuse_build_error" => { + timing.record_count("publication_point_cache_miss_reuse_build_error", 1) + } + _ => timing.record_count("publication_point_cache_miss_other", 1), + } + } + } + + fn current_publication_point_cache_identity( + &self, + ca: &CaInstanceHandle, + ) -> Result { + let ca_cert_sha256 = match ca.ca_certificate_rsync_uri.as_deref() { + Some(uri) => self + .current_hash_for_uri(uri) + .or_else(|| ca.ca_certificate_sha256_32()) + .ok_or_else(|| "current_ca_certificate_hash_missing".to_string())?, + None => ca + .ca_certificate_sha256_32() + .ok_or_else(|| "current_ca_certificate_hash_missing".to_string())?, + }; + let manifest_sha256 = self + .current_hash_for_uri(&ca.manifest_rsync_uri) + .ok_or_else(|| "current_manifest_missing".to_string())?; + Ok(PublicationPointCacheIdentity { + ca_cert_sha256, + manifest_sha256, + ta_context_digest: ta_context_digest_for_ca(ca), + ca_validation_context_digest: ca_validation_context_digest_for_ca(ca), + policy_fingerprint: publication_point_cache_policy_fingerprint(self.policy), + }) + } + + fn current_hash_for_uri(&self, uri: &str) -> Option<[u8; 32]> { + if let Some(index) = self.current_repo_index.as_ref() { + if let Ok(index) = index.read() { + if let Some(entry) = index.get_by_uri(uri) { + return Some(entry.current_hash); + } + } + } + self.store + .load_current_object_with_hash_by_uri(uri) + .ok() + .flatten() + .map(|entry| entry.current_hash) + } + + fn build_publication_point_cache_result( + &self, + ca: &CaInstanceHandle, + projection: PublicationPointCacheProjection, + repo_sync_source: Option<&str>, + repo_sync_phase: Option<&str>, + repo_sync_duration_ms: u64, + repo_sync_err: Option<&str>, + warnings: &[Warning], + ) -> Result { + let build_started = std::time::Instant::now(); + let mut warnings = warnings.to_vec(); + let output_reuse_count = projection.outputs.len() as u64; + let child_reuse_count = projection.children.len() as u64; + let related_object_reuse_count = projection.related_objects.len() as u64; + let build_objects_started = std::time::Instant::now(); + let mut objects = build_objects_output_from_publication_point_cache_projection( + &projection, + self.validation_time, + &mut warnings, + ); + let build_objects_ms = self.record_publication_point_cache_phase_ms( + "publication_point_cache_build_objects_total", + build_objects_started, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "publication_point_cache_build_objects", + build_objects_ms, + ); + let restore_children_started = std::time::Instant::now(); + let child_restore_workers = self.publication_point_cache_child_restore_worker_count(); + let (discovered_children, child_audits) = restore_children_from_publication_point_cache( + self.store, + ca, + &projection, + self.validation_time, + &mut warnings, + child_restore_workers, + self.timing.as_ref(), + ); + let restore_children_ms = self.record_publication_point_cache_phase_ms( + "publication_point_cache_restore_children_total", + restore_children_started, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "publication_point_cache_restore_children", + restore_children_ms, + ); + let ccr_projection = projection.ccr_manifest_projection.clone(); + let ccr_append_started = std::time::Instant::now(); + self.append_ccr_manifest_projection(&ccr_projection)?; + let ccr_append_ms = self.record_publication_point_cache_phase_ms( + "publication_point_cache_ccr_append_total", + ccr_append_started, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "publication_point_cache_ccr_append", + ccr_append_ms, + ); + let audit_build_started = std::time::Instant::now(); + let audit = build_publication_point_audit_from_publication_point_cache_projection( + ca, + PublicationPointSource::PublicationPointCache, + repo_sync_source, + repo_sync_phase, + Some(repo_sync_duration_ms), + repo_sync_err, + &projection, + self.validation_time, + &warnings, + &objects, + &child_audits, + ); + let audit_build_ms = self.record_publication_point_cache_phase_ms( + "publication_point_cache_audit_build_total", + audit_build_started, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "publication_point_cache_audit_build", + audit_build_ms, + ); + let audit_object_count = audit.objects.len() as u64; + let cir_cached_objects = audit.objects.clone(); + let cir_cached_objects_count = cir_cached_objects.len() as u64; + objects.local_outputs_cache.clear(); + let total_ms = self.record_publication_point_cache_phase_ms( + "publication_point_cache_reuse_build_total", + build_started, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "publication_point_cache_reuse_build", + total_ms, + ); + if let Some(timing) = self.timing.as_ref() { + timing.record_count("publication_point_cache_outputs_reused", output_reuse_count); + timing.record_count("publication_point_cache_children_reused", child_reuse_count); + timing.record_count( + "publication_point_cache_related_objects_reused", + related_object_reuse_count, + ); + timing.record_count( + "publication_point_cache_audit_objects_reused", + audit_object_count, + ); + if total_ms > 0 { + timing.record_count("publication_point_cache_reuse_nonzero_ms_hits", 1); + timing.record_count("publication_point_cache_reuse_nonzero_ms_total", total_ms); + } + } + if total_ms >= crate::progress_log::pp_cache_slow_threshold_ms() { + crate::progress_log::emit( + "publication_point_cache_reuse_slow", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri.as_str(), + "publication_point_rsync_uri": ca.publication_point_rsync_uri.as_str(), + "repo_sync_source": repo_sync_source, + "repo_sync_phase": repo_sync_phase, + "repo_sync_duration_ms": repo_sync_duration_ms, + "repo_sync_err": repo_sync_err, + "outputs_reused": output_reuse_count, + "children_reused": child_reuse_count, + "related_objects_reused": related_object_reuse_count, + "audit_objects_reused": audit_object_count, + "build_objects_ms": build_objects_ms, + "restore_children_ms": restore_children_ms, + "restore_children_workers": child_restore_workers, + "ccr_append_ms": ccr_append_ms, + "audit_build_ms": audit_build_ms, + "total_ms": total_ms, + "cir_cached_objects": cir_cached_objects_count, + "slow_threshold_ms": crate::progress_log::pp_cache_slow_threshold_ms(), + }), + ); + } + Ok(PublicationPointRunResult { + source: PublicationPointSource::PublicationPointCache, + snapshot: None, + warnings, + objects, + audit, + cir_fresh_objects: Vec::new(), + cir_cached_objects, + discovered_children, + }) + } + + fn record_publication_point_cache_phase_ms( + &self, + phase: &'static str, + started: std::time::Instant, + ) -> u64 { + let nanos = started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64; + if let Some(timing) = self.timing.as_ref() { + timing.record_phase_nanos(phase, nanos); + } + nanos / 1_000_000 + } + + pub(crate) fn record_publication_point_total_ms(&self, manifest_rsync_uri: &str, ms: u64) { + if let Some(timing) = self.timing.as_ref() { + timing.record_publication_point_nanos(manifest_rsync_uri, ms.saturating_mul(1_000_000)); + } + } + + pub(crate) fn record_publication_point_step_ms( + &self, + manifest_rsync_uri: &str, + step: &'static str, + ms: u64, + ) { + if let Some(timing) = self.timing.as_ref() { + timing.record_publication_point_step_nanos( + manifest_rsync_uri, + step, + ms.saturating_mul(1_000_000), + ); + } + } + +} diff --git a/crates/panda-rpki-validator/src/validation/tree_runner/cache_reuse_methods.rs b/crates/panda-rpki-validator/src/validation/tree_runner/cache_reuse_methods.rs new file mode 100644 index 0000000..175437d --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_runner/cache_reuse_methods.rs @@ -0,0 +1,481 @@ +impl<'a> Rpkiv1PublicationPointRunner<'a> { + fn publication_point_cache_child_restore_worker_count(&self) -> usize { + self.parallel_phase2_config + .as_ref() + .map(|config| config.object_workers) + .unwrap_or(1) + .clamp(1, PUBLICATION_POINT_CACHE_CHILD_RESTORE_MAX_WORKERS) + } + + pub(crate) fn ccr_accumulator_snapshot(&self) -> Option { + self.ccr_accumulator + .as_ref() + .and_then(|accumulator| accumulator.lock().ok().map(|guard| guard.clone())) + } + + pub(crate) fn append_ccr_manifest_projection( + &self, + projection: &VcirCcrManifestProjection, + ) -> Result<(), String> { + if let Some(accumulator) = self.ccr_accumulator.as_ref() { + accumulator + .lock() + .map_err(|_| "lock CCR accumulator failed".to_string())? + .append_manifest_projection(projection)?; + } + Ok(()) + } + + fn append_ccr_manifest_projection_from_reuse( + &self, + projection: &VcirReuseProjection, + ) -> Result<(), String> { + match projection.source { + PublicationPointSource::Fresh => Err( + "invalid reuse projection source: fresh does not belong to failed-fetch reuse" + .to_string(), + ), + PublicationPointSource::PublicationPointCache => self.append_ccr_manifest_projection( + projection.ccr_manifest_projection.as_ref().ok_or_else(|| { + "publication-point cache reuse is missing CCR manifest projection".to_string() + })?, + ), + PublicationPointSource::VcirCurrentInstance => self.append_ccr_manifest_projection( + projection.ccr_manifest_projection.as_ref().ok_or_else(|| { + "vcir current-instance reuse is missing CCR manifest projection".to_string() + })?, + ), + PublicationPointSource::FailedFetchNoCache => Ok(()), + } + } + + fn current_manifest_hash_hex_for_audit(&self, ca: &CaInstanceHandle) -> Option { + if let Some(index_handle) = self.current_repo_index.as_ref() + && let Ok(index) = index_handle.read() + && let Some(entry) = index.get_by_uri(&ca.manifest_rsync_uri) + { + return Some(entry.current_hash_hex.clone()); + } + + self.store + .load_current_object_with_hash_by_uri(&ca.manifest_rsync_uri) + .ok() + .flatten() + .map(|current| current.current_hash_hex) + } + + fn rejected_manifest_audit_entry_for_failed_fetch( + &self, + ca: &CaInstanceHandle, + fresh_err: &ManifestFreshError, + ) -> Option { + let sha256_hex = self.current_manifest_hash_hex_for_audit(ca)?; + Some(ObjectAuditEntry { + rsync_uri: ca.manifest_rsync_uri.clone(), + sha256_hex, + kind: AuditObjectKind::Manifest, + result: AuditObjectResult::Error, + detail: Some(fresh_err.to_string()), + }) + } + + fn fresh_failure_audit_entries_for_cir( + &self, + ca: &CaInstanceHandle, + fresh_err: &ManifestFreshError, + ) -> Vec { + if !fresh_err.should_warn_when_current_instance_reused() { + return Vec::new(); + } + self.rejected_manifest_audit_entry_for_failed_fetch(ca, fresh_err) + .into_iter() + .collect() + } + + pub(crate) fn stage_fresh_publication_point_after_repo_ready( + &self, + ca: &CaInstanceHandle, + repo_sync_ok: bool, + repo_sync_err: Option<&str>, + ) -> Result { + let snapshot_prepare_started = std::time::Instant::now(); + let issuer_ca_der = ca_certificate_der_for_validation(ca, self.store, self.timing.as_ref()) + .map_err(|detail| FreshPublicationPointStageError { + error: ManifestFreshError::IssuerCaLoadFailed { detail }, + snapshot_prepare_ms: snapshot_prepare_started.elapsed().as_millis() as u64, + })?; + let issuer_ca_der: Arc<[u8]> = Arc::from(issuer_ca_der.as_ref()); + let fresh_publication_point = { + let _manifest_total = self + .timing + .as_ref() + .map(|t| t.span_phase("manifest_processing_total")); + process_manifest_publication_point_fresh_after_repo_sync_with_timing( + self.store, + &ca.manifest_rsync_uri, + &ca.publication_point_rsync_uri, + self.current_repo_index.as_ref(), + issuer_ca_der.as_ref(), + ca.ca_certificate_rsync_uri.as_deref(), + self.validation_time, + repo_sync_ok, + repo_sync_err, + ) + }; + let snapshot_prepare_ms = snapshot_prepare_started.elapsed().as_millis() as u64; + let (fresh_point, snapshot_prepare_timing) = + fresh_publication_point.map_err(|error| FreshPublicationPointStageError { + error, + snapshot_prepare_ms, + })?; + if let Some(timing) = self.timing.as_ref() { + timing.record_phase_nanos( + "fresh_snapshot_prepare_total", + snapshot_prepare_ms.saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_snapshot_manifest_load_total", + snapshot_prepare_timing + .manifest_load_ms + .saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_snapshot_manifest_decode_total", + snapshot_prepare_timing + .manifest_decode_ms + .saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_snapshot_replay_guard_total", + snapshot_prepare_timing + .replay_guard_ms + .saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_snapshot_manifest_entries_total", + snapshot_prepare_timing + .manifest_entries_ms + .saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_snapshot_pack_files_total", + snapshot_prepare_timing + .pack_files_ms + .saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_snapshot_ee_path_validate_total", + snapshot_prepare_timing + .ee_path_validate_ms + .saturating_mul(1_000_000), + ); + timing.record_count("fresh_publication_points", 1); + timing.record_count( + "fresh_manifest_files_total", + snapshot_prepare_timing.manifest_file_count as u64, + ); + } + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_snapshot_prepare", + snapshot_prepare_ms, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_snapshot_manifest_load", + snapshot_prepare_timing.manifest_load_ms, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_snapshot_manifest_decode", + snapshot_prepare_timing.manifest_decode_ms, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_snapshot_replay_guard", + snapshot_prepare_timing.replay_guard_ms, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_snapshot_manifest_entries", + snapshot_prepare_timing.manifest_entries_ms, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_snapshot_pack_files", + snapshot_prepare_timing.pack_files_ms, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_snapshot_ee_path_validate", + snapshot_prepare_timing.ee_path_validate_ms, + ); + + let child_discovery_started = std::time::Instant::now(); + let out = { + let _child_disc_total = self + .timing + .as_ref() + .map(|t| t.span_phase("child_discovery_total")); + discover_children_from_fresh_snapshot_with_audit_cached_with_issuer_der( + ca, + issuer_ca_der.as_ref(), + &fresh_point, + self.validation_time, + self.timing.as_ref(), + self.policy, + if self.enable_child_certificate_validation_cache { + Some(ChildCertificateValidationCacheContext { + store: self.store, + issuer_ca_sha256: sha256_digest_32(issuer_ca_der.as_ref()), + ca_validation_context_digest: ca_validation_context_digest_for_ca(ca), + policy_fingerprint: publication_point_cache_policy_fingerprint(self.policy), + }) + } else { + None + }, + ) + }; + let (discovered_children, child_audits, discovered_router_keys, warnings) = match out { + Ok(out) => (out.children, out.audits, out.router_keys, Vec::new()), + Err(e) => ( + Vec::new(), + Vec::new(), + Vec::new(), + vec![ + Warning::new(format!("child CA discovery failed: {e}")) + .with_rfc_refs(&[RfcRef("RFC 6487 §7.2")]) + .with_context(&ca.manifest_rsync_uri), + ], + ), + }; + let child_discovery_ms = child_discovery_started.elapsed().as_millis() as u64; + if let Some(timing) = self.timing.as_ref() { + timing.record_phase_nanos( + "fresh_child_discovery_total", + child_discovery_ms.saturating_mul(1_000_000), + ); + timing.record_count( + "fresh_children_discovered", + discovered_children.len() as u64, + ); + timing.record_count("fresh_child_audits", child_audits.len() as u64); + timing.record_count( + "fresh_router_keys_discovered", + discovered_router_keys.len() as u64, + ); + } + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_child_discovery", + child_discovery_ms, + ); + + Ok(FreshPublicationPointStage { + fresh_point, + issuer_ca_der, + snapshot_prepare_timing, + snapshot_prepare_ms, + discovered_children, + child_audits, + discovered_router_keys, + child_discovery_ms, + warnings, + }) + } + + pub(crate) fn finalize_fresh_publication_point_from_reducer( + &self, + ca: &CaInstanceHandle, + fresh_point: &FreshValidatedPublicationPoint, + warnings: Vec, + mut objects: crate::validation::objects::ObjectsOutput, + child_audits: Vec, + discovered_children: Vec, + repo_sync_source: Option<&str>, + repo_sync_phase: Option<&str>, + repo_sync_duration_ms: u64, + repo_sync_err: Option<&str>, + ) -> Result { + let snapshot_pack_started = std::time::Instant::now(); + let pack = fresh_point.to_publication_point_snapshot(); + let snapshot_pack_ms = snapshot_pack_started.elapsed().as_millis() as u64; + if let Some(timing) = self.timing.as_ref() { + timing.record_phase_nanos( + "fresh_snapshot_pack_total", + snapshot_pack_ms.saturating_mul(1_000_000), + ); + } + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_snapshot_pack", + snapshot_pack_ms, + ); + + let persist_vcir_started = std::time::Instant::now(); + let persist_vcir_timing = if self.persist_vcir { + persist_vcir_for_fresh_result_with_timing( + self.store, + self.policy, + ca, + &pack, + &mut objects, + &warnings, + &child_audits, + &discovered_children, + self.validation_time, + self.publication_point_cache_observe_only + || self.enable_publication_point_validation_cache, + ) + .map_err(|e| format!("persist VCIR failed: {e}"))? + } else { + PersistVcirTimingBreakdown::default() + }; + let persist_vcir_ms = persist_vcir_started.elapsed().as_millis() as u64; + if let Some(timing) = self.timing.as_ref() { + timing.record_phase_nanos( + "fresh_persist_vcir_total", + persist_vcir_ms.saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_persist_embedded_store_total", + persist_vcir_timing + .embedded_store_ms + .saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_persist_build_vcir_total", + persist_vcir_timing.build_vcir_ms.saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_persist_replace_vcir_total", + persist_vcir_timing + .replace_vcir_ms + .saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_persist_local_outputs_total", + persist_vcir_timing + .build_vcir + .local_outputs_ms + .saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_persist_child_entries_total", + persist_vcir_timing + .build_vcir + .child_entries_ms + .saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_persist_related_artifacts_total", + persist_vcir_timing + .build_vcir + .related_artifacts_ms + .saturating_mul(1_000_000), + ); + if persist_vcir_timing.publication_point_cache_future_notbefore_guarded { + timing.record_count("publication_point_cache_future_notbefore_guarded", 1); + } + } + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_persist_vcir", + persist_vcir_ms, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_persist_build_vcir", + persist_vcir_timing.build_vcir_ms, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_persist_replace_vcir", + persist_vcir_timing.replace_vcir_ms, + ); + + // local_outputs_cache only exists to build/persist VCIR. Release it before the + // publication point result is retained for the rest of the run. + let _released_local_outputs = std::mem::take(&mut objects.local_outputs_cache); + let _released_roa_cache_object_meta = std::mem::take(&mut objects.roa_cache_object_meta); + + let mut ccr_projection_build_ms = 0; + let mut ccr_append_ms = 0; + if self.ccr_accumulator.is_some() { + let ccr_projection_build_started = std::time::Instant::now(); + let child_entries = + build_vcir_child_entries(self.store, &discovered_children, self.validation_time)?; + let ccr_manifest_projection = + build_vcir_ccr_manifest_projection_from_fresh(ca, &pack, &child_entries)?; + ccr_projection_build_ms = ccr_projection_build_started.elapsed().as_millis() as u64; + let ccr_append_started = std::time::Instant::now(); + self.append_ccr_manifest_projection(&ccr_manifest_projection)?; + ccr_append_ms = ccr_append_started.elapsed().as_millis() as u64; + } + if let Some(timing) = self.timing.as_ref() { + timing.record_phase_nanos( + "fresh_ccr_projection_build_total", + ccr_projection_build_ms.saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_ccr_append_total", + ccr_append_ms.saturating_mul(1_000_000), + ); + } + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_ccr_projection_build", + ccr_projection_build_ms, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_ccr_append", + ccr_append_ms, + ); + + let audit_build_started = std::time::Instant::now(); + let audit = build_publication_point_audit_from_snapshot( + ca, + PublicationPointSource::Fresh, + repo_sync_source, + repo_sync_phase, + Some(repo_sync_duration_ms), + repo_sync_err, + &pack, + &warnings, + &objects, + &child_audits, + ); + let audit_build_ms = audit_build_started.elapsed().as_millis() as u64; + if let Some(timing) = self.timing.as_ref() { + timing.record_phase_nanos( + "fresh_audit_build_total", + audit_build_ms.saturating_mul(1_000_000), + ); + } + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_audit_build", + audit_build_ms, + ); + + Ok(FreshPublicationPointFinalizeOutput { + result: PublicationPointRunResult { + source: PublicationPointSource::Fresh, + snapshot: Some(pack), + warnings, + objects, + audit, + cir_fresh_objects: Vec::new(), + cir_cached_objects: Vec::new(), + discovered_children, + }, + snapshot_pack_ms, + persist_vcir_ms, + persist_vcir_timing, + ccr_projection_build_ms, + ccr_append_ms, + audit_build_ms, + }) + } +} diff --git a/crates/panda-rpki-validator/src/validation/tree_runner/cache_types.rs b/crates/panda-rpki-validator/src/validation/tree_runner/cache_types.rs new file mode 100644 index 0000000..c729c46 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_runner/cache_types.rs @@ -0,0 +1,339 @@ +struct ChildDiscoveryOutput { + children: Vec, + audits: Vec, + router_keys: Vec, +} + +#[derive(Clone, Debug)] +struct VerifiedIssuerCrl { + crl: crate::data_model::crl::RpkixCrl, + revoked_serials: std::collections::HashSet>, + sha256_hex: String, +} + +#[derive(Clone, Debug)] +enum CachedIssuerCrl { + Pending { + bytes: Vec, + sha256_hex: Option, + }, + Ok(VerifiedIssuerCrl), +} + +impl CachedIssuerCrl { + fn current_sha256_hex(&mut self) -> &str { + match self { + CachedIssuerCrl::Pending { bytes, sha256_hex } => { + if sha256_hex.is_none() { + *sha256_hex = Some(crate::audit::sha256_hex(bytes)); + } + sha256_hex + .as_deref() + .expect("pending CRL sha256 must be populated") + } + CachedIssuerCrl::Ok(verified) => verified.sha256_hex.as_str(), + } + } +} + +struct PublicationPointCacheIdentity { + ca_cert_sha256: [u8; 32], + manifest_sha256: [u8; 32], + ta_context_digest: [u8; 32], + ca_validation_context_digest: [u8; 32], + policy_fingerprint: [u8; 32], +} + +fn sha256_digest_32(bytes: impl AsRef<[u8]>) -> [u8; 32] { + let digest = sha2::Sha256::digest(bytes.as_ref()); + let mut out = [0u8; 32]; + out.copy_from_slice(&digest); + out +} + +fn hash_serialized_parts(parts: &[(&str, &[u8])]) -> [u8; 32] { + let mut hasher = sha2::Sha256::new(); + for (label, value) in parts { + hasher.update((label.len() as u64).to_be_bytes()); + hasher.update(label.as_bytes()); + hasher.update((value.len() as u64).to_be_bytes()); + hasher.update(*value); + } + let digest = hasher.finalize(); + let mut out = [0u8; 32]; + out.copy_from_slice(&digest); + out +} + +fn cbor_or_debug_bytes(value: &T) -> Vec { + serde_cbor::to_vec(value).unwrap_or_else(|_| format!("{value:?}").into_bytes()) +} + +fn ta_context_digest_for_ca(ca: &CaInstanceHandle) -> [u8; 32] { + hash_serialized_parts(&[ + ("version", b"publication-point-cache-ta-v1"), + ("tal_id", ca.tal_id.as_bytes()), + ]) +} + +pub(crate) fn ca_validation_context_digest_for_ca(ca: &CaInstanceHandle) -> [u8; 32] { + let parent_manifest = ca + .parent_manifest_rsync_uri + .as_deref() + .unwrap_or("") + .as_bytes(); + let effective_ip = cbor_or_debug_bytes(&ca.effective_ip_resources); + let effective_as = cbor_or_debug_bytes(&ca.effective_as_resources); + hash_serialized_parts(&[ + ("version", b"publication-point-cache-parent-context-v1"), + ("tal_id", ca.tal_id.as_bytes()), + ("parent_manifest", parent_manifest), + ("effective_ip", effective_ip.as_slice()), + ("effective_as", effective_as.as_slice()), + ]) +} + +pub(crate) fn publication_point_cache_policy_fingerprint(policy: &Policy) -> [u8; 32] { + let policy_bytes = cbor_or_debug_bytes(policy); + hash_serialized_parts(&[ + ("version", b"publication-point-cache-policy-v3"), + ("policy", policy_bytes.as_slice()), + ("ta_constraints", policy.ta_constraints.fingerprint_bytes()), + ]) +} + +fn publication_point_cache_projection_items_valid( + projection: &PublicationPointCacheProjection, + validation_time: time::OffsetDateTime, +) -> Result<(), &'static str> { + for output in &projection.outputs { + if !pack_time_window_contains( + &output.item_effective_not_before, + &output.item_effective_until, + validation_time, + ) { + return Err("output_time_gate_miss"); + } + } + for child in &projection.children { + if !pack_time_window_contains( + &child.child_effective_not_before, + &child.child_effective_until, + validation_time, + ) { + return Err("child_time_gate_miss"); + } + } + Ok(()) +} + +fn pack_time_window_contains( + not_before: &PackTime, + until: &PackTime, + validation_time: time::OffsetDateTime, +) -> bool { + let Ok(not_before) = parse_snapshot_time_value(not_before) else { + return false; + }; + let Ok(until) = parse_snapshot_time_value(until) else { + return false; + }; + validation_time >= not_before && validation_time < until +} + +#[derive(Clone, Copy)] +struct ChildCertificateValidationCacheContext<'a> { + store: &'a RocksStore, + issuer_ca_sha256: [u8; 32], + ca_validation_context_digest: [u8; 32], + policy_fingerprint: [u8; 32], +} + +fn child_certificate_cache_key_sha256_hex( + child_cert_uri: &str, + child_cert_sha256: &[u8; 32], + issuer_ca_sha256: &[u8; 32], + ca_validation_context_digest: &[u8; 32], + policy_fingerprint: &[u8; 32], +) -> String { + let digest = hash_serialized_parts(&[ + ("version", b"child-certificate-cache-key-v1"), + ("child_cert_uri", child_cert_uri.as_bytes()), + ("child_cert_sha256", child_cert_sha256), + ("issuer_ca_sha256", issuer_ca_sha256), + // Keep the persisted cache-key label stable; this change only renames Rust identifiers. + ("parent_context_digest", ca_validation_context_digest), + ("policy_fingerprint", policy_fingerprint), + ]); + sha256_hex_from_32(&digest) +} + +#[derive(Clone, Debug)] +struct ChildCertificateCacheCandidate { + projection: Option, +} + +fn remember_child_certificate_cache_dirty_projection( + dirty_projections: &mut Option>, + projection: &ChildCertificateCacheProjection, +) { + if let Some(dirty_projections) = dirty_projections { + dirty_projections.insert(projection.cache_key_sha256_hex.clone(), projection.clone()); + } +} + +fn load_child_certificate_der_for_discovery<'a>( + file: &'a PackFile, + elapsed_nanos: &mut u64, + count: &mut u64, +) -> Result<&'a [u8], String> { + let started = std::time::Instant::now(); + let bytes = file + .bytes() + .map_err(|e| format!("child certificate bytes load failed: {e}"))?; + *elapsed_nanos = + elapsed_nanos.saturating_add(started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); + *count = count.saturating_add(1); + Ok(bytes) +} + +fn ca_certificate_der_for_validation<'a>( + ca: &'a CaInstanceHandle, + store: &RocksStore, + timing: Option<&TimingHandle>, +) -> Result, String> { + let started = std::time::Instant::now(); + let was_lazy = ca.ca_certificate_sha256_hex().is_some(); + let der = ca.ca_certificate_der(store)?; + if was_lazy { + let elapsed = started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64; + if let Some(timing) = timing { + timing.record_count("ca_certificate_lazy_load_count", 1); + timing.record_count("ca_certificate_lazy_load_bytes", der.len() as u64); + timing.record_phase_nanos("ca_certificate_lazy_load_total", elapsed); + } + } + Ok(der) +} + +fn child_certificate_cache_certificate_window( + child_not_before: time::OffsetDateTime, + child_not_after: time::OffsetDateTime, + issuer_not_before: time::OffsetDateTime, + issuer_not_after: time::OffsetDateTime, +) -> (PackTime, PackTime) { + let effective_not_before = child_not_before.max(issuer_not_before); + let effective_until = child_not_after.min(issuer_not_after); + ( + PackTime::from_utc_offset_datetime(effective_not_before), + PackTime::from_utc_offset_datetime(effective_until), + ) +} + +fn get_current_crl_sha256_hex( + crl_rsync_uri: &str, + crl_cache: &mut std::collections::HashMap, +) -> Option { + crl_cache + .get_mut(crl_rsync_uri) + .map(|entry| entry.current_sha256_hex().to_string()) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ChildCertificateCacheCrlGate { + Unchanged, + ChangedValid, + Expired, + Invalid, + Missing, +} + +#[derive(Default)] +struct ChildCertificateCacheCrlGateSet { + gates_by_uri_and_expected_hash: + std::collections::HashMap<(String, String), ChildCertificateCacheCrlGate>, +} + +impl ChildCertificateCacheCrlGateSet { + fn evaluate( + &mut self, + projection: &ChildCertificateCacheProjection, + crl_cache: &mut std::collections::HashMap, + issuer_ca_der: &[u8], + validation_time: time::OffsetDateTime, + ) -> (ChildCertificateCacheCrlGate, bool) { + let key = ( + projection.issuer_crl_uri.clone(), + projection.issuer_crl_sha256_hex.clone(), + ); + if let Some(gate) = self.gates_by_uri_and_expected_hash.get(&key) { + return (*gate, true); + } + + let gate = evaluate_child_certificate_cache_crl_gate( + projection, + crl_cache, + issuer_ca_der, + validation_time, + ); + self.gates_by_uri_and_expected_hash.insert(key, gate); + (gate, false) + } +} + +fn evaluate_child_certificate_cache_crl_gate( + projection: &ChildCertificateCacheProjection, + crl_cache: &mut std::collections::HashMap, + issuer_ca_der: &[u8], + validation_time: time::OffsetDateTime, +) -> ChildCertificateCacheCrlGate { + let Some(current_crl_hash) = get_current_crl_sha256_hex(&projection.issuer_crl_uri, crl_cache) + else { + return ChildCertificateCacheCrlGate::Missing; + }; + if current_crl_hash == projection.issuer_crl_sha256_hex { + let verified_crl = match ensure_issuer_crl_verified( + &projection.issuer_crl_uri, + crl_cache, + issuer_ca_der, + ) { + Ok(verified_crl) => verified_crl, + Err(_) => return ChildCertificateCacheCrlGate::Invalid, + }; + if !crl_valid_at_time_for_cache(&verified_crl.crl, validation_time) { + return ChildCertificateCacheCrlGate::Expired; + } + return ChildCertificateCacheCrlGate::Unchanged; + } + + let verified_crl = + match ensure_issuer_crl_verified(&projection.issuer_crl_uri, crl_cache, issuer_ca_der) { + Ok(verified_crl) => verified_crl, + Err(_) => return ChildCertificateCacheCrlGate::Invalid, + }; + if !crl_valid_at_time_for_cache(&verified_crl.crl, validation_time) { + return ChildCertificateCacheCrlGate::Expired; + } + ChildCertificateCacheCrlGate::ChangedValid +} + +fn crl_valid_at_time_for_cache( + crl: &crate::data_model::crl::RpkixCrl, + validation_time: time::OffsetDateTime, +) -> bool { + let this_update = crl.this_update.utc.to_offset(time::UtcOffset::UTC); + let next_update = crl.next_update.utc.to_offset(time::UtcOffset::UTC); + validation_time >= this_update && validation_time < next_update +} + +#[derive(Clone, Debug)] +struct VcirReuseProjection { + source: PublicationPointSource, + vcir: Option, + ccr_manifest_projection: Option, + snapshot: Option, + objects: crate::validation::objects::ObjectsOutput, + child_audits: Vec, + discovered_children: Vec, + warnings: Vec, +} diff --git a/crates/panda-rpki-validator/src/validation/tree_runner/child_validation.rs b/crates/panda-rpki-validator/src/validation/tree_runner/child_validation.rs new file mode 100644 index 0000000..abf5dc5 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_runner/child_validation.rs @@ -0,0 +1,176 @@ +fn is_non_router_certificate(err: &BgpsecRouterCertificatePathError) -> bool { + matches!( + err, + BgpsecRouterCertificatePathError::Decode(BgpsecRouterCertificateDecodeError::Validate( + BgpsecRouterCertificateProfileError::NotEe + | BgpsecRouterCertificateProfileError::MissingExtendedKeyUsage + | BgpsecRouterCertificateProfileError::MissingBgpsecRouterEku + )) + ) +} + +fn router_asns_for_resource_mode( + router_asns: &[u32], + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + mode: ResourceValidationMode, +) -> Result, String> { + let Some(issuer_effective_as) = issuer_effective_as else { + return Err("issuer has no effective AS resources".to_string()); + }; + + match mode { + ResourceValidationMode::Rfc6487 => { + let outside: Vec = router_asns + .iter() + .copied() + .filter(|asn| !as_resource_set_contains_asn(issuer_effective_as, *asn)) + .collect(); + if outside.is_empty() { + Ok(router_asns.to_vec()) + } else { + Err(format!( + "router AS resources are not a subset of issuer effective AS resources: {outside:?}" + )) + } + } + ResourceValidationMode::ValidationUpdate03 => { + let filtered: Vec = router_asns + .iter() + .copied() + .filter(|asn| as_resource_set_contains_asn(issuer_effective_as, *asn)) + .collect(); + if filtered.is_empty() { + Err("router AS resources have empty validated resource set".to_string()) + } else { + Ok(filtered) + } + } + } +} + +fn as_resource_set_contains_asn( + resources: &crate::data_model::rc::AsResourceSet, + asn: u32, +) -> bool { + let Some(choice) = resources.asnum.as_ref() else { + return false; + }; + match choice { + crate::data_model::rc::AsIdentifierChoice::Inherit => false, + crate::data_model::rc::AsIdentifierChoice::AsIdsOrRanges(items) => { + items.iter().any(|item| match item { + crate::data_model::rc::AsIdOrRange::Id(id) => *id == asn, + crate::data_model::rc::AsIdOrRange::Range { min, max } => { + *min <= asn && asn <= *max + } + }) + } + } +} + +fn select_issuer_crl_uri_for_child<'a>( + child: &'a crate::data_model::rc::ResourceCertificate, + crl_cache: &std::collections::HashMap, +) -> Result<&'a str, String> { + if crl_cache.is_empty() { + return Err( + "no CRL available in publication point snapshot (cannot validate certificates) (RFC 9286 §7; RFC 6487 §4.8.6)" + .to_string(), + ); + } + let Some(crldp_uris) = child.tbs.extensions.crl_distribution_points_uris.as_ref() else { + return Err( + "child certificate CRLDistributionPoints missing (RFC 6487 §4.8.6)".to_string(), + ); + }; + + for u in crldp_uris { + let s = u.as_str(); + if crl_cache.contains_key(s) { + return Ok(s); + } + } + + Err(format!( + "CRL referenced by child certificate CRLDistributionPoints not found in publication point snapshot: {} (RFC 6487 §4.8.6; RFC 9286 §4.2.1)", + crldp_uris + .iter() + .map(|u| u.as_str()) + .collect::>() + .join(", ") + )) +} + +fn ensure_issuer_crl_verified<'a>( + crl_rsync_uri: &str, + crl_cache: &'a mut std::collections::HashMap, + issuer_ca_der: &[u8], +) -> Result<&'a VerifiedIssuerCrl, CaPathError> { + let entry = crl_cache + .get_mut(crl_rsync_uri) + .expect("CRL must exist in cache"); + match entry { + CachedIssuerCrl::Ok(v) => Ok(v), + CachedIssuerCrl::Pending { + bytes, + sha256_hex: cached_sha256_hex, + } => { + let der = std::mem::take(bytes); + let crl = crate::data_model::crl::RpkixCrl::decode_der(&der)?; + crl.verify_signature_with_issuer_certificate_der(issuer_ca_der)?; + let sha256_hex = cached_sha256_hex + .take() + .unwrap_or_else(|| crate::audit::sha256_hex(&der)); + + let mut revoked_serials: std::collections::HashSet> = + std::collections::HashSet::with_capacity(crl.revoked_certs.len()); + for rc in &crl.revoked_certs { + revoked_serials.insert(rc.serial_number.bytes_be.clone()); + } + + *entry = CachedIssuerCrl::Ok(VerifiedIssuerCrl { + crl, + revoked_serials, + sha256_hex, + }); + match entry { + CachedIssuerCrl::Ok(v) => Ok(v), + _ => unreachable!(), + } + } + } +} + +fn validate_subordinate_ca_cert_with_cached_issuer( + child_ca_der: &[u8], + child_ca: crate::data_model::rc::ResourceCertificate, + issuer_ca_der: &[u8], + issuer_ca: &crate::data_model::rc::ResourceCertificate, + issuer_spki: &SubjectPublicKeyInfo<'_>, + issuer_crl_rsync_uri: &str, + crl_cache: &mut std::collections::HashMap, + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + issuer_resources_index: &IssuerEffectiveResourcesIndex, + validation_time: time::OffsetDateTime, + resource_validation_mode: crate::policy::ResourceValidationMode, +) -> Result { + let verified_crl = ensure_issuer_crl_verified(issuer_crl_rsync_uri, crl_cache, issuer_ca_der)?; + + validate_subordinate_ca_cert_with_prevalidated_issuer_and_resources( + child_ca_der, + child_ca, + issuer_ca, + issuer_spki, + &verified_crl.crl, + &verified_crl.revoked_serials, + issuer_ca_rsync_uri, + issuer_crl_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + issuer_resources_index, + validation_time, + resource_validation_mode, + ) +} diff --git a/crates/panda-rpki-validator/src/validation/tree_runner/discovery.rs b/crates/panda-rpki-validator/src/validation/tree_runner/discovery.rs new file mode 100644 index 0000000..11fe9b4 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_runner/discovery.rs @@ -0,0 +1,332 @@ +// Child discovery orchestration and cache setup. The processing loop lives in +// `discovery/body_loop.rs`; it is expanded as a local macro so the original local +// state remains in the function scope while keeping each source part reviewable. + +fn discover_children_from_fresh_snapshot_with_audit_cached_with_issuer_der< + P: PublicationPointData, +>( + issuer: &CaInstanceHandle, + issuer_ca_der: &[u8], + publication_point: &P, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + policy: &Policy, + cache_context: Option>, +) -> Result { + let locked_files = publication_point.files(); + // Issuer CA is only required when we actually attempt to validate a subordinate CA. For some + // audit-only error paths (e.g., missing CRL in the snapshot), we still want discovery to succeed. + let issuer_ca_decode_error: Option; + let issuer_ca = match crate::data_model::rc::ResourceCertificate::decode_der(issuer_ca_der) { + Ok(v) => { + match v.validate_rfc6487_profile(crate::data_model::rc::ResourceCertificateRole::Ca) { + Ok(()) => { + issuer_ca_decode_error = None; + Some(v) + } + Err(e) => { + issuer_ca_decode_error = Some(format!( + "issuer CA profile validation failed: {e} (RFC 6487 §4.8)" + )); + None + } + } + } + Err(e) => { + issuer_ca_decode_error = Some(format!( + "issuer CA decode failed: {e} (RFC 5280 §4.1; RFC 6487 §4)" + )); + None + } + }; + + let issuer_spki_error: Option; + let issuer_spki: Option> = if let Some(ca) = issuer_ca.as_ref() { + match SubjectPublicKeyInfo::from_der(&ca.tbs.subject_public_key_info) { + Ok((rem, spki)) if rem.is_empty() => { + issuer_spki_error = None; + Some(spki) + } + Ok((rem, _)) => { + issuer_spki_error = Some(format!( + "trailing bytes after issuer SubjectPublicKeyInfo DER: {} bytes (DER; RFC 5280 §4.1.2.7)", + rem.len() + )); + None + } + Err(e) => { + issuer_spki_error = Some(format!( + "issuer SubjectPublicKeyInfo parse error: {e} (RFC 5280 §4.1.2.7)" + )); + None + } + } + } else { + issuer_spki_error = issuer_ca_decode_error.clone(); + None + }; + + let mut crl_cache: std::collections::HashMap = locked_files + .iter() + .filter(|f| f.rsync_uri.ends_with(".crl")) + .map(|f| -> Result<(String, CachedIssuerCrl), String> { + let bytes = f + .bytes_cloned() + .map_err(|e| format!("snapshot CRL bytes load failed: {e}"))?; + Ok(( + f.rsync_uri.clone(), + CachedIssuerCrl::Pending { + bytes, + sha256_hex: Some(sha256_hex_from_32(&f.sha256)), + }, + )) + }) + .collect::>()?; + + let mut out: Vec = Vec::new(); + let mut audits: Vec = Vec::new(); + let mut router_keys: Vec = Vec::new(); + let issuer_resources_index = IssuerEffectiveResourcesIndex::from_effective_resources( + issuer.effective_ip_resources.as_ref(), + issuer.effective_as_resources.as_ref(), + ) + .map_err(|e| format!("build issuer effective resources index failed: {e}"))?; + + let mut cer_seen: u64 = 0; + let mut ca_skipped_not_ca: u64 = 0; + let mut ca_ok: u64 = 0; + let mut ca_error: u64 = 0; + let mut router_ok: u64 = 0; + let mut router_error: u64 = 0; + let mut router_skipped_non_router: u64 = 0; + let mut crl_select_error: u64 = 0; + let mut uri_discovery_error: u64 = 0; + let mut child_cert_cache_lookup: u64 = 0; + let mut child_cert_cache_hit: u64 = 0; + let mut child_cert_cache_hit_ca: u64 = 0; + let mut child_cert_cache_hit_router: u64 = 0; + let mut child_cert_cache_miss_not_found: u64 = 0; + let mut child_cert_cache_miss_time_gate: u64 = 0; + let mut child_cert_cache_miss_crl_missing: u64 = 0; + let mut child_cert_cache_miss_crl_invalid: u64 = 0; + let mut child_cert_cache_miss_crl_expired: u64 = 0; + let mut child_cert_cache_miss_revoked: u64 = 0; + let mut child_cert_cache_crl_recheck_hit: u64 = 0; + let mut child_cert_cache_crl_gate_reused: u64 = 0; + let mut child_cert_cache_load_error: u64 = 0; + let mut child_cert_cache_write_ok: u64 = 0; + let mut child_cert_cache_write_error: u64 = 0; + + let mut select_crl_nanos: u64 = 0; + let mut child_decode_nanos: u64 = 0; + let mut validate_sub_ca_nanos: u64 = 0; + let mut validate_router_nanos: u64 = 0; + let mut uri_discovery_nanos: u64 = 0; + let mut enqueue_nanos: u64 = 0; + let mut child_cert_cache_lookup_nanos: u64 = 0; + let mut child_cert_cache_write_nanos: u64 = 0; + let child_cert_der_load_cache_hit_nanos: u64 = 0; + let mut child_cert_der_load_fresh_nanos: u64 = 0; + let child_cert_der_load_cache_hit_count: u64 = 0; + let mut child_cert_der_load_fresh_count: u64 = 0; + + let mut eff_ip_items_bucket_le_10: u64 = 0; + let mut eff_ip_items_bucket_le_100: u64 = 0; + let mut eff_ip_items_bucket_gt_100: u64 = 0; + let mut eff_as_items_bucket_le_10: u64 = 0; + let mut eff_as_items_bucket_le_100: u64 = 0; + let mut eff_as_items_bucket_gt_100: u64 = 0; + + let mut child_cache_candidates: HashMap = + HashMap::new(); + let mut child_cache_segment_keys = Vec::::new(); + let mut child_cache_segment_dirty_projections: Option< + HashMap, + > = None; + let mut child_cert_cache_batch_lookup_publication_points: u64 = 0; + let mut child_cert_cache_batch_lookup_entries: u64 = 0; + let mut child_cert_cache_batch_lookup_errors: u64 = 0; + let mut child_cert_cache_batch_lookup_nanos: u64 = 0; + let mut child_cert_cache_mmap_lookup_publication_points: u64 = 0; + let mut child_cert_cache_mmap_lookup_entries: u64 = 0; + let mut child_cert_cache_mmap_lookup_hits: u64 = 0; + let mut child_cert_cache_mmap_lookup_misses: u64 = 0; + let mut child_cert_cache_mmap_lookup_missing_segments: u64 = 0; + let mut child_cert_cache_mmap_lookup_errors: u64 = 0; + let mut child_cert_cache_mmap_lookup_file_bytes: u64 = 0; + let mut child_cert_cache_mmap_lookup_nanos: u64 = 0; + let mut child_cert_cache_mmap_write_entries: u64 = 0; + let mut child_cert_cache_mmap_write_errors: u64 = 0; + let mut child_cert_cache_mmap_write_file_bytes: u64 = 0; + let mut child_cert_cache_mmap_write_nanos: u64 = 0; + + if let Some(cache) = cache_context { + let mut uris = Vec::new(); + let mut keys = Vec::new(); + for f in locked_files + .iter() + .filter(|file| file.rsync_uri.ends_with(".cer")) + { + uris.push(f.rsync_uri.clone()); + keys.push(child_certificate_cache_key_sha256_hex( + &f.rsync_uri, + &f.sha256, + &cache.issuer_ca_sha256, + &cache.ca_validation_context_digest, + &cache.policy_fingerprint, + )); + } + let use_mmap_segment = keys.len() >= CHILD_CERTIFICATE_CACHE_MMAP_MIN_CER_COUNT; + if use_mmap_segment { + child_cache_segment_keys = keys.clone(); + child_cache_segment_dirty_projections = Some(HashMap::new()); + } + + let mut db_lookup_indices: Vec = Vec::new(); + if !keys.is_empty() { + if use_mmap_segment { + let mmap_lookup_started = std::time::Instant::now(); + child_cert_cache_mmap_lookup_publication_points = 1; + child_cert_cache_mmap_lookup_entries = keys.len() as u64; + match cache + .store + .get_child_certificate_cache_projections_mmap_segment( + publication_point.manifest_rsync_uri(), + &keys, + ) { + Ok(Some(lookup)) => { + child_cert_cache_mmap_lookup_hits = lookup.hits as u64; + child_cert_cache_mmap_lookup_misses = lookup.misses as u64; + child_cert_cache_mmap_lookup_file_bytes = lookup.file_bytes; + for (idx, ((uri, _key), projection)) in uris + .iter() + .zip(keys.iter()) + .zip(lookup.projections.into_iter()) + .enumerate() + { + if let Some(projection) = projection { + child_cache_candidates.insert( + uri.clone(), + ChildCertificateCacheCandidate { + projection: Some(projection), + }, + ); + } else { + db_lookup_indices.push(idx); + } + } + } + Ok(None) => { + child_cert_cache_mmap_lookup_missing_segments = 1; + db_lookup_indices.extend(0..keys.len()); + } + Err(_) => { + child_cert_cache_mmap_lookup_errors = + child_cert_cache_mmap_lookup_errors.saturating_add(keys.len() as u64); + db_lookup_indices.extend(0..keys.len()); + } + } + child_cert_cache_mmap_lookup_nanos = mmap_lookup_started + .elapsed() + .as_nanos() + .min(u128::from(u64::MAX)) + as u64; + } else { + db_lookup_indices.extend(0..keys.len()); + } + + if !db_lookup_indices.is_empty() { + let batch_lookup_started = std::time::Instant::now(); + child_cert_cache_batch_lookup_publication_points = 1; + child_cert_cache_batch_lookup_entries = db_lookup_indices.len() as u64; + let db_keys = db_lookup_indices + .iter() + .map(|idx| keys[*idx].clone()) + .collect::>(); + match cache + .store + .get_child_certificate_cache_projections_batch(&db_keys) + { + Ok(projections) => { + for (idx, projection) in db_lookup_indices + .iter() + .copied() + .zip(projections.into_iter()) + { + if let Some(projection) = projection { + remember_child_certificate_cache_dirty_projection( + &mut child_cache_segment_dirty_projections, + &projection, + ); + child_cache_candidates.insert( + uris[idx].clone(), + ChildCertificateCacheCandidate { + projection: Some(projection), + }, + ); + } + } + } + Err(_) => { + child_cert_cache_batch_lookup_errors = child_cert_cache_batch_lookup_errors + .saturating_add(child_cert_cache_batch_lookup_entries); + } + } + child_cert_cache_batch_lookup_nanos = child_cert_cache_batch_lookup_nanos + .saturating_add( + batch_lookup_started + .elapsed() + .as_nanos() + .min(u128::from(u64::MAX)) as u64, + ); + } + } + } + + fn bucketize(v: usize) -> u8 { + if v <= 10 { + 0 + } else if v <= 100 { + 1 + } else { + 2 + } + } + + fn ip_item_count(ip: Option<&crate::data_model::rc::IpResourceSet>) -> usize { + let Some(ip) = ip else { return 0 }; + ip.families + .iter() + .map(|f| match &f.choice { + crate::data_model::rc::IpAddressChoice::Inherit => 0usize, + crate::data_model::rc::IpAddressChoice::AddressesOrRanges(items) => items.len(), + }) + .sum() + } + + fn as_item_count(asr: Option<&crate::data_model::rc::AsResourceSet>) -> usize { + let Some(asr) = asr else { return 0 }; + let mut n = 0usize; + if let Some(c) = asr.asnum.as_ref() { + if let crate::data_model::rc::AsIdentifierChoice::AsIdsOrRanges(items) = c { + n = n.saturating_add(items.len()); + } + } + if let Some(c) = asr.rdi.as_ref() { + if let crate::data_model::rc::AsIdentifierChoice::AsIdsOrRanges(items) = c { + n = n.saturating_add(items.len()); + } + } + n + } + + let mut child_cert_crl_gate_set = ChildCertificateCacheCrlGateSet::default(); + + macro_rules! child_discovery_body_loop { + () => { + include!("discovery/body_loop.rs") + }; + } + + child_discovery_body_loop!() +} diff --git a/crates/panda-rpki-validator/src/validation/tree_runner/discovery/body_loop.rs b/crates/panda-rpki-validator/src/validation/tree_runner/discovery/body_loop.rs new file mode 100644 index 0000000..109a354 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_runner/discovery/body_loop.rs @@ -0,0 +1,923 @@ +{ + for f in locked_files { + if !f.rsync_uri.ends_with(".cer") { + continue; + } + cer_seen = cer_seen.saturating_add(1); + let child_cert_sha256_hex = sha256_hex_from_32(&f.sha256); + + if cache_context.is_some() { + let lookup_started = std::time::Instant::now(); + child_cert_cache_lookup = child_cert_cache_lookup.saturating_add(1); + let cache_lookup_failed = child_cert_cache_batch_lookup_errors > 0 + && !child_cache_candidates.contains_key(&f.rsync_uri); + if cache_lookup_failed { + child_cert_cache_load_error = child_cert_cache_load_error.saturating_add(1); + } else { + match child_cache_candidates + .get(&f.rsync_uri) + .and_then(|candidate| candidate.projection.as_ref()) + { + Some(projection) => { + let time_gate_ok = pack_time_window_contains( + &projection.effective_not_before, + &projection.effective_until, + validation_time, + ); + if !time_gate_ok { + child_cert_cache_miss_time_gate = + child_cert_cache_miss_time_gate.saturating_add(1); + } else { + let (crl_gate, gate_reused) = child_cert_crl_gate_set.evaluate( + projection, + &mut crl_cache, + issuer_ca_der, + validation_time, + ); + if gate_reused { + child_cert_cache_crl_gate_reused = + child_cert_cache_crl_gate_reused.saturating_add(1); + } + let mut crl_gate_allows_reuse = false; + match crl_gate { + ChildCertificateCacheCrlGate::Unchanged => { + crl_gate_allows_reuse = true; + } + ChildCertificateCacheCrlGate::ChangedValid => { + match ensure_issuer_crl_verified( + &projection.issuer_crl_uri, + &mut crl_cache, + issuer_ca_der, + ) { + Ok(verified_crl) => { + if verified_crl + .revoked_serials + .contains(&projection.child_cert_serial) + { + child_cert_cache_miss_revoked = + child_cert_cache_miss_revoked.saturating_add(1); + } else { + child_cert_cache_crl_recheck_hit = + child_cert_cache_crl_recheck_hit + .saturating_add(1); + crl_gate_allows_reuse = true; + } + } + Err(_) => { + child_cert_cache_miss_crl_invalid = + child_cert_cache_miss_crl_invalid.saturating_add(1); + } + } + } + ChildCertificateCacheCrlGate::Expired => { + child_cert_cache_miss_crl_expired = + child_cert_cache_miss_crl_expired.saturating_add(1); + } + ChildCertificateCacheCrlGate::Invalid => { + child_cert_cache_miss_crl_invalid = + child_cert_cache_miss_crl_invalid.saturating_add(1); + } + ChildCertificateCacheCrlGate::Missing => { + child_cert_cache_miss_crl_missing = + child_cert_cache_miss_crl_missing.saturating_add(1); + } + } + + if crl_gate_allows_reuse { + match &projection.payload { + ChildCertificateCachePayload::ChildCa { + child_manifest_rsync_uri, + child_ski, + child_rsync_base_uri, + child_publication_point_rsync_uri, + child_rrdp_notification_uri, + child_effective_ip_resources, + child_effective_as_resources, + .. + } => { + child_cert_cache_lookup_nanos = + child_cert_cache_lookup_nanos.saturating_add( + lookup_started + .elapsed() + .as_nanos() + .min(u128::from(u64::MAX)) + as u64, + ); + out.push(DiscoveredChildCaInstance { + handle: CaInstanceHandle { + depth: 0, + tal_id: issuer.tal_id.clone(), + parent_manifest_rsync_uri: Some( + issuer.manifest_rsync_uri.clone(), + ), + ca_certificate: CaCertificateRef::repo_bytes( + child_cert_sha256_hex.clone(), + ), + ca_certificate_rsync_uri: Some(f.rsync_uri.clone()), + effective_ip_resources: + child_effective_ip_resources.clone(), + effective_as_resources: + child_effective_as_resources.clone(), + rsync_base_uri: child_rsync_base_uri.clone(), + manifest_rsync_uri: child_manifest_rsync_uri + .clone(), + publication_point_rsync_uri: + child_publication_point_rsync_uri.clone(), + rrdp_notification_uri: child_rrdp_notification_uri + .clone(), + }, + discovered_from: crate::audit::DiscoveredFrom { + parent_manifest_rsync_uri: issuer + .manifest_rsync_uri + .clone(), + child_ca_certificate_rsync_uri: f.rsync_uri.clone(), + child_ca_certificate_sha256_hex: + child_cert_sha256_hex.clone(), + }, + child_entry_projection: Some( + DiscoveredChildEntryProjection { + child_ski: child_ski.clone(), + }, + ), + }); + ca_ok = ca_ok.saturating_add(1); + child_cert_cache_hit = + child_cert_cache_hit.saturating_add(1); + child_cert_cache_hit_ca = + child_cert_cache_hit_ca.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: child_cert_sha256_hex.clone(), + kind: AuditObjectKind::Certificate, + result: AuditObjectResult::Ok, + detail: Some( + "restored subordinate CA discovery from child certificate validation cache" + .to_string(), + ), + }); + continue; + } + ChildCertificateCachePayload::Router { + router_keys: cached_keys, + } => { + for cached_key in cached_keys { + router_keys.push(RouterKeyPayload { + as_id: cached_key.as_id, + ski: cached_key.ski.clone(), + spki_der: cached_key.spki_der.clone(), + source_object_uri: f.rsync_uri.clone(), + source_object_hash: child_cert_sha256_hex.clone(), + source_ee_cert_hash: child_cert_sha256_hex.clone(), + item_effective_until: cached_key + .item_effective_until + .clone(), + }); + } + router_ok = router_ok.saturating_add(1); + child_cert_cache_hit = + child_cert_cache_hit.saturating_add(1); + child_cert_cache_hit_router = + child_cert_cache_hit_router.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: child_cert_sha256_hex.clone(), + kind: AuditObjectKind::RouterCertificate, + result: AuditObjectResult::Ok, + detail: Some( + "restored BGPsec router certificate from child certificate validation cache" + .to_string(), + ), + }); + child_cert_cache_lookup_nanos = + child_cert_cache_lookup_nanos.saturating_add( + lookup_started + .elapsed() + .as_nanos() + .min(u128::from(u64::MAX)) + as u64, + ); + continue; + } + } + } + } + } + None => { + child_cert_cache_miss_not_found = + child_cert_cache_miss_not_found.saturating_add(1); + } + } + } + child_cert_cache_lookup_nanos = child_cert_cache_lookup_nanos.saturating_add( + lookup_started + .elapsed() + .as_nanos() + .min(u128::from(u64::MAX)) as u64, + ); + } + + let child_der = load_child_certificate_der_for_discovery( + f, + &mut child_cert_der_load_fresh_nanos, + &mut child_cert_der_load_fresh_count, + )?; + + let tdecode = std::time::Instant::now(); + let child_cert = match crate::data_model::rc::ResourceCertificate::decode_der(child_der) { + Ok(v) => v, + Err(e) => { + ca_error = ca_error.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Certificate, + result: AuditObjectResult::Error, + detail: Some(format!("child certificate decode failed: {e}")), + }); + continue; + } + }; + child_decode_nanos = child_decode_nanos + .saturating_add(tdecode.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); + + let t0 = std::time::Instant::now(); + let issuer_crl_uri = match select_issuer_crl_uri_for_child(&child_cert, &crl_cache) { + Ok(v) => v.to_string(), + Err(e) => { + crl_select_error = crl_select_error.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Certificate, + result: AuditObjectResult::Error, + detail: Some(format!( + "cannot select issuer CRL for child certificate: {e}" + )), + }); + continue; + } + }; + select_crl_nanos = select_crl_nanos + .saturating_add(t0.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); + + let t1 = std::time::Instant::now(); + let Some(issuer_ca_ref) = issuer_ca.as_ref() else { + ca_error = ca_error.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Certificate, + result: AuditObjectResult::Error, + detail: Some( + issuer_ca_decode_error + .clone() + .unwrap_or_else(|| "issuer CA decode failed".to_string()), + ), + }); + continue; + }; + let Some(issuer_spki_ref) = issuer_spki.as_ref() else { + ca_error = ca_error.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Certificate, + result: AuditObjectResult::Error, + detail: Some( + issuer_spki_error + .clone() + .unwrap_or_else(|| "issuer SubjectPublicKeyInfo unavailable".to_string()), + ), + }); + continue; + }; + let validated = match validate_subordinate_ca_cert_with_cached_issuer( + child_der, + child_cert, + issuer_ca_der, + issuer_ca_ref, + issuer_spki_ref, + issuer_crl_uri.as_str(), + &mut crl_cache, + issuer.ca_certificate_rsync_uri.as_deref(), + issuer.effective_ip_resources.as_ref(), + issuer.effective_as_resources.as_ref(), + &issuer_resources_index, + validation_time, + policy.resource_validation_mode, + ) { + Ok(v) => v, + Err(CaPathError::ChildNotCa) => { + let tr = std::time::Instant::now(); + let router_result = match ensure_issuer_crl_verified( + issuer_crl_uri.as_str(), + &mut crl_cache, + issuer_ca_der, + ) { + Ok(verified_crl) => { + BgpsecRouterCertificate::validate_path_with_prevalidated_issuer( + child_der, + issuer_ca_ref, + issuer_spki_ref, + &verified_crl.crl, + &verified_crl.revoked_serials, + issuer.ca_certificate_rsync_uri.as_deref(), + Some(issuer_crl_uri.as_str()), + validation_time, + ) + } + Err(err) => { + validate_router_nanos = validate_router_nanos.saturating_add( + tr.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64, + ); + router_error = router_error.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::RouterCertificate, + result: AuditObjectResult::Error, + detail: Some(format!( + "router certificate issuer CRL validation failed: {err}" + )), + }); + continue; + } + }; + validate_router_nanos = validate_router_nanos + .saturating_add(tr.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); + + match router_result { + Ok(router) => { + if let Some(ta_constraints) = policy.ta_constraints.for_tal(&issuer.tal_id) + { + if let Err(error) = + ta_constraints.validate_ee_certificate(&router.resource_cert) + { + router_error = router_error.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::RouterCertificate, + result: AuditObjectResult::Error, + detail: Some(format!( + "router certificate violates TA constraints: {error}" + )), + }); + continue; + } + } + let router_asns = match router_asns_for_resource_mode( + &router.asns, + issuer.effective_as_resources.as_ref(), + policy.resource_validation_mode, + ) { + Ok(v) => v, + Err(detail) => { + router_error = router_error.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::RouterCertificate, + result: AuditObjectResult::Error, + detail: Some(format!( + "router certificate AS resource validation failed: {detail}" + )), + }); + continue; + } + }; + router_ok = router_ok.saturating_add(1); + let source_object_hash = sha256_hex_from_32(&f.sha256); + let item_effective_until = PackTime::from_utc_offset_datetime( + router.resource_cert.tbs.validity_not_after, + ); + for as_id in &router_asns { + router_keys.push(RouterKeyPayload { + as_id: *as_id, + ski: router.subject_key_identifier.clone(), + spki_der: router.spki_der.clone(), + source_object_uri: f.rsync_uri.clone(), + source_object_hash: source_object_hash.clone(), + source_ee_cert_hash: source_object_hash.clone(), + item_effective_until: item_effective_until.clone(), + }); + } + if let Some(cache) = cache_context { + let write_started = std::time::Instant::now(); + if let Ok(verified_crl) = ensure_issuer_crl_verified( + issuer_crl_uri.as_str(), + &mut crl_cache, + issuer_ca_der, + ) { + let (effective_not_before, effective_until) = + child_certificate_cache_certificate_window( + router.resource_cert.tbs.validity_not_before, + router.resource_cert.tbs.validity_not_after, + issuer_ca_ref.tbs.validity_not_before, + issuer_ca_ref.tbs.validity_not_after, + ); + let cache_key_sha256_hex = child_certificate_cache_key_sha256_hex( + &f.rsync_uri, + &f.sha256, + &cache.issuer_ca_sha256, + &cache.ca_validation_context_digest, + &cache.policy_fingerprint, + ); + let projection = ChildCertificateCacheProjection { + schema_version: CHILD_CERTIFICATE_CACHE_SCHEMA_VERSION, + algorithm_version: CHILD_CERTIFICATE_CACHE_ALGORITHM_VERSION, + cache_key_sha256_hex, + child_cert_uri: f.rsync_uri.clone(), + child_cert_sha256_hex: child_cert_sha256_hex.clone(), + child_cert_serial: BigUnsigned::from_biguint( + &router.resource_cert.tbs.serial_number, + ) + .bytes_be, + issuer_ca_sha256_hex: sha256_hex_from_32( + &cache.issuer_ca_sha256, + ), + issuer_crl_uri: issuer_crl_uri.clone(), + issuer_crl_sha256_hex: verified_crl.sha256_hex.clone(), + ca_validation_context_digest: cache + .ca_validation_context_digest, + validation_policy_fingerprint: cache.policy_fingerprint, + effective_not_before, + effective_until, + payload: ChildCertificateCachePayload::Router { + router_keys: router_asns + .iter() + .map(|as_id| ChildCertificateCacheRouterKeyProjection { + as_id: *as_id, + ski: router.subject_key_identifier.clone(), + spki_der: router.spki_der.clone(), + item_effective_until: item_effective_until.clone(), + }) + .collect(), + }, + }; + if cache + .store + .put_child_certificate_cache_projection(&projection) + .is_ok() + { + remember_child_certificate_cache_dirty_projection( + &mut child_cache_segment_dirty_projections, + &projection, + ); + child_cert_cache_write_ok = + child_cert_cache_write_ok.saturating_add(1); + } else { + child_cert_cache_write_error = + child_cert_cache_write_error.saturating_add(1); + } + } else { + child_cert_cache_write_error = + child_cert_cache_write_error.saturating_add(1); + } + child_cert_cache_write_nanos = child_cert_cache_write_nanos + .saturating_add( + write_started.elapsed().as_nanos().min(u128::from(u64::MAX)) + as u64, + ); + } + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::RouterCertificate, + result: AuditObjectResult::Ok, + detail: Some( + "validated BGPsec router certificate (RFC 8209); no child CA instance enqueued" + .to_string(), + ), + }); + } + Err(err) if is_non_router_certificate(&err) => { + ca_skipped_not_ca = ca_skipped_not_ca.saturating_add(1); + router_skipped_non_router = router_skipped_non_router.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Certificate, + result: AuditObjectResult::Skipped, + detail: Some( + "skipped: not a CA resource certificate or BGPsec router certificate" + .to_string(), + ), + }); + } + Err(err) => { + router_error = router_error.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::RouterCertificate, + result: AuditObjectResult::Error, + detail: Some(format!("router certificate validation failed: {err}")), + }); + } + } + continue; + } + Err(e) => { + ca_error = ca_error.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Certificate, + result: AuditObjectResult::Error, + detail: Some(format!("child CA validation failed: {e}")), + }); + continue; + } + }; + validate_sub_ca_nanos = validate_sub_ca_nanos + .saturating_add(t1.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); + + if !validated.resource_warnings.is_empty() { + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Certificate, + result: AuditObjectResult::Ok, + detail: Some(format!( + "resource validation warning ({:?}): {}", + policy.resource_validation_mode, + validated.resource_warnings.summary() + )), + }); + } + + let eff_ip_items = ip_item_count(validated.effective_ip_resources.as_ref()); + match bucketize(eff_ip_items) { + 0 => eff_ip_items_bucket_le_10 = eff_ip_items_bucket_le_10.saturating_add(1), + 1 => eff_ip_items_bucket_le_100 = eff_ip_items_bucket_le_100.saturating_add(1), + _ => eff_ip_items_bucket_gt_100 = eff_ip_items_bucket_gt_100.saturating_add(1), + } + let eff_as_items = as_item_count(validated.effective_as_resources.as_ref()); + match bucketize(eff_as_items) { + 0 => eff_as_items_bucket_le_10 = eff_as_items_bucket_le_10.saturating_add(1), + 1 => eff_as_items_bucket_le_100 = eff_as_items_bucket_le_100.saturating_add(1), + _ => eff_as_items_bucket_gt_100 = eff_as_items_bucket_gt_100.saturating_add(1), + } + + let t2 = std::time::Instant::now(); + let uris = match ca_instance_uris_from_ca_certificate(&validated.child_ca) { + Ok(v) => v, + Err(e) => { + uri_discovery_error = uri_discovery_error.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Certificate, + result: AuditObjectResult::Error, + detail: Some(format!("CA instance URI discovery failed: {e}")), + }); + continue; + } + }; + uri_discovery_nanos = uri_discovery_nanos + .saturating_add(t2.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); + + let t3 = std::time::Instant::now(); + let child_rsync_base_uri = uris.rsync_base_uri.clone(); + let child_manifest_rsync_uri = uris.manifest_rsync_uri.clone(); + let child_publication_point_rsync_uri = uris.publication_point_rsync_uri.clone(); + let child_rrdp_notification_uri = uris.rrdp_notification_uri.clone(); + out.push(DiscoveredChildCaInstance { + handle: CaInstanceHandle { + depth: 0, + tal_id: issuer.tal_id.clone(), + parent_manifest_rsync_uri: Some(issuer.manifest_rsync_uri.clone()), + ca_certificate: CaCertificateRef::inline_der(child_der.to_vec()), + ca_certificate_rsync_uri: Some(f.rsync_uri.clone()), + effective_ip_resources: validated.effective_ip_resources.clone(), + effective_as_resources: validated.effective_as_resources.clone(), + rsync_base_uri: child_rsync_base_uri.clone(), + manifest_rsync_uri: child_manifest_rsync_uri.clone(), + publication_point_rsync_uri: child_publication_point_rsync_uri.clone(), + rrdp_notification_uri: child_rrdp_notification_uri.clone(), + }, + discovered_from: crate::audit::DiscoveredFrom { + parent_manifest_rsync_uri: issuer.manifest_rsync_uri.clone(), + child_ca_certificate_rsync_uri: f.rsync_uri.clone(), + child_ca_certificate_sha256_hex: child_cert_sha256_hex.clone(), + }, + child_entry_projection: validated + .child_ca + .tbs + .extensions + .subject_key_identifier + .as_ref() + .map(|child_ski| DiscoveredChildEntryProjection { + child_ski: hex::encode(child_ski), + }), + }); + enqueue_nanos = + enqueue_nanos.saturating_add(t3.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); + + if let Some(cache) = cache_context { + let write_started = std::time::Instant::now(); + if let Ok(verified_crl) = + ensure_issuer_crl_verified(issuer_crl_uri.as_str(), &mut crl_cache, issuer_ca_der) + { + if let Some(child_ski) = validated + .child_ca + .tbs + .extensions + .subject_key_identifier + .as_ref() + { + let (effective_not_before, effective_until) = + child_certificate_cache_certificate_window( + validated.child_ca.tbs.validity_not_before, + validated.child_ca.tbs.validity_not_after, + issuer_ca_ref.tbs.validity_not_before, + issuer_ca_ref.tbs.validity_not_after, + ); + let cache_key_sha256_hex = child_certificate_cache_key_sha256_hex( + &f.rsync_uri, + &f.sha256, + &cache.issuer_ca_sha256, + &cache.ca_validation_context_digest, + &cache.policy_fingerprint, + ); + let projection = ChildCertificateCacheProjection { + schema_version: CHILD_CERTIFICATE_CACHE_SCHEMA_VERSION, + algorithm_version: CHILD_CERTIFICATE_CACHE_ALGORITHM_VERSION, + cache_key_sha256_hex, + child_cert_uri: f.rsync_uri.clone(), + child_cert_sha256_hex: child_cert_sha256_hex.clone(), + child_cert_serial: BigUnsigned::from_biguint( + &validated.child_ca.tbs.serial_number, + ) + .bytes_be, + issuer_ca_sha256_hex: sha256_hex_from_32(&cache.issuer_ca_sha256), + issuer_crl_uri: issuer_crl_uri.clone(), + issuer_crl_sha256_hex: verified_crl.sha256_hex.clone(), + ca_validation_context_digest: cache.ca_validation_context_digest, + validation_policy_fingerprint: cache.policy_fingerprint, + effective_not_before, + effective_until, + payload: ChildCertificateCachePayload::ChildCa { + child_manifest_rsync_uri, + child_ski: hex::encode(child_ski), + child_rsync_base_uri, + child_publication_point_rsync_uri, + child_rrdp_notification_uri, + child_effective_ip_resources: validated.effective_ip_resources.clone(), + child_effective_as_resources: validated.effective_as_resources.clone(), + }, + }; + if cache + .store + .put_child_certificate_cache_projection(&projection) + .is_ok() + { + remember_child_certificate_cache_dirty_projection( + &mut child_cache_segment_dirty_projections, + &projection, + ); + child_cert_cache_write_ok = child_cert_cache_write_ok.saturating_add(1); + } else { + child_cert_cache_write_error = + child_cert_cache_write_error.saturating_add(1); + } + } else { + child_cert_cache_write_error = child_cert_cache_write_error.saturating_add(1); + } + } else { + child_cert_cache_write_error = child_cert_cache_write_error.saturating_add(1); + } + child_cert_cache_write_nanos = + child_cert_cache_write_nanos.saturating_add( + write_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64, + ); + } + + ca_ok = ca_ok.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: child_cert_sha256_hex.clone(), + kind: AuditObjectKind::Certificate, + result: AuditObjectResult::Ok, + detail: Some("validated subordinate CA certificate; enqueued CA instance".to_string()), + }); + } + + if let (Some(cache), Some(dirty_projections)) = + (cache_context, child_cache_segment_dirty_projections.take()) + { + if !dirty_projections.is_empty() { + let write_started = std::time::Instant::now(); + let projections = dirty_projections.into_values().collect::>(); + child_cert_cache_mmap_write_entries = projections.len() as u64; + match cache + .store + .write_child_certificate_cache_mmap_segment_overlay( + publication_point.manifest_rsync_uri(), + &child_cache_segment_keys, + &projections, + ) { + Ok(stats) => { + child_cert_cache_mmap_write_file_bytes = stats.file_bytes; + } + Err(_) => { + child_cert_cache_mmap_write_errors = + child_cert_cache_mmap_write_errors.saturating_add(1); + } + } + child_cert_cache_mmap_write_nanos = + write_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64; + } + } + + if let Some(t) = timing { + t.record_count("child_cer_seen", cer_seen); + t.record_count("child_ca_ok", ca_ok); + t.record_count("child_ca_error", ca_error); + t.record_count("child_ca_skipped_not_ca", ca_skipped_not_ca); + t.record_count("child_router_ok", router_ok); + t.record_count("child_router_error", router_error); + t.record_count("child_router_skipped_non_router", router_skipped_non_router); + t.record_count("child_crl_select_error", crl_select_error); + t.record_count("child_uri_discovery_error", uri_discovery_error); + t.record_count("child_certificate_cache_lookup", child_cert_cache_lookup); + t.record_count("child_certificate_cache_hit", child_cert_cache_hit); + t.record_count("child_certificate_cache_hit_ca", child_cert_cache_hit_ca); + t.record_count( + "child_certificate_cache_hit_router", + child_cert_cache_hit_router, + ); + t.record_count( + "child_certificate_cache_miss_not_found", + child_cert_cache_miss_not_found, + ); + t.record_count( + "child_certificate_cache_miss_time_gate", + child_cert_cache_miss_time_gate, + ); + t.record_count( + "child_certificate_cache_miss_crl_missing", + child_cert_cache_miss_crl_missing, + ); + t.record_count( + "child_certificate_cache_miss_crl_invalid", + child_cert_cache_miss_crl_invalid, + ); + t.record_count( + "child_certificate_cache_miss_crl_expired", + child_cert_cache_miss_crl_expired, + ); + t.record_count( + "child_certificate_cache_miss_revoked", + child_cert_cache_miss_revoked, + ); + t.record_count( + "child_certificate_cache_crl_recheck_hit", + child_cert_cache_crl_recheck_hit, + ); + t.record_count( + "child_certificate_cache_crl_gate_reused", + child_cert_cache_crl_gate_reused, + ); + t.record_count( + "child_certificate_cache_load_error", + child_cert_cache_load_error, + ); + t.record_count( + "child_certificate_cache_write_ok", + child_cert_cache_write_ok, + ); + t.record_count( + "child_certificate_cache_write_error", + child_cert_cache_write_error, + ); + t.record_count( + "child_certificate_cache_batch_lookup_publication_points", + child_cert_cache_batch_lookup_publication_points, + ); + t.record_count( + "child_certificate_cache_batch_lookup_entries", + child_cert_cache_batch_lookup_entries, + ); + t.record_count( + "child_certificate_cache_batch_lookup_errors", + child_cert_cache_batch_lookup_errors, + ); + t.record_count( + "child_certificate_cache_mmap_lookup_publication_points", + child_cert_cache_mmap_lookup_publication_points, + ); + t.record_count( + "child_certificate_cache_mmap_lookup_entries", + child_cert_cache_mmap_lookup_entries, + ); + t.record_count( + "child_certificate_cache_mmap_lookup_hits", + child_cert_cache_mmap_lookup_hits, + ); + t.record_count( + "child_certificate_cache_mmap_lookup_misses", + child_cert_cache_mmap_lookup_misses, + ); + t.record_count( + "child_certificate_cache_mmap_lookup_missing_segments", + child_cert_cache_mmap_lookup_missing_segments, + ); + t.record_count( + "child_certificate_cache_mmap_lookup_errors", + child_cert_cache_mmap_lookup_errors, + ); + t.record_count( + "child_certificate_cache_mmap_lookup_file_bytes", + child_cert_cache_mmap_lookup_file_bytes, + ); + t.record_count( + "child_certificate_cache_mmap_write_entries", + child_cert_cache_mmap_write_entries, + ); + t.record_count( + "child_certificate_cache_mmap_write_errors", + child_cert_cache_mmap_write_errors, + ); + t.record_count( + "child_certificate_cache_mmap_write_file_bytes", + child_cert_cache_mmap_write_file_bytes, + ); + t.record_count( + "child_certificate_der_load_cache_hit_count", + child_cert_der_load_cache_hit_count, + ); + t.record_count( + "child_certificate_der_load_fresh_count", + child_cert_der_load_fresh_count, + ); + + t.record_count("child_effective_ip_items_le_10", eff_ip_items_bucket_le_10); + t.record_count( + "child_effective_ip_items_le_100", + eff_ip_items_bucket_le_100, + ); + t.record_count( + "child_effective_ip_items_gt_100", + eff_ip_items_bucket_gt_100, + ); + t.record_count("child_effective_as_items_le_10", eff_as_items_bucket_le_10); + t.record_count( + "child_effective_as_items_le_100", + eff_as_items_bucket_le_100, + ); + t.record_count( + "child_effective_as_items_gt_100", + eff_as_items_bucket_gt_100, + ); + + t.record_phase_nanos("child_select_issuer_crl_total", select_crl_nanos); + t.record_phase_nanos("child_decode_certificate_total", child_decode_nanos); + t.record_phase_nanos("child_validate_subordinate_total", validate_sub_ca_nanos); + t.record_phase_nanos( + "child_validate_router_certificate_total", + validate_router_nanos, + ); + t.record_phase_nanos("child_ca_instance_uri_discovery_total", uri_discovery_nanos); + t.record_phase_nanos("child_enqueue_total", enqueue_nanos); + t.record_phase_nanos( + "child_certificate_cache_lookup_total", + child_cert_cache_lookup_nanos, + ); + t.record_phase_nanos( + "child_certificate_cache_write_total", + child_cert_cache_write_nanos, + ); + t.record_phase_nanos( + "child_certificate_cache_batch_lookup_total", + child_cert_cache_batch_lookup_nanos, + ); + t.record_phase_nanos( + "child_certificate_cache_mmap_lookup_total", + child_cert_cache_mmap_lookup_nanos, + ); + t.record_phase_nanos( + "child_certificate_cache_mmap_write_total", + child_cert_cache_mmap_write_nanos, + ); + t.record_phase_nanos( + "child_certificate_der_load_cache_hit_total", + child_cert_der_load_cache_hit_nanos, + ); + t.record_phase_nanos( + "child_certificate_der_load_fresh_total", + child_cert_der_load_fresh_nanos, + ); + t.record_phase_nanos( + "child_certificate_der_load_total", + child_cert_der_load_cache_hit_nanos.saturating_add(child_cert_der_load_fresh_nanos), + ); + } + + Ok(ChildDiscoveryOutput { + children: out, + audits, + router_keys, + }) +} diff --git a/crates/panda-rpki-validator/src/validation/tree_runner/discovery/wrappers.rs b/crates/panda-rpki-validator/src/validation/tree_runner/discovery/wrappers.rs new file mode 100644 index 0000000..dfebe14 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_runner/discovery/wrappers.rs @@ -0,0 +1,48 @@ +fn discover_children_from_fresh_snapshot_with_audit( + issuer: &CaInstanceHandle, + publication_point: &P, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, +) -> Result { + let issuer_ca_der = match &issuer.ca_certificate { + CaCertificateRef::InlineDer(bytes) => bytes.as_slice(), + CaCertificateRef::RepoBytes { .. } => { + return Err("lazy CA certificate requires store-backed child discovery".to_string()); + } + }; + let default_policy = Policy::default(); + discover_children_from_fresh_snapshot_with_audit_cached_with_issuer_der( + issuer, + issuer_ca_der, + publication_point, + validation_time, + timing, + &default_policy, + None, + ) +} + +fn discover_children_from_fresh_snapshot_with_audit_cached( + issuer: &CaInstanceHandle, + publication_point: &P, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + cache_context: Option>, +) -> Result { + let issuer_ca_der = match &issuer.ca_certificate { + CaCertificateRef::InlineDer(bytes) => bytes.as_slice(), + CaCertificateRef::RepoBytes { .. } => { + return Err("lazy CA certificate requires store-backed child discovery".to_string()); + } + }; + let default_policy = Policy::default(); + discover_children_from_fresh_snapshot_with_audit_cached_with_issuer_der( + issuer, + issuer_ca_der, + publication_point, + validation_time, + timing, + &default_policy, + cache_context, + ) +} diff --git a/crates/panda-rpki-validator/src/validation/tree_runner/labels.rs b/crates/panda-rpki-validator/src/validation/tree_runner/labels.rs new file mode 100644 index 0000000..c88360b --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_runner/labels.rs @@ -0,0 +1,114 @@ +//! Stable labels used by validation audit output. +//! +//! These helpers are pure mappings. Keeping them separate from publication +//! point traversal makes the output vocabulary easy to review without +//! changing traversal or validation control flow. + +use crate::audit::{AuditObjectKind, AuditObjectResult}; +use crate::storage::{VcirArtifactKind, VcirArtifactValidationStatus}; +use crate::validation::manifest::PublicationPointSource; + +pub(super) fn kind_from_rsync_uri(uri: &str) -> AuditObjectKind { + if uri.ends_with(".crl") { + AuditObjectKind::Crl + } else if uri.ends_with(".cer") { + AuditObjectKind::Certificate + } else if uri.ends_with(".roa") { + AuditObjectKind::Roa + } else if uri.ends_with(".asa") { + AuditObjectKind::Aspa + } else { + AuditObjectKind::Other + } +} + +pub(super) fn source_label(source: PublicationPointSource) -> String { + match source { + PublicationPointSource::Fresh => "fresh".to_string(), + PublicationPointSource::PublicationPointCache => "publication_point_cache".to_string(), + PublicationPointSource::VcirCurrentInstance => "vcir_current_instance".to_string(), + PublicationPointSource::FailedFetchNoCache => "failed_fetch_no_cache".to_string(), + } +} + +pub(super) fn repo_sync_phase_label(phase: crate::sync::repo::RepoSyncPhase) -> &'static str { + match phase { + crate::sync::repo::RepoSyncPhase::RrdpOk => "rrdp_ok", + crate::sync::repo::RepoSyncPhase::RrdpFailedRsyncOk => "rrdp_failed_rsync_ok", + crate::sync::repo::RepoSyncPhase::RsyncOnlyOk => "rsync_only_ok", + crate::sync::repo::RepoSyncPhase::ReplayRrdpOk => "replay_rrdp_ok", + crate::sync::repo::RepoSyncPhase::ReplayRsyncOk => "replay_rsync_ok", + crate::sync::repo::RepoSyncPhase::ReplayNoopRrdp => "replay_noop_rrdp", + crate::sync::repo::RepoSyncPhase::ReplayNoopRsync => "replay_noop_rsync", + } +} + +pub(super) fn repo_sync_failure_phase_label( + attempted_rrdp: bool, + original_notification_uri: Option<&str>, + effective_notification_uri: Option<&str>, +) -> &'static str { + if attempted_rrdp && original_notification_uri.is_some() && effective_notification_uri.is_some() + { + "rrdp_failed_rsync_failed" + } else if attempted_rrdp + && original_notification_uri.is_some() + && effective_notification_uri.is_none() + { + "rsync_only_failed_after_rrdp_dedup" + } else { + "rsync_only_failed" + } +} + +pub(super) fn terminal_state_label(source: PublicationPointSource) -> &'static str { + match source { + PublicationPointSource::Fresh => "fresh", + PublicationPointSource::PublicationPointCache => "publication_point_cache", + PublicationPointSource::VcirCurrentInstance => "fallback_current_instance", + PublicationPointSource::FailedFetchNoCache => "failed_no_cache", + } +} + +pub(super) fn repo_sync_source_label(source: crate::sync::repo::RepoSyncSource) -> &'static str { + match source { + crate::sync::repo::RepoSyncSource::Rrdp => "rrdp", + crate::sync::repo::RepoSyncSource::Rsync => "rsync", + } +} + +pub(super) fn effective_repo_sync_duration_ms( + elapsed_ms: u64, + runtime_reported_duration_ms: Option, + repo_sync_ok: bool, +) -> u64 { + if repo_sync_ok { + return elapsed_ms; + } + runtime_reported_duration_ms + .map(|runtime_ms| elapsed_ms.max(runtime_ms)) + .unwrap_or(elapsed_ms) +} + +pub(super) fn kind_from_vcir_artifact_kind(kind: VcirArtifactKind) -> AuditObjectKind { + match kind { + VcirArtifactKind::Mft => AuditObjectKind::Manifest, + VcirArtifactKind::Crl => AuditObjectKind::Crl, + VcirArtifactKind::Cer => AuditObjectKind::Certificate, + VcirArtifactKind::Roa => AuditObjectKind::Roa, + VcirArtifactKind::Aspa => AuditObjectKind::Aspa, + VcirArtifactKind::Gbr | VcirArtifactKind::Tal | VcirArtifactKind::Other => { + AuditObjectKind::Other + } + } +} + +pub(super) fn audit_result_from_vcir_status( + status: VcirArtifactValidationStatus, +) -> AuditObjectResult { + match status { + VcirArtifactValidationStatus::Accepted => AuditObjectResult::Ok, + VcirArtifactValidationStatus::Rejected => AuditObjectResult::Error, + VcirArtifactValidationStatus::WarningOnly => AuditObjectResult::Skipped, + } +} diff --git a/crates/panda-rpki-validator/src/validation/tree_runner/publication_point_runner.rs b/crates/panda-rpki-validator/src/validation/tree_runner/publication_point_runner.rs new file mode 100644 index 0000000..7713872 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_runner/publication_point_runner.rs @@ -0,0 +1,731 @@ +impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> { + fn prefetch_discovered_children( + &self, + children: &[DiscoveredChildCaInstance], + ) -> Result<(), String> { + if let Some(runtime) = self.repo_sync_runtime.as_ref() { + runtime.prefetch_discovered_children(children)?; + } + Ok(()) + } + + fn run_publication_point( + &self, + ca: &CaInstanceHandle, + ) -> Result { + let publication_point_started = std::time::Instant::now(); + let _pp_total = self + .timing + .as_ref() + .map(|t| t.span_publication_point(&ca.manifest_rsync_uri)); + if let Some(t) = self.timing.as_ref() { + t.record_count("publication_points_seen", 1); + if ca.rrdp_notification_uri.is_some() { + t.record_count("publication_points_rrdp_notify_present_total", 1); + } else { + t.record_count("publication_points_rrdp_notify_missing_total", 1); + } + } + + let mut warnings: Vec = Vec::new(); + crate::progress_log::emit( + "publication_point_start", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "rsync_base_uri": ca.rsync_base_uri, + "rrdp_notification_uri": ca.rrdp_notification_uri, + }), + ); + + let attempted_rrdp = + self.policy.sync_preference == crate::policy::SyncPreference::RrdpThenRsync; + let original_notification_uri = ca.rrdp_notification_uri.as_deref(); + let mut effective_notification_uri = if attempted_rrdp { + original_notification_uri + } else { + None + }; + let mut skip_sync_due_to_dedup = false; + + if attempted_rrdp && self.rrdp_dedup { + if let Some(notification_uri) = original_notification_uri { + if let Some(rrdp_ok) = self + .rrdp_repo_cache + .lock() + .expect("rrdp_repo_cache lock") + .get(notification_uri) + .copied() + { + if let Some(t) = self.timing.as_ref() { + t.record_count("rrdp_repo_dedup_hits", 1); + } + if rrdp_ok { + if let Some(t) = self.timing.as_ref() { + t.record_count("rrdp_repo_dedup_rrdp_ok_skip", 1); + } + skip_sync_due_to_dedup = true; + } else { + if let Some(t) = self.timing.as_ref() { + t.record_count("rrdp_repo_dedup_rrdp_failed_skip", 1); + } + effective_notification_uri = None; + } + } else if let Some(t) = self.timing.as_ref() { + t.record_count("rrdp_repo_dedup_misses", 1); + } + } + } + + if !skip_sync_due_to_dedup && effective_notification_uri.is_none() && self.rsync_dedup { + let base = self.rsync_fetcher.dedup_key(&ca.rsync_base_uri); + let hit_ok = self + .rsync_repo_cache + .lock() + .expect("rsync_repo_cache lock") + .get(&base) + .copied() + .unwrap_or(false); + if hit_ok { + if let Some(t) = self.timing.as_ref() { + t.record_count("rsync_repo_dedup_hits", 1); + t.record_count("rsync_repo_dedup_skipped_sync", 1); + } + skip_sync_due_to_dedup = true; + } else if let Some(t) = self.timing.as_ref() { + t.record_count("rsync_repo_dedup_misses", 1); + } + } + + let repo_sync_started = std::time::Instant::now(); + let mut runtime_repo_sync_duration_ms = None; + let (repo_sync_ok, repo_sync_err, repo_sync_source, repo_sync_phase): ( + bool, + Option, + Option, + Option, + ) = if let Some(runtime) = self.repo_sync_runtime.as_ref() { + let RepoSyncRuntimeOutcome { + repo_sync_ok, + repo_sync_err, + repo_sync_source, + repo_sync_phase, + repo_sync_duration_ms, + warnings: repo_warnings, + } = runtime.sync_publication_point_repo(ca)?; + runtime_repo_sync_duration_ms = Some(repo_sync_duration_ms); + warnings.extend(repo_warnings); + ( + repo_sync_ok, + repo_sync_err, + repo_sync_source, + repo_sync_phase, + ) + } else if skip_sync_due_to_dedup { + let source = if effective_notification_uri.is_some() { + Some("rrdp_dedup_skip".to_string()) + } else { + Some("rsync_dedup_skip".to_string()) + }; + let phase = source.clone(); + (true, None, source, phase) + } else { + let repo_key = effective_notification_uri.unwrap_or_else(|| ca.rsync_base_uri.as_str()); + let _repo_total = self + .timing + .as_ref() + .map(|t| t.span_phase("repo_sync_total")); + let _repo_span = self.timing.as_ref().map(|t| t.span_rrdp_repo(repo_key)); + + match if let Some(delta_index) = self.replay_delta_index.as_ref() { + sync_publication_point_replay_delta( + self.store, + delta_index, + effective_notification_uri, + &ca.rsync_base_uri, + self.http_fetcher, + self.rsync_fetcher, + self.timing.as_ref(), + self.download_log.as_ref(), + ) + } else if let Some(replay_index) = self.replay_archive_index.as_ref() { + sync_publication_point_replay( + self.store, + replay_index, + effective_notification_uri, + &ca.rsync_base_uri, + self.http_fetcher, + self.rsync_fetcher, + self.timing.as_ref(), + self.download_log.as_ref(), + ) + } else { + sync_publication_point( + self.store, + self.policy, + effective_notification_uri, + &ca.rsync_base_uri, + self.http_fetcher, + self.rsync_fetcher, + self.timing.as_ref(), + self.download_log.as_ref(), + ) + } { + Ok(res) => { + if self.rsync_dedup && res.source == crate::sync::repo::RepoSyncSource::Rsync { + let base = self.rsync_fetcher.dedup_key(&ca.rsync_base_uri); + self.rsync_repo_cache + .lock() + .expect("rsync_repo_cache lock") + .insert(base, true); + if let Some(t) = self.timing.as_ref() { + t.record_count("rsync_repo_dedup_mark_ok", 1); + } + } + + if attempted_rrdp && self.rrdp_dedup { + if let Some(notification_uri) = original_notification_uri { + if effective_notification_uri.is_some() { + let rrdp_ok = res.source == crate::sync::repo::RepoSyncSource::Rrdp; + self.rrdp_repo_cache + .lock() + .expect("rrdp_repo_cache lock") + .insert(notification_uri.to_string(), rrdp_ok); + if let Some(t) = self.timing.as_ref() { + if rrdp_ok { + t.record_count("rrdp_repo_dedup_mark_ok", 1); + } else { + t.record_count("rrdp_repo_dedup_mark_failed", 1); + } + } + } + } + } + + warnings.extend(res.warnings); + ( + true, + None, + Some(repo_sync_source_label(res.source).to_string()), + Some(repo_sync_phase_label(res.phase).to_string()), + ) + } + Err(e) => { + if attempted_rrdp && self.rrdp_dedup { + if let Some(notification_uri) = original_notification_uri { + if effective_notification_uri.is_some() { + self.rrdp_repo_cache + .lock() + .expect("rrdp_repo_cache lock") + .insert(notification_uri.to_string(), false); + } + } + } + warnings.push( + Warning::new(format!("repo sync failed (fresh processing stopped): {e}")) + .with_rfc_refs(&[RfcRef("RFC 8182 §3.4.5"), RfcRef("RFC 9286 §6.6")]) + .with_context(&ca.rsync_base_uri), + ); + ( + false, + Some(e.to_string()), + None, + Some( + repo_sync_failure_phase_label( + attempted_rrdp, + original_notification_uri, + effective_notification_uri, + ) + .to_string(), + ), + ) + } + } + }; + let repo_sync_duration_ms = effective_repo_sync_duration_ms( + repo_sync_started.elapsed().as_millis() as u64, + runtime_repo_sync_duration_ms, + repo_sync_ok, + ); + crate::progress_log::emit( + "publication_point_repo_sync_done", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "repo_sync_ok": repo_sync_ok, + "repo_sync_source": repo_sync_source, + "repo_sync_phase": repo_sync_phase, + "repo_sync_error": repo_sync_err, + "repo_sync_duration_ms": repo_sync_duration_ms, + }), + ); + + if let Some(result) = self.observe_or_reuse_publication_point_cache( + ca, + repo_sync_source.as_deref(), + repo_sync_phase.as_deref(), + repo_sync_duration_ms, + repo_sync_err.as_deref(), + &warnings, + ) { + let total_duration_ms = publication_point_started.elapsed().as_millis() as u64; + crate::progress_log::emit( + "publication_point_finish", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "source": source_label(result.source), + "repo_sync_source": repo_sync_source, + "repo_sync_phase": repo_sync_phase, + "repo_sync_duration_ms": repo_sync_duration_ms, + "total_duration_ms": total_duration_ms, + "post_repo_duration_ms": total_duration_ms.saturating_sub(repo_sync_duration_ms), + "warning_count": result.warnings.len(), + "vrp_count": result.objects.vrps.len(), + "vap_count": result.objects.aspas.len(), + "router_key_count": result.objects.router_keys.len(), + "child_count": result.discovered_children.len(), + }), + ); + return Ok(result); + } + + let fresh_stage = self.stage_fresh_publication_point_after_repo_ready( + ca, + repo_sync_ok, + repo_sync_err.as_deref(), + ); + + match fresh_stage { + Ok(stage) => { + let FreshPublicationPointStage { + fresh_point, + issuer_ca_der, + snapshot_prepare_timing, + snapshot_prepare_ms, + discovered_children, + child_audits, + discovered_router_keys, + child_discovery_ms, + warnings: stage_warnings, + } = stage; + warnings.extend(stage_warnings); + + let has_roa = fresh_point + .files() + .iter() + .any(|file| file.rsync_uri.ends_with(".roa")); + if self.enable_roa_validation_cache { + if let Some(timing) = self.timing.as_ref() { + if has_roa { + timing.record_count( + "roa_validation_cache_roa_candidate_publication_points", + 1, + ); + } else { + timing.record_count( + "roa_validation_cache_skipped_no_roa_publication_points", + 1, + ); + } + } + } + let roa_cache_view = if has_roa { + self.roa_validation_cache_view_for_fresh_point(fresh_point.manifest_rsync_uri()) + } else { + None + }; + let roa_cache = if self.enable_roa_validation_cache && has_roa { + RoaValidationCacheInput::enabled_with_context( + roa_cache_view.as_ref(), + ca_validation_context_digest_for_ca(ca), + publication_point_cache_policy_fingerprint(self.policy), + ) + } else { + RoaValidationCacheInput::disabled() + }; + let objects_processing_started = std::time::Instant::now(); + let ta_constraints = self.policy.ta_constraints.for_tal(&ca.tal_id); + let mut objects = { + let _objects_total = self + .timing + .as_ref() + .map(|t| t.span_phase("objects_processing_total")); + if let Some(ta_constraints) = ta_constraints { + // This method is the serial/fallback publication-point path. The + // phase-2 scheduler uses the stage-specific parallel prepare path, + // which carries the same immutable per-TAL snapshot into workers. + process_publication_point_for_issuer_with_cache_options_and_ta_constraints( + &fresh_point, + self.policy, + issuer_ca_der.as_ref(), + ca.ca_certificate_rsync_uri.as_deref(), + ca.effective_ip_resources.as_ref(), + ca.effective_as_resources.as_ref(), + self.validation_time, + self.timing.as_ref(), + false, + roa_cache, + Some(ta_constraints), + ) + } else if let Some(phase2_pool) = self.parallel_roa_worker_pool.as_ref() { + process_publication_point_for_issuer_parallel_roa_with_pool_cache_options( + &fresh_point, + self.policy, + issuer_ca_der.as_ref(), + ca.ca_certificate_rsync_uri.as_deref(), + ca.effective_ip_resources.as_ref(), + ca.effective_as_resources.as_ref(), + self.validation_time, + self.timing.as_ref(), + phase2_pool, + false, + roa_cache, + ) + } else if let Some(phase2_config) = self.parallel_phase2_config.as_ref() { + process_publication_point_for_issuer_parallel_roa_with_cache_options( + &fresh_point, + self.policy, + issuer_ca_der.as_ref(), + ca.ca_certificate_rsync_uri.as_deref(), + ca.effective_ip_resources.as_ref(), + ca.effective_as_resources.as_ref(), + self.validation_time, + self.timing.as_ref(), + phase2_config, + false, + roa_cache, + ) + } else { + crate::validation::objects::process_publication_point_for_issuer_with_cache_options( + &fresh_point, + self.policy, + issuer_ca_der.as_ref(), + ca.ca_certificate_rsync_uri.as_deref(), + ca.effective_ip_resources.as_ref(), + ca.effective_as_resources.as_ref(), + self.validation_time, + self.timing.as_ref(), + false, + roa_cache, + ) + } + }; + let objects_processing_ms = objects_processing_started.elapsed().as_millis() as u64; + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_objects_processing", + objects_processing_ms, + ); + + objects.router_keys.extend(discovered_router_keys); + objects + .local_outputs_cache + .extend(build_router_key_local_outputs(ca, &objects.router_keys)); + + let finalized = self.finalize_fresh_publication_point_from_reducer( + ca, + &fresh_point, + warnings, + objects, + child_audits, + discovered_children, + repo_sync_source.as_deref(), + repo_sync_phase.as_deref(), + repo_sync_duration_ms, + repo_sync_err.as_deref(), + )?; + let FreshPublicationPointFinalizeOutput { + result, + snapshot_pack_ms, + persist_vcir_ms, + persist_vcir_timing, + ccr_projection_build_ms, + ccr_append_ms, + audit_build_ms, + } = finalized; + let total_duration_ms = publication_point_started.elapsed().as_millis() as u64; + crate::progress_log::emit( + "publication_point_finish", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "source": "fresh", + "repo_sync_source": repo_sync_source, + "repo_sync_phase": repo_sync_phase, + "repo_sync_duration_ms": repo_sync_duration_ms, + "total_duration_ms": total_duration_ms, + "post_repo_duration_ms": total_duration_ms.saturating_sub(repo_sync_duration_ms), + "snapshot_prepare_ms": snapshot_prepare_ms, + "snapshot_manifest_load_ms": snapshot_prepare_timing.manifest_load_ms, + "snapshot_manifest_decode_ms": snapshot_prepare_timing.manifest_decode_ms, + "snapshot_replay_guard_ms": snapshot_prepare_timing.replay_guard_ms, + "snapshot_manifest_entries_ms": snapshot_prepare_timing.manifest_entries_ms, + "snapshot_pack_files_ms": snapshot_prepare_timing.pack_files_ms, + "snapshot_ee_path_validate_ms": snapshot_prepare_timing.ee_path_validate_ms, + "objects_processing_ms": objects_processing_ms, + "child_discovery_ms": child_discovery_ms, + "snapshot_pack_ms": snapshot_pack_ms, + "persist_vcir_ms": persist_vcir_ms, + "persist_embedded_collect_ms": persist_vcir_timing.embedded_collect_ms, + "persist_embedded_store_ms": persist_vcir_timing.embedded_store_ms, + "persist_build_vcir_ms": persist_vcir_timing.build_vcir_ms, + "persist_replace_vcir_ms": persist_vcir_timing.replace_vcir_ms, + "persist_select_crl_ms": persist_vcir_timing.build_vcir.select_crl_ms, + "persist_current_ca_decode_ms": persist_vcir_timing.build_vcir.current_ca_decode_ms, + "persist_local_outputs_ms": persist_vcir_timing.build_vcir.local_outputs_ms, + "persist_child_entries_ms": persist_vcir_timing.build_vcir.child_entries_ms, + "persist_related_artifacts_ms": persist_vcir_timing.build_vcir.related_artifacts_ms, + "persist_vcir_struct_ms": persist_vcir_timing.build_vcir.struct_build_ms, + "persist_replace_breakdown": &persist_vcir_timing.replace_vcir, + "publication_point_cache_future_notbefore_guarded": persist_vcir_timing.publication_point_cache_future_notbefore_guarded, + "ccr_projection_build_ms": ccr_projection_build_ms, + "ccr_append_ms": ccr_append_ms, + "audit_build_ms": audit_build_ms, + "warning_count": result.warnings.len(), + "vrp_count": result.objects.vrps.len(), + "vap_count": result.objects.aspas.len(), + "router_key_count": result.objects.router_keys.len(), + "child_count": result.discovered_children.len(), + }), + ); + if (total_duration_ms as f64) / 1000.0 >= crate::progress_log::slow_threshold_secs() + { + crate::progress_log::emit( + "publication_point_slow", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "source": "fresh", + "repo_sync_source": repo_sync_source, + "repo_sync_phase": repo_sync_phase, + "repo_sync_duration_ms": repo_sync_duration_ms, + "total_duration_ms": total_duration_ms, + "post_repo_duration_ms": total_duration_ms.saturating_sub(repo_sync_duration_ms), + "snapshot_prepare_ms": snapshot_prepare_ms, + "snapshot_manifest_load_ms": snapshot_prepare_timing.manifest_load_ms, + "snapshot_manifest_decode_ms": snapshot_prepare_timing.manifest_decode_ms, + "snapshot_replay_guard_ms": snapshot_prepare_timing.replay_guard_ms, + "snapshot_manifest_entries_ms": snapshot_prepare_timing.manifest_entries_ms, + "snapshot_pack_files_ms": snapshot_prepare_timing.pack_files_ms, + "snapshot_ee_path_validate_ms": snapshot_prepare_timing.ee_path_validate_ms, + "objects_processing_ms": objects_processing_ms, + "child_discovery_ms": child_discovery_ms, + "snapshot_pack_ms": snapshot_pack_ms, + "persist_vcir_ms": persist_vcir_ms, + "persist_embedded_collect_ms": persist_vcir_timing.embedded_collect_ms, + "persist_embedded_store_ms": persist_vcir_timing.embedded_store_ms, + "persist_build_vcir_ms": persist_vcir_timing.build_vcir_ms, + "persist_replace_vcir_ms": persist_vcir_timing.replace_vcir_ms, + "persist_select_crl_ms": persist_vcir_timing.build_vcir.select_crl_ms, + "persist_current_ca_decode_ms": persist_vcir_timing.build_vcir.current_ca_decode_ms, + "persist_local_outputs_ms": persist_vcir_timing.build_vcir.local_outputs_ms, + "persist_child_entries_ms": persist_vcir_timing.build_vcir.child_entries_ms, + "persist_related_artifacts_ms": persist_vcir_timing.build_vcir.related_artifacts_ms, + "persist_vcir_struct_ms": persist_vcir_timing.build_vcir.struct_build_ms, + "persist_replace_breakdown": &persist_vcir_timing.replace_vcir, + "publication_point_cache_future_notbefore_guarded": persist_vcir_timing.publication_point_cache_future_notbefore_guarded, + "ccr_projection_build_ms": ccr_projection_build_ms, + "ccr_append_ms": ccr_append_ms, + "audit_build_ms": audit_build_ms, + }), + ); + } + Ok(result) + } + Err(stage_err) => { + let snapshot_prepare_ms = stage_err.snapshot_prepare_ms; + let fresh_err = stage_err.error; + match self.policy.ca_failed_fetch_policy { + crate::policy::CaFailedFetchPolicy::StopAllOutput => { + let total_duration_ms = + publication_point_started.elapsed().as_millis() as u64; + crate::progress_log::emit( + "publication_point_finish", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "source": "error", + "repo_sync_source": repo_sync_source, + "repo_sync_phase": repo_sync_phase, + "repo_sync_duration_ms": repo_sync_duration_ms, + "total_duration_ms": total_duration_ms, + "post_repo_duration_ms": total_duration_ms.saturating_sub(repo_sync_duration_ms), + "snapshot_prepare_ms": snapshot_prepare_ms, + "projection_ms": 0, + "audit_build_ms": 0, + "error": fresh_err.to_string(), + }), + ); + crate::progress_log::emit( + "repo_terminal_failure", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "repo_sync_source": repo_sync_source, + "repo_sync_phase": repo_sync_phase, + "repo_sync_error": repo_sync_err, + "repo_sync_duration_ms": repo_sync_duration_ms, + "terminal_state": "stop_all_output", + "error": fresh_err.to_string(), + }), + ); + Err(format!("{fresh_err}")) + } + crate::policy::CaFailedFetchPolicy::ReuseCurrentInstanceVcir => { + let projection_started = std::time::Instant::now(); + let projection = project_current_instance_vcir_on_failed_fetch( + self.store, + ca, + &fresh_err, + self.policy, + self.validation_time, + ) + .map_err(|e| format!("failed fetch VCIR projection failed: {e}"))?; + let fresh_failure_audits = + self.fresh_failure_audit_entries_for_cir(ca, &fresh_err); + self.append_ccr_manifest_projection_from_reuse(&projection)?; + let projection_ms = projection_started.elapsed().as_millis() as u64; + warnings.extend(projection.warnings.clone()); + let audit_build_started = std::time::Instant::now(); + let audit = build_publication_point_audit_from_vcir( + ca, + projection.source, + repo_sync_source.as_deref(), + repo_sync_phase.as_deref(), + Some(repo_sync_duration_ms), + repo_sync_err.as_deref(), + projection.vcir.as_ref(), + projection.snapshot.as_ref(), + &warnings, + &projection.objects, + &projection.child_audits, + &fresh_failure_audits, + ); + let audit_build_ms = audit_build_started.elapsed().as_millis() as u64; + let cir_cached_objects = + if projection.source == PublicationPointSource::VcirCurrentInstance { + audit + .objects + .iter() + .filter(|entry| { + !fresh_failure_audits.iter().any(|fresh| fresh == *entry) + }) + .cloned() + .collect() + } else { + Vec::new() + }; + let result = PublicationPointRunResult { + source: projection.source, + snapshot: projection.snapshot, + warnings, + objects: projection.objects, + audit, + cir_fresh_objects: if projection.source + == PublicationPointSource::VcirCurrentInstance + { + fresh_failure_audits + } else { + Vec::new() + }, + cir_cached_objects, + discovered_children: projection.discovered_children, + }; + let total_duration_ms = + publication_point_started.elapsed().as_millis() as u64; + crate::progress_log::emit( + "publication_point_finish", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "source": source_label(result.source), + "repo_sync_source": repo_sync_source, + "repo_sync_phase": repo_sync_phase, + "repo_sync_duration_ms": repo_sync_duration_ms, + "total_duration_ms": total_duration_ms, + "post_repo_duration_ms": total_duration_ms.saturating_sub(repo_sync_duration_ms), + "snapshot_prepare_ms": snapshot_prepare_ms, + "projection_ms": projection_ms, + "audit_build_ms": audit_build_ms, + "warning_count": result.warnings.len(), + "vrp_count": result.objects.vrps.len(), + "vap_count": result.objects.aspas.len(), + "router_key_count": result.objects.router_keys.len(), + "child_count": result.discovered_children.len(), + }), + ); + match result.source { + PublicationPointSource::VcirCurrentInstance if !repo_sync_ok => { + crate::progress_log::emit( + "rsync_failed_fallback_current_instance", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "repo_sync_source": repo_sync_source, + "repo_sync_phase": repo_sync_phase, + "repo_sync_error": repo_sync_err, + "repo_sync_duration_ms": repo_sync_duration_ms, + "terminal_state": "fallback_current_instance", + }), + ); + } + PublicationPointSource::FailedFetchNoCache => { + if !repo_sync_ok { + crate::progress_log::emit( + "rsync_failed_no_cache", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "repo_sync_source": repo_sync_source, + "repo_sync_phase": repo_sync_phase, + "repo_sync_error": repo_sync_err, + "repo_sync_duration_ms": repo_sync_duration_ms, + "terminal_state": "failed_no_cache", + }), + ); + } + crate::progress_log::emit( + "repo_terminal_failure", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "repo_sync_source": repo_sync_source, + "repo_sync_phase": repo_sync_phase, + "repo_sync_error": repo_sync_err, + "repo_sync_duration_ms": repo_sync_duration_ms, + "terminal_state": "failed_no_cache", + }), + ); + } + PublicationPointSource::Fresh => {} + PublicationPointSource::PublicationPointCache => {} + PublicationPointSource::VcirCurrentInstance => {} + } + if (total_duration_ms as f64) / 1000.0 + >= crate::progress_log::slow_threshold_secs() + { + crate::progress_log::emit( + "publication_point_slow", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "source": source_label(result.source), + "repo_sync_source": repo_sync_source, + "repo_sync_phase": repo_sync_phase, + "repo_sync_duration_ms": repo_sync_duration_ms, + "total_duration_ms": total_duration_ms, + "post_repo_duration_ms": total_duration_ms.saturating_sub(repo_sync_duration_ms), + "snapshot_prepare_ms": snapshot_prepare_ms, + "projection_ms": projection_ms, + "audit_build_ms": audit_build_ms, + }), + ); + } + Ok(result) + } + } + } + } + } +} diff --git a/crates/panda-rpki-validator/src/validation/tree_runner/tests.rs b/crates/panda-rpki-validator/src/validation/tree_runner/tests.rs index 8a7ecd9..ca7c110 100644 --- a/crates/panda-rpki-validator/src/validation/tree_runner/tests.rs +++ b/crates/panda-rpki-validator/src/validation/tree_runner/tests.rs @@ -1,5869 +1,8 @@ -use super::*; -use crate::data_model::oid::OID_AD_SIGNED_OBJECT; -use crate::data_model::rc::{ - AccessDescription, AsIdOrRange, AsIdentifierChoice, AsResourceSet, ResourceCertificate, -}; -use crate::data_model::roa::RoaAfi; -use crate::fetch::rsync::LocalDirRsyncFetcher; -use crate::fetch::rsync::{RsyncFetchError, RsyncFetcher}; -use crate::storage::{ - PackFile, PackTime, PublicationPointCacheProjection, - PublicationPointCacheProjectionWriteAction, RawByHashEntry, RepositoryViewEntry, - RepositoryViewState, RocksStore, ValidatedCaInstanceResult, ValidatedManifestMeta, - VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus, VcirAuditSummary, - VcirChildEntry, VcirInstanceGate, VcirLocalOutput, VcirLocalOutputPayload, VcirOutputType, - VcirRelatedArtifact, VcirSourceObjectType, VcirSummary, -}; -use crate::sync::rrdp::Fetcher; -use crate::validation::publication_point::PublicationPointSnapshot; -use crate::validation::tree::{DiscoveredChildEntryProjection, PublicationPointRunner}; - -use std::process::Command; -use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; - -fn sha256_32(input: &[u8]) -> [u8; 32] { - sha256_hex_to_32(&sha256_hex(input)) -} - -fn ipv4_addr(octets: [u8; 4]) -> [u8; 16] { - let mut addr = [0u8; 16]; - addr[..4].copy_from_slice(&octets); - addr -} - -#[test] -fn publication_point_cache_policy_fingerprint_includes_resource_validation_mode() { - let mut strict_policy = Policy::default(); - strict_policy.resource_validation_mode = ResourceValidationMode::Rfc6487; - let mut vrs_policy = Policy::default(); - vrs_policy.resource_validation_mode = ResourceValidationMode::ValidationUpdate03; - - assert_ne!( - publication_point_cache_policy_fingerprint(&strict_policy), - publication_point_cache_policy_fingerprint(&vrs_policy) - ); -} - -#[test] -fn router_asns_for_resource_mode_filters_vrs_and_rejects_strict_overclaim() { - let issuer_as = AsResourceSet { - asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ - AsIdOrRange::Range { - min: 64500, - max: 64510, - }, - ])), - rdi: None, - }; - - let strict = router_asns_for_resource_mode( - &[64505, 64520], - Some(&issuer_as), - ResourceValidationMode::Rfc6487, - ) - .unwrap_err(); - assert!(strict.contains("not a subset"), "{strict}"); - - let vrs = router_asns_for_resource_mode( - &[64505, 64520], - Some(&issuer_as), - ResourceValidationMode::ValidationUpdate03, - ) - .expect("vrs filters router asns"); - assert_eq!(vrs, vec![64505]); -} - -struct NeverHttpFetcher; -impl Fetcher for NeverHttpFetcher { - fn fetch(&self, _uri: &str) -> Result, String> { - Err("http fetch disabled in test".to_string()) - } -} - -struct FailingRsyncFetcher; -impl RsyncFetcher for FailingRsyncFetcher { - fn fetch_objects( - &self, - _rsync_base_uri: &str, - ) -> Result)>, RsyncFetchError> { - Err(RsyncFetchError::Fetch("rsync disabled in test".to_string())) - } -} - -fn sample_runner_with_ccr_accumulator<'a>( - store: &'a RocksStore, - policy: &'a Policy, -) -> Rpkiv1PublicationPointRunner<'a> { - Rpkiv1PublicationPointRunner { - store, - policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &FailingRsyncFetcher, - validation_time: time::OffsetDateTime::now_utc(), - timing: None, - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: false, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: Some(Mutex::new(CcrAccumulator::new(Vec::new()))), - persist_vcir: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - } -} - -fn openssl_available() -> bool { - Command::new("openssl") - .arg("version") - .output() - .map(|o| o.status.success()) - .unwrap_or(false) -} - -struct Generated { - issuer_ca_der: Vec, - child_ca_der: Vec, - issuer_crl_der: Vec, - issuer_crl_der_next: Vec, -} - -fn run(cmd: &mut Command) { - let out = cmd.output().expect("run command"); - if !out.status.success() { - panic!( - "command failed: {:?}\nstdout={}\nstderr={}", - cmd, - String::from_utf8_lossy(&out.stdout), - String::from_utf8_lossy(&out.stderr) - ); - } -} - -fn generate_chain_and_crl() -> Generated { - assert!(openssl_available(), "openssl is required for this test"); - - let td = tempfile::tempdir().expect("tempdir"); - let dir = td.path(); - - std::fs::create_dir_all(dir.join("newcerts")).expect("newcerts"); - std::fs::write(dir.join("index.txt"), b"").expect("index"); - std::fs::write(dir.join("serial"), b"1000\n").expect("serial"); - std::fs::write(dir.join("crlnumber"), b"1000\n").expect("crlnumber"); - - let cnf = format!( - r#" -[ ca ] -default_ca = CA_default - -[ CA_default ] -dir = {dir} -database = $dir/index.txt -new_certs_dir = $dir/newcerts -certificate = $dir/issuer.pem -private_key = $dir/issuer.key -serial = $dir/serial -crlnumber = $dir/crlnumber -default_md = sha256 -default_days = 365 -default_crl_days = 1 -policy = policy_any -x509_extensions = v3_issuer_ca -crl_extensions = crl_ext -unique_subject = no -copy_extensions = none - -[ policy_any ] -commonName = supplied - -[ req ] -prompt = no -distinguished_name = dn - -[ dn ] -CN = Test Issuer CA - -[ v3_issuer_ca ] -basicConstraints = critical,CA:true -keyUsage = critical, keyCertSign, cRLSign -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid:always -certificatePolicies = critical, 1.3.6.1.5.5.7.14.2 -subjectInfoAccess = caRepository;URI:rsync://example.test/repo/issuer/, rpkiManifest;URI:rsync://example.test/repo/issuer/issuer.mft, rpkiNotify;URI:https://example.test/notification.xml -sbgp-ipAddrBlock = critical, IPv4:10.0.0.0/8 -sbgp-autonomousSysNum = critical, AS:64496-64511 - -[ v3_child_ca ] -basicConstraints = critical,CA:true -keyUsage = critical, keyCertSign, cRLSign -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid:always -crlDistributionPoints = URI:rsync://example.test/repo/issuer/issuer.crl -authorityInfoAccess = caIssuers;URI:rsync://example.test/repo/issuer/issuer.cer -certificatePolicies = critical, 1.3.6.1.5.5.7.14.2 -subjectInfoAccess = caRepository;URI:rsync://example.test/repo/child/, rpkiManifest;URI:rsync://example.test/repo/child/child.mft, rpkiNotify;URI:https://example.test/notification.xml -sbgp-ipAddrBlock = critical, IPv4:10.0.0.0/16 -sbgp-autonomousSysNum = critical, AS:64496 - -[ crl_ext ] -authorityKeyIdentifier = keyid:always -"#, - dir = dir.display() - ); - std::fs::write(dir.join("openssl.cnf"), cnf.as_bytes()).expect("write cnf"); - - run(Command::new("openssl") - .arg("genrsa") - .arg("-out") - .arg(dir.join("issuer.key")) - .arg("2048")); - run(Command::new("openssl") - .arg("req") - .arg("-new") - .arg("-x509") - .arg("-sha256") - .arg("-days") - .arg("365") - .arg("-key") - .arg(dir.join("issuer.key")) - .arg("-config") - .arg(dir.join("openssl.cnf")) - .arg("-extensions") - .arg("v3_issuer_ca") - .arg("-out") - .arg(dir.join("issuer.pem"))); - - run(Command::new("openssl") - .arg("genrsa") - .arg("-out") - .arg(dir.join("child.key")) - .arg("2048")); - run(Command::new("openssl") - .arg("req") - .arg("-new") - .arg("-key") - .arg(dir.join("child.key")) - .arg("-subj") - .arg("/CN=Test Child CA") - .arg("-out") - .arg(dir.join("child.csr"))); - - run(Command::new("openssl") - .arg("ca") - .arg("-batch") - .arg("-config") - .arg(dir.join("openssl.cnf")) - .arg("-in") - .arg(dir.join("child.csr")) - .arg("-extensions") - .arg("v3_child_ca") - .arg("-out") - .arg(dir.join("child.pem"))); - - run(Command::new("openssl") - .arg("x509") - .arg("-in") - .arg(dir.join("issuer.pem")) - .arg("-outform") - .arg("DER") - .arg("-out") - .arg(dir.join("issuer.cer"))); - run(Command::new("openssl") - .arg("x509") - .arg("-in") - .arg(dir.join("child.pem")) - .arg("-outform") - .arg("DER") - .arg("-out") - .arg(dir.join("child.cer"))); - - run(Command::new("openssl") - .arg("ca") - .arg("-gencrl") - .arg("-config") - .arg(dir.join("openssl.cnf")) - .arg("-out") - .arg(dir.join("issuer.crl.pem"))); - run(Command::new("openssl") - .arg("crl") - .arg("-in") - .arg(dir.join("issuer.crl.pem")) - .arg("-outform") - .arg("DER") - .arg("-out") - .arg(dir.join("issuer.crl"))); - run(Command::new("openssl") - .arg("ca") - .arg("-gencrl") - .arg("-config") - .arg(dir.join("openssl.cnf")) - .arg("-out") - .arg(dir.join("issuer-next.crl.pem"))); - run(Command::new("openssl") - .arg("crl") - .arg("-in") - .arg(dir.join("issuer-next.crl.pem")) - .arg("-outform") - .arg("DER") - .arg("-out") - .arg(dir.join("issuer-next.crl"))); - - Generated { - issuer_ca_der: std::fs::read(dir.join("issuer.cer")).expect("read issuer der"), - child_ca_der: std::fs::read(dir.join("child.cer")).expect("read child der"), - issuer_crl_der: std::fs::read(dir.join("issuer.crl")).expect("read crl der"), - issuer_crl_der_next: std::fs::read(dir.join("issuer-next.crl")).expect("read next crl der"), - } -} - -struct GeneratedRouter { - issuer_ca_der: Vec, - router_der: Vec, - issuer_crl_der: Vec, -} - -fn generate_router_cert_with_variant(key_spec: &str, include_eku: bool) -> GeneratedRouter { - assert!(openssl_available(), "openssl is required for this test"); - - let td = tempfile::tempdir().expect("tempdir"); - let dir = td.path(); - - std::fs::create_dir_all(dir.join("newcerts")).expect("newcerts"); - std::fs::write(dir.join("index.txt"), b"").expect("index"); - std::fs::write(dir.join("serial"), b"1000\n").expect("serial"); - std::fs::write(dir.join("crlnumber"), b"1000\n").expect("crlnumber"); - - let eku_line = if include_eku { - "extendedKeyUsage = 1.3.6.1.5.5.7.3.30" - } else { - "" - }; - let cnf = format!( - r#" -[ ca ] -default_ca = CA_default - -[ CA_default ] -dir = {dir} -database = $dir/index.txt -new_certs_dir = $dir/newcerts -certificate = $dir/issuer.pem -private_key = $dir/issuer.key -serial = $dir/serial -crlnumber = $dir/crlnumber -default_md = sha256 -default_days = 365 -default_crl_days = 1 -policy = policy_any -x509_extensions = v3_issuer_ca -crl_extensions = crl_ext -unique_subject = no -copy_extensions = none - -[ policy_any ] -commonName = supplied - -[ req ] -prompt = no -distinguished_name = dn - -[ dn ] -CN = Test Issuer CA - -[ v3_issuer_ca ] -basicConstraints = critical,CA:true -keyUsage = critical, keyCertSign, cRLSign -subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid:always -certificatePolicies = critical, 1.3.6.1.5.5.7.14.2 -subjectInfoAccess = caRepository;URI:rsync://example.test/repo/issuer/, rpkiManifest;URI:rsync://example.test/repo/issuer/issuer.mft, rpkiNotify;URI:https://example.test/notification.xml -sbgp-ipAddrBlock = critical, IPv4:10.0.0.0/8 -sbgp-autonomousSysNum = critical, AS:64496-64511 - -[ v3_router ] -keyUsage = critical, digitalSignature -{eku_line} -authorityKeyIdentifier = keyid:always -crlDistributionPoints = URI:rsync://example.test/repo/issuer/issuer.crl -authorityInfoAccess = caIssuers;URI:rsync://example.test/repo/issuer/issuer.cer -certificatePolicies = critical, 1.3.6.1.5.5.7.14.2 -sbgp-autonomousSysNum = critical, AS:64496 - -[ crl_ext ] -authorityKeyIdentifier = keyid:always -"#, - dir = dir.display(), - eku_line = eku_line, - ); - std::fs::write(dir.join("openssl.cnf"), cnf.as_bytes()).expect("write cnf"); - - run(Command::new("openssl") - .arg("genrsa") - .arg("-out") - .arg(dir.join("issuer.key")) - .arg("2048")); - run(Command::new("openssl") - .arg("req") - .arg("-new") - .arg("-x509") - .arg("-sha256") - .arg("-days") - .arg("365") - .arg("-key") - .arg(dir.join("issuer.key")) - .arg("-config") - .arg(dir.join("openssl.cnf")) - .arg("-extensions") - .arg("v3_issuer_ca") - .arg("-out") - .arg(dir.join("issuer.pem"))); - - match key_spec { - "ec-p256" => run(Command::new("openssl") - .arg("ecparam") - .arg("-name") - .arg("prime256v1") - .arg("-genkey") - .arg("-noout") - .arg("-out") - .arg(dir.join("router.key"))), - "ec-p384" => run(Command::new("openssl") - .arg("ecparam") - .arg("-name") - .arg("secp384r1") - .arg("-genkey") - .arg("-noout") - .arg("-out") - .arg(dir.join("router.key"))), - other => panic!("unsupported key_spec {other}"), - } - - run(Command::new("openssl") - .arg("req") - .arg("-new") - .arg("-key") - .arg(dir.join("router.key")) - .arg("-subj") - .arg("/CN=ROUTER-0000FC10/serialNumber=01020304") - .arg("-out") - .arg(dir.join("router.csr"))); - - run(Command::new("openssl") - .arg("ca") - .arg("-batch") - .arg("-config") - .arg(dir.join("openssl.cnf")) - .arg("-in") - .arg(dir.join("router.csr")) - .arg("-extensions") - .arg("v3_router") - .arg("-out") - .arg(dir.join("router.pem"))); - - run(Command::new("openssl") - .arg("x509") - .arg("-in") - .arg(dir.join("issuer.pem")) - .arg("-outform") - .arg("DER") - .arg("-out") - .arg(dir.join("issuer.cer"))); - run(Command::new("openssl") - .arg("x509") - .arg("-in") - .arg(dir.join("router.pem")) - .arg("-outform") - .arg("DER") - .arg("-out") - .arg(dir.join("router.cer"))); - - run(Command::new("openssl") - .arg("ca") - .arg("-gencrl") - .arg("-config") - .arg(dir.join("openssl.cnf")) - .arg("-out") - .arg(dir.join("issuer.crl.pem"))); - run(Command::new("openssl") - .arg("crl") - .arg("-in") - .arg(dir.join("issuer.crl.pem")) - .arg("-outform") - .arg("DER") - .arg("-out") - .arg(dir.join("issuer.crl"))); - - GeneratedRouter { - issuer_ca_der: std::fs::read(dir.join("issuer.cer")).expect("read issuer der"), - router_der: std::fs::read(dir.join("router.cer")).expect("read router der"), - issuer_crl_der: std::fs::read(dir.join("issuer.crl")).expect("read crl der"), - } -} -fn dummy_pack_with_files(files: Vec) -> PublicationPointSnapshot { - let now = time::OffsetDateTime::now_utc(); - PublicationPointSnapshot { - format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_number_be: vec![1], - this_update: PackTime::from_utc_offset_datetime(now), - next_update: PackTime::from_utc_offset_datetime(now + time::Duration::hours(1)), - verified_at: PackTime::from_utc_offset_datetime(now), - manifest_bytes: vec![0x01], - files, - } -} - -fn cernet_publication_point_snapshot_for_vcir_tests() --> (PublicationPointSnapshot, Vec, time::OffsetDateTime) { - let dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); - let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/"; - let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; - let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); - let manifest_bytes = std::fs::read(dir.join(manifest_file)).expect("read manifest fixture"); - let manifest = ManifestObject::decode_der(&manifest_bytes).expect("decode manifest fixture"); - let candidate = manifest.manifest.this_update + time::Duration::seconds(60); - let validation_time = if candidate < manifest.manifest.next_update { - candidate - } else { - manifest.manifest.this_update - }; - - let issuer_ca_der = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - ) - .expect("read issuer ca fixture"); - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy { - sync_preference: crate::policy::SyncPreference::RsyncOnly, - ..Policy::default() - }; - - sync_publication_point( - &store, - &policy, - None, - rsync_base_uri, - &NeverHttpFetcher, - &LocalDirRsyncFetcher::new(&dir), - None, - None, - ) - .expect("sync cernet fixture"); - - let pp = crate::validation::manifest::process_manifest_publication_point( - &store, - &policy, - &manifest_rsync_uri, - rsync_base_uri, - issuer_ca_der.as_slice(), - Some( - "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - validation_time, - ) - .expect("process manifest publication point"); - - (pp.snapshot, issuer_ca_der, validation_time) -} - -fn sample_vcir_for_projection( - now: time::OffsetDateTime, - child_cert_hash: &str, -) -> ValidatedCaInstanceResult { - let manifest_uri = "rsync://example.test/repo/issuer/issuer.mft".to_string(); - let current_crl_uri = "rsync://example.test/repo/issuer/issuer.crl".to_string(); - let child_cert_uri = "rsync://example.test/repo/issuer/child.cer".to_string(); - let child_manifest_uri = "rsync://example.test/repo/child/child.mft".to_string(); - let roa_uri = "rsync://example.test/repo/issuer/a.roa".to_string(); - let aspa_uri = "rsync://example.test/repo/issuer/a.asa".to_string(); - let router_uri = "rsync://example.test/repo/issuer/router.cer".to_string(); - let manifest_hash = sha256_hex(b"manifest-bytes"); - let current_crl_hash = sha256_hex(b"current-crl-bytes"); - let roa_hash = sha256_hex(b"roa-bytes"); - let aspa_hash = sha256_hex(b"aspa-bytes"); - let router_hash = sha256_hex(b"router-bytes"); - let ee_hash = sha256_hex(b"ee-cert-bytes"); - let gate_until = PackTime::from_utc_offset_datetime(now + time::Duration::hours(1)); - let ccr_manifest_projection = VcirCcrManifestProjection { - manifest_rsync_uri: manifest_uri.clone(), - manifest_sha256: hex::decode(&manifest_hash).expect("decode manifest hash"), - manifest_size: 2048, - manifest_ee_aki: vec![0x11; 20], - manifest_number_be: vec![1], - manifest_this_update: PackTime::from_utc_offset_datetime(now), - manifest_sia_locations_der: vec![ - crate::ccr::manifest_location::encode_access_description_der(&AccessDescription { - access_method_oid: OID_AD_SIGNED_OBJECT.to_string(), - access_location: manifest_uri.clone(), - }) - .expect("encode signedObject"), - ], - subordinate_skis: vec![vec![0x33; 20]], - }; - ValidatedCaInstanceResult { - manifest_rsync_uri: manifest_uri.clone(), - parent_manifest_rsync_uri: None, - tal_id: "test-tal".to_string(), - ca_subject_name: "CN=Issuer".to_string(), - ca_ski: "11".repeat(20), - issuer_ski: "22".repeat(20), - last_successful_validation_time: PackTime::from_utc_offset_datetime(now), - current_manifest_rsync_uri: manifest_uri.clone(), - current_crl_rsync_uri: current_crl_uri.clone(), - validated_manifest_meta: ValidatedManifestMeta { - validated_manifest_number: vec![1], - validated_manifest_this_update: PackTime::from_utc_offset_datetime(now), - validated_manifest_next_update: gate_until.clone(), - }, - ccr_manifest_projection, - instance_gate: VcirInstanceGate { - manifest_next_update: gate_until.clone(), - current_crl_next_update: gate_until.clone(), - self_ca_not_after: PackTime::from_utc_offset_datetime(now + time::Duration::hours(2)), - instance_effective_until: gate_until.clone(), - }, - child_entries: vec![VcirChildEntry { - child_manifest_rsync_uri: child_manifest_uri, - child_cert_rsync_uri: child_cert_uri.clone(), - child_cert_hash: child_cert_hash.to_string(), - child_ski: "33".repeat(20), - child_rsync_base_uri: "rsync://example.test/repo/child/".to_string(), - child_publication_point_rsync_uri: "rsync://example.test/repo/child/".to_string(), - child_rrdp_notification_uri: Some("https://example.test/child-notify.xml".to_string()), - child_effective_ip_resources: None, - child_effective_as_resources: None, - accepted_at_validation_time: PackTime::from_utc_offset_datetime(now), - }], - local_outputs: vec![ - VcirLocalOutput { - output_type: VcirOutputType::Vrp, - item_effective_until: PackTime::from_utc_offset_datetime( - now + time::Duration::minutes(30), - ), - source_object_uri: roa_uri.clone(), - source_object_type: VcirSourceObjectType::Roa, - source_object_hash: sha256_hex_to_32(&roa_hash), - source_ee_cert_hash: sha256_hex_to_32(&ee_hash), - payload: VcirLocalOutputPayload::Vrp { - asn: 64496, - afi: RoaAfi::Ipv4, - prefix_len: 24, - addr: ipv4_addr([203, 0, 113, 0]), - max_length: 24, - }, - rule_hash: sha256_32(b"roa-rule"), - }, - VcirLocalOutput { - output_type: VcirOutputType::Aspa, - item_effective_until: PackTime::from_utc_offset_datetime( - now + time::Duration::minutes(30), - ), - source_object_uri: aspa_uri.clone(), - source_object_type: VcirSourceObjectType::Aspa, - source_object_hash: sha256_hex_to_32(&aspa_hash), - source_ee_cert_hash: sha256_hex_to_32(&ee_hash), - payload: VcirLocalOutputPayload::Aspa { - customer_as_id: 64496, - provider_as_ids: vec![64497, 64498], - }, - rule_hash: sha256_32(b"aspa-rule"), - }, - VcirLocalOutput { - output_type: VcirOutputType::RouterKey, - item_effective_until: PackTime::from_utc_offset_datetime( - now + time::Duration::minutes(30), - ), - source_object_uri: router_uri.clone(), - source_object_type: VcirSourceObjectType::RouterKey, - source_object_hash: sha256_hex_to_32(&router_hash), - source_ee_cert_hash: sha256_hex_to_32(&router_hash), - payload: VcirLocalOutputPayload::RouterKey { - as_id: 64496, - ski: vec![0x11; 20], - spki_der: vec![0x30, 0x00], - }, - rule_hash: sha256_32(b"router-key-rule"), - }, - ], - related_artifacts: vec![ - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::Manifest, - artifact_kind: VcirArtifactKind::Mft, - uri: Some(manifest_uri.clone()), - sha256: manifest_hash, - object_type: Some("mft".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - reject_reason: None, - }, - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::CurrentCrl, - artifact_kind: VcirArtifactKind::Crl, - uri: Some(current_crl_uri), - sha256: current_crl_hash, - object_type: Some("crl".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - reject_reason: None, - }, - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::ChildCaCert, - artifact_kind: VcirArtifactKind::Cer, - uri: Some(child_cert_uri), - sha256: child_cert_hash.to_string(), - object_type: Some("cer".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - reject_reason: None, - }, - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::SignedObject, - artifact_kind: VcirArtifactKind::Roa, - uri: Some(roa_uri), - sha256: roa_hash, - object_type: Some("roa".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - reject_reason: None, - }, - VcirRelatedArtifact { - artifact_role: VcirArtifactRole::SignedObject, - artifact_kind: VcirArtifactKind::Aspa, - uri: Some(aspa_uri), - sha256: aspa_hash, - object_type: Some("aspa".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - reject_reason: None, - }, - ], - summary: VcirSummary { - local_vrp_count: 1, - local_aspa_count: 1, - local_router_key_count: 1, - child_count: 1, - accepted_object_count: 4, - rejected_object_count: 0, - }, - audit_summary: VcirAuditSummary { - failed_fetch_eligible: true, - last_failed_fetch_reason: None, - warning_count: 0, - audit_flags: Vec::new(), - }, - } -} - -fn put_vcir_for_failed_fetch_reuse( - store: &RocksStore, - ca: &CaInstanceHandle, - policy: &Policy, - vcir: &ValidatedCaInstanceResult, -) { - let validation_time = vcir - .last_successful_validation_time - .parse() - .expect("parse VCIR validation time"); - let identity = failed_fetch_reuse_identity_for_fresh_result( - ca, - policy, - validation_time, - vcir.instance_gate.instance_effective_until.clone(), - ) - .expect("build VCIR failed-fetch reuse identity"); - store - .put_vcir_with_failed_fetch_reuse_identity(vcir, &identity) - .expect("put reusable VCIR"); -} - -fn sample_ca_for_failed_fetch_reuse(vcir: &ValidatedCaInstanceResult) -> CaInstanceHandle { - CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(Vec::new()), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), - } -} - -#[test] -fn never_http_fetcher_returns_error() { - let f = NeverHttpFetcher; - let err = f.fetch("https://example.test/").unwrap_err(); - assert!(err.contains("disabled"), "{err}"); -} - -#[test] -fn kind_from_rsync_uri_classifies_known_extensions() { - assert_eq!( - kind_from_rsync_uri("rsync://example.test/x.crl"), - AuditObjectKind::Crl - ); - assert_eq!( - kind_from_rsync_uri("rsync://example.test/x.cer"), - AuditObjectKind::Certificate - ); - assert_eq!( - kind_from_rsync_uri("rsync://example.test/x.roa"), - AuditObjectKind::Roa - ); - assert_eq!( - kind_from_rsync_uri("rsync://example.test/x.asa"), - AuditObjectKind::Aspa - ); - assert_eq!( - kind_from_rsync_uri("rsync://example.test/x.bin"), - AuditObjectKind::Other - ); -} - -#[test] -fn build_vcir_local_outputs_prefers_cached_outputs() { - let pack = dummy_pack_with_files(vec![]); - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(vec![1]), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: pack.publication_point_rsync_uri.clone(), - manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), - rrdp_notification_uri: None, - }; - let cached = vec![VcirLocalOutput { - output_type: VcirOutputType::Vrp, - item_effective_until: pack.next_update.clone(), - source_object_uri: "rsync://example.test/repo/issuer/a.roa".to_string(), - source_object_type: VcirSourceObjectType::Roa, - source_object_hash: sha256_32(b"cached-roa"), - source_ee_cert_hash: sha256_32(b"cached-ee"), - payload: VcirLocalOutputPayload::Vrp { - asn: 64500, - afi: RoaAfi::Ipv4, - prefix_len: 24, - addr: ipv4_addr([203, 0, 113, 0]), - max_length: 24, - }, - rule_hash: sha256_32(b"cached-rule"), - }]; - let mut objects = crate::validation::objects::ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: cached.clone(), - warnings: Vec::new(), - stats: crate::validation::objects::ObjectsStats::default(), - audit: Vec::new(), - roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), - roa_cache_object_meta: Vec::new(), - }; - let outputs = - take_or_build_vcir_local_outputs(&ca, &pack, &mut objects).expect("reuse cached outputs"); - assert_eq!(outputs, cached); - assert!(objects.local_outputs_cache.is_empty()); -} - -#[test] -fn persist_vcir_non_repository_evidence_stores_current_ca_cert_only() { - let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests(); - let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); - let objects = crate::validation::objects::process_publication_point_snapshot_for_issuer( - &pack, - &Policy::default(), - issuer_ca_der.as_slice(), - Some( - "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - issuer_ca.tbs.extensions.ip_resources.as_ref(), - issuer_ca.tbs.extensions.as_resources.as_ref(), - validation_time, - None, - ); - assert!( - !objects.local_outputs_cache.is_empty(), - "expected local outputs from signed objects" - ); - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(issuer_ca_der.clone()), - ca_certificate_rsync_uri: Some( - "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(), - ), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: pack.publication_point_rsync_uri.clone(), - manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), - rrdp_notification_uri: None, - }; - persist_vcir_non_repository_evidence(&store, &ca).expect("persist embedded evidence"); - - let issuer_hash = sha256_hex(&issuer_ca_der); - let issuer_entry = store - .get_raw_by_hash_entry(&issuer_hash) - .expect("load issuer raw entry") - .expect("issuer raw entry present"); - assert!( - issuer_entry - .origin_uris - .iter() - .any(|uri| uri.ends_with("BfycW4hQb3wNP4YsiJW-1n6fjro.cer")) - ); - let first_output = objects - .local_outputs_cache - .first() - .expect("first local output"); - assert!( - store - .get_raw_by_hash_entry(&first_output.source_ee_cert_hash_hex()) - .expect("load source ee raw") - .is_none() - ); -} - -#[test] -fn build_router_key_local_outputs_encodes_router_key_payloads() { - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(Vec::new()), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - let outputs = build_router_key_local_outputs( - &ca, - &[RouterKeyPayload { - as_id: 64496, - ski: vec![0x11; 20], - spki_der: vec![0x30, 0x00], - source_object_uri: "rsync://example.test/repo/issuer/router.cer".to_string(), - source_object_hash: "11".repeat(32), - source_ee_cert_hash: "11".repeat(32), - item_effective_until: PackTime { - rfc3339_utc: "2026-12-31T00:00:00Z".to_string(), - }, - }], - ); - assert_eq!(outputs.len(), 1); - assert_eq!(outputs[0].output_type, VcirOutputType::RouterKey); - assert_eq!( - outputs[0].source_object_type, - VcirSourceObjectType::RouterKey - ); - assert!(outputs[0].payload_json().contains("spki_der_base64")); -} - -#[test] -fn build_vcir_local_outputs_falls_back_to_decoding_accepted_objects_when_cache_is_empty() { - let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests(); - let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); - let objects = crate::validation::objects::process_publication_point_snapshot_for_issuer( - &pack, - &Policy::default(), - issuer_ca_der.as_slice(), - Some( - "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - issuer_ca.tbs.extensions.ip_resources.as_ref(), - issuer_ca.tbs.extensions.as_resources.as_ref(), - validation_time, - None, - ); - let mut objects_without_cache = objects.clone(); - objects_without_cache.local_outputs_cache.clear(); - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), - ca_certificate_rsync_uri: Some( - "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(), - ), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: pack.publication_point_rsync_uri.clone(), - manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), - rrdp_notification_uri: None, - }; - let local_outputs = build_vcir_local_outputs(&ca, &pack, &objects_without_cache) - .expect("rebuild vcir local outputs"); - assert!(!local_outputs.is_empty()); - assert_eq!(local_outputs.len(), objects.vrps.len()); - assert!( - local_outputs - .iter() - .all(|output| output.output_type == VcirOutputType::Vrp) - ); -} - -#[test] -fn finalize_fresh_publication_point_releases_local_outputs_cache_after_persist() { - let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests(); - let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); - let mut objects = crate::validation::objects::process_publication_point_snapshot_for_issuer( - &pack, - &Policy::default(), - issuer_ca_der.as_slice(), - Some( - "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - issuer_ca.tbs.extensions.ip_resources.as_ref(), - issuer_ca.tbs.extensions.as_resources.as_ref(), - validation_time, - None, - ); - assert!( - !objects.local_outputs_cache.is_empty(), - "expected local outputs from signed objects" - ); - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy::default(); - let runner = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &FailingRsyncFetcher, - validation_time, - timing: None, - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: false, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - }; - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(issuer_ca_der.clone()), - ca_certificate_rsync_uri: Some( - "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(), - ), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: pack.publication_point_rsync_uri.clone(), - manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), - rrdp_notification_uri: None, - }; - let fresh_point = FreshValidatedPublicationPoint { - manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), - manifest_number_be: pack.manifest_number_be.clone(), - this_update: pack.this_update.clone(), - next_update: pack.next_update.clone(), - verified_at: pack.verified_at.clone(), - manifest_bytes: pack.manifest_bytes.clone(), - files: pack.files.clone(), - }; - - objects.local_outputs_cache.shrink_to_fit(); - let original_cache_capacity = objects.local_outputs_cache.capacity(); - let finalized = runner - .finalize_fresh_publication_point_from_reducer( - &ca, - &fresh_point, - Vec::new(), - objects, - Vec::new(), - Vec::new(), - None, - None, - 0, - None, - ) - .expect("finalize fresh publication point"); - - assert!( - finalized.result.objects.local_outputs_cache.is_empty(), - "local outputs cache should be released after VCIR persistence" - ); - assert_eq!( - finalized.result.objects.local_outputs_cache.capacity(), - 0, - "released cache should not keep its backing allocation" - ); - assert!(original_cache_capacity > 0); - - let persisted = store - .get_vcir(&pack.manifest_rsync_uri) - .expect("load persisted vcir") - .expect("persisted vcir"); - assert!( - !persisted.local_outputs.is_empty(), - "VCIR should still persist local outputs before cache release" - ); -} - -#[test] -fn persist_vcir_for_fresh_result_stores_vcir_and_replay_meta_for_real_snapshot() { - let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests(); - let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); - let objects = crate::validation::objects::process_publication_point_snapshot_for_issuer( - &pack, - &Policy::default(), - issuer_ca_der.as_slice(), - Some( - "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - issuer_ca.tbs.extensions.ip_resources.as_ref(), - issuer_ca.tbs.extensions.as_resources.as_ref(), - validation_time, - None, - ); - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(issuer_ca_der.clone()), - ca_certificate_rsync_uri: Some( - "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(), - ), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: pack.publication_point_rsync_uri.clone(), - manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), - rrdp_notification_uri: None, - }; - - let mut objects = objects; - persist_vcir_for_fresh_result_with_timing( - &store, - &Policy::default(), - &ca, - &pack, - &mut objects, - &[], - &[], - &[], - validation_time, - false, - ) - .map(|_timing| ()) - .expect("persist vcir for fresh result"); - - let vcir = store - .get_vcir(&pack.manifest_rsync_uri) - .expect("get vcir") - .expect("vcir exists"); - assert_eq!(vcir.manifest_rsync_uri, pack.manifest_rsync_uri); - assert_eq!(vcir.summary.local_vrp_count as usize, objects.vrps.len()); - assert_eq!( - vcir.ccr_manifest_projection.manifest_rsync_uri, - pack.manifest_rsync_uri - ); - assert_eq!( - vcir.ccr_manifest_projection.manifest_number_be, - pack.manifest_number_be - ); - assert_eq!( - vcir.ccr_manifest_projection.manifest_this_update, - pack.this_update - ); - assert_eq!( - vcir.ccr_manifest_projection.manifest_size, - pack.manifest_bytes.len() as u64 - ); - assert!(vcir.local_outputs.first().is_some(), "local outputs stored"); - let replay_meta = store - .get_manifest_replay_meta(&pack.manifest_rsync_uri) - .expect("get replay meta") - .expect("replay meta exists"); - assert_eq!(replay_meta.manifest_rsync_uri, pack.manifest_rsync_uri); - assert_eq!( - replay_meta.manifest_sha256, - sha2::Sha256::digest(&pack.manifest_bytes).to_vec() - ); -} - -#[test] -fn build_vcir_ccr_manifest_projection_from_fresh_real_snapshot_matches_manifest_contents() { - let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests(); - let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), - ca_certificate_rsync_uri: Some( - "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(), - ), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: pack.publication_point_rsync_uri.clone(), - manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), - rrdp_notification_uri: None, - }; - let child_discovery = - discover_children_from_fresh_snapshot_with_audit(&ca, &pack, validation_time, None) - .expect("discover children"); - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let child_entries = - build_vcir_child_entries(&store, &child_discovery.children, validation_time) - .expect("build child entries"); - - let projection = build_vcir_ccr_manifest_projection_from_fresh(&ca, &pack, &child_entries) - .expect("build ccr manifest projection"); - let manifest = ManifestObject::decode_der(&pack.manifest_bytes).expect("decode manifest"); - let expected_locations = match manifest.signed_object.signed_data.certificates[0] - .resource_cert - .tbs - .extensions - .subject_info_access - .as_ref() - .expect("manifest sia") - { - SubjectInfoAccess::Ee(ee_sia) => vec![ - crate::ccr::manifest_location::select_manifest_signed_object_location( - &pack.manifest_rsync_uri, - &ee_sia.access_descriptions, - ) - .expect("select manifest signedObject"), - ], - SubjectInfoAccess::Ca(_) => panic!("manifest ee SIA should not be CA variant"), - }; - - assert_eq!(projection.manifest_rsync_uri, pack.manifest_rsync_uri); - assert_eq!( - projection.manifest_sha256, - sha2::Sha256::digest(&pack.manifest_bytes).to_vec() - ); - assert_eq!(projection.manifest_size, pack.manifest_bytes.len() as u64); - assert_eq!( - projection.manifest_ee_aki, - manifest.signed_object.signed_data.certificates[0] - .resource_cert - .tbs - .extensions - .authority_key_identifier - .clone() - .expect("manifest aki") - ); - assert_eq!( - projection.manifest_number_be, - manifest.manifest.manifest_number.bytes_be - ); - assert_eq!(projection.manifest_this_update, pack.this_update); - assert_eq!(projection.manifest_sia_locations_der, expected_locations); - let expected_subordinate_skis = child_entries - .iter() - .map(|child| hex::decode(&child.child_ski).expect("decode child ski")) - .collect::>(); - assert_eq!(projection.subordinate_skis, expected_subordinate_skis); -} - -#[test] -fn build_vcir_child_entries_uses_projection_without_repo_bytes() { - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let validation_time = time::OffsetDateTime::parse( - "2026-06-26T00:00:00Z", - &time::format_description::well_known::Rfc3339, - ) - .expect("parse time"); - let child = DiscoveredChildCaInstance { - handle: CaInstanceHandle { - depth: 1, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: Some("rsync://example.test/repo/root.mft".to_string()), - ca_certificate: CaCertificateRef::repo_bytes("aa".repeat(32)), - ca_certificate_rsync_uri: Some("rsync://example.test/repo/child.cer".to_string()), - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/child/".to_string(), - manifest_rsync_uri: "rsync://example.test/repo/child/child.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/child/".to_string(), - rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), - }, - discovered_from: crate::audit::DiscoveredFrom { - parent_manifest_rsync_uri: "rsync://example.test/repo/root.mft".to_string(), - child_ca_certificate_rsync_uri: "rsync://example.test/repo/child.cer".to_string(), - child_ca_certificate_sha256_hex: "aa".repeat(32), - }, - child_entry_projection: Some(DiscoveredChildEntryProjection { - child_ski: "11".repeat(20), - }), - }; - - let entries = build_vcir_child_entries(&store, &[child], validation_time) - .expect("projection should avoid repo-bytes load"); - assert_eq!(entries.len(), 1); - assert_eq!(entries[0].child_ski, "11".repeat(20)); -} - -#[test] -fn build_vcir_related_artifacts_classifies_snapshot_files_and_audit_statuses() { - let manifest_bytes = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft", - ), - ) - .expect("read manifest fixture"); - let crl_bytes = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.crl", - ), - ) - .expect("read crl fixture"); - let pack = PublicationPointSnapshot { - format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_number_be: vec![1], - this_update: PackTime::from_utc_offset_datetime(time::OffsetDateTime::now_utc()), - next_update: PackTime::from_utc_offset_datetime( - time::OffsetDateTime::now_utc() + time::Duration::hours(1), - ), - verified_at: PackTime::from_utc_offset_datetime(time::OffsetDateTime::now_utc()), - manifest_bytes, - files: vec![ - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/issuer.crl", - crl_bytes, - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/child.cer", - vec![1u8, 2], - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/a.roa", - vec![3u8, 4], - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/a.asa", - vec![5u8, 6], - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/a.gbr", - vec![7u8, 8], - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/extra.bin", - vec![9u8], - ), - ], - }; - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(vec![0x11, 0x22]), - ca_certificate_rsync_uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()), - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: pack.publication_point_rsync_uri.clone(), - manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), - rrdp_notification_uri: None, - }; - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let objects = crate::validation::objects::ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings: Vec::new(), - stats: crate::validation::objects::ObjectsStats::default(), - audit: vec![ - ObjectAuditEntry { - rsync_uri: "rsync://example.test/repo/issuer/a.roa".to_string(), - sha256_hex: sha256_hex_from_32(&pack.files[2].sha256), - kind: AuditObjectKind::Roa, - result: AuditObjectResult::Error, - detail: Some("bad roa".to_string()), - }, - ObjectAuditEntry { - rsync_uri: "rsync://example.test/repo/issuer/a.asa".to_string(), - sha256_hex: sha256_hex_from_32(&pack.files[3].sha256), - kind: AuditObjectKind::Aspa, - result: AuditObjectResult::Skipped, - detail: Some("skipped aspa".to_string()), - }, - ], - roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), - roa_cache_object_meta: Vec::new(), - }; - let artifacts = build_vcir_related_artifacts( - &store, - &ca, - &pack, - "rsync://example.test/repo/issuer/issuer.crl", - &objects, - &[], - ); - assert!( - artifacts - .iter() - .any(|artifact| artifact.artifact_role == VcirArtifactRole::Manifest) - ); - assert!( - artifacts - .iter() - .any(|artifact| artifact.artifact_role == VcirArtifactRole::TrustAnchorCert) - ); - assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() - == Some("rsync://example.test/repo/issuer/issuer.crl") - && artifact.artifact_role == VcirArtifactRole::CurrentCrl)); - assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() - == Some("rsync://example.test/repo/issuer/child.cer") - && artifact.artifact_role == VcirArtifactRole::ChildCaCert)); - assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() - == Some("rsync://example.test/repo/issuer/a.roa") - && artifact.validation_status == VcirArtifactValidationStatus::Rejected - && artifact.reject_reason.as_deref() == Some("bad roa"))); - assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() - == Some("rsync://example.test/repo/issuer/a.asa") - && artifact.validation_status == VcirArtifactValidationStatus::WarningOnly - && artifact.reject_reason.is_none())); - assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() - == Some("rsync://example.test/repo/issuer/a.gbr") - && artifact.artifact_kind == VcirArtifactKind::Gbr)); - assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() - == Some("rsync://example.test/repo/issuer/extra.bin") - && artifact.artifact_kind == VcirArtifactKind::Other)); - assert!( - !artifacts - .iter() - .any(|artifact| artifact.uri.is_none() - && artifact.sha256 == sha256_hex(b"embedded-ee")), - "embedded EE cert artifacts should no longer be persisted separately" - ); -} - -#[test] -fn select_issuer_crl_from_snapshot_reports_missing_crldp_for_self_signed_cert() { - let ta_der = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/ta/apnic-ta.cer"), - ) - .expect("read TA fixture"); - - let pack = dummy_pack_with_files(vec![]); - let err = select_issuer_crl_from_snapshot(&ta_der, &pack).unwrap_err(); - assert!(err.contains("CRLDistributionPoints missing"), "{err}"); -} - -#[test] -fn select_issuer_crl_from_snapshot_finds_matching_crl() { - // Use real fixtures to ensure child cert has CRLDP rsync URI and CRL exists. - let child_cert_der = - std::fs::read(std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/R-lVU1XGsAeqzV1Fv0HjOD6ZFkE.cer", - )) - .expect("read child cert fixture"); - let crl_der = - std::fs::read(std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.crl", - )) - .expect("read crl fixture"); - - let pack = dummy_pack_with_files(vec![PackFile::from_bytes_compute_sha256( - "rsync://ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.crl", - crl_der.clone(), - )]); - - let (uri, found) = - select_issuer_crl_from_snapshot(child_cert_der.as_slice(), &pack).expect("find crl"); - assert_eq!( - uri, - "rsync://ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.crl" - ); - assert_eq!(found, crl_der.as_slice()); -} - -#[test] -fn discover_children_from_fresh_pack_discovers_child_ca() { - let g = generate_chain_and_crl(); - - let pack = dummy_pack_with_files(vec![ - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/issuer.crl", - g.issuer_crl_der.clone(), - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/child.cer", - g.child_ca_der.clone(), - ), - ]); - - let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); - let issuer = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), - ca_certificate_rsync_uri: None, - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let now = time::OffsetDateTime::now_utc(); - let children = discover_children_from_fresh_snapshot_with_audit(&issuer, &pack, now, None) - .expect("discover children") - .children; - assert_eq!(children.len(), 1); - assert_eq!( - children[0].discovered_from.parent_manifest_rsync_uri, - issuer.manifest_rsync_uri - ); - assert_eq!( - children[0].discovered_from.child_ca_certificate_rsync_uri, - "rsync://example.test/repo/issuer/child.cer" - ); - assert_eq!( - children[0].handle.rsync_base_uri, - "rsync://example.test/repo/child/".to_string() - ); - assert_eq!( - children[0].handle.manifest_rsync_uri, - "rsync://example.test/repo/child/child.mft".to_string() - ); - assert_eq!( - children[0].handle.publication_point_rsync_uri, - "rsync://example.test/repo/child/".to_string() - ); - assert_eq!( - children[0].handle.rrdp_notification_uri.as_deref(), - Some("https://example.test/notification.xml") - ); -} - -#[test] -fn discover_children_child_certificate_cache_reuses_successful_child_ca() { - let g = generate_chain_and_crl(); - let pack = dummy_pack_with_files(vec![ - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/issuer.crl", - g.issuer_crl_der.clone(), - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/child.cer", - g.child_ca_der.clone(), - ), - ]); - let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); - let issuer = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), - ca_certificate_rsync_uri: None, - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy::default(); - let cache_context = ChildCertificateValidationCacheContext { - store: &store, - issuer_ca_sha256: issuer.ca_certificate_sha256_32().unwrap(), - ca_validation_context_digest: ca_validation_context_digest_for_ca(&issuer), - policy_fingerprint: publication_point_cache_policy_fingerprint(&policy), - }; - let validation_time = time::OffsetDateTime::now_utc(); - - let first = discover_children_from_fresh_snapshot_with_audit_cached( - &issuer, - &pack, - validation_time, - None, - Some(cache_context), - ) - .expect("first discovery writes cache"); - assert_eq!(first.children.len(), 1); - - let child_file = pack - .files - .iter() - .find(|file| file.rsync_uri.ends_with("child.cer")) - .expect("child file"); - let cache_key = child_certificate_cache_key_sha256_hex( - &child_file.rsync_uri, - &child_file.sha256, - &cache_context.issuer_ca_sha256, - &cache_context.ca_validation_context_digest, - &cache_context.policy_fingerprint, - ); - assert!( - store - .get_child_certificate_cache_projection(&cache_key) - .expect("get projection") - .is_some() - ); - - let timing = TimingHandle::new(crate::analysis::timing::TimingMeta { - recorded_at_utc_rfc3339: "2026-06-24T00:00:00Z".to_string(), - validation_time_utc_rfc3339: "2026-06-24T00:00:00Z".to_string(), - tal_url: None, - db_path: None, - }); - let second = discover_children_from_fresh_snapshot_with_audit_cached( - &issuer, - &pack, - validation_time, - Some(&timing), - Some(cache_context), - ) - .expect("second discovery reuses cache"); - assert_eq!(second.children.len(), 1); - assert_eq!( - second.children[0].handle.manifest_rsync_uri, - first.children[0].handle.manifest_rsync_uri - ); - assert!(second.audits.iter().any(|audit| { - audit - .detail - .as_deref() - .unwrap_or("") - .contains("child certificate validation cache") - })); - let counts = timing.counts_snapshot(); - assert_eq!( - counts.get("child_certificate_cache_hit_ca").copied(), - Some(1) - ); - assert_eq!( - counts - .get("child_certificate_cache_batch_lookup_publication_points") - .copied(), - Some(1) - ); - assert_eq!( - counts - .get("child_certificate_cache_batch_lookup_entries") - .copied(), - Some(1) - ); - assert_eq!( - counts - .get("child_certificate_der_load_fresh_count") - .copied(), - Some(0) - ); -} - -#[test] -fn discover_children_child_certificate_cache_rechecks_changed_valid_crl() { - let g = generate_chain_and_crl(); - let pack = dummy_pack_with_files(vec![ - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/issuer.crl", - g.issuer_crl_der.clone(), - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/child.cer", - g.child_ca_der.clone(), - ), - ]); - let mut changed_pack = pack.clone(); - changed_pack.files[0] = PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/issuer.crl", - g.issuer_crl_der_next.clone(), - ); - let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); - let issuer = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), - ca_certificate_rsync_uri: None, - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy::default(); - let cache_context = ChildCertificateValidationCacheContext { - store: &store, - issuer_ca_sha256: issuer.ca_certificate_sha256_32().unwrap(), - ca_validation_context_digest: ca_validation_context_digest_for_ca(&issuer), - policy_fingerprint: publication_point_cache_policy_fingerprint(&policy), - }; - let validation_time = time::OffsetDateTime::now_utc(); - discover_children_from_fresh_snapshot_with_audit_cached( - &issuer, - &pack, - validation_time, - None, - Some(cache_context), - ) - .expect("populate cache"); - let child_file = pack - .files - .iter() - .find(|file| file.rsync_uri.ends_with("child.cer")) - .expect("child file"); - let cache_key = child_certificate_cache_key_sha256_hex( - &child_file.rsync_uri, - &child_file.sha256, - &cache_context.issuer_ca_sha256, - &cache_context.ca_validation_context_digest, - &cache_context.policy_fingerprint, - ); - let projection = store - .get_child_certificate_cache_projection(&cache_key) - .expect("get child certificate cache projection") - .expect("child certificate cache projection present"); - let projected_until = - parse_snapshot_time_value(&projection.effective_until).expect("parse projected until"); - let initial_crl = crate::data_model::crl::RpkixCrl::decode_der(&g.issuer_crl_der) - .expect("decode initial crl"); - assert!( - projected_until > initial_crl.next_update.utc, - "child certificate cache projection must not be hard-capped by the issuing CRL nextUpdate" - ); - - let timing = TimingHandle::new(crate::analysis::timing::TimingMeta { - recorded_at_utc_rfc3339: "2026-06-24T00:00:00Z".to_string(), - validation_time_utc_rfc3339: "2026-06-24T00:00:00Z".to_string(), - tal_url: None, - db_path: None, - }); - let out = discover_children_from_fresh_snapshot_with_audit_cached( - &issuer, - &changed_pack, - validation_time, - Some(&timing), - Some(cache_context), - ) - .expect("changed valid CRL should recheck revocation and reuse cache"); - assert_eq!(out.children.len(), 1); - assert!( - out.audits - .iter() - .any(|audit| audit.detail.as_deref().unwrap_or("").contains("cache")) - ); - let counts = timing.counts_snapshot(); - assert_eq!( - counts - .get("child_certificate_cache_crl_recheck_hit") - .copied(), - Some(1) - ); - assert_eq!( - counts.get("child_certificate_cache_hit_ca").copied(), - Some(1) - ); -} - -#[test] -fn discover_children_child_certificate_cache_misses_when_current_crl_invalid() { - let g = generate_chain_and_crl(); - let pack = dummy_pack_with_files(vec![ - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/issuer.crl", - g.issuer_crl_der.clone(), - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/child.cer", - g.child_ca_der.clone(), - ), - ]); - let mut changed_pack = pack.clone(); - changed_pack.files[0] = PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/issuer.crl", - b"not-a-valid-crl".to_vec(), - ); - let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); - let issuer = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), - ca_certificate_rsync_uri: None, - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy::default(); - let cache_context = ChildCertificateValidationCacheContext { - store: &store, - issuer_ca_sha256: issuer.ca_certificate_sha256_32().unwrap(), - ca_validation_context_digest: ca_validation_context_digest_for_ca(&issuer), - policy_fingerprint: publication_point_cache_policy_fingerprint(&policy), - }; - let validation_time = time::OffsetDateTime::now_utc(); - discover_children_from_fresh_snapshot_with_audit_cached( - &issuer, - &pack, - validation_time, - None, - Some(cache_context), - ) - .expect("populate cache"); - - let timing = TimingHandle::new(crate::analysis::timing::TimingMeta { - recorded_at_utc_rfc3339: "2026-06-24T00:00:00Z".to_string(), - validation_time_utc_rfc3339: "2026-06-24T00:00:00Z".to_string(), - tal_url: None, - db_path: None, - }); - let out = discover_children_from_fresh_snapshot_with_audit_cached( - &issuer, - &changed_pack, - validation_time, - Some(&timing), - Some(cache_context), - ) - .expect("invalid CRL should miss cache and continue with audit error"); - assert!(out.children.is_empty()); - assert!( - out.audits - .iter() - .all(|audit| !audit.detail.as_deref().unwrap_or("").contains("cache")) - ); - let counts = timing.counts_snapshot(); - assert_eq!( - counts - .get("child_certificate_cache_miss_crl_invalid") - .copied(), - Some(1) - ); -} - -#[test] -fn discover_children_child_certificate_cache_misses_when_unchanged_crl_expired() { - let g = generate_chain_and_crl(); - let pack = dummy_pack_with_files(vec![ - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/issuer.crl", - g.issuer_crl_der.clone(), - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/child.cer", - g.child_ca_der.clone(), - ), - ]); - let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); - let issuer = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), - ca_certificate_rsync_uri: None, - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy::default(); - let cache_context = ChildCertificateValidationCacheContext { - store: &store, - issuer_ca_sha256: issuer.ca_certificate_sha256_32().unwrap(), - ca_validation_context_digest: ca_validation_context_digest_for_ca(&issuer), - policy_fingerprint: publication_point_cache_policy_fingerprint(&policy), - }; - let validation_time = time::OffsetDateTime::now_utc(); - discover_children_from_fresh_snapshot_with_audit_cached( - &issuer, - &pack, - validation_time, - None, - Some(cache_context), - ) - .expect("populate cache"); - - let timing = TimingHandle::new(crate::analysis::timing::TimingMeta { - recorded_at_utc_rfc3339: "2026-06-24T00:00:00Z".to_string(), - validation_time_utc_rfc3339: "2026-06-24T00:00:00Z".to_string(), - tal_url: None, - db_path: None, - }); - let out = discover_children_from_fresh_snapshot_with_audit_cached( - &issuer, - &pack, - validation_time + time::Duration::days(2), - Some(&timing), - Some(cache_context), - ) - .expect("expired unchanged CRL should miss cache and continue with audit error"); - assert!(out.children.is_empty()); - assert!( - out.audits - .iter() - .all(|audit| !audit.detail.as_deref().unwrap_or("").contains("cache")) - ); - let counts = timing.counts_snapshot(); - assert_eq!( - counts - .get("child_certificate_cache_miss_crl_expired") - .copied(), - Some(1) - ); -} - -#[test] -fn discover_children_with_audit_records_missing_crl_for_child_certificate() { - let now = time::OffsetDateTime::now_utc(); - - let child_ca_der = - std::fs::read(std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/R-lVU1XGsAeqzV1Fv0HjOD6ZFkE.cer", - )) - .expect("read child ca fixture"); - - // Pack contains the child CA cert but does not contain the CRL referenced by the child - // certificate CRLDistributionPoints extension. - let pack = dummy_pack_with_files(vec![PackFile::from_bytes_compute_sha256( - "rsync://ca.rg.net/rpki/RGnet-OU/R-lVU1XGsAeqzV1Fv0HjOD6ZFkE.cer", - child_ca_der, - )]); - - let issuer = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(vec![1]), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), - rrdp_notification_uri: None, - }; - - let out = discover_children_from_fresh_snapshot_with_audit(&issuer, &pack, now, None) - .expect("discovery should succeed with audit error"); - assert_eq!(out.children.len(), 0); - assert_eq!(out.audits.len(), 1); - assert_eq!( - out.audits[0].rsync_uri, - "rsync://ca.rg.net/rpki/RGnet-OU/R-lVU1XGsAeqzV1Fv0HjOD6ZFkE.cer" - ); - assert_eq!(out.audits[0].result, AuditObjectResult::Error); - assert!( - out.audits[0] - .detail - .as_deref() - .unwrap_or("") - .contains("cannot select issuer CRL"), - "expected deterministic CRL selection failure to be recorded" - ); -} - -#[test] -fn runner_offline_rsync_fixture_produces_pack_and_warnings() { - let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); - assert!(fixture_dir.is_dir(), "fixture directory must exist"); - - let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); - let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; - let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); - - // Pick a validation_time inside the fixture manifest's validity window to keep this - // test stable across wall-clock time. - let fixture_manifest_bytes = - std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); - let fixture_manifest = - crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) - .expect("decode manifest fixture"); - let validation_time = { - let this_update = fixture_manifest.manifest.this_update; - let next_update = fixture_manifest.manifest.next_update; - let candidate = this_update + time::Duration::seconds(60); - if candidate < next_update { - candidate - } else { - this_update - } - }; - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy { - sync_preference: crate::policy::SyncPreference::RsyncOnly, - ..Policy::default() - }; - - let runner = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), - validation_time, - timing: None, - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: false, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - }; - - // For this fixture-driven smoke, we provide the correct issuer CA certificate (the CA for - // this publication point) so ROA EE certificate paths can validate. - let issuer_ca_der = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - ) - .expect("read issuer ca fixture"); - let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); - - let handle = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), - ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: rsync_base_uri.clone(), - manifest_rsync_uri: manifest_rsync_uri.clone(), - publication_point_rsync_uri: rsync_base_uri.clone(), - rrdp_notification_uri: None, - }; - - let out = runner - .run_publication_point(&handle) - .expect("run publication point"); - assert_eq!(out.source, PublicationPointSource::Fresh); - let pack = out.snapshot.expect("fresh run pack"); - assert_eq!(pack.manifest_rsync_uri, manifest_rsync_uri); - assert!(pack.files.len() > 1); - assert!( - out.objects.vrps.len() > 1, - "expected to extract VRPs from ROAs" - ); - - let vcir = store - .get_vcir(&manifest_rsync_uri) - .expect("get vcir") - .expect("vcir exists after fresh run"); - assert_eq!(vcir.manifest_rsync_uri, manifest_rsync_uri); - assert_eq!(vcir.tal_id, "test-tal"); - assert!( - vcir.local_outputs - .iter() - .any(|output| output.output_type == crate::storage::VcirOutputType::Vrp), - "expected VCIR local_outputs to contain VRP entries" - ); - let first_vrp = vcir - .local_outputs - .iter() - .find(|output| output.output_type == crate::storage::VcirOutputType::Vrp) - .expect("first VCIR VRP output"); - assert!(!first_vrp.rule_hash_hex().is_empty()); - assert!(!first_vrp.output_id().is_empty()); - let replay_meta = store - .get_manifest_replay_meta(&manifest_rsync_uri) - .expect("get replay meta") - .expect("replay meta exists"); - assert_eq!(replay_meta.manifest_rsync_uri, manifest_rsync_uri); -} - -#[test] -fn runner_roa_validation_cache_reuses_vcir_outputs_on_second_fixture_run() { - let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); - assert!(fixture_dir.is_dir(), "fixture directory must exist"); - - let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); - let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; - let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); - - let fixture_manifest_bytes = - std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); - let fixture_manifest = - crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) - .expect("decode manifest fixture"); - let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy { - sync_preference: crate::policy::SyncPreference::RsyncOnly, - ..Policy::default() - }; - - let issuer_ca_der = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - ) - .expect("read issuer ca fixture"); - let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); - - let handle = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), - ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: rsync_base_uri.clone(), - manifest_rsync_uri: manifest_rsync_uri.clone(), - publication_point_rsync_uri: rsync_base_uri, - rrdp_notification_uri: None, - }; - - let first_runner = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), - validation_time, - timing: None, - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: false, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - }; - let first = first_runner - .run_publication_point(&handle) - .expect("first fresh run"); - assert!(first.objects.vrps.len() > 1); - assert_eq!(first.objects.roa_cache_stats.hit_roas, 0); - - let second_runner = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), - validation_time, - timing: None, - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: false, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: true, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - }; - let second = second_runner - .run_publication_point(&handle) - .expect("second cache-enabled run"); - - assert_eq!(second.objects.vrps, first.objects.vrps); - assert_eq!(second.objects.roa_cache_stats.enabled_publication_points, 1); - assert_eq!( - second.objects.roa_cache_stats.vcir_hit_publication_points, - 1 - ); - assert_eq!( - second.objects.roa_cache_stats.vcir_miss_publication_points, - 0 - ); - assert!(second.objects.roa_cache_stats.hit_roas > 1); - assert_eq!(second.objects.roa_cache_stats.miss_roas, 0); - assert_eq!(second.objects.roa_cache_stats.blocked_roas, 0); - assert_eq!(second.objects.roa_cache_stats.fresh_roas, 0); -} - -#[test] -fn runner_publication_point_cache_observe_and_reuse_path() { - let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); - assert!(fixture_dir.is_dir(), "fixture directory must exist"); - - let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); - let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; - let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); - let fixture_manifest_bytes = - std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); - let fixture_manifest = - crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) - .expect("decode manifest fixture"); - let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy { - sync_preference: crate::policy::SyncPreference::RsyncOnly, - ..Policy::default() - }; - let issuer_ca_der = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - ) - .expect("read issuer ca fixture"); - let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); - let handle = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), - ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: rsync_base_uri.clone(), - manifest_rsync_uri: manifest_rsync_uri.clone(), - publication_point_rsync_uri: rsync_base_uri, - rrdp_notification_uri: None, - }; - - let first_runner = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), - validation_time, - timing: None, - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: false, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: true, - enable_publication_point_validation_cache: false, - }; - let first = first_runner - .run_publication_point(&handle) - .expect("first fresh run"); - assert_eq!(first.source, PublicationPointSource::Fresh); - assert!( - store - .get_publication_point_cache_projection(&manifest_rsync_uri) - .expect("load publication-point projection") - .is_some() - ); - - let timing = crate::analysis::timing::TimingHandle::new(crate::analysis::timing::TimingMeta { - recorded_at_utc_rfc3339: "2026-01-01T00:00:00Z".to_string(), - validation_time_utc_rfc3339: "2026-01-01T00:00:00Z".to_string(), - tal_url: None, - db_path: None, - }); - let observe_runner = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), - validation_time, - timing: Some(timing.clone()), - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: false, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: true, - enable_publication_point_validation_cache: false, - }; - let observed = observe_runner - .run_publication_point(&handle) - .expect("observe-only run"); - assert_eq!(observed.source, PublicationPointSource::Fresh); - assert_eq!(observed.objects.vrps, first.objects.vrps); - assert_eq!( - timing - .counts_snapshot() - .get("publication_point_cache_theoretical_hits") - .copied(), - Some(1) - ); - - let cache_runner = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), - validation_time, - timing: None, - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: false, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: true, - }; - let cached = cache_runner - .run_publication_point(&handle) - .expect("publication-point cache run"); - assert_eq!(cached.source, PublicationPointSource::PublicationPointCache); - assert_eq!(cached.objects.vrps, first.objects.vrps); - assert_eq!(cached.objects.aspas, first.objects.aspas); - assert_eq!( - cached.discovered_children.len(), - first.discovered_children.len() - ); - assert!(!cached.cir_cached_objects.is_empty()); -} - -fn seed_publication_point_cache_projection( - store: &RocksStore, - policy: &Policy, - ca: &CaInstanceHandle, - validation_time: time::OffsetDateTime, -) -> ValidatedCaInstanceResult { - let child_bytes = b"child-cert".to_vec(); - let child_hash = sha256_hex(&child_bytes); - let vcir = sample_vcir_for_projection(validation_time, &child_hash); - store - .put_blob_bytes_batch(&[ - (child_hash, child_bytes), - (sha256_hex(b"manifest-bytes"), b"manifest-bytes".to_vec()), - ]) - .expect("put cache bytes"); - store - .put_repository_view_entry(&RepositoryViewEntry { - rsync_uri: ca.manifest_rsync_uri.clone(), - current_hash: Some(sha256_hex(b"manifest-bytes")), - repository_source: Some(ca.publication_point_rsync_uri.clone()), - object_type: Some("mft".to_string()), - state: RepositoryViewState::Present, - }) - .expect("put manifest current view"); - let projection = PublicationPointCacheProjection::from_vcir_with_context( - &vcir, - ca.publication_point_rsync_uri.clone(), - ca.ca_certificate_rsync_uri.clone(), - ca.ca_certificate_sha256_32().unwrap(), - sha256_32(b"manifest-bytes"), - ta_context_digest_for_ca(ca), - ca_validation_context_digest_for_ca(ca), - publication_point_cache_policy_fingerprint(policy), - ) - .expect("build publication point projection"); - store - .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) - .expect("put publication point projection"); - vcir -} - -#[test] -fn publication_point_cache_future_notbefore_guard_detects_future_roa_notbefore_only() { - let uri = "rsync://example.test/repo/issuer/future.roa"; - let bytes = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"), - ) - .expect("read ROA fixture"); - let roa = RoaObject::decode_der(&bytes).expect("decode ROA fixture"); - let ee = &roa.signed_object.signed_data.certificates[0].resource_cert; - let file = PackFile::from_bytes_compute_sha256(uri, bytes); - let pack = dummy_pack_with_files(vec![file]); - let objects = crate::validation::objects::ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings: Vec::new(), - stats: Default::default(), - audit: vec![ObjectAuditEntry { - rsync_uri: uri.to_string(), - sha256_hex: sha256_hex_from_32(&pack.files[0].sha256), - kind: AuditObjectKind::Roa, - result: AuditObjectResult::Error, - detail: Some( - "EE certificate path validation failed: certificate not valid at validation_time" - .to_string(), - ), - }], - roa_cache_stats: Default::default(), - roa_cache_object_meta: Vec::new(), - }; - - assert!(publication_point_cache_has_future_not_before_risk( - &pack, - &objects, - &[], - ee.tbs.validity_not_before - time::Duration::seconds(1), - &Policy::default(), - )); - assert!(!publication_point_cache_has_future_not_before_risk( - &pack, - &objects, - &[], - ee.tbs.validity_not_after + time::Duration::seconds(1), - &Policy::default(), - )); -} - -#[test] -fn publication_point_cache_delete_action_removes_existing_projection() { - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy::default(); - let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::minutes(1); - let ca = publication_point_cache_fixture_ca(); - let vcir = seed_publication_point_cache_projection(&store, &policy, &ca, validation_time); - - assert!( - store - .get_publication_point_cache_projection_cached(&ca.manifest_rsync_uri) - .expect("load projection") - .is_some() - ); - - store - .replace_vcir_manifest_replay_meta_and_projection_action( - &vcir, - None, - PublicationPointCacheProjectionWriteAction::Delete { - manifest_rsync_uri: &vcir.manifest_rsync_uri, - }, - ) - .expect("delete projection"); - - assert!( - store - .get_publication_point_cache_projection_cached(&ca.manifest_rsync_uri) - .expect("load projection after delete") - .is_none() - ); -} - -fn publication_point_cache_fixture_ca() -> CaInstanceHandle { - CaInstanceHandle { - depth: 1, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(b"ca-cert".to_vec()), - ca_certificate_rsync_uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()), - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: Some("https://example.test/notification.xml".to_string()), - } -} - -#[test] -fn runner_publication_point_cache_reuses_projection_outputs_children_and_ccr() { - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy::default(); - let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::minutes(1); - let ca = publication_point_cache_fixture_ca(); - let vcir = seed_publication_point_cache_projection(&store, &policy, &ca, validation_time); - let timing = crate::analysis::timing::TimingHandle::new(crate::analysis::timing::TimingMeta { - recorded_at_utc_rfc3339: "2026-01-01T00:00:00Z".to_string(), - validation_time_utc_rfc3339: "2026-01-01T00:00:00Z".to_string(), - tal_url: None, - db_path: None, - }); - let runner = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &FailingRsyncFetcher, - validation_time, - timing: Some(timing.clone()), - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: false, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: Some(Mutex::new(CcrAccumulator::new(Vec::new()))), - persist_vcir: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: true, - }; - - let result = runner - .observe_or_reuse_publication_point_cache(&ca, Some("rrdp"), Some("delta"), 7, None, &[]) - .expect("cache result"); - - assert_eq!(result.source, PublicationPointSource::PublicationPointCache); - assert_eq!(result.objects.vrps.len(), 1); - assert_eq!(result.objects.aspas.len(), 1); - assert_eq!(result.objects.router_keys.len(), 1); - assert_eq!(result.discovered_children.len(), 1); - assert_eq!( - result.discovered_children[0].handle.manifest_rsync_uri, - vcir.child_entries[0].child_manifest_rsync_uri - ); - assert!(result.cir_fresh_objects.is_empty()); - assert!(!result.cir_cached_objects.is_empty()); - assert_eq!( - runner - .ccr_accumulator_snapshot() - .expect("ccr accumulator") - .manifest_count(), - 1 - ); - let counts = timing.counts_snapshot(); - assert_eq!( - counts.get("publication_point_cache_reuse_hits").copied(), - Some(1) - ); - assert_eq!( - counts - .get("publication_point_cache_outputs_reused") - .copied(), - Some(3) - ); - assert_eq!( - counts - .get("publication_point_cache_children_reused") - .copied(), - Some(1) - ); - assert_eq!( - counts - .get("publication_point_cache_related_objects_reused") - .copied(), - Some(vcir.related_artifacts.len() as u64) - ); - assert_eq!( - counts - .get("publication_point_cache_audit_objects_reused") - .copied(), - Some(result.cir_cached_objects.len() as u64) - ); - let timing_dir = tempfile::tempdir().expect("timing dir"); - let timing_path = timing_dir.path().join("timing.json"); - timing.write_json(&timing_path, 20).expect("write timing"); - let timing_json: serde_json::Value = - serde_json::from_slice(&std::fs::read(&timing_path).expect("read timing")) - .expect("parse timing"); - let phase_keys = timing_json["phases"] - .as_object() - .expect("phases") - .keys() - .map(|key| key.as_str()) - .collect::>(); - assert!(phase_keys.contains("publication_point_cache_lookup_hit_total")); - assert!(phase_keys.contains("publication_point_cache_reuse_build_total")); - assert!(phase_keys.contains("publication_point_cache_build_objects_total")); -} - -#[test] -fn publication_point_cache_restore_children_parallel_keeps_order_and_audit() { - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy::default(); - let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::minutes(1); - let ca = publication_point_cache_fixture_ca(); - seed_publication_point_cache_projection(&store, &policy, &ca, validation_time); - let mut projection = store - .get_publication_point_cache_projection(&ca.manifest_rsync_uri) - .expect("load projection") - .expect("projection"); - let template = projection.children[0].clone(); - let mut blobs = Vec::new(); - let mut children = Vec::new(); - for index in 0..300 { - let bytes = format!("child-cert-{index}").into_bytes(); - let child_hash = sha256_hex(&bytes); - let mut child = template.clone(); - child.child_cert_hash = child_hash.clone(); - child.child_cert_rsync_uri = format!("rsync://example.test/repo/issuer/child-{index}.cer"); - child.child_manifest_rsync_uri = - format!("rsync://example.test/repo/child-{index}/child.mft"); - child.child_publication_point_rsync_uri = - format!("rsync://example.test/repo/child-{index}/"); - child.child_rsync_base_uri = child.child_publication_point_rsync_uri.clone(); - blobs.push((child_hash, bytes)); - children.push(child); - } - projection.children = children; - store.put_blob_bytes_batch(&blobs).expect("put child blobs"); - let timing = crate::analysis::timing::TimingHandle::new(crate::analysis::timing::TimingMeta { - recorded_at_utc_rfc3339: "2026-01-01T00:00:00Z".to_string(), - validation_time_utc_rfc3339: "2026-01-01T00:00:00Z".to_string(), - tal_url: None, - db_path: None, - }); - let mut warnings = Vec::new(); - - let (restored_children, audits) = restore_children_from_publication_point_cache( - &store, - &ca, - &projection, - validation_time, - &mut warnings, - 4, - Some(&timing), - ); - - assert!(warnings.is_empty()); - assert_eq!(restored_children.len(), 300); - assert_eq!(audits.len(), 300); - assert_eq!( - restored_children[0].handle.ca_certificate_sha256_hex(), - Some(sha256_hex(b"child-cert-0").as_str()) - ); - assert_eq!( - restored_children[299] - .handle - .ca_certificate_der(&store) - .unwrap() - .as_ref(), - b"child-cert-299" - ); - assert_eq!( - restored_children[0].handle.manifest_rsync_uri, - "rsync://example.test/repo/child-0/child.mft" - ); - assert!( - audits - .iter() - .all(|audit| audit.result == AuditObjectResult::Ok) - ); - let counts = timing.counts_snapshot(); - assert_eq!( - counts - .get("publication_point_cache_restore_children_parallel_publication_points") - .copied(), - Some(1) - ); - assert_eq!( - counts - .get("publication_point_cache_restore_children_parallel_children") - .copied(), - Some(300) - ); - assert_eq!( - counts - .get("publication_point_cache_restore_children_workers_total") - .copied(), - Some(4) - ); -} - -#[test] -fn publication_point_cache_restore_children_does_not_require_child_der_bytes() { - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy::default(); - let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::minutes(1); - let ca = publication_point_cache_fixture_ca(); - seed_publication_point_cache_projection(&store, &policy, &ca, validation_time); - let mut projection = store - .get_publication_point_cache_projection(&ca.manifest_rsync_uri) - .expect("load projection") - .expect("projection"); - projection.children[0].child_cert_hash = "22".repeat(32); - let mut warnings = Vec::new(); - - let (restored_children, audits) = restore_children_from_publication_point_cache( - &store, - &ca, - &projection, - validation_time, - &mut warnings, - 1, - None, - ); - - assert!(warnings.is_empty()); - assert!(!restored_children.is_empty()); - assert_eq!(audits.len(), restored_children.len()); - assert_eq!( - restored_children[0].handle.ca_certificate_sha256_hex(), - Some(projection.children[0].child_cert_hash.as_str()) - ); - assert!( - restored_children[0] - .handle - .ca_certificate_der(&store) - .is_err(), - "lazy child handle should not require repo bytes until a fresh path asks for DER" - ); -} - -#[test] -fn runner_publication_point_cache_blocks_parent_policy_and_output_time_mismatch() { - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy::default(); - let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::minutes(1); - let ca = publication_point_cache_fixture_ca(); - seed_publication_point_cache_projection(&store, &policy, &ca, validation_time); - - let timing = crate::analysis::timing::TimingHandle::new(crate::analysis::timing::TimingMeta { - recorded_at_utc_rfc3339: "2026-01-01T00:00:00Z".to_string(), - validation_time_utc_rfc3339: "2026-01-01T00:00:00Z".to_string(), - tal_url: None, - db_path: None, - }); - let mut parent_changed_ca = ca.clone(); - parent_changed_ca.parent_manifest_rsync_uri = - Some("rsync://example.test/repo/other-parent.mft".to_string()); - let parent_changed_runner = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &FailingRsyncFetcher, - validation_time, - timing: Some(timing.clone()), - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: false, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: true, - }; - assert!( - parent_changed_runner - .observe_or_reuse_publication_point_cache( - &parent_changed_ca, - Some("rrdp"), - Some("delta"), - 7, - None, - &[] - ) - .is_none() - ); - assert_eq!( - timing - .counts_snapshot() - .get("publication_point_cache_miss_parent_context_mismatch") - .copied(), - Some(1) - ); - - let strict_policy = Policy { - strict: crate::policy::StrictPolicy::all(), - ..Policy::default() - }; - let policy_changed_runner = Rpkiv1PublicationPointRunner { - store: &store, - policy: &strict_policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &FailingRsyncFetcher, - validation_time, - timing: Some(timing.clone()), - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: false, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: true, - }; - assert!( - policy_changed_runner - .observe_or_reuse_publication_point_cache( - &ca, - Some("rrdp"), - Some("delta"), - 7, - None, - &[] - ) - .is_none() - ); - assert_eq!( - timing - .counts_snapshot() - .get("publication_point_cache_miss_policy_mismatch") - .copied(), - Some(1) - ); - - let output_expired_runner = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &FailingRsyncFetcher, - validation_time: validation_time + time::Duration::minutes(40), - timing: Some(timing.clone()), - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: false, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: true, - }; - assert!( - output_expired_runner - .observe_or_reuse_publication_point_cache( - &ca, - Some("rrdp"), - Some("delta"), - 7, - None, - &[] - ) - .is_none() - ); - assert_eq!( - timing - .counts_snapshot() - .get("publication_point_cache_miss_output_time_gate") - .copied(), - Some(1) - ); -} - -#[test] -fn runner_rsync_dedup_skips_second_sync_for_same_base() { - let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); - assert!(fixture_dir.is_dir(), "fixture directory must exist"); - - let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); - let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; - let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); - - let fixture_manifest_bytes = - std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); - let fixture_manifest = - crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) - .expect("decode manifest fixture"); - let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy { - sync_preference: crate::policy::SyncPreference::RsyncOnly, - ..Policy::default() - }; - - let issuer_ca_der = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - ) - .expect("read issuer ca fixture"); - let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); - - let handle = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), - ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: rsync_base_uri.clone(), - manifest_rsync_uri: manifest_rsync_uri.clone(), - publication_point_rsync_uri: rsync_base_uri.clone(), - rrdp_notification_uri: None, - }; - - struct CountingRsyncFetcher { - inner: LocalDirRsyncFetcher, - calls: Arc, - } - impl RsyncFetcher for CountingRsyncFetcher { - fn fetch_objects( - &self, - rsync_base_uri: &str, - ) -> Result)>, RsyncFetchError> { - self.calls.fetch_add(1, Ordering::SeqCst); - self.inner.fetch_objects(rsync_base_uri) - } - } - - let calls = Arc::new(AtomicUsize::new(0)); - let rsync = CountingRsyncFetcher { - inner: LocalDirRsyncFetcher::new(&fixture_dir), - calls: calls.clone(), - }; - - let runner = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &rsync, - validation_time, - timing: None, - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: true, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - }; - - let first = runner.run_publication_point(&handle).expect("first run ok"); - assert_eq!(first.source, PublicationPointSource::Fresh); - - let second = runner - .run_publication_point(&handle) - .expect("second run ok"); - assert_eq!(second.source, PublicationPointSource::Fresh); - - assert_eq!( - calls.load(Ordering::SeqCst), - 1, - "rsync should be called once" - ); -} - -#[test] -fn runner_rsync_dedup_skips_second_sync_for_same_module_scope() { - let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); - assert!(fixture_dir.is_dir(), "fixture directory must exist"); - - let first_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); - let second_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/sub/".to_string(); - let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; - let manifest_rsync_uri = format!("{first_base_uri}{manifest_file}"); - - let fixture_manifest_bytes = - std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); - let fixture_manifest = - crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) - .expect("decode manifest fixture"); - let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy { - sync_preference: crate::policy::SyncPreference::RsyncOnly, - ..Policy::default() - }; - - let issuer_ca_der = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - ) - .expect("read issuer ca fixture"); - let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); - - let handle = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), - ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: first_base_uri.clone(), - manifest_rsync_uri: manifest_rsync_uri.clone(), - publication_point_rsync_uri: first_base_uri.clone(), - rrdp_notification_uri: None, - }; - let second_handle = CaInstanceHandle { - rsync_base_uri: second_base_uri.clone(), - publication_point_rsync_uri: second_base_uri.clone(), - ..handle.clone() - }; - - struct ModuleScopeRsyncFetcher { - inner: LocalDirRsyncFetcher, - calls: Arc, - } - impl RsyncFetcher for ModuleScopeRsyncFetcher { - fn fetch_objects( - &self, - rsync_base_uri: &str, - ) -> Result)>, RsyncFetchError> { - self.calls.fetch_add(1, Ordering::SeqCst); - self.inner.fetch_objects(rsync_base_uri) - } - - fn dedup_key(&self, _rsync_base_uri: &str) -> String { - "rsync://rpki.cernet.net/repo/".to_string() - } - } - - let calls = Arc::new(AtomicUsize::new(0)); - let rsync = ModuleScopeRsyncFetcher { - inner: LocalDirRsyncFetcher::new(&fixture_dir), - calls: calls.clone(), - }; - - let runner = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &rsync, - validation_time, - timing: None, - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: true, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - }; - - let first = runner.run_publication_point(&handle).expect("first run ok"); - assert_eq!(first.source, PublicationPointSource::Fresh); - - let second = runner - .run_publication_point(&second_handle) - .expect("second run ok"); - assert!(matches!( - second.source, - PublicationPointSource::Fresh | PublicationPointSource::VcirCurrentInstance - )); - - assert_eq!( - calls.load(Ordering::SeqCst), - 1, - "module-scope dedup should skip second sync" - ); -} - -#[test] -fn runner_rsync_dedup_works_in_rsync_only_mode_even_when_rrdp_notify_exists() { - let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); - assert!(fixture_dir.is_dir(), "fixture directory must exist"); - - let first_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); - let second_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/sub/".to_string(); - let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; - let manifest_rsync_uri = format!("{first_base_uri}{manifest_file}"); - - let fixture_manifest_bytes = - std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); - let fixture_manifest = - crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) - .expect("decode manifest fixture"); - let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy { - sync_preference: crate::policy::SyncPreference::RsyncOnly, - ..Policy::default() - }; - - let issuer_ca_der = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - ) - .expect("read issuer ca fixture"); - let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); - - let handle = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), - ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: first_base_uri.clone(), - manifest_rsync_uri: manifest_rsync_uri.clone(), - publication_point_rsync_uri: first_base_uri.clone(), - rrdp_notification_uri: Some("https://rrdp.example.test/notification.xml".to_string()), - }; - let second_handle = CaInstanceHandle { - rsync_base_uri: second_base_uri.clone(), - publication_point_rsync_uri: second_base_uri.clone(), - ..handle.clone() - }; - - struct ModuleScopeRsyncFetcher { - inner: LocalDirRsyncFetcher, - calls: Arc, - } - impl RsyncFetcher for ModuleScopeRsyncFetcher { - fn fetch_objects( - &self, - rsync_base_uri: &str, - ) -> Result)>, RsyncFetchError> { - self.calls.fetch_add(1, Ordering::SeqCst); - self.inner.fetch_objects(rsync_base_uri) - } - - fn dedup_key(&self, _rsync_base_uri: &str) -> String { - "rsync://rpki.cernet.net/repo/".to_string() - } - } - - let calls = Arc::new(AtomicUsize::new(0)); - let rsync = ModuleScopeRsyncFetcher { - inner: LocalDirRsyncFetcher::new(&fixture_dir), - calls: calls.clone(), - }; - - let runner = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &rsync, - validation_time, - timing: None, - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: true, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: true, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - }; - - let first = runner.run_publication_point(&handle).expect("first run ok"); - assert_eq!(first.source, PublicationPointSource::Fresh); - - let second = runner - .run_publication_point(&second_handle) - .expect("second run ok"); - assert!(matches!( - second.source, - PublicationPointSource::Fresh | PublicationPointSource::VcirCurrentInstance - )); - - assert_eq!( - calls.load(Ordering::SeqCst), - 1, - "rsync-only mode must deduplicate by rsync scope even when RRDP notification is present" - ); -} - -#[test] -fn runner_when_repo_sync_fails_uses_current_instance_vcir_and_keeps_children_empty_for_fixture() { - let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); - assert!(fixture_dir.is_dir(), "fixture directory must exist"); - - let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); - let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; - let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); - - let fixture_manifest_bytes = - std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); - let fixture_manifest = - crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) - .expect("decode manifest fixture"); - let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy { - sync_preference: crate::policy::SyncPreference::RsyncOnly, - ..Policy::default() - }; - - let issuer_ca_der = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - ) - .expect("read issuer ca fixture"); - let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); - - let handle = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), - ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: rsync_base_uri.clone(), - manifest_rsync_uri: manifest_rsync_uri.clone(), - publication_point_rsync_uri: rsync_base_uri.clone(), - rrdp_notification_uri: None, - }; - - // First: successful fresh run to populate the latest VCIR baseline. - let ok_runner = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), - validation_time, - timing: None, - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: false, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - }; - let first = ok_runner - .run_publication_point(&handle) - .expect("first run ok"); - assert_eq!(first.source, PublicationPointSource::Fresh); - assert!( - first.discovered_children.is_empty(), - "fixture has no child .cer" - ); - - // Second: repo sync fails, but we can still reuse current-instance VCIR. - let bad_runner = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &FailingRsyncFetcher, - validation_time, - timing: None, - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: false, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - }; - let second = bad_runner - .run_publication_point(&handle) - .expect("should reuse current-instance VCIR"); - assert_eq!(second.source, PublicationPointSource::VcirCurrentInstance); - assert!(second.discovered_children.is_empty()); - assert!( - second - .warnings - .iter() - .any(|w| w.message.contains("repo sync failed")), - "expected warning about repo sync failure" - ); -} - -#[test] -fn build_publication_point_audit_emits_no_audit_entry_for_duplicate_pack_uri() { - let pack = dummy_pack_with_files(vec![ - PackFile::from_bytes_compute_sha256("rsync://example.test/repo/dup.roa", vec![1u8]), - PackFile::from_bytes_compute_sha256("rsync://example.test/repo/dup.roa", vec![2u8]), - ]); - let pp = crate::validation::manifest::PublicationPointResult { - source: crate::validation::manifest::PublicationPointSource::VcirCurrentInstance, - snapshot: pack.clone(), - warnings: Vec::new(), - }; - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(vec![1]), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: pack.publication_point_rsync_uri.clone(), - manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), - rrdp_notification_uri: None, - }; - let objects = crate::validation::objects::ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings: Vec::new(), - stats: crate::validation::objects::ObjectsStats::default(), - audit: Vec::new(), - roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), - roa_cache_object_meta: Vec::new(), - }; - - let audit = build_publication_point_audit_from_snapshot( - &ca, - pp.source, - None, - None, - None, - None, - &pp.snapshot, - &[], - &objects, - &[], - ); - assert_eq!(audit.source, "vcir_current_instance"); - assert_eq!(audit.repo_sync_phase, None); - assert_eq!(audit.repo_terminal_state, "fallback_current_instance"); - assert!( - audit - .objects - .iter() - .any(|e| e.detail.as_deref() == Some("skipped: no audit entry")), - "expected a duplicate key to produce a 'no audit entry' placeholder" - ); -} - -#[test] -fn build_publication_point_audit_marks_invalid_crl_as_error_and_overlays_roa_audit() { - let now = time::OffsetDateTime::now_utc(); - let pack = dummy_pack_with_files(vec![ - PackFile::from_bytes_compute_sha256("rsync://example.test/repo/issuer/bad.crl", vec![0u8]), - PackFile::from_bytes_compute_sha256("rsync://example.test/repo/issuer/x.roa", vec![1u8]), - ]); - - let pp = crate::validation::manifest::PublicationPointResult { - source: crate::validation::manifest::PublicationPointSource::Fresh, - snapshot: pack.clone(), - warnings: Vec::new(), - }; - - let issuer = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(vec![1]), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), - rrdp_notification_uri: None, - }; - - let objects = crate::validation::objects::ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings: Vec::new(), - stats: crate::validation::objects::ObjectsStats::default(), - audit: vec![ObjectAuditEntry { - rsync_uri: "rsync://example.test/repo/issuer/x.roa".to_string(), - sha256_hex: sha256_hex_from_32(&pack.files[1].sha256), - kind: AuditObjectKind::Roa, - result: AuditObjectResult::Ok, - detail: None, - }], - roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), - roa_cache_object_meta: Vec::new(), - }; - - let audit = build_publication_point_audit_from_snapshot( - &issuer, - pp.source, - Some("rsync"), - Some("rsync_only_ok"), - Some(123), - Some("none"), - &pp.snapshot, - &[], - &objects, - &[], - ); - assert_eq!(audit.objects[0].kind, AuditObjectKind::Manifest); - assert_eq!(audit.repo_sync_source.as_deref(), Some("rsync")); - assert_eq!(audit.repo_sync_phase.as_deref(), Some("rsync_only_ok")); - assert_eq!(audit.repo_sync_duration_ms, Some(123)); - assert_eq!(audit.repo_sync_error.as_deref(), Some("none")); - assert_eq!(audit.repo_terminal_state, "fresh"); - - let crl = audit - .objects - .iter() - .find(|e| e.rsync_uri.ends_with("bad.crl")) - .expect("crl entry"); - assert!(matches!(crl.result, AuditObjectResult::Error)); - - let roa = audit - .objects - .iter() - .find(|e| e.rsync_uri.ends_with("x.roa")) - .expect("roa entry"); - assert!(matches!(roa.result, AuditObjectResult::Ok)); - - // Smoke that time fields are populated from pack. - assert!(audit.verified_at_rfc3339_utc.contains('T')); - assert!(audit.this_update_rfc3339_utc.contains('T')); - assert!(audit.next_update_rfc3339_utc.contains('T')); - let _ = now; -} - -#[test] -fn discover_children_with_router_certificate_records_ok_audit_and_no_child() { - let g = generate_router_cert_with_variant("ec-p256", true); - let pack = dummy_pack_with_files(vec![ - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/issuer.crl", - g.issuer_crl_der.clone(), - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/router.cer", - g.router_der.clone(), - ), - ]); - - let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); - let issuer = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), - ca_certificate_rsync_uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let out = discover_children_from_fresh_snapshot_with_audit( - &issuer, - &pack, - time::OffsetDateTime::now_utc(), - None, - ) - .expect("discover router cert"); - assert!(out.children.is_empty()); - assert_eq!(out.audits.len(), 1); - assert!(matches!(out.audits[0].result, AuditObjectResult::Ok)); - assert!( - out.audits[0] - .detail - .as_deref() - .unwrap_or("") - .contains("validated BGPsec router certificate") - ); -} - -#[test] -fn discover_children_with_non_router_ee_certificate_records_skipped_audit() { - let g = generate_router_cert_with_variant("ec-p256", false); - let pack = dummy_pack_with_files(vec![ - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/issuer.crl", - g.issuer_crl_der.clone(), - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/router-no-eku.cer", - g.router_der.clone(), - ), - ]); - - let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); - let issuer = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), - ca_certificate_rsync_uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let out = discover_children_from_fresh_snapshot_with_audit( - &issuer, - &pack, - time::OffsetDateTime::now_utc(), - None, - ) - .expect("discover non-router cert"); - assert!(out.children.is_empty()); - assert_eq!(out.audits.len(), 1); - assert!(matches!(out.audits[0].result, AuditObjectResult::Skipped)); - assert!( - out.audits[0] - .detail - .as_deref() - .unwrap_or("") - .contains("not a CA resource certificate or BGPsec router certificate") - ); -} - -#[test] -fn discover_children_with_invalid_router_certificate_records_error_audit() { - let g = generate_router_cert_with_variant("ec-p384", true); - let pack = dummy_pack_with_files(vec![ - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/issuer.crl", - g.issuer_crl_der.clone(), - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/router-invalid.cer", - g.router_der.clone(), - ), - ]); - - let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); - let issuer = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), - ca_certificate_rsync_uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let out = discover_children_from_fresh_snapshot_with_audit( - &issuer, - &pack, - time::OffsetDateTime::now_utc(), - None, - ) - .expect("discover invalid router cert"); - assert!(out.children.is_empty()); - assert_eq!(out.audits.len(), 1); - assert!(matches!(out.audits[0].result, AuditObjectResult::Error)); - assert!( - out.audits[0] - .detail - .as_deref() - .unwrap_or("") - .contains("router certificate validation failed") - ); -} - -#[test] -fn discover_children_with_audit_records_decode_error_for_corrupt_cer() { - let g = generate_chain_and_crl(); - - let pack = dummy_pack_with_files(vec![ - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/issuer.crl", - g.issuer_crl_der.clone(), - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/corrupt.cer", - vec![0u8], - ), - ]); - - let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); - let issuer = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), - ca_certificate_rsync_uri: None, - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let now = time::OffsetDateTime::now_utc(); - let out = discover_children_from_fresh_snapshot_with_audit(&issuer, &pack, now, None) - .expect("discover children"); - assert!(out.children.is_empty()); - assert_eq!(out.audits.len(), 1); - assert!(matches!(out.audits[0].result, AuditObjectResult::Error)); -} - -#[test] -fn select_issuer_crl_uri_for_child_covers_missing_and_not_found_paths() { - let g = generate_chain_and_crl(); - let child = ResourceCertificate::decode_der(&g.child_ca_der).expect("decode child cert"); - - let empty: std::collections::HashMap = - std::collections::HashMap::new(); - let err = select_issuer_crl_uri_for_child(&child, &empty).unwrap_err(); - assert!(err.contains("no CRL available"), "{err}"); - - let ta_der = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/ta/apnic-ta.cer"), - ) - .expect("read TA fixture"); - let ta = ResourceCertificate::decode_der(&ta_der).expect("decode TA fixture"); - let mut cache = std::collections::HashMap::new(); - cache.insert( - "rsync://example.test/repo/issuer/issuer.crl".to_string(), - CachedIssuerCrl::Pending { - bytes: g.issuer_crl_der.clone(), - sha256_hex: None, - }, - ); - let err = select_issuer_crl_uri_for_child(&ta, &cache).unwrap_err(); - assert!(err.contains("CRLDistributionPoints missing"), "{err}"); - - let mut wrong = std::collections::HashMap::new(); - wrong.insert( - "rsync://example.test/repo/issuer/other.crl".to_string(), - CachedIssuerCrl::Pending { - bytes: g.issuer_crl_der, - sha256_hex: None, - }, - ); - let err = select_issuer_crl_uri_for_child(&child, &wrong).unwrap_err(); - assert!( - err.contains("not found in publication point snapshot"), - "{err}" - ); -} - -#[test] -fn ensure_issuer_crl_verified_promotes_pending_cache_entry() { - let g = generate_chain_and_crl(); - let mut cache = std::collections::HashMap::new(); - let crl_uri = "rsync://example.test/repo/issuer/issuer.crl".to_string(); - cache.insert( - crl_uri.clone(), - CachedIssuerCrl::Pending { - bytes: g.issuer_crl_der.clone(), - sha256_hex: None, - }, - ); - - let first = ensure_issuer_crl_verified(&crl_uri, &mut cache, &g.issuer_ca_der) - .expect("verify pending CRL"); - assert!(first.revoked_serials.is_empty()); - assert!(matches!(cache.get(&crl_uri), Some(CachedIssuerCrl::Ok(_)))); - - let second = ensure_issuer_crl_verified(&crl_uri, &mut cache, &g.issuer_ca_der) - .expect("reuse verified CRL"); - assert!(second.revoked_serials.is_empty()); -} - -#[test] -fn discover_children_with_invalid_issuer_der_records_error_audit() { - let g = generate_chain_and_crl(); - let pack = dummy_pack_with_files(vec![ - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/issuer.crl", - g.issuer_crl_der.clone(), - ), - PackFile::from_bytes_compute_sha256( - "rsync://example.test/repo/issuer/child.cer", - g.child_ca_der.clone(), - ), - ]); - - let issuer = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(vec![0u8]), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let out = discover_children_from_fresh_snapshot_with_audit( - &issuer, - &pack, - time::OffsetDateTime::now_utc(), - None, - ) - .expect("discover children with invalid issuer der"); - assert!(out.children.is_empty()); - assert_eq!(out.audits.len(), 1); - assert!(matches!(out.audits[0].result, AuditObjectResult::Error)); - assert!( - out.audits[0] - .detail - .as_deref() - .unwrap_or("") - .contains("issuer CA decode failed") - ); -} - -#[test] -fn project_current_instance_vcir_reuses_local_outputs_and_restores_children() { - let now = time::OffsetDateTime::now_utc(); - let g = generate_chain_and_crl(); - let child_cert_hash = sha256_hex(&g.child_ca_der); - let vcir = sample_vcir_for_projection(now, &child_cert_hash); - - let store_dir = tempfile::tempdir().expect("store dir"); - let main_db = store_dir.path().join("work-db"); - let repo_bytes_db = store_dir.path().join("repo-bytes.db"); - let store = RocksStore::open_with_external_repo_bytes(&main_db, &repo_bytes_db) - .expect("open rocksdb with external repo bytes"); - store - .put_blob_bytes_batch(&[(child_cert_hash.clone(), g.child_ca_der.clone())]) - .expect("put child cert repo bytes"); - assert!( - store - .get_raw_by_hash_entry(&child_cert_hash) - .expect("lookup child raw_by_hash") - .is_none(), - "child cert restoration should not require raw_by_hash entries" - ); - - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(Vec::new()), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), - }; - let policy = Policy::default(); - put_vcir_for_failed_fetch_reuse(&store, &ca, &policy, &vcir); - - let projection = project_current_instance_vcir_on_failed_fetch( - &store, - &ca, - &ManifestFreshError::RepoSyncFailed { - detail: "synthetic".to_string(), - }, - &policy, - now, - ) - .expect("project vcir"); - - assert_eq!( - projection.source, - PublicationPointSource::VcirCurrentInstance - ); - assert_eq!(projection.objects.vrps.len(), 1); - assert_eq!(projection.objects.aspas.len(), 1); - assert_eq!(projection.objects.router_keys.len(), 1); - assert_eq!(projection.discovered_children.len(), 1); - assert_eq!( - projection.discovered_children[0].handle.manifest_rsync_uri, - "rsync://example.test/repo/child/child.mft" - ); - assert_eq!( - projection.ccr_manifest_projection.as_ref(), - Some(&vcir.ccr_manifest_projection) - ); - assert!( - projection.snapshot.is_none(), - "current-instance reuse should not reconstruct a byte-backed snapshot" - ); - assert!( - !projection - .warnings - .iter() - .any(|warning| warning.message.contains("manifest failed fetch")), - "successful current-instance reuse should not duplicate the fresh fetch error" - ); - assert!( - !projection - .warnings - .iter() - .any(|warning| warning.message.contains("using latest validated result")), - "successful current-instance reuse should be tracked by source, not warning" - ); - assert!( - !projection - .warnings - .iter() - .any(|warning| warning.message.contains("manifest raw bytes missing")), - "successful current-instance reuse should not load repo bytes for audit reconstruction" - ); - assert!( - !projection - .warnings - .iter() - .any(|warning| warning.message.contains("child certificate bytes missing")), - "child discovery restoration should read child certs from repo bytes" - ); -} - -#[test] -fn project_current_instance_vcir_does_not_reuse_without_identity() { - let now = time::OffsetDateTime::now_utc(); - let child_cert_hash = sha256_hex(b"child-cert"); - let vcir = sample_vcir_for_projection(now, &child_cert_hash); - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - store.put_vcir(&vcir).expect("put legacy-style VCIR"); - let ca = sample_ca_for_failed_fetch_reuse(&vcir); - - let projection = project_current_instance_vcir_on_failed_fetch( - &store, - &ca, - &ManifestFreshError::RepoSyncFailed { - detail: "synthetic".to_string(), - }, - &Policy::default(), - now, - ) - .expect("project VCIR"); - - assert_eq!( - projection.source, - PublicationPointSource::FailedFetchNoCache - ); - assert!(projection.objects.vrps.is_empty()); - assert!(projection.discovered_children.is_empty()); - assert!( - projection - .warnings - .iter() - .any(|warning| warning.message.contains("reuse identity is missing")) - ); -} - -#[test] -fn project_current_instance_vcir_does_not_reuse_when_ta_context_changes() { - let now = time::OffsetDateTime::now_utc(); - let child_cert_hash = sha256_hex(b"child-cert"); - let vcir = sample_vcir_for_projection(now, &child_cert_hash); - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy::default(); - let original_ca = sample_ca_for_failed_fetch_reuse(&vcir); - put_vcir_for_failed_fetch_reuse(&store, &original_ca, &policy, &vcir); - let mut changed_ca = original_ca; - changed_ca.tal_id = "different-tal".to_string(); - - let projection = project_current_instance_vcir_on_failed_fetch( - &store, - &changed_ca, - &ManifestFreshError::RepoSyncFailed { - detail: "synthetic".to_string(), - }, - &policy, - now, - ) - .expect("project VCIR"); - - assert_eq!( - projection.source, - PublicationPointSource::FailedFetchNoCache - ); - assert!(projection.objects.vrps.is_empty()); - assert!(projection.discovered_children.is_empty()); - assert!( - projection - .warnings - .iter() - .any(|warning| warning.message.contains("identity does not match")) - ); -} - -#[test] -fn project_current_instance_vcir_returns_no_output_when_instance_gate_expired() { - let now = time::OffsetDateTime::now_utc(); - let child_cert_hash = sha256_hex(b"child-cert"); - let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); - let this_update = PackTime::from_utc_offset_datetime(now - time::Duration::minutes(2)); - let expired = PackTime::from_utc_offset_datetime(now - time::Duration::minutes(1)); - vcir.validated_manifest_meta.validated_manifest_this_update = this_update; - vcir.validated_manifest_meta.validated_manifest_next_update = expired.clone(); - vcir.instance_gate.manifest_next_update = expired.clone(); - vcir.instance_gate.current_crl_next_update = expired.clone(); - vcir.instance_gate.instance_effective_until = expired; - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - store.put_vcir(&vcir).expect("put vcir"); - - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(Vec::new()), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let projection = project_current_instance_vcir_on_failed_fetch( - &store, - &ca, - &ManifestFreshError::RepoSyncFailed { - detail: "synthetic".to_string(), - }, - &Policy::default(), - now, - ) - .expect("project vcir"); - - assert_eq!( - projection.source, - PublicationPointSource::FailedFetchNoCache - ); - assert!(projection.ccr_manifest_projection.is_none()); - assert!(projection.objects.vrps.is_empty()); - assert!(projection.objects.aspas.is_empty()); - assert!(projection.discovered_children.is_empty()); -} - -#[test] -fn project_current_instance_vcir_keeps_real_fresh_validation_warning() { - let now = time::OffsetDateTime::now_utc(); - let child_cert_hash = sha256_hex(b"child-cert"); - let vcir = sample_vcir_for_projection(now, &child_cert_hash); - - let store_dir = tempfile::tempdir().expect("store dir"); - let main_db = store_dir.path().join("work-db"); - let repo_bytes_db = store_dir.path().join("repo-bytes.db"); - let store = RocksStore::open_with_external_repo_bytes(&main_db, &repo_bytes_db) - .expect("open rocksdb with external repo bytes"); - store - .put_blob_bytes_batch(&[(child_cert_hash, b"child-cert".to_vec())]) - .expect("put child cert repo bytes"); - - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(Vec::new()), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - let policy = Policy::default(); - put_vcir_for_failed_fetch_reuse(&store, &ca, &policy, &vcir); - - let projection = project_current_instance_vcir_on_failed_fetch( - &store, - &ca, - &ManifestFreshError::HashMismatch { - rsync_uri: "rsync://example.test/repo/issuer/a.roa".to_string(), - }, - &policy, - now, - ) - .expect("project vcir"); - - assert_eq!( - projection.source, - PublicationPointSource::VcirCurrentInstance - ); - assert!( - projection - .warnings - .iter() - .any(|warning| { warning.message.contains("manifest file hash mismatch") }) - ); - assert!( - !projection - .warnings - .iter() - .any(|warning| warning.message.contains("using latest validated result")), - "successful current-instance reuse should not emit bookkeeping warnings" - ); -} - -#[test] -fn project_current_instance_vcir_returns_no_output_when_latest_result_missing() { - let now = time::OffsetDateTime::now_utc(); - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(Vec::new()), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let projection = project_current_instance_vcir_on_failed_fetch( - &store, - &ca, - &ManifestFreshError::RepoSyncFailed { - detail: "synthetic".to_string(), - }, - &Policy::default(), - now, - ) - .expect("project without cached vcir"); - - assert_eq!( - projection.source, - PublicationPointSource::FailedFetchNoCache - ); - assert!(projection.vcir.is_none()); - assert!(projection.ccr_manifest_projection.is_none()); - assert!(projection.snapshot.is_none()); - assert!(projection.objects.audit.is_empty()); - assert!(projection.discovered_children.is_empty()); - assert!(projection.warnings.iter().any(|warning| { - warning - .message - .contains("no latest validated result for current CA instance") - })); -} - -#[test] -fn project_current_instance_vcir_returns_no_output_when_latest_result_is_ineligible() { - let now = time::OffsetDateTime::now_utc(); - let child_cert_hash = sha256_hex(b"child-cert"); - let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); - vcir.audit_summary.failed_fetch_eligible = false; - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - store.put_vcir(&vcir).expect("put vcir"); - - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(Vec::new()), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let projection = project_current_instance_vcir_on_failed_fetch( - &store, - &ca, - &ManifestFreshError::RepoSyncFailed { - detail: "synthetic".to_string(), - }, - &Policy::default(), - now, - ) - .expect("project ineligible vcir"); - - assert_eq!( - projection.source, - PublicationPointSource::FailedFetchNoCache - ); - assert!(projection.vcir.is_some()); - assert!(projection.ccr_manifest_projection.is_none()); - assert!(projection.snapshot.is_none()); - assert!(projection.discovered_children.is_empty()); - assert!(projection.warnings.iter().any(|warning| { - warning - .message - .contains("latest VCIR is not marked failed-fetch eligible") - })); -} - -#[test] -fn project_current_instance_vcir_rejects_mismatched_ccr_projection_uri() { - let now = time::OffsetDateTime::now_utc(); - let child_cert_hash = sha256_hex(b"child-cert"); - let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); - vcir.ccr_manifest_projection.manifest_rsync_uri = - "rsync://example.test/repo/issuer/other.mft".to_string(); - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(Vec::new()), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - let policy = Policy::default(); - put_vcir_for_failed_fetch_reuse(&store, &ca, &policy, &vcir); - - let err = project_current_instance_vcir_on_failed_fetch( - &store, - &ca, - &ManifestFreshError::RepoSyncFailed { - detail: "synthetic".to_string(), - }, - &policy, - now, - ) - .unwrap_err(); - - assert!( - err.contains("vcir CCR manifest projection URI mismatch"), - "{err}" - ); -} - -#[test] -fn fresh_and_reuse_paths_produce_equivalent_ccr_manifest_projection() { - let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests(); - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), - ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: pack.publication_point_rsync_uri.clone(), - manifest_rsync_uri: pack.manifest_rsync_uri.clone(), - publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), - rrdp_notification_uri: None, - }; - let child_discovery = - discover_children_from_fresh_snapshot_with_audit(&ca, &pack, validation_time, None) - .expect("discover children"); - let mut objects = empty_objects_output(); - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let (fresh_vcir, _timing) = build_vcir_from_fresh_result_with_timing( - &store, - &ca, - &pack, - &mut objects, - &[], - &child_discovery.audits, - &child_discovery.children, - validation_time, - ) - .expect("build fresh vcir"); - - let reuse_projection = reuse_ccr_manifest_projection_from_vcir(&ca, &fresh_vcir) - .expect("reuse projection from vcir"); - - assert_eq!(fresh_vcir.ccr_manifest_projection, reuse_projection); -} - -#[test] -fn append_ccr_manifest_projection_from_reuse_requires_projection_for_current_instance() { - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy::default(); - let runner = sample_runner_with_ccr_accumulator(&store, &policy); - - let err = runner - .append_ccr_manifest_projection_from_reuse(&VcirReuseProjection { - source: PublicationPointSource::VcirCurrentInstance, - vcir: None, - ccr_manifest_projection: None, - snapshot: None, - objects: empty_objects_output(), - child_audits: Vec::new(), - discovered_children: Vec::new(), - warnings: Vec::new(), - }) - .unwrap_err(); - - assert!(err.contains("missing CCR manifest projection"), "{err}"); - assert_eq!( - runner - .ccr_accumulator_snapshot() - .expect("ccr accumulator snapshot") - .manifest_count(), - 0 - ); -} - -#[test] -fn append_ccr_manifest_projection_from_reuse_skips_failed_fetch_no_cache() { - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy::default(); - let runner = sample_runner_with_ccr_accumulator(&store, &policy); - - runner - .append_ccr_manifest_projection_from_reuse(&VcirReuseProjection { - source: PublicationPointSource::FailedFetchNoCache, - vcir: None, - ccr_manifest_projection: None, - snapshot: None, - objects: empty_objects_output(), - child_audits: Vec::new(), - discovered_children: Vec::new(), - warnings: Vec::new(), - }) - .expect("failed-fetch no-cache should not append"); - - assert_eq!( - runner - .ccr_accumulator_snapshot() - .expect("ccr accumulator snapshot") - .manifest_count(), - 0 - ); -} - -#[test] -fn parse_snapshot_time_value_reports_invalid_timestamp() { - let err = parse_snapshot_time_value(&PackTime { - rfc3339_utc: "not-a-time".to_string(), - }) - .unwrap_err(); - - assert!(err.contains("invalid RFC3339 time 'not-a-time'"), "{err}"); -} - -#[test] -fn runner_roa_validation_cache_uses_projection_not_full_vcir_fallback() { - let now = time::OffsetDateTime::now_utc(); - let child_cert_hash = sha256_hex(b"child-cert"); - let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); - vcir.local_outputs - .retain(|output| output.source_object_type != VcirSourceObjectType::Roa); - vcir.summary.local_vrp_count = 0; - vcir.summary.local_aspa_count = 1; - vcir.summary.local_router_key_count = 1; - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - store.put_vcir(&vcir).expect("put vcir without projection"); - assert!( - store - .get_vcir(&vcir.manifest_rsync_uri) - .expect("get vcir") - .is_some() - ); - assert!( - store - .get_roa_cache_projection(&vcir.manifest_rsync_uri) - .expect("get projection") - .is_none() - ); - - let timing = crate::analysis::timing::TimingHandle::new(crate::analysis::timing::TimingMeta { - recorded_at_utc_rfc3339: "2026-06-07T00:00:00Z".to_string(), - validation_time_utc_rfc3339: "2026-06-07T00:00:00Z".to_string(), - tal_url: None, - db_path: None, - }); - let policy = Policy::default(); - let runner = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &FailingRsyncFetcher, - validation_time: now, - timing: Some(timing.clone()), - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: false, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: true, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - }; - - assert!( - runner - .roa_validation_cache_view_for_fresh_point(&vcir.manifest_rsync_uri) - .is_none() - ); - let dir = tempfile::tempdir().expect("timing dir"); - let path = dir.path().join("timing.json"); - timing.write_json(&path, 10).expect("write timing"); - let report: serde_json::Value = - serde_json::from_slice(&std::fs::read(path).expect("read timing")).expect("parse timing"); - assert_eq!( - report["counts"]["roa_validation_cache_projection_missing_publication_points"], - 1 - ); - assert!( - report["phases"]["roa_validation_cache_projection_load_total"]["count"] - .as_u64() - .unwrap_or_default() - >= 1 - ); -} - -#[test] -fn build_objects_output_from_vcir_tracks_expired_and_invalid_cached_outputs() { - let now = time::OffsetDateTime::now_utc(); - let child_cert_hash = sha256_hex(b"child-cert"); - let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); - - let bad_time_uri = "rsync://example.test/repo/issuer/bad-time.roa".to_string(); - let expired_uri = "rsync://example.test/repo/issuer/expired.asa".to_string(); - let bad_json_uri = "rsync://example.test/repo/issuer/bad-json.roa".to_string(); - let bad_prefix_uri = "rsync://example.test/repo/issuer/bad-prefix.roa".to_string(); - let bad_aspa_uri = "rsync://example.test/repo/issuer/bad-aspa.asa".to_string(); - - for (uri, kind) in [ - (bad_time_uri.clone(), VcirArtifactKind::Roa), - (expired_uri.clone(), VcirArtifactKind::Aspa), - (bad_json_uri.clone(), VcirArtifactKind::Roa), - (bad_prefix_uri.clone(), VcirArtifactKind::Roa), - (bad_aspa_uri.clone(), VcirArtifactKind::Aspa), - ] { - vcir.related_artifacts.push(VcirRelatedArtifact { - artifact_role: VcirArtifactRole::SignedObject, - artifact_kind: kind, - uri: Some(uri.clone()), - sha256: sha256_hex(uri.as_bytes()), - object_type: Some( - match kind { - VcirArtifactKind::Aspa => "aspa", - _ => "roa", - } - .to_string(), - ), - validation_status: VcirArtifactValidationStatus::Accepted, - reject_reason: None, - }); - } - - vcir.local_outputs.push(VcirLocalOutput { - output_type: VcirOutputType::Vrp, - item_effective_until: PackTime { - rfc3339_utc: "bad-time-value".to_string(), - }, - source_object_uri: bad_time_uri.clone(), - source_object_type: VcirSourceObjectType::Roa, - source_object_hash: sha256_32(b"bad-time-src"), - source_ee_cert_hash: sha256_32(b"bad-time-ee"), - payload: VcirLocalOutputPayload::Vrp { - asn: 64496, - afi: RoaAfi::Ipv4, - prefix_len: 24, - addr: ipv4_addr([203, 0, 113, 0]), - max_length: 24, - }, - rule_hash: sha256_32(b"bad-time-rule"), - }); - vcir.local_outputs.push(VcirLocalOutput { - output_type: VcirOutputType::Aspa, - item_effective_until: PackTime::from_utc_offset_datetime(now - time::Duration::minutes(1)), - source_object_uri: expired_uri.clone(), - source_object_type: VcirSourceObjectType::Aspa, - source_object_hash: sha256_32(b"expired-src"), - source_ee_cert_hash: sha256_32(b"expired-ee"), - payload: VcirLocalOutputPayload::Aspa { - customer_as_id: 64500, - provider_as_ids: vec![64501], - }, - rule_hash: sha256_32(b"expired-rule"), - }); - vcir.local_outputs.push(VcirLocalOutput { - output_type: VcirOutputType::Vrp, - item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(5)), - source_object_uri: bad_json_uri.clone(), - source_object_type: VcirSourceObjectType::Roa, - source_object_hash: sha256_32(b"bad-json-src"), - source_ee_cert_hash: sha256_32(b"bad-json-ee"), - payload: VcirLocalOutputPayload::Aspa { - customer_as_id: 64510, - provider_as_ids: vec![64511], - }, - rule_hash: sha256_32(b"bad-json-rule"), - }); - vcir.local_outputs.push(VcirLocalOutput { - output_type: VcirOutputType::Vrp, - item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(5)), - source_object_uri: bad_prefix_uri.clone(), - source_object_type: VcirSourceObjectType::Roa, - source_object_hash: sha256_32(b"bad-prefix-src"), - source_ee_cert_hash: sha256_32(b"bad-prefix-ee"), - payload: VcirLocalOutputPayload::Aspa { - customer_as_id: 64512, - provider_as_ids: vec![64513], - }, - rule_hash: sha256_32(b"bad-prefix-rule"), - }); - vcir.local_outputs.push(VcirLocalOutput { - output_type: VcirOutputType::Aspa, - item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(5)), - source_object_uri: bad_aspa_uri.clone(), - source_object_type: VcirSourceObjectType::Aspa, - source_object_hash: sha256_32(b"bad-aspa-src"), - source_ee_cert_hash: sha256_32(b"bad-aspa-ee"), - payload: VcirLocalOutputPayload::Vrp { - asn: 64520, - afi: RoaAfi::Ipv4, - prefix_len: 24, - addr: ipv4_addr([198, 51, 100, 0]), - max_length: 24, - }, - rule_hash: sha256_32(b"bad-aspa-rule"), - }); - - let mut warnings = Vec::new(); - let output = build_objects_output_from_vcir(&vcir, now, &mut warnings); - - assert_eq!(output.vrps.len(), 1); - assert_eq!(output.aspas.len(), 1); - assert_eq!(output.stats.roa_total, 4); - assert_eq!(output.stats.roa_ok, 1); - assert_eq!(output.stats.aspa_total, 3); - assert_eq!(output.stats.aspa_ok, 1); - assert!(warnings.iter().any(|warning| { - warning - .message - .contains("cached local output has invalid item_effective_until") - })); - assert!(warnings.iter().any(|warning| { - warning - .message - .contains("cached ROA local output parse failed") - })); - assert!(warnings.iter().any(|warning| { - warning - .message - .contains("cached ASPA local output parse failed") - })); - assert!(output.audit.iter().any(|entry| { - entry.rsync_uri == expired_uri - && matches!(entry.result, AuditObjectResult::Skipped) - && entry.detail.as_deref() == Some("skipped: cached local output expired") - })); - assert!(output.audit.iter().any(|entry| { - entry.rsync_uri == bad_time_uri && matches!(entry.result, AuditObjectResult::Error) - })); - assert!(output.audit.iter().any(|entry| { - entry.rsync_uri == bad_prefix_uri - && matches!(entry.result, AuditObjectResult::Error) - && entry - .detail - .as_deref() - .unwrap_or("") - .contains("cached ROA local output parse failed") - })); -} - -#[test] -fn build_publication_point_audit_from_vcir_uses_vcir_metadata_and_overlays_child_and_object_audits() -{ - let now = time::OffsetDateTime::now_utc(); - let child_cert_hash = sha256_hex(b"child-cert"); - let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); - vcir.related_artifacts - .retain(|artifact| artifact.artifact_role != VcirArtifactRole::Manifest); - - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(Vec::new()), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), - }; - let runner_warnings = vec![Warning::new("runner warning")]; - let objects = crate::validation::objects::ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings: vec![Warning::new("objects warning")], - stats: crate::validation::objects::ObjectsStats::default(), - audit: vec![ObjectAuditEntry { - rsync_uri: "rsync://example.test/repo/issuer/a.roa".to_string(), - sha256_hex: sha256_hex(b"override-roa"), - kind: AuditObjectKind::Roa, - result: AuditObjectResult::Error, - detail: Some("overridden from object audit".to_string()), - }], - roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), - roa_cache_object_meta: Vec::new(), - }; - let child_audits = vec![ObjectAuditEntry { - rsync_uri: vcir.child_entries[0].child_cert_rsync_uri.clone(), - sha256_hex: vcir.child_entries[0].child_cert_hash.clone(), - kind: AuditObjectKind::Certificate, - result: AuditObjectResult::Ok, - detail: Some("restored child CA instance from VCIR".to_string()), - }]; - - let audit = build_publication_point_audit_from_vcir( - &ca, - PublicationPointSource::VcirCurrentInstance, - Some("rsync"), - Some("rrdp_failed_rsync_failed"), - Some(456), - Some("rsync failed"), - Some(&vcir), - None, - &runner_warnings, - &objects, - &child_audits, - &[], - ); - - assert_eq!(audit.source, "vcir_current_instance"); - assert_eq!(audit.repo_sync_source.as_deref(), Some("rsync")); - assert_eq!( - audit.repo_sync_phase.as_deref(), - Some("rrdp_failed_rsync_failed") - ); - assert_eq!(audit.repo_sync_duration_ms, Some(456)); - assert_eq!(audit.repo_sync_error.as_deref(), Some("rsync failed")); - assert_eq!(audit.repo_terminal_state, "fallback_current_instance"); - assert_eq!(audit.objects[0].rsync_uri, vcir.current_manifest_rsync_uri); - assert_eq!(audit.objects[0].kind, AuditObjectKind::Manifest); - assert_eq!( - audit.this_update_rfc3339_utc, - vcir.validated_manifest_meta - .validated_manifest_this_update - .rfc3339_utc - ); - assert_eq!( - audit.next_update_rfc3339_utc, - vcir.validated_manifest_meta - .validated_manifest_next_update - .rfc3339_utc - ); - assert_eq!( - audit.verified_at_rfc3339_utc, - vcir.last_successful_validation_time.rfc3339_utc - ); - assert_eq!(audit.warnings.len(), 2); - assert!(audit.objects.iter().any(|entry| { - entry.rsync_uri == "rsync://example.test/repo/issuer/a.roa" - && matches!(entry.result, AuditObjectResult::Error) - && entry.detail.as_deref() == Some("overridden from object audit") - })); - assert!(audit.objects.iter().any(|entry| { - entry.rsync_uri == vcir.child_entries[0].child_cert_rsync_uri - && matches!(entry.result, AuditObjectResult::Ok) - })); -} - -#[test] -fn build_publication_point_audit_from_vcir_restores_reject_reason_with_legacy_fallback() { - let now = time::OffsetDateTime::now_utc(); - let child_cert_hash = sha256_hex(b"child-cert"); - let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); - vcir.related_artifacts.push(VcirRelatedArtifact { - artifact_role: VcirArtifactRole::SignedObject, - artifact_kind: VcirArtifactKind::Roa, - uri: Some("rsync://example.test/repo/issuer/rejected-with-reason.roa".to_string()), - sha256: sha256_hex(b"rejected-with-reason"), - object_type: Some("roa".to_string()), - validation_status: VcirArtifactValidationStatus::Rejected, - reject_reason: Some("EE certificate path validation failed: test".to_string()), - }); - vcir.related_artifacts.push(VcirRelatedArtifact { - artifact_role: VcirArtifactRole::SignedObject, - artifact_kind: VcirArtifactKind::Roa, - uri: Some("rsync://example.test/repo/issuer/rejected-legacy.roa".to_string()), - sha256: sha256_hex(b"rejected-legacy"), - object_type: Some("roa".to_string()), - validation_status: VcirArtifactValidationStatus::Rejected, - reject_reason: None, - }); - - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(Vec::new()), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - let objects = crate::validation::objects::ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings: Vec::new(), - stats: crate::validation::objects::ObjectsStats::default(), - audit: Vec::new(), - roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), - roa_cache_object_meta: Vec::new(), - }; - - let audit = build_publication_point_audit_from_vcir( - &ca, - PublicationPointSource::VcirCurrentInstance, - None, - None, - None, - None, - Some(&vcir), - None, - &[], - &objects, - &[], - &[], - ); - - assert!(audit.objects.iter().any(|entry| { - entry.rsync_uri == "rsync://example.test/repo/issuer/rejected-with-reason.roa" - && matches!(entry.result, AuditObjectResult::Error) - && entry.detail.as_deref() == Some("EE certificate path validation failed: test") - })); - assert!(audit.objects.iter().any(|entry| { - entry.rsync_uri == "rsync://example.test/repo/issuer/rejected-legacy.roa" - && matches!(entry.result, AuditObjectResult::Error) - && entry.detail.as_deref() == Some(CACHED_REJECT_REASON_NOT_RECORDED) - })); - assert!( - audit.objects.iter().all(|entry| { - !matches!(entry.result, AuditObjectResult::Ok) || entry.detail.is_none() - }) - ); -} - -#[test] -fn build_publication_point_audit_from_pp_cache_projection_restores_reject_reason() { - let now = time::OffsetDateTime::now_utc(); - let child_cert_hash = sha256_hex(b"child-cert"); - let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); - vcir.related_artifacts.push(VcirRelatedArtifact { - artifact_role: VcirArtifactRole::SignedObject, - artifact_kind: VcirArtifactKind::Roa, - uri: Some("rsync://example.test/repo/issuer/rejected-with-reason.roa".to_string()), - sha256: sha256_hex(b"rejected-with-reason"), - object_type: Some("roa".to_string()), - validation_status: VcirArtifactValidationStatus::Rejected, - reject_reason: Some("EE certificate path validation failed: test".to_string()), - }); - vcir.related_artifacts.push(VcirRelatedArtifact { - artifact_role: VcirArtifactRole::SignedObject, - artifact_kind: VcirArtifactKind::Roa, - uri: Some("rsync://example.test/repo/issuer/rejected-legacy.roa".to_string()), - sha256: sha256_hex(b"rejected-legacy"), - object_type: Some("roa".to_string()), - validation_status: VcirArtifactValidationStatus::Rejected, - reject_reason: None, - }); - let projection = PublicationPointCacheProjection::from_vcir_with_context( - &vcir, - "rsync://example.test/repo/issuer/".to_string(), - None, - [0x11; 32], - [0x22; 32], - [0x33; 32], - [0x44; 32], - [0x55; 32], - ) - .expect("build publication point projection"); - - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(Vec::new()), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - let objects = crate::validation::objects::ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings: Vec::new(), - stats: crate::validation::objects::ObjectsStats::default(), - audit: Vec::new(), - roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), - roa_cache_object_meta: Vec::new(), - }; - - let audit = build_publication_point_audit_from_publication_point_cache_projection( - &ca, - PublicationPointSource::PublicationPointCache, - None, - None, - None, - None, - &projection, - now, - &[], - &objects, - &[], - ); - - assert!(audit.objects.iter().any(|entry| { - entry.rsync_uri == "rsync://example.test/repo/issuer/rejected-with-reason.roa" - && matches!(entry.result, AuditObjectResult::Error) - && entry.detail.as_deref() == Some("EE certificate path validation failed: test") - })); - assert!(audit.objects.iter().any(|entry| { - entry.rsync_uri == "rsync://example.test/repo/issuer/rejected-legacy.roa" - && matches!(entry.result, AuditObjectResult::Error) - && entry.detail.as_deref() == Some(CACHED_REJECT_REASON_NOT_RECORDED) - })); -} - -#[test] -fn build_publication_point_audit_from_vcir_failed_no_cache_keeps_current_reject_only() { - let now = time::OffsetDateTime::now_utc(); - let child_cert_hash = sha256_hex(b"child-cert"); - let vcir = sample_vcir_for_projection(now, &child_cert_hash); - - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(Vec::new()), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), - }; - let objects = crate::validation::objects::ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings: Vec::new(), - stats: crate::validation::objects::ObjectsStats::default(), - audit: vec![ObjectAuditEntry { - rsync_uri: vcir.current_manifest_rsync_uri.clone(), - sha256_hex: sha256_hex(b"current-manifest"), - kind: AuditObjectKind::Manifest, - result: AuditObjectResult::Error, - detail: Some("manifest is not valid at validation_time".to_string()), - }], - roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), - roa_cache_object_meta: Vec::new(), - }; - - let audit = build_publication_point_audit_from_vcir( - &ca, - PublicationPointSource::FailedFetchNoCache, - Some("rsync"), - Some("rsync_only_ok"), - Some(123), - None, - Some(&vcir), - None, - &[Warning::new("latest VCIR instance_gate expired")], - &objects, - &[], - &[], - ); - - assert_eq!(audit.source, "failed_fetch_no_cache"); - assert_eq!(audit.repo_terminal_state, "failed_no_cache"); - assert_eq!( - audit.this_update_rfc3339_utc, - vcir.validated_manifest_meta - .validated_manifest_this_update - .rfc3339_utc - ); - assert_eq!(audit.objects.len(), 1); - assert_eq!(audit.objects[0].rsync_uri, vcir.current_manifest_rsync_uri); - assert!(matches!(audit.objects[0].result, AuditObjectResult::Error)); - assert!( - !audit - .objects - .iter() - .any(|entry| entry.rsync_uri == "rsync://example.test/repo/issuer/a.roa"), - "failed-no-cache must not expand old VCIR related artifacts into current-run audit", - ); - assert!( - !audit - .objects - .iter() - .any(|entry| entry.rsync_uri == "rsync://example.test/repo/issuer/issuer.crl"), - "failed-no-cache must not expose old CRL as current-run CIR input", - ); -} - -#[test] -fn rejected_manifest_audit_entry_for_failed_fetch_uses_current_repo_hash() { - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let policy = Policy::default(); - let runner = sample_runner_with_ccr_accumulator(&store, &policy); - let manifest_uri = "rsync://example.test/repo/issuer/issuer.mft"; - let manifest_hash = sha256_hex(b"manifest-bytes"); - store - .put_blob_bytes_batch(&[(manifest_hash.clone(), b"manifest-bytes".to_vec())]) - .expect("put manifest bytes"); - store - .put_repository_view_entry(&crate::storage::RepositoryViewEntry { - rsync_uri: manifest_uri.to_string(), - current_hash: Some(manifest_hash.clone()), - repository_source: Some("rsync://example.test/repo/issuer/".to_string()), - object_type: Some("mft".to_string()), - state: crate::storage::RepositoryViewState::Present, - }) - .expect("put repository view"); - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(Vec::new()), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: manifest_uri.to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let entry = runner - .rejected_manifest_audit_entry_for_failed_fetch( - &ca, - &ManifestFreshError::StaleOrEarly { - this_update_rfc3339_utc: "2026-05-27T08:37:07Z".to_string(), - next_update_rfc3339_utc: "2026-05-28T10:01:07Z".to_string(), - validation_time_rfc3339_utc: "2026-05-28T10:11:00Z".to_string(), - }, - ) - .expect("rejected manifest audit entry"); - - assert_eq!(entry.rsync_uri, manifest_uri); - assert_eq!(entry.sha256_hex, manifest_hash); - assert_eq!(entry.kind, AuditObjectKind::Manifest); - assert_eq!(entry.result, AuditObjectResult::Error); - assert!( - entry - .detail - .as_deref() - .unwrap_or("") - .contains("manifest is not valid at validation_time") - ); -} - -#[test] -fn build_publication_point_audit_from_vcir_without_cached_inputs_returns_empty_listing() { - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(Vec::new()), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let audit = build_publication_point_audit_from_vcir( - &ca, - PublicationPointSource::FailedFetchNoCache, - Some("rsync"), - Some("rsync_only_failed"), - Some(789), - Some("load from network failed, fallback to cache"), - None, - None, - &[Warning::new("runner warning")], - &crate::validation::objects::ObjectsOutput { - vrps: Vec::new(), - aspas: Vec::new(), - router_keys: Vec::new(), - local_outputs_cache: Vec::new(), - warnings: vec![Warning::new("object warning")], - stats: crate::validation::objects::ObjectsStats::default(), - audit: Vec::new(), - roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), - roa_cache_object_meta: Vec::new(), - }, - &[], - &[], - ); - - assert_eq!(audit.source, "failed_fetch_no_cache"); - assert_eq!(audit.repo_sync_source.as_deref(), Some("rsync")); - assert_eq!(audit.repo_sync_phase.as_deref(), Some("rsync_only_failed")); - assert_eq!(audit.repo_sync_duration_ms, Some(789)); - assert_eq!( - audit.repo_sync_error.as_deref(), - Some("load from network failed, fallback to cache") - ); - assert_eq!(audit.repo_terminal_state, "failed_no_cache"); - assert!(audit.this_update_rfc3339_utc.is_empty()); - assert!(audit.next_update_rfc3339_utc.is_empty()); - assert!(audit.verified_at_rfc3339_utc.is_empty()); - assert_eq!(audit.warnings.len(), 2); - assert!(audit.objects.is_empty()); -} - -#[test] -fn effective_repo_sync_duration_uses_runtime_duration_for_failures() { - assert_eq!(effective_repo_sync_duration_ms(0, Some(12), false), 12); - assert_eq!(effective_repo_sync_duration_ms(5, Some(12), false), 12); - assert_eq!(effective_repo_sync_duration_ms(20, Some(12), false), 20); - assert_eq!(effective_repo_sync_duration_ms(5, None, false), 5); - assert_eq!(effective_repo_sync_duration_ms(5, Some(12), true), 5); -} - -#[test] -fn reconstruct_snapshot_from_vcir_reports_missing_manifest_and_related_raw_bytes() { - let now = time::OffsetDateTime::now_utc(); - let child_cert_hash = sha256_hex(b"child-cert"); - let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); - let dup_uri = "rsync://example.test/repo/issuer/dup.roa".to_string(); - vcir.related_artifacts.push(VcirRelatedArtifact { - artifact_role: VcirArtifactRole::SignedObject, - artifact_kind: VcirArtifactKind::Roa, - uri: Some(dup_uri.clone()), - sha256: sha256_hex(b"dup-roa-1"), - object_type: Some("roa".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - reject_reason: None, - }); - vcir.related_artifacts.push(VcirRelatedArtifact { - artifact_role: VcirArtifactRole::SignedObject, - artifact_kind: VcirArtifactKind::Roa, - uri: Some(dup_uri.clone()), - sha256: sha256_hex(b"dup-roa-2"), - object_type: Some("roa".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - reject_reason: None, - }); - vcir.related_artifacts.push(VcirRelatedArtifact { - artifact_role: VcirArtifactRole::IssuerCert, - artifact_kind: VcirArtifactKind::Cer, - uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()), - sha256: sha256_hex(b"issuer-cert"), - object_type: Some("cer".to_string()), - validation_status: VcirArtifactValidationStatus::Accepted, - reject_reason: None, - }); - - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(Vec::new()), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let mut warnings = Vec::new(); - assert!(reconstruct_snapshot_from_vcir(&store, &ca, &vcir, &mut warnings).is_none()); - assert!(warnings.iter().any(|warning| { - warning - .message - .contains("manifest raw bytes missing for VCIR audit reconstruction") - })); - - let manifest_bytes = b"manifest-bytes".to_vec(); - let current_crl_bytes = b"current-crl-bytes".to_vec(); - let child_bytes = b"child-cert".to_vec(); - let roa_bytes = b"roa-bytes".to_vec(); - for (bytes, uri, object_type) in [ - ( - manifest_bytes.clone(), - Some(vcir.manifest_rsync_uri.clone()), - Some("mft".to_string()), - ), - ( - current_crl_bytes, - Some(vcir.current_crl_rsync_uri.clone()), - Some("crl".to_string()), - ), - ( - child_bytes, - Some(vcir.child_entries[0].child_cert_rsync_uri.clone()), - Some("cer".to_string()), - ), - ( - roa_bytes, - Some("rsync://example.test/repo/issuer/a.roa".to_string()), - Some("roa".to_string()), - ), - ] { - let mut entry = RawByHashEntry::from_bytes(sha256_hex(&bytes), bytes); - if let Some(uri) = uri { - entry.origin_uris.push(uri); - } - entry.object_type = object_type; - entry.encoding = Some("der".to_string()); - store.put_raw_by_hash_entry(&entry).expect("put raw entry"); - } - - warnings.clear(); - let pack = reconstruct_snapshot_from_vcir(&store, &ca, &vcir, &mut warnings) - .expect("reconstruct pack with partial related artifacts"); - assert_eq!(pack.manifest_bytes, manifest_bytes); - assert_eq!(pack.files.len(), 3, "crl + child cert + roa only"); - assert!( - pack.files - .iter() - .any(|file| file.rsync_uri.ends_with("issuer.crl")) - ); - assert!( - pack.files - .iter() - .any(|file| file.rsync_uri.ends_with("child.cer")) - ); - assert!( - pack.files - .iter() - .any(|file| file.rsync_uri.ends_with("a.roa")) - ); - assert!( - !pack - .files - .iter() - .any(|file| file.rsync_uri.ends_with("issuer.cer")) - ); - assert!(warnings.iter().any(|warning| { - warning - .message - .contains("related artifact raw bytes missing for VCIR audit reconstruction") - })); -} - -#[test] -fn reconstruct_snapshot_from_vcir_reads_repo_bytes_without_raw_entries() { - let now = time::OffsetDateTime::now_utc(); - let child_cert_hash = sha256_hex(b"child-cert"); - let vcir = sample_vcir_for_projection(now, &child_cert_hash); - let ca = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(Vec::new()), - ca_certificate_rsync_uri: None, - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), - manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), - publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), - rrdp_notification_uri: None, - }; - - let store_dir = tempfile::tempdir().expect("store dir"); - let main_db = store_dir.path().join("work-db"); - let repo_bytes_db = store_dir.path().join("repo-bytes.db"); - let store = RocksStore::open_with_external_repo_bytes(&main_db, &repo_bytes_db) - .expect("open rocksdb with external repo bytes"); - let repo_blobs = [ - b"manifest-bytes".to_vec(), - b"current-crl-bytes".to_vec(), - b"child-cert".to_vec(), - b"roa-bytes".to_vec(), - b"aspa-bytes".to_vec(), - ] - .into_iter() - .map(|bytes| (sha256_hex(&bytes), bytes)) - .collect::>(); - store - .put_blob_bytes_batch(&repo_blobs) - .expect("put external repo bytes"); - - let manifest_hash = vcir - .related_artifacts - .iter() - .find(|artifact| artifact.artifact_role == VcirArtifactRole::Manifest) - .expect("manifest artifact") - .sha256 - .clone(); - assert!( - store - .get_raw_by_hash_entry(&manifest_hash) - .expect("raw manifest lookup") - .is_none(), - "repo object bytes must not require raw_by_hash entries" - ); - - let mut warnings = Vec::new(); - let pack = reconstruct_snapshot_from_vcir(&store, &ca, &vcir, &mut warnings) - .expect("reconstruct pack from external repo bytes"); - assert_eq!(pack.manifest_bytes, b"manifest-bytes".to_vec()); - assert_eq!(pack.files.len(), 4, "crl + child cert + roa + aspa"); - assert!( - warnings.iter().all(|warning| { - !warning - .message - .contains("raw bytes missing for VCIR audit reconstruction") - }), - "external repo bytes should satisfy VCIR audit reconstruction without raw warnings" - ); -} - -#[test] -fn runner_dedup_paths_execute_with_timing_enabled() { - let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); - let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); - let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; - let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); - let fixture_manifest_bytes = - std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); - let fixture_manifest = - crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) - .expect("decode manifest fixture"); - let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); - let store_dir = tempfile::tempdir().expect("store dir"); - let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); - let issuer_ca_der = std::fs::read( - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( - "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", - ), - ) - .expect("read issuer ca fixture"); - let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); - let handle = CaInstanceHandle { - depth: 0, - tal_id: "test-tal".to_string(), - parent_manifest_rsync_uri: None, - ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), - ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), - effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), - effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), - rsync_base_uri: rsync_base_uri.clone(), - manifest_rsync_uri: manifest_rsync_uri.clone(), - publication_point_rsync_uri: rsync_base_uri.clone(), - rrdp_notification_uri: Some("https://example.test/notification.xml".to_string()), - }; - let timing = crate::analysis::timing::TimingHandle::new(crate::analysis::timing::TimingMeta { - recorded_at_utc_rfc3339: "2026-03-11T00:00:00Z".to_string(), - validation_time_utc_rfc3339: "2026-03-11T00:00:00Z".to_string(), - tal_url: None, - db_path: None, - }); - let policy_rrdp = Policy::default(); - let runner_rrdp = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy_rrdp, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), - validation_time, - timing: Some(timing.clone()), - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: true, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: false, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - }; - let first = runner_rrdp - .run_publication_point(&handle) - .expect("rrdp fallback to rsync"); - assert_eq!(first.source, PublicationPointSource::Fresh); - let second = runner_rrdp - .run_publication_point(&handle) - .expect("rrdp dedup skip"); - assert_eq!(second.source, PublicationPointSource::Fresh); - - let policy_rsync = Policy { - sync_preference: crate::policy::SyncPreference::RsyncOnly, - ..Policy::default() - }; - let runner_rsync = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy_rsync, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), - validation_time, - timing: Some(timing), - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: true, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: None, - persist_vcir: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: false, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: false, - }; - let third = runner_rsync - .run_publication_point(&handle) - .expect("rsync first run"); - assert_eq!(third.source, PublicationPointSource::Fresh); - let fourth = runner_rsync - .run_publication_point(&handle) - .expect("rsync dedup run"); - assert_eq!(fourth.source, PublicationPointSource::Fresh); - assert_eq!( - crate::fetch::rsync::normalize_rsync_base_uri("rsync://example.test/repo"), - "rsync://example.test/repo/" - ); -} - -#[derive(Debug, serde::Deserialize)] -#[serde(rename_all = "camelCase")] -struct RipeRootFinalizeFixtureEvent { - event_type: String, - validation_time: String, - pp_manifest_uri: Option, - object_uri: Option, - sha256: Option, - object_type: Option, - result: Option, - reason: Option, -} - -#[derive(Debug)] -struct RipeRootFinalizeObject { - uri: String, - sha256_hex: String, - object_type: String, - result: String, - reason: Option, -} - -struct RipeRootFinalizeFixture { - manifest_uri: String, - publication_point_uri: String, - validation_time: time::OffsetDateTime, - manifest_sha256_hex: String, - objects: Vec, -} - -fn load_ripe_root_finalize_fixture( - fixture_root: &std::path::Path, - run_id: &str, -) -> RipeRootFinalizeFixture { - let events_path = fixture_root - .join(format!("run_{run_id}")) - .join("ripe-root-events.jsonl"); - let file = std::fs::File::open(&events_path) - .unwrap_or_else(|e| panic!("open fixture events {} failed: {e}", events_path.display())); - let reader = std::io::BufReader::new(file); - let mut manifest_uri = None; - let mut validation_time = None; - let mut manifest_sha256_hex = None; - let mut objects = Vec::new(); - - for line in std::io::BufRead::lines(reader) { - let line = line.expect("read fixture event line"); - let event: RipeRootFinalizeFixtureEvent = - serde_json::from_str(&line).expect("decode fixture event"); - if event.event_type == "publication_point" { - manifest_uri = event.pp_manifest_uri; - validation_time = Some( - time::OffsetDateTime::parse( - &event.validation_time, - &time::format_description::well_known::Rfc3339, - ) - .expect("parse validation_time"), - ); - continue; - } - if event.event_type != "object" { - continue; - } - let uri = event.object_uri.expect("fixture object uri"); - let sha256_hex = event.sha256.expect("fixture object sha256"); - let object_type = event.object_type.expect("fixture object type"); - let result = event.result.expect("fixture object result"); - if object_type == "manifest" { - manifest_sha256_hex = Some(sha256_hex.clone()); - } - objects.push(RipeRootFinalizeObject { - uri, - sha256_hex, - object_type, - result, - reason: event.reason, - }); - } - - let manifest_uri = manifest_uri.expect("fixture publication_point event"); - let publication_point_uri = manifest_uri - .rsplit_once('/') - .map(|(parent, _)| format!("{parent}/")) - .expect("manifest uri parent"); - RipeRootFinalizeFixture { - manifest_uri, - publication_point_uri, - validation_time: validation_time.expect("fixture validation_time"), - manifest_sha256_hex: manifest_sha256_hex.expect("fixture manifest object"), - objects, - } -} - -fn pack_file_from_fixture_object( - object: &RipeRootFinalizeObject, - repo_bytes: &Arc, -) -> PackFile { - PackFile::from_lazy_repo_bytes( - object.uri.clone(), - object.sha256_hex.clone(), - sha256_hex_to_32(&object.sha256_hex), - repo_bytes.clone(), - ) -} - -fn child_audit_from_fixture_object(object: &RipeRootFinalizeObject) -> ObjectAuditEntry { - ObjectAuditEntry { - rsync_uri: object.uri.clone(), - sha256_hex: object.sha256_hex.clone(), - kind: AuditObjectKind::Certificate, - result: match object.result.as_str() { - "ok" => AuditObjectResult::Ok, - "skipped" => AuditObjectResult::Skipped, - _ => AuditObjectResult::Error, - }, - detail: object.reason.clone(), - } -} - -fn discovered_child_from_fixture_object( - issuer: &CaInstanceHandle, - object: &RipeRootFinalizeObject, - child_entry_projection: Option, -) -> DiscoveredChildCaInstance { - let stem = object - .uri - .rsplit_once('/') - .map(|(_, file)| file.trim_end_matches(".cer")) - .unwrap_or("child"); - let child_publication_point = format!("{}synthetic-child-{stem}/", issuer.rsync_base_uri); - let child_manifest = format!("{child_publication_point}child.mft"); - DiscoveredChildCaInstance { - handle: CaInstanceHandle { - depth: issuer.depth + 1, - tal_id: issuer.tal_id.clone(), - parent_manifest_rsync_uri: Some(issuer.manifest_rsync_uri.clone()), - ca_certificate: CaCertificateRef::repo_bytes(object.sha256_hex.clone()), - ca_certificate_rsync_uri: Some(object.uri.clone()), - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: child_publication_point.clone(), - manifest_rsync_uri: child_manifest, - publication_point_rsync_uri: child_publication_point, - rrdp_notification_uri: issuer.rrdp_notification_uri.clone(), - }, - discovered_from: crate::audit::DiscoveredFrom { - parent_manifest_rsync_uri: issuer.manifest_rsync_uri.clone(), - child_ca_certificate_rsync_uri: object.uri.clone(), - child_ca_certificate_sha256_hex: object.sha256_hex.clone(), - }, - child_entry_projection, - } -} - -fn child_entry_projection_from_fixture_object( - store: &RocksStore, - object: &RipeRootFinalizeObject, -) -> DiscoveredChildEntryProjection { - let child_der = store - .get_blob_bytes(&object.sha256_hex) - .expect("load child certificate bytes for projection") - .expect("child certificate bytes exist for projection"); - let child_cert = - ResourceCertificate::decode_der(&child_der).expect("decode child certificate projection"); - let child_ski = child_cert - .tbs - .extensions - .subject_key_identifier - .as_ref() - .expect("child certificate projection SKI"); - DiscoveredChildEntryProjection { - child_ski: hex::encode(child_ski), - } -} - -struct RipeRootChildEntryProfile { - count: usize, - load_der_nanos: u128, - decode_cert_nanos: u128, - build_entry_nanos: u128, -} - -fn profile_ripe_root_child_entry_build( - store: &RocksStore, - discovered_children: &[DiscoveredChildCaInstance], - validation_time: time::OffsetDateTime, -) -> Result { - let mut out = Vec::with_capacity(discovered_children.len()); - let mut load_der_nanos = 0; - let mut decode_cert_nanos = 0; - let mut build_entry_nanos = 0; - for child in discovered_children { - let load_started = std::time::Instant::now(); - let child_der = child.handle.ca_certificate_der(store)?; - load_der_nanos += load_started.elapsed().as_nanos(); - - let decode_started = std::time::Instant::now(); - let child_cert = ResourceCertificate::decode_der(child_der.as_ref()) - .map_err(|e| format!("decode child certificate for VCIR failed: {e}"))?; - decode_cert_nanos += decode_started.elapsed().as_nanos(); - - let build_started = std::time::Instant::now(); - let child_ski = child_cert - .tbs - .extensions - .subject_key_identifier - .as_ref() - .ok_or_else(|| "child certificate missing SubjectKeyIdentifier".to_string())?; - out.push(VcirChildEntry { - child_manifest_rsync_uri: child.handle.manifest_rsync_uri.clone(), - child_cert_rsync_uri: child.discovered_from.child_ca_certificate_rsync_uri.clone(), - child_cert_hash: child - .discovered_from - .child_ca_certificate_sha256_hex - .clone(), - child_ski: hex::encode(child_ski), - child_rsync_base_uri: child.handle.rsync_base_uri.clone(), - child_publication_point_rsync_uri: child.handle.publication_point_rsync_uri.clone(), - child_rrdp_notification_uri: child.handle.rrdp_notification_uri.clone(), - child_effective_ip_resources: child.handle.effective_ip_resources.clone(), - child_effective_as_resources: child.handle.effective_as_resources.clone(), - accepted_at_validation_time: PackTime::from_utc_offset_datetime(validation_time), - }); - build_entry_nanos += build_started.elapsed().as_nanos(); - } - Ok(RipeRootChildEntryProfile { - count: out.len(), - load_der_nanos, - decode_cert_nanos, - build_entry_nanos, - }) -} - -#[test] -#[ignore = "manual performance repro: requires target/ripe-root-finalize-repro repo-bytes fixture"] -fn ripe_root_finalize_repro_from_remote_fixture() { - let fixture_root = std::env::var("RPKI_RIPE_ROOT_FINALIZE_FIXTURE") - .map(std::path::PathBuf::from) - .unwrap_or_else(|_| { - std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("target/ripe-root-finalize-repro") - }); - let run_id = std::env::var("RPKI_RIPE_ROOT_FINALIZE_RUN").unwrap_or_else(|_| "0262".into()); - assert!( - fixture_root.exists(), - "fixture root missing: {}; expected copied remote fixture", - fixture_root.display() - ); - - let fixture = load_ripe_root_finalize_fixture(&fixture_root, &run_id); - let repo_bytes_db = fixture_root.join("db/repo-bytes.db"); - assert!( - repo_bytes_db.exists(), - "repo-bytes fixture missing: {}", - repo_bytes_db.display() - ); - - let store_dir = tempfile::tempdir().expect("store dir"); - let work_db = store_dir.path().join("work-db"); - let store = RocksStore::open_with_external_repo_bytes(&work_db, &repo_bytes_db) - .expect("open work-db with external repo-bytes"); - let repo_bytes = Arc::new( - store - .external_repo_bytes_ref() - .expect("external repo bytes") - .clone(), - ); - - let manifest_bytes = store - .get_blob_bytes(&fixture.manifest_sha256_hex) - .expect("load fixture manifest bytes") - .expect("fixture manifest bytes exist"); - let manifest = ManifestObject::decode_der(&manifest_bytes).expect("decode fixture manifest"); - let mut files = Vec::with_capacity(fixture.objects.len().saturating_sub(1)); - let mut child_audits = Vec::new(); - let current_ca_hash = "007ad0c291b01ede4bd60e1204074ce3f7192186a022c9577cef5d8e91d5171a"; - let current_ca_uri = "rsync://rpki.ripe.net/repository/aca/KpSo3VVK5wEHIJnHC2QHVV3d5mk.cer"; - let parent_manifest_uri = - "rsync://rpki.ripe.net/repository/aca/7DNNDzoYvgAht7joQih2Qayxcxo.mft"; - let ca = CaInstanceHandle { - depth: 1, - tal_id: "ripe-ncc".to_string(), - parent_manifest_rsync_uri: Some(parent_manifest_uri.to_string()), - ca_certificate: CaCertificateRef::repo_bytes(current_ca_hash.to_string()), - ca_certificate_rsync_uri: Some(current_ca_uri.to_string()), - effective_ip_resources: None, - effective_as_resources: None, - rsync_base_uri: fixture.publication_point_uri.clone(), - manifest_rsync_uri: fixture.manifest_uri.clone(), - publication_point_rsync_uri: fixture.publication_point_uri.clone(), - rrdp_notification_uri: Some("https://rrdp.ripe.net/notification.xml".to_string()), - }; - let mut discovered_children = Vec::new(); - let use_child_projection = std::env::var("RPKI_RIPE_ROOT_FINALIZE_USE_CHILD_PROJECTION") - .map(|value| value != "0" && value.to_ascii_lowercase() != "false") - .unwrap_or(false); - for object in &fixture.objects { - if object.uri == fixture.manifest_uri { - continue; - } - files.push(pack_file_from_fixture_object(object, &repo_bytes)); - if object.object_type == "certificate" { - child_audits.push(child_audit_from_fixture_object(object)); - if object.result == "ok" { - let child_entry_projection = use_child_projection - .then(|| child_entry_projection_from_fixture_object(&store, object)); - discovered_children.push(discovered_child_from_fixture_object( - &ca, - object, - child_entry_projection, - )); - } - } - } - if std::env::var("RPKI_RIPE_ROOT_FINALIZE_PROFILE_CHILD").is_ok() { - let profile_children = fixture - .objects - .iter() - .filter(|object| object.object_type == "certificate" && object.result == "ok") - .map(|object| discovered_child_from_fixture_object(&ca, object, None)) - .collect::>(); - let profile_started = std::time::Instant::now(); - let profile = - profile_ripe_root_child_entry_build(&store, &profile_children, fixture.validation_time) - .expect("profile child entry build"); - eprintln!( - "ripe root child-entry profile: run={} count={} total_ms={} load_der_ms={:.3} decode_cert_ms={:.3} build_entry_ms={:.3}", - run_id, - profile.count, - profile_started.elapsed().as_millis(), - profile.load_der_nanos as f64 / 1_000_000.0, - profile.decode_cert_nanos as f64 / 1_000_000.0, - profile.build_entry_nanos as f64 / 1_000_000.0, - ); - if std::env::var("RPKI_RIPE_ROOT_FINALIZE_ONLY_CHILD_PROFILE").is_ok() { - assert!(profile.count > 22_000); - return; - } - } - - let fresh_point = FreshValidatedPublicationPoint { - manifest_rsync_uri: fixture.manifest_uri.clone(), - publication_point_rsync_uri: fixture.publication_point_uri.clone(), - manifest_number_be: manifest.manifest.manifest_number.bytes_be.clone(), - this_update: PackTime::from_utc_offset_datetime(manifest.manifest.this_update), - next_update: PackTime::from_utc_offset_datetime(manifest.manifest.next_update), - verified_at: PackTime::from_utc_offset_datetime(fixture.validation_time), - manifest_bytes, - files, - }; - let policy = Policy::default(); - let enable_ccr_accumulator = std::env::var("RPKI_RIPE_ROOT_FINALIZE_CCR") - .map(|value| value != "0" && value.to_ascii_lowercase() != "false") - .unwrap_or(true); - let runner = Rpkiv1PublicationPointRunner { - store: &store, - policy: &policy, - http_fetcher: &NeverHttpFetcher, - rsync_fetcher: &FailingRsyncFetcher, - validation_time: fixture.validation_time, - timing: None, - download_log: None, - replay_archive_index: None, - replay_delta_index: None, - rrdp_dedup: false, - rrdp_repo_cache: Mutex::new(HashMap::new()), - rsync_dedup: false, - rsync_repo_cache: Mutex::new(HashMap::new()), - current_repo_index: None, - repo_sync_runtime: None, - parallel_phase2_config: None, - parallel_roa_worker_pool: None, - ccr_accumulator: enable_ccr_accumulator - .then(|| Mutex::new(CcrAccumulator::new(Vec::new()))), - persist_vcir: true, - enable_roa_validation_cache: false, - enable_child_certificate_validation_cache: true, - publication_point_cache_observe_only: false, - enable_publication_point_validation_cache: true, - }; - - eprintln!( - "ripe root finalize repro setup: run={} ccr_accumulator={} child_projection={} objects={} files={} child_audits={} discovered_children={}", - run_id, - enable_ccr_accumulator, - use_child_projection, - fixture.objects.len(), - fresh_point.files.len(), - child_audits.len(), - discovered_children.len() - ); - let started = std::time::Instant::now(); - let output = runner - .finalize_fresh_publication_point_from_reducer( - &ca, - &fresh_point, - Vec::new(), - empty_objects_output(), - child_audits, - discovered_children, - Some("rrdp"), - Some("rrdp_ok"), - 0, - None, - ) - .expect("finalize fixture publication point"); - let finalize_ms = started.elapsed().as_millis(); - eprintln!( - "ripe root finalize repro timing: run={} finalize_ms={} snapshot_pack_ms={} persist_vcir_ms={} build_vcir_ms={} child_entries_ms={} related_artifacts_ms={} replace_vcir_ms={} replace_vcir_encode_ms={} replace_vcir_write_batch_ms={} ccr_projection_build_ms={} audit_build_ms={}", - run_id, - finalize_ms, - output.snapshot_pack_ms, - output.persist_vcir_ms, - output.persist_vcir_timing.build_vcir_ms, - output.persist_vcir_timing.build_vcir.child_entries_ms, - output.persist_vcir_timing.build_vcir.related_artifacts_ms, - output.persist_vcir_timing.replace_vcir_ms, - output.persist_vcir_timing.replace_vcir.vcir_encode_ms, - output.persist_vcir_timing.replace_vcir.write_batch_ms, - output.ccr_projection_build_ms, - output.audit_build_ms, - ); - eprintln!( - "ripe root finalize repro result: audit_objects={} discovered_children={} ccr_manifest_count={}", - output.result.audit.objects.len(), - output.result.discovered_children.len(), - runner - .ccr_accumulator_snapshot() - .map(|snapshot| snapshot.manifest_count()) - .unwrap_or(0), - ); - - assert!(fresh_point.files.len() > 22_000); - assert!(output.result.discovered_children.len() > 22_000); - if enable_ccr_accumulator { - assert_eq!( - runner - .ccr_accumulator_snapshot() - .expect("ccr snapshot") - .manifest_count(), - 1 - ); - } -} +include!("tests/helpers.rs"); +include!("tests/cache_basics.rs"); +include!("tests/cache_behaviour.rs"); +include!("tests/publication_cache.rs"); +include!("tests/runner_behaviour.rs"); +include!("tests/projection.rs"); +include!("tests/projection_tail.rs"); +include!("tests/fixture_repro.rs"); diff --git a/crates/panda-rpki-validator/src/validation/tree_runner/tests/cache_basics.rs b/crates/panda-rpki-validator/src/validation/tree_runner/tests/cache_basics.rs new file mode 100644 index 0000000..184220b --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_runner/tests/cache_basics.rs @@ -0,0 +1,897 @@ +#[test] +fn never_http_fetcher_returns_error() { + let f = NeverHttpFetcher; + let err = f.fetch("https://example.test/").unwrap_err(); + assert!(err.contains("disabled"), "{err}"); +} + +#[test] +fn kind_from_rsync_uri_classifies_known_extensions() { + assert_eq!( + kind_from_rsync_uri("rsync://example.test/x.crl"), + AuditObjectKind::Crl + ); + assert_eq!( + kind_from_rsync_uri("rsync://example.test/x.cer"), + AuditObjectKind::Certificate + ); + assert_eq!( + kind_from_rsync_uri("rsync://example.test/x.roa"), + AuditObjectKind::Roa + ); + assert_eq!( + kind_from_rsync_uri("rsync://example.test/x.asa"), + AuditObjectKind::Aspa + ); + assert_eq!( + kind_from_rsync_uri("rsync://example.test/x.bin"), + AuditObjectKind::Other + ); +} + +#[test] +fn build_vcir_local_outputs_prefers_cached_outputs() { + let pack = dummy_pack_with_files(vec![]); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(vec![1]), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + let cached = vec![VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: pack.next_update.clone(), + source_object_uri: "rsync://example.test/repo/issuer/a.roa".to_string(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: sha256_32(b"cached-roa"), + source_ee_cert_hash: sha256_32(b"cached-ee"), + payload: VcirLocalOutputPayload::Vrp { + asn: 64500, + afi: RoaAfi::Ipv4, + prefix_len: 24, + addr: ipv4_addr([203, 0, 113, 0]), + max_length: 24, + }, + rule_hash: sha256_32(b"cached-rule"), + }]; + let mut objects = crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: cached.clone(), + warnings: Vec::new(), + stats: crate::validation::objects::ObjectsStats::default(), + audit: Vec::new(), + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + }; + let outputs = + take_or_build_vcir_local_outputs(&ca, &pack, &mut objects).expect("reuse cached outputs"); + assert_eq!(outputs, cached); + assert!(objects.local_outputs_cache.is_empty()); +} + +#[test] +fn persist_vcir_non_repository_evidence_stores_current_ca_cert_only() { + let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests(); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + let objects = crate::validation::objects::process_publication_point_snapshot_for_issuer( + &pack, + &Policy::default(), + issuer_ca_der.as_slice(), + Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + issuer_ca.tbs.extensions.ip_resources.as_ref(), + issuer_ca.tbs.extensions.as_resources.as_ref(), + validation_time, + None, + ); + assert!( + !objects.local_outputs_cache.is_empty(), + "expected local outputs from signed objects" + ); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der.clone()), + ca_certificate_rsync_uri: Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(), + ), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + persist_vcir_non_repository_evidence(&store, &ca).expect("persist embedded evidence"); + + let issuer_hash = sha256_hex(&issuer_ca_der); + let issuer_entry = store + .get_raw_by_hash_entry(&issuer_hash) + .expect("load issuer raw entry") + .expect("issuer raw entry present"); + assert!( + issuer_entry + .origin_uris + .iter() + .any(|uri| uri.ends_with("BfycW4hQb3wNP4YsiJW-1n6fjro.cer")) + ); + let first_output = objects + .local_outputs_cache + .first() + .expect("first local output"); + assert!( + store + .get_raw_by_hash_entry(&first_output.source_ee_cert_hash_hex()) + .expect("load source ee raw") + .is_none() + ); +} + +#[test] +fn build_router_key_local_outputs_encodes_router_key_payloads() { + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + let outputs = build_router_key_local_outputs( + &ca, + &[RouterKeyPayload { + as_id: 64496, + ski: vec![0x11; 20], + spki_der: vec![0x30, 0x00], + source_object_uri: "rsync://example.test/repo/issuer/router.cer".to_string(), + source_object_hash: "11".repeat(32), + source_ee_cert_hash: "11".repeat(32), + item_effective_until: PackTime { + rfc3339_utc: "2026-12-31T00:00:00Z".to_string(), + }, + }], + ); + assert_eq!(outputs.len(), 1); + assert_eq!(outputs[0].output_type, VcirOutputType::RouterKey); + assert_eq!( + outputs[0].source_object_type, + VcirSourceObjectType::RouterKey + ); + assert!(outputs[0].payload_json().contains("spki_der_base64")); +} + +#[test] +fn build_vcir_local_outputs_falls_back_to_decoding_accepted_objects_when_cache_is_empty() { + let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests(); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + let objects = crate::validation::objects::process_publication_point_snapshot_for_issuer( + &pack, + &Policy::default(), + issuer_ca_der.as_slice(), + Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + issuer_ca.tbs.extensions.ip_resources.as_ref(), + issuer_ca.tbs.extensions.as_resources.as_ref(), + validation_time, + None, + ); + let mut objects_without_cache = objects.clone(); + objects_without_cache.local_outputs_cache.clear(); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), + ca_certificate_rsync_uri: Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(), + ), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + let local_outputs = build_vcir_local_outputs(&ca, &pack, &objects_without_cache) + .expect("rebuild vcir local outputs"); + assert!(!local_outputs.is_empty()); + assert_eq!(local_outputs.len(), objects.vrps.len()); + assert!( + local_outputs + .iter() + .all(|output| output.output_type == VcirOutputType::Vrp) + ); +} + +#[test] +fn finalize_fresh_publication_point_releases_local_outputs_cache_after_persist() { + let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests(); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + let mut objects = crate::validation::objects::process_publication_point_snapshot_for_issuer( + &pack, + &Policy::default(), + issuer_ca_der.as_slice(), + Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + issuer_ca.tbs.extensions.ip_resources.as_ref(), + issuer_ca.tbs.extensions.as_resources.as_ref(), + validation_time, + None, + ); + assert!( + !objects.local_outputs_cache.is_empty(), + "expected local outputs from signed objects" + ); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &FailingRsyncFetcher, + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + }; + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der.clone()), + ca_certificate_rsync_uri: Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(), + ), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + let fresh_point = FreshValidatedPublicationPoint { + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + manifest_number_be: pack.manifest_number_be.clone(), + this_update: pack.this_update.clone(), + next_update: pack.next_update.clone(), + verified_at: pack.verified_at.clone(), + manifest_bytes: pack.manifest_bytes.clone(), + files: pack.files.clone(), + }; + + objects.local_outputs_cache.shrink_to_fit(); + let original_cache_capacity = objects.local_outputs_cache.capacity(); + let finalized = runner + .finalize_fresh_publication_point_from_reducer( + &ca, + &fresh_point, + Vec::new(), + objects, + Vec::new(), + Vec::new(), + None, + None, + 0, + None, + ) + .expect("finalize fresh publication point"); + + assert!( + finalized.result.objects.local_outputs_cache.is_empty(), + "local outputs cache should be released after VCIR persistence" + ); + assert_eq!( + finalized.result.objects.local_outputs_cache.capacity(), + 0, + "released cache should not keep its backing allocation" + ); + assert!(original_cache_capacity > 0); + + let persisted = store + .get_vcir(&pack.manifest_rsync_uri) + .expect("load persisted vcir") + .expect("persisted vcir"); + assert!( + !persisted.local_outputs.is_empty(), + "VCIR should still persist local outputs before cache release" + ); +} + +#[test] +fn persist_vcir_for_fresh_result_stores_vcir_and_replay_meta_for_real_snapshot() { + let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests(); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + let objects = crate::validation::objects::process_publication_point_snapshot_for_issuer( + &pack, + &Policy::default(), + issuer_ca_der.as_slice(), + Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + issuer_ca.tbs.extensions.ip_resources.as_ref(), + issuer_ca.tbs.extensions.as_resources.as_ref(), + validation_time, + None, + ); + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der.clone()), + ca_certificate_rsync_uri: Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(), + ), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + + let mut objects = objects; + persist_vcir_for_fresh_result_with_timing( + &store, + &Policy::default(), + &ca, + &pack, + &mut objects, + &[], + &[], + &[], + validation_time, + false, + ) + .map(|_timing| ()) + .expect("persist vcir for fresh result"); + + let vcir = store + .get_vcir(&pack.manifest_rsync_uri) + .expect("get vcir") + .expect("vcir exists"); + assert_eq!(vcir.manifest_rsync_uri, pack.manifest_rsync_uri); + assert_eq!(vcir.summary.local_vrp_count as usize, objects.vrps.len()); + assert_eq!( + vcir.ccr_manifest_projection.manifest_rsync_uri, + pack.manifest_rsync_uri + ); + assert_eq!( + vcir.ccr_manifest_projection.manifest_number_be, + pack.manifest_number_be + ); + assert_eq!( + vcir.ccr_manifest_projection.manifest_this_update, + pack.this_update + ); + assert_eq!( + vcir.ccr_manifest_projection.manifest_size, + pack.manifest_bytes.len() as u64 + ); + assert!(vcir.local_outputs.first().is_some(), "local outputs stored"); + let replay_meta = store + .get_manifest_replay_meta(&pack.manifest_rsync_uri) + .expect("get replay meta") + .expect("replay meta exists"); + assert_eq!(replay_meta.manifest_rsync_uri, pack.manifest_rsync_uri); + assert_eq!( + replay_meta.manifest_sha256, + sha2::Sha256::digest(&pack.manifest_bytes).to_vec() + ); +} + +#[test] +fn build_vcir_ccr_manifest_projection_from_fresh_real_snapshot_matches_manifest_contents() { + let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests(); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), + ca_certificate_rsync_uri: Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(), + ), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + let child_discovery = + discover_children_from_fresh_snapshot_with_audit(&ca, &pack, validation_time, None) + .expect("discover children"); + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let child_entries = + build_vcir_child_entries(&store, &child_discovery.children, validation_time) + .expect("build child entries"); + + let projection = build_vcir_ccr_manifest_projection_from_fresh(&ca, &pack, &child_entries) + .expect("build ccr manifest projection"); + let manifest = ManifestObject::decode_der(&pack.manifest_bytes).expect("decode manifest"); + let expected_locations = match manifest.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .extensions + .subject_info_access + .as_ref() + .expect("manifest sia") + { + SubjectInfoAccess::Ee(ee_sia) => vec![ + crate::ccr::manifest_location::select_manifest_signed_object_location( + &pack.manifest_rsync_uri, + &ee_sia.access_descriptions, + ) + .expect("select manifest signedObject"), + ], + SubjectInfoAccess::Ca(_) => panic!("manifest ee SIA should not be CA variant"), + }; + + assert_eq!(projection.manifest_rsync_uri, pack.manifest_rsync_uri); + assert_eq!( + projection.manifest_sha256, + sha2::Sha256::digest(&pack.manifest_bytes).to_vec() + ); + assert_eq!(projection.manifest_size, pack.manifest_bytes.len() as u64); + assert_eq!( + projection.manifest_ee_aki, + manifest.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .extensions + .authority_key_identifier + .clone() + .expect("manifest aki") + ); + assert_eq!( + projection.manifest_number_be, + manifest.manifest.manifest_number.bytes_be + ); + assert_eq!(projection.manifest_this_update, pack.this_update); + assert_eq!(projection.manifest_sia_locations_der, expected_locations); + let expected_subordinate_skis = child_entries + .iter() + .map(|child| hex::decode(&child.child_ski).expect("decode child ski")) + .collect::>(); + assert_eq!(projection.subordinate_skis, expected_subordinate_skis); +} + +#[test] +fn build_vcir_child_entries_uses_projection_without_repo_bytes() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let validation_time = time::OffsetDateTime::parse( + "2026-06-26T00:00:00Z", + &time::format_description::well_known::Rfc3339, + ) + .expect("parse time"); + let child = DiscoveredChildCaInstance { + handle: CaInstanceHandle { + depth: 1, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: Some("rsync://example.test/repo/root.mft".to_string()), + ca_certificate: CaCertificateRef::repo_bytes("aa".repeat(32)), + ca_certificate_rsync_uri: Some("rsync://example.test/repo/child.cer".to_string()), + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/child/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/child/child.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/child/".to_string(), + rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), + }, + discovered_from: crate::audit::DiscoveredFrom { + parent_manifest_rsync_uri: "rsync://example.test/repo/root.mft".to_string(), + child_ca_certificate_rsync_uri: "rsync://example.test/repo/child.cer".to_string(), + child_ca_certificate_sha256_hex: "aa".repeat(32), + }, + child_entry_projection: Some(DiscoveredChildEntryProjection { + child_ski: "11".repeat(20), + }), + }; + + let entries = build_vcir_child_entries(&store, &[child], validation_time) + .expect("projection should avoid repo-bytes load"); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].child_ski, "11".repeat(20)); +} + +#[test] +fn build_vcir_related_artifacts_classifies_snapshot_files_and_audit_statuses() { + let manifest_bytes = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft", + ), + ) + .expect("read manifest fixture"); + let crl_bytes = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.crl", + ), + ) + .expect("read crl fixture"); + let pack = PublicationPointSnapshot { + format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_number_be: vec![1], + this_update: PackTime::from_utc_offset_datetime(time::OffsetDateTime::now_utc()), + next_update: PackTime::from_utc_offset_datetime( + time::OffsetDateTime::now_utc() + time::Duration::hours(1), + ), + verified_at: PackTime::from_utc_offset_datetime(time::OffsetDateTime::now_utc()), + manifest_bytes, + files: vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + crl_bytes, + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/child.cer", + vec![1u8, 2], + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/a.roa", + vec![3u8, 4], + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/a.asa", + vec![5u8, 6], + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/a.gbr", + vec![7u8, 8], + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/extra.bin", + vec![9u8], + ), + ], + }; + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(vec![0x11, 0x22]), + ca_certificate_rsync_uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()), + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let objects = crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: Vec::new(), + stats: crate::validation::objects::ObjectsStats::default(), + audit: vec![ + ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/issuer/a.roa".to_string(), + sha256_hex: sha256_hex_from_32(&pack.files[2].sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Error, + detail: Some("bad roa".to_string()), + }, + ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/issuer/a.asa".to_string(), + sha256_hex: sha256_hex_from_32(&pack.files[3].sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Skipped, + detail: Some("skipped aspa".to_string()), + }, + ], + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + }; + let artifacts = build_vcir_related_artifacts( + &store, + &ca, + &pack, + "rsync://example.test/repo/issuer/issuer.crl", + &objects, + &[], + ); + assert!( + artifacts + .iter() + .any(|artifact| artifact.artifact_role == VcirArtifactRole::Manifest) + ); + assert!( + artifacts + .iter() + .any(|artifact| artifact.artifact_role == VcirArtifactRole::TrustAnchorCert) + ); + assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() + == Some("rsync://example.test/repo/issuer/issuer.crl") + && artifact.artifact_role == VcirArtifactRole::CurrentCrl)); + assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() + == Some("rsync://example.test/repo/issuer/child.cer") + && artifact.artifact_role == VcirArtifactRole::ChildCaCert)); + assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() + == Some("rsync://example.test/repo/issuer/a.roa") + && artifact.validation_status == VcirArtifactValidationStatus::Rejected + && artifact.reject_reason.as_deref() == Some("bad roa"))); + assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() + == Some("rsync://example.test/repo/issuer/a.asa") + && artifact.validation_status == VcirArtifactValidationStatus::WarningOnly + && artifact.reject_reason.is_none())); + assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() + == Some("rsync://example.test/repo/issuer/a.gbr") + && artifact.artifact_kind == VcirArtifactKind::Gbr)); + assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() + == Some("rsync://example.test/repo/issuer/extra.bin") + && artifact.artifact_kind == VcirArtifactKind::Other)); + assert!( + !artifacts + .iter() + .any(|artifact| artifact.uri.is_none() + && artifact.sha256 == sha256_hex(b"embedded-ee")), + "embedded EE cert artifacts should no longer be persisted separately" + ); +} + +#[test] +fn select_issuer_crl_from_snapshot_reports_missing_crldp_for_self_signed_cert() { + let ta_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/ta/apnic-ta.cer"), + ) + .expect("read TA fixture"); + + let pack = dummy_pack_with_files(vec![]); + let err = select_issuer_crl_from_snapshot(&ta_der, &pack).unwrap_err(); + assert!(err.contains("CRLDistributionPoints missing"), "{err}"); +} + +#[test] +fn select_issuer_crl_from_snapshot_finds_matching_crl() { + // Use real fixtures to ensure child cert has CRLDP rsync URI and CRL exists. + let child_cert_der = + std::fs::read(std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/R-lVU1XGsAeqzV1Fv0HjOD6ZFkE.cer", + )) + .expect("read child cert fixture"); + let crl_der = + std::fs::read(std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.crl", + )) + .expect("read crl fixture"); + + let pack = dummy_pack_with_files(vec![PackFile::from_bytes_compute_sha256( + "rsync://ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.crl", + crl_der.clone(), + )]); + + let (uri, found) = + select_issuer_crl_from_snapshot(child_cert_der.as_slice(), &pack).expect("find crl"); + assert_eq!( + uri, + "rsync://ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.crl" + ); + assert_eq!(found, crl_der.as_slice()); +} + +#[test] +fn discover_children_from_fresh_pack_discovers_child_ca() { + let g = generate_chain_and_crl(); + + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/child.cer", + g.child_ca_der.clone(), + ), + ]); + + let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), + ca_certificate_rsync_uri: None, + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let now = time::OffsetDateTime::now_utc(); + let children = discover_children_from_fresh_snapshot_with_audit(&issuer, &pack, now, None) + .expect("discover children") + .children; + assert_eq!(children.len(), 1); + assert_eq!( + children[0].discovered_from.parent_manifest_rsync_uri, + issuer.manifest_rsync_uri + ); + assert_eq!( + children[0].discovered_from.child_ca_certificate_rsync_uri, + "rsync://example.test/repo/issuer/child.cer" + ); + assert_eq!( + children[0].handle.rsync_base_uri, + "rsync://example.test/repo/child/".to_string() + ); + assert_eq!( + children[0].handle.manifest_rsync_uri, + "rsync://example.test/repo/child/child.mft".to_string() + ); + assert_eq!( + children[0].handle.publication_point_rsync_uri, + "rsync://example.test/repo/child/".to_string() + ); + assert_eq!( + children[0].handle.rrdp_notification_uri.as_deref(), + Some("https://example.test/notification.xml") + ); +} + +#[test] +fn discover_children_child_certificate_cache_reuses_successful_child_ca() { + let g = generate_chain_and_crl(); + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/child.cer", + g.child_ca_der.clone(), + ), + ]); + let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), + ca_certificate_rsync_uri: None, + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let cache_context = ChildCertificateValidationCacheContext { + store: &store, + issuer_ca_sha256: issuer.ca_certificate_sha256_32().unwrap(), + ca_validation_context_digest: ca_validation_context_digest_for_ca(&issuer), + policy_fingerprint: publication_point_cache_policy_fingerprint(&policy), + }; + let validation_time = time::OffsetDateTime::now_utc(); + + let first = discover_children_from_fresh_snapshot_with_audit_cached( + &issuer, + &pack, + validation_time, + None, + Some(cache_context), + ) + .expect("first discovery writes cache"); + assert_eq!(first.children.len(), 1); + + let child_file = pack + .files + .iter() + .find(|file| file.rsync_uri.ends_with("child.cer")) + .expect("child file"); + let cache_key = child_certificate_cache_key_sha256_hex( + &child_file.rsync_uri, + &child_file.sha256, + &cache_context.issuer_ca_sha256, + &cache_context.ca_validation_context_digest, + &cache_context.policy_fingerprint, + ); + assert!( + store + .get_child_certificate_cache_projection(&cache_key) + .expect("get projection") + .is_some() + ); + + let timing = TimingHandle::new(crate::analysis::timing::TimingMeta { + recorded_at_utc_rfc3339: "2026-06-24T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-06-24T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + let second = discover_children_from_fresh_snapshot_with_audit_cached( + &issuer, + &pack, + validation_time, + Some(&timing), + Some(cache_context), + ) + .expect("second discovery reuses cache"); + assert_eq!(second.children.len(), 1); + assert_eq!( + second.children[0].handle.manifest_rsync_uri, + first.children[0].handle.manifest_rsync_uri + ); + assert!(second.audits.iter().any(|audit| { + audit + .detail + .as_deref() + .unwrap_or("") + .contains("child certificate validation cache") + })); + let counts = timing.counts_snapshot(); + assert_eq!( + counts.get("child_certificate_cache_hit_ca").copied(), + Some(1) + ); + assert_eq!( + counts + .get("child_certificate_cache_batch_lookup_publication_points") + .copied(), + Some(1) + ); + assert_eq!( + counts + .get("child_certificate_cache_batch_lookup_entries") + .copied(), + Some(1) + ); + assert_eq!( + counts + .get("child_certificate_der_load_fresh_count") + .copied(), + Some(0) + ); +} diff --git a/crates/panda-rpki-validator/src/validation/tree_runner/tests/cache_behaviour.rs b/crates/panda-rpki-validator/src/validation/tree_runner/tests/cache_behaviour.rs new file mode 100644 index 0000000..3f6472f --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_runner/tests/cache_behaviour.rs @@ -0,0 +1,718 @@ +#[test] +fn discover_children_child_certificate_cache_rechecks_changed_valid_crl() { + let g = generate_chain_and_crl(); + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/child.cer", + g.child_ca_der.clone(), + ), + ]); + let mut changed_pack = pack.clone(); + changed_pack.files[0] = PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der_next.clone(), + ); + let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), + ca_certificate_rsync_uri: None, + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let cache_context = ChildCertificateValidationCacheContext { + store: &store, + issuer_ca_sha256: issuer.ca_certificate_sha256_32().unwrap(), + ca_validation_context_digest: ca_validation_context_digest_for_ca(&issuer), + policy_fingerprint: publication_point_cache_policy_fingerprint(&policy), + }; + let validation_time = time::OffsetDateTime::now_utc(); + discover_children_from_fresh_snapshot_with_audit_cached( + &issuer, + &pack, + validation_time, + None, + Some(cache_context), + ) + .expect("populate cache"); + let child_file = pack + .files + .iter() + .find(|file| file.rsync_uri.ends_with("child.cer")) + .expect("child file"); + let cache_key = child_certificate_cache_key_sha256_hex( + &child_file.rsync_uri, + &child_file.sha256, + &cache_context.issuer_ca_sha256, + &cache_context.ca_validation_context_digest, + &cache_context.policy_fingerprint, + ); + let projection = store + .get_child_certificate_cache_projection(&cache_key) + .expect("get child certificate cache projection") + .expect("child certificate cache projection present"); + let projected_until = + parse_snapshot_time_value(&projection.effective_until).expect("parse projected until"); + let initial_crl = crate::data_model::crl::RpkixCrl::decode_der(&g.issuer_crl_der) + .expect("decode initial crl"); + assert!( + projected_until > initial_crl.next_update.utc, + "child certificate cache projection must not be hard-capped by the issuing CRL nextUpdate" + ); + + let timing = TimingHandle::new(crate::analysis::timing::TimingMeta { + recorded_at_utc_rfc3339: "2026-06-24T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-06-24T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + let out = discover_children_from_fresh_snapshot_with_audit_cached( + &issuer, + &changed_pack, + validation_time, + Some(&timing), + Some(cache_context), + ) + .expect("changed valid CRL should recheck revocation and reuse cache"); + assert_eq!(out.children.len(), 1); + assert!( + out.audits + .iter() + .any(|audit| audit.detail.as_deref().unwrap_or("").contains("cache")) + ); + let counts = timing.counts_snapshot(); + assert_eq!( + counts + .get("child_certificate_cache_crl_recheck_hit") + .copied(), + Some(1) + ); + assert_eq!( + counts.get("child_certificate_cache_hit_ca").copied(), + Some(1) + ); +} + +#[test] +fn discover_children_child_certificate_cache_misses_when_current_crl_invalid() { + let g = generate_chain_and_crl(); + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/child.cer", + g.child_ca_der.clone(), + ), + ]); + let mut changed_pack = pack.clone(); + changed_pack.files[0] = PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + b"not-a-valid-crl".to_vec(), + ); + let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), + ca_certificate_rsync_uri: None, + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let cache_context = ChildCertificateValidationCacheContext { + store: &store, + issuer_ca_sha256: issuer.ca_certificate_sha256_32().unwrap(), + ca_validation_context_digest: ca_validation_context_digest_for_ca(&issuer), + policy_fingerprint: publication_point_cache_policy_fingerprint(&policy), + }; + let validation_time = time::OffsetDateTime::now_utc(); + discover_children_from_fresh_snapshot_with_audit_cached( + &issuer, + &pack, + validation_time, + None, + Some(cache_context), + ) + .expect("populate cache"); + + let timing = TimingHandle::new(crate::analysis::timing::TimingMeta { + recorded_at_utc_rfc3339: "2026-06-24T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-06-24T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + let out = discover_children_from_fresh_snapshot_with_audit_cached( + &issuer, + &changed_pack, + validation_time, + Some(&timing), + Some(cache_context), + ) + .expect("invalid CRL should miss cache and continue with audit error"); + assert!(out.children.is_empty()); + assert!( + out.audits + .iter() + .all(|audit| !audit.detail.as_deref().unwrap_or("").contains("cache")) + ); + let counts = timing.counts_snapshot(); + assert_eq!( + counts + .get("child_certificate_cache_miss_crl_invalid") + .copied(), + Some(1) + ); +} + +#[test] +fn discover_children_child_certificate_cache_misses_when_unchanged_crl_expired() { + let g = generate_chain_and_crl(); + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/child.cer", + g.child_ca_der.clone(), + ), + ]); + let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), + ca_certificate_rsync_uri: None, + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let cache_context = ChildCertificateValidationCacheContext { + store: &store, + issuer_ca_sha256: issuer.ca_certificate_sha256_32().unwrap(), + ca_validation_context_digest: ca_validation_context_digest_for_ca(&issuer), + policy_fingerprint: publication_point_cache_policy_fingerprint(&policy), + }; + let validation_time = time::OffsetDateTime::now_utc(); + discover_children_from_fresh_snapshot_with_audit_cached( + &issuer, + &pack, + validation_time, + None, + Some(cache_context), + ) + .expect("populate cache"); + + let timing = TimingHandle::new(crate::analysis::timing::TimingMeta { + recorded_at_utc_rfc3339: "2026-06-24T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-06-24T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + let out = discover_children_from_fresh_snapshot_with_audit_cached( + &issuer, + &pack, + validation_time + time::Duration::days(2), + Some(&timing), + Some(cache_context), + ) + .expect("expired unchanged CRL should miss cache and continue with audit error"); + assert!(out.children.is_empty()); + assert!( + out.audits + .iter() + .all(|audit| !audit.detail.as_deref().unwrap_or("").contains("cache")) + ); + let counts = timing.counts_snapshot(); + assert_eq!( + counts + .get("child_certificate_cache_miss_crl_expired") + .copied(), + Some(1) + ); +} + +#[test] +fn discover_children_with_audit_records_missing_crl_for_child_certificate() { + let now = time::OffsetDateTime::now_utc(); + + let child_ca_der = + std::fs::read(std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/R-lVU1XGsAeqzV1Fv0HjOD6ZFkE.cer", + )) + .expect("read child ca fixture"); + + // Pack contains the child CA cert but does not contain the CRL referenced by the child + // certificate CRLDistributionPoints extension. + let pack = dummy_pack_with_files(vec![PackFile::from_bytes_compute_sha256( + "rsync://ca.rg.net/rpki/RGnet-OU/R-lVU1XGsAeqzV1Fv0HjOD6ZFkE.cer", + child_ca_der, + )]); + + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(vec![1]), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + + let out = discover_children_from_fresh_snapshot_with_audit(&issuer, &pack, now, None) + .expect("discovery should succeed with audit error"); + assert_eq!(out.children.len(), 0); + assert_eq!(out.audits.len(), 1); + assert_eq!( + out.audits[0].rsync_uri, + "rsync://ca.rg.net/rpki/RGnet-OU/R-lVU1XGsAeqzV1Fv0HjOD6ZFkE.cer" + ); + assert_eq!(out.audits[0].result, AuditObjectResult::Error); + assert!( + out.audits[0] + .detail + .as_deref() + .unwrap_or("") + .contains("cannot select issuer CRL"), + "expected deterministic CRL selection failure to be recorded" + ); +} + +#[test] +fn runner_offline_rsync_fixture_produces_pack_and_warnings() { + let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + assert!(fixture_dir.is_dir(), "fixture directory must exist"); + + let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); + + // Pick a validation_time inside the fixture manifest's validity window to keep this + // test stable across wall-clock time. + let fixture_manifest_bytes = + std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); + let fixture_manifest = + crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) + .expect("decode manifest fixture"); + let validation_time = { + let this_update = fixture_manifest.manifest.this_update; + let next_update = fixture_manifest.manifest.next_update; + let candidate = this_update + time::Duration::seconds(60); + if candidate < next_update { + candidate + } else { + this_update + } + }; + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + + let runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + }; + + // For this fixture-driven smoke, we provide the correct issuer CA certificate (the CA for + // this publication point) so ROA EE certificate paths can validate. + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + + let handle = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: rsync_base_uri.clone(), + manifest_rsync_uri: manifest_rsync_uri.clone(), + publication_point_rsync_uri: rsync_base_uri.clone(), + rrdp_notification_uri: None, + }; + + let out = runner + .run_publication_point(&handle) + .expect("run publication point"); + assert_eq!(out.source, PublicationPointSource::Fresh); + let pack = out.snapshot.expect("fresh run pack"); + assert_eq!(pack.manifest_rsync_uri, manifest_rsync_uri); + assert!(pack.files.len() > 1); + assert!( + out.objects.vrps.len() > 1, + "expected to extract VRPs from ROAs" + ); + + let vcir = store + .get_vcir(&manifest_rsync_uri) + .expect("get vcir") + .expect("vcir exists after fresh run"); + assert_eq!(vcir.manifest_rsync_uri, manifest_rsync_uri); + assert_eq!(vcir.tal_id, "test-tal"); + assert!( + vcir.local_outputs + .iter() + .any(|output| output.output_type == crate::storage::VcirOutputType::Vrp), + "expected VCIR local_outputs to contain VRP entries" + ); + let first_vrp = vcir + .local_outputs + .iter() + .find(|output| output.output_type == crate::storage::VcirOutputType::Vrp) + .expect("first VCIR VRP output"); + assert!(!first_vrp.rule_hash_hex().is_empty()); + assert!(!first_vrp.output_id().is_empty()); + let replay_meta = store + .get_manifest_replay_meta(&manifest_rsync_uri) + .expect("get replay meta") + .expect("replay meta exists"); + assert_eq!(replay_meta.manifest_rsync_uri, manifest_rsync_uri); +} + +#[test] +fn runner_roa_validation_cache_reuses_vcir_outputs_on_second_fixture_run() { + let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + assert!(fixture_dir.is_dir(), "fixture directory must exist"); + + let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); + + let fixture_manifest_bytes = + std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); + let fixture_manifest = + crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) + .expect("decode manifest fixture"); + let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + + let handle = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: rsync_base_uri.clone(), + manifest_rsync_uri: manifest_rsync_uri.clone(), + publication_point_rsync_uri: rsync_base_uri, + rrdp_notification_uri: None, + }; + + let first_runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + }; + let first = first_runner + .run_publication_point(&handle) + .expect("first fresh run"); + assert!(first.objects.vrps.len() > 1); + assert_eq!(first.objects.roa_cache_stats.hit_roas, 0); + + let second_runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: true, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + }; + let second = second_runner + .run_publication_point(&handle) + .expect("second cache-enabled run"); + + assert_eq!(second.objects.vrps, first.objects.vrps); + assert_eq!(second.objects.roa_cache_stats.enabled_publication_points, 1); + assert_eq!( + second.objects.roa_cache_stats.vcir_hit_publication_points, + 1 + ); + assert_eq!( + second.objects.roa_cache_stats.vcir_miss_publication_points, + 0 + ); + assert!(second.objects.roa_cache_stats.hit_roas > 1); + assert_eq!(second.objects.roa_cache_stats.miss_roas, 0); + assert_eq!(second.objects.roa_cache_stats.blocked_roas, 0); + assert_eq!(second.objects.roa_cache_stats.fresh_roas, 0); +} + +#[test] +fn runner_publication_point_cache_observe_and_reuse_path() { + let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + assert!(fixture_dir.is_dir(), "fixture directory must exist"); + + let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); + let fixture_manifest_bytes = + std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); + let fixture_manifest = + crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) + .expect("decode manifest fixture"); + let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + let handle = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: rsync_base_uri.clone(), + manifest_rsync_uri: manifest_rsync_uri.clone(), + publication_point_rsync_uri: rsync_base_uri, + rrdp_notification_uri: None, + }; + + let first_runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: true, + enable_publication_point_validation_cache: false, + }; + let first = first_runner + .run_publication_point(&handle) + .expect("first fresh run"); + assert_eq!(first.source, PublicationPointSource::Fresh); + assert!( + store + .get_publication_point_cache_projection(&manifest_rsync_uri) + .expect("load publication-point projection") + .is_some() + ); + + let timing = crate::analysis::timing::TimingHandle::new(crate::analysis::timing::TimingMeta { + recorded_at_utc_rfc3339: "2026-01-01T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-01-01T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + let observe_runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), + validation_time, + timing: Some(timing.clone()), + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: true, + enable_publication_point_validation_cache: false, + }; + let observed = observe_runner + .run_publication_point(&handle) + .expect("observe-only run"); + assert_eq!(observed.source, PublicationPointSource::Fresh); + assert_eq!(observed.objects.vrps, first.objects.vrps); + assert_eq!( + timing + .counts_snapshot() + .get("publication_point_cache_theoretical_hits") + .copied(), + Some(1) + ); + + let cache_runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: true, + }; + let cached = cache_runner + .run_publication_point(&handle) + .expect("publication-point cache run"); + assert_eq!(cached.source, PublicationPointSource::PublicationPointCache); + assert_eq!(cached.objects.vrps, first.objects.vrps); + assert_eq!(cached.objects.aspas, first.objects.aspas); + assert_eq!( + cached.discovered_children.len(), + first.discovered_children.len() + ); + assert!(!cached.cir_cached_objects.is_empty()); +} diff --git a/crates/panda-rpki-validator/src/validation/tree_runner/tests/fixture_repro.rs b/crates/panda-rpki-validator/src/validation/tree_runner/tests/fixture_repro.rs new file mode 100644 index 0000000..b52efdf --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_runner/tests/fixture_repro.rs @@ -0,0 +1,444 @@ +#[derive(Debug, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +struct RipeRootFinalizeFixtureEvent { + event_type: String, + validation_time: String, + pp_manifest_uri: Option, + object_uri: Option, + sha256: Option, + object_type: Option, + result: Option, + reason: Option, +} + +#[derive(Debug)] +struct RipeRootFinalizeObject { + uri: String, + sha256_hex: String, + object_type: String, + result: String, + reason: Option, +} + +struct RipeRootFinalizeFixture { + manifest_uri: String, + publication_point_uri: String, + validation_time: time::OffsetDateTime, + manifest_sha256_hex: String, + objects: Vec, +} + +fn load_ripe_root_finalize_fixture( + fixture_root: &std::path::Path, + run_id: &str, +) -> RipeRootFinalizeFixture { + let events_path = fixture_root + .join(format!("run_{run_id}")) + .join("ripe-root-events.jsonl"); + let file = std::fs::File::open(&events_path) + .unwrap_or_else(|e| panic!("open fixture events {} failed: {e}", events_path.display())); + let reader = std::io::BufReader::new(file); + let mut manifest_uri = None; + let mut validation_time = None; + let mut manifest_sha256_hex = None; + let mut objects = Vec::new(); + + for line in std::io::BufRead::lines(reader) { + let line = line.expect("read fixture event line"); + let event: RipeRootFinalizeFixtureEvent = + serde_json::from_str(&line).expect("decode fixture event"); + if event.event_type == "publication_point" { + manifest_uri = event.pp_manifest_uri; + validation_time = Some( + time::OffsetDateTime::parse( + &event.validation_time, + &time::format_description::well_known::Rfc3339, + ) + .expect("parse validation_time"), + ); + continue; + } + if event.event_type != "object" { + continue; + } + let uri = event.object_uri.expect("fixture object uri"); + let sha256_hex = event.sha256.expect("fixture object sha256"); + let object_type = event.object_type.expect("fixture object type"); + let result = event.result.expect("fixture object result"); + if object_type == "manifest" { + manifest_sha256_hex = Some(sha256_hex.clone()); + } + objects.push(RipeRootFinalizeObject { + uri, + sha256_hex, + object_type, + result, + reason: event.reason, + }); + } + + let manifest_uri = manifest_uri.expect("fixture publication_point event"); + let publication_point_uri = manifest_uri + .rsplit_once('/') + .map(|(parent, _)| format!("{parent}/")) + .expect("manifest uri parent"); + RipeRootFinalizeFixture { + manifest_uri, + publication_point_uri, + validation_time: validation_time.expect("fixture validation_time"), + manifest_sha256_hex: manifest_sha256_hex.expect("fixture manifest object"), + objects, + } +} + +fn pack_file_from_fixture_object( + object: &RipeRootFinalizeObject, + repo_bytes: &Arc, +) -> PackFile { + PackFile::from_lazy_repo_bytes( + object.uri.clone(), + object.sha256_hex.clone(), + sha256_hex_to_32(&object.sha256_hex), + repo_bytes.clone(), + ) +} + +fn child_audit_from_fixture_object(object: &RipeRootFinalizeObject) -> ObjectAuditEntry { + ObjectAuditEntry { + rsync_uri: object.uri.clone(), + sha256_hex: object.sha256_hex.clone(), + kind: AuditObjectKind::Certificate, + result: match object.result.as_str() { + "ok" => AuditObjectResult::Ok, + "skipped" => AuditObjectResult::Skipped, + _ => AuditObjectResult::Error, + }, + detail: object.reason.clone(), + } +} + +fn discovered_child_from_fixture_object( + issuer: &CaInstanceHandle, + object: &RipeRootFinalizeObject, + child_entry_projection: Option, +) -> DiscoveredChildCaInstance { + let stem = object + .uri + .rsplit_once('/') + .map(|(_, file)| file.trim_end_matches(".cer")) + .unwrap_or("child"); + let child_publication_point = format!("{}synthetic-child-{stem}/", issuer.rsync_base_uri); + let child_manifest = format!("{child_publication_point}child.mft"); + DiscoveredChildCaInstance { + handle: CaInstanceHandle { + depth: issuer.depth + 1, + tal_id: issuer.tal_id.clone(), + parent_manifest_rsync_uri: Some(issuer.manifest_rsync_uri.clone()), + ca_certificate: CaCertificateRef::repo_bytes(object.sha256_hex.clone()), + ca_certificate_rsync_uri: Some(object.uri.clone()), + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: child_publication_point.clone(), + manifest_rsync_uri: child_manifest, + publication_point_rsync_uri: child_publication_point, + rrdp_notification_uri: issuer.rrdp_notification_uri.clone(), + }, + discovered_from: crate::audit::DiscoveredFrom { + parent_manifest_rsync_uri: issuer.manifest_rsync_uri.clone(), + child_ca_certificate_rsync_uri: object.uri.clone(), + child_ca_certificate_sha256_hex: object.sha256_hex.clone(), + }, + child_entry_projection, + } +} + +fn child_entry_projection_from_fixture_object( + store: &RocksStore, + object: &RipeRootFinalizeObject, +) -> DiscoveredChildEntryProjection { + let child_der = store + .get_blob_bytes(&object.sha256_hex) + .expect("load child certificate bytes for projection") + .expect("child certificate bytes exist for projection"); + let child_cert = + ResourceCertificate::decode_der(&child_der).expect("decode child certificate projection"); + let child_ski = child_cert + .tbs + .extensions + .subject_key_identifier + .as_ref() + .expect("child certificate projection SKI"); + DiscoveredChildEntryProjection { + child_ski: hex::encode(child_ski), + } +} + +struct RipeRootChildEntryProfile { + count: usize, + load_der_nanos: u128, + decode_cert_nanos: u128, + build_entry_nanos: u128, +} + +fn profile_ripe_root_child_entry_build( + store: &RocksStore, + discovered_children: &[DiscoveredChildCaInstance], + validation_time: time::OffsetDateTime, +) -> Result { + let mut out = Vec::with_capacity(discovered_children.len()); + let mut load_der_nanos = 0; + let mut decode_cert_nanos = 0; + let mut build_entry_nanos = 0; + for child in discovered_children { + let load_started = std::time::Instant::now(); + let child_der = child.handle.ca_certificate_der(store)?; + load_der_nanos += load_started.elapsed().as_nanos(); + + let decode_started = std::time::Instant::now(); + let child_cert = ResourceCertificate::decode_der(child_der.as_ref()) + .map_err(|e| format!("decode child certificate for VCIR failed: {e}"))?; + decode_cert_nanos += decode_started.elapsed().as_nanos(); + + let build_started = std::time::Instant::now(); + let child_ski = child_cert + .tbs + .extensions + .subject_key_identifier + .as_ref() + .ok_or_else(|| "child certificate missing SubjectKeyIdentifier".to_string())?; + out.push(VcirChildEntry { + child_manifest_rsync_uri: child.handle.manifest_rsync_uri.clone(), + child_cert_rsync_uri: child.discovered_from.child_ca_certificate_rsync_uri.clone(), + child_cert_hash: child + .discovered_from + .child_ca_certificate_sha256_hex + .clone(), + child_ski: hex::encode(child_ski), + child_rsync_base_uri: child.handle.rsync_base_uri.clone(), + child_publication_point_rsync_uri: child.handle.publication_point_rsync_uri.clone(), + child_rrdp_notification_uri: child.handle.rrdp_notification_uri.clone(), + child_effective_ip_resources: child.handle.effective_ip_resources.clone(), + child_effective_as_resources: child.handle.effective_as_resources.clone(), + accepted_at_validation_time: PackTime::from_utc_offset_datetime(validation_time), + }); + build_entry_nanos += build_started.elapsed().as_nanos(); + } + Ok(RipeRootChildEntryProfile { + count: out.len(), + load_der_nanos, + decode_cert_nanos, + build_entry_nanos, + }) +} + +#[test] +#[ignore = "manual performance repro: requires target/ripe-root-finalize-repro repo-bytes fixture"] +fn ripe_root_finalize_repro_from_remote_fixture() { + let fixture_root = std::env::var("RPKI_RIPE_ROOT_FINALIZE_FIXTURE") + .map(std::path::PathBuf::from) + .unwrap_or_else(|_| { + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("target/ripe-root-finalize-repro") + }); + let run_id = std::env::var("RPKI_RIPE_ROOT_FINALIZE_RUN").unwrap_or_else(|_| "0262".into()); + assert!( + fixture_root.exists(), + "fixture root missing: {}; expected copied remote fixture", + fixture_root.display() + ); + + let fixture = load_ripe_root_finalize_fixture(&fixture_root, &run_id); + let repo_bytes_db = fixture_root.join("db/repo-bytes.db"); + assert!( + repo_bytes_db.exists(), + "repo-bytes fixture missing: {}", + repo_bytes_db.display() + ); + + let store_dir = tempfile::tempdir().expect("store dir"); + let work_db = store_dir.path().join("work-db"); + let store = RocksStore::open_with_external_repo_bytes(&work_db, &repo_bytes_db) + .expect("open work-db with external repo-bytes"); + let repo_bytes = Arc::new( + store + .external_repo_bytes_ref() + .expect("external repo bytes") + .clone(), + ); + + let manifest_bytes = store + .get_blob_bytes(&fixture.manifest_sha256_hex) + .expect("load fixture manifest bytes") + .expect("fixture manifest bytes exist"); + let manifest = ManifestObject::decode_der(&manifest_bytes).expect("decode fixture manifest"); + let mut files = Vec::with_capacity(fixture.objects.len().saturating_sub(1)); + let mut child_audits = Vec::new(); + let current_ca_hash = "007ad0c291b01ede4bd60e1204074ce3f7192186a022c9577cef5d8e91d5171a"; + let current_ca_uri = "rsync://rpki.ripe.net/repository/aca/KpSo3VVK5wEHIJnHC2QHVV3d5mk.cer"; + let parent_manifest_uri = + "rsync://rpki.ripe.net/repository/aca/7DNNDzoYvgAht7joQih2Qayxcxo.mft"; + let ca = CaInstanceHandle { + depth: 1, + tal_id: "ripe-ncc".to_string(), + parent_manifest_rsync_uri: Some(parent_manifest_uri.to_string()), + ca_certificate: CaCertificateRef::repo_bytes(current_ca_hash.to_string()), + ca_certificate_rsync_uri: Some(current_ca_uri.to_string()), + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: fixture.publication_point_uri.clone(), + manifest_rsync_uri: fixture.manifest_uri.clone(), + publication_point_rsync_uri: fixture.publication_point_uri.clone(), + rrdp_notification_uri: Some("https://rrdp.ripe.net/notification.xml".to_string()), + }; + let mut discovered_children = Vec::new(); + let use_child_projection = std::env::var("RPKI_RIPE_ROOT_FINALIZE_USE_CHILD_PROJECTION") + .map(|value| value != "0" && value.to_ascii_lowercase() != "false") + .unwrap_or(false); + for object in &fixture.objects { + if object.uri == fixture.manifest_uri { + continue; + } + files.push(pack_file_from_fixture_object(object, &repo_bytes)); + if object.object_type == "certificate" { + child_audits.push(child_audit_from_fixture_object(object)); + if object.result == "ok" { + let child_entry_projection = use_child_projection + .then(|| child_entry_projection_from_fixture_object(&store, object)); + discovered_children.push(discovered_child_from_fixture_object( + &ca, + object, + child_entry_projection, + )); + } + } + } + if std::env::var("RPKI_RIPE_ROOT_FINALIZE_PROFILE_CHILD").is_ok() { + let profile_children = fixture + .objects + .iter() + .filter(|object| object.object_type == "certificate" && object.result == "ok") + .map(|object| discovered_child_from_fixture_object(&ca, object, None)) + .collect::>(); + let profile_started = std::time::Instant::now(); + let profile = + profile_ripe_root_child_entry_build(&store, &profile_children, fixture.validation_time) + .expect("profile child entry build"); + eprintln!( + "ripe root child-entry profile: run={} count={} total_ms={} load_der_ms={:.3} decode_cert_ms={:.3} build_entry_ms={:.3}", + run_id, + profile.count, + profile_started.elapsed().as_millis(), + profile.load_der_nanos as f64 / 1_000_000.0, + profile.decode_cert_nanos as f64 / 1_000_000.0, + profile.build_entry_nanos as f64 / 1_000_000.0, + ); + if std::env::var("RPKI_RIPE_ROOT_FINALIZE_ONLY_CHILD_PROFILE").is_ok() { + assert!(profile.count > 22_000); + return; + } + } + + let fresh_point = FreshValidatedPublicationPoint { + manifest_rsync_uri: fixture.manifest_uri.clone(), + publication_point_rsync_uri: fixture.publication_point_uri.clone(), + manifest_number_be: manifest.manifest.manifest_number.bytes_be.clone(), + this_update: PackTime::from_utc_offset_datetime(manifest.manifest.this_update), + next_update: PackTime::from_utc_offset_datetime(manifest.manifest.next_update), + verified_at: PackTime::from_utc_offset_datetime(fixture.validation_time), + manifest_bytes, + files, + }; + let policy = Policy::default(); + let enable_ccr_accumulator = std::env::var("RPKI_RIPE_ROOT_FINALIZE_CCR") + .map(|value| value != "0" && value.to_ascii_lowercase() != "false") + .unwrap_or(true); + let runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &FailingRsyncFetcher, + validation_time: fixture.validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: enable_ccr_accumulator + .then(|| Mutex::new(CcrAccumulator::new(Vec::new()))), + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: true, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: true, + }; + + eprintln!( + "ripe root finalize repro setup: run={} ccr_accumulator={} child_projection={} objects={} files={} child_audits={} discovered_children={}", + run_id, + enable_ccr_accumulator, + use_child_projection, + fixture.objects.len(), + fresh_point.files.len(), + child_audits.len(), + discovered_children.len() + ); + let started = std::time::Instant::now(); + let output = runner + .finalize_fresh_publication_point_from_reducer( + &ca, + &fresh_point, + Vec::new(), + empty_objects_output(), + child_audits, + discovered_children, + Some("rrdp"), + Some("rrdp_ok"), + 0, + None, + ) + .expect("finalize fixture publication point"); + let finalize_ms = started.elapsed().as_millis(); + eprintln!( + "ripe root finalize repro timing: run={} finalize_ms={} snapshot_pack_ms={} persist_vcir_ms={} build_vcir_ms={} child_entries_ms={} related_artifacts_ms={} replace_vcir_ms={} replace_vcir_encode_ms={} replace_vcir_write_batch_ms={} ccr_projection_build_ms={} audit_build_ms={}", + run_id, + finalize_ms, + output.snapshot_pack_ms, + output.persist_vcir_ms, + output.persist_vcir_timing.build_vcir_ms, + output.persist_vcir_timing.build_vcir.child_entries_ms, + output.persist_vcir_timing.build_vcir.related_artifacts_ms, + output.persist_vcir_timing.replace_vcir_ms, + output.persist_vcir_timing.replace_vcir.vcir_encode_ms, + output.persist_vcir_timing.replace_vcir.write_batch_ms, + output.ccr_projection_build_ms, + output.audit_build_ms, + ); + eprintln!( + "ripe root finalize repro result: audit_objects={} discovered_children={} ccr_manifest_count={}", + output.result.audit.objects.len(), + output.result.discovered_children.len(), + runner + .ccr_accumulator_snapshot() + .map(|snapshot| snapshot.manifest_count()) + .unwrap_or(0), + ); + + assert!(fresh_point.files.len() > 22_000); + assert!(output.result.discovered_children.len() > 22_000); + if enable_ccr_accumulator { + assert_eq!( + runner + .ccr_accumulator_snapshot() + .expect("ccr snapshot") + .manifest_count(), + 1 + ); + } +} diff --git a/crates/panda-rpki-validator/src/validation/tree_runner/tests/helpers.rs b/crates/panda-rpki-validator/src/validation/tree_runner/tests/helpers.rs new file mode 100644 index 0000000..a601f10 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_runner/tests/helpers.rs @@ -0,0 +1,803 @@ +use super::*; +use crate::data_model::oid::OID_AD_SIGNED_OBJECT; +use crate::data_model::rc::{ + AccessDescription, AsIdOrRange, AsIdentifierChoice, AsResourceSet, ResourceCertificate, +}; +use crate::data_model::roa::RoaAfi; +use crate::fetch::rsync::LocalDirRsyncFetcher; +use crate::fetch::rsync::{RsyncFetchError, RsyncFetcher}; +use crate::storage::{ + PackFile, PackTime, PublicationPointCacheProjection, + PublicationPointCacheProjectionWriteAction, RawByHashEntry, RepositoryViewEntry, + RepositoryViewState, RocksStore, ValidatedCaInstanceResult, ValidatedManifestMeta, + VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus, VcirAuditSummary, + VcirChildEntry, VcirInstanceGate, VcirLocalOutput, VcirLocalOutputPayload, VcirOutputType, + VcirRelatedArtifact, VcirSourceObjectType, VcirSummary, +}; +use crate::sync::rrdp::Fetcher; +use crate::validation::publication_point::PublicationPointSnapshot; +use crate::validation::tree::{DiscoveredChildEntryProjection, PublicationPointRunner}; + +use std::process::Command; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +fn sha256_32(input: &[u8]) -> [u8; 32] { + sha256_hex_to_32(&sha256_hex(input)) +} + +fn ipv4_addr(octets: [u8; 4]) -> [u8; 16] { + let mut addr = [0u8; 16]; + addr[..4].copy_from_slice(&octets); + addr +} + +#[test] +fn publication_point_cache_policy_fingerprint_includes_resource_validation_mode() { + let mut strict_policy = Policy::default(); + strict_policy.resource_validation_mode = ResourceValidationMode::Rfc6487; + let mut vrs_policy = Policy::default(); + vrs_policy.resource_validation_mode = ResourceValidationMode::ValidationUpdate03; + + assert_ne!( + publication_point_cache_policy_fingerprint(&strict_policy), + publication_point_cache_policy_fingerprint(&vrs_policy) + ); +} + +#[test] +fn router_asns_for_resource_mode_filters_vrs_and_rejects_strict_overclaim() { + let issuer_as = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { + min: 64500, + max: 64510, + }, + ])), + rdi: None, + }; + + let strict = router_asns_for_resource_mode( + &[64505, 64520], + Some(&issuer_as), + ResourceValidationMode::Rfc6487, + ) + .unwrap_err(); + assert!(strict.contains("not a subset"), "{strict}"); + + let vrs = router_asns_for_resource_mode( + &[64505, 64520], + Some(&issuer_as), + ResourceValidationMode::ValidationUpdate03, + ) + .expect("vrs filters router asns"); + assert_eq!(vrs, vec![64505]); +} + +struct NeverHttpFetcher; +impl Fetcher for NeverHttpFetcher { + fn fetch(&self, _uri: &str) -> Result, String> { + Err("http fetch disabled in test".to_string()) + } +} + +struct FailingRsyncFetcher; +impl RsyncFetcher for FailingRsyncFetcher { + fn fetch_objects( + &self, + _rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + Err(RsyncFetchError::Fetch("rsync disabled in test".to_string())) + } +} + +fn sample_runner_with_ccr_accumulator<'a>( + store: &'a RocksStore, + policy: &'a Policy, +) -> Rpkiv1PublicationPointRunner<'a> { + Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &FailingRsyncFetcher, + validation_time: time::OffsetDateTime::now_utc(), + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: Some(Mutex::new(CcrAccumulator::new(Vec::new()))), + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + } +} + +fn openssl_available() -> bool { + Command::new("openssl") + .arg("version") + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +struct Generated { + issuer_ca_der: Vec, + child_ca_der: Vec, + issuer_crl_der: Vec, + issuer_crl_der_next: Vec, +} + +fn run(cmd: &mut Command) { + let out = cmd.output().expect("run command"); + if !out.status.success() { + panic!( + "command failed: {:?}\nstdout={}\nstderr={}", + cmd, + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + } +} + +fn generate_chain_and_crl() -> Generated { + assert!(openssl_available(), "openssl is required for this test"); + + let td = tempfile::tempdir().expect("tempdir"); + let dir = td.path(); + + std::fs::create_dir_all(dir.join("newcerts")).expect("newcerts"); + std::fs::write(dir.join("index.txt"), b"").expect("index"); + std::fs::write(dir.join("serial"), b"1000\n").expect("serial"); + std::fs::write(dir.join("crlnumber"), b"1000\n").expect("crlnumber"); + + let cnf = format!( + r#" +[ ca ] +default_ca = CA_default + +[ CA_default ] +dir = {dir} +database = $dir/index.txt +new_certs_dir = $dir/newcerts +certificate = $dir/issuer.pem +private_key = $dir/issuer.key +serial = $dir/serial +crlnumber = $dir/crlnumber +default_md = sha256 +default_days = 365 +default_crl_days = 1 +policy = policy_any +x509_extensions = v3_issuer_ca +crl_extensions = crl_ext +unique_subject = no +copy_extensions = none + +[ policy_any ] +commonName = supplied + +[ req ] +prompt = no +distinguished_name = dn + +[ dn ] +CN = Test Issuer CA + +[ v3_issuer_ca ] +basicConstraints = critical,CA:true +keyUsage = critical, keyCertSign, cRLSign +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always +certificatePolicies = critical, 1.3.6.1.5.5.7.14.2 +subjectInfoAccess = caRepository;URI:rsync://example.test/repo/issuer/, rpkiManifest;URI:rsync://example.test/repo/issuer/issuer.mft, rpkiNotify;URI:https://example.test/notification.xml +sbgp-ipAddrBlock = critical, IPv4:10.0.0.0/8 +sbgp-autonomousSysNum = critical, AS:64496-64511 + +[ v3_child_ca ] +basicConstraints = critical,CA:true +keyUsage = critical, keyCertSign, cRLSign +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always +crlDistributionPoints = URI:rsync://example.test/repo/issuer/issuer.crl +authorityInfoAccess = caIssuers;URI:rsync://example.test/repo/issuer/issuer.cer +certificatePolicies = critical, 1.3.6.1.5.5.7.14.2 +subjectInfoAccess = caRepository;URI:rsync://example.test/repo/child/, rpkiManifest;URI:rsync://example.test/repo/child/child.mft, rpkiNotify;URI:https://example.test/notification.xml +sbgp-ipAddrBlock = critical, IPv4:10.0.0.0/16 +sbgp-autonomousSysNum = critical, AS:64496 + +[ crl_ext ] +authorityKeyIdentifier = keyid:always +"#, + dir = dir.display() + ); + std::fs::write(dir.join("openssl.cnf"), cnf.as_bytes()).expect("write cnf"); + + run(Command::new("openssl") + .arg("genrsa") + .arg("-out") + .arg(dir.join("issuer.key")) + .arg("2048")); + run(Command::new("openssl") + .arg("req") + .arg("-new") + .arg("-x509") + .arg("-sha256") + .arg("-days") + .arg("365") + .arg("-key") + .arg(dir.join("issuer.key")) + .arg("-config") + .arg(dir.join("openssl.cnf")) + .arg("-extensions") + .arg("v3_issuer_ca") + .arg("-out") + .arg(dir.join("issuer.pem"))); + + run(Command::new("openssl") + .arg("genrsa") + .arg("-out") + .arg(dir.join("child.key")) + .arg("2048")); + run(Command::new("openssl") + .arg("req") + .arg("-new") + .arg("-key") + .arg(dir.join("child.key")) + .arg("-subj") + .arg("/CN=Test Child CA") + .arg("-out") + .arg(dir.join("child.csr"))); + + run(Command::new("openssl") + .arg("ca") + .arg("-batch") + .arg("-config") + .arg(dir.join("openssl.cnf")) + .arg("-in") + .arg(dir.join("child.csr")) + .arg("-extensions") + .arg("v3_child_ca") + .arg("-out") + .arg(dir.join("child.pem"))); + + run(Command::new("openssl") + .arg("x509") + .arg("-in") + .arg(dir.join("issuer.pem")) + .arg("-outform") + .arg("DER") + .arg("-out") + .arg(dir.join("issuer.cer"))); + run(Command::new("openssl") + .arg("x509") + .arg("-in") + .arg(dir.join("child.pem")) + .arg("-outform") + .arg("DER") + .arg("-out") + .arg(dir.join("child.cer"))); + + run(Command::new("openssl") + .arg("ca") + .arg("-gencrl") + .arg("-config") + .arg(dir.join("openssl.cnf")) + .arg("-out") + .arg(dir.join("issuer.crl.pem"))); + run(Command::new("openssl") + .arg("crl") + .arg("-in") + .arg(dir.join("issuer.crl.pem")) + .arg("-outform") + .arg("DER") + .arg("-out") + .arg(dir.join("issuer.crl"))); + run(Command::new("openssl") + .arg("ca") + .arg("-gencrl") + .arg("-config") + .arg(dir.join("openssl.cnf")) + .arg("-out") + .arg(dir.join("issuer-next.crl.pem"))); + run(Command::new("openssl") + .arg("crl") + .arg("-in") + .arg(dir.join("issuer-next.crl.pem")) + .arg("-outform") + .arg("DER") + .arg("-out") + .arg(dir.join("issuer-next.crl"))); + + Generated { + issuer_ca_der: std::fs::read(dir.join("issuer.cer")).expect("read issuer der"), + child_ca_der: std::fs::read(dir.join("child.cer")).expect("read child der"), + issuer_crl_der: std::fs::read(dir.join("issuer.crl")).expect("read crl der"), + issuer_crl_der_next: std::fs::read(dir.join("issuer-next.crl")).expect("read next crl der"), + } +} + +struct GeneratedRouter { + issuer_ca_der: Vec, + router_der: Vec, + issuer_crl_der: Vec, +} + +fn generate_router_cert_with_variant(key_spec: &str, include_eku: bool) -> GeneratedRouter { + assert!(openssl_available(), "openssl is required for this test"); + + let td = tempfile::tempdir().expect("tempdir"); + let dir = td.path(); + + std::fs::create_dir_all(dir.join("newcerts")).expect("newcerts"); + std::fs::write(dir.join("index.txt"), b"").expect("index"); + std::fs::write(dir.join("serial"), b"1000\n").expect("serial"); + std::fs::write(dir.join("crlnumber"), b"1000\n").expect("crlnumber"); + + let eku_line = if include_eku { + "extendedKeyUsage = 1.3.6.1.5.5.7.3.30" + } else { + "" + }; + let cnf = format!( + r#" +[ ca ] +default_ca = CA_default + +[ CA_default ] +dir = {dir} +database = $dir/index.txt +new_certs_dir = $dir/newcerts +certificate = $dir/issuer.pem +private_key = $dir/issuer.key +serial = $dir/serial +crlnumber = $dir/crlnumber +default_md = sha256 +default_days = 365 +default_crl_days = 1 +policy = policy_any +x509_extensions = v3_issuer_ca +crl_extensions = crl_ext +unique_subject = no +copy_extensions = none + +[ policy_any ] +commonName = supplied + +[ req ] +prompt = no +distinguished_name = dn + +[ dn ] +CN = Test Issuer CA + +[ v3_issuer_ca ] +basicConstraints = critical,CA:true +keyUsage = critical, keyCertSign, cRLSign +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always +certificatePolicies = critical, 1.3.6.1.5.5.7.14.2 +subjectInfoAccess = caRepository;URI:rsync://example.test/repo/issuer/, rpkiManifest;URI:rsync://example.test/repo/issuer/issuer.mft, rpkiNotify;URI:https://example.test/notification.xml +sbgp-ipAddrBlock = critical, IPv4:10.0.0.0/8 +sbgp-autonomousSysNum = critical, AS:64496-64511 + +[ v3_router ] +keyUsage = critical, digitalSignature +{eku_line} +authorityKeyIdentifier = keyid:always +crlDistributionPoints = URI:rsync://example.test/repo/issuer/issuer.crl +authorityInfoAccess = caIssuers;URI:rsync://example.test/repo/issuer/issuer.cer +certificatePolicies = critical, 1.3.6.1.5.5.7.14.2 +sbgp-autonomousSysNum = critical, AS:64496 + +[ crl_ext ] +authorityKeyIdentifier = keyid:always +"#, + dir = dir.display(), + eku_line = eku_line, + ); + std::fs::write(dir.join("openssl.cnf"), cnf.as_bytes()).expect("write cnf"); + + run(Command::new("openssl") + .arg("genrsa") + .arg("-out") + .arg(dir.join("issuer.key")) + .arg("2048")); + run(Command::new("openssl") + .arg("req") + .arg("-new") + .arg("-x509") + .arg("-sha256") + .arg("-days") + .arg("365") + .arg("-key") + .arg(dir.join("issuer.key")) + .arg("-config") + .arg(dir.join("openssl.cnf")) + .arg("-extensions") + .arg("v3_issuer_ca") + .arg("-out") + .arg(dir.join("issuer.pem"))); + + match key_spec { + "ec-p256" => run(Command::new("openssl") + .arg("ecparam") + .arg("-name") + .arg("prime256v1") + .arg("-genkey") + .arg("-noout") + .arg("-out") + .arg(dir.join("router.key"))), + "ec-p384" => run(Command::new("openssl") + .arg("ecparam") + .arg("-name") + .arg("secp384r1") + .arg("-genkey") + .arg("-noout") + .arg("-out") + .arg(dir.join("router.key"))), + other => panic!("unsupported key_spec {other}"), + } + + run(Command::new("openssl") + .arg("req") + .arg("-new") + .arg("-key") + .arg(dir.join("router.key")) + .arg("-subj") + .arg("/CN=ROUTER-0000FC10/serialNumber=01020304") + .arg("-out") + .arg(dir.join("router.csr"))); + + run(Command::new("openssl") + .arg("ca") + .arg("-batch") + .arg("-config") + .arg(dir.join("openssl.cnf")) + .arg("-in") + .arg(dir.join("router.csr")) + .arg("-extensions") + .arg("v3_router") + .arg("-out") + .arg(dir.join("router.pem"))); + + run(Command::new("openssl") + .arg("x509") + .arg("-in") + .arg(dir.join("issuer.pem")) + .arg("-outform") + .arg("DER") + .arg("-out") + .arg(dir.join("issuer.cer"))); + run(Command::new("openssl") + .arg("x509") + .arg("-in") + .arg(dir.join("router.pem")) + .arg("-outform") + .arg("DER") + .arg("-out") + .arg(dir.join("router.cer"))); + + run(Command::new("openssl") + .arg("ca") + .arg("-gencrl") + .arg("-config") + .arg(dir.join("openssl.cnf")) + .arg("-out") + .arg(dir.join("issuer.crl.pem"))); + run(Command::new("openssl") + .arg("crl") + .arg("-in") + .arg(dir.join("issuer.crl.pem")) + .arg("-outform") + .arg("DER") + .arg("-out") + .arg(dir.join("issuer.crl"))); + + GeneratedRouter { + issuer_ca_der: std::fs::read(dir.join("issuer.cer")).expect("read issuer der"), + router_der: std::fs::read(dir.join("router.cer")).expect("read router der"), + issuer_crl_der: std::fs::read(dir.join("issuer.crl")).expect("read crl der"), + } +} +fn dummy_pack_with_files(files: Vec) -> PublicationPointSnapshot { + let now = time::OffsetDateTime::now_utc(); + PublicationPointSnapshot { + format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_number_be: vec![1], + this_update: PackTime::from_utc_offset_datetime(now), + next_update: PackTime::from_utc_offset_datetime(now + time::Duration::hours(1)), + verified_at: PackTime::from_utc_offset_datetime(now), + manifest_bytes: vec![0x01], + files, + } +} + +fn cernet_publication_point_snapshot_for_vcir_tests() +-> (PublicationPointSnapshot, Vec, time::OffsetDateTime) { + let dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/"; + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); + let manifest_bytes = std::fs::read(dir.join(manifest_file)).expect("read manifest fixture"); + let manifest = ManifestObject::decode_der(&manifest_bytes).expect("decode manifest fixture"); + let candidate = manifest.manifest.this_update + time::Duration::seconds(60); + let validation_time = if candidate < manifest.manifest.next_update { + candidate + } else { + manifest.manifest.this_update + }; + + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + + sync_publication_point( + &store, + &policy, + None, + rsync_base_uri, + &NeverHttpFetcher, + &LocalDirRsyncFetcher::new(&dir), + None, + None, + ) + .expect("sync cernet fixture"); + + let pp = crate::validation::manifest::process_manifest_publication_point( + &store, + &policy, + &manifest_rsync_uri, + rsync_base_uri, + issuer_ca_der.as_slice(), + Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + validation_time, + ) + .expect("process manifest publication point"); + + (pp.snapshot, issuer_ca_der, validation_time) +} + +fn sample_vcir_for_projection( + now: time::OffsetDateTime, + child_cert_hash: &str, +) -> ValidatedCaInstanceResult { + let manifest_uri = "rsync://example.test/repo/issuer/issuer.mft".to_string(); + let current_crl_uri = "rsync://example.test/repo/issuer/issuer.crl".to_string(); + let child_cert_uri = "rsync://example.test/repo/issuer/child.cer".to_string(); + let child_manifest_uri = "rsync://example.test/repo/child/child.mft".to_string(); + let roa_uri = "rsync://example.test/repo/issuer/a.roa".to_string(); + let aspa_uri = "rsync://example.test/repo/issuer/a.asa".to_string(); + let router_uri = "rsync://example.test/repo/issuer/router.cer".to_string(); + let manifest_hash = sha256_hex(b"manifest-bytes"); + let current_crl_hash = sha256_hex(b"current-crl-bytes"); + let roa_hash = sha256_hex(b"roa-bytes"); + let aspa_hash = sha256_hex(b"aspa-bytes"); + let router_hash = sha256_hex(b"router-bytes"); + let ee_hash = sha256_hex(b"ee-cert-bytes"); + let gate_until = PackTime::from_utc_offset_datetime(now + time::Duration::hours(1)); + let ccr_manifest_projection = VcirCcrManifestProjection { + manifest_rsync_uri: manifest_uri.clone(), + manifest_sha256: hex::decode(&manifest_hash).expect("decode manifest hash"), + manifest_size: 2048, + manifest_ee_aki: vec![0x11; 20], + manifest_number_be: vec![1], + manifest_this_update: PackTime::from_utc_offset_datetime(now), + manifest_sia_locations_der: vec![ + crate::ccr::manifest_location::encode_access_description_der(&AccessDescription { + access_method_oid: OID_AD_SIGNED_OBJECT.to_string(), + access_location: manifest_uri.clone(), + }) + .expect("encode signedObject"), + ], + subordinate_skis: vec![vec![0x33; 20]], + }; + ValidatedCaInstanceResult { + manifest_rsync_uri: manifest_uri.clone(), + parent_manifest_rsync_uri: None, + tal_id: "test-tal".to_string(), + ca_subject_name: "CN=Issuer".to_string(), + ca_ski: "11".repeat(20), + issuer_ski: "22".repeat(20), + last_successful_validation_time: PackTime::from_utc_offset_datetime(now), + current_manifest_rsync_uri: manifest_uri.clone(), + current_crl_rsync_uri: current_crl_uri.clone(), + validated_manifest_meta: ValidatedManifestMeta { + validated_manifest_number: vec![1], + validated_manifest_this_update: PackTime::from_utc_offset_datetime(now), + validated_manifest_next_update: gate_until.clone(), + }, + ccr_manifest_projection, + instance_gate: VcirInstanceGate { + manifest_next_update: gate_until.clone(), + current_crl_next_update: gate_until.clone(), + self_ca_not_after: PackTime::from_utc_offset_datetime(now + time::Duration::hours(2)), + instance_effective_until: gate_until.clone(), + }, + child_entries: vec![VcirChildEntry { + child_manifest_rsync_uri: child_manifest_uri, + child_cert_rsync_uri: child_cert_uri.clone(), + child_cert_hash: child_cert_hash.to_string(), + child_ski: "33".repeat(20), + child_rsync_base_uri: "rsync://example.test/repo/child/".to_string(), + child_publication_point_rsync_uri: "rsync://example.test/repo/child/".to_string(), + child_rrdp_notification_uri: Some("https://example.test/child-notify.xml".to_string()), + child_effective_ip_resources: None, + child_effective_as_resources: None, + accepted_at_validation_time: PackTime::from_utc_offset_datetime(now), + }], + local_outputs: vec![ + VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: PackTime::from_utc_offset_datetime( + now + time::Duration::minutes(30), + ), + source_object_uri: roa_uri.clone(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: sha256_hex_to_32(&roa_hash), + source_ee_cert_hash: sha256_hex_to_32(&ee_hash), + payload: VcirLocalOutputPayload::Vrp { + asn: 64496, + afi: RoaAfi::Ipv4, + prefix_len: 24, + addr: ipv4_addr([203, 0, 113, 0]), + max_length: 24, + }, + rule_hash: sha256_32(b"roa-rule"), + }, + VcirLocalOutput { + output_type: VcirOutputType::Aspa, + item_effective_until: PackTime::from_utc_offset_datetime( + now + time::Duration::minutes(30), + ), + source_object_uri: aspa_uri.clone(), + source_object_type: VcirSourceObjectType::Aspa, + source_object_hash: sha256_hex_to_32(&aspa_hash), + source_ee_cert_hash: sha256_hex_to_32(&ee_hash), + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: 64496, + provider_as_ids: vec![64497, 64498], + }, + rule_hash: sha256_32(b"aspa-rule"), + }, + VcirLocalOutput { + output_type: VcirOutputType::RouterKey, + item_effective_until: PackTime::from_utc_offset_datetime( + now + time::Duration::minutes(30), + ), + source_object_uri: router_uri.clone(), + source_object_type: VcirSourceObjectType::RouterKey, + source_object_hash: sha256_hex_to_32(&router_hash), + source_ee_cert_hash: sha256_hex_to_32(&router_hash), + payload: VcirLocalOutputPayload::RouterKey { + as_id: 64496, + ski: vec![0x11; 20], + spki_der: vec![0x30, 0x00], + }, + rule_hash: sha256_32(b"router-key-rule"), + }, + ], + related_artifacts: vec![ + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::Manifest, + artifact_kind: VcirArtifactKind::Mft, + uri: Some(manifest_uri.clone()), + sha256: manifest_hash, + object_type: Some("mft".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::CurrentCrl, + artifact_kind: VcirArtifactKind::Crl, + uri: Some(current_crl_uri), + sha256: current_crl_hash, + object_type: Some("crl".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::ChildCaCert, + artifact_kind: VcirArtifactKind::Cer, + uri: Some(child_cert_uri), + sha256: child_cert_hash.to_string(), + object_type: Some("cer".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: VcirArtifactKind::Roa, + uri: Some(roa_uri), + sha256: roa_hash, + object_type: Some("roa".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: VcirArtifactKind::Aspa, + uri: Some(aspa_uri), + sha256: aspa_hash, + object_type: Some("aspa".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }, + ], + summary: VcirSummary { + local_vrp_count: 1, + local_aspa_count: 1, + local_router_key_count: 1, + child_count: 1, + accepted_object_count: 4, + rejected_object_count: 0, + }, + audit_summary: VcirAuditSummary { + failed_fetch_eligible: true, + last_failed_fetch_reason: None, + warning_count: 0, + audit_flags: Vec::new(), + }, + } +} + +fn put_vcir_for_failed_fetch_reuse( + store: &RocksStore, + ca: &CaInstanceHandle, + policy: &Policy, + vcir: &ValidatedCaInstanceResult, +) { + let validation_time = vcir + .last_successful_validation_time + .parse() + .expect("parse VCIR validation time"); + let identity = failed_fetch_reuse_identity_for_fresh_result( + ca, + policy, + validation_time, + vcir.instance_gate.instance_effective_until.clone(), + ) + .expect("build VCIR failed-fetch reuse identity"); + store + .put_vcir_with_failed_fetch_reuse_identity(vcir, &identity) + .expect("put reusable VCIR"); +} + +fn sample_ca_for_failed_fetch_reuse(vcir: &ValidatedCaInstanceResult) -> CaInstanceHandle { + CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), + } +} diff --git a/crates/panda-rpki-validator/src/validation/tree_runner/tests/projection.rs b/crates/panda-rpki-validator/src/validation/tree_runner/tests/projection.rs new file mode 100644 index 0000000..cbd43c7 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_runner/tests/projection.rs @@ -0,0 +1,904 @@ +#[test] +fn project_current_instance_vcir_does_not_reuse_when_ta_context_changes() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let vcir = sample_vcir_for_projection(now, &child_cert_hash); + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let original_ca = sample_ca_for_failed_fetch_reuse(&vcir); + put_vcir_for_failed_fetch_reuse(&store, &original_ca, &policy, &vcir); + let mut changed_ca = original_ca; + changed_ca.tal_id = "different-tal".to_string(); + + let projection = project_current_instance_vcir_on_failed_fetch( + &store, + &changed_ca, + &ManifestFreshError::RepoSyncFailed { + detail: "synthetic".to_string(), + }, + &policy, + now, + ) + .expect("project VCIR"); + + assert_eq!( + projection.source, + PublicationPointSource::FailedFetchNoCache + ); + assert!(projection.objects.vrps.is_empty()); + assert!(projection.discovered_children.is_empty()); + assert!( + projection + .warnings + .iter() + .any(|warning| warning.message.contains("identity does not match")) + ); +} + +#[test] +fn project_current_instance_vcir_returns_no_output_when_instance_gate_expired() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); + let this_update = PackTime::from_utc_offset_datetime(now - time::Duration::minutes(2)); + let expired = PackTime::from_utc_offset_datetime(now - time::Duration::minutes(1)); + vcir.validated_manifest_meta.validated_manifest_this_update = this_update; + vcir.validated_manifest_meta.validated_manifest_next_update = expired.clone(); + vcir.instance_gate.manifest_next_update = expired.clone(); + vcir.instance_gate.current_crl_next_update = expired.clone(); + vcir.instance_gate.instance_effective_until = expired; + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + store.put_vcir(&vcir).expect("put vcir"); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let projection = project_current_instance_vcir_on_failed_fetch( + &store, + &ca, + &ManifestFreshError::RepoSyncFailed { + detail: "synthetic".to_string(), + }, + &Policy::default(), + now, + ) + .expect("project vcir"); + + assert_eq!( + projection.source, + PublicationPointSource::FailedFetchNoCache + ); + assert!(projection.ccr_manifest_projection.is_none()); + assert!(projection.objects.vrps.is_empty()); + assert!(projection.objects.aspas.is_empty()); + assert!(projection.discovered_children.is_empty()); +} + +#[test] +fn project_current_instance_vcir_keeps_real_fresh_validation_warning() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let vcir = sample_vcir_for_projection(now, &child_cert_hash); + + let store_dir = tempfile::tempdir().expect("store dir"); + let main_db = store_dir.path().join("work-db"); + let repo_bytes_db = store_dir.path().join("repo-bytes.db"); + let store = RocksStore::open_with_external_repo_bytes(&main_db, &repo_bytes_db) + .expect("open rocksdb with external repo bytes"); + store + .put_blob_bytes_batch(&[(child_cert_hash, b"child-cert".to_vec())]) + .expect("put child cert repo bytes"); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + let policy = Policy::default(); + put_vcir_for_failed_fetch_reuse(&store, &ca, &policy, &vcir); + + let projection = project_current_instance_vcir_on_failed_fetch( + &store, + &ca, + &ManifestFreshError::HashMismatch { + rsync_uri: "rsync://example.test/repo/issuer/a.roa".to_string(), + }, + &policy, + now, + ) + .expect("project vcir"); + + assert_eq!( + projection.source, + PublicationPointSource::VcirCurrentInstance + ); + assert!( + projection + .warnings + .iter() + .any(|warning| { warning.message.contains("manifest file hash mismatch") }) + ); + assert!( + !projection + .warnings + .iter() + .any(|warning| warning.message.contains("using latest validated result")), + "successful current-instance reuse should not emit bookkeeping warnings" + ); +} + +#[test] +fn project_current_instance_vcir_returns_no_output_when_latest_result_missing() { + let now = time::OffsetDateTime::now_utc(); + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let projection = project_current_instance_vcir_on_failed_fetch( + &store, + &ca, + &ManifestFreshError::RepoSyncFailed { + detail: "synthetic".to_string(), + }, + &Policy::default(), + now, + ) + .expect("project without cached vcir"); + + assert_eq!( + projection.source, + PublicationPointSource::FailedFetchNoCache + ); + assert!(projection.vcir.is_none()); + assert!(projection.ccr_manifest_projection.is_none()); + assert!(projection.snapshot.is_none()); + assert!(projection.objects.audit.is_empty()); + assert!(projection.discovered_children.is_empty()); + assert!(projection.warnings.iter().any(|warning| { + warning + .message + .contains("no latest validated result for current CA instance") + })); +} + +#[test] +fn project_current_instance_vcir_returns_no_output_when_latest_result_is_ineligible() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); + vcir.audit_summary.failed_fetch_eligible = false; + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + store.put_vcir(&vcir).expect("put vcir"); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let projection = project_current_instance_vcir_on_failed_fetch( + &store, + &ca, + &ManifestFreshError::RepoSyncFailed { + detail: "synthetic".to_string(), + }, + &Policy::default(), + now, + ) + .expect("project ineligible vcir"); + + assert_eq!( + projection.source, + PublicationPointSource::FailedFetchNoCache + ); + assert!(projection.vcir.is_some()); + assert!(projection.ccr_manifest_projection.is_none()); + assert!(projection.snapshot.is_none()); + assert!(projection.discovered_children.is_empty()); + assert!(projection.warnings.iter().any(|warning| { + warning + .message + .contains("latest VCIR is not marked failed-fetch eligible") + })); +} + +#[test] +fn project_current_instance_vcir_rejects_mismatched_ccr_projection_uri() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); + vcir.ccr_manifest_projection.manifest_rsync_uri = + "rsync://example.test/repo/issuer/other.mft".to_string(); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + let policy = Policy::default(); + put_vcir_for_failed_fetch_reuse(&store, &ca, &policy, &vcir); + + let err = project_current_instance_vcir_on_failed_fetch( + &store, + &ca, + &ManifestFreshError::RepoSyncFailed { + detail: "synthetic".to_string(), + }, + &policy, + now, + ) + .unwrap_err(); + + assert!( + err.contains("vcir CCR manifest projection URI mismatch"), + "{err}" + ); +} + +#[test] +fn fresh_and_reuse_paths_produce_equivalent_ccr_manifest_projection() { + let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests(); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + let child_discovery = + discover_children_from_fresh_snapshot_with_audit(&ca, &pack, validation_time, None) + .expect("discover children"); + let mut objects = empty_objects_output(); + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let (fresh_vcir, _timing) = build_vcir_from_fresh_result_with_timing( + &store, + &ca, + &pack, + &mut objects, + &[], + &child_discovery.audits, + &child_discovery.children, + validation_time, + ) + .expect("build fresh vcir"); + + let reuse_projection = reuse_ccr_manifest_projection_from_vcir(&ca, &fresh_vcir) + .expect("reuse projection from vcir"); + + assert_eq!(fresh_vcir.ccr_manifest_projection, reuse_projection); +} + +#[test] +fn append_ccr_manifest_projection_from_reuse_requires_projection_for_current_instance() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let runner = sample_runner_with_ccr_accumulator(&store, &policy); + + let err = runner + .append_ccr_manifest_projection_from_reuse(&VcirReuseProjection { + source: PublicationPointSource::VcirCurrentInstance, + vcir: None, + ccr_manifest_projection: None, + snapshot: None, + objects: empty_objects_output(), + child_audits: Vec::new(), + discovered_children: Vec::new(), + warnings: Vec::new(), + }) + .unwrap_err(); + + assert!(err.contains("missing CCR manifest projection"), "{err}"); + assert_eq!( + runner + .ccr_accumulator_snapshot() + .expect("ccr accumulator snapshot") + .manifest_count(), + 0 + ); +} + +#[test] +fn append_ccr_manifest_projection_from_reuse_skips_failed_fetch_no_cache() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let runner = sample_runner_with_ccr_accumulator(&store, &policy); + + runner + .append_ccr_manifest_projection_from_reuse(&VcirReuseProjection { + source: PublicationPointSource::FailedFetchNoCache, + vcir: None, + ccr_manifest_projection: None, + snapshot: None, + objects: empty_objects_output(), + child_audits: Vec::new(), + discovered_children: Vec::new(), + warnings: Vec::new(), + }) + .expect("failed-fetch no-cache should not append"); + + assert_eq!( + runner + .ccr_accumulator_snapshot() + .expect("ccr accumulator snapshot") + .manifest_count(), + 0 + ); +} + +#[test] +fn parse_snapshot_time_value_reports_invalid_timestamp() { + let err = parse_snapshot_time_value(&PackTime { + rfc3339_utc: "not-a-time".to_string(), + }) + .unwrap_err(); + + assert!(err.contains("invalid RFC3339 time 'not-a-time'"), "{err}"); +} + +#[test] +fn runner_roa_validation_cache_uses_projection_not_full_vcir_fallback() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); + vcir.local_outputs + .retain(|output| output.source_object_type != VcirSourceObjectType::Roa); + vcir.summary.local_vrp_count = 0; + vcir.summary.local_aspa_count = 1; + vcir.summary.local_router_key_count = 1; + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + store.put_vcir(&vcir).expect("put vcir without projection"); + assert!( + store + .get_vcir(&vcir.manifest_rsync_uri) + .expect("get vcir") + .is_some() + ); + assert!( + store + .get_roa_cache_projection(&vcir.manifest_rsync_uri) + .expect("get projection") + .is_none() + ); + + let timing = crate::analysis::timing::TimingHandle::new(crate::analysis::timing::TimingMeta { + recorded_at_utc_rfc3339: "2026-06-07T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-06-07T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + let policy = Policy::default(); + let runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &FailingRsyncFetcher, + validation_time: now, + timing: Some(timing.clone()), + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: true, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + }; + + assert!( + runner + .roa_validation_cache_view_for_fresh_point(&vcir.manifest_rsync_uri) + .is_none() + ); + let dir = tempfile::tempdir().expect("timing dir"); + let path = dir.path().join("timing.json"); + timing.write_json(&path, 10).expect("write timing"); + let report: serde_json::Value = + serde_json::from_slice(&std::fs::read(path).expect("read timing")).expect("parse timing"); + assert_eq!( + report["counts"]["roa_validation_cache_projection_missing_publication_points"], + 1 + ); + assert!( + report["phases"]["roa_validation_cache_projection_load_total"]["count"] + .as_u64() + .unwrap_or_default() + >= 1 + ); +} + +#[test] +fn build_objects_output_from_vcir_tracks_expired_and_invalid_cached_outputs() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); + + let bad_time_uri = "rsync://example.test/repo/issuer/bad-time.roa".to_string(); + let expired_uri = "rsync://example.test/repo/issuer/expired.asa".to_string(); + let bad_json_uri = "rsync://example.test/repo/issuer/bad-json.roa".to_string(); + let bad_prefix_uri = "rsync://example.test/repo/issuer/bad-prefix.roa".to_string(); + let bad_aspa_uri = "rsync://example.test/repo/issuer/bad-aspa.asa".to_string(); + + for (uri, kind) in [ + (bad_time_uri.clone(), VcirArtifactKind::Roa), + (expired_uri.clone(), VcirArtifactKind::Aspa), + (bad_json_uri.clone(), VcirArtifactKind::Roa), + (bad_prefix_uri.clone(), VcirArtifactKind::Roa), + (bad_aspa_uri.clone(), VcirArtifactKind::Aspa), + ] { + vcir.related_artifacts.push(VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: kind, + uri: Some(uri.clone()), + sha256: sha256_hex(uri.as_bytes()), + object_type: Some( + match kind { + VcirArtifactKind::Aspa => "aspa", + _ => "roa", + } + .to_string(), + ), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }); + } + + vcir.local_outputs.push(VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: PackTime { + rfc3339_utc: "bad-time-value".to_string(), + }, + source_object_uri: bad_time_uri.clone(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: sha256_32(b"bad-time-src"), + source_ee_cert_hash: sha256_32(b"bad-time-ee"), + payload: VcirLocalOutputPayload::Vrp { + asn: 64496, + afi: RoaAfi::Ipv4, + prefix_len: 24, + addr: ipv4_addr([203, 0, 113, 0]), + max_length: 24, + }, + rule_hash: sha256_32(b"bad-time-rule"), + }); + vcir.local_outputs.push(VcirLocalOutput { + output_type: VcirOutputType::Aspa, + item_effective_until: PackTime::from_utc_offset_datetime(now - time::Duration::minutes(1)), + source_object_uri: expired_uri.clone(), + source_object_type: VcirSourceObjectType::Aspa, + source_object_hash: sha256_32(b"expired-src"), + source_ee_cert_hash: sha256_32(b"expired-ee"), + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: 64500, + provider_as_ids: vec![64501], + }, + rule_hash: sha256_32(b"expired-rule"), + }); + vcir.local_outputs.push(VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(5)), + source_object_uri: bad_json_uri.clone(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: sha256_32(b"bad-json-src"), + source_ee_cert_hash: sha256_32(b"bad-json-ee"), + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: 64510, + provider_as_ids: vec![64511], + }, + rule_hash: sha256_32(b"bad-json-rule"), + }); + vcir.local_outputs.push(VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(5)), + source_object_uri: bad_prefix_uri.clone(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: sha256_32(b"bad-prefix-src"), + source_ee_cert_hash: sha256_32(b"bad-prefix-ee"), + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: 64512, + provider_as_ids: vec![64513], + }, + rule_hash: sha256_32(b"bad-prefix-rule"), + }); + vcir.local_outputs.push(VcirLocalOutput { + output_type: VcirOutputType::Aspa, + item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(5)), + source_object_uri: bad_aspa_uri.clone(), + source_object_type: VcirSourceObjectType::Aspa, + source_object_hash: sha256_32(b"bad-aspa-src"), + source_ee_cert_hash: sha256_32(b"bad-aspa-ee"), + payload: VcirLocalOutputPayload::Vrp { + asn: 64520, + afi: RoaAfi::Ipv4, + prefix_len: 24, + addr: ipv4_addr([198, 51, 100, 0]), + max_length: 24, + }, + rule_hash: sha256_32(b"bad-aspa-rule"), + }); + + let mut warnings = Vec::new(); + let output = build_objects_output_from_vcir(&vcir, now, &mut warnings); + + assert_eq!(output.vrps.len(), 1); + assert_eq!(output.aspas.len(), 1); + assert_eq!(output.stats.roa_total, 4); + assert_eq!(output.stats.roa_ok, 1); + assert_eq!(output.stats.aspa_total, 3); + assert_eq!(output.stats.aspa_ok, 1); + assert!(warnings.iter().any(|warning| { + warning + .message + .contains("cached local output has invalid item_effective_until") + })); + assert!(warnings.iter().any(|warning| { + warning + .message + .contains("cached ROA local output parse failed") + })); + assert!(warnings.iter().any(|warning| { + warning + .message + .contains("cached ASPA local output parse failed") + })); + assert!(output.audit.iter().any(|entry| { + entry.rsync_uri == expired_uri + && matches!(entry.result, AuditObjectResult::Skipped) + && entry.detail.as_deref() == Some("skipped: cached local output expired") + })); + assert!(output.audit.iter().any(|entry| { + entry.rsync_uri == bad_time_uri && matches!(entry.result, AuditObjectResult::Error) + })); + assert!(output.audit.iter().any(|entry| { + entry.rsync_uri == bad_prefix_uri + && matches!(entry.result, AuditObjectResult::Error) + && entry + .detail + .as_deref() + .unwrap_or("") + .contains("cached ROA local output parse failed") + })); +} + +#[test] +fn build_publication_point_audit_from_vcir_uses_vcir_metadata_and_overlays_child_and_object_audits() +{ + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); + vcir.related_artifacts + .retain(|artifact| artifact.artifact_role != VcirArtifactRole::Manifest); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), + }; + let runner_warnings = vec![Warning::new("runner warning")]; + let objects = crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: vec![Warning::new("objects warning")], + stats: crate::validation::objects::ObjectsStats::default(), + audit: vec![ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/issuer/a.roa".to_string(), + sha256_hex: sha256_hex(b"override-roa"), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Error, + detail: Some("overridden from object audit".to_string()), + }], + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + }; + let child_audits = vec![ObjectAuditEntry { + rsync_uri: vcir.child_entries[0].child_cert_rsync_uri.clone(), + sha256_hex: vcir.child_entries[0].child_cert_hash.clone(), + kind: AuditObjectKind::Certificate, + result: AuditObjectResult::Ok, + detail: Some("restored child CA instance from VCIR".to_string()), + }]; + + let audit = build_publication_point_audit_from_vcir( + &ca, + PublicationPointSource::VcirCurrentInstance, + Some("rsync"), + Some("rrdp_failed_rsync_failed"), + Some(456), + Some("rsync failed"), + Some(&vcir), + None, + &runner_warnings, + &objects, + &child_audits, + &[], + ); + + assert_eq!(audit.source, "vcir_current_instance"); + assert_eq!(audit.repo_sync_source.as_deref(), Some("rsync")); + assert_eq!( + audit.repo_sync_phase.as_deref(), + Some("rrdp_failed_rsync_failed") + ); + assert_eq!(audit.repo_sync_duration_ms, Some(456)); + assert_eq!(audit.repo_sync_error.as_deref(), Some("rsync failed")); + assert_eq!(audit.repo_terminal_state, "fallback_current_instance"); + assert_eq!(audit.objects[0].rsync_uri, vcir.current_manifest_rsync_uri); + assert_eq!(audit.objects[0].kind, AuditObjectKind::Manifest); + assert_eq!( + audit.this_update_rfc3339_utc, + vcir.validated_manifest_meta + .validated_manifest_this_update + .rfc3339_utc + ); + assert_eq!( + audit.next_update_rfc3339_utc, + vcir.validated_manifest_meta + .validated_manifest_next_update + .rfc3339_utc + ); + assert_eq!( + audit.verified_at_rfc3339_utc, + vcir.last_successful_validation_time.rfc3339_utc + ); + assert_eq!(audit.warnings.len(), 2); + assert!(audit.objects.iter().any(|entry| { + entry.rsync_uri == "rsync://example.test/repo/issuer/a.roa" + && matches!(entry.result, AuditObjectResult::Error) + && entry.detail.as_deref() == Some("overridden from object audit") + })); + assert!(audit.objects.iter().any(|entry| { + entry.rsync_uri == vcir.child_entries[0].child_cert_rsync_uri + && matches!(entry.result, AuditObjectResult::Ok) + })); +} + +#[test] +fn build_publication_point_audit_from_vcir_restores_reject_reason_with_legacy_fallback() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); + vcir.related_artifacts.push(VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: VcirArtifactKind::Roa, + uri: Some("rsync://example.test/repo/issuer/rejected-with-reason.roa".to_string()), + sha256: sha256_hex(b"rejected-with-reason"), + object_type: Some("roa".to_string()), + validation_status: VcirArtifactValidationStatus::Rejected, + reject_reason: Some("EE certificate path validation failed: test".to_string()), + }); + vcir.related_artifacts.push(VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: VcirArtifactKind::Roa, + uri: Some("rsync://example.test/repo/issuer/rejected-legacy.roa".to_string()), + sha256: sha256_hex(b"rejected-legacy"), + object_type: Some("roa".to_string()), + validation_status: VcirArtifactValidationStatus::Rejected, + reject_reason: None, + }); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + let objects = crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: Vec::new(), + stats: crate::validation::objects::ObjectsStats::default(), + audit: Vec::new(), + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + }; + + let audit = build_publication_point_audit_from_vcir( + &ca, + PublicationPointSource::VcirCurrentInstance, + None, + None, + None, + None, + Some(&vcir), + None, + &[], + &objects, + &[], + &[], + ); + + assert!(audit.objects.iter().any(|entry| { + entry.rsync_uri == "rsync://example.test/repo/issuer/rejected-with-reason.roa" + && matches!(entry.result, AuditObjectResult::Error) + && entry.detail.as_deref() == Some("EE certificate path validation failed: test") + })); + assert!(audit.objects.iter().any(|entry| { + entry.rsync_uri == "rsync://example.test/repo/issuer/rejected-legacy.roa" + && matches!(entry.result, AuditObjectResult::Error) + && entry.detail.as_deref() == Some(CACHED_REJECT_REASON_NOT_RECORDED) + })); + assert!( + audit.objects.iter().all(|entry| { + !matches!(entry.result, AuditObjectResult::Ok) || entry.detail.is_none() + }) + ); +} + +#[test] +fn build_publication_point_audit_from_pp_cache_projection_restores_reject_reason() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); + vcir.related_artifacts.push(VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: VcirArtifactKind::Roa, + uri: Some("rsync://example.test/repo/issuer/rejected-with-reason.roa".to_string()), + sha256: sha256_hex(b"rejected-with-reason"), + object_type: Some("roa".to_string()), + validation_status: VcirArtifactValidationStatus::Rejected, + reject_reason: Some("EE certificate path validation failed: test".to_string()), + }); + vcir.related_artifacts.push(VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: VcirArtifactKind::Roa, + uri: Some("rsync://example.test/repo/issuer/rejected-legacy.roa".to_string()), + sha256: sha256_hex(b"rejected-legacy"), + object_type: Some("roa".to_string()), + validation_status: VcirArtifactValidationStatus::Rejected, + reject_reason: None, + }); + let projection = PublicationPointCacheProjection::from_vcir_with_context( + &vcir, + "rsync://example.test/repo/issuer/".to_string(), + None, + [0x11; 32], + [0x22; 32], + [0x33; 32], + [0x44; 32], + [0x55; 32], + ) + .expect("build publication point projection"); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + let objects = crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: Vec::new(), + stats: crate::validation::objects::ObjectsStats::default(), + audit: Vec::new(), + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + }; + + let audit = build_publication_point_audit_from_publication_point_cache_projection( + &ca, + PublicationPointSource::PublicationPointCache, + None, + None, + None, + None, + &projection, + now, + &[], + &objects, + &[], + ); + + assert!(audit.objects.iter().any(|entry| { + entry.rsync_uri == "rsync://example.test/repo/issuer/rejected-with-reason.roa" + && matches!(entry.result, AuditObjectResult::Error) + && entry.detail.as_deref() == Some("EE certificate path validation failed: test") + })); + assert!(audit.objects.iter().any(|entry| { + entry.rsync_uri == "rsync://example.test/repo/issuer/rejected-legacy.roa" + && matches!(entry.result, AuditObjectResult::Error) + && entry.detail.as_deref() == Some(CACHED_REJECT_REASON_NOT_RECORDED) + })); +} diff --git a/crates/panda-rpki-validator/src/validation/tree_runner/tests/projection_tail.rs b/crates/panda-rpki-validator/src/validation/tree_runner/tests/projection_tail.rs new file mode 100644 index 0000000..99018ac --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_runner/tests/projection_tail.rs @@ -0,0 +1,515 @@ +#[test] +fn build_publication_point_audit_from_vcir_failed_no_cache_keeps_current_reject_only() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let vcir = sample_vcir_for_projection(now, &child_cert_hash); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), + }; + let objects = crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: Vec::new(), + stats: crate::validation::objects::ObjectsStats::default(), + audit: vec![ObjectAuditEntry { + rsync_uri: vcir.current_manifest_rsync_uri.clone(), + sha256_hex: sha256_hex(b"current-manifest"), + kind: AuditObjectKind::Manifest, + result: AuditObjectResult::Error, + detail: Some("manifest is not valid at validation_time".to_string()), + }], + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + }; + + let audit = build_publication_point_audit_from_vcir( + &ca, + PublicationPointSource::FailedFetchNoCache, + Some("rsync"), + Some("rsync_only_ok"), + Some(123), + None, + Some(&vcir), + None, + &[Warning::new("latest VCIR instance_gate expired")], + &objects, + &[], + &[], + ); + + assert_eq!(audit.source, "failed_fetch_no_cache"); + assert_eq!(audit.repo_terminal_state, "failed_no_cache"); + assert_eq!( + audit.this_update_rfc3339_utc, + vcir.validated_manifest_meta + .validated_manifest_this_update + .rfc3339_utc + ); + assert_eq!(audit.objects.len(), 1); + assert_eq!(audit.objects[0].rsync_uri, vcir.current_manifest_rsync_uri); + assert!(matches!(audit.objects[0].result, AuditObjectResult::Error)); + assert!( + !audit + .objects + .iter() + .any(|entry| entry.rsync_uri == "rsync://example.test/repo/issuer/a.roa"), + "failed-no-cache must not expand old VCIR related artifacts into current-run audit", + ); + assert!( + !audit + .objects + .iter() + .any(|entry| entry.rsync_uri == "rsync://example.test/repo/issuer/issuer.crl"), + "failed-no-cache must not expose old CRL as current-run CIR input", + ); +} + +#[test] +fn rejected_manifest_audit_entry_for_failed_fetch_uses_current_repo_hash() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let runner = sample_runner_with_ccr_accumulator(&store, &policy); + let manifest_uri = "rsync://example.test/repo/issuer/issuer.mft"; + let manifest_hash = sha256_hex(b"manifest-bytes"); + store + .put_blob_bytes_batch(&[(manifest_hash.clone(), b"manifest-bytes".to_vec())]) + .expect("put manifest bytes"); + store + .put_repository_view_entry(&crate::storage::RepositoryViewEntry { + rsync_uri: manifest_uri.to_string(), + current_hash: Some(manifest_hash.clone()), + repository_source: Some("rsync://example.test/repo/issuer/".to_string()), + object_type: Some("mft".to_string()), + state: crate::storage::RepositoryViewState::Present, + }) + .expect("put repository view"); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: manifest_uri.to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let entry = runner + .rejected_manifest_audit_entry_for_failed_fetch( + &ca, + &ManifestFreshError::StaleOrEarly { + this_update_rfc3339_utc: "2026-05-27T08:37:07Z".to_string(), + next_update_rfc3339_utc: "2026-05-28T10:01:07Z".to_string(), + validation_time_rfc3339_utc: "2026-05-28T10:11:00Z".to_string(), + }, + ) + .expect("rejected manifest audit entry"); + + assert_eq!(entry.rsync_uri, manifest_uri); + assert_eq!(entry.sha256_hex, manifest_hash); + assert_eq!(entry.kind, AuditObjectKind::Manifest); + assert_eq!(entry.result, AuditObjectResult::Error); + assert!( + entry + .detail + .as_deref() + .unwrap_or("") + .contains("manifest is not valid at validation_time") + ); +} + +#[test] +fn build_publication_point_audit_from_vcir_without_cached_inputs_returns_empty_listing() { + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let audit = build_publication_point_audit_from_vcir( + &ca, + PublicationPointSource::FailedFetchNoCache, + Some("rsync"), + Some("rsync_only_failed"), + Some(789), + Some("load from network failed, fallback to cache"), + None, + None, + &[Warning::new("runner warning")], + &crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: vec![Warning::new("object warning")], + stats: crate::validation::objects::ObjectsStats::default(), + audit: Vec::new(), + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + }, + &[], + &[], + ); + + assert_eq!(audit.source, "failed_fetch_no_cache"); + assert_eq!(audit.repo_sync_source.as_deref(), Some("rsync")); + assert_eq!(audit.repo_sync_phase.as_deref(), Some("rsync_only_failed")); + assert_eq!(audit.repo_sync_duration_ms, Some(789)); + assert_eq!( + audit.repo_sync_error.as_deref(), + Some("load from network failed, fallback to cache") + ); + assert_eq!(audit.repo_terminal_state, "failed_no_cache"); + assert!(audit.this_update_rfc3339_utc.is_empty()); + assert!(audit.next_update_rfc3339_utc.is_empty()); + assert!(audit.verified_at_rfc3339_utc.is_empty()); + assert_eq!(audit.warnings.len(), 2); + assert!(audit.objects.is_empty()); +} + +#[test] +fn effective_repo_sync_duration_uses_runtime_duration_for_failures() { + assert_eq!(effective_repo_sync_duration_ms(0, Some(12), false), 12); + assert_eq!(effective_repo_sync_duration_ms(5, Some(12), false), 12); + assert_eq!(effective_repo_sync_duration_ms(20, Some(12), false), 20); + assert_eq!(effective_repo_sync_duration_ms(5, None, false), 5); + assert_eq!(effective_repo_sync_duration_ms(5, Some(12), true), 5); +} + +#[test] +fn reconstruct_snapshot_from_vcir_reports_missing_manifest_and_related_raw_bytes() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); + let dup_uri = "rsync://example.test/repo/issuer/dup.roa".to_string(); + vcir.related_artifacts.push(VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: VcirArtifactKind::Roa, + uri: Some(dup_uri.clone()), + sha256: sha256_hex(b"dup-roa-1"), + object_type: Some("roa".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }); + vcir.related_artifacts.push(VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: VcirArtifactKind::Roa, + uri: Some(dup_uri.clone()), + sha256: sha256_hex(b"dup-roa-2"), + object_type: Some("roa".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }); + vcir.related_artifacts.push(VcirRelatedArtifact { + artifact_role: VcirArtifactRole::IssuerCert, + artifact_kind: VcirArtifactKind::Cer, + uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()), + sha256: sha256_hex(b"issuer-cert"), + object_type: Some("cer".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let mut warnings = Vec::new(); + assert!(reconstruct_snapshot_from_vcir(&store, &ca, &vcir, &mut warnings).is_none()); + assert!(warnings.iter().any(|warning| { + warning + .message + .contains("manifest raw bytes missing for VCIR audit reconstruction") + })); + + let manifest_bytes = b"manifest-bytes".to_vec(); + let current_crl_bytes = b"current-crl-bytes".to_vec(); + let child_bytes = b"child-cert".to_vec(); + let roa_bytes = b"roa-bytes".to_vec(); + for (bytes, uri, object_type) in [ + ( + manifest_bytes.clone(), + Some(vcir.manifest_rsync_uri.clone()), + Some("mft".to_string()), + ), + ( + current_crl_bytes, + Some(vcir.current_crl_rsync_uri.clone()), + Some("crl".to_string()), + ), + ( + child_bytes, + Some(vcir.child_entries[0].child_cert_rsync_uri.clone()), + Some("cer".to_string()), + ), + ( + roa_bytes, + Some("rsync://example.test/repo/issuer/a.roa".to_string()), + Some("roa".to_string()), + ), + ] { + let mut entry = RawByHashEntry::from_bytes(sha256_hex(&bytes), bytes); + if let Some(uri) = uri { + entry.origin_uris.push(uri); + } + entry.object_type = object_type; + entry.encoding = Some("der".to_string()); + store.put_raw_by_hash_entry(&entry).expect("put raw entry"); + } + + warnings.clear(); + let pack = reconstruct_snapshot_from_vcir(&store, &ca, &vcir, &mut warnings) + .expect("reconstruct pack with partial related artifacts"); + assert_eq!(pack.manifest_bytes, manifest_bytes); + assert_eq!(pack.files.len(), 3, "crl + child cert + roa only"); + assert!( + pack.files + .iter() + .any(|file| file.rsync_uri.ends_with("issuer.crl")) + ); + assert!( + pack.files + .iter() + .any(|file| file.rsync_uri.ends_with("child.cer")) + ); + assert!( + pack.files + .iter() + .any(|file| file.rsync_uri.ends_with("a.roa")) + ); + assert!( + !pack + .files + .iter() + .any(|file| file.rsync_uri.ends_with("issuer.cer")) + ); + assert!(warnings.iter().any(|warning| { + warning + .message + .contains("related artifact raw bytes missing for VCIR audit reconstruction") + })); +} + +#[test] +fn reconstruct_snapshot_from_vcir_reads_repo_bytes_without_raw_entries() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let vcir = sample_vcir_for_projection(now, &child_cert_hash); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let store_dir = tempfile::tempdir().expect("store dir"); + let main_db = store_dir.path().join("work-db"); + let repo_bytes_db = store_dir.path().join("repo-bytes.db"); + let store = RocksStore::open_with_external_repo_bytes(&main_db, &repo_bytes_db) + .expect("open rocksdb with external repo bytes"); + let repo_blobs = [ + b"manifest-bytes".to_vec(), + b"current-crl-bytes".to_vec(), + b"child-cert".to_vec(), + b"roa-bytes".to_vec(), + b"aspa-bytes".to_vec(), + ] + .into_iter() + .map(|bytes| (sha256_hex(&bytes), bytes)) + .collect::>(); + store + .put_blob_bytes_batch(&repo_blobs) + .expect("put external repo bytes"); + + let manifest_hash = vcir + .related_artifacts + .iter() + .find(|artifact| artifact.artifact_role == VcirArtifactRole::Manifest) + .expect("manifest artifact") + .sha256 + .clone(); + assert!( + store + .get_raw_by_hash_entry(&manifest_hash) + .expect("raw manifest lookup") + .is_none(), + "repo object bytes must not require raw_by_hash entries" + ); + + let mut warnings = Vec::new(); + let pack = reconstruct_snapshot_from_vcir(&store, &ca, &vcir, &mut warnings) + .expect("reconstruct pack from external repo bytes"); + assert_eq!(pack.manifest_bytes, b"manifest-bytes".to_vec()); + assert_eq!(pack.files.len(), 4, "crl + child cert + roa + aspa"); + assert!( + warnings.iter().all(|warning| { + !warning + .message + .contains("raw bytes missing for VCIR audit reconstruction") + }), + "external repo bytes should satisfy VCIR audit reconstruction without raw warnings" + ); +} + +#[test] +fn runner_dedup_paths_execute_with_timing_enabled() { + let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); + let fixture_manifest_bytes = + std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); + let fixture_manifest = + crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) + .expect("decode manifest fixture"); + let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + let handle = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: rsync_base_uri.clone(), + manifest_rsync_uri: manifest_rsync_uri.clone(), + publication_point_rsync_uri: rsync_base_uri.clone(), + rrdp_notification_uri: Some("https://example.test/notification.xml".to_string()), + }; + let timing = crate::analysis::timing::TimingHandle::new(crate::analysis::timing::TimingMeta { + recorded_at_utc_rfc3339: "2026-03-11T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-03-11T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + let policy_rrdp = Policy::default(); + let runner_rrdp = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy_rrdp, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), + validation_time, + timing: Some(timing.clone()), + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + }; + let first = runner_rrdp + .run_publication_point(&handle) + .expect("rrdp fallback to rsync"); + assert_eq!(first.source, PublicationPointSource::Fresh); + let second = runner_rrdp + .run_publication_point(&handle) + .expect("rrdp dedup skip"); + assert_eq!(second.source, PublicationPointSource::Fresh); + + let policy_rsync = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + let runner_rsync = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy_rsync, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), + validation_time, + timing: Some(timing), + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + }; + let third = runner_rsync + .run_publication_point(&handle) + .expect("rsync first run"); + assert_eq!(third.source, PublicationPointSource::Fresh); + let fourth = runner_rsync + .run_publication_point(&handle) + .expect("rsync dedup run"); + assert_eq!(fourth.source, PublicationPointSource::Fresh); + assert_eq!( + crate::fetch::rsync::normalize_rsync_base_uri("rsync://example.test/repo"), + "rsync://example.test/repo/" + ); +} diff --git a/crates/panda-rpki-validator/src/validation/tree_runner/tests/publication_cache.rs b/crates/panda-rpki-validator/src/validation/tree_runner/tests/publication_cache.rs new file mode 100644 index 0000000..06f48d3 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_runner/tests/publication_cache.rs @@ -0,0 +1,887 @@ +fn seed_publication_point_cache_projection( + store: &RocksStore, + policy: &Policy, + ca: &CaInstanceHandle, + validation_time: time::OffsetDateTime, +) -> ValidatedCaInstanceResult { + let child_bytes = b"child-cert".to_vec(); + let child_hash = sha256_hex(&child_bytes); + let vcir = sample_vcir_for_projection(validation_time, &child_hash); + store + .put_blob_bytes_batch(&[ + (child_hash, child_bytes), + (sha256_hex(b"manifest-bytes"), b"manifest-bytes".to_vec()), + ]) + .expect("put cache bytes"); + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: ca.manifest_rsync_uri.clone(), + current_hash: Some(sha256_hex(b"manifest-bytes")), + repository_source: Some(ca.publication_point_rsync_uri.clone()), + object_type: Some("mft".to_string()), + state: RepositoryViewState::Present, + }) + .expect("put manifest current view"); + let projection = PublicationPointCacheProjection::from_vcir_with_context( + &vcir, + ca.publication_point_rsync_uri.clone(), + ca.ca_certificate_rsync_uri.clone(), + ca.ca_certificate_sha256_32().unwrap(), + sha256_32(b"manifest-bytes"), + ta_context_digest_for_ca(ca), + ca_validation_context_digest_for_ca(ca), + publication_point_cache_policy_fingerprint(policy), + ) + .expect("build publication point projection"); + store + .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) + .expect("put publication point projection"); + vcir +} + +#[test] +fn publication_point_cache_future_notbefore_guard_detects_future_roa_notbefore_only() { + let uri = "rsync://example.test/repo/issuer/future.roa"; + let bytes = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"), + ) + .expect("read ROA fixture"); + let roa = RoaObject::decode_der(&bytes).expect("decode ROA fixture"); + let ee = &roa.signed_object.signed_data.certificates[0].resource_cert; + let file = PackFile::from_bytes_compute_sha256(uri, bytes); + let pack = dummy_pack_with_files(vec![file]); + let objects = crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: Vec::new(), + stats: Default::default(), + audit: vec![ObjectAuditEntry { + rsync_uri: uri.to_string(), + sha256_hex: sha256_hex_from_32(&pack.files[0].sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Error, + detail: Some( + "EE certificate path validation failed: certificate not valid at validation_time" + .to_string(), + ), + }], + roa_cache_stats: Default::default(), + roa_cache_object_meta: Vec::new(), + }; + + assert!(publication_point_cache_has_future_not_before_risk( + &pack, + &objects, + &[], + ee.tbs.validity_not_before - time::Duration::seconds(1), + &Policy::default(), + )); + assert!(!publication_point_cache_has_future_not_before_risk( + &pack, + &objects, + &[], + ee.tbs.validity_not_after + time::Duration::seconds(1), + &Policy::default(), + )); +} + +#[test] +fn publication_point_cache_delete_action_removes_existing_projection() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::minutes(1); + let ca = publication_point_cache_fixture_ca(); + let vcir = seed_publication_point_cache_projection(&store, &policy, &ca, validation_time); + + assert!( + store + .get_publication_point_cache_projection_cached(&ca.manifest_rsync_uri) + .expect("load projection") + .is_some() + ); + + store + .replace_vcir_manifest_replay_meta_and_projection_action( + &vcir, + None, + PublicationPointCacheProjectionWriteAction::Delete { + manifest_rsync_uri: &vcir.manifest_rsync_uri, + }, + ) + .expect("delete projection"); + + assert!( + store + .get_publication_point_cache_projection_cached(&ca.manifest_rsync_uri) + .expect("load projection after delete") + .is_none() + ); +} + +fn publication_point_cache_fixture_ca() -> CaInstanceHandle { + CaInstanceHandle { + depth: 1, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(b"ca-cert".to_vec()), + ca_certificate_rsync_uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()), + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: Some("https://example.test/notification.xml".to_string()), + } +} + +#[test] +fn runner_publication_point_cache_reuses_projection_outputs_children_and_ccr() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::minutes(1); + let ca = publication_point_cache_fixture_ca(); + let vcir = seed_publication_point_cache_projection(&store, &policy, &ca, validation_time); + let timing = crate::analysis::timing::TimingHandle::new(crate::analysis::timing::TimingMeta { + recorded_at_utc_rfc3339: "2026-01-01T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-01-01T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + let runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &FailingRsyncFetcher, + validation_time, + timing: Some(timing.clone()), + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: Some(Mutex::new(CcrAccumulator::new(Vec::new()))), + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: true, + }; + + let result = runner + .observe_or_reuse_publication_point_cache(&ca, Some("rrdp"), Some("delta"), 7, None, &[]) + .expect("cache result"); + + assert_eq!(result.source, PublicationPointSource::PublicationPointCache); + assert_eq!(result.objects.vrps.len(), 1); + assert_eq!(result.objects.aspas.len(), 1); + assert_eq!(result.objects.router_keys.len(), 1); + assert_eq!(result.discovered_children.len(), 1); + assert_eq!( + result.discovered_children[0].handle.manifest_rsync_uri, + vcir.child_entries[0].child_manifest_rsync_uri + ); + assert!(result.cir_fresh_objects.is_empty()); + assert!(!result.cir_cached_objects.is_empty()); + assert_eq!( + runner + .ccr_accumulator_snapshot() + .expect("ccr accumulator") + .manifest_count(), + 1 + ); + let counts = timing.counts_snapshot(); + assert_eq!( + counts.get("publication_point_cache_reuse_hits").copied(), + Some(1) + ); + assert_eq!( + counts + .get("publication_point_cache_outputs_reused") + .copied(), + Some(3) + ); + assert_eq!( + counts + .get("publication_point_cache_children_reused") + .copied(), + Some(1) + ); + assert_eq!( + counts + .get("publication_point_cache_related_objects_reused") + .copied(), + Some(vcir.related_artifacts.len() as u64) + ); + assert_eq!( + counts + .get("publication_point_cache_audit_objects_reused") + .copied(), + Some(result.cir_cached_objects.len() as u64) + ); + let timing_dir = tempfile::tempdir().expect("timing dir"); + let timing_path = timing_dir.path().join("timing.json"); + timing.write_json(&timing_path, 20).expect("write timing"); + let timing_json: serde_json::Value = + serde_json::from_slice(&std::fs::read(&timing_path).expect("read timing")) + .expect("parse timing"); + let phase_keys = timing_json["phases"] + .as_object() + .expect("phases") + .keys() + .map(|key| key.as_str()) + .collect::>(); + assert!(phase_keys.contains("publication_point_cache_lookup_hit_total")); + assert!(phase_keys.contains("publication_point_cache_reuse_build_total")); + assert!(phase_keys.contains("publication_point_cache_build_objects_total")); +} + +#[test] +fn publication_point_cache_restore_children_parallel_keeps_order_and_audit() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::minutes(1); + let ca = publication_point_cache_fixture_ca(); + seed_publication_point_cache_projection(&store, &policy, &ca, validation_time); + let mut projection = store + .get_publication_point_cache_projection(&ca.manifest_rsync_uri) + .expect("load projection") + .expect("projection"); + let template = projection.children[0].clone(); + let mut blobs = Vec::new(); + let mut children = Vec::new(); + for index in 0..300 { + let bytes = format!("child-cert-{index}").into_bytes(); + let child_hash = sha256_hex(&bytes); + let mut child = template.clone(); + child.child_cert_hash = child_hash.clone(); + child.child_cert_rsync_uri = format!("rsync://example.test/repo/issuer/child-{index}.cer"); + child.child_manifest_rsync_uri = + format!("rsync://example.test/repo/child-{index}/child.mft"); + child.child_publication_point_rsync_uri = + format!("rsync://example.test/repo/child-{index}/"); + child.child_rsync_base_uri = child.child_publication_point_rsync_uri.clone(); + blobs.push((child_hash, bytes)); + children.push(child); + } + projection.children = children; + store.put_blob_bytes_batch(&blobs).expect("put child blobs"); + let timing = crate::analysis::timing::TimingHandle::new(crate::analysis::timing::TimingMeta { + recorded_at_utc_rfc3339: "2026-01-01T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-01-01T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + let mut warnings = Vec::new(); + + let (restored_children, audits) = restore_children_from_publication_point_cache( + &store, + &ca, + &projection, + validation_time, + &mut warnings, + 4, + Some(&timing), + ); + + assert!(warnings.is_empty()); + assert_eq!(restored_children.len(), 300); + assert_eq!(audits.len(), 300); + assert_eq!( + restored_children[0].handle.ca_certificate_sha256_hex(), + Some(sha256_hex(b"child-cert-0").as_str()) + ); + assert_eq!( + restored_children[299] + .handle + .ca_certificate_der(&store) + .unwrap() + .as_ref(), + b"child-cert-299" + ); + assert_eq!( + restored_children[0].handle.manifest_rsync_uri, + "rsync://example.test/repo/child-0/child.mft" + ); + assert!( + audits + .iter() + .all(|audit| audit.result == AuditObjectResult::Ok) + ); + let counts = timing.counts_snapshot(); + assert_eq!( + counts + .get("publication_point_cache_restore_children_parallel_publication_points") + .copied(), + Some(1) + ); + assert_eq!( + counts + .get("publication_point_cache_restore_children_parallel_children") + .copied(), + Some(300) + ); + assert_eq!( + counts + .get("publication_point_cache_restore_children_workers_total") + .copied(), + Some(4) + ); +} + +#[test] +fn publication_point_cache_restore_children_does_not_require_child_der_bytes() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::minutes(1); + let ca = publication_point_cache_fixture_ca(); + seed_publication_point_cache_projection(&store, &policy, &ca, validation_time); + let mut projection = store + .get_publication_point_cache_projection(&ca.manifest_rsync_uri) + .expect("load projection") + .expect("projection"); + projection.children[0].child_cert_hash = "22".repeat(32); + let mut warnings = Vec::new(); + + let (restored_children, audits) = restore_children_from_publication_point_cache( + &store, + &ca, + &projection, + validation_time, + &mut warnings, + 1, + None, + ); + + assert!(warnings.is_empty()); + assert!(!restored_children.is_empty()); + assert_eq!(audits.len(), restored_children.len()); + assert_eq!( + restored_children[0].handle.ca_certificate_sha256_hex(), + Some(projection.children[0].child_cert_hash.as_str()) + ); + assert!( + restored_children[0] + .handle + .ca_certificate_der(&store) + .is_err(), + "lazy child handle should not require repo bytes until a fresh path asks for DER" + ); +} + +#[test] +fn runner_publication_point_cache_blocks_parent_policy_and_output_time_mismatch() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::minutes(1); + let ca = publication_point_cache_fixture_ca(); + seed_publication_point_cache_projection(&store, &policy, &ca, validation_time); + + let timing = crate::analysis::timing::TimingHandle::new(crate::analysis::timing::TimingMeta { + recorded_at_utc_rfc3339: "2026-01-01T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-01-01T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + let mut parent_changed_ca = ca.clone(); + parent_changed_ca.parent_manifest_rsync_uri = + Some("rsync://example.test/repo/other-parent.mft".to_string()); + let parent_changed_runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &FailingRsyncFetcher, + validation_time, + timing: Some(timing.clone()), + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: true, + }; + assert!( + parent_changed_runner + .observe_or_reuse_publication_point_cache( + &parent_changed_ca, + Some("rrdp"), + Some("delta"), + 7, + None, + &[] + ) + .is_none() + ); + assert_eq!( + timing + .counts_snapshot() + .get("publication_point_cache_miss_parent_context_mismatch") + .copied(), + Some(1) + ); + + let strict_policy = Policy { + strict: crate::policy::StrictPolicy::all(), + ..Policy::default() + }; + let policy_changed_runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &strict_policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &FailingRsyncFetcher, + validation_time, + timing: Some(timing.clone()), + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: true, + }; + assert!( + policy_changed_runner + .observe_or_reuse_publication_point_cache( + &ca, + Some("rrdp"), + Some("delta"), + 7, + None, + &[] + ) + .is_none() + ); + assert_eq!( + timing + .counts_snapshot() + .get("publication_point_cache_miss_policy_mismatch") + .copied(), + Some(1) + ); + + let output_expired_runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &FailingRsyncFetcher, + validation_time: validation_time + time::Duration::minutes(40), + timing: Some(timing.clone()), + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: true, + }; + assert!( + output_expired_runner + .observe_or_reuse_publication_point_cache( + &ca, + Some("rrdp"), + Some("delta"), + 7, + None, + &[] + ) + .is_none() + ); + assert_eq!( + timing + .counts_snapshot() + .get("publication_point_cache_miss_output_time_gate") + .copied(), + Some(1) + ); +} + +#[test] +fn runner_rsync_dedup_skips_second_sync_for_same_base() { + let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + assert!(fixture_dir.is_dir(), "fixture directory must exist"); + + let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); + + let fixture_manifest_bytes = + std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); + let fixture_manifest = + crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) + .expect("decode manifest fixture"); + let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + + let handle = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: rsync_base_uri.clone(), + manifest_rsync_uri: manifest_rsync_uri.clone(), + publication_point_rsync_uri: rsync_base_uri.clone(), + rrdp_notification_uri: None, + }; + + struct CountingRsyncFetcher { + inner: LocalDirRsyncFetcher, + calls: Arc, + } + impl RsyncFetcher for CountingRsyncFetcher { + fn fetch_objects( + &self, + rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + self.calls.fetch_add(1, Ordering::SeqCst); + self.inner.fetch_objects(rsync_base_uri) + } + } + + let calls = Arc::new(AtomicUsize::new(0)); + let rsync = CountingRsyncFetcher { + inner: LocalDirRsyncFetcher::new(&fixture_dir), + calls: calls.clone(), + }; + + let runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &rsync, + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + }; + + let first = runner.run_publication_point(&handle).expect("first run ok"); + assert_eq!(first.source, PublicationPointSource::Fresh); + + let second = runner + .run_publication_point(&handle) + .expect("second run ok"); + assert_eq!(second.source, PublicationPointSource::Fresh); + + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "rsync should be called once" + ); +} + +#[test] +fn runner_rsync_dedup_skips_second_sync_for_same_module_scope() { + let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + assert!(fixture_dir.is_dir(), "fixture directory must exist"); + + let first_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); + let second_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/sub/".to_string(); + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{first_base_uri}{manifest_file}"); + + let fixture_manifest_bytes = + std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); + let fixture_manifest = + crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) + .expect("decode manifest fixture"); + let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + + let handle = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: first_base_uri.clone(), + manifest_rsync_uri: manifest_rsync_uri.clone(), + publication_point_rsync_uri: first_base_uri.clone(), + rrdp_notification_uri: None, + }; + let second_handle = CaInstanceHandle { + rsync_base_uri: second_base_uri.clone(), + publication_point_rsync_uri: second_base_uri.clone(), + ..handle.clone() + }; + + struct ModuleScopeRsyncFetcher { + inner: LocalDirRsyncFetcher, + calls: Arc, + } + impl RsyncFetcher for ModuleScopeRsyncFetcher { + fn fetch_objects( + &self, + rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + self.calls.fetch_add(1, Ordering::SeqCst); + self.inner.fetch_objects(rsync_base_uri) + } + + fn dedup_key(&self, _rsync_base_uri: &str) -> String { + "rsync://rpki.cernet.net/repo/".to_string() + } + } + + let calls = Arc::new(AtomicUsize::new(0)); + let rsync = ModuleScopeRsyncFetcher { + inner: LocalDirRsyncFetcher::new(&fixture_dir), + calls: calls.clone(), + }; + + let runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &rsync, + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + }; + + let first = runner.run_publication_point(&handle).expect("first run ok"); + assert_eq!(first.source, PublicationPointSource::Fresh); + + let second = runner + .run_publication_point(&second_handle) + .expect("second run ok"); + assert!(matches!( + second.source, + PublicationPointSource::Fresh | PublicationPointSource::VcirCurrentInstance + )); + + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "module-scope dedup should skip second sync" + ); +} + +#[test] +fn runner_rsync_dedup_works_in_rsync_only_mode_even_when_rrdp_notify_exists() { + let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + assert!(fixture_dir.is_dir(), "fixture directory must exist"); + + let first_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); + let second_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/sub/".to_string(); + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{first_base_uri}{manifest_file}"); + + let fixture_manifest_bytes = + std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); + let fixture_manifest = + crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) + .expect("decode manifest fixture"); + let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + + let handle = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: first_base_uri.clone(), + manifest_rsync_uri: manifest_rsync_uri.clone(), + publication_point_rsync_uri: first_base_uri.clone(), + rrdp_notification_uri: Some("https://rrdp.example.test/notification.xml".to_string()), + }; + let second_handle = CaInstanceHandle { + rsync_base_uri: second_base_uri.clone(), + publication_point_rsync_uri: second_base_uri.clone(), + ..handle.clone() + }; + + struct ModuleScopeRsyncFetcher { + inner: LocalDirRsyncFetcher, + calls: Arc, + } + impl RsyncFetcher for ModuleScopeRsyncFetcher { + fn fetch_objects( + &self, + rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + self.calls.fetch_add(1, Ordering::SeqCst); + self.inner.fetch_objects(rsync_base_uri) + } + + fn dedup_key(&self, _rsync_base_uri: &str) -> String { + "rsync://rpki.cernet.net/repo/".to_string() + } + } + + let calls = Arc::new(AtomicUsize::new(0)); + let rsync = ModuleScopeRsyncFetcher { + inner: LocalDirRsyncFetcher::new(&fixture_dir), + calls: calls.clone(), + }; + + let runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &rsync, + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + }; + + let first = runner.run_publication_point(&handle).expect("first run ok"); + assert_eq!(first.source, PublicationPointSource::Fresh); + + let second = runner + .run_publication_point(&second_handle) + .expect("second run ok"); + assert!(matches!( + second.source, + PublicationPointSource::Fresh | PublicationPointSource::VcirCurrentInstance + )); + + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "rsync-only mode must deduplicate by rsync scope even when RRDP notification is present" + ); +} diff --git a/crates/panda-rpki-validator/src/validation/tree_runner/tests/runner_behaviour.rs b/crates/panda-rpki-validator/src/validation/tree_runner/tests/runner_behaviour.rs new file mode 100644 index 0000000..fb12aaf --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_runner/tests/runner_behaviour.rs @@ -0,0 +1,694 @@ +#[test] +fn runner_when_repo_sync_fails_uses_current_instance_vcir_and_keeps_children_empty_for_fixture() { + let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + assert!(fixture_dir.is_dir(), "fixture directory must exist"); + + let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); + + let fixture_manifest_bytes = + std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); + let fixture_manifest = + crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) + .expect("decode manifest fixture"); + let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + + let handle = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: rsync_base_uri.clone(), + manifest_rsync_uri: manifest_rsync_uri.clone(), + publication_point_rsync_uri: rsync_base_uri.clone(), + rrdp_notification_uri: None, + }; + + // First: successful fresh run to populate the latest VCIR baseline. + let ok_runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + }; + let first = ok_runner + .run_publication_point(&handle) + .expect("first run ok"); + assert_eq!(first.source, PublicationPointSource::Fresh); + assert!( + first.discovered_children.is_empty(), + "fixture has no child .cer" + ); + + // Second: repo sync fails, but we can still reuse current-instance VCIR. + let bad_runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &FailingRsyncFetcher, + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + }; + let second = bad_runner + .run_publication_point(&handle) + .expect("should reuse current-instance VCIR"); + assert_eq!(second.source, PublicationPointSource::VcirCurrentInstance); + assert!(second.discovered_children.is_empty()); + assert!( + second + .warnings + .iter() + .any(|w| w.message.contains("repo sync failed")), + "expected warning about repo sync failure" + ); +} + +#[test] +fn build_publication_point_audit_emits_no_audit_entry_for_duplicate_pack_uri() { + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256("rsync://example.test/repo/dup.roa", vec![1u8]), + PackFile::from_bytes_compute_sha256("rsync://example.test/repo/dup.roa", vec![2u8]), + ]); + let pp = crate::validation::manifest::PublicationPointResult { + source: crate::validation::manifest::PublicationPointSource::VcirCurrentInstance, + snapshot: pack.clone(), + warnings: Vec::new(), + }; + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(vec![1]), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + let objects = crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: Vec::new(), + stats: crate::validation::objects::ObjectsStats::default(), + audit: Vec::new(), + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + }; + + let audit = build_publication_point_audit_from_snapshot( + &ca, + pp.source, + None, + None, + None, + None, + &pp.snapshot, + &[], + &objects, + &[], + ); + assert_eq!(audit.source, "vcir_current_instance"); + assert_eq!(audit.repo_sync_phase, None); + assert_eq!(audit.repo_terminal_state, "fallback_current_instance"); + assert!( + audit + .objects + .iter() + .any(|e| e.detail.as_deref() == Some("skipped: no audit entry")), + "expected a duplicate key to produce a 'no audit entry' placeholder" + ); +} + +#[test] +fn build_publication_point_audit_marks_invalid_crl_as_error_and_overlays_roa_audit() { + let now = time::OffsetDateTime::now_utc(); + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256("rsync://example.test/repo/issuer/bad.crl", vec![0u8]), + PackFile::from_bytes_compute_sha256("rsync://example.test/repo/issuer/x.roa", vec![1u8]), + ]); + + let pp = crate::validation::manifest::PublicationPointResult { + source: crate::validation::manifest::PublicationPointSource::Fresh, + snapshot: pack.clone(), + warnings: Vec::new(), + }; + + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(vec![1]), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + + let objects = crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: Vec::new(), + stats: crate::validation::objects::ObjectsStats::default(), + audit: vec![ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/issuer/x.roa".to_string(), + sha256_hex: sha256_hex_from_32(&pack.files[1].sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Ok, + detail: None, + }], + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + }; + + let audit = build_publication_point_audit_from_snapshot( + &issuer, + pp.source, + Some("rsync"), + Some("rsync_only_ok"), + Some(123), + Some("none"), + &pp.snapshot, + &[], + &objects, + &[], + ); + assert_eq!(audit.objects[0].kind, AuditObjectKind::Manifest); + assert_eq!(audit.repo_sync_source.as_deref(), Some("rsync")); + assert_eq!(audit.repo_sync_phase.as_deref(), Some("rsync_only_ok")); + assert_eq!(audit.repo_sync_duration_ms, Some(123)); + assert_eq!(audit.repo_sync_error.as_deref(), Some("none")); + assert_eq!(audit.repo_terminal_state, "fresh"); + + let crl = audit + .objects + .iter() + .find(|e| e.rsync_uri.ends_with("bad.crl")) + .expect("crl entry"); + assert!(matches!(crl.result, AuditObjectResult::Error)); + + let roa = audit + .objects + .iter() + .find(|e| e.rsync_uri.ends_with("x.roa")) + .expect("roa entry"); + assert!(matches!(roa.result, AuditObjectResult::Ok)); + + // Smoke that time fields are populated from pack. + assert!(audit.verified_at_rfc3339_utc.contains('T')); + assert!(audit.this_update_rfc3339_utc.contains('T')); + assert!(audit.next_update_rfc3339_utc.contains('T')); + let _ = now; +} + +#[test] +fn discover_children_with_router_certificate_records_ok_audit_and_no_child() { + let g = generate_router_cert_with_variant("ec-p256", true); + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/router.cer", + g.router_der.clone(), + ), + ]); + + let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), + ca_certificate_rsync_uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let out = discover_children_from_fresh_snapshot_with_audit( + &issuer, + &pack, + time::OffsetDateTime::now_utc(), + None, + ) + .expect("discover router cert"); + assert!(out.children.is_empty()); + assert_eq!(out.audits.len(), 1); + assert!(matches!(out.audits[0].result, AuditObjectResult::Ok)); + assert!( + out.audits[0] + .detail + .as_deref() + .unwrap_or("") + .contains("validated BGPsec router certificate") + ); +} + +#[test] +fn discover_children_with_non_router_ee_certificate_records_skipped_audit() { + let g = generate_router_cert_with_variant("ec-p256", false); + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/router-no-eku.cer", + g.router_der.clone(), + ), + ]); + + let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), + ca_certificate_rsync_uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let out = discover_children_from_fresh_snapshot_with_audit( + &issuer, + &pack, + time::OffsetDateTime::now_utc(), + None, + ) + .expect("discover non-router cert"); + assert!(out.children.is_empty()); + assert_eq!(out.audits.len(), 1); + assert!(matches!(out.audits[0].result, AuditObjectResult::Skipped)); + assert!( + out.audits[0] + .detail + .as_deref() + .unwrap_or("") + .contains("not a CA resource certificate or BGPsec router certificate") + ); +} + +#[test] +fn discover_children_with_invalid_router_certificate_records_error_audit() { + let g = generate_router_cert_with_variant("ec-p384", true); + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/router-invalid.cer", + g.router_der.clone(), + ), + ]); + + let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), + ca_certificate_rsync_uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let out = discover_children_from_fresh_snapshot_with_audit( + &issuer, + &pack, + time::OffsetDateTime::now_utc(), + None, + ) + .expect("discover invalid router cert"); + assert!(out.children.is_empty()); + assert_eq!(out.audits.len(), 1); + assert!(matches!(out.audits[0].result, AuditObjectResult::Error)); + assert!( + out.audits[0] + .detail + .as_deref() + .unwrap_or("") + .contains("router certificate validation failed") + ); +} + +#[test] +fn discover_children_with_audit_records_decode_error_for_corrupt_cer() { + let g = generate_chain_and_crl(); + + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/corrupt.cer", + vec![0u8], + ), + ]); + + let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), + ca_certificate_rsync_uri: None, + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let now = time::OffsetDateTime::now_utc(); + let out = discover_children_from_fresh_snapshot_with_audit(&issuer, &pack, now, None) + .expect("discover children"); + assert!(out.children.is_empty()); + assert_eq!(out.audits.len(), 1); + assert!(matches!(out.audits[0].result, AuditObjectResult::Error)); +} + +#[test] +fn select_issuer_crl_uri_for_child_covers_missing_and_not_found_paths() { + let g = generate_chain_and_crl(); + let child = ResourceCertificate::decode_der(&g.child_ca_der).expect("decode child cert"); + + let empty: std::collections::HashMap = + std::collections::HashMap::new(); + let err = select_issuer_crl_uri_for_child(&child, &empty).unwrap_err(); + assert!(err.contains("no CRL available"), "{err}"); + + let ta_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/ta/apnic-ta.cer"), + ) + .expect("read TA fixture"); + let ta = ResourceCertificate::decode_der(&ta_der).expect("decode TA fixture"); + let mut cache = std::collections::HashMap::new(); + cache.insert( + "rsync://example.test/repo/issuer/issuer.crl".to_string(), + CachedIssuerCrl::Pending { + bytes: g.issuer_crl_der.clone(), + sha256_hex: None, + }, + ); + let err = select_issuer_crl_uri_for_child(&ta, &cache).unwrap_err(); + assert!(err.contains("CRLDistributionPoints missing"), "{err}"); + + let mut wrong = std::collections::HashMap::new(); + wrong.insert( + "rsync://example.test/repo/issuer/other.crl".to_string(), + CachedIssuerCrl::Pending { + bytes: g.issuer_crl_der, + sha256_hex: None, + }, + ); + let err = select_issuer_crl_uri_for_child(&child, &wrong).unwrap_err(); + assert!( + err.contains("not found in publication point snapshot"), + "{err}" + ); +} + +#[test] +fn ensure_issuer_crl_verified_promotes_pending_cache_entry() { + let g = generate_chain_and_crl(); + let mut cache = std::collections::HashMap::new(); + let crl_uri = "rsync://example.test/repo/issuer/issuer.crl".to_string(); + cache.insert( + crl_uri.clone(), + CachedIssuerCrl::Pending { + bytes: g.issuer_crl_der.clone(), + sha256_hex: None, + }, + ); + + let first = ensure_issuer_crl_verified(&crl_uri, &mut cache, &g.issuer_ca_der) + .expect("verify pending CRL"); + assert!(first.revoked_serials.is_empty()); + assert!(matches!(cache.get(&crl_uri), Some(CachedIssuerCrl::Ok(_)))); + + let second = ensure_issuer_crl_verified(&crl_uri, &mut cache, &g.issuer_ca_der) + .expect("reuse verified CRL"); + assert!(second.revoked_serials.is_empty()); +} + +#[test] +fn discover_children_with_invalid_issuer_der_records_error_audit() { + let g = generate_chain_and_crl(); + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/child.cer", + g.child_ca_der.clone(), + ), + ]); + + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(vec![0u8]), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let out = discover_children_from_fresh_snapshot_with_audit( + &issuer, + &pack, + time::OffsetDateTime::now_utc(), + None, + ) + .expect("discover children with invalid issuer der"); + assert!(out.children.is_empty()); + assert_eq!(out.audits.len(), 1); + assert!(matches!(out.audits[0].result, AuditObjectResult::Error)); + assert!( + out.audits[0] + .detail + .as_deref() + .unwrap_or("") + .contains("issuer CA decode failed") + ); +} + +#[test] +fn project_current_instance_vcir_reuses_local_outputs_and_restores_children() { + let now = time::OffsetDateTime::now_utc(); + let g = generate_chain_and_crl(); + let child_cert_hash = sha256_hex(&g.child_ca_der); + let vcir = sample_vcir_for_projection(now, &child_cert_hash); + + let store_dir = tempfile::tempdir().expect("store dir"); + let main_db = store_dir.path().join("work-db"); + let repo_bytes_db = store_dir.path().join("repo-bytes.db"); + let store = RocksStore::open_with_external_repo_bytes(&main_db, &repo_bytes_db) + .expect("open rocksdb with external repo bytes"); + store + .put_blob_bytes_batch(&[(child_cert_hash.clone(), g.child_ca_der.clone())]) + .expect("put child cert repo bytes"); + assert!( + store + .get_raw_by_hash_entry(&child_cert_hash) + .expect("lookup child raw_by_hash") + .is_none(), + "child cert restoration should not require raw_by_hash entries" + ); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), + }; + let policy = Policy::default(); + put_vcir_for_failed_fetch_reuse(&store, &ca, &policy, &vcir); + + let projection = project_current_instance_vcir_on_failed_fetch( + &store, + &ca, + &ManifestFreshError::RepoSyncFailed { + detail: "synthetic".to_string(), + }, + &policy, + now, + ) + .expect("project vcir"); + + assert_eq!( + projection.source, + PublicationPointSource::VcirCurrentInstance + ); + assert_eq!(projection.objects.vrps.len(), 1); + assert_eq!(projection.objects.aspas.len(), 1); + assert_eq!(projection.objects.router_keys.len(), 1); + assert_eq!(projection.discovered_children.len(), 1); + assert_eq!( + projection.discovered_children[0].handle.manifest_rsync_uri, + "rsync://example.test/repo/child/child.mft" + ); + assert_eq!( + projection.ccr_manifest_projection.as_ref(), + Some(&vcir.ccr_manifest_projection) + ); + assert!( + projection.snapshot.is_none(), + "current-instance reuse should not reconstruct a byte-backed snapshot" + ); + assert!( + !projection + .warnings + .iter() + .any(|warning| warning.message.contains("manifest failed fetch")), + "successful current-instance reuse should not duplicate the fresh fetch error" + ); + assert!( + !projection + .warnings + .iter() + .any(|warning| warning.message.contains("using latest validated result")), + "successful current-instance reuse should be tracked by source, not warning" + ); + assert!( + !projection + .warnings + .iter() + .any(|warning| warning.message.contains("manifest raw bytes missing")), + "successful current-instance reuse should not load repo bytes for audit reconstruction" + ); + assert!( + !projection + .warnings + .iter() + .any(|warning| warning.message.contains("child certificate bytes missing")), + "child discovery restoration should read child certs from repo bytes" + ); +} + +#[test] +fn project_current_instance_vcir_does_not_reuse_without_identity() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let vcir = sample_vcir_for_projection(now, &child_cert_hash); + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + store.put_vcir(&vcir).expect("put legacy-style VCIR"); + let ca = sample_ca_for_failed_fetch_reuse(&vcir); + + let projection = project_current_instance_vcir_on_failed_fetch( + &store, + &ca, + &ManifestFreshError::RepoSyncFailed { + detail: "synthetic".to_string(), + }, + &Policy::default(), + now, + ) + .expect("project VCIR"); + + assert_eq!( + projection.source, + PublicationPointSource::FailedFetchNoCache + ); + assert!(projection.objects.vrps.is_empty()); + assert!(projection.discovered_children.is_empty()); + assert!( + projection + .warnings + .iter() + .any(|warning| warning.message.contains("reuse identity is missing")) + ); +} diff --git a/crates/panda-rpki-validator/src/validation/tree_runner/types.rs b/crates/panda-rpki-validator/src/validation/tree_runner/types.rs new file mode 100644 index 0000000..13f5547 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_runner/types.rs @@ -0,0 +1,101 @@ +const PUBLICATION_POINT_CACHE_CHILD_RESTORE_PARALLEL_MIN_CHILDREN: usize = 256; +const PUBLICATION_POINT_CACHE_CHILD_RESTORE_MAX_WORKERS: usize = 16; +const CHILD_CERTIFICATE_CACHE_MMAP_MIN_CER_COUNT: usize = 2048; + +fn sha256_hex_to_32(hex_value: &str) -> [u8; 32] { + let mut out = [0u8; 32]; + hex::decode_to_slice(hex_value, &mut out).expect("internal sha256 hex should decode"); + out +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct BuildVcirTimingBreakdown { + pub(crate) select_crl_ms: u64, + pub(crate) current_ca_decode_ms: u64, + pub(crate) local_outputs_ms: u64, + pub(crate) child_entries_ms: u64, + pub(crate) related_artifacts_ms: u64, + pub(crate) struct_build_ms: u64, +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct PersistVcirTimingBreakdown { + pub(crate) embedded_collect_ms: u64, + pub(crate) embedded_store_ms: u64, + pub(crate) build_vcir_ms: u64, + pub(crate) replace_vcir_ms: u64, + pub(crate) publication_point_cache_future_notbefore_guarded: bool, + pub(crate) build_vcir: BuildVcirTimingBreakdown, + pub(crate) replace_vcir: VcirReplaceTimingBreakdown, +} + +#[derive(Clone, Debug)] +pub(crate) struct FreshPublicationPointStage { + pub(crate) fresh_point: FreshValidatedPublicationPoint, + pub(crate) issuer_ca_der: Arc<[u8]>, + pub(crate) snapshot_prepare_timing: FreshPublicationPointTimingBreakdown, + pub(crate) snapshot_prepare_ms: u64, + pub(crate) discovered_children: Vec, + pub(crate) child_audits: Vec, + pub(crate) discovered_router_keys: Vec, + pub(crate) child_discovery_ms: u64, + pub(crate) warnings: Vec, +} + +#[derive(Debug)] +pub(crate) struct FreshPublicationPointStageError { + pub(crate) error: ManifestFreshError, + pub(crate) snapshot_prepare_ms: u64, +} + +#[derive(Clone, Debug)] +pub(crate) struct FreshPublicationPointFinalizeOutput { + pub(crate) result: PublicationPointRunResult, + pub(crate) snapshot_pack_ms: u64, + pub(crate) persist_vcir_ms: u64, + pub(crate) persist_vcir_timing: PersistVcirTimingBreakdown, + pub(crate) ccr_projection_build_ms: u64, + pub(crate) ccr_append_ms: u64, + pub(crate) audit_build_ms: u64, +} + +pub struct Rpkiv1PublicationPointRunner<'a> { + pub store: &'a RocksStore, + pub policy: &'a Policy, + pub http_fetcher: &'a dyn Fetcher, + pub rsync_fetcher: &'a dyn RsyncFetcher, + pub validation_time: time::OffsetDateTime, + pub timing: Option, + pub download_log: Option, + pub replay_archive_index: Option>, + pub replay_delta_index: Option>, + /// In-run RRDP dedup: when RRDP is enabled, only sync each `rrdp_notification_uri` once per run. + /// + /// - If RRDP succeeded for a repo, later publication points referencing that same RRDP repo + /// skip network fetches and reuse the already-populated current repository view. + /// - If RRDP failed for a repo, later publication points skip RRDP attempts and go straight + /// to rsync for their own `rsync_base_uri` (still per-publication-point). + pub rrdp_dedup: bool, + pub rrdp_repo_cache: Mutex>, // notification_uri -> rrdp_ok + + /// In-run rsync dedup: when rsync is used, only sync each `rsync_base_uri` once per run. + /// + /// This reduces duplicate rsync network fetches when multiple publication points share the + /// same `rsync_base_uri` (observed in APNIC full sync timing reports). + pub rsync_dedup: bool, + pub rsync_repo_cache: Mutex>, // rsync_base_uri -> rsync_ok + pub current_repo_index: Option, + pub repo_sync_runtime: Option>, + pub parallel_phase2_config: Option, + pub parallel_roa_worker_pool: Option, + pub ccr_accumulator: Option>, + /// When false, skip VCIR persistence and per-output VCIR projection building. + /// + /// This is intended for replay/compare-only runs where the caller does not need + /// the resulting DB to be reused by a later delta run. + pub persist_vcir: bool, + pub enable_roa_validation_cache: bool, + pub enable_child_certificate_validation_cache: bool, + pub publication_point_cache_observe_only: bool, + pub enable_publication_point_validation_cache: bool, +} diff --git a/crates/panda-rpki-validator/src/validation/tree_runner/vcir_outputs.rs b/crates/panda-rpki-validator/src/validation/tree_runner/vcir_outputs.rs new file mode 100644 index 0000000..d41a0c8 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_runner/vcir_outputs.rs @@ -0,0 +1,933 @@ +fn audit_kind_for_vcir_output_type(output_type: VcirOutputType) -> AuditObjectKind { + match output_type { + VcirOutputType::Vrp => AuditObjectKind::Roa, + VcirOutputType::Aspa => AuditObjectKind::Aspa, + VcirOutputType::RouterKey => AuditObjectKind::RouterCertificate, + } +} + +fn build_objects_output_from_vcir( + vcir: &ValidatedCaInstanceResult, + validation_time: time::OffsetDateTime, + warnings: &mut Vec, +) -> crate::validation::objects::ObjectsOutput { + let mut output = empty_objects_output(); + let mut audit_by_uri: HashMap = HashMap::new(); + let mut roa_total: HashSet = HashSet::new(); + let mut aspa_total: HashSet = HashSet::new(); + let mut roa_ok: HashSet = HashSet::new(); + let mut aspa_ok: HashSet = HashSet::new(); + + for artifact in &vcir.related_artifacts { + if artifact.artifact_role != VcirArtifactRole::SignedObject { + continue; + } + if let Some(uri) = artifact.uri.as_ref() { + match artifact.artifact_kind { + VcirArtifactKind::Roa => { + roa_total.insert(uri.clone()); + } + VcirArtifactKind::Aspa => { + aspa_total.insert(uri.clone()); + } + _ => {} + } + } + } + + for local in &vcir.local_outputs { + let effective_until = match parse_snapshot_time_value(&local.item_effective_until) { + Ok(v) => v, + Err(e) => { + warnings.push( + Warning::new(format!( + "cached local output has invalid item_effective_until: {e}" + )) + .with_context(&local.source_object_uri), + ); + audit_by_uri.insert( + local.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: local.source_object_uri.clone(), + sha256_hex: local.source_object_hash_hex(), + kind: audit_kind_for_vcir_output_type(local.output_type), + result: AuditObjectResult::Error, + detail: Some( + "cached local output has invalid item_effective_until".to_string(), + ), + }, + ); + continue; + } + }; + if validation_time > effective_until { + audit_by_uri + .entry(local.source_object_uri.clone()) + .or_insert_with(|| ObjectAuditEntry { + rsync_uri: local.source_object_uri.clone(), + sha256_hex: local.source_object_hash_hex(), + kind: audit_kind_for_vcir_output_type(local.output_type), + result: AuditObjectResult::Skipped, + detail: Some("skipped: cached local output expired".to_string()), + }); + continue; + } + + match local.output_type { + VcirOutputType::Vrp => match parse_vcir_vrp_output(local) { + Ok(vrp) => { + roa_ok.insert(local.source_object_uri.clone()); + output.vrps.push(vrp); + audit_by_uri.insert( + local.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: local.source_object_uri.clone(), + sha256_hex: local.source_object_hash_hex(), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Ok, + detail: None, + }, + ); + } + Err(e) => { + warnings.push( + Warning::new(format!("cached ROA local output parse failed: {e}")) + .with_context(&local.source_object_uri), + ); + audit_by_uri.insert( + local.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: local.source_object_uri.clone(), + sha256_hex: local.source_object_hash_hex(), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Error, + detail: Some(format!("cached ROA local output parse failed: {e}")), + }, + ); + } + }, + VcirOutputType::Aspa => match parse_vcir_aspa_output(local) { + Ok(aspa) => { + aspa_ok.insert(local.source_object_uri.clone()); + output.aspas.push(aspa); + audit_by_uri.insert( + local.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: local.source_object_uri.clone(), + sha256_hex: local.source_object_hash_hex(), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Ok, + detail: None, + }, + ); + } + Err(e) => { + warnings.push( + Warning::new(format!("cached ASPA local output parse failed: {e}")) + .with_context(&local.source_object_uri), + ); + audit_by_uri.insert( + local.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: local.source_object_uri.clone(), + sha256_hex: local.source_object_hash_hex(), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Error, + detail: Some(format!("cached ASPA local output parse failed: {e}")), + }, + ); + } + }, + VcirOutputType::RouterKey => match parse_vcir_router_key_output(local) { + Ok(router_key) => { + output.router_keys.push(router_key); + audit_by_uri.insert( + local.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: local.source_object_uri.clone(), + sha256_hex: local.source_object_hash_hex(), + kind: AuditObjectKind::RouterCertificate, + result: AuditObjectResult::Ok, + detail: Some("cached Router Key local output restored".to_string()), + }, + ); + } + Err(e) => { + warnings.push( + Warning::new(format!("cached Router Key local output parse failed: {e}")) + .with_context(&local.source_object_uri), + ); + audit_by_uri.insert( + local.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: local.source_object_uri.clone(), + sha256_hex: local.source_object_hash_hex(), + kind: AuditObjectKind::RouterCertificate, + result: AuditObjectResult::Error, + detail: Some(format!( + "cached Router Key local output parse failed: {e}" + )), + }, + ); + } + }, + } + } + + output.stats.roa_total = roa_total.len(); + output.stats.roa_ok = roa_ok.len(); + output.stats.aspa_total = aspa_total.len(); + output.stats.aspa_ok = aspa_ok.len(); + let mut audit: Vec<_> = audit_by_uri.into_values().collect(); + audit.sort_by(|a, b| a.rsync_uri.cmp(&b.rsync_uri)); + output.audit = audit; + output +} + +fn build_objects_output_from_publication_point_cache_projection( + projection: &PublicationPointCacheProjection, + validation_time: time::OffsetDateTime, + warnings: &mut Vec, +) -> crate::validation::objects::ObjectsOutput { + let mut output = empty_objects_output(); + let mut audit_by_uri: HashMap = HashMap::new(); + let mut roa_total: HashSet = HashSet::new(); + let mut aspa_total: HashSet = HashSet::new(); + let mut roa_ok: HashSet = HashSet::new(); + let mut aspa_ok: HashSet = HashSet::new(); + + for artifact in &projection.related_objects { + if artifact.artifact_role != VcirArtifactRole::SignedObject { + continue; + } + if let Some(uri) = artifact.uri.as_ref() { + match artifact.artifact_kind { + VcirArtifactKind::Roa => { + roa_total.insert(uri.clone()); + } + VcirArtifactKind::Aspa => { + aspa_total.insert(uri.clone()); + } + _ => {} + } + } + } + + for projected in &projection.outputs { + let effective_until = match parse_snapshot_time_value(&projected.item_effective_until) { + Ok(value) => value, + Err(err) => { + warnings.push( + Warning::new(format!( + "publication-point cached local output has invalid item_effective_until: {err}" + )) + .with_context(&projected.source_object_uri), + ); + audit_by_uri.insert( + projected.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: projected.source_object_uri.clone(), + sha256_hex: hex::encode(projected.source_object_hash), + kind: audit_kind_for_vcir_output_type(projected.output_type), + result: AuditObjectResult::Error, + detail: Some( + "publication-point cached local output has invalid item_effective_until" + .to_string(), + ), + }, + ); + continue; + } + }; + if validation_time > effective_until { + audit_by_uri + .entry(projected.source_object_uri.clone()) + .or_insert_with(|| ObjectAuditEntry { + rsync_uri: projected.source_object_uri.clone(), + sha256_hex: hex::encode(projected.source_object_hash), + kind: audit_kind_for_vcir_output_type(projected.output_type), + result: AuditObjectResult::Skipped, + detail: Some( + "skipped: publication-point cached local output expired".to_string(), + ), + }); + continue; + } + + match projected.output_type { + VcirOutputType::Vrp => match parse_publication_point_cache_vrp_output(projected) { + Ok(vrp) => { + roa_ok.insert(projected.source_object_uri.clone()); + output.vrps.push(vrp); + audit_by_uri.insert( + projected.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: projected.source_object_uri.clone(), + sha256_hex: hex::encode(projected.source_object_hash), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Ok, + detail: None, + }, + ); + } + Err(err) => { + warnings.push( + Warning::new(format!( + "publication-point cached ROA local output parse failed: {err}" + )) + .with_context(&projected.source_object_uri), + ); + audit_by_uri.insert( + projected.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: projected.source_object_uri.clone(), + sha256_hex: hex::encode(projected.source_object_hash), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Error, + detail: Some(format!( + "publication-point cached ROA local output parse failed: {err}" + )), + }, + ); + } + }, + VcirOutputType::Aspa => match parse_publication_point_cache_aspa_output(projected) { + Ok(aspa) => { + aspa_ok.insert(projected.source_object_uri.clone()); + output.aspas.push(aspa); + audit_by_uri.insert( + projected.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: projected.source_object_uri.clone(), + sha256_hex: hex::encode(projected.source_object_hash), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Ok, + detail: None, + }, + ); + } + Err(err) => { + warnings.push( + Warning::new(format!( + "publication-point cached ASPA local output parse failed: {err}" + )) + .with_context(&projected.source_object_uri), + ); + audit_by_uri.insert( + projected.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: projected.source_object_uri.clone(), + sha256_hex: hex::encode(projected.source_object_hash), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Error, + detail: Some(format!( + "publication-point cached ASPA local output parse failed: {err}" + )), + }, + ); + } + }, + VcirOutputType::RouterKey => { + match parse_publication_point_cache_router_key_output(projected) { + Ok(router_key) => { + output.router_keys.push(router_key); + audit_by_uri.insert( + projected.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: projected.source_object_uri.clone(), + sha256_hex: hex::encode(projected.source_object_hash), + kind: AuditObjectKind::RouterCertificate, + result: AuditObjectResult::Ok, + detail: Some( + "publication-point cached Router Key local output restored" + .to_string(), + ), + }, + ); + } + Err(err) => { + warnings.push( + Warning::new(format!( + "publication-point cached Router Key local output parse failed: {err}" + )) + .with_context(&projected.source_object_uri), + ); + audit_by_uri.insert( + projected.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: projected.source_object_uri.clone(), + sha256_hex: hex::encode(projected.source_object_hash), + kind: AuditObjectKind::RouterCertificate, + result: AuditObjectResult::Error, + detail: Some(format!( + "publication-point cached Router Key local output parse failed: {err}" + )), + }, + ); + } + } + } + } + } + + output.stats.roa_total = roa_total.len(); + output.stats.roa_ok = roa_ok.len(); + output.stats.aspa_total = aspa_total.len(); + output.stats.aspa_ok = aspa_ok.len(); + let mut audit: Vec<_> = audit_by_uri.into_values().collect(); + audit.sort_by(|left, right| left.rsync_uri.cmp(&right.rsync_uri)); + output.audit = audit; + output +} + +fn parse_vcir_vrp_output(local: &VcirLocalOutput) -> Result { + match &local.payload { + VcirLocalOutputPayload::Vrp { + asn, + afi, + prefix_len, + addr, + max_length, + } => Ok(Vrp { + asn: *asn, + prefix: crate::data_model::roa::IpPrefix { + afi: *afi, + prefix_len: *prefix_len, + addr: *addr, + }, + max_length: *max_length, + }), + _ => Err("VCIR local output payload is not VRP".to_string()), + } +} + +fn parse_vcir_aspa_output(local: &VcirLocalOutput) -> Result { + match &local.payload { + VcirLocalOutputPayload::Aspa { + customer_as_id, + provider_as_ids, + } => Ok(AspaAttestation { + customer_as_id: *customer_as_id, + provider_as_ids: provider_as_ids.clone(), + }), + _ => Err("VCIR local output payload is not ASPA".to_string()), + } +} + +fn parse_vcir_router_key_output(local: &VcirLocalOutput) -> Result { + match &local.payload { + VcirLocalOutputPayload::RouterKey { + as_id, + ski, + spki_der, + } => Ok(RouterKeyPayload { + as_id: *as_id, + ski: ski.clone(), + spki_der: spki_der.clone(), + source_object_uri: local.source_object_uri.clone(), + source_object_hash: local.source_object_hash_hex(), + source_ee_cert_hash: local.source_ee_cert_hash_hex(), + item_effective_until: local.item_effective_until.clone(), + }), + _ => Err("VCIR local output payload is not Router Key".to_string()), + } +} + +fn parse_publication_point_cache_vrp_output( + projected: &PublicationPointCacheOutput, +) -> Result { + match &projected.payload { + VcirLocalOutputPayload::Vrp { + asn, + afi, + prefix_len, + addr, + max_length, + } => Ok(Vrp { + asn: *asn, + prefix: crate::data_model::roa::IpPrefix { + afi: *afi, + prefix_len: *prefix_len, + addr: *addr, + }, + max_length: *max_length, + }), + _ => Err("publication-point cache output payload is not VRP".to_string()), + } +} + +fn parse_publication_point_cache_aspa_output( + projected: &PublicationPointCacheOutput, +) -> Result { + match &projected.payload { + VcirLocalOutputPayload::Aspa { + customer_as_id, + provider_as_ids, + } => Ok(AspaAttestation { + customer_as_id: *customer_as_id, + provider_as_ids: provider_as_ids.clone(), + }), + _ => Err("publication-point cache output payload is not ASPA".to_string()), + } +} + +fn parse_publication_point_cache_router_key_output( + projected: &PublicationPointCacheOutput, +) -> Result { + match &projected.payload { + VcirLocalOutputPayload::RouterKey { + as_id, + ski, + spki_der, + } => Ok(RouterKeyPayload { + as_id: *as_id, + ski: ski.clone(), + spki_der: spki_der.clone(), + source_object_uri: projected.source_object_uri.clone(), + source_object_hash: hex::encode(projected.source_object_hash), + source_ee_cert_hash: hex::encode(projected.source_ee_cert_hash), + item_effective_until: projected.item_effective_until.clone(), + }), + _ => Err("publication-point cache output payload is not Router Key".to_string()), + } +} + +fn restore_children_from_vcir( + _store: &RocksStore, + ca: &CaInstanceHandle, + vcir: &ValidatedCaInstanceResult, + _warnings: &mut Vec, +) -> (Vec, Vec) { + let mut children = Vec::new(); + let mut audits = Vec::new(); + for child in &vcir.child_entries { + children.push(DiscoveredChildCaInstance { + handle: CaInstanceHandle { + depth: 0, + tal_id: ca.tal_id.clone(), + parent_manifest_rsync_uri: Some(ca.manifest_rsync_uri.clone()), + ca_certificate: CaCertificateRef::repo_bytes(child.child_cert_hash.clone()), + ca_certificate_rsync_uri: Some(child.child_cert_rsync_uri.clone()), + effective_ip_resources: child.child_effective_ip_resources.clone(), + effective_as_resources: child.child_effective_as_resources.clone(), + rsync_base_uri: child.child_rsync_base_uri.clone(), + manifest_rsync_uri: child.child_manifest_rsync_uri.clone(), + publication_point_rsync_uri: child.child_publication_point_rsync_uri.clone(), + rrdp_notification_uri: child.child_rrdp_notification_uri.clone(), + }, + discovered_from: crate::audit::DiscoveredFrom { + parent_manifest_rsync_uri: ca.manifest_rsync_uri.clone(), + child_ca_certificate_rsync_uri: child.child_cert_rsync_uri.clone(), + child_ca_certificate_sha256_hex: child.child_cert_hash.clone(), + }, + child_entry_projection: Some(DiscoveredChildEntryProjection { + child_ski: child.child_ski.clone(), + }), + }); + audits.push(ObjectAuditEntry { + rsync_uri: child.child_cert_rsync_uri.clone(), + sha256_hex: child.child_cert_hash.clone(), + kind: AuditObjectKind::Certificate, + result: AuditObjectResult::Ok, + detail: Some("restored child CA instance from VCIR".to_string()), + }); + } + (children, audits) +} + +fn restore_children_from_publication_point_cache( + store: &RocksStore, + ca: &CaInstanceHandle, + projection: &PublicationPointCacheProjection, + validation_time: time::OffsetDateTime, + warnings: &mut Vec, + worker_count: usize, + timing: Option<&TimingHandle>, +) -> (Vec, Vec) { + let worker_count = worker_count + .clamp(1, PUBLICATION_POINT_CACHE_CHILD_RESTORE_MAX_WORKERS) + .min(projection.children.len().max(1)); + let outcomes = if worker_count > 1 + && projection.children.len() >= PUBLICATION_POINT_CACHE_CHILD_RESTORE_PARALLEL_MIN_CHILDREN + { + if let Some(timing) = timing { + timing.record_count( + "publication_point_cache_restore_children_parallel_publication_points", + 1, + ); + timing.record_count( + "publication_point_cache_restore_children_parallel_children", + projection.children.len() as u64, + ); + timing.record_count( + "publication_point_cache_restore_children_workers_total", + worker_count as u64, + ); + } + restore_publication_point_cache_children_parallel( + store, + ca, + &projection.children, + validation_time, + worker_count, + ) + } else { + if let Some(timing) = timing { + timing.record_count( + "publication_point_cache_restore_children_batch_publication_points", + 1, + ); + timing.record_count( + "publication_point_cache_restore_children_batch_children", + projection.children.len() as u64, + ); + } + restore_publication_point_cache_children_chunk( + store, + ca, + &projection.children, + validation_time, + ) + }; + collect_publication_point_cache_child_restore_outcomes(outcomes, warnings) +} + +fn restore_publication_point_cache_children_parallel( + _store: &RocksStore, + ca: &CaInstanceHandle, + children: &[PublicationPointCacheChild], + validation_time: time::OffsetDateTime, + worker_count: usize, +) -> Vec { + let chunk_size = children.len().div_ceil(worker_count).max(1); + let mut chunk_results = Vec::new(); + std::thread::scope(|scope| { + let mut handles = Vec::new(); + for chunk in children.chunks(chunk_size) { + handles.push(scope.spawn(move || { + restore_publication_point_cache_children_chunk(_store, ca, chunk, validation_time) + })); + } + for handle in handles { + chunk_results.extend( + handle + .join() + .expect("publication-point cache child restore worker panicked"), + ); + } + }); + chunk_results +} + +fn restore_publication_point_cache_children_chunk( + _store: &RocksStore, + ca: &CaInstanceHandle, + children: &[PublicationPointCacheChild], + validation_time: time::OffsetDateTime, +) -> Vec { + let mut outcomes = vec![None; children.len()]; + for (position, child) in children.iter().enumerate() { + let effective_not_before = + match parse_snapshot_time_value(&child.child_effective_not_before) { + Ok(value) => value, + Err(e) => { + outcomes[position] = Some(PublicationPointCacheChildRestoreOutcome { + child: None, + audit: None, + warning: Some( + Warning::new(format!( + "publication-point cache child has invalid effective notBefore: {e}" + )) + .with_context(&child.child_cert_rsync_uri), + ), + }); + continue; + } + }; + let effective_until = match parse_snapshot_time_value(&child.child_effective_until) { + Ok(value) => value, + Err(e) => { + outcomes[position] = Some(PublicationPointCacheChildRestoreOutcome { + child: None, + audit: None, + warning: Some( + Warning::new(format!( + "publication-point cache child has invalid effective until: {e}" + )) + .with_context(&child.child_cert_rsync_uri), + ), + }); + continue; + } + }; + if validation_time < effective_not_before || validation_time > effective_until { + outcomes[position] = Some(PublicationPointCacheChildRestoreOutcome { + child: None, + warning: None, + audit: Some(publication_point_cache_child_audit( + child, + AuditObjectResult::Skipped, + Some("skipped: publication-point cache child expired".to_string()), + )), + }); + continue; + } + outcomes[position] = Some(PublicationPointCacheChildRestoreOutcome { + child: Some(publication_point_cache_discovered_child(ca, child)), + warning: None, + audit: Some(publication_point_cache_child_audit( + child, + AuditObjectResult::Ok, + Some("restored child CA instance from publication-point cache".to_string()), + )), + }); + } + + outcomes + .into_iter() + .flatten() + .collect::>() +} + +fn collect_publication_point_cache_child_restore_outcomes( + outcomes: Vec, + warnings: &mut Vec, +) -> (Vec, Vec) { + let mut children = Vec::new(); + let mut audits = Vec::new(); + for outcome in outcomes { + if let Some(warning) = outcome.warning { + warnings.push(warning); + } + if let Some(audit) = outcome.audit { + audits.push(audit); + } + if let Some(child) = outcome.child { + children.push(child); + } + } + (children, audits) +} + +#[derive(Clone)] +struct PublicationPointCacheChildRestoreOutcome { + child: Option, + audit: Option, + warning: Option, +} + +fn publication_point_cache_discovered_child( + ca: &CaInstanceHandle, + child: &PublicationPointCacheChild, +) -> DiscoveredChildCaInstance { + DiscoveredChildCaInstance { + handle: CaInstanceHandle { + depth: 0, + tal_id: ca.tal_id.clone(), + parent_manifest_rsync_uri: Some(ca.manifest_rsync_uri.clone()), + ca_certificate: CaCertificateRef::repo_bytes(child.child_cert_hash.clone()), + ca_certificate_rsync_uri: Some(child.child_cert_rsync_uri.clone()), + effective_ip_resources: child.child_effective_ip_resources.clone(), + effective_as_resources: child.child_effective_as_resources.clone(), + rsync_base_uri: child.child_rsync_base_uri.clone(), + manifest_rsync_uri: child.child_manifest_rsync_uri.clone(), + publication_point_rsync_uri: child.child_publication_point_rsync_uri.clone(), + rrdp_notification_uri: child.child_rrdp_notification_uri.clone(), + }, + discovered_from: crate::audit::DiscoveredFrom { + parent_manifest_rsync_uri: ca.manifest_rsync_uri.clone(), + child_ca_certificate_rsync_uri: child.child_cert_rsync_uri.clone(), + child_ca_certificate_sha256_hex: child.child_cert_hash.clone(), + }, + child_entry_projection: Some(DiscoveredChildEntryProjection { + child_ski: child.child_ski.clone(), + }), + } +} + +fn publication_point_cache_child_audit( + child: &PublicationPointCacheChild, + result: AuditObjectResult, + detail: Option, +) -> ObjectAuditEntry { + ObjectAuditEntry { + rsync_uri: child.child_cert_rsync_uri.clone(), + sha256_hex: child.child_cert_hash.clone(), + kind: AuditObjectKind::Certificate, + result, + detail, + } +} + +fn persist_vcir_for_fresh_result_with_timing( + store: &RocksStore, + policy: &Policy, + ca: &CaInstanceHandle, + pack: &PublicationPointSnapshot, + objects: &mut crate::validation::objects::ObjectsOutput, + warnings: &[Warning], + child_audits: &[ObjectAuditEntry], + discovered_children: &[DiscoveredChildCaInstance], + validation_time: time::OffsetDateTime, + write_publication_point_cache_projection: bool, +) -> Result { + let mut timing = PersistVcirTimingBreakdown::default(); + + if objects.stats.publication_point_dropped { + return Ok(timing); + } + + let embedded_store_started = std::time::Instant::now(); + persist_vcir_non_repository_evidence(store, ca) + .map_err(|e| format!("store VCIR audit evidence failed: {e}"))?; + timing.embedded_store_ms = embedded_store_started.elapsed().as_millis() as u64; + + let build_vcir_started = std::time::Instant::now(); + let (vcir, build_vcir_timing) = build_vcir_from_fresh_result_with_timing( + store, + ca, + pack, + objects, + warnings, + child_audits, + discovered_children, + validation_time, + )?; + timing.build_vcir_ms = build_vcir_started.elapsed().as_millis() as u64; + timing.build_vcir = build_vcir_timing; + + let replace_vcir_started = std::time::Instant::now(); + let future_not_before_cache_guard = write_publication_point_cache_projection + && publication_point_cache_has_future_not_before_risk( + pack, + objects, + child_audits, + validation_time, + policy, + ); + timing.publication_point_cache_future_notbefore_guarded = future_not_before_cache_guard; + let publication_point_cache_projection = + if write_publication_point_cache_projection && !future_not_before_cache_guard { + Some(build_publication_point_cache_projection_from_fresh( + policy, ca, pack, &vcir, + )?) + } else { + None + }; + let publication_point_cache_projection_action = if !write_publication_point_cache_projection { + PublicationPointCacheProjectionWriteAction::Keep + } else if future_not_before_cache_guard { + PublicationPointCacheProjectionWriteAction::Delete { + manifest_rsync_uri: &vcir.manifest_rsync_uri, + } + } else { + PublicationPointCacheProjectionWriteAction::Write( + publication_point_cache_projection + .as_ref() + .expect("publication point projection must exist when guard is not active"), + ) + }; + let failed_fetch_reuse_identity = failed_fetch_reuse_identity_for_fresh_result( + ca, + policy, + validation_time, + vcir.instance_gate.instance_effective_until.clone(), + )?; + let replace_timing = store + .replace_vcir_manifest_replay_meta_and_projection_action_with_failed_fetch_reuse_identity( + &vcir, + Some(&RoaCacheProjectionContext { + ca_validation_context_digest: ca_validation_context_digest_for_ca(ca), + policy_fingerprint: publication_point_cache_policy_fingerprint(policy), + object_meta: objects.roa_cache_object_meta.clone(), + }), + publication_point_cache_projection_action, + Some(&failed_fetch_reuse_identity), + ) + .map_err(|e| format!("store VCIR and manifest replay meta failed: {e}"))?; + timing.replace_vcir_ms = replace_vcir_started.elapsed().as_millis() as u64; + timing.replace_vcir = replace_timing; + + Ok(timing) +} + +fn build_publication_point_cache_projection_from_fresh( + policy: &Policy, + ca: &CaInstanceHandle, + pack: &PublicationPointSnapshot, + vcir: &ValidatedCaInstanceResult, +) -> Result { + let ca_cert_sha256 = ca + .ca_certificate_sha256_32() + .ok_or_else(|| "current CA certificate hash unavailable".to_string())?; + let manifest_sha256 = sha256_digest_32(&pack.manifest_bytes); + PublicationPointCacheProjection::from_vcir_with_context( + vcir, + pack.publication_point_rsync_uri.clone(), + ca.ca_certificate_rsync_uri.clone(), + ca_cert_sha256, + manifest_sha256, + ta_context_digest_for_ca(ca), + ca_validation_context_digest_for_ca(ca), + publication_point_cache_policy_fingerprint(policy), + ) + .map_err(|e| e.to_string()) +} + +fn publication_point_cache_has_future_not_before_risk( + pack: &PublicationPointSnapshot, + objects: &crate::validation::objects::ObjectsOutput, + child_audits: &[ObjectAuditEntry], + validation_time: time::OffsetDateTime, + policy: &Policy, +) -> bool { + let mut files_by_uri: HashMap<&str, &PackFile> = HashMap::new(); + for file in &pack.files { + files_by_uri.insert(file.rsync_uri.as_str(), file); + } + + objects + .audit + .iter() + .chain(child_audits.iter()) + .any(|entry| { + audit_entry_has_certificate_time_error(entry) + && files_by_uri + .get(entry.rsync_uri.as_str()) + .map(|file| { + audit_entry_has_future_not_before(entry, file, validation_time, policy) + }) + .unwrap_or(true) + }) +} + +fn audit_entry_has_certificate_time_error(entry: &ObjectAuditEntry) -> bool { + entry.result == AuditObjectResult::Error + && entry + .detail + .as_deref() + .is_some_and(|detail| detail.contains("certificate not valid at validation_time")) +} + +fn audit_entry_has_future_not_before( + entry: &ObjectAuditEntry, + file: &PackFile, + validation_time: time::OffsetDateTime, + policy: &Policy, +) -> bool { + match entry.kind { + AuditObjectKind::Roa => signed_object_ee_not_before(file, policy, SignedObjectKind::Roa) + .map(|not_before| validation_time < not_before) + .unwrap_or(true), + AuditObjectKind::Aspa => signed_object_ee_not_before(file, policy, SignedObjectKind::Aspa) + .map(|not_before| validation_time < not_before) + .unwrap_or(true), + AuditObjectKind::Certificate | AuditObjectKind::RouterCertificate => file + .bytes() + .ok() + .and_then(|bytes| ResourceCertificate::decode_der(bytes).ok()) + .map(|cert| validation_time < cert.tbs.validity_not_before) + .unwrap_or(true), + _ => true, + } +} diff --git a/crates/panda-rpki-validator/src/validation/tree_runner/vcir_persistence.rs b/crates/panda-rpki-validator/src/validation/tree_runner/vcir_persistence.rs new file mode 100644 index 0000000..cc3b489 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_runner/vcir_persistence.rs @@ -0,0 +1,688 @@ +#[derive(Clone, Copy)] +enum SignedObjectKind { + Roa, + Aspa, +} + +fn signed_object_ee_not_before( + file: &PackFile, + policy: &Policy, + kind: SignedObjectKind, +) -> Option { + let bytes = file.bytes().ok()?; + match kind { + SignedObjectKind::Roa => { + let object = RoaObject::decode_der_with_strict_options( + bytes, + policy.strict.cms_der, + policy.strict.name, + ) + .ok()?; + Some( + object.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .validity_not_before, + ) + } + SignedObjectKind::Aspa => { + let object = AspaObject::decode_der_with_strict_options( + bytes, + policy.strict.cms_der, + policy.strict.name, + ) + .ok()?; + Some( + object.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .validity_not_before, + ) + } + } +} + +fn build_vcir_from_fresh_result_with_timing( + store: &RocksStore, + ca: &CaInstanceHandle, + pack: &PublicationPointSnapshot, + objects: &mut crate::validation::objects::ObjectsOutput, + warnings: &[Warning], + child_audits: &[ObjectAuditEntry], + discovered_children: &[DiscoveredChildCaInstance], + validation_time: time::OffsetDateTime, +) -> Result<(ValidatedCaInstanceResult, BuildVcirTimingBreakdown), String> { + let mut timing = BuildVcirTimingBreakdown::default(); + + let select_crl_started = std::time::Instant::now(); + let current_crl = select_manifest_current_crl_from_snapshot(pack)?; + timing.select_crl_ms = select_crl_started.elapsed().as_millis() as u64; + + let current_ca_decode_started = std::time::Instant::now(); + let ca_der = ca.ca_certificate_der(store)?; + let ca_cert = ResourceCertificate::decode_der(ca_der.as_ref()) + .map_err(|e| format!("decode current CA certificate failed: {e}"))?; + timing.current_ca_decode_ms = current_ca_decode_started.elapsed().as_millis() as u64; + + let local_outputs_started = std::time::Instant::now(); + let local_outputs = take_or_build_vcir_local_outputs(ca, pack, objects)?; + timing.local_outputs_ms = local_outputs_started.elapsed().as_millis() as u64; + + let child_entries_started = std::time::Instant::now(); + let child_entries = build_vcir_child_entries(store, discovered_children, validation_time)?; + timing.child_entries_ms = child_entries_started.elapsed().as_millis() as u64; + + let related_artifacts_started = std::time::Instant::now(); + let related_artifacts = build_vcir_related_artifacts( + store, + ca, + pack, + current_crl.file.rsync_uri.as_str(), + objects, + child_audits, + ); + timing.related_artifacts_ms = related_artifacts_started.elapsed().as_millis() as u64; + let ccr_manifest_projection = + build_vcir_ccr_manifest_projection_from_fresh(ca, pack, &child_entries)?; + let local_vrp_count = local_outputs + .iter() + .filter(|output| output.output_type == VcirOutputType::Vrp) + .count() as u32; + let local_aspa_count = local_outputs + .iter() + .filter(|output| output.output_type == VcirOutputType::Aspa) + .count() as u32; + let local_router_key_count = local_outputs + .iter() + .filter(|output| output.output_type == VcirOutputType::RouterKey) + .count() as u32; + let accepted_object_count = related_artifacts + .iter() + .filter(|artifact| artifact.validation_status == VcirArtifactValidationStatus::Accepted) + .count() as u32; + let rejected_object_count = related_artifacts + .iter() + .filter(|artifact| artifact.validation_status == VcirArtifactValidationStatus::Rejected) + .count() as u32; + let ca_ski = hex::encode( + ca_cert + .tbs + .extensions + .subject_key_identifier + .as_ref() + .ok_or_else(|| "current CA certificate missing SubjectKeyIdentifier".to_string())?, + ); + let issuer_ski = hex::encode( + ca_cert + .tbs + .extensions + .authority_key_identifier + .as_ref() + .or(ca_cert.tbs.extensions.subject_key_identifier.as_ref()) + .ok_or_else(|| "current CA certificate missing AuthorityKeyIdentifier".to_string())?, + ); + + let struct_build_started = std::time::Instant::now(); + let vcir = ValidatedCaInstanceResult { + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + parent_manifest_rsync_uri: ca.parent_manifest_rsync_uri.clone(), + tal_id: ca.tal_id.clone(), + ca_subject_name: ca_cert.tbs.subject_name.to_string(), + ca_ski, + issuer_ski, + last_successful_validation_time: PackTime::from_utc_offset_datetime(validation_time), + current_manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + current_crl_rsync_uri: current_crl.file.rsync_uri.clone(), + validated_manifest_meta: crate::storage::ValidatedManifestMeta { + validated_manifest_number: pack.manifest_number_be.clone(), + validated_manifest_this_update: pack.this_update.clone(), + validated_manifest_next_update: pack.next_update.clone(), + }, + ccr_manifest_projection, + instance_gate: VcirInstanceGate { + manifest_next_update: pack.next_update.clone(), + current_crl_next_update: PackTime::from_utc_offset_datetime( + current_crl.crl.next_update.utc, + ), + self_ca_not_after: PackTime::from_utc_offset_datetime(ca_cert.tbs.validity_not_after), + instance_effective_until: PackTime::from_utc_offset_datetime( + pack.next_update + .parse() + .map_err(|e| format!("parse snapshot next_update failed: {e}"))? + .min(current_crl.crl.next_update.utc) + .min(ca_cert.tbs.validity_not_after), + ), + }, + child_entries, + local_outputs, + related_artifacts, + summary: VcirSummary { + local_vrp_count, + local_aspa_count, + local_router_key_count, + child_count: discovered_children.len() as u32, + accepted_object_count, + rejected_object_count, + }, + audit_summary: VcirAuditSummary { + failed_fetch_eligible: true, + last_failed_fetch_reason: None, + warning_count: (warnings.len() + objects.warnings.len()) as u32, + audit_flags: Vec::new(), + }, + }; + vcir.validate_internal().map_err(|e| e.to_string())?; + timing.struct_build_ms = struct_build_started.elapsed().as_millis() as u64; + Ok((vcir, timing)) +} + +fn take_or_build_vcir_local_outputs( + ca: &CaInstanceHandle, + pack: &PublicationPointSnapshot, + objects: &mut crate::validation::objects::ObjectsOutput, +) -> Result, String> { + let mut cached_outputs = std::mem::take(&mut objects.local_outputs_cache); + if cached_outputs.is_empty() { + return build_vcir_local_outputs(ca, pack, objects); + } + + let covered_roa_uris: HashSet = cached_outputs + .iter() + .filter(|output| output.source_object_type == VcirSourceObjectType::Roa) + .map(|output| output.source_object_uri.clone()) + .collect(); + let covered_aspa_uris: HashSet = cached_outputs + .iter() + .filter(|output| output.source_object_type == VcirSourceObjectType::Aspa) + .map(|output| output.source_object_uri.clone()) + .collect(); + cached_outputs.extend(build_vcir_local_outputs_excluding( + ca, + pack, + objects, + &covered_roa_uris, + &covered_aspa_uris, + )?); + Ok(cached_outputs) +} + +fn build_vcir_ccr_manifest_projection_from_fresh( + ca: &CaInstanceHandle, + pack: &PublicationPointSnapshot, + child_entries: &[VcirChildEntry], +) -> Result { + let manifest = ManifestObject::decode_der(&pack.manifest_bytes) + .map_err(|e| format!("decode manifest for VCIR CCR projection failed: {e}"))?; + let ee = &manifest.signed_object.signed_data.certificates[0].resource_cert; + let manifest_ee_aki = ee + .tbs + .extensions + .authority_key_identifier + .clone() + .ok_or_else(|| "manifest EE certificate missing AuthorityKeyIdentifier".to_string())?; + let manifest_sia_locations_der = match ee + .tbs + .extensions + .subject_info_access + .as_ref() + .ok_or_else(|| "manifest EE certificate missing Subject Information Access".to_string())? + { + SubjectInfoAccess::Ee(ee_sia) => vec![select_manifest_signed_object_location( + &ca.manifest_rsync_uri, + &ee_sia.access_descriptions, + )?], + SubjectInfoAccess::Ca(_) => { + return Err( + "manifest EE certificate Subject Information Access has CA variant".to_string(), + ); + } + }; + + let mut subordinate_skis = child_entries + .iter() + .map(|child| { + hex::decode(&child.child_ski) + .map_err(|e| format!("decode child_ski for VCIR CCR projection failed: {e}")) + }) + .collect::, _>>()?; + subordinate_skis.sort(); + subordinate_skis.dedup(); + + Ok(VcirCcrManifestProjection { + manifest_rsync_uri: ca.manifest_rsync_uri.clone(), + manifest_sha256: sha2::Sha256::digest(&pack.manifest_bytes).to_vec(), + manifest_size: pack.manifest_bytes.len() as u64, + manifest_ee_aki, + manifest_number_be: pack.manifest_number_be.clone(), + manifest_this_update: pack.this_update.clone(), + manifest_sia_locations_der, + subordinate_skis, + }) +} + +struct CurrentCrlRef<'a> { + file: &'a PackFile, + crl: RpkixCrl, +} + +fn select_manifest_current_crl_from_snapshot( + pack: &PublicationPointSnapshot, +) -> Result, String> { + let manifest = ManifestObject::decode_der(&pack.manifest_bytes) + .map_err(|e| format!("decode snapshot manifest for VCIR failed: {e}"))?; + let ee = &manifest.signed_object.signed_data.certificates[0].resource_cert; + let crldp_uris = ee + .tbs + .extensions + .crl_distribution_points_uris + .as_ref() + .ok_or_else(|| "manifest EE certificate missing CRLDistributionPoints".to_string())?; + for uri in crldp_uris { + if let Some(file) = pack + .files + .iter() + .find(|candidate| candidate.rsync_uri == *uri) + { + let crl = RpkixCrl::decode_der( + file.bytes() + .map_err(|e| format!("load current CRL bytes for VCIR failed: {e}"))?, + ) + .map_err(|e| format!("decode current CRL for VCIR failed: {e}"))?; + return Ok(CurrentCrlRef { file, crl }); + } + } + Err(format!( + "manifest EE certificate CRLDistributionPoints not found in pack: {}", + crldp_uris.join(", ") + )) +} + +fn build_vcir_local_outputs( + _ca: &CaInstanceHandle, + pack: &PublicationPointSnapshot, + objects: &crate::validation::objects::ObjectsOutput, +) -> Result, String> { + build_vcir_local_outputs_excluding(_ca, pack, objects, &HashSet::new(), &HashSet::new()) +} + +fn build_vcir_local_outputs_excluding( + _ca: &CaInstanceHandle, + pack: &PublicationPointSnapshot, + objects: &crate::validation::objects::ObjectsOutput, + covered_roa_uris: &HashSet, + covered_aspa_uris: &HashSet, +) -> Result, String> { + let accepted_roa_uris: HashSet<&str> = objects + .audit + .iter() + .filter(|entry| entry.kind == AuditObjectKind::Roa && entry.result == AuditObjectResult::Ok) + .map(|entry| entry.rsync_uri.as_str()) + .collect(); + let accepted_aspa_uris: HashSet<&str> = objects + .audit + .iter() + .filter(|entry| { + entry.kind == AuditObjectKind::Aspa && entry.result == AuditObjectResult::Ok + }) + .map(|entry| entry.rsync_uri.as_str()) + .collect(); + + let mut out = Vec::new(); + for file in &pack.files { + let source_object_hash = sha256_hex_from_32(&file.sha256); + if accepted_roa_uris.contains(file.rsync_uri.as_str()) + && !covered_roa_uris.contains(file.rsync_uri.as_str()) + { + let roa = RoaObject::decode_der( + file.bytes() + .map_err(|e| format!("load accepted ROA bytes for VCIR failed: {e}"))?, + ) + .map_err(|e| format!("decode accepted ROA for VCIR failed: {e}"))?; + let ee = &roa.signed_object.signed_data.certificates[0]; + let source_ee_cert_hash = sha256_hex(ee.raw_der.as_slice()); + let item_effective_until = + PackTime::from_utc_offset_datetime(ee.resource_cert.tbs.validity_not_after); + for vrp in roa_to_vrps_for_vcir(&roa) { + let prefix = vrp_prefix_to_string(&vrp); + let rule_hash = sha256_hex( + format!( + "roa-rule:{}:{}:{}:{}", + source_object_hash, vrp.asn, prefix, vrp.max_length + ) + .as_bytes(), + ); + out.push(VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: item_effective_until.clone(), + source_object_uri: file.rsync_uri.clone(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: file.sha256, + source_ee_cert_hash: sha256_hex_to_32(&source_ee_cert_hash), + payload: VcirLocalOutputPayload::Vrp { + asn: vrp.asn, + afi: vrp.prefix.afi, + prefix_len: vrp.prefix.prefix_len, + addr: vrp.prefix.addr, + max_length: vrp.max_length, + }, + rule_hash: sha256_hex_to_32(&rule_hash), + }); + } + } else if accepted_aspa_uris.contains(file.rsync_uri.as_str()) + && !covered_aspa_uris.contains(file.rsync_uri.as_str()) + { + let aspa = AspaObject::decode_der( + file.bytes() + .map_err(|e| format!("load accepted ASPA bytes for VCIR failed: {e}"))?, + ) + .map_err(|e| format!("decode accepted ASPA for VCIR failed: {e}"))?; + let ee = &aspa.signed_object.signed_data.certificates[0]; + let source_ee_cert_hash = sha256_hex(ee.raw_der.as_slice()); + let item_effective_until = + PackTime::from_utc_offset_datetime(ee.resource_cert.tbs.validity_not_after); + let providers = aspa + .aspa + .provider_as_ids + .iter() + .map(u32::to_string) + .collect::>() + .join(","); + let rule_hash = sha256_hex( + format!( + "aspa-rule:{}:{}:{}", + source_object_hash, aspa.aspa.customer_as_id, providers + ) + .as_bytes(), + ); + out.push(VcirLocalOutput { + output_type: VcirOutputType::Aspa, + item_effective_until, + source_object_uri: file.rsync_uri.clone(), + source_object_type: VcirSourceObjectType::Aspa, + source_object_hash: file.sha256, + source_ee_cert_hash: sha256_hex_to_32(&source_ee_cert_hash), + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: aspa.aspa.customer_as_id, + provider_as_ids: aspa.aspa.provider_as_ids.clone(), + }, + rule_hash: sha256_hex_to_32(&rule_hash), + }); + } + } + Ok(out) +} + +pub(crate) fn build_router_key_local_outputs( + _ca: &CaInstanceHandle, + router_keys: &[RouterKeyPayload], +) -> Vec { + router_keys + .iter() + .map(|router_key| { + let ski_hex = hex::encode(&router_key.ski); + let spki_der_base64 = + base64::engine::general_purpose::STANDARD.encode(&router_key.spki_der); + let rule_hash = sha256_hex( + format!( + "router-key-rule:{}:{}:{}:{}", + router_key.source_object_hash, router_key.as_id, ski_hex, spki_der_base64 + ) + .as_bytes(), + ); + VcirLocalOutput { + output_type: VcirOutputType::RouterKey, + item_effective_until: router_key.item_effective_until.clone(), + source_object_uri: router_key.source_object_uri.clone(), + source_object_type: VcirSourceObjectType::RouterKey, + source_object_hash: sha256_hex_to_32(&router_key.source_object_hash), + source_ee_cert_hash: sha256_hex_to_32(&router_key.source_ee_cert_hash), + payload: VcirLocalOutputPayload::RouterKey { + as_id: router_key.as_id, + ski: router_key.ski.clone(), + spki_der: router_key.spki_der.clone(), + }, + rule_hash: sha256_hex_to_32(&rule_hash), + } + }) + .collect() +} + +fn build_vcir_child_entries( + store: &RocksStore, + discovered_children: &[DiscoveredChildCaInstance], + validation_time: time::OffsetDateTime, +) -> Result, String> { + let mut out = Vec::with_capacity(discovered_children.len()); + for child in discovered_children { + let child_ski = match child.child_entry_projection.as_ref() { + Some(projection) => projection.child_ski.clone(), + None => { + let child_der = child.handle.ca_certificate_der(store)?; + let child_cert = ResourceCertificate::decode_der(child_der.as_ref()) + .map_err(|e| format!("decode child certificate for VCIR failed: {e}"))?; + let child_ski = child_cert + .tbs + .extensions + .subject_key_identifier + .as_ref() + .ok_or_else(|| "child certificate missing SubjectKeyIdentifier".to_string())?; + hex::encode(child_ski) + } + }; + out.push(VcirChildEntry { + child_manifest_rsync_uri: child.handle.manifest_rsync_uri.clone(), + child_cert_rsync_uri: child.discovered_from.child_ca_certificate_rsync_uri.clone(), + child_cert_hash: child + .discovered_from + .child_ca_certificate_sha256_hex + .clone(), + child_ski, + child_rsync_base_uri: child.handle.rsync_base_uri.clone(), + child_publication_point_rsync_uri: child.handle.publication_point_rsync_uri.clone(), + child_rrdp_notification_uri: child.handle.rrdp_notification_uri.clone(), + child_effective_ip_resources: child.handle.effective_ip_resources.clone(), + child_effective_as_resources: child.handle.effective_as_resources.clone(), + accepted_at_validation_time: PackTime::from_utc_offset_datetime(validation_time), + }); + } + Ok(out) +} + +fn persist_vcir_non_repository_evidence( + store: &RocksStore, + ca: &CaInstanceHandle, +) -> Result<(), String> { + let ca_der = ca.ca_certificate_der(store)?; + let current_ca_hash = sha256_hex(ca_der.as_ref()); + let mut current_ca_entry = RawByHashEntry::from_bytes(current_ca_hash, ca_der.to_vec()); + if let Some(uri) = ca.ca_certificate_rsync_uri.as_ref() { + current_ca_entry.origin_uris.push(uri.clone()); + } + current_ca_entry.object_type = Some("cer".to_string()); + current_ca_entry.encoding = Some("der".to_string()); + upsert_raw_by_hash_entry(store, current_ca_entry)?; + Ok(()) +} + +fn upsert_raw_by_hash_entry(store: &RocksStore, entry: RawByHashEntry) -> Result<(), String> { + match store.get_raw_by_hash_entry(&entry.sha256_hex) { + Ok(Some(existing)) => { + if existing.bytes != entry.bytes { + return Err(format!( + "raw_by_hash collision for sha256 {} while storing VCIR audit evidence", + entry.sha256_hex + )); + } + let mut merged = existing; + let mut changed = false; + for uri in entry.origin_uris { + if !merged + .origin_uris + .iter() + .any(|existing_uri| existing_uri == &uri) + { + merged.origin_uris.push(uri); + changed = true; + } + } + if merged.object_type.is_none() && entry.object_type.is_some() { + merged.object_type = entry.object_type; + changed = true; + } + if merged.encoding.is_none() && entry.encoding.is_some() { + merged.encoding = entry.encoding; + changed = true; + } + if changed { + store + .put_raw_by_hash_entry(&merged) + .map_err(|e| format!("update raw_by_hash entry failed: {e}"))?; + } + Ok(()) + } + Ok(None) => store + .put_raw_by_hash_entry(&entry) + .map_err(|e| format!("store raw_by_hash entry failed: {e}")), + Err(e) => Err(format!("load raw_by_hash entry failed: {e}")), + } +} + +fn build_vcir_related_artifacts( + store: &RocksStore, + ca: &CaInstanceHandle, + pack: &PublicationPointSnapshot, + current_crl_rsync_uri: &str, + objects: &crate::validation::objects::ObjectsOutput, + child_audits: &[ObjectAuditEntry], +) -> Vec { + let mut audit_by_uri: HashMap<&str, &ObjectAuditEntry> = HashMap::new(); + for entry in child_audits.iter().chain(objects.audit.iter()) { + audit_by_uri.insert(entry.rsync_uri.as_str(), entry); + } + + let mut artifacts = Vec::with_capacity(pack.files.len() + 2); + artifacts.push(VcirRelatedArtifact { + artifact_role: VcirArtifactRole::Manifest, + artifact_kind: VcirArtifactKind::Mft, + uri: Some(pack.manifest_rsync_uri.clone()), + sha256: sha256_hex(&pack.manifest_bytes), + object_type: Some("mft".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }); + artifacts.push(VcirRelatedArtifact { + artifact_role: if ca.parent_manifest_rsync_uri.is_none() { + VcirArtifactRole::TrustAnchorCert + } else { + VcirArtifactRole::IssuerCert + }, + artifact_kind: VcirArtifactKind::Cer, + uri: ca.ca_certificate_rsync_uri.clone(), + sha256: ca + .ca_certificate_sha256_hex() + .map(str::to_string) + .unwrap_or_else(|| { + ca.ca_certificate_der(store) + .map(|bytes| sha256_hex(bytes.as_ref())) + .unwrap_or_default() + }), + object_type: Some("cer".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }); + + for file in &pack.files { + let audit_entry = audit_by_uri.get(file.rsync_uri.as_str()).copied(); + let result = audit_entry + .map(|entry| entry.result.clone()) + .unwrap_or(AuditObjectResult::Ok); + let validation_status = audit_result_to_vcir_status(&result); + let reject_reason = if validation_status == VcirArtifactValidationStatus::Rejected { + audit_entry.and_then(|entry| entry.detail.clone()) + } else { + None + }; + let (artifact_role, artifact_kind) = artifact_role_and_kind(file, current_crl_rsync_uri); + artifacts.push(VcirRelatedArtifact { + artifact_role, + artifact_kind, + uri: Some(file.rsync_uri.clone()), + sha256: sha256_hex_from_32(&file.sha256), + object_type: object_type_from_uri(file.rsync_uri.as_str()), + validation_status, + reject_reason, + }); + } + + artifacts +} + +fn artifact_role_and_kind( + file: &PackFile, + current_crl_rsync_uri: &str, +) -> (VcirArtifactRole, VcirArtifactKind) { + if file.rsync_uri == current_crl_rsync_uri { + (VcirArtifactRole::CurrentCrl, VcirArtifactKind::Crl) + } else if file.rsync_uri.ends_with(".cer") { + (VcirArtifactRole::ChildCaCert, VcirArtifactKind::Cer) + } else if file.rsync_uri.ends_with(".roa") { + (VcirArtifactRole::SignedObject, VcirArtifactKind::Roa) + } else if file.rsync_uri.ends_with(".asa") { + (VcirArtifactRole::SignedObject, VcirArtifactKind::Aspa) + } else if file.rsync_uri.ends_with(".gbr") { + (VcirArtifactRole::SignedObject, VcirArtifactKind::Gbr) + } else if file.rsync_uri.ends_with(".crl") { + (VcirArtifactRole::Other, VcirArtifactKind::Crl) + } else if file.rsync_uri.ends_with(".mft") { + (VcirArtifactRole::Manifest, VcirArtifactKind::Mft) + } else { + (VcirArtifactRole::Other, VcirArtifactKind::Other) + } +} + +fn object_type_from_uri(uri: &str) -> Option { + uri.rsplit_once('.') + .map(|(_, ext)| ext.to_ascii_lowercase()) +} + +fn audit_result_to_vcir_status(result: &AuditObjectResult) -> VcirArtifactValidationStatus { + match result { + AuditObjectResult::Ok => VcirArtifactValidationStatus::Accepted, + AuditObjectResult::Error => VcirArtifactValidationStatus::Rejected, + AuditObjectResult::Skipped => VcirArtifactValidationStatus::WarningOnly, + } +} + +fn roa_to_vrps_for_vcir(roa: &RoaObject) -> Vec { + let asn = roa.roa.as_id; + let mut out = Vec::new(); + for fam in &roa.roa.ip_addr_blocks { + for entry in &fam.addresses { + let max_length = entry.max_length.unwrap_or(entry.prefix.prefix_len); + out.push(Vrp { + asn, + prefix: entry.prefix.clone(), + max_length, + }); + } + } + out +} + +fn vrp_prefix_to_string(vrp: &Vrp) -> String { + match vrp.prefix.afi { + RoaAfi::Ipv4 => { + let addr = std::net::Ipv4Addr::new( + vrp.prefix.addr[0], + vrp.prefix.addr[1], + vrp.prefix.addr[2], + vrp.prefix.addr[3], + ); + format!("{addr}/{}", vrp.prefix.prefix_len) + } + RoaAfi::Ipv6 => { + let addr = std::net::Ipv6Addr::from(vrp.prefix.addr); + format!("{addr}/{}", vrp.prefix.prefix_len) + } + } +} diff --git a/crates/panda-rpki-validator/tests/fixtures/ta_constraints/afrinic-current-ipv4-deny.constraints b/crates/panda-rpki-validator/tests/fixtures/ta_constraints/afrinic-current-ipv4-deny.constraints index 26ce0a9..19a688f 100644 --- a/crates/panda-rpki-validator/tests/fixtures/ta_constraints/afrinic-current-ipv4-deny.constraints +++ b/crates/panda-rpki-validator/tests/fixtures/ta_constraints/afrinic-current-ipv4-deny.constraints @@ -1,4 +1,4 @@ -# Feature #151 M7 current AFRINIC IPv4 holdings deny fixture. +# Current AFRINIC IPv4 holdings deny fixture. # # Source: AFRINIC delegated extended latest. # URL: https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest diff --git a/crates/panda-rpki-validator/tests/fixtures/ta_constraints/afrinic-full-ipv4-deny.constraints b/crates/panda-rpki-validator/tests/fixtures/ta_constraints/afrinic-full-ipv4-deny.constraints index 463ba07..d010ce1 100644 --- a/crates/panda-rpki-validator/tests/fixtures/ta_constraints/afrinic-full-ipv4-deny.constraints +++ b/crates/panda-rpki-validator/tests/fixtures/ta_constraints/afrinic-full-ipv4-deny.constraints @@ -1,4 +1,4 @@ -# Feature #151 remote all5 performance fixture. +# AFRINIC IPv4 holdings deny fixture for deterministic tests. # # Source: official rpki-client 9.8 afrinic.constraints # ($OpenBSD: afrinic.constraints,v 1.4 2026/03/11 21:46:36 job Exp $). diff --git a/crates/panda-rpki-validator/tests/fixtures/ta_constraints/local-custom-allow.constraints b/crates/panda-rpki-validator/tests/fixtures/ta_constraints/local-custom-allow.constraints index 5b64975..ae2e7b7 100644 --- a/crates/panda-rpki-validator/tests/fixtures/ta_constraints/local-custom-allow.constraints +++ b/crates/panda-rpki-validator/tests/fixtures/ta_constraints/local-custom-allow.constraints @@ -1,2 +1,2 @@ -# local baseline-v1: the single valid ROA EE certificate has this IPv4 block +# Minimal local fixture: the single valid ROA EE certificate has this IPv4 block allow 203.0.113.0/24 diff --git a/crates/panda-rpki-validator/tests/fixtures/ta_constraints/local-custom-deny.constraints b/crates/panda-rpki-validator/tests/fixtures/ta_constraints/local-custom-deny.constraints index 9a4ac6c..4b95af4 100644 --- a/crates/panda-rpki-validator/tests/fixtures/ta_constraints/local-custom-deny.constraints +++ b/crates/panda-rpki-validator/tests/fixtures/ta_constraints/local-custom-deny.constraints @@ -1,3 +1,3 @@ -# local baseline-v1 negative control: deny wins over a covering allow rule +# Minimal local negative control: deny wins over a covering allow rule allow 0.0.0.0/0 deny 203.0.113.0/24 diff --git a/crates/panda-rpki-validator/tests/fixtures/ta_constraints/ripe-ncc-afrinic-deny.constraints b/crates/panda-rpki-validator/tests/fixtures/ta_constraints/ripe-ncc-afrinic-deny.constraints index 1a9384b..2b54a3f 100644 --- a/crates/panda-rpki-validator/tests/fixtures/ta_constraints/ripe-ncc-afrinic-deny.constraints +++ b/crates/panda-rpki-validator/tests/fixtures/ta_constraints/ripe-ncc-afrinic-deny.constraints @@ -1,4 +1,4 @@ -# Remote RIPE TA soak policy for feature #151. +# Deterministic RIPE NCC trust-anchor deny fixture. # # Start permissive for every INR kind, then carve out several /8 blocks # allocated to AFRINIC. These denies should not match normal RIPE NCC diff --git a/deploy/docker/.env.example b/deploy/docker/.env.example index 091f5c0..f9b7403 100644 --- a/deploy/docker/.env.example +++ b/deploy/docker/.env.example @@ -1,4 +1,4 @@ -# Staging/runtime configuration. Copy to .env; never commit the copied file. +# Runtime configuration. Copy to .env; never commit the copied file. VALIDATOR_IMAGE=panda-rpki-validator:0.1.0-dirty-amd64 VALIDATOR_PLATFORM=linux/amd64 COMPOSE_PROJECT_NAME=panda-rpki-validator @@ -6,7 +6,7 @@ HOST_DATA_DIR=/var/lib/panda-rpki-validator RESTART_POLICY=no VALIDATOR_COMMAND=run -# Normal sync/validation contract (the defaults match the existing runtime). +# Normal synchronization and validation settings. MAX_RUNS=3 INTERVAL_SECS=0 RIRS=afrinic,apnic,arin,lacnic,ripe diff --git a/docker/base-images.toml b/docker/base-images.toml index 25424e8..505e1db 100644 --- a/docker/base-images.toml +++ b/docker/base-images.toml @@ -1,6 +1,5 @@ -# M4 staging build inputs. Replace tags with immutable digest references before M7. -# The digest and verification date are intentionally empty until the release -# registry/base-image policy is approved. +# Development build inputs. A supported release must use immutable digest +# references and record their verification in release provenance. builder_image = "rust:1-bookworm" runtime_image = "debian:bookworm-slim" builder_digest = "" diff --git a/docker/validator-runtime.Dockerfile b/docker/validator-runtime.Dockerfile index d3fbe80..f694f6b 100644 --- a/docker/validator-runtime.Dockerfile +++ b/docker/validator-runtime.Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1.7 -# Staging defaults. M7 release builds must replace these tags with pinned +# Development defaults. A supported release must replace these tags with # immutable references recorded in docker/base-images.toml. ARG BUILDER_IMAGE=rust:1-bookworm ARG RUNTIME_IMAGE=debian:bookworm-slim diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..25acfcb --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,18 @@ +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = _build + +.PHONY: help html linkcheck clean + +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +html: + @$(SPHINXBUILD) -W --keep-going -b html "$(SOURCEDIR)" "$(BUILDDIR)/html" $(SPHINXOPTS) $(O) + +linkcheck: + @$(SPHINXBUILD) -W --keep-going -b linkcheck "$(SOURCEDIR)" "$(BUILDDIR)/linkcheck" $(SPHINXOPTS) $(O) + +clean: + rm -rf "$(BUILDDIR)" diff --git a/docs/architecture.md b/docs/architecture.md deleted file mode 100644 index 27406bc..0000000 --- a/docs/architecture.md +++ /dev/null @@ -1,21 +0,0 @@ -# panda-rpki-validator M5 architecture - -The first public package is intentionally a single Cargo package. Its internal -boundaries are: - -1. `model-crypto`: TAL/TA, ASN.1/RPKI objects, signatures and resources. -2. `transport`: HTTP, RRDP, rsync and offline replay inputs. -3. `sync-validation-engine`: snapshot/delta orchestration, CA-tree validation, - policy/TA constraints and parallel scheduling. -4. `state-cache`: repository bytes, RocksDB state and cache lifecycle. -5. `runtime-lifecycle`: run numbering, retention, TA refresh, failure reset and - artifact materialization for native and Docker execution. - -M4 contains the extracted normal synchronization and validation implementation -across all five boundaries, plus the daemon and shell lifecycle wrapper used -by the Docker image. Metrics, query, UI, RTR, Prometheus/Grafana, old installer -control plane, remote scripts and `verification-only` are not dependencies of -this package. M5 has established the first canonical normal-run and -multi-architecture baseline; the remaining profile/performance gates against -the old runtime must pass before this repository can be considered -release-ready. diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..c9a2945 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,19 @@ +@ECHO OFF +pushd %~dp0 + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=_build + +if "%1" == "" goto help + +%SPHINXBUILD% -W --keep-going -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/output-abi.md b/docs/output-abi.md deleted file mode 100644 index 29968f1..0000000 --- a/docs/output-abi.md +++ /dev/null @@ -1,61 +0,0 @@ -# Normal run output ABI - -This document defines the compatibility surface between the existing `rpki` -runtime and `panda-rpki-validator`. It covers normal snapshot and delta runs; -`verification-only` is deliberately outside this component. - -## Persistent volume - -The image exposes one data root, `/var/lib/panda-rpki-validator` by default. -The root may be changed by deployment configuration, but the relative layout -must remain: - -```text -/ -├── state/ -│ ├── db/ # work DB and repo-bytes state -│ ├── rsync-mirror/ # reusable rsync mirror when enabled -│ └── ... # lifecycle, TA refresh and failure-isolation state -├── runs/run_XXXX/ # retained normal run directories -├── logs/ -└── tmp/ -``` - -The component and the old runtime must never share a state root during A/B -comparison. - -## Required normal-run files - -Every successful snapshot/delta run must produce the following files under its -run directory: - -| File | Contract | -|---|---| -| `run-meta.json` | run id/sequence, status, sync mode, timing and lifecycle metadata | -| `run-summary.json` | status, counts, artifact index, stage timing and exit information | -| `daemon-status.json` | current runner status and last-run information | -| `report.json` | validation report and object/publication-point summaries | -| `input.cir` | canonical synchronized validation input | -| `result.ccr` | canonical validated state/cache output | -| `vrps.csv` | VRP payload with the existing header, order and canonical sorting | -| `vaps.csv` | VAP payload with the existing header, order and canonical sorting | -| `validation-contract.json` | effective validation configuration and binding | -| `stage-timing.json` | stage-level timing used by diagnostics and performance gates | -| `stdout.log` / `stderr.log` | child process logs | -| `process-time.txt` | resource/time measurement when enabled | - -`db-stats-estimate.txt` and lifecycle helper files remain part of the diagnostic -surface whenever the corresponding feature is enabled. Missing required files, -changed schema, changed CSV columns/order, or an undeclared semantic difference -fails compatibility even if a VRP count is produced. - -## Canonical comparison - -The compatibility harness compares exit code, sync mode, publication-point -outcome, accepted/rejected objects, VRP/VAP, report, CIR/CCR and state/cache -digests. It normalizes only lifecycle wall-clock timestamps, CCR `producedAt`, -absolute paths, fields explicitly marked as component/image version or -provenance, and runtime duration/resource telemetry (including digests derived -from timestamped audit events). The validation time is a fixed input and is -compared exactly. Every normalization is recorded in the baseline manifest; -output differences outside that list are failures. diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..ceffc08 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +Sphinx==9.1.0 +sphinx-rtd-theme==3.1.0 diff --git a/docs/source/architecture.rst b/docs/source/architecture.rst new file mode 100644 index 0000000..b23bc6d --- /dev/null +++ b/docs/source/architecture.rst @@ -0,0 +1,25 @@ +Architecture +============ + +The repository uses a Rust workspace. The primary validator crate is +crates/panda-rpki-validator and is organized around these responsibilities: + +* command-line parsing and operator-facing commands; +* trust-anchor and publication-point retrieval; +* parsing and validation of RPKI signed objects; +* validation-tree scheduling and traversal; +* storage, caches, and run state; and +* report generation and output serialization. + +Source boundaries should follow these responsibilities. Large orchestration +modules are refactored in small, test-protected steps so public behavior and +fixed-input results remain stable. + +Data flow +--------- + +Trust-anchor inputs seed repository retrieval. Parsed objects enter the +validation tree, which applies cryptographic and resource-validation rules. +The resulting state is persisted and rendered into the configured reports. +Network transport, cache lifetime, and output retention are operational +concerns and must remain explicit in their respective modules. diff --git a/docs/source/cli.rst b/docs/source/cli.rst new file mode 100644 index 0000000..ffc6aae --- /dev/null +++ b/docs/source/cli.rst @@ -0,0 +1,22 @@ +Command-line interface +====================== + +The panda-rpki-validator binary is the primary command-line entry point. +Commands and flags are part of the operator-facing compatibility surface. +Discover the exact interface for a checkout with: + +.. code-block:: console + + cargo run --locked -p panda-rpki-validator -- --help + +The repository also provides scripts/runtime/run_validator.sh for repeatable +local executions and scripts/docker/build_image.sh for local container image +builds. Scripts validate required commands and their documented environment +variables before starting a run. + +The container entrypoint accepts run to select the runtime wrapper. That +subcommand belongs to the container entrypoint, not the Rust validator binary. + +Automation should pin a reviewed source revision, retain generated reports, +and treat command failures as operational signals rather than silently +retrying forever. diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..d036336 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,17 @@ +project = "panda-rpki" +copyright = "2026, panda-rpki contributors" +author = "panda-rpki contributors" +release = "0.1.0" +language = "en" + +extensions = [] +templates_path = [] +exclude_patterns = ["_build"] + +html_theme = "sphinx_rtd_theme" +html_title = "panda-rpki documentation" +html_static_path = [] +html_theme_options = { + "navigation_depth": 3, + "collapse_navigation": False, +} diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst new file mode 100644 index 0000000..d5c1068 --- /dev/null +++ b/docs/source/configuration.rst @@ -0,0 +1,32 @@ +Configuration +============= + +The runtime helper reads configuration from .env at the package root by +default. Set ENV_FILE to select another file: + +.. code-block:: console + + ENV_FILE=/path/to/.env bash scripts/runtime/run_validator.sh + +The Compose deployment has a separate deployment configuration file at +deploy/docker/.env. It supplies Compose variables as well as environment +variables for the runtime wrapper. The helper recognizes controls for run +retention, RIR selection, TAL input, resource-validation mode, storage +locations, mirror reuse, retry intervals, and progress logging. Inspect +deploy/docker/.env.example and the helper's usage output before changing +production-like settings. + +Runtime state +------------- + +By default, state, run reports, logs, and temporary files are placed below +the package root. Set RUN_ROOT to move them elsewhere. These locations are +runtime data, not source inputs, and should be retained or removed according +to the operator's own recovery policy. + +Network safety +-------------- + +Use bounded timeouts and explicit network access policies in automation. +Publication points are untrusted network inputs, so test configuration +changes against known fixtures before applying them to a long-running service. diff --git a/docs/source/contributing.rst b/docs/source/contributing.rst new file mode 100644 index 0000000..fbb9c92 --- /dev/null +++ b/docs/source/contributing.rst @@ -0,0 +1,11 @@ +Contributing +============ + +This checkout is being prepared for public release and does not yet accept +external contributions. The source is distributed under the BSD 3-Clause +License (see the repository ``LICENSE`` file); the contribution workflow and +code of conduct must be established before maintainers solicit or accept +contributions. + +For local development, keep changes focused, add tests before refactoring +behavior-sensitive code, and run the checks in the testing guide. diff --git a/docs/source/development.rst b/docs/source/development.rst new file mode 100644 index 0000000..c648193 --- /dev/null +++ b/docs/source/development.rst @@ -0,0 +1,17 @@ +Development +=========== + +The Rust workspace contains the validator implementation, supporting crates, +integration fixtures, and scripts for local and container workflows. + +Before proposing a behavioral change, add or extend focused unit tests, run +the workspace test gates, and exercise a fixed-input compatibility comparison +when storage, scheduling, validation traversal, or output behavior changes. + +Keep source comments focused on current behavior and invariants. Avoid +embedding personal paths, internal work-item labels, historical branches, or +deployment-specific addresses in public-facing source and documentation. + +The CI public-tree check rejects known internal-history patterns from the +public text surface. If a new public document needs a term that resembles a +rejected pattern, revise the check deliberately and explain why in review. diff --git a/docs/source/getting-started.rst b/docs/source/getting-started.rst new file mode 100644 index 0000000..057bc6f --- /dev/null +++ b/docs/source/getting-started.rst @@ -0,0 +1,46 @@ +Getting started +=============== + +Prerequisites +------------- + +Install the Rust toolchain named by rust-toolchain.toml, Docker for the +container workflow, and Python 3.12 or later for documentation builds. + +Build and test +-------------- + +From the repository root, run: + +.. code-block:: console + + cargo build --locked --workspace + cargo test --locked --workspace + +The validator crate is located at crates/panda-rpki-validator. Its binary +accepts trust-anchor inputs and produces validated RPKI output. Use the local +help command to inspect the command surface available in the checkout: + +.. code-block:: console + + cargo run --locked -p panda-rpki-validator -- --help + +The bootstrap TAL and trust-anchor files in fixtures are not an offline +repository snapshot. To make a deterministic offline validation run, supply a +reviewed payload replay archive and locks file through the documented command +line options. Do not treat live RRDP or rsync retrieval as reproducible input. + +Local container workflow +------------------------ + +The Docker scripts build a local image and run the repository verification +workflow: + +.. code-block:: console + + bash scripts/docker/build_image.sh --arch amd64 --allow-dirty + +For the Compose workflow, copy deploy/docker/.env.example to deploy/docker/.env +and use deploy/docker/scripts/start.sh. The image name in that file must match +the image tag built locally. Do not place credentials or generated outputs +under version control. diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..4a1e2b5 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,27 @@ +panda-rpki documentation +========================= + +panda-rpki is a Rust implementation of an RPKI relying-party validator. +It retrieves RPKI publication data, validates the associated objects, and +emits machine-readable reports for operational systems. + +This documentation describes the current source tree and supported local +development workflows. Public release terms, release artifacts, and a public +security reporting channel are not established yet. + +Built with Sphinx using a theme provided by Read the Docs. + +.. toctree:: + :maxdepth: 2 + :caption: Contents + + getting-started + configuration + cli + operations + testing + development + architecture + output-abi + contributing + security diff --git a/docs/source/operations.rst b/docs/source/operations.rst new file mode 100644 index 0000000..0198fff --- /dev/null +++ b/docs/source/operations.rst @@ -0,0 +1,19 @@ +Operations +========== + +An operational run obtains RPKI repository data through the configured +transport, validates objects from the selected trust anchors, and writes +reports and state below RUN_ROOT. The helper script can repeat a run at an +interval and retain a bounded run history. + +Before operating a long-running instance: + +* choose explicit paths for state and logs; +* set bounded network timeouts; +* preserve reports needed for diagnosis; +* monitor exit status, validation failures, and storage growth; and +* test upgrades with the compatibility fixtures described in the testing guide. + +The current checkout is not a published production release. Operators are +responsible for reviewing the source revision and deployment configuration +they use. diff --git a/docs/source/output-abi.rst b/docs/source/output-abi.rst new file mode 100644 index 0000000..8040895 --- /dev/null +++ b/docs/source/output-abi.rst @@ -0,0 +1,19 @@ +Output compatibility +==================== + +Reports and serialized records are consumed by automation, so their fields, +ordering assumptions, and error semantics are compatibility-sensitive. +Changes to output code require focused tests and a fixed-input comparison +before they are treated as behavior-preserving. + +When changing an output format: + +* document the intended consumer impact; +* preserve stable fields where possible; +* add a narrow unit or integration test for the changed contract; +* compare output from equivalent fixed inputs; and +* state any deliberate incompatibility in release notes once a public release + process exists. + +Do not interpret this document as a versioned external API promise. A public +versioning and release policy has not been established for this checkout. diff --git a/docs/source/security.rst b/docs/source/security.rst new file mode 100644 index 0000000..1e8b911 --- /dev/null +++ b/docs/source/security.rst @@ -0,0 +1,11 @@ +Security +======== + +panda-rpki processes cryptographic material and untrusted network data. +Treat credentials, trust-anchor inputs, runtime state, logs, and generated +reports as potentially sensitive operational data. + +No public vulnerability-reporting channel is established for this pre-release +checkout. Do not include a private contact address in a future public release +until its ownership, response process, and disclosure policy have been +approved. diff --git a/docs/source/testing.rst b/docs/source/testing.rst new file mode 100644 index 0000000..4811211 --- /dev/null +++ b/docs/source/testing.rst @@ -0,0 +1,33 @@ +Testing +======== + +Fast checks +----------- + +Run the workspace checks from the repository root: + +.. code-block:: console + + cargo fmt --all -- --check + cargo test --locked --workspace + cargo clippy --locked --workspace --all-targets -- -D warnings + +Compatibility fixtures +---------------------- + +The tests/compat directory defines fixed-input, two-run comparisons. They +protect runtime equivalence while changes are made to traversal, storage, and +output code. The fixture manifest records only test inputs and comparison +scope; it does not describe a historical source baseline. + +Documentation checks +-------------------- + +Install the documentation dependencies and build both local documentation +targets: + +.. code-block:: console + + python -m pip install -r docs/requirements.txt + sphinx-build -W --keep-going -b html docs/source docs/_build/html + sphinx-build -W --keep-going -b linkcheck docs/source docs/_build/linkcheck diff --git a/fixtures/manifest.toml b/fixtures/manifest.toml index 2c9fe26..fbc4908 100644 --- a/fixtures/manifest.toml +++ b/fixtures/manifest.toml @@ -4,7 +4,6 @@ redistribution_status = "pending-license-and-provenance-review" contains_private_data = false contains_live_rir_archive = false verification_only = "not-included" -source_baseline = "provenance/source-baseline.toml" expected_profiles = ["snapshot", "delta-warm", "rrdp-fallback-rsync", "payload-replay"] -runtime_fixture_policy = "image ships TAL/TA only; repository fixtures are explicit staging bind mounts" +runtime_fixture_policy = "image ships TAL/TA only; repository fixtures are explicitly mounted" native_fixture_path = "crates/panda-rpki-validator/tests/fixtures" diff --git a/fixtures/minimal/README.md b/fixtures/minimal/README.md index 8bac42f..a410315 100644 --- a/fixtures/minimal/README.md +++ b/fixtures/minimal/README.md @@ -1,11 +1,11 @@ # Minimal fixture policy -This directory reserves the deterministic, locally generated RPKI fixture -contract. It intentionally contains no live RIR archive, private TAL/TA, key, -customer data, or source-repository bytes. The M4 Docker smoke uses standard -TAL/TA files from `fixtures/` and an explicit bind-mounted repository from the -native test fixture tree; that repository is not shipped in the image while -its redistribution and provenance are pending. M5 must add or approve the -smallest redistributable snapshot/delta fixture and record its generator, -hashes and license in `fixtures/manifest.toml` before using it as public -compatibility evidence. +This directory reserves the contract for deterministic, locally generated RPKI +fixtures. It must not contain live repository archives, private trust anchors, +private keys, customer data, or source-repository bytes. + +Public compatibility evidence requires the smallest suitable snapshot and +delta fixture together with its generator, hashes, license, and redistribution +status in fixtures/manifest.toml. Repository fixtures used only by native tests +are supplied explicitly; the runtime image ships only the declared TAL and TA +files. diff --git a/migration/allowlist.toml b/migration/allowlist.toml deleted file mode 100644 index b036e1a..0000000 --- a/migration/allowlist.toml +++ /dev/null @@ -1,77 +0,0 @@ -schema_version = 1 -source_commit = "74cbebbd3334ac0063761c1a97a88ee000cc2a57" -source_history_policy = "do-not-copy" -license_status = "pending" -verification_only = "defer-to-follow-up-backlog" - -[[entry]] -source = "src/data_model/" -destination = "crates/panda-rpki-validator/src/data_model/" -classification = "replace" -reason = "RPKI object and resource model copied without private repository coupling" - -[[entry]] -source = "src/fetch/" -destination = "crates/panda-rpki-validator/src/fetch/" -classification = "replace" -reason = "RRDP/rsync transport and replay input" - -[[entry]] -source = "src/sync/" -destination = "crates/panda-rpki-validator/src/sync/" -classification = "replace" -reason = "snapshot/delta synchronization orchestration" - -[[entry]] -source = "src/validation/" -destination = "crates/panda-rpki-validator/src/validation/" -classification = "replace" -reason = "RPKI path and signed-object validation" - -[[entry]] -source = "src/parallel/" -destination = "crates/panda-rpki-validator/src/parallel/" -classification = "replace" -reason = "sync and validation scheduling" - -[[entry]] -source = "src/storage.rs and src/storage/" -destination = "crates/panda-rpki-validator/src/storage.rs and crates/panda-rpki-validator/src/storage/" -classification = "replace" -reason = "state, repo-bytes and cache backend" - -[[entry]] -source = "src/tools/rpki_daemon.rs and scripts/soak/run_soak.sh" -destination = "crates/panda-rpki-validator/src/daemon.rs and scripts/runtime/run_validator.sh" -classification = "replace" -reason = "preserve normal run lifecycle and artifacts without old daemon/private paths" - -[[entry]] -source = "docker/ours-rp-runtime.Dockerfile and scripts/docker/build_docker_runtime_image.sh" -destination = "docker/validator-runtime.Dockerfile and scripts/docker/build_image.sh" -classification = "replace" -reason = "retain complete runtime/multi-arch delivery with new public boundary" - -[[entry]] -source = "deploy/docker-installer/" -destination = "(none in this component)" -classification = "defer-private" -reason = "multi-service installer, monitor and upgrade control plane" - -[[entry]] -source = "src/verification_only.rs and verification-only scripts" -destination = "(none in this component)" -classification = "defer-private" -reason = "explicitly deferred by product decision" - -[[entry]] -source = "metrics, Prometheus/Grafana, Explorer, query, RTR, remote and experiment control" -destination = "(none in this component)" -classification = "drop" -reason = "not part of the validator runtime closure" - -[[entry]] -source = "(new) tests/compat/verify_run_abi.sh and tests/compat/compare_runs.py" -destination = "tests/compat/" -classification = "replace" -reason = "component-side normal-run ABI and original/component canonical comparison harness" diff --git a/provenance/source-baseline.toml b/provenance/source-baseline.toml deleted file mode 100644 index 5afa6ef..0000000 --- a/provenance/source-baseline.toml +++ /dev/null @@ -1,17 +0,0 @@ -schema_version = 1 -component = "panda-rpki-validator" -staging_repository = "panda-rpki" -staging_remote = "https://git.nasp.fit/yuyr/panda-rpki.git" -staging_visibility = "private" -staging_freeze = "initial-main-freeze-2026-09-01" -source_workspace = "private rpki workspace (redacted)" -source_commit = "74cbebbd3334ac0063761c1a97a88ee000cc2a57" -source_branch = "dev_1.0.0_yuyr_parallel" -source_history_copied = false -source_remote_recorded = false -license_status = "pending" -verification_only_migrated = false -normal_output_abi = "docs/output-abi.md" -allowlist = "migration/allowlist.toml" -status = "m5-compatibility-multiarch-baseline" -compatibility_report = "../specs/develop/20260831/m5_compatibility_multiarch_baseline_milestone_report.md" diff --git a/scripts/ci/check_public_tree.sh b/scripts/ci/check_public_tree.sh new file mode 100644 index 0000000..4700a59 --- /dev/null +++ b/scripts/ci/check_public_tree.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +cd "$REPO_ROOT" + +readonly PATTERN='git\.nasp\.fit|specs/develop|dev_1\.0\.0_yuyr|private[[:space:]]+(staging|repository)|remote[- ]?(231|211)|Feature #[0-9]{3}|#[0-9]{3}[[:space:]]+M[0-9]+' + +readonly -a PUBLIC_TEXT_PATHS=( + README.md + CONTRIBUTING.md + SECURITY.md + docs + docker + deploy + scripts + tests/compat + fixtures + crates/panda-rpki-validator/src + crates/panda-rpki-validator/tests/fixtures/ta_constraints +) + +if matches="$(rg -n -I -g '!scripts/ci/check_public_tree.sh' -e "$PATTERN" "${PUBLIC_TEXT_PATHS[@]}")"; then + echo "public-tree check found internal-history wording:" >&2 + echo "$matches" >&2 + exit 1 +fi + +echo "public-tree check passed" diff --git a/scripts/docker/build_image.sh b/scripts/docker/build_image.sh index 98dee0f..5d1d43d 100755 --- a/scripts/docker/build_image.sh +++ b/scripts/docker/build_image.sh @@ -24,9 +24,9 @@ Options: --image Image tag (default: panda-rpki-validator:-) --out-dir Image archive/metadata directory (default: docker-out) --dockerfile Dockerfile path - --builder-image Builder base image (M4 staging tag; pin before release) - --runtime-image Runtime base image (M4 staging tag; pin before release) - --allow-dirty Permit staging build and mark tag/metadata dirty + --builder-image Builder base image (pin for a supported release) + --runtime-image Runtime base image (pin for a supported release) + --allow-dirty Permit a local build and mark tag/metadata dirty --no-binfmt Do not install binfmt/qemu for a cross-architecture build --no-save Do not write docker save archive --no-load Build without loading the image into the local daemon @@ -119,7 +119,7 @@ if [[ -n "$(git -C "$REPO_ROOT" status --short 2>/dev/null)" || "$SOURCE_COMMIT_ SOURCE_DIRTY=true fi if [[ "$SOURCE_DIRTY" == true && "$ALLOW_DIRTY" != 1 ]]; then - echo "refusing dirty staging tree; pass --allow-dirty for M4 development" >&2 + echo "refusing dirty tree; pass --allow-dirty for a local development build" >&2 exit 2 fi diff --git a/scripts/runtime/run_validator.sh b/scripts/runtime/run_validator.sh index 7452ef7..0057521 100755 --- a/scripts/runtime/run_validator.sh +++ b/scripts/runtime/run_validator.sh @@ -69,7 +69,8 @@ usage() { Usage: ./run_validator.sh -配置来自 package 根目录下的 .env；也可以用 ENV_FILE=/path/to/.env 覆盖。 +Configuration is read from .env at the package root. Set +ENV_FILE=/path/to/.env to use a different file. USAGE } @@ -1381,7 +1382,7 @@ run_one_round() { daemon_args+=(--db-stats-exact-every "$DB_STATS_EXACT_EVERY") fi fi - # Dead-repo transport blacklist (#141): opt-in via env; the daemon forwards + # Dead-repository transport blacklist: opt in through the environment; the daemon forwards # the enable flag and threshold to the child unless already present there. if [[ -n "${DEAD_REPO_BLACKLIST_PATH:-}" ]]; then daemon_args+=(--dead-repo-blacklist "$DEAD_REPO_BLACKLIST_PATH") diff --git a/tests/compat/README.md b/tests/compat/README.md index 13719e9..54f473b 100644 --- a/tests/compat/README.md +++ b/tests/compat/README.md @@ -1,49 +1,28 @@ -# Compatibility harness (M5) +# Compatibility harness -`verify_run_abi.sh` validates the required files and semantic fields for a -successful normal run. It is intentionally independent of absolute paths and -wall-clock timestamps, so it can be used for native and Docker run directories: +The compatibility harness verifies that two retained validator runs implement +the same declared run-artifact contract. It is useful when comparing native and +container execution, or when validating a refactor against a fixed offline +input. -```bash -tests/compat/verify_run_abi.sh /path/to/runs/run_0001 -``` +verify_run_abi.sh validates the required files and semantic fields for one +successful run. It is independent of absolute paths and wall-clock timestamps: -`compare_runs.py` compares an original-runtime run with a component run. It -checks the normal-run status/count contract, report and validation contract, -and canonical CIR/CCR/VRP/VAP payloads. It normalises only the exceptions in -`baseline-manifest.toml` (including runtime telemetry that is measured -separately). Validation time is treated as a fixed input, while only lifecycle -timestamps and CCR `producedAt` are normalised; all other semantic payload -differences are failures: + tests/compat/verify_run_abi.sh /path/to/runs/run_0001 -```bash -tests/compat/compare_runs.py \ - /path/to/original/runs/run_0001 \ - /path/to/panda/runs/run_0001 -``` +compare_runs.py compares two run directories. It checks run status and counts, +the validation contract, report content, and canonical CIR, CCR, VRP, and VAP +payloads. Only the documented canonicalization exceptions in +baseline-manifest.toml are normalized: -The M5 harness will run the same frozen input against the original `rpki` -runtime, native `panda-rpki-validator`, and the Docker runtime. It will compare -the normal-run output ABI in `docs/output-abi.md`, then record stage timing, -CPU, RSS, state size and artifact size for five serial repetitions per profile. + tests/compat/compare_runs.py \ + /path/to/left/runs/run_0001 \ + /path/to/right/runs/run_0001 -M4 provides the ABI verifier and a real snapshot/warm-delta staging run. The -canonical comparator is now available for M5; five-run performance comparison -and the complete profile matrix remain M5 deliverables. -`verification-only` is excluded. +Use a fixed validation time and a reproducible input for equivalence testing. +Live RRDP and rsync sources can change during a comparison; investigate an +input difference before interpreting a file-level mismatch as validator +behaviour. -For live-RIR evidence, run both images with the same `RIRS` and fixed -`--validation-time`, then compare `input.cir` and the decoded `result.ccr` in -separate steps. `compare_runs.py` deliberately remains strict about report, -validation-event and CSV order, so a live RRDP/rsync source race can produce a -file-level mismatch even when the CCR state digest matches. Use -`ccr_state_compare` plus `triage_ccr_cir_pair` to distinguish source/input -drift from validator behavior; the 2026-09-01 APNIC/all5 evidence is recorded in -`specs/develop/20260901/m5_live_rir_image_artifact_comparison_milestone_report.md`. -The follow-up remote-231 serial all5 run (old image first, then new image; -one snapshot plus three deltas per side with cache/prefetch/parallel flags) is -recorded in -`specs/develop/20260901_2/m5_live_all5_cache_prefetch_serial_4run_milestone_report.md`. -Its strict file mismatches remain failures until the same frozen input is -replayed on both sides; live CCR/CIR differences must first be triaged with -the state comparator rather than treated as code parity. +The test policy and the output contract are documented in the Sphinx site: +docs/source/testing.rst and docs/source/output-abi.rst. diff --git a/tests/compat/baseline-manifest.toml b/tests/compat/baseline-manifest.toml index c5e3824..7cebb62 100644 --- a/tests/compat/baseline-manifest.toml +++ b/tests/compat/baseline-manifest.toml @@ -1,7 +1,6 @@ schema_version = 1 -source_commit = "74cbebbd3334ac0063761c1a97a88ee000cc2a57" component = "panda-rpki-validator" -comparison_status = "m5-canonical-snapshot-delta-baseline-in-progress" +comparison_scope = "fixed-input-runtime-equivalence" verification_only = "excluded" normal_profiles = [ "snapshot",