commit 15c8de82ba08363502a6ffc4bccda011b7828028 Author: yuyr Date: Tue Sep 1 12:13:53 2026 +0800 freeze initial panda-rpki staging diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6c241b3 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,20 @@ +.git +.git/** +.gitignore +target +artifacts +state +runs +logs +tmp +docker-out +provenance/generated +crates/panda-rpki-validator/tests +tests +specs +.env +.env.* +!.env.example +*.local +*.tmp +*.swp diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a71364a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,27 @@ +name: ci + +on: + push: + pull_request: + +jobs: + rust: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + toolchain: 1.92.0 + components: rustfmt, clippy + - run: cargo fmt --all --check + - run: cargo build --locked -p panda-rpki-validator + - run: cargo test --locked -p panda-rpki-validator + - run: cargo clippy --locked -p panda-rpki-validator --all-targets -- -D warnings + + docker-runtime: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: docker buildx build --platform linux/amd64 --load -t panda-rpki-validator:ci -f docker/validator-runtime.Dockerfile . + - run: ./scripts/docker/verify_image.sh --image panda-rpki-validator:ci + - run: docker run --rm panda-rpki-validator:ci run --help diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bb7541e --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +/target/ +/artifacts/ +/state/ +/runs/ +/logs/ +/tmp/ +/docker-out/ +/provenance/generated/ +.env +.env.* +!.env.example +*.local +*.tmp +*.swp +__pycache__/ +*.py[cod] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..e4e9298 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,10 @@ +# Contributing to panda-rpki + +This repository is private staging for the `panda-rpki-validator` extraction. +External contributions are not enabled while ownership, license and contribution +governance are pending. + +During M3–M6, changes must preserve `docs/output-abi.md`, update the relevant +allowlist/provenance entry, and pass the locked Rust build and Docker smoke +checks. Do not add live RIR data, private state, credentials, internal hosts, +or copied history. diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..b568397 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2633 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "aligned-vec" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" +dependencies = [ + "equator", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.20", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "async-compression" +version = "0.4.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3976abdc8fe7d1133d43d304afd42abdf5bc3e1319d263d223bde07b5efc4be8" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide 0.8.9", + "object", + "rustc-demangle", + "windows-link", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags 2.13.1", + "cexpr", + "clang-sys", + "itertools", + "lazy_static", + "lazycell", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex 1.3.0", + "syn 2.0.119", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "cc" +version = "1.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex 2.0.1", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core", +] + +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "compression-codecs" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc14f565cf027a105f7a44ccf9e5b424348421a1d8952a8fc9d499d313107789" + +[[package]] +name = "cookie-factory" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2" +dependencies = [ + "futures", +] + +[[package]] +name = "cpp_demangle" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2bb79cb74d735044c972aae58ed0aaa9a837e85b01106a54c39e42e97f62253" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "uuid", +] + +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "cookie-factory", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + +[[package]] +name = "equator" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" +dependencies = [ + "equator-macro", +] + +[[package]] +name = "equator-macro" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "find-msvc-tools" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" + +[[package]] +name = "findshlibs" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64" +dependencies = [ + "cc", + "lazy_static", + "libc", + "winapi", +] + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide 0.9.1", + "zlib-rs", +] + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core", + "wasm-bindgen", +] + +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "half" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b43ede17f21864e81be2fa654110bf1e793774238d86ef8555c37e6519c0403" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", + "webpki-roots", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07aa2048142242915a31d35844fb311e0e53fcca590c3a0a40dcf1b841fa09eb" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "inferno" +version = "0.11.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" +dependencies = [ + "ahash", + "indexmap", + "is-terminal", + "itoa", + "log", + "num-format", + "once_cell", + "quick-xml 0.26.0", + "rgb", + "str_stack", +] + +[[package]] +name = "ipnet" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" + +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "librocksdb-sys" +version = "0.16.0+8.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce3d60bc059831dc1c83903fb45c103f75db65c5a7bf22272764d9cc683e348c" +dependencies = [ + "bindgen", + "bzip2-sys", + "cc", + "glob", + "libc", + "libz-sys", + "lz4-sys", +] + +[[package]] +name = "libz-sys" +version = "1.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bc9657773828b90eeb625adff10eeac83cc21bbfd8e23a03eaa8a33c9e28d9" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "nix" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-format" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" +dependencies = [ + "arrayvec", + "itoa", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "panda-rpki-validator" +version = "0.1.0" +dependencies = [ + "asn1-rs", + "base64", + "der-parser", + "flate2", + "hex", + "memmap2", + "pprof", + "quick-xml 0.37.5", + "reqwest", + "ring", + "rocksdb", + "roxmltree", + "serde", + "serde_cbor", + "serde_json", + "sha2", + "tempfile", + "thiserror 2.0.20", + "time", + "toml", + "url", + "uuid", + "x509-parser", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "pprof" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afad4d4df7b31280028245f152d5a575083e2abb822d05736f5e47653e77689f" +dependencies = [ + "aligned-vec", + "backtrace", + "cfg-if", + "findshlibs", + "inferno", + "libc", + "log", + "nix", + "once_cell", + "prost", + "prost-build", + "prost-derive", + "sha2", + "smallvec", + "spin", + "symbolic-demangle", + "tempfile", + "thiserror 1.0.69", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" +dependencies = [ + "bytes", + "heck", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" +dependencies = [ + "prost", +] + +[[package]] +name = "quick-xml" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" +dependencies = [ + "memchr", +] + +[[package]] +name = "quick-xml" +version = "0.37.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +dependencies = [ + "memchr", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash 2.1.3", + "rustls", + "socket2", + "thiserror 2.0.20", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand", + "rand_pcg", + "ring", + "rustc-hash 2.1.3", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.20", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webpki-roots", +] + +[[package]] +name = "rgb" +version = "0.8.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rocksdb" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd13e55d6d7b8cd0ea569161127567cd587676c99f4472f779a0279aa60a7a7" +dependencies = [ + "libc", + "librocksdb-sys", +] + +[[package]] +name = "roxmltree" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97" + +[[package]] +name = "rustc-demangle" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.13.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_cbor" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" +dependencies = [ + "half", + "serde", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" +dependencies = [ + "lock_api", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "str_stack" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f446288b699d66d0fd2e30d1cfe7869194312524b3b9252594868ed26ef056a" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "symbolic-common" +version = "12.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "332615d90111d8eeaf86a84dc9bbe9f65d0d8c5cf11b4caccedc37754eb0dcfd" +dependencies = [ + "debugid", + "memmap2", + "stable_deref_trait", + "uuid", +] + +[[package]] +name = "symbolic-demangle" +version = "12.18.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "912017718eb4d21930546245af9a3475c9dccf15675a5c215664e76621afc471" +dependencies = [ + "cpp_demangle", + "rustc-demangle", + "symbolic-common", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl 2.0.20", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "async-compression", + "bitflags 2.13.1", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tokio", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "ring", + "rusticata-macros", + "thiserror 2.0.20", + "time", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "zlib-rs" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..4b685a0 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,4 @@ +[workspace] +members = ["crates/panda-rpki-validator"] +default-members = ["crates/panda-rpki-validator"] +resolver = "3" diff --git a/README.md b/README.md new file mode 100644 index 0000000..abf8f87 --- /dev/null +++ b/README.md @@ -0,0 +1,100 @@ +# panda-rpki + +`panda-rpki` is the staging workspace for the independently distributable RPKI +synchronization and validation component. The first component is +`panda-rpki-validator` (Cargo package and CLI); its Rust import path is +`panda_rpki_validator`. + +This directory is currently a **private staging repository**. It has no copied +history from the existing private `rpki` repository. The normal synchronization +and validation path has now been extracted, together with its daemon and +Docker lifecycle wrapper. The first M5 canonical snapshot/delta baseline and +amd64/arm64 staging image checks are complete; remaining profile/performance +gates still block any public release. Do not use this staging build as a +production validator. + +## Current milestone + +M4/M5 contain the first functional extraction from source commit +`74cbebbd3334ac0063761c1a97a88ee000cc2a57`: normal RRDP/rsync synchronization, +RPKI validation, RocksDB state, CIR/CCR/report/CSV outputs, the run daemon, and +the snapshot/delta lifecycle wrapper. The `verification-only` mode is +intentionally deferred. The project license is intentionally **TBD**; no +external contributions or public release are accepted until ownership and +licensing are approved. + +## Build and test the staging component + +```bash +cargo build --locked -p panda-rpki-validator +cargo test --locked -p panda-rpki-validator +cargo fmt --all --check +cargo clippy --locked -p panda-rpki-validator --all-targets -- -D warnings +./scripts/docker/build_image.sh --arch amd64 --allow-dirty --no-save +``` + +The extracted test suite currently passes `735` tests with `1` ignored test in +the normal profile. The APNIC offline snapshot/delta profile has also passed +the M5 canonical comparator and a five-run pinned release smoke baseline; the +full cache/fallback/failure/profile matrix is still pending. + +For a deterministic single-RIR run against the checked-in test repository, +build the binaries first and invoke the lifecycle wrapper with +`RPKI_EXTRA_ARGS=--disable-rrdp --rsync-local-dir ...`; the wrapper writes the +normal run ABI under `RUN_ROOT/runs/run_0001/`. + +## Docker smoke + +```bash +./scripts/docker/build_image.sh --arch amd64 --allow-dirty +./scripts/docker/verify_image.sh \ + --image panda-rpki-validator:0.1.0-dirty-amd64 +``` + +The runtime image contains both `panda-rpki-validator` and +`panda-rpki-validator-daemon`, the complete normal-run wrapper, and the +redistributable TAL/TA fixtures. It uses one persistent data root mounted at +`/var/lib/panda-rpki-validator`; `run`, `run-validator`, and `daemon` are +entrypoint subcommands. The wrapper preserves the output contract documented +in +[`docs/output-abi.md`](docs/output-abi.md). + +For native or container A/B output checks, run the ABI verifier and canonical +comparator against two retained run directories: + +```bash +tests/compat/verify_run_abi.sh /path/to/run_0001 +tests/compat/compare_runs.py \ + /path/to/original/runs/run_0001 \ + /path/to/panda/runs/run_0001 +``` + +The comparator has matched the original runtime for the APNIC offline +snapshot/delta baseline. Five serial release samples on that small single-RIR +profile stay within the current wall-time gate under a pinned CPU. A live APNIC +run also produced an identical decoded CCR state (MFT/VRP/VAP/TA/RK); only the +time-bearing `producedAt` byte differed in the raw DER. A concurrent all-RIR +(`all5`) run is retained as network/fallback evidence, but its source was not an +atomic snapshot (the original image timed out on the ARIN RRDP notification), +so its different CCR state is not a code-parity verdict. See the detailed +[`M5 live RIR comparison report`](../specs/develop/20260901/m5_live_rir_image_artifact_comparison_milestone_report.md). +The follow-up remote-231 serial all5 run (old image first, then this image; +one snapshot plus three warm deltas per image with cache/prefetch/parallel +flags) also completed 4/4 runs on each side and retained full artifacts. Its +live CCR/CIR and timing differences are documented separately and are not a +frozen-input parity result: [`remote-231 all5 cache/prefetch report`](../specs/develop/20260901_2/m5_live_all5_cache_prefetch_serial_4run_milestone_report.md). +The new image also completed a separate remote-231 all5 long sequence of one +snapshot plus ten deltas; the timing and cache counters are retained as live +health evidence, not as a frozen-input parity gate: [`one snapshot + ten delta +timing report`](../specs/develop/20260901_2/m5_new_image_all5_snapshot_10delta_timing_report.md). +Cache, RRDP/fallback, constraints, replay and failure-path profiles remain in +M5. + +## Repository status + +- `verification-only`: deferred to a separate backlog item. +- License and copyright owner: TBD; do not add a speculative `LICENSE` file. +- Staging Git remote: `https://git.nasp.fit/yuyr/panda-rpki.git` (private + staging target selected); public visibility, registry prefix, signing identity, + and release tags remain unselected until release governance is complete. +- Source provenance baseline: `provenance/source-baseline.toml`. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..b54d660 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,12 @@ +# Security policy + +This project is not public yet and does not have a public vulnerability intake +address. Do not publish vulnerability details in an issue or commit. Report +urgent findings to the project owner through the private project channel and +include the affected commit, image tag/digest, reproduction steps, and whether +state or output data is exposed. + +The public contact, response SLA, supported release branches, and disclosure +process will be added after ownership and license approval. Container images +must never contain credentials, production state, private registry settings, +or internal hostnames. diff --git a/crates/panda-rpki-validator/Cargo.toml b/crates/panda-rpki-validator/Cargo.toml new file mode 100644 index 0000000..1733d42 --- /dev/null +++ b/crates/panda-rpki-validator/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "panda-rpki-validator" +version = "0.1.0" +edition = "2024" +rust-version = "1.92" +description = "Panda RPKI synchronization and validation runtime" +publish = false + +[lib] +name = "panda_rpki_validator" +path = "src/lib.rs" + +[[bin]] +name = "panda-rpki-validator" +path = "src/main.rs" + +[[bin]] +name = "panda-rpki-validator-daemon" +path = "src/bin/panda-rpki-validator-daemon.rs" + +[features] +default = ["full"] +full = [] +profile = ["dep:pprof", "dep:flate2"] + +[dependencies] +asn1-rs = "0.7.1" +der-parser = { version = "10.0.0", features = ["serialize"] } +hex = "0.4.3" +base64 = "0.22.1" +sha2 = "0.10.8" +thiserror = "2.0.18" +time = "0.3.45" +ring = "0.17.14" +x509-parser = { version = "0.18.0", features = ["verify"] } +url = "2.5.8" +serde = { version = "1.0.218", features = ["derive"] } +serde_json = { version = "1.0.140", features = ["raw_value"] } +toml = "0.8.20" +rocksdb = { version = "0.22.0", default-features = false, features = ["lz4"] } +serde_cbor = "0.11.2" +memmap2 = "0.9.10" +roxmltree = "0.20.0" +quick-xml = "0.37.2" +uuid = { version = "1.7.0", features = ["v4"] } +reqwest = { version = "0.12.12", default-features = false, features = ["blocking", "rustls-tls", "gzip", "brotli", "deflate"] } +pprof = { version = "0.14.1", optional = true, features = ["flamegraph", "prost-codec"] } +flate2 = { version = "1.0.35", optional = true } +tempfile = "3.16.0" diff --git a/crates/panda-rpki-validator/src/analysis/mod.rs b/crates/panda-rpki-validator/src/analysis/mod.rs new file mode 100644 index 0000000..fca4906 --- /dev/null +++ b/crates/panda-rpki-validator/src/analysis/mod.rs @@ -0,0 +1 @@ +pub mod timing; diff --git a/crates/panda-rpki-validator/src/analysis/timing.rs b/crates/panda-rpki-validator/src/analysis/timing.rs new file mode 100644 index 0000000..e16b86a --- /dev/null +++ b/crates/panda-rpki-validator/src/analysis/timing.rs @@ -0,0 +1,353 @@ +use std::collections::HashMap; +use std::path::Path; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use serde::{Deserialize, Serialize}; + +#[derive(Clone)] +pub struct TimingHandle { + inner: Arc>, +} + +impl TimingHandle { + pub fn new(meta: TimingMeta) -> Self { + Self { + inner: Arc::new(Mutex::new(TimingCollector::new(meta))), + } + } + + pub fn span_phase(&self, phase: &'static str) -> TimingSpanGuard<'_> { + TimingSpanGuard { + handle: self.clone(), + kind: TimingSpanKind::Phase(phase), + start: Instant::now(), + } + } + + pub fn span_rrdp_repo<'a>(&self, repo_uri: &'a str) -> TimingSpanGuard<'a> { + TimingSpanGuard { + handle: self.clone(), + kind: TimingSpanKind::RrdpRepo(repo_uri), + start: Instant::now(), + } + } + + pub fn span_rrdp_repo_step<'a>( + &self, + repo_uri: &'a str, + step: &'static str, + ) -> TimingSpanGuard<'a> { + TimingSpanGuard { + handle: self.clone(), + kind: TimingSpanKind::RrdpRepoStep { repo_uri, step }, + start: Instant::now(), + } + } + + pub fn span_publication_point<'a>(&self, manifest_rsync_uri: &'a str) -> TimingSpanGuard<'a> { + TimingSpanGuard { + handle: self.clone(), + kind: TimingSpanKind::PublicationPoint(manifest_rsync_uri), + start: Instant::now(), + } + } + + pub fn set_meta(&self, update: TimingMetaUpdate<'_>) { + let mut g = self.inner.lock().expect("timing lock"); + if let Some(v) = update.tal_url { + g.meta.tal_url = Some(v.to_string()); + } + if let Some(v) = update.db_path { + g.meta.db_path = Some(v.to_string()); + } + } + + pub fn record_count(&self, key: &'static str, inc: u64) { + let mut g = self.inner.lock().expect("timing lock"); + g.counts + .entry(key) + .and_modify(|v| *v = v.saturating_add(inc)) + .or_insert(inc); + } + + pub fn counts_snapshot(&self) -> HashMap { + let g = self.inner.lock().expect("timing lock"); + g.counts + .iter() + .map(|(key, value)| ((*key).to_string(), *value)) + .collect() + } + + pub fn report_snapshot(&self, top_n: usize) -> TimingReportV1 { + let g = self.inner.lock().expect("timing lock"); + g.to_report(top_n) + } + + /// Record a phase duration directly in nanoseconds. + /// + /// This is useful when aggregating sub-phase timings locally (to reduce lock contention) + /// and then emitting a single record per publication point. + pub fn record_phase_nanos(&self, phase: &'static str, nanos: u64) { + let mut g = self.inner.lock().expect("timing lock"); + g.phases.record(phase, nanos); + } + + pub fn record_publication_point_nanos(&self, manifest_rsync_uri: &str, nanos: u64) { + let mut g = self.inner.lock().expect("timing lock"); + g.publication_points.record(manifest_rsync_uri, nanos); + } + + pub fn record_publication_point_step_nanos( + &self, + manifest_rsync_uri: &str, + step: &'static str, + nanos: u64, + ) { + let mut g = self.inner.lock().expect("timing lock"); + g.publication_point_steps + .record(&format!("{manifest_rsync_uri}::{step}"), nanos); + } + + pub fn write_json(&self, path: &Path, top_n: usize) -> Result<(), String> { + let report = { + let g = self.inner.lock().expect("timing lock"); + g.to_report(top_n) + }; + + let f = std::fs::File::create(path) + .map_err(|e| format!("create timing json failed: {}: {e}", path.display()))?; + serde_json::to_writer_pretty(f, &report) + .map_err(|e| format!("write timing json failed: {e}"))?; + Ok(()) + } + + fn record_duration(&self, kind: TimingSpanKind<'_>, duration: Duration) { + let nanos_u64 = duration.as_nanos().min(u128::from(u64::MAX)) as u64; + let mut g = self.inner.lock().expect("timing lock"); + match kind { + TimingSpanKind::Phase(name) => g.phases.record(name, nanos_u64), + TimingSpanKind::RrdpRepo(uri) => g.rrdp_repos.record(uri, nanos_u64), + TimingSpanKind::RrdpRepoStep { repo_uri, step } => g + .rrdp_repo_steps + .record(&format!("{repo_uri}::{step}"), nanos_u64), + TimingSpanKind::PublicationPoint(uri) => g.publication_points.record(uri, nanos_u64), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TimingMeta { + pub recorded_at_utc_rfc3339: String, + pub validation_time_utc_rfc3339: String, + pub tal_url: Option, + pub db_path: Option, +} + +#[derive(Clone, Debug, Default)] +pub struct TimingMetaUpdate<'a> { + pub tal_url: Option<&'a str>, + pub db_path: Option<&'a str>, +} + +pub struct TimingSpanGuard<'a> { + handle: TimingHandle, + kind: TimingSpanKind<'a>, + start: Instant, +} + +impl Drop for TimingSpanGuard<'_> { + fn drop(&mut self) { + self.handle + .record_duration(self.kind.clone(), self.start.elapsed()); + } +} + +#[derive(Clone, Debug)] +enum TimingSpanKind<'a> { + Phase(&'static str), + RrdpRepo(&'a str), + RrdpRepoStep { + repo_uri: &'a str, + step: &'static str, + }, + PublicationPoint(&'a str), +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct DurationStats { + pub count: u64, + pub total_nanos: u64, +} + +impl DurationStats { + fn record(&mut self, nanos: u64) { + self.count = self.count.saturating_add(1); + self.total_nanos = self.total_nanos.saturating_add(nanos); + } +} + +#[derive(Clone, Debug, Default)] +struct DurationStatsMap { + map: HashMap, +} + +impl DurationStatsMap { + fn record(&mut self, key: &str, nanos: u64) { + self.map.entry(key.to_string()).or_default().record(nanos); + } + + fn top(&self, n: usize) -> Vec { + let mut v = self + .map + .iter() + .map(|(k, s)| TopDurationEntry { + key: k.clone(), + count: s.count, + total_nanos: s.total_nanos, + }) + .collect::>(); + v.sort_by(|a, b| b.total_nanos.cmp(&a.total_nanos)); + v.truncate(n); + v + } +} + +struct TimingCollector { + meta: TimingMeta, + counts: HashMap<&'static str, u64>, + phases: DurationStatsMap, + rrdp_repos: DurationStatsMap, + rrdp_repo_steps: DurationStatsMap, + publication_points: DurationStatsMap, + publication_point_steps: DurationStatsMap, +} + +impl TimingCollector { + fn new(meta: TimingMeta) -> Self { + Self { + meta, + counts: HashMap::new(), + phases: DurationStatsMap::default(), + rrdp_repos: DurationStatsMap::default(), + rrdp_repo_steps: DurationStatsMap::default(), + publication_points: DurationStatsMap::default(), + publication_point_steps: DurationStatsMap::default(), + } + } + + fn to_report(&self, top_n: usize) -> TimingReportV1 { + TimingReportV1 { + format_version: 1, + meta: self.meta.clone(), + counts: self + .counts + .iter() + .map(|(k, v)| ((*k).to_string(), *v)) + .collect(), + phases: self + .phases + .map + .iter() + .map(|(k, s)| (k.clone(), s.clone())) + .collect(), + top_rrdp_repos: self.rrdp_repos.top(top_n), + top_rrdp_repo_steps: self.rrdp_repo_steps.top(top_n), + top_publication_points: self.publication_points.top(top_n), + top_publication_point_steps: self.publication_point_steps.top(top_n), + } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct TimingReportV1 { + pub format_version: u64, + pub meta: TimingMeta, + pub counts: HashMap, + pub phases: HashMap, + pub top_rrdp_repos: Vec, + pub top_rrdp_repo_steps: Vec, + pub top_publication_points: Vec, + pub top_publication_point_steps: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TopDurationEntry { + pub key: String, + pub count: u64, + pub total_nanos: u64, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn timing_handle_writes_json_with_phases_and_tops() { + let meta = TimingMeta { + recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + tal_url: Some("https://example.test/x.tal".to_string()), + db_path: Some("db".to_string()), + }; + let h = TimingHandle::new(meta); + + { + let _p = h.span_phase("tal_bootstrap"); + } + { + let _r = h.span_rrdp_repo("https://rrdp.example.test/notification.xml"); + } + { + let _s = h.span_rrdp_repo_step( + "https://rrdp.example.test/notification.xml", + "fetch_notification", + ); + } + { + let _pp = h.span_publication_point("rsync://example.test/repo/manifest.mft"); + } + h.record_count("vrps", 42); + h.record_publication_point_nanos("rsync://example.test/repo/manifest.mft", 1_000_000); + h.record_publication_point_step_nanos( + "rsync://example.test/repo/manifest.mft", + "fresh_snapshot_prepare", + 1_000_000, + ); + + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("timing.json"); + h.write_json(&path, 10).expect("write_json"); + + let rep: TimingReportV1 = + serde_json::from_slice(&std::fs::read(&path).expect("read timing.json")) + .expect("parse timing.json"); + assert_eq!(rep.format_version, 1); + assert!(rep.phases.contains_key("tal_bootstrap")); + assert_eq!(rep.counts.get("vrps").copied(), Some(42)); + assert!( + rep.top_rrdp_repos + .iter() + .any(|e| e.key.contains("rrdp.example.test")), + "expected repo in top list" + ); + assert!( + rep.top_rrdp_repo_steps + .iter() + .any(|e| e.key.contains("fetch_notification")), + "expected repo step in top list" + ); + assert!( + rep.top_publication_points + .iter() + .any(|e| e.key.contains("manifest.mft")), + "expected PP in top list" + ); + assert!( + rep.top_publication_point_steps + .iter() + .any(|e| e.key.contains("fresh_snapshot_prepare")), + "expected PP step in top list" + ); + } +} diff --git a/crates/panda-rpki-validator/src/audit.rs b/crates/panda-rpki-validator/src/audit.rs new file mode 100644 index 0000000..9825025 --- /dev/null +++ b/crates/panda-rpki-validator/src/audit.rs @@ -0,0 +1,342 @@ +use serde::Serialize; +use sha2::Digest; + +use crate::policy::Policy; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AuditObjectKind { + Manifest, + Crl, + Certificate, + RouterCertificate, + Roa, + Aspa, + Other, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AuditObjectResult { + Ok, + Skipped, + Error, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct ObjectAuditEntry { + pub rsync_uri: String, + pub sha256_hex: String, + pub kind: AuditObjectKind, + pub result: AuditObjectResult, + #[serde(skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct AuditWarning { + pub message: String, + pub category: String, + pub rfc_refs: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub context: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct QueryAuditManifest { + pub schema_version: u32, + pub status: String, + pub events_path: String, + pub events_count: u64, + pub events_sha256: String, + pub writer_version: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ValidationEventCounts { + #[serde(skip_serializing_if = "Option::is_none")] + pub objects: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub warnings: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub vrps: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub aspas: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ValidationEvent { + pub schema_version: u32, + pub seq: u64, + pub event_type: String, + pub validation_time: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub pp_node_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub pp_manifest_uri: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub pp_rsync_base_uri: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub repo_sync_phase: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub repo_terminal_state: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub object_uri: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub sha256: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub object_type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub result: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub reason: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub counts: Option, +} + +impl From<&crate::report::Warning> for AuditWarning { + fn from(w: &crate::report::Warning) -> Self { + Self { + message: w.message.clone(), + category: w.category.as_str().to_string(), + rfc_refs: w.rfc_refs.iter().map(|r| r.0.to_string()).collect(), + context: w.context.clone(), + } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct PublicationPointAudit { + /// Monotonic node ID assigned by the traversal engine. + /// + /// Present when running via the Stage2 tree engine; may be absent in ad-hoc runs. + #[serde(skip_serializing_if = "Option::is_none")] + pub node_id: Option, + /// Parent node ID in the traversal tree. + #[serde(skip_serializing_if = "Option::is_none")] + pub parent_node_id: Option, + /// Provenance metadata for non-root nodes (how this CA instance was discovered). + #[serde(skip_serializing_if = "Option::is_none")] + pub discovered_from: Option, + + pub rsync_base_uri: String, + pub manifest_rsync_uri: String, + pub publication_point_rsync_uri: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub rrdp_notification_uri: Option, + + pub source: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub repo_sync_source: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub repo_sync_phase: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub repo_sync_duration_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub repo_sync_error: Option, + pub repo_terminal_state: String, + pub this_update_rfc3339_utc: String, + pub next_update_rfc3339_utc: String, + pub verified_at_rfc3339_utc: String, + + pub warnings: Vec, + pub objects: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct DiscoveredFrom { + pub parent_manifest_rsync_uri: String, + pub child_ca_certificate_rsync_uri: String, + pub child_ca_certificate_sha256_hex: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct TreeSummary { + pub instances_processed: usize, + pub instances_failed: usize, + pub warnings: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct AuditRunMeta { + pub validation_time_rfc3339_utc: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AuditDownloadKind { + RrdpNotification, + RrdpSnapshot, + RrdpDelta, + Rsync, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct AuditDownloadObjectsStat { + pub objects_count: u64, + pub objects_bytes_total: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct AuditDownloadEvent { + pub kind: AuditDownloadKind, + pub uri: String, + pub started_at_rfc3339_utc: String, + pub finished_at_rfc3339_utc: String, + pub duration_ms: u64, + pub success: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub bytes: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub objects: Option, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct AuditDownloadKindStats { + pub ok_total: u64, + pub fail_total: u64, + pub duration_ms_total: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub bytes_total: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub objects_count_total: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub objects_bytes_total: Option, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct AuditDownloadStats { + pub events_total: u64, + /// Statistics keyed by serialized `AuditDownloadKind` string (e.g. "rrdp_snapshot"). + pub by_kind: std::collections::BTreeMap, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn publication_point_audit_serializes_object_audit_only() { + let audit = PublicationPointAudit { + objects: vec![ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/fresh.roa".to_string(), + sha256_hex: "11".repeat(32), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Ok, + detail: None, + }], + ..PublicationPointAudit::default() + }; + + let value = serde_json::to_value(&audit).expect("serialize audit"); + assert!(value.get("objects").is_some()); + assert!(value.get("cir_fresh_objects").is_none()); + assert!(value.get("cir_cached_objects").is_none()); + } + + #[test] + fn warning_audit_keeps_the_warning_category() { + let warning = crate::report::Warning::new("BER-compatible CMS accepted") + .with_category(crate::report::WarningCategory::BerCompatibleCmsEncoding); + + let audit_warning = AuditWarning::from(&warning); + assert_eq!(audit_warning.category, "ber_compatible_cms_encoding"); + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct AuditRepoSyncStateStat { + pub count: u64, + pub duration_ms_total: u64, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct AuditRepoSyncStats { + pub publication_points_total: u64, + pub by_phase: std::collections::BTreeMap, + pub by_terminal_state: std::collections::BTreeMap, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct AuditReportV1 { + pub format_version: u32, + pub meta: AuditRunMeta, + pub policy: Policy, + pub tree: TreeSummary, + pub publication_points: Vec, + + pub vrps: Vec, + pub aspas: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct AuditReportV2 { + pub format_version: u32, + pub meta: AuditRunMeta, + pub policy: Policy, + pub tree: TreeSummary, + pub publication_points: Vec, + + pub vrps: Vec, + pub aspas: Vec, + + pub downloads: Vec, + pub download_stats: AuditDownloadStats, + pub repo_sync_stats: AuditRepoSyncStats, + #[serde(rename = "queryAudit", skip_serializing_if = "Option::is_none")] + pub query_audit: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct VrpOutput { + pub asn: u32, + pub prefix: String, + pub max_length: u16, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct AspaOutput { + pub customer_as_id: u32, + pub provider_as_ids: Vec, +} + +pub fn sha256_hex_from_32(bytes: &[u8; 32]) -> String { + hex::encode(bytes) +} + +pub fn sha256_hex(bytes: &[u8]) -> String { + let digest = sha2::Sha256::digest(bytes); + hex::encode(digest) +} + +pub fn format_roa_ip_prefix(p: &crate::data_model::roa::IpPrefix) -> String { + let addr = p.addr_bytes(); + match p.afi { + crate::data_model::roa::RoaAfi::Ipv4 => { + format!( + "{}.{}.{}.{}{}", + addr[0], + addr[1], + addr[2], + addr[3], + format!("/{}", p.prefix_len) + ) + } + crate::data_model::roa::RoaAfi::Ipv6 => { + let mut parts = Vec::with_capacity(8); + for i in 0..8 { + let hi = addr[i * 2] as u16; + let lo = addr[i * 2 + 1] as u16; + parts.push(format!("{:x}", (hi << 8) | lo)); + } + format!("{}{}", parts.join(":"), format!("/{}", p.prefix_len)) + } + } +} diff --git a/crates/panda-rpki-validator/src/audit_downloads.rs b/crates/panda-rpki-validator/src/audit_downloads.rs new file mode 100644 index 0000000..41f7412 --- /dev/null +++ b/crates/panda-rpki-validator/src/audit_downloads.rs @@ -0,0 +1,170 @@ +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use crate::audit::{ + AuditDownloadEvent, AuditDownloadKind, AuditDownloadKindStats, AuditDownloadObjectsStat, + AuditDownloadStats, +}; + +#[derive(Clone, Debug, Default)] +pub struct DownloadLogHandle { + inner: Arc>>, +} + +impl DownloadLogHandle { + pub fn new() -> Self { + Self::default() + } + + pub fn record_event(&self, event: AuditDownloadEvent) { + self.inner.lock().expect("download log lock").push(event); + } + + pub fn snapshot_events(&self) -> Vec { + self.inner.lock().expect("download log lock").clone() + } + + pub fn stats_from_events(events: &[AuditDownloadEvent]) -> AuditDownloadStats { + let mut out = AuditDownloadStats { + events_total: events.len() as u64, + by_kind: BTreeMap::new(), + }; + for e in events { + let kind_key = match e.kind { + AuditDownloadKind::RrdpNotification => "rrdp_notification", + AuditDownloadKind::RrdpSnapshot => "rrdp_snapshot", + AuditDownloadKind::RrdpDelta => "rrdp_delta", + AuditDownloadKind::Rsync => "rsync", + } + .to_string(); + + let st = out + .by_kind + .entry(kind_key) + .or_insert_with(|| AuditDownloadKindStats { + ok_total: 0, + fail_total: 0, + duration_ms_total: 0, + bytes_total: None, + objects_count_total: None, + objects_bytes_total: None, + }); + if e.success { + st.ok_total = st.ok_total.saturating_add(1); + } else { + st.fail_total = st.fail_total.saturating_add(1); + } + st.duration_ms_total = st.duration_ms_total.saturating_add(e.duration_ms); + if let Some(b) = e.bytes { + st.bytes_total = Some(st.bytes_total.unwrap_or(0).saturating_add(b)); + } + if let Some(objects) = &e.objects { + st.objects_count_total = Some( + st.objects_count_total + .unwrap_or(0) + .saturating_add(objects.objects_count), + ); + st.objects_bytes_total = Some( + st.objects_bytes_total + .unwrap_or(0) + .saturating_add(objects.objects_bytes_total), + ); + } + } + out + } + + pub fn stats(&self) -> AuditDownloadStats { + let events = self.snapshot_events(); + Self::stats_from_events(&events) + } + + pub fn span_download<'a>( + &'a self, + kind: AuditDownloadKind, + uri: &'a str, + ) -> DownloadSpanGuard<'a> { + DownloadSpanGuard { + handle: self, + kind, + uri, + start_instant: Instant::now(), + started_at: time::OffsetDateTime::now_utc(), + bytes: None, + objects: None, + error: None, + success: None, + } + } +} + +pub struct DownloadSpanGuard<'a> { + handle: &'a DownloadLogHandle, + kind: AuditDownloadKind, + uri: &'a str, + start_instant: Instant, + started_at: time::OffsetDateTime, + bytes: Option, + objects: Option, + error: Option, + success: Option, +} + +impl DownloadSpanGuard<'_> { + pub fn set_bytes(&mut self, bytes: u64) { + self.bytes = Some(bytes); + } + + pub fn set_objects(&mut self, objects_count: u64, objects_bytes_total: u64) { + self.objects = Some(AuditDownloadObjectsStat { + objects_count, + objects_bytes_total, + }); + } + + pub fn set_ok(&mut self) { + self.success = Some(true); + } + + pub fn set_err(&mut self, msg: impl Into) { + self.success = Some(false); + self.error = Some(msg.into()); + } +} + +impl Drop for DownloadSpanGuard<'_> { + fn drop(&mut self) { + use time::format_description::well_known::Rfc3339; + let finished_at = time::OffsetDateTime::now_utc(); + let dur = self.start_instant.elapsed(); + let duration_ms = duration_to_ms(dur); + let started_at_rfc3339_utc = self + .started_at + .to_offset(time::UtcOffset::UTC) + .format(&Rfc3339) + .unwrap_or_else(|_| "".to_string()); + let finished_at_rfc3339_utc = finished_at + .to_offset(time::UtcOffset::UTC) + .format(&Rfc3339) + .unwrap_or_else(|_| "".to_string()); + let success = self.success.unwrap_or(false); + let event = AuditDownloadEvent { + kind: self.kind.clone(), + uri: self.uri.to_string(), + started_at_rfc3339_utc, + finished_at_rfc3339_utc, + duration_ms, + success, + error: if success { None } else { self.error.clone() }, + bytes: self.bytes, + objects: self.objects.clone(), + }; + self.handle.record_event(event); + } +} + +fn duration_to_ms(d: Duration) -> u64 { + let ms = d.as_millis(); + ms.min(u128::from(u64::MAX)) as u64 +} diff --git a/crates/panda-rpki-validator/src/bin/panda-rpki-validator-daemon.rs b/crates/panda-rpki-validator/src/bin/panda-rpki-validator-daemon.rs new file mode 100644 index 0000000..66d1426 --- /dev/null +++ b/crates/panda-rpki-validator/src/bin/panda-rpki-validator-daemon.rs @@ -0,0 +1,6 @@ +fn main() { + let code = panda_rpki_validator::daemon::main_entry(); + if code != 0 { + std::process::exit(code); + } +} diff --git a/crates/panda-rpki-validator/src/blob_store.rs b/crates/panda-rpki-validator/src/blob_store.rs new file mode 100644 index 0000000..01ecf9b --- /dev/null +++ b/crates/panda-rpki-validator/src/blob_store.rs @@ -0,0 +1,839 @@ +use std::path::PathBuf; +use std::sync::Arc; + +use rocksdb::{DB, Options, WriteBatch}; + +use crate::storage::{ + RawByHashEntry, RocksDbMemoryDbSnapshot, RocksStore, StorageError, StorageResult, + memory_db_snapshot_for_column_families, +}; + +const RAW_BY_HASH_KEY_PREFIX: &str = "rawbyhash:"; +const RAW_BLOB_KEY_PREFIX: &str = "rawblob:"; +const REPO_BYTES_KEY_PREFIX: &str = "sha256:"; + +fn raw_by_hash_key(sha256_hex: &str) -> String { + format!("{RAW_BY_HASH_KEY_PREFIX}{sha256_hex}") +} + +fn raw_blob_key(sha256_hex: &str) -> String { + format!("{RAW_BLOB_KEY_PREFIX}{sha256_hex}") +} + +fn repo_bytes_key(sha256_hex: &str) -> String { + format!("{REPO_BYTES_KEY_PREFIX}{sha256_hex}") +} + +fn validate_blob_sha256_hex(sha256_hex: &str) -> StorageResult<()> { + if sha256_hex.len() != 64 || !sha256_hex.as_bytes().iter().all(u8::is_ascii_hexdigit) { + return Err(StorageError::InvalidData { + entity: "raw_blob", + detail: format!("invalid sha256 hex: {sha256_hex}"), + }); + } + Ok(()) +} + +fn validate_blob_bytes(bytes: &[u8]) -> StorageResult<()> { + if bytes.is_empty() { + return Err(StorageError::InvalidData { + entity: "raw_blob", + detail: "bytes must not be empty".to_string(), + }); + } + Ok(()) +} + +pub trait RawObjectStore { + fn get_raw_entry(&self, sha256_hex: &str) -> StorageResult>; + + fn get_raw_entries_batch( + &self, + sha256_hexes: &[String], + ) -> StorageResult>>; + + fn get_blob_bytes(&self, sha256_hex: &str) -> StorageResult>> { + self.get_raw_entry(sha256_hex) + .map(|entry| entry.map(|entry| entry.bytes)) + } + + fn get_blob_bytes_batch(&self, sha256_hexes: &[String]) -> StorageResult>>> { + self.get_raw_entries_batch(sha256_hexes).map(|entries| { + entries + .into_iter() + .map(|entry| entry.map(|entry| entry.bytes)) + .collect() + }) + } +} + +#[derive(Clone, Debug)] +pub struct ExternalRawStoreDb { + path: PathBuf, + db: Arc, +} + +#[derive(Clone, Debug)] +pub struct ExternalRepoBytesDb { + path: PathBuf, + db: Arc, + read_only: bool, + secondary: bool, +} + +impl ExternalRawStoreDb { + pub fn open(path: impl Into) -> StorageResult { + let path = path.into(); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|e| StorageError::RocksDb(e.to_string()))?; + } + let mut opts = Options::default(); + opts.create_if_missing(true); + opts.set_compression_type(rocksdb::DBCompressionType::Lz4); + opts.set_max_open_files(512); + let db = DB::open(&opts, &path).map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(Self { + path, + db: Arc::new(db), + }) + } + + pub fn put_raw_entry(&self, entry: &RawByHashEntry) -> StorageResult<()> { + entry.validate_internal()?; + let key = raw_by_hash_key(&entry.sha256_hex); + let blob_key = raw_blob_key(&entry.sha256_hex); + let value = serde_cbor::to_vec(entry).map_err(|e| StorageError::Codec { + entity: "raw_by_hash", + detail: e.to_string(), + })?; + let blob_value = entry.bytes.clone(); + self.db + .write({ + let mut batch = WriteBatch::default(); + batch.put(key.as_bytes(), value); + batch.put(blob_key.as_bytes(), blob_value); + batch + }) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(()) + } + + pub fn put_raw_entries_batch(&self, entries: &[RawByHashEntry]) -> StorageResult<()> { + if entries.is_empty() { + return Ok(()); + } + let mut batch = WriteBatch::default(); + for entry in entries { + entry.validate_internal()?; + let key = raw_by_hash_key(&entry.sha256_hex); + let blob_key = raw_blob_key(&entry.sha256_hex); + let value = serde_cbor::to_vec(entry).map_err(|e| StorageError::Codec { + entity: "raw_by_hash", + detail: e.to_string(), + })?; + batch.put(key.as_bytes(), value); + batch.put(blob_key.as_bytes(), entry.bytes.as_slice()); + } + self.db + .write(batch) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(()) + } + + pub fn put_blob_bytes_batch(&self, blobs: &[(String, Vec)]) -> StorageResult<()> { + if blobs.is_empty() { + return Ok(()); + } + let mut batch = WriteBatch::default(); + for (sha256_hex, bytes) in blobs { + validate_blob_sha256_hex(sha256_hex)?; + validate_blob_bytes(bytes)?; + let blob_key = raw_blob_key(sha256_hex); + batch.put(blob_key.as_bytes(), bytes.as_slice()); + } + self.db + .write(batch) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(()) + } + + pub fn delete_raw_entry(&self, sha256_hex: &str) -> StorageResult<()> { + let key = raw_by_hash_key(sha256_hex); + let blob_key = raw_blob_key(sha256_hex); + self.db + .write({ + let mut batch = WriteBatch::default(); + batch.delete(key.as_bytes()); + batch.delete(blob_key.as_bytes()); + batch + }) + .map_err(|e| StorageError::RocksDb(e.to_string())) + } + + pub fn path(&self) -> &PathBuf { + &self.path + } + + pub(crate) fn memory_snapshot(&self, label: impl Into) -> RocksDbMemoryDbSnapshot { + memory_db_snapshot_for_column_families(label, self.db.as_ref(), None) + } +} + +impl ExternalRepoBytesDb { + pub fn open(path: impl Into) -> StorageResult { + let path = path.into(); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|e| StorageError::RocksDb(e.to_string()))?; + } + let mut opts = Options::default(); + opts.create_if_missing(true); + opts.set_compression_type(rocksdb::DBCompressionType::Lz4); + opts.set_max_open_files(512); + let db = DB::open(&opts, &path).map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(Self { + path, + db: Arc::new(db), + read_only: false, + secondary: false, + }) + } + + pub fn open_read_only(path: impl Into) -> StorageResult { + let path = path.into(); + let mut opts = Options::default(); + opts.set_compression_type(rocksdb::DBCompressionType::Lz4); + opts.set_max_open_files(512); + let db = DB::open_for_read_only(&opts, &path, false) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(Self { + path, + db: Arc::new(db), + read_only: true, + secondary: false, + }) + } + + /// Open the repo-bytes DB as a RocksDB secondary instance. + /// + /// Unlike `open_read_only` (a frozen point-in-time view), a secondary + /// instance can follow the live primary via `try_catch_up_with_primary`, + /// which is required when the soak keeps writing new object bytes while + /// the query service is running. + pub fn open_as_secondary( + path: impl Into, + secondary_path: impl Into, + ) -> StorageResult { + let path = path.into(); + let secondary_path = secondary_path.into(); + if let Some(parent) = secondary_path.parent() { + std::fs::create_dir_all(parent).map_err(|e| StorageError::RocksDb(e.to_string()))?; + } + let mut opts = Options::default(); + opts.set_compression_type(rocksdb::DBCompressionType::Lz4); + opts.set_max_open_files(512); + let db = DB::open_as_secondary(&opts, &path, &secondary_path) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(Self { + path, + db: Arc::new(db), + read_only: true, + secondary: true, + }) + } + + /// Pull the secondary view up to the primary's current state. No-op for + /// non-secondary handles. + pub fn try_catch_up_with_primary(&self) -> StorageResult<()> { + if self.secondary { + self.db + .try_catch_up_with_primary() + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + } + Ok(()) + } + + pub(crate) fn is_secondary(&self) -> bool { + self.secondary + } + + pub fn put_blob_bytes_batch(&self, blobs: &[(String, Vec)]) -> StorageResult<()> { + if blobs.is_empty() { + return Ok(()); + } + if self.read_only { + return Err(StorageError::RocksDb(format!( + "repo-bytes DB is read-only: {}", + self.path.display() + ))); + } + let mut batch = WriteBatch::default(); + for (sha256_hex, bytes) in blobs { + validate_blob_sha256_hex(sha256_hex)?; + validate_blob_bytes(bytes)?; + let key = repo_bytes_key(sha256_hex); + batch.put(key.as_bytes(), bytes.as_slice()); + } + self.db + .write(batch) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(()) + } + + pub(crate) fn is_read_only(&self) -> bool { + self.read_only + } + + pub(crate) fn require_existing_blob_bytes_batch( + &self, + blobs: &[(String, Vec)], + ) -> StorageResult<()> { + if blobs.is_empty() { + return Ok(()); + } + let hashes = blobs + .iter() + .map(|(sha256_hex, _)| sha256_hex.clone()) + .collect::>(); + let existing = self.get_blob_bytes_batch(&hashes)?; + for ((sha256_hex, expected_bytes), actual_bytes) in blobs.iter().zip(existing) { + match actual_bytes { + Some(actual_bytes) if actual_bytes == *expected_bytes => {} + Some(_) => { + return Err(StorageError::InvalidData { + entity: "read_only_repo_bytes", + detail: format!("existing bytes differ for SHA-256 {sha256_hex}"), + }); + } + None => { + return Err(StorageError::InvalidData { + entity: "read_only_repo_bytes", + detail: format!("blob is missing for SHA-256 {sha256_hex}"), + }); + } + } + } + Ok(()) + } + + pub fn get_blob_bytes(&self, sha256_hex: &str) -> StorageResult>> { + validate_blob_sha256_hex(sha256_hex)?; + let key = repo_bytes_key(sha256_hex); + self.db + .get(key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string())) + } + + pub fn get_blob_bytes_batch( + &self, + sha256_hexes: &[String], + ) -> StorageResult>>> { + if sha256_hexes.is_empty() { + return Ok(Vec::new()); + } + let keys: Vec = sha256_hexes + .iter() + .map(|hash| { + validate_blob_sha256_hex(hash)?; + Ok::(repo_bytes_key(hash)) + }) + .collect::>()?; + self.db + .multi_get(keys.iter().map(|key| key.as_bytes())) + .into_iter() + .map(|res| res.map_err(|e| StorageError::RocksDb(e.to_string()))) + .collect() + } + + pub fn path(&self) -> &PathBuf { + &self.path + } + + pub(crate) fn memory_snapshot(&self, label: impl Into) -> RocksDbMemoryDbSnapshot { + memory_db_snapshot_for_column_families(label, self.db.as_ref(), None) + } +} + +impl RawObjectStore for RocksStore { + fn get_raw_entry(&self, sha256_hex: &str) -> StorageResult> { + self.get_raw_by_hash_entry(sha256_hex) + } + + fn get_raw_entries_batch( + &self, + sha256_hexes: &[String], + ) -> StorageResult>> { + self.get_raw_by_hash_entries_batch(sha256_hexes) + } + + fn get_blob_bytes(&self, sha256_hex: &str) -> StorageResult>> { + RocksStore::get_blob_bytes(self, sha256_hex) + } + + fn get_blob_bytes_batch(&self, sha256_hexes: &[String]) -> StorageResult>>> { + RocksStore::get_blob_bytes_batch(self, sha256_hexes) + } +} + +impl RawObjectStore for ExternalRawStoreDb { + fn get_raw_entry(&self, sha256_hex: &str) -> StorageResult> { + let key = raw_by_hash_key(sha256_hex); + let Some(bytes) = self + .db + .get(key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + else { + return Ok(None); + }; + let entry = + serde_cbor::from_slice::(&bytes).map_err(|e| StorageError::Codec { + entity: "raw_by_hash", + detail: e.to_string(), + })?; + entry.validate_internal()?; + Ok(Some(entry)) + } + + fn get_raw_entries_batch( + &self, + sha256_hexes: &[String], + ) -> StorageResult>> { + if sha256_hexes.is_empty() { + return Ok(Vec::new()); + } + let keys: Vec = sha256_hexes + .iter() + .map(|hash| raw_by_hash_key(hash)) + .collect(); + self.db + .multi_get(keys.iter().map(|key| key.as_bytes())) + .into_iter() + .map(|res| { + let maybe = res.map_err(|e| StorageError::RocksDb(e.to_string()))?; + match maybe { + Some(bytes) => { + let entry = + serde_cbor::from_slice::(&bytes).map_err(|e| { + StorageError::Codec { + entity: "raw_by_hash", + detail: e.to_string(), + } + })?; + entry.validate_internal()?; + Ok(Some(entry)) + } + None => Ok(None), + } + }) + .collect() + } + + fn get_blob_bytes(&self, sha256_hex: &str) -> StorageResult>> { + let key = raw_blob_key(sha256_hex); + self.db + .get(key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string())) + } + + fn get_blob_bytes_batch(&self, sha256_hexes: &[String]) -> StorageResult>>> { + if sha256_hexes.is_empty() { + return Ok(Vec::new()); + } + let keys: Vec = sha256_hexes.iter().map(|hash| raw_blob_key(hash)).collect(); + self.db + .multi_get(keys.iter().map(|key| key.as_bytes())) + .into_iter() + .map(|res| res.map_err(|e| StorageError::RocksDb(e.to_string()))) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::{ExternalRawStoreDb, ExternalRepoBytesDb, RawObjectStore}; + use crate::storage::{RawByHashEntry, RocksStore, StorageError, StorageResult}; + use std::collections::HashMap; + + fn sha256_hex(bytes: &[u8]) -> String { + use sha2::{Digest, Sha256}; + hex::encode(Sha256::digest(bytes)) + } + + #[derive(Default)] + struct MockRawStore { + entries: HashMap, + } + + impl RawObjectStore for MockRawStore { + fn get_raw_entry(&self, sha256_hex: &str) -> StorageResult> { + Ok(self.entries.get(sha256_hex).cloned()) + } + + fn get_raw_entries_batch( + &self, + sha256_hexes: &[String], + ) -> StorageResult>> { + Ok(sha256_hexes + .iter() + .map(|hash| self.entries.get(hash).cloned()) + .collect()) + } + } + + #[test] + fn rocks_store_raw_object_store_reads_single_and_batch_entries() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let a = b"object-a".to_vec(); + let b = b"object-b".to_vec(); + let a_hash = sha256_hex(&a); + let b_hash = sha256_hex(&b); + + store + .put_raw_by_hash_entry(&RawByHashEntry::from_bytes(a_hash.clone(), a.clone())) + .expect("put a"); + store + .put_raw_by_hash_entry(&RawByHashEntry::from_bytes(b_hash.clone(), b.clone())) + .expect("put b"); + + let single = store + .get_raw_entry(&a_hash) + .expect("get single") + .expect("present"); + assert_eq!(single.bytes, a); + + let batch = store + .get_raw_entries_batch(&[a_hash.clone(), "00".repeat(32), b_hash.clone()]) + .expect("get batch"); + assert_eq!(batch.len(), 3); + assert_eq!( + batch[0].as_ref().map(|entry| entry.bytes.as_slice()), + Some(a.as_slice()) + ); + assert!(batch[1].is_none()); + assert_eq!( + batch[2].as_ref().map(|entry| entry.bytes.as_slice()), + Some(b.as_slice()) + ); + } + + #[test] + fn external_raw_store_db_roundtrips_entries() { + let td = tempfile::tempdir().expect("tempdir"); + let raw_store = + ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store"); + + let mut entry = RawByHashEntry::from_bytes(sha256_hex(b"blob"), b"blob".to_vec()); + entry + .origin_uris + .push("rsync://example.test/repo/a.cer".to_string()); + entry.object_type = Some("cer".to_string()); + raw_store.put_raw_entry(&entry).expect("put raw entry"); + + let got = raw_store + .get_raw_entry(&entry.sha256_hex) + .expect("read raw entry") + .expect("entry exists"); + assert_eq!(got, entry); + } + + #[test] + fn external_raw_store_db_batch_writes_and_reads() { + let td = tempfile::tempdir().expect("tempdir"); + let raw_store = + ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store"); + + let a = RawByHashEntry::from_bytes(sha256_hex(b"a"), b"a".to_vec()); + let b = RawByHashEntry::from_bytes(sha256_hex(b"b"), b"b".to_vec()); + raw_store + .put_raw_entries_batch(&[a.clone(), b.clone()]) + .expect("batch put"); + + let batch = raw_store + .get_raw_entries_batch(&[a.sha256_hex.clone(), b.sha256_hex.clone()]) + .expect("batch get"); + assert_eq!(batch.len(), 2); + assert_eq!(batch[0], Some(a)); + assert_eq!(batch[1], Some(b)); + } + + #[test] + fn raw_object_store_default_blob_helpers_return_bytes_only() { + let td = tempfile::tempdir().expect("tempdir"); + let raw_store = ExternalRawStoreDb::open(td.path().join("nested/raw-store.db")) + .expect("open raw store"); + + let mut entry = RawByHashEntry::from_bytes(sha256_hex(b"blob"), b"blob".to_vec()); + entry + .origin_uris + .push("rsync://example.test/repo/blob.roa".to_string()); + raw_store.put_raw_entry(&entry).expect("put raw entry"); + + let single = raw_store + .get_blob_bytes(&entry.sha256_hex) + .expect("get blob bytes") + .expect("entry exists"); + assert_eq!(single, b"blob".to_vec()); + + let batch = raw_store + .get_blob_bytes_batch(&[entry.sha256_hex.clone(), "00".repeat(32)]) + .expect("get blob bytes batch"); + assert_eq!(batch, vec![Some(b"blob".to_vec()), None]); + } + + #[test] + fn raw_object_store_default_blob_helpers_work_for_custom_store() { + let mut store = MockRawStore::default(); + let a = RawByHashEntry::from_bytes(sha256_hex(b"a"), b"a".to_vec()); + let b = RawByHashEntry::from_bytes(sha256_hex(b"b"), b"b".to_vec()); + store.entries.insert(a.sha256_hex.clone(), a.clone()); + store.entries.insert(b.sha256_hex.clone(), b.clone()); + + let single = store + .get_blob_bytes(&a.sha256_hex) + .expect("single blob bytes") + .expect("present"); + assert_eq!(single, b"a".to_vec()); + + let batch = store + .get_blob_bytes_batch(&[a.sha256_hex.clone(), "00".repeat(32), b.sha256_hex.clone()]) + .expect("batch blob bytes"); + assert_eq!(batch, vec![Some(b"a".to_vec()), None, Some(b"b".to_vec())]); + } + + #[test] + fn rocks_store_blob_helpers_use_external_raw_store_fast_path() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open_with_external_raw_store( + &td.path().join("db"), + &td.path().join("raw-store.db"), + ) + .expect("open store with external raw store"); + + let entry = RawByHashEntry::from_bytes(sha256_hex(b"blob-fast"), b"blob-fast".to_vec()); + store.put_raw_by_hash_entry(&entry).expect("put"); + + let single = store + .get_blob_bytes(&entry.sha256_hex) + .expect("single blob bytes") + .expect("present"); + assert_eq!(single, b"blob-fast".to_vec()); + + let batch = store + .get_blob_bytes_batch(&[entry.sha256_hex.clone(), "00".repeat(32)]) + .expect("batch blob bytes"); + assert_eq!(batch, vec![Some(b"blob-fast".to_vec()), None]); + } + + #[test] + fn external_raw_store_db_delete_removes_entry() { + let td = tempfile::tempdir().expect("tempdir"); + let raw_store = + ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store"); + + let entry = RawByHashEntry::from_bytes(sha256_hex(b"gone"), b"gone".to_vec()); + raw_store.put_raw_entry(&entry).expect("put"); + assert!( + raw_store + .get_raw_entry(&entry.sha256_hex) + .unwrap() + .is_some() + ); + + raw_store + .delete_raw_entry(&entry.sha256_hex) + .expect("delete entry"); + assert!( + raw_store + .get_raw_entry(&entry.sha256_hex) + .unwrap() + .is_none() + ); + } + + #[test] + fn put_blob_bytes_batch_round_trips_without_raw_entry() { + let td = tempfile::tempdir().expect("tempdir"); + let raw_store = + ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store"); + + let a = (sha256_hex(b"blob-a"), b"blob-a".to_vec()); + let b = (sha256_hex(b"blob-b"), b"blob-b".to_vec()); + raw_store + .put_blob_bytes_batch(&[a.clone(), b.clone()]) + .expect("put blobs"); + + assert_eq!( + raw_store.get_blob_bytes(&a.0).expect("get blob a"), + Some(a.1.clone()) + ); + assert_eq!( + raw_store.get_blob_bytes(&b.0).expect("get blob b"), + Some(b.1.clone()) + ); + assert!(raw_store.get_raw_entry(&a.0).expect("get raw a").is_none()); + assert!(raw_store.get_raw_entry(&b.0).expect("get raw b").is_none()); + } + + #[test] + fn put_blob_bytes_batch_rejects_invalid_inputs() { + let td = tempfile::tempdir().expect("tempdir"); + let raw_store = + ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store"); + + let err = raw_store + .put_blob_bytes_batch(&[("zz".repeat(32), b"blob".to_vec())]) + .expect_err("invalid hash should fail"); + assert!(matches!(err, StorageError::InvalidData { .. })); + + let err = raw_store + .put_blob_bytes_batch(&[(sha256_hex(b"blob"), Vec::new())]) + .expect_err("empty bytes should fail"); + assert!(matches!(err, StorageError::InvalidData { .. })); + } + + #[test] + fn external_raw_store_db_rejects_invalid_entry_on_put() { + let td = tempfile::tempdir().expect("tempdir"); + let raw_store = + ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store"); + + let bad = RawByHashEntry { + sha256_hex: "11".repeat(32), + bytes: b"blob".to_vec(), + origin_uris: Vec::new(), + object_type: None, + encoding: None, + }; + let err = raw_store + .put_raw_entry(&bad) + .expect_err("invalid hash should fail"); + assert!(matches!(err, StorageError::InvalidData { .. })); + } + + #[test] + fn external_raw_store_db_reports_codec_error_for_corrupt_value() { + let td = tempfile::tempdir().expect("tempdir"); + let raw_store = + ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store"); + raw_store + .db + .put(b"rawbyhash:deadbeef", b"not-cbor") + .expect("inject corrupt bytes"); + + let err = raw_store + .get_raw_entry("deadbeef") + .expect_err("corrupt value should fail"); + assert!(matches!( + err, + StorageError::Codec { + entity: "raw_by_hash", + .. + } + )); + } + + #[test] + fn external_raw_store_db_batch_returns_empty_for_empty_request() { + let td = tempfile::tempdir().expect("tempdir"); + let raw_store = + ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store"); + let entries = raw_store + .get_raw_entries_batch(&[]) + .expect("empty batch succeeds"); + assert!(entries.is_empty()); + raw_store + .put_raw_entries_batch(&[]) + .expect("empty put succeeds"); + } + + #[test] + fn external_repo_bytes_db_roundtrips_blob_bytes_without_raw_entry() { + let td = tempfile::tempdir().expect("tempdir"); + let repo_bytes = + ExternalRepoBytesDb::open(td.path().join("repo-bytes.db")).expect("open repo bytes"); + let bytes = b"repo-bytes-object".to_vec(); + let hash = sha256_hex(&bytes); + + repo_bytes + .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) + .expect("put repo bytes"); + + assert_eq!( + repo_bytes.get_blob_bytes(&hash).expect("get repo bytes"), + Some(bytes.clone()) + ); + assert_eq!( + repo_bytes + .get_blob_bytes_batch(&[hash, "00".repeat(32)]) + .expect("get repo bytes batch"), + vec![Some(bytes), None] + ); + } + + #[test] + fn external_repo_bytes_db_rejects_invalid_inputs() { + let td = tempfile::tempdir().expect("tempdir"); + let repo_bytes = + ExternalRepoBytesDb::open(td.path().join("repo-bytes.db")).expect("open repo bytes"); + + assert!( + repo_bytes + .put_blob_bytes_batch(&[("not-a-valid-hash".to_string(), b"blob".to_vec())]) + .is_err() + ); + assert!( + repo_bytes + .put_blob_bytes_batch(&[(sha256_hex(b"blob"), Vec::new())]) + .is_err() + ); + assert!(repo_bytes.get_blob_bytes("not-a-valid-hash").is_err()); + } + + #[test] + fn external_repo_bytes_db_secondary_catches_up_with_live_primary() { + let td = tempfile::tempdir().expect("tempdir"); + let primary_path = td.path().join("repo-bytes.db"); + let secondary_path = td.path().join("repo-bytes.secondary"); + + let primary = ExternalRepoBytesDb::open(&primary_path).expect("open primary"); + let bytes_a = b"repo-bytes-a".to_vec(); + let hash_a = sha256_hex(&bytes_a); + primary + .put_blob_bytes_batch(&[(hash_a.clone(), bytes_a.clone())]) + .expect("put a"); + + let secondary = ExternalRepoBytesDb::open_as_secondary(&primary_path, &secondary_path) + .expect("open secondary"); + assert!(secondary.is_secondary()); + + // A secondary open does not necessarily see pre-existing data until it + // catches up with the primary's manifest. + secondary + .try_catch_up_with_primary() + .expect("initial catch up"); + assert_eq!( + secondary + .get_blob_bytes(&hash_a) + .expect("get a via secondary"), + Some(bytes_a) + ); + + // Bytes written by the primary *after* the secondary open become + // visible after another catch-up (this is the live-soak scenario). + let bytes_b = b"repo-bytes-b".to_vec(); + let hash_b = sha256_hex(&bytes_b); + primary + .put_blob_bytes_batch(&[(hash_b.clone(), bytes_b.clone())]) + .expect("put b after secondary open"); + secondary + .try_catch_up_with_primary() + .expect("second catch up"); + assert_eq!( + secondary + .get_blob_bytes(&hash_b) + .expect("get b via secondary"), + Some(bytes_b) + ); + } +} diff --git a/crates/panda-rpki-validator/src/ccr/accumulator.rs b/crates/panda-rpki-validator/src/ccr/accumulator.rs new file mode 100644 index 0000000..ce4587c --- /dev/null +++ b/crates/panda-rpki-validator/src/ccr/accumulator.rs @@ -0,0 +1,516 @@ +use std::collections::BTreeMap; + +use crate::ccr::build::{ + build_aspa_payload_state, build_roa_payload_state, build_router_key_state_from_runtime, + build_trust_anchor_state, +}; +use crate::ccr::encode::encode_manifest_state_payload_der; +use crate::ccr::hash::compute_state_hash; +use crate::ccr::manifest_location::select_manifest_signed_object_location_from_der; +use crate::ccr::model::{ + CcrDigestAlgorithm, ManifestInstance, ManifestState, RpkiCanonicalCacheRepresentation, +}; +use crate::data_model::common::BigUnsigned; +use crate::data_model::ta::TrustAnchor; +use crate::storage::VcirCcrManifestProjection; +use crate::validation::objects::{AspaAttestation, RouterKeyPayload, Vrp}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CcrManifestContribution { + pub manifest_rsync_uri: String, + pub hash: Vec, + pub size: u64, + pub aki: Vec, + pub manifest_number_be: Vec, + pub this_update: time::OffsetDateTime, + pub locations_der: Vec>, + pub subordinate_skis: Vec>, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CcrAccumulatorMemoryStats { + pub trust_anchor_count: u64, + pub manifest_count: u64, + pub estimated_heap_bytes: u64, + pub string_bytes: u64, + pub string_capacity_bytes: u64, + pub vec_payload_bytes: u64, + pub vec_capacity_bytes: u64, + pub locations_der_count: u64, + pub subordinate_ski_count: u64, + pub btree_key_capacity_bytes: u64, + pub btree_entry_shallow_bytes: u64, +} + +impl CcrManifestContribution { + fn from_projection(projection: &VcirCcrManifestProjection) -> Result { + let this_update = projection + .manifest_this_update + .parse() + .map_err(|e| format!("parse projection manifest_this_update failed: {e}"))?; + Ok(Self { + manifest_rsync_uri: projection.manifest_rsync_uri.clone(), + hash: projection.manifest_sha256.clone(), + size: projection.manifest_size, + aki: projection.manifest_ee_aki.clone(), + manifest_number_be: projection.manifest_number_be.clone(), + this_update, + locations_der: vec![select_manifest_signed_object_location_from_der( + &projection.manifest_rsync_uri, + &projection.manifest_sia_locations_der, + )?], + subordinate_skis: projection.subordinate_skis.clone(), + }) + } + + fn to_manifest_instance(&self) -> ManifestInstance { + ManifestInstance { + hash: self.hash.clone(), + size: self.size, + aki: self.aki.clone(), + manifest_number: BigUnsigned { + bytes_be: self.manifest_number_be.clone(), + }, + this_update: self.this_update, + locations: self.locations_der.clone(), + subordinates: self.subordinate_skis.clone(), + } + } + + fn add_memory_stats(&self, stats: &mut CcrAccumulatorMemoryStats) { + stats.string_bytes += self.manifest_rsync_uri.len() as u64; + stats.string_capacity_bytes += self.manifest_rsync_uri.capacity() as u64; + stats.estimated_heap_bytes += self.manifest_rsync_uri.capacity() as u64; + + add_vec_stats(&self.hash, stats); + add_vec_stats(&self.aki, stats); + add_vec_stats(&self.manifest_number_be, stats); + add_vec_of_vec_stats(&self.locations_der, stats); + add_vec_of_vec_stats(&self.subordinate_skis, stats); + stats.locations_der_count += self.locations_der.len() as u64; + stats.subordinate_ski_count += self.subordinate_skis.len() as u64; + } +} + +fn add_vec_stats(value: &Vec, stats: &mut CcrAccumulatorMemoryStats) { + stats.vec_payload_bytes += value.len() as u64; + stats.vec_capacity_bytes += value.capacity() as u64; + stats.estimated_heap_bytes += value.capacity() as u64; +} + +fn add_vec_of_vec_stats(values: &Vec>, stats: &mut CcrAccumulatorMemoryStats) { + let outer_capacity = values.capacity() * std::mem::size_of::>(); + stats.vec_payload_bytes += (values.len() * std::mem::size_of::>()) as u64; + stats.vec_capacity_bytes += outer_capacity as u64; + stats.estimated_heap_bytes += outer_capacity as u64; + for value in values { + add_vec_stats(value, stats); + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CcrAccumulator { + trust_anchors: Vec, + manifests_by_hash: BTreeMap, CcrManifestContribution>, + most_recent_update: time::OffsetDateTime, +} + +impl CcrAccumulator { + pub fn new(trust_anchors: Vec) -> Self { + Self { + trust_anchors, + manifests_by_hash: BTreeMap::new(), + most_recent_update: time::OffsetDateTime::UNIX_EPOCH, + } + } + + pub fn append_manifest_projection( + &mut self, + projection: &VcirCcrManifestProjection, + ) -> Result<(), String> { + let contribution = CcrManifestContribution::from_projection(projection)?; + match self.manifests_by_hash.get(contribution.hash.as_slice()) { + Some(existing) if existing != &contribution => { + return Err(format!( + "duplicate manifest hash with conflicting content for URI: {}", + contribution.manifest_rsync_uri + )); + } + Some(_) => {} + None => { + self.manifests_by_hash + .insert(contribution.hash.clone(), contribution.clone()); + } + } + if contribution.this_update > self.most_recent_update { + self.most_recent_update = contribution.this_update; + } + Ok(()) + } + + pub fn finish( + &self, + produced_at: time::OffsetDateTime, + vrps: &[Vrp], + aspas: &[AspaAttestation], + router_keys: &[RouterKeyPayload], + ) -> Result { + let manifest_instances = self + .manifests_by_hash + .values() + .map(CcrManifestContribution::to_manifest_instance) + .collect::>(); + let manifest_payload_der = encode_manifest_state_payload_der(&manifest_instances) + .map_err(|e| format!("manifest state encoding failed: {e}"))?; + let manifest_state = ManifestState { + mis: manifest_instances, + most_recent_update: self.most_recent_update, + hash: compute_state_hash(&manifest_payload_der), + }; + let vrp_state = build_roa_payload_state(vrps).map_err(|e| e.to_string())?; + let aspa_state = build_aspa_payload_state(aspas).map_err(|e| e.to_string())?; + let ta_state = build_trust_anchor_state(&self.trust_anchors).map_err(|e| e.to_string())?; + let router_key_state = + build_router_key_state_from_runtime(router_keys).map_err(|e| e.to_string())?; + Ok(RpkiCanonicalCacheRepresentation { + version: 0, + hash_alg: CcrDigestAlgorithm::Sha256, + produced_at, + mfts: Some(manifest_state), + vrps: Some(vrp_state), + vaps: Some(aspa_state), + tas: Some(ta_state), + rks: Some(router_key_state), + }) + } + + pub fn manifest_count(&self) -> usize { + self.manifests_by_hash.len() + } + + pub fn memory_stats(&self) -> CcrAccumulatorMemoryStats { + let mut stats = CcrAccumulatorMemoryStats { + trust_anchor_count: self.trust_anchors.len() as u64, + manifest_count: self.manifests_by_hash.len() as u64, + ..CcrAccumulatorMemoryStats::default() + }; + stats.estimated_heap_bytes += + (self.trust_anchors.capacity() * std::mem::size_of::()) as u64; + for trust_anchor in &self.trust_anchors { + add_vec_stats(&trust_anchor.tal.raw, &mut stats); + add_vec_of_string_stats(&trust_anchor.tal.comments, &mut stats); + stats.vec_payload_bytes += + (trust_anchor.tal.ta_uris.len() * std::mem::size_of::()) as u64; + stats.vec_capacity_bytes += + (trust_anchor.tal.ta_uris.capacity() * std::mem::size_of::()) as u64; + stats.estimated_heap_bytes += + (trust_anchor.tal.ta_uris.capacity() * std::mem::size_of::()) as u64; + for uri in &trust_anchor.tal.ta_uris { + stats.string_bytes += uri.as_str().len() as u64; + stats.string_capacity_bytes += uri.as_str().len() as u64; + stats.estimated_heap_bytes += uri.as_str().len() as u64; + } + add_vec_stats(&trust_anchor.tal.subject_public_key_info_der, &mut stats); + add_vec_stats(&trust_anchor.ta_certificate.raw_der, &mut stats); + if let Some(uri) = &trust_anchor.resolved_ta_uri { + stats.string_bytes += uri.as_str().len() as u64; + stats.string_capacity_bytes += uri.as_str().len() as u64; + stats.estimated_heap_bytes += uri.as_str().len() as u64; + } + } + + stats.btree_entry_shallow_bytes = (self.manifests_by_hash.len() + * (std::mem::size_of::>() + std::mem::size_of::())) + as u64; + stats.estimated_heap_bytes += stats.btree_entry_shallow_bytes; + for (key, contribution) in &self.manifests_by_hash { + stats.btree_key_capacity_bytes += key.capacity() as u64; + stats.estimated_heap_bytes += key.capacity() as u64; + contribution.add_memory_stats(&mut stats); + } + stats + } +} + +fn add_vec_of_string_stats(values: &Vec, stats: &mut CcrAccumulatorMemoryStats) { + let outer_capacity = values.capacity() * std::mem::size_of::(); + stats.vec_payload_bytes += (values.len() * std::mem::size_of::()) as u64; + stats.vec_capacity_bytes += outer_capacity as u64; + stats.estimated_heap_bytes += outer_capacity as u64; + for value in values { + stats.string_bytes += value.len() as u64; + stats.string_capacity_bytes += value.capacity() as u64; + stats.estimated_heap_bytes += value.capacity() as u64; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ccr::export::build_ccr_from_run; + use crate::ccr::verify::verify_content_info; + use crate::data_model::manifest::ManifestObject; + use crate::data_model::oid::{OID_AD_RPKI_NOTIFY, OID_AD_SIGNED_OBJECT}; + use crate::data_model::rc::{AccessDescription, SubjectInfoAccess}; + use crate::data_model::roa::{IpPrefix, RoaAfi}; + use crate::data_model::ta::TrustAnchor; + use crate::data_model::tal::Tal; + use crate::storage::{ + PackTime, RawByHashEntry, RocksStore, ValidatedCaInstanceResult, ValidatedManifestMeta, + VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus, VcirAuditSummary, + VcirCcrManifestProjection, VcirChildEntry, VcirInstanceGate, VcirRelatedArtifact, + VcirSummary, + }; + use sha2::Digest; + + fn sample_trust_anchor() -> TrustAnchor { + let base = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let tal_bytes = std::fs::read(base.join("tests/fixtures/tal/apnic-rfc7730-https.tal")) + .expect("read tal"); + let ta_der = std::fs::read(base.join("tests/fixtures/ta/apnic-ta.cer")).expect("read ta"); + let tal = Tal::decode_bytes(&tal_bytes).expect("decode tal"); + TrustAnchor::bind_der(tal, &ta_der, None).expect("bind ta") + } + + fn sample_vcir_and_manifest( + store: &RocksStore, + ) -> ( + ValidatedCaInstanceResult, + Vec, + Vec, + Vec, + ) { + let base = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let manifest_der = std::fs::read( + base.join( + "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft", + ), + ) + .expect("read manifest"); + let manifest = ManifestObject::decode_der(&manifest_der).expect("decode manifest"); + let manifest_uri = match manifest.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .extensions + .subject_info_access + .as_ref() + .expect("manifest sia") + { + SubjectInfoAccess::Ee(ee_sia) => ee_sia + .access_descriptions + .iter() + .find(|ad| { + ad.access_method_oid == OID_AD_SIGNED_OBJECT + && ad.access_location.starts_with("rsync://") + }) + .expect("manifest rsync signedObject") + .access_location + .clone(), + SubjectInfoAccess::Ca(_) => panic!("manifest ee sia should not be CA variant"), + }; + let manifest_hash = hex::encode(sha2::Sha256::digest(&manifest_der)); + let mut raw = RawByHashEntry::from_bytes(manifest_hash.clone(), manifest_der.clone()); + raw.origin_uris.push(manifest_uri.clone()); + raw.object_type = Some("mft".to_string()); + raw.encoding = Some("der".to_string()); + store.put_raw_by_hash_entry(&raw).expect("put raw"); + + let projection = VcirCcrManifestProjection { + manifest_rsync_uri: manifest_uri.clone(), + manifest_sha256: sha2::Sha256::digest(&manifest_der).to_vec(), + manifest_size: manifest_der.len() as u64, + manifest_ee_aki: manifest.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .extensions + .authority_key_identifier + .clone() + .expect("manifest aki"), + manifest_number_be: manifest.manifest.manifest_number.bytes_be.clone(), + manifest_this_update: PackTime::from_utc_offset_datetime(manifest.manifest.this_update), + manifest_sia_locations_der: match manifest.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .extensions + .subject_info_access + .as_ref() + .expect("manifest sia") + { + SubjectInfoAccess::Ee(ee_sia) => vec![ + crate::ccr::manifest_location::select_manifest_signed_object_location( + &manifest_uri, + &ee_sia.access_descriptions, + ) + .expect("select manifest signedObject"), + ], + SubjectInfoAccess::Ca(_) => panic!("manifest ee sia should not be CA variant"), + }, + subordinate_skis: vec![vec![0x33; 20]], + }; + + let vcir = ValidatedCaInstanceResult { + manifest_rsync_uri: manifest_uri.clone(), + parent_manifest_rsync_uri: None, + tal_id: "apnic".to_string(), + ca_subject_name: "CN=test".to_string(), + ca_ski: "11".repeat(20), + issuer_ski: "22".repeat(20), + last_successful_validation_time: PackTime::from_utc_offset_datetime( + manifest.manifest.this_update, + ), + current_manifest_rsync_uri: manifest_uri.clone(), + current_crl_rsync_uri: format!("{manifest_uri}.crl"), + validated_manifest_meta: ValidatedManifestMeta { + validated_manifest_number: manifest.manifest.manifest_number.bytes_be.clone(), + validated_manifest_this_update: PackTime::from_utc_offset_datetime( + manifest.manifest.this_update, + ), + validated_manifest_next_update: PackTime::from_utc_offset_datetime( + manifest.manifest.next_update, + ), + }, + ccr_manifest_projection: projection.clone(), + instance_gate: VcirInstanceGate { + manifest_next_update: PackTime::from_utc_offset_datetime( + manifest.manifest.next_update, + ), + current_crl_next_update: PackTime::from_utc_offset_datetime( + manifest.manifest.next_update, + ), + self_ca_not_after: PackTime::from_utc_offset_datetime( + manifest.manifest.next_update, + ), + instance_effective_until: PackTime::from_utc_offset_datetime( + manifest.manifest.next_update, + ), + }, + child_entries: vec![VcirChildEntry { + child_manifest_rsync_uri: "rsync://example.test/repo/child.mft".to_string(), + child_cert_rsync_uri: "rsync://example.test/repo/child.cer".to_string(), + child_cert_hash: "aa".repeat(32), + child_ski: "33".repeat(20), + child_rsync_base_uri: "rsync://example.test/repo/".to_string(), + child_publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), + child_rrdp_notification_uri: None, + child_effective_ip_resources: None, + child_effective_as_resources: None, + accepted_at_validation_time: PackTime::from_utc_offset_datetime( + manifest.manifest.this_update, + ), + }], + local_outputs: Vec::new(), + related_artifacts: vec![VcirRelatedArtifact { + artifact_role: VcirArtifactRole::Manifest, + artifact_kind: VcirArtifactKind::Mft, + uri: Some(manifest_uri.clone()), + sha256: manifest_hash, + object_type: Some("mft".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }], + summary: VcirSummary { + local_vrp_count: 0, + local_aspa_count: 0, + local_router_key_count: 0, + child_count: 1, + accepted_object_count: 1, + rejected_object_count: 0, + }, + audit_summary: VcirAuditSummary { + failed_fetch_eligible: true, + last_failed_fetch_reason: None, + warning_count: 0, + audit_flags: Vec::new(), + }, + }; + + let vrps = vec![Vrp { + asn: 64496, + prefix: IpPrefix { + afi: RoaAfi::Ipv4, + prefix_len: 8, + addr: [10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + }, + max_length: 8, + }]; + let aspas = vec![AspaAttestation { + customer_as_id: 64496, + provider_as_ids: vec![64497], + }]; + let router_keys = vec![RouterKeyPayload { + as_id: 64496, + ski: vec![0x11; 20], + spki_der: vec![0x30, 0x00], + source_object_uri: "rsync://example.test/repo/router.cer".to_string(), + source_object_hash: hex::encode([0x11; 32]), + source_ee_cert_hash: hex::encode([0x11; 32]), + item_effective_until: PackTime::from_utc_offset_datetime( + time::OffsetDateTime::now_utc() + time::Duration::hours(1), + ), + }]; + (vcir, vrps, aspas, router_keys) + } + + #[test] + fn accumulator_finish_matches_builder_on_fresh_vcir_inputs() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let (vcir, vrps, aspas, router_keys) = sample_vcir_and_manifest(&store); + store.put_vcir(&vcir).expect("put vcir"); + let trust_anchor = sample_trust_anchor(); + + let builder_ccr = build_ccr_from_run( + &store, + &[trust_anchor.clone()], + &vrps, + &aspas, + &router_keys, + time::OffsetDateTime::now_utc(), + ) + .expect("build ccr from run"); + + let mut accumulator = CcrAccumulator::new(vec![trust_anchor]); + accumulator + .append_manifest_projection(&vcir.ccr_manifest_projection) + .expect("append manifest projection"); + let accumulated_ccr = accumulator + .finish(time::OffsetDateTime::now_utc(), &vrps, &aspas, &router_keys) + .expect("finish accumulator"); + + assert_eq!(builder_ccr.mfts, accumulated_ccr.mfts); + assert_eq!(builder_ccr.vrps, accumulated_ccr.vrps); + assert_eq!(builder_ccr.vaps, accumulated_ccr.vaps); + assert_eq!(builder_ccr.tas, accumulated_ccr.tas); + assert_eq!(builder_ccr.rks, accumulated_ccr.rks); + let ci = crate::ccr::model::CcrContentInfo::new(accumulated_ccr); + verify_content_info(&ci).expect("verify accumulated ccr"); + } + + #[test] + fn accumulator_sanitizes_historical_projection_with_rpki_notify() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let (mut vcir, _, _, _) = sample_vcir_and_manifest(&store); + let notify_der = + crate::ccr::manifest_location::encode_access_description_der(&AccessDescription { + access_method_oid: OID_AD_RPKI_NOTIFY.to_string(), + access_location: "https://rrdp.example.test/notification.xml".to_string(), + }) + .expect("encode rpkiNotify"); + let expected_location = vcir.ccr_manifest_projection.manifest_sia_locations_der[0].clone(); + vcir.ccr_manifest_projection + .manifest_sia_locations_der + .push(notify_der); + + let mut accumulator = CcrAccumulator::new(Vec::new()); + accumulator + .append_manifest_projection(&vcir.ccr_manifest_projection) + .expect("sanitize historical projection"); + let contribution = accumulator + .manifests_by_hash + .values() + .next() + .expect("manifest contribution"); + assert_eq!(contribution.locations_der, vec![expected_location]); + } +} diff --git a/crates/panda-rpki-validator/src/ccr/build.rs b/crates/panda-rpki-validator/src/ccr/build.rs new file mode 100644 index 0000000..cb82450 --- /dev/null +++ b/crates/panda-rpki-validator/src/ccr/build.rs @@ -0,0 +1,975 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use serde::Serialize; +use sha2::Digest; + +use crate::ccr::encode::{ + encode_aspa_payload_state_payload_der, encode_manifest_state_payload_der, + encode_roa_payload_state_payload_der, encode_router_key_state_payload_der, + encode_trust_anchor_state_payload_der, +}; +use crate::ccr::hash::compute_state_hash; +use crate::ccr::manifest_location::select_manifest_signed_object_location; +use crate::ccr::model::{ + AspaPayloadSet, AspaPayloadState, ManifestInstance, ManifestState, RoaPayloadSet, + RoaPayloadState, RouterKey, RouterKeySet, RouterKeyState, TrustAnchorState, +}; +use crate::data_model::manifest::ManifestObject; +use crate::data_model::rc::SubjectInfoAccess; +use crate::data_model::roa::RoaAfi; +use crate::data_model::router_cert::BgpsecRouterCertificate; +use crate::data_model::ta::TrustAnchor; +use crate::storage::{RocksStore, ValidatedCaInstanceResult, VcirArtifactRole}; +use crate::validation::objects::{AspaAttestation, RouterKeyPayload, Vrp}; + +#[derive(Debug, thiserror::Error)] +pub enum CcrBuildError { + #[error("trust anchor set must not be empty")] + EmptyTrustAnchors, + + #[error("trust anchor certificate missing SubjectKeyIdentifier")] + MissingTrustAnchorSki, + + #[error("ROA payload state encoding failed: {0}")] + RoaEncode(String), + + #[error("ASPA payload state encoding failed: {0}")] + AspaEncode(String), + + #[error("TrustAnchor state encoding failed: {0}")] + TrustAnchorEncode(String), + + #[error("manifest artifact missing in VCIR: {0}")] + MissingManifestArtifact(String), + + #[error("manifest raw bytes missing in store for {manifest_rsync_uri}: {sha256_hex}")] + MissingManifestRawBytes { + manifest_rsync_uri: String, + sha256_hex: String, + }, + + #[error("manifest raw bytes load failed for {manifest_rsync_uri}: {detail}")] + LoadManifestRawBytes { + manifest_rsync_uri: String, + detail: String, + }, + + #[error("manifest decode failed for {manifest_rsync_uri}: {detail}")] + ManifestDecode { + manifest_rsync_uri: String, + detail: String, + }, + + #[error("manifest EE certificate missing AuthorityKeyIdentifier: {0}")] + ManifestEeMissingAki(String), + + #[error("manifest EE certificate missing Subject Information Access extension: {0}")] + ManifestEeMissingSia(String), + + #[error("manifest EE certificate SIA is not the EE form: {0}")] + ManifestEeSiaWrongVariant(String), + + #[error( + "manifest EE certificate SIA location selection failed for {manifest_rsync_uri}: {detail}" + )] + ManifestSiaLocationSelection { + manifest_rsync_uri: String, + detail: String, + }, + + #[error("manifest child SKI is not valid lowercase hex or not 20 bytes: {0}")] + InvalidChildSki(String), + + #[error("manifest state encoding failed: {0}")] + ManifestEncode(String), + + #[error("duplicate manifest hash with conflicting content for URI: {0}")] + DuplicateManifestHashConflict(String), + + #[error("router key state encoding failed: {0}")] + RouterKeyEncode(String), +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct ManifestStateBuildBreakdown { + pub vcir_count: usize, + pub unique_manifest_count: usize, + pub find_manifest_artifact_ms: u64, + pub load_manifest_blob_ms: u64, + pub decode_manifest_der_ms: u64, + pub build_manifest_instance_ms: u64, + pub dedup_manifest_instance_ms: u64, + pub encode_manifest_state_ms: u64, + pub total_ms: u64, +} + +pub fn build_roa_payload_state(vrps: &[Vrp]) -> Result { + let mut grouped: BTreeMap> = BTreeMap::new(); + for vrp in vrps { + grouped + .entry(vrp.asn) + .or_default() + .insert(RoaPayloadKey::from_vrp(vrp)); + } + + let mut rps = Vec::with_capacity(grouped.len()); + for (asn, entries) in grouped { + let mut families: BTreeMap> = BTreeMap::new(); + for entry in entries { + families.entry(entry.afi).or_default().push(entry); + } + let mut ip_addr_blocks = Vec::with_capacity(families.len()); + for (afi, entries) in families { + ip_addr_blocks.push(encode_roa_ip_address_family(afi, &entries)); + } + rps.push(RoaPayloadSet { + as_id: asn, + ip_addr_blocks, + }); + } + + let payload_der = encode_roa_payload_state_payload_der(&rps) + .map_err(|e| CcrBuildError::RoaEncode(e.to_string()))?; + Ok(RoaPayloadState { + rps, + hash: compute_state_hash(&payload_der), + }) +} + +pub fn build_aspa_payload_state( + attestations: &[AspaAttestation], +) -> Result { + let mut grouped: BTreeMap> = BTreeMap::new(); + for attestation in attestations { + grouped + .entry(attestation.customer_as_id) + .or_default() + .extend(attestation.provider_as_ids.iter().copied()); + } + + let aps: Vec = grouped + .into_iter() + .map(|(customer_as_id, providers)| AspaPayloadSet { + customer_as_id, + providers: providers.into_iter().collect(), + }) + .collect(); + + let payload_der = encode_aspa_payload_state_payload_der(&aps) + .map_err(|e| CcrBuildError::AspaEncode(e.to_string()))?; + Ok(AspaPayloadState { + aps, + hash: compute_state_hash(&payload_der), + }) +} + +pub fn build_trust_anchor_state( + trust_anchors: &[TrustAnchor], +) -> Result { + if trust_anchors.is_empty() { + return Err(CcrBuildError::EmptyTrustAnchors); + } + let mut skis: BTreeSet> = BTreeSet::new(); + for ta in trust_anchors { + let ski = ta + .ta_certificate + .rc_ca + .tbs + .extensions + .subject_key_identifier + .clone() + .ok_or(CcrBuildError::MissingTrustAnchorSki)?; + skis.insert(ski); + } + let skis: Vec> = skis.into_iter().collect(); + let payload_der = encode_trust_anchor_state_payload_der(&skis) + .map_err(|e| CcrBuildError::TrustAnchorEncode(e.to_string()))?; + Ok(TrustAnchorState { + skis, + hash: compute_state_hash(&payload_der), + }) +} + +pub fn build_manifest_state_from_vcirs( + store: &RocksStore, + vcirs: &[ValidatedCaInstanceResult], +) -> Result { + build_manifest_state_from_vcirs_with_breakdown(store, vcirs).map(|(state, _)| state) +} + +pub fn build_manifest_state_from_vcirs_with_breakdown( + store: &RocksStore, + vcirs: &[ValidatedCaInstanceResult], +) -> Result<(ManifestState, ManifestStateBuildBreakdown), CcrBuildError> { + let total_started = std::time::Instant::now(); + let mut breakdown = ManifestStateBuildBreakdown { + vcir_count: vcirs.len(), + ..ManifestStateBuildBreakdown::default() + }; + let mut find_manifest_artifact_duration = std::time::Duration::ZERO; + let mut load_manifest_blob_duration = std::time::Duration::ZERO; + let mut decode_manifest_der_duration = std::time::Duration::ZERO; + let mut build_manifest_instance_duration = std::time::Duration::ZERO; + let mut dedup_manifest_instance_duration = std::time::Duration::ZERO; + let mut mis_by_hash: BTreeMap, ManifestInstance> = BTreeMap::new(); + let mut most_recent_update = time::OffsetDateTime::UNIX_EPOCH; + + for vcir in vcirs { + let started = std::time::Instant::now(); + let manifest_artifact = vcir + .related_artifacts + .iter() + .find(|artifact| { + artifact.artifact_role == VcirArtifactRole::Manifest + && artifact.uri.as_deref() == Some(vcir.current_manifest_rsync_uri.as_str()) + }) + .ok_or_else(|| { + CcrBuildError::MissingManifestArtifact(vcir.current_manifest_rsync_uri.clone()) + })?; + find_manifest_artifact_duration += started.elapsed(); + + let started = std::time::Instant::now(); + let raw_bytes = store + .get_blob_bytes(&manifest_artifact.sha256) + .map_err(|e| CcrBuildError::LoadManifestRawBytes { + manifest_rsync_uri: vcir.current_manifest_rsync_uri.clone(), + detail: e.to_string(), + })? + .ok_or_else(|| CcrBuildError::MissingManifestRawBytes { + manifest_rsync_uri: vcir.current_manifest_rsync_uri.clone(), + sha256_hex: manifest_artifact.sha256.clone(), + })?; + load_manifest_blob_duration += started.elapsed(); + + let started = std::time::Instant::now(); + let manifest = + ManifestObject::decode_der(&raw_bytes).map_err(|e| CcrBuildError::ManifestDecode { + manifest_rsync_uri: vcir.current_manifest_rsync_uri.clone(), + detail: e.to_string(), + })?; + decode_manifest_der_duration += started.elapsed(); + + let started = std::time::Instant::now(); + let ee = &manifest.signed_object.signed_data.certificates[0].resource_cert; + let aki = ee + .tbs + .extensions + .authority_key_identifier + .clone() + .ok_or_else(|| { + CcrBuildError::ManifestEeMissingAki(vcir.current_manifest_rsync_uri.clone()) + })?; + let sia = ee + .tbs + .extensions + .subject_info_access + .as_ref() + .ok_or_else(|| { + CcrBuildError::ManifestEeMissingSia(vcir.current_manifest_rsync_uri.clone()) + })?; + let locations = match sia { + SubjectInfoAccess::Ee(ee_sia) => vec![ + select_manifest_signed_object_location( + &vcir.current_manifest_rsync_uri, + &ee_sia.access_descriptions, + ) + .map_err(|detail| CcrBuildError::ManifestSiaLocationSelection { + manifest_rsync_uri: vcir.current_manifest_rsync_uri.clone(), + detail, + })?, + ], + SubjectInfoAccess::Ca(_) => { + return Err(CcrBuildError::ManifestEeSiaWrongVariant( + vcir.current_manifest_rsync_uri.clone(), + )); + } + }; + + let subordinates = collect_subordinate_ski_bytes(vcir)?; + let this_update = vcir + .validated_manifest_meta + .validated_manifest_this_update + .parse() + .map_err(|e| CcrBuildError::ManifestDecode { + manifest_rsync_uri: vcir.current_manifest_rsync_uri.clone(), + detail: format!("invalid validated_manifest_this_update: {e}"), + })?; + if this_update > most_recent_update { + most_recent_update = this_update; + } + + let instance = ManifestInstance { + hash: sha2::Sha256::digest(&raw_bytes).to_vec(), + size: raw_bytes.len() as u64, + aki, + manifest_number: crate::data_model::common::BigUnsigned { + bytes_be: vcir + .validated_manifest_meta + .validated_manifest_number + .clone(), + }, + this_update, + locations, + subordinates, + }; + build_manifest_instance_duration += started.elapsed(); + + let started = std::time::Instant::now(); + match mis_by_hash.get(instance.hash.as_slice()) { + Some(existing) if existing != &instance => { + return Err(CcrBuildError::DuplicateManifestHashConflict( + vcir.current_manifest_rsync_uri.clone(), + )); + } + Some(_) => {} + None => { + mis_by_hash.insert(instance.hash.clone(), instance); + } + } + dedup_manifest_instance_duration += started.elapsed(); + } + + let started = std::time::Instant::now(); + let mis: Vec = mis_by_hash.into_values().collect(); + let payload_der = encode_manifest_state_payload_der(&mis) + .map_err(|e| CcrBuildError::ManifestEncode(e.to_string()))?; + let state = ManifestState { + mis, + most_recent_update, + hash: compute_state_hash(&payload_der), + }; + breakdown.encode_manifest_state_ms = started.elapsed().as_millis() as u64; + breakdown.find_manifest_artifact_ms = find_manifest_artifact_duration.as_millis() as u64; + breakdown.load_manifest_blob_ms = load_manifest_blob_duration.as_millis() as u64; + breakdown.decode_manifest_der_ms = decode_manifest_der_duration.as_millis() as u64; + breakdown.build_manifest_instance_ms = build_manifest_instance_duration.as_millis() as u64; + breakdown.dedup_manifest_instance_ms = dedup_manifest_instance_duration.as_millis() as u64; + breakdown.unique_manifest_count = state.mis.len(); + breakdown.total_ms = total_started.elapsed().as_millis() as u64; + Ok((state, breakdown)) +} + +fn collect_subordinate_ski_bytes( + vcir: &ValidatedCaInstanceResult, +) -> Result>, CcrBuildError> { + let mut skis = BTreeSet::new(); + for child in &vcir.child_entries { + let bytes = hex::decode(&child.child_ski) + .map_err(|_| CcrBuildError::InvalidChildSki(child.child_ski.clone()))?; + if bytes.len() != 20 { + return Err(CcrBuildError::InvalidChildSki(child.child_ski.clone())); + } + skis.insert(bytes); + } + Ok(skis.into_iter().collect()) +} + +pub fn build_router_key_state( + router_certs: &[BgpsecRouterCertificate], +) -> Result { + let mut grouped: BTreeMap> = BTreeMap::new(); + for cert in router_certs { + let key = RouterKey { + ski: cert.subject_key_identifier.clone(), + spki_der: cert.spki_der.clone(), + }; + for asn in &cert.asns { + grouped.entry(*asn).or_default().insert(key.clone()); + } + } + + let rksets: Vec = grouped + .into_iter() + .map(|(as_id, router_keys)| RouterKeySet { + as_id, + router_keys: router_keys.into_iter().collect(), + }) + .collect(); + + let payload_der = encode_router_key_state_payload_der(&rksets) + .map_err(|e| CcrBuildError::RouterKeyEncode(e.to_string()))?; + Ok(RouterKeyState { + rksets, + hash: compute_state_hash(&payload_der), + }) +} + +pub fn build_router_key_state_from_runtime( + router_keys: &[RouterKeyPayload], +) -> Result { + let mut grouped: BTreeMap> = BTreeMap::new(); + for router_key in router_keys { + grouped + .entry(router_key.as_id) + .or_default() + .insert(RouterKey { + ski: router_key.ski.clone(), + spki_der: router_key.spki_der.clone(), + }); + } + + let rksets: Vec = grouped + .into_iter() + .map(|(as_id, router_keys)| RouterKeySet { + as_id, + router_keys: router_keys.into_iter().collect(), + }) + .collect(); + + let payload_der = encode_router_key_state_payload_der(&rksets) + .map_err(|e| CcrBuildError::RouterKeyEncode(e.to_string()))?; + Ok(RouterKeyState { + rksets, + hash: compute_state_hash(&payload_der), + }) +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +struct RoaPayloadKey { + afi: u16, + addr: Vec, + prefix_len: u8, + max_length: u8, +} + +impl RoaPayloadKey { + fn from_vrp(vrp: &Vrp) -> Self { + let afi = match vrp.prefix.afi { + RoaAfi::Ipv4 => 1, + RoaAfi::Ipv6 => 2, + }; + Self { + afi, + addr: vrp.prefix.addr.to_vec(), + prefix_len: vrp.prefix.prefix_len as u8, + max_length: vrp.max_length as u8, + } + } +} + +fn encode_roa_ip_address_family(afi: u16, entries: &[RoaPayloadKey]) -> Vec { + let address_family = afi.to_be_bytes(); + let addresses = entries + .iter() + .map(encode_roa_ip_address) + .collect::>(); + encode_sequence(&[ + encode_octet_string(&address_family), + encode_sequence(&addresses), + ]) +} + +fn encode_roa_ip_address(entry: &RoaPayloadKey) -> Vec { + let (unused_bits, content) = encode_prefix_bit_string(&entry.addr, entry.prefix_len); + let mut fields = vec![encode_bit_string(unused_bits, &content)]; + if entry.max_length != entry.prefix_len { + fields.push(encode_integer_u8(entry.max_length)); + } + encode_sequence(&fields) +} + +fn encode_prefix_bit_string(addr: &[u8], prefix_len: u8) -> (u8, Vec) { + if prefix_len == 0 { + return (0, Vec::new()); + } + let octets = ((prefix_len as usize) + 7) / 8; + let mut content = addr[..octets].to_vec(); + let rem = prefix_len % 8; + let unused = if rem == 0 { 0 } else { 8 - rem }; + if unused > 0 { + let mask = 0xFFu8 << unused; + let last = content.last_mut().expect("octets > 0"); + *last &= mask; + } + (unused, content) +} + +fn encode_integer_u8(v: u8) -> Vec { + encode_integer_bytes(vec![v]) +} + +fn encode_integer_bytes(mut bytes: Vec) -> Vec { + if bytes.is_empty() { + bytes.push(0); + } + if bytes[0] & 0x80 != 0 { + bytes.insert(0, 0); + } + encode_tlv(0x02, bytes) +} + +fn encode_bit_string(unused_bits: u8, content: &[u8]) -> Vec { + let mut value = Vec::with_capacity(content.len() + 1); + value.push(unused_bits); + value.extend_from_slice(content); + encode_tlv(0x03, value) +} + +fn encode_octet_string(bytes: &[u8]) -> Vec { + encode_tlv(0x04, bytes.to_vec()) +} + +fn encode_sequence(elements: &[Vec]) -> Vec { + let total_len: usize = elements.iter().map(Vec::len).sum(); + let mut buf = Vec::with_capacity(total_len); + for element in elements { + buf.extend_from_slice(element); + } + encode_tlv(0x30, buf) +} + +fn encode_tlv(tag: u8, value: Vec) -> Vec { + let mut out = Vec::with_capacity(1 + 9 + value.len()); + out.push(tag); + encode_length(value.len(), &mut out); + out.extend_from_slice(&value); + out +} + +fn encode_length(len: usize, out: &mut Vec) { + if len < 0x80 { + out.push(len as u8); + return; + } + let mut bytes = Vec::new(); + let mut value = len; + while value > 0 { + bytes.push((value & 0xFF) as u8); + value >>= 8; + } + bytes.reverse(); + out.push(0x80 | (bytes.len() as u8)); + out.extend_from_slice(&bytes); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ccr::decode::decode_content_info; + use crate::ccr::encode::{ + encode_aspa_payload_state, encode_content_info, encode_trust_anchor_state, + }; + use crate::ccr::model::{CcrContentInfo, CcrDigestAlgorithm, RpkiCanonicalCacheRepresentation}; + use crate::data_model::roa::{IpPrefix, RoaAfi}; + use crate::data_model::ta::TrustAnchor; + use crate::data_model::tal::Tal; + + fn sample_vrp_v4(asn: u32, a: [u8; 4], prefix_len: u16, max_length: u16) -> Vrp { + Vrp { + asn, + prefix: IpPrefix { + afi: RoaAfi::Ipv4, + prefix_len, + addr: [a[0], a[1], a[2], a[3], 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + }, + max_length, + } + } + + fn sample_vrp_v6(asn: u32, a: [u8; 16], prefix_len: u16, max_length: u16) -> Vrp { + Vrp { + asn, + prefix: IpPrefix { + afi: RoaAfi::Ipv6, + prefix_len, + addr: a, + }, + max_length, + } + } + + fn decode_family_block(block: &[u8]) -> (u16, Vec<(u8, Vec, Option)>) { + let mut top = crate::data_model::common::DerReader::new(block); + let mut seq = top.take_sequence().expect("family seq"); + assert!(top.is_empty()); + let afi_bytes = seq.take_octet_string().expect("afi octet string"); + let afi = u16::from_be_bytes([afi_bytes[0], afi_bytes[1]]); + let mut addrs = seq.take_sequence().expect("addresses seq"); + let mut entries = Vec::new(); + while !addrs.is_empty() { + let mut addr_seq = addrs.take_sequence().expect("roa ip addr seq"); + let (unused_bits, content) = addr_seq.take_bit_string().expect("bit string"); + let prefix_len = (content.len() * 8) as u8 - unused_bits; + let max_len = if addr_seq.is_empty() { + None + } else { + Some(addr_seq.take_uint_u64().expect("maxlen") as u8) + }; + entries.push((prefix_len, content.to_vec(), max_len)); + } + (afi, entries) + } + + fn sample_trust_anchor(path_tal: &str, path_ta: &str) -> TrustAnchor { + let base = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let tal_bytes = std::fs::read(base.join(path_tal)).expect("read tal"); + let ta_der = std::fs::read(base.join(path_ta)).expect("read ta"); + let tal = Tal::decode_bytes(&tal_bytes).expect("decode tal"); + TrustAnchor::bind_der(tal, &ta_der, None).expect("bind ta") + } + + fn sample_router_cert(asns: Vec, ski_fill: u8, spki_fill: u8) -> BgpsecRouterCertificate { + let ta = sample_trust_anchor( + "tests/fixtures/tal/apnic-rfc7730-https.tal", + "tests/fixtures/ta/apnic-ta.cer", + ); + BgpsecRouterCertificate { + raw_der: vec![0x01, 0x02], + resource_cert: ta.ta_certificate.rc_ca, + subject_key_identifier: vec![ski_fill; 20], + spki_der: vec![0x30, 0x03, 0x03, 0x01, spki_fill], + asns, + } + } + + fn sample_manifest_vcir(manifest_der: &[u8], child_ski_hex: &str) -> ValidatedCaInstanceResult { + let manifest = ManifestObject::decode_der(manifest_der).expect("decode manifest fixture"); + let manifest_uri = match manifest.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .extensions + .subject_info_access + .as_ref() + .expect("manifest sia") + { + SubjectInfoAccess::Ee(ee_sia) => ee_sia + .access_descriptions + .iter() + .find(|ad| { + ad.access_method_oid == crate::data_model::oid::OID_AD_SIGNED_OBJECT + && ad.access_location.starts_with("rsync://") + }) + .expect("manifest rsync signedObject") + .access_location + .clone(), + SubjectInfoAccess::Ca(_) => panic!("manifest ee sia should not be CA variant"), + }; + let hash = hex::encode(sha2::Sha256::digest(manifest_der)); + let now = manifest.manifest.this_update; + let next = manifest.manifest.next_update; + let projection = crate::storage::VcirCcrManifestProjection { + manifest_rsync_uri: manifest_uri.clone(), + manifest_sha256: sha2::Sha256::digest(manifest_der).to_vec(), + manifest_size: manifest_der.len() as u64, + manifest_ee_aki: manifest.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .extensions + .authority_key_identifier + .clone() + .expect("manifest aki"), + manifest_number_be: manifest.manifest.manifest_number.bytes_be.clone(), + manifest_this_update: crate::storage::PackTime::from_utc_offset_datetime(now), + manifest_sia_locations_der: match manifest.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .extensions + .subject_info_access + .as_ref() + .expect("manifest sia") + { + SubjectInfoAccess::Ee(ee_sia) => vec![ + select_manifest_signed_object_location( + &manifest_uri, + &ee_sia.access_descriptions, + ) + .expect("select manifest signedObject"), + ], + SubjectInfoAccess::Ca(_) => panic!("manifest ee sia should not be CA variant"), + }, + subordinate_skis: vec![hex::decode(child_ski_hex).expect("decode child ski")], + }; + crate::storage::ValidatedCaInstanceResult { + manifest_rsync_uri: manifest_uri.clone(), + parent_manifest_rsync_uri: None, + tal_id: "test-tal".to_string(), + ca_subject_name: "CN=test".to_string(), + ca_ski: "11".repeat(20), + issuer_ski: "22".repeat(20), + last_successful_validation_time: crate::storage::PackTime::from_utc_offset_datetime( + now, + ), + current_manifest_rsync_uri: manifest_uri.clone(), + current_crl_rsync_uri: format!("{manifest_uri}.crl"), + validated_manifest_meta: crate::storage::ValidatedManifestMeta { + validated_manifest_number: manifest.manifest.manifest_number.bytes_be.clone(), + validated_manifest_this_update: crate::storage::PackTime::from_utc_offset_datetime( + now, + ), + validated_manifest_next_update: crate::storage::PackTime::from_utc_offset_datetime( + next, + ), + }, + ccr_manifest_projection: projection, + instance_gate: crate::storage::VcirInstanceGate { + manifest_next_update: crate::storage::PackTime::from_utc_offset_datetime(next), + current_crl_next_update: crate::storage::PackTime::from_utc_offset_datetime(next), + self_ca_not_after: crate::storage::PackTime::from_utc_offset_datetime(next), + instance_effective_until: crate::storage::PackTime::from_utc_offset_datetime(next), + }, + child_entries: vec![crate::storage::VcirChildEntry { + child_manifest_rsync_uri: format!("{manifest_uri}/child.mft"), + child_cert_rsync_uri: format!("{manifest_uri}/child.cer"), + child_cert_hash: "aa".repeat(32), + child_ski: child_ski_hex.to_string(), + child_rsync_base_uri: format!("{manifest_uri}/"), + child_publication_point_rsync_uri: format!("{manifest_uri}/"), + child_rrdp_notification_uri: None, + child_effective_ip_resources: None, + child_effective_as_resources: None, + accepted_at_validation_time: crate::storage::PackTime::from_utc_offset_datetime( + now, + ), + }], + local_outputs: Vec::new(), + related_artifacts: vec![crate::storage::VcirRelatedArtifact { + artifact_role: crate::storage::VcirArtifactRole::Manifest, + artifact_kind: crate::storage::VcirArtifactKind::Mft, + uri: Some(manifest_uri.clone()), + sha256: hash, + object_type: Some("mft".to_string()), + validation_status: crate::storage::VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }], + summary: crate::storage::VcirSummary { + local_vrp_count: 0, + local_aspa_count: 0, + local_router_key_count: 0, + child_count: 1, + accepted_object_count: 1, + rejected_object_count: 0, + }, + audit_summary: crate::storage::VcirAuditSummary { + failed_fetch_eligible: true, + last_failed_fetch_reason: None, + warning_count: 0, + audit_flags: Vec::new(), + }, + } + } + + #[test] + fn build_manifest_state_from_vcirs_collects_current_manifests_and_hashes_payload() { + let base = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let manifest_a = std::fs::read(base.join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft")).expect("read manifest a"); + let manifest_b = std::fs::read(base.join( + "tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.mft", + )) + .expect("read manifest b"); + let vcir_a = sample_manifest_vcir(&manifest_a, &"33".repeat(20)); + let vcir_b = sample_manifest_vcir(&manifest_b, &"44".repeat(20)); + let store_dir = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + for (vcir, bytes) in [(&vcir_a, &manifest_a), (&vcir_b, &manifest_b)] { + let artifact = &vcir.related_artifacts[0]; + let mut raw = + crate::storage::RawByHashEntry::from_bytes(artifact.sha256.clone(), bytes.to_vec()); + raw.origin_uris + .push(vcir.current_manifest_rsync_uri.clone()); + raw.object_type = Some("mft".to_string()); + raw.encoding = Some("der".to_string()); + store.put_raw_by_hash_entry(&raw).expect("put raw manifest"); + } + + let state = build_manifest_state_from_vcirs(&store, &[vcir_a.clone(), vcir_b.clone()]) + .expect("build manifest state"); + assert_eq!(state.mis.len(), 2); + assert!(state.mis[0].hash < state.mis[1].hash); + let expected_subordinates = [ + hex::decode(vcir_a.child_entries[0].child_ski.clone()).unwrap(), + hex::decode(vcir_b.child_entries[0].child_ski.clone()).unwrap(), + ]; + let actual_subordinates = state + .mis + .iter() + .map(|mi| { + assert_eq!(mi.subordinates.len(), 1); + assert!(!mi.locations.is_empty()); + mi.subordinates[0].clone() + }) + .collect::>(); + let expected_subordinates = expected_subordinates + .into_iter() + .collect::>(); + assert_eq!(actual_subordinates, expected_subordinates); + let payload_der = + encode_manifest_state_payload_der(&state.mis).expect("encode mis payload"); + assert!(crate::ccr::verify_state_hash(&state.hash, &payload_der)); + let max_time = [ + vcir_a + .validated_manifest_meta + .validated_manifest_this_update + .parse() + .unwrap(), + vcir_b + .validated_manifest_meta + .validated_manifest_this_update + .parse() + .unwrap(), + ] + .into_iter() + .max() + .unwrap(); + assert_eq!(state.most_recent_update, max_time); + } + + #[test] + fn build_manifest_state_from_vcirs_empty_uses_epoch() { + let store_dir = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let state = build_manifest_state_from_vcirs(&store, &[]).expect("empty manifest state"); + assert!(state.mis.is_empty()); + assert_eq!(state.most_recent_update, time::OffsetDateTime::UNIX_EPOCH); + let payload_der = + encode_manifest_state_payload_der(&state.mis).expect("encode mis payload"); + assert!(crate::ccr::verify_state_hash(&state.hash, &payload_der)); + } + + #[test] + fn build_router_key_state_groups_dedupes_and_sorts() { + let certs = vec![ + sample_router_cert(vec![64497], 0x21, 0xA1), + sample_router_cert(vec![64496, 64497], 0x11, 0xA0), + sample_router_cert(vec![64496], 0x11, 0xA0), + sample_router_cert(vec![64496], 0x11, 0xA2), + ]; + let state = build_router_key_state(&certs).expect("build router key state"); + assert_eq!(state.rksets.len(), 2); + assert_eq!(state.rksets[0].as_id, 64496); + assert_eq!(state.rksets[1].as_id, 64497); + assert_eq!(state.rksets[0].router_keys.len(), 2); + assert_eq!(state.rksets[1].router_keys.len(), 2); + assert!(state.rksets[0].router_keys[0].ski <= state.rksets[0].router_keys[1].ski); + let payload_der = + encode_router_key_state_payload_der(&state.rksets).expect("encode rk payload"); + assert!(crate::ccr::verify_state_hash(&state.hash, &payload_der)); + } + + #[test] + fn build_router_key_state_empty_is_valid_and_hashes_empty_sequence() { + let state = build_router_key_state(&[]).expect("empty router key state"); + assert!(state.rksets.is_empty()); + let payload_der = + encode_router_key_state_payload_der(&state.rksets).expect("encode rk payload"); + assert!(crate::ccr::verify_state_hash(&state.hash, &payload_der)); + } + + #[test] + fn build_roa_payload_state_groups_dedupes_and_sorts() { + let vrps = vec![ + sample_vrp_v4(64497, [10, 1, 0, 0], 16, 16), + sample_vrp_v4(64496, [10, 0, 0, 0], 8, 8), + sample_vrp_v4(64496, [10, 0, 0, 0], 8, 8), + sample_vrp_v4(64496, [10, 1, 0, 0], 16, 24), + sample_vrp_v6( + 64496, + [0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + 32, + 48, + ), + ]; + let state = build_roa_payload_state(&vrps).expect("build roa state"); + assert_eq!(state.rps.len(), 2); + assert_eq!(state.rps[0].as_id, 64496); + assert_eq!(state.rps[1].as_id, 64497); + assert_eq!(state.rps[0].ip_addr_blocks.len(), 2); + let (afi4, entries4) = decode_family_block(&state.rps[0].ip_addr_blocks[0]); + let (afi6, entries6) = decode_family_block(&state.rps[0].ip_addr_blocks[1]); + assert_eq!(afi4, 1); + assert_eq!(afi6, 2); + assert_eq!(entries4.len(), 2); + assert_eq!(entries4[0], (8, vec![10], None)); + assert_eq!(entries4[1], (16, vec![10, 1], Some(24))); + assert_eq!(entries6.len(), 1); + assert_eq!(entries6[0], (32, vec![0x20, 0x01, 0x0d, 0xb8], Some(48))); + let payload_der = encode_roa_payload_state_payload_der(&state.rps).expect("encode payload"); + assert!(crate::ccr::verify_state_hash(&state.hash, &payload_der)); + } + + #[test] + fn build_roa_payload_state_empty_is_valid_and_hashes_empty_sequence() { + let state = build_roa_payload_state(&[]).expect("empty roa state"); + assert!(state.rps.is_empty()); + let payload_der = encode_roa_payload_state_payload_der(&state.rps).expect("encode payload"); + assert!(crate::ccr::verify_state_hash(&state.hash, &payload_der)); + } + + #[test] + fn build_aspa_payload_state_merges_and_sorts() { + let aspas = vec![ + AspaAttestation { + customer_as_id: 64497, + provider_as_ids: vec![65002], + }, + AspaAttestation { + customer_as_id: 64496, + provider_as_ids: vec![65003, 65001], + }, + AspaAttestation { + customer_as_id: 64496, + provider_as_ids: vec![65002, 65001], + }, + ]; + let state = build_aspa_payload_state(&aspas).expect("build aspa state"); + assert_eq!(state.aps.len(), 2); + assert_eq!(state.aps[0].customer_as_id, 64496); + assert_eq!(state.aps[0].providers, vec![65001, 65002, 65003]); + assert_eq!(state.aps[1].customer_as_id, 64497); + let encoded = encode_aspa_payload_state(&state).expect("encode aspa state"); + let decoded = decode_content_info( + &encode_content_info(&CcrContentInfo::new(RpkiCanonicalCacheRepresentation { + version: 0, + hash_alg: CcrDigestAlgorithm::Sha256, + produced_at: time::OffsetDateTime::now_utc(), + mfts: None, + vrps: None, + vaps: Some(state.clone()), + tas: None, + rks: None, + })) + .expect("encode ccr"), + ) + .expect("decode ccr"); + assert_eq!(decoded.content.vaps, Some(state)); + assert!(!encoded.is_empty()); + } + + #[test] + fn build_trust_anchor_state_collects_sorted_unique_skis() { + let apnic = sample_trust_anchor( + "tests/fixtures/tal/apnic-rfc7730-https.tal", + "tests/fixtures/ta/apnic-ta.cer", + ); + let arin = sample_trust_anchor( + "tests/fixtures/tal/arin.tal", + "tests/fixtures/ta/arin-ta.cer", + ); + let state = build_trust_anchor_state(&[apnic.clone(), arin.clone(), apnic]) + .expect("build ta state"); + assert_eq!(state.skis.len(), 2); + assert!(state.skis[0] < state.skis[1]); + let payload_der = + encode_trust_anchor_state_payload_der(&state.skis).expect("encode ta payload"); + assert!(crate::ccr::verify_state_hash(&state.hash, &payload_der)); + let encoded = encode_trust_anchor_state(&state).expect("encode ta state"); + assert!(!encoded.is_empty()); + } + + #[test] + fn build_trust_anchor_state_rejects_empty_and_missing_ski() { + let err = build_trust_anchor_state(&[]).expect_err("empty TAs must fail"); + assert!(err.to_string().contains("trust anchor set"), "{err}"); + + let mut ta = sample_trust_anchor( + "tests/fixtures/tal/apnic-rfc7730-https.tal", + "tests/fixtures/ta/apnic-ta.cer", + ); + ta.ta_certificate + .rc_ca + .tbs + .extensions + .subject_key_identifier = None; + let err = build_trust_anchor_state(&[ta]).expect_err("missing ski must fail"); + assert!(err.to_string().contains("SubjectKeyIdentifier"), "{err}"); + } +} diff --git a/crates/panda-rpki-validator/src/ccr/compare_view.rs b/crates/panda-rpki-validator/src/ccr/compare_view.rs new file mode 100644 index 0000000..b424f49 --- /dev/null +++ b/crates/panda-rpki-validator/src/ccr/compare_view.rs @@ -0,0 +1,235 @@ +use std::collections::BTreeSet; +use std::io::Write; +use std::path::Path; + +use crate::ccr::{CcrContentInfo, extract_vrp_rows}; +use crate::validation::objects::{AspaAttestation, Vrp}; + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct VrpCompareRow { + pub asn: String, + pub ip_prefix: String, + pub max_length: String, + pub trust_anchor: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct VapCompareRow { + pub customer_asn: String, + pub providers: String, + pub trust_anchor: String, +} + +fn normalize_asn(asn: u32) -> String { + format!("AS{asn}") +} + +pub fn canonical_vrp_prefix(prefix: &crate::data_model::roa::IpPrefix) -> String { + let mut addr = prefix.addr_bytes().to_vec(); + let total_bits = match prefix.afi { + crate::data_model::roa::RoaAfi::Ipv4 => 32usize, + crate::data_model::roa::RoaAfi::Ipv6 => 128usize, + }; + let keep = usize::from(prefix.prefix_len); + for bit in keep..total_bits { + let byte = bit / 8; + let offset = 7 - (bit % 8); + addr[byte] &= !(1u8 << offset); + } + match prefix.afi { + crate::data_model::roa::RoaAfi::Ipv4 => { + let ipv4 = std::net::Ipv4Addr::new(addr[0], addr[1], addr[2], addr[3]); + format!("{ipv4}/{}", prefix.prefix_len) + } + crate::data_model::roa::RoaAfi::Ipv6 => { + let mut bytes = [0u8; 16]; + bytes.copy_from_slice(&addr[..16]); + let ipv6 = std::net::Ipv6Addr::from(bytes); + format!("{ipv6}/{}", prefix.prefix_len) + } + } +} + +pub fn build_vrp_compare_rows(vrps: &[Vrp]) -> BTreeSet { + vrps.iter() + .map(|vrp| VrpCompareRow { + asn: normalize_asn(vrp.asn), + ip_prefix: canonical_vrp_prefix(&vrp.prefix), + max_length: vrp.max_length.to_string(), + trust_anchor: "unknown".to_string(), + }) + .collect() +} + +pub fn build_vap_compare_rows(aspas: &[AspaAttestation]) -> BTreeSet { + aspas + .iter() + .map(|aspa| { + let mut providers = aspa.provider_as_ids.iter().copied().collect::>(); + providers.sort_unstable(); + providers.dedup(); + VapCompareRow { + customer_asn: normalize_asn(aspa.customer_as_id), + providers: providers + .into_iter() + .map(normalize_asn) + .collect::>() + .join(";"), + trust_anchor: "unknown".to_string(), + } + }) + .collect() +} + +pub fn decode_ccr_compare_views( + content_info: &CcrContentInfo, +) -> Result<(BTreeSet, BTreeSet), String> { + let vrps = extract_vrp_rows(content_info) + .map_err(|e| format!("extract vrp rows from ccr failed: {e}"))? + .into_iter() + .map(|(asn, prefix, max_length)| VrpCompareRow { + asn: normalize_asn(asn), + ip_prefix: prefix, + max_length: max_length.to_string(), + trust_anchor: "unknown".to_string(), + }) + .collect::>(); + + let vaps = content_info + .content + .vaps + .as_ref() + .map(|state| { + state + .aps + .iter() + .map(|vap| VapCompareRow { + customer_asn: normalize_asn(vap.customer_as_id), + providers: vap + .providers + .iter() + .copied() + .map(normalize_asn) + .collect::>() + .join(";"), + trust_anchor: "unknown".to_string(), + }) + .collect::>() + }) + .unwrap_or_default(); + + Ok((vrps, vaps)) +} + +pub fn write_vrp_csv(path: &Path, rows: &BTreeSet) -> Result<(), String> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| format!("create parent dirs failed: {}: {e}", parent.display()))?; + } + let mut file = std::io::BufWriter::new( + std::fs::File::create(path) + .map_err(|e| format!("create file failed: {}: {e}", path.display()))?, + ); + writeln!(file, "ASN,IP Prefix,Max Length,Trust Anchor").map_err(|e| e.to_string())?; + for row in rows { + writeln!( + file, + "{},{},{},{}", + row.asn, row.ip_prefix, row.max_length, row.trust_anchor + ) + .map_err(|e| e.to_string())?; + } + Ok(()) +} + +pub fn write_vap_csv(path: &Path, rows: &BTreeSet) -> Result<(), String> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| format!("create parent dirs failed: {}: {e}", parent.display()))?; + } + let mut file = std::io::BufWriter::new( + std::fs::File::create(path) + .map_err(|e| format!("create file failed: {}: {e}", path.display()))?, + ); + writeln!(file, "Customer ASN,Providers,Trust Anchor").map_err(|e| e.to_string())?; + for row in rows { + writeln!( + file, + "{},{},{}", + row.customer_asn, row.providers, row.trust_anchor + ) + .map_err(|e| e.to_string())?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ccr::{ + CcrContentInfo, CcrDigestAlgorithm, RpkiCanonicalCacheRepresentation, + build_aspa_payload_state, build_roa_payload_state, + }; + use crate::data_model::roa::{IpPrefix, RoaAfi}; + + #[test] + fn build_vap_compare_rows_sorts_and_dedups_providers() { + let rows = build_vap_compare_rows(&[AspaAttestation { + customer_as_id: 64496, + provider_as_ids: vec![64498, 64497, 64498], + }]); + let row = rows.iter().next().expect("one row"); + assert_eq!(row.customer_asn, "AS64496"); + assert_eq!(row.providers, "AS64497;AS64498"); + assert_eq!(row.trust_anchor, "unknown"); + } + + #[test] + fn decode_ccr_compare_views_extracts_vrps_and_vaps() { + let vrps = build_roa_payload_state(&[Vrp { + asn: 64496, + prefix: IpPrefix { + afi: RoaAfi::Ipv4, + prefix_len: 24, + addr: [192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + }, + max_length: 24, + }]) + .expect("build vrps"); + let vaps = build_aspa_payload_state(&[AspaAttestation { + customer_as_id: 64496, + provider_as_ids: vec![64497], + }]) + .expect("build vaps"); + let content = CcrContentInfo::new(RpkiCanonicalCacheRepresentation { + version: 0, + hash_alg: CcrDigestAlgorithm::Sha256, + produced_at: time::OffsetDateTime::now_utc(), + mfts: None, + vrps: Some(vrps), + vaps: Some(vaps), + tas: None, + rks: None, + }); + let (vrp_rows, vap_rows) = + decode_ccr_compare_views(&content).expect("decode compare views"); + assert_eq!(vrp_rows.len(), 1); + assert_eq!(vap_rows.len(), 1); + assert_eq!(vap_rows.iter().next().unwrap().providers, "AS64497"); + } + + #[test] + fn build_vrp_compare_rows_canonicalizes_ipv6_prefix_text() { + let rows = build_vrp_compare_rows(&[Vrp { + asn: 64496, + prefix: IpPrefix { + afi: RoaAfi::Ipv6, + prefix_len: 32, + addr: [0x20, 0x01, 0x0d, 0xb8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], + }, + max_length: 48, + }]); + let row = rows.iter().next().expect("row"); + assert_eq!(row.ip_prefix, "2001:db8::/32"); + } +} diff --git a/crates/panda-rpki-validator/src/ccr/decode.rs b/crates/panda-rpki-validator/src/ccr/decode.rs new file mode 100644 index 0000000..e999add --- /dev/null +++ b/crates/panda-rpki-validator/src/ccr/decode.rs @@ -0,0 +1,541 @@ +use crate::ccr::model::{ + AspaPayloadSet, AspaPayloadState, CCR_VERSION_V0, CcrContentInfo, CcrDigestAlgorithm, + ManifestInstance, ManifestState, RoaPayloadSet, RoaPayloadState, RouterKey, RouterKeySet, + RouterKeyState, RpkiCanonicalCacheRepresentation, TrustAnchorState, +}; +use crate::data_model::common::{BigUnsigned, DerReader}; +use crate::data_model::oid::{OID_CT_RPKI_CCR, OID_CT_RPKI_CCR_RAW, OID_SHA256, OID_SHA256_RAW}; +use der_parser::der::parse_der_oid; + +#[derive(Debug, thiserror::Error)] +pub enum CcrDecodeError { + #[error("DER parse error: {0}")] + Parse(String), + + #[error("unexpected contentType OID: expected {expected}, got {actual}")] + UnexpectedContentType { + expected: &'static str, + actual: String, + }, + + #[error("unexpected digest algorithm OID: expected {expected}, got {actual}")] + UnexpectedDigestAlgorithm { + expected: &'static str, + actual: String, + }, + + #[error("CCR model validation failed after decode: {0}")] + Validate(String), +} + +pub fn decode_content_info(der: &[u8]) -> Result { + let mut top = DerReader::new(der); + let mut seq = top.take_sequence().map_err(CcrDecodeError::Parse)?; + if !top.is_empty() { + return Err(CcrDecodeError::Parse( + "trailing bytes after ContentInfo".into(), + )); + } + let content_type_raw = seq.take_tag(0x06).map_err(CcrDecodeError::Parse)?; + if content_type_raw != OID_CT_RPKI_CCR_RAW { + return Err(CcrDecodeError::UnexpectedContentType { + expected: OID_CT_RPKI_CCR, + actual: oid_string(content_type_raw)?, + }); + } + let inner = seq.take_tag(0xA0).map_err(CcrDecodeError::Parse)?; + if !seq.is_empty() { + return Err(CcrDecodeError::Parse( + "trailing fields in ContentInfo".into(), + )); + } + let content = decode_ccr(inner)?; + let ci = CcrContentInfo::new(content); + ci.validate().map_err(CcrDecodeError::Validate)?; + Ok(ci) +} + +pub fn decode_ccr(der: &[u8]) -> Result { + let mut top = DerReader::new(der); + let mut seq = top.take_sequence().map_err(CcrDecodeError::Parse)?; + if !top.is_empty() { + return Err(CcrDecodeError::Parse("trailing bytes after CCR".into())); + } + + let version = if !seq.is_empty() && seq.peek_tag().map_err(CcrDecodeError::Parse)? == 0xA0 { + let explicit = seq.take_tag(0xA0).map_err(CcrDecodeError::Parse)?; + let mut inner = DerReader::new(explicit); + let version = inner.take_uint_u64().map_err(CcrDecodeError::Parse)? as u32; + if !inner.is_empty() { + return Err(CcrDecodeError::Parse( + "trailing bytes inside CCR version EXPLICIT".into(), + )); + } + version + } else { + CCR_VERSION_V0 + }; + + let hash_alg = decode_digest_algorithm(seq.take_sequence().map_err(CcrDecodeError::Parse)?)?; + let produced_at = parse_generalized_time(seq.take_tag(0x18).map_err(CcrDecodeError::Parse)?)?; + + let mut mfts = None; + let mut vrps = None; + let mut vaps = None; + let mut tas = None; + let mut rks = None; + while !seq.is_empty() { + let tag = seq.peek_tag().map_err(CcrDecodeError::Parse)?; + let (tag_read, value) = seq.take_any().map_err(CcrDecodeError::Parse)?; + debug_assert_eq!(tag, tag_read); + match tag { + 0xA1 => mfts = Some(decode_manifest_state(value)?), + 0xA2 => vrps = Some(decode_roa_payload_state(value)?), + 0xA3 => vaps = Some(decode_aspa_payload_state(value)?), + 0xA4 => tas = Some(decode_trust_anchor_state(value)?), + 0xA5 => rks = Some(decode_router_key_state(value)?), + _ => { + return Err(CcrDecodeError::Parse(format!( + "unexpected CCR field tag 0x{tag:02X}" + ))); + } + } + } + + let ccr = RpkiCanonicalCacheRepresentation { + version, + hash_alg, + produced_at, + mfts, + vrps, + vaps, + tas, + rks, + }; + ccr.validate().map_err(CcrDecodeError::Validate)?; + Ok(ccr) +} + +fn decode_manifest_state(explicit_der: &[u8]) -> Result { + let mut outer = DerReader::new(explicit_der); + let mut seq = outer.take_sequence().map_err(CcrDecodeError::Parse)?; + if !outer.is_empty() { + return Err(CcrDecodeError::Parse( + "trailing bytes after ManifestState".into(), + )); + } + let mis_der = seq.take_tag(0x30).map_err(CcrDecodeError::Parse)?; + let mut mis_reader = DerReader::new(mis_der); + let mut mis = Vec::new(); + while !mis_reader.is_empty() { + let (_tag, full, _value) = mis_reader.take_any_full().map_err(CcrDecodeError::Parse)?; + mis.push(decode_manifest_instance(full)?); + } + let most_recent_update = + parse_generalized_time(seq.take_tag(0x18).map_err(CcrDecodeError::Parse)?)?; + let hash = seq + .take_octet_string() + .map_err(CcrDecodeError::Parse)? + .to_vec(); + if !seq.is_empty() { + return Err(CcrDecodeError::Parse( + "trailing fields in ManifestState".into(), + )); + } + Ok(ManifestState { + mis, + most_recent_update, + hash, + }) +} + +fn decode_manifest_instance(der: &[u8]) -> Result { + let mut top = DerReader::new(der); + let mut seq = top.take_sequence().map_err(CcrDecodeError::Parse)?; + if !top.is_empty() { + return Err(CcrDecodeError::Parse( + "trailing bytes after ManifestInstance".into(), + )); + } + let hash = seq + .take_octet_string() + .map_err(CcrDecodeError::Parse)? + .to_vec(); + let size = seq.take_uint_u64().map_err(CcrDecodeError::Parse)?; + let aki = seq + .take_octet_string() + .map_err(CcrDecodeError::Parse)? + .to_vec(); + let manifest_number = decode_big_unsigned(seq.take_tag(0x02).map_err(CcrDecodeError::Parse)?)?; + let this_update = parse_generalized_time(seq.take_tag(0x18).map_err(CcrDecodeError::Parse)?)?; + let locations_der = seq.take_tag(0x30).map_err(CcrDecodeError::Parse)?; + let mut locations_reader = DerReader::new(locations_der); + let mut locations = Vec::new(); + while !locations_reader.is_empty() { + let (_tag, full, _value) = locations_reader + .take_any_full() + .map_err(CcrDecodeError::Parse)?; + locations.push(full.to_vec()); + } + let subordinates = if !seq.is_empty() { + let subordinate_der = seq.take_tag(0x30).map_err(CcrDecodeError::Parse)?; + let mut reader = DerReader::new(subordinate_der); + let mut out = Vec::new(); + while !reader.is_empty() { + out.push( + reader + .take_octet_string() + .map_err(CcrDecodeError::Parse)? + .to_vec(), + ); + } + out + } else { + Vec::new() + }; + Ok(ManifestInstance { + hash, + size, + aki, + manifest_number, + this_update, + locations, + subordinates, + }) +} + +fn decode_roa_payload_state(explicit_der: &[u8]) -> Result { + let mut outer = DerReader::new(explicit_der); + let mut seq = outer.take_sequence().map_err(CcrDecodeError::Parse)?; + if !outer.is_empty() { + return Err(CcrDecodeError::Parse( + "trailing bytes after ROAPayloadState".into(), + )); + } + let payload_der = seq.take_tag(0x30).map_err(CcrDecodeError::Parse)?; + let mut reader = DerReader::new(payload_der); + let mut rps = Vec::new(); + while !reader.is_empty() { + let (_tag, full, _value) = reader.take_any_full().map_err(CcrDecodeError::Parse)?; + rps.push(decode_roa_payload_set(full)?); + } + let hash = seq + .take_octet_string() + .map_err(CcrDecodeError::Parse)? + .to_vec(); + Ok(RoaPayloadState { rps, hash }) +} + +fn decode_roa_payload_set(der: &[u8]) -> Result { + let mut top = DerReader::new(der); + let mut seq = top.take_sequence().map_err(CcrDecodeError::Parse)?; + if !top.is_empty() { + return Err(CcrDecodeError::Parse( + "trailing bytes after ROAPayloadSet".into(), + )); + } + let as_id = seq.take_uint_u64().map_err(CcrDecodeError::Parse)? as u32; + let blocks_der = seq.take_tag(0x30).map_err(CcrDecodeError::Parse)?; + let mut reader = DerReader::new(blocks_der); + let mut ip_addr_blocks = Vec::new(); + while !reader.is_empty() { + let (_tag, full, _value) = reader.take_any_full().map_err(CcrDecodeError::Parse)?; + ip_addr_blocks.push(full.to_vec()); + } + Ok(RoaPayloadSet { + as_id, + ip_addr_blocks, + }) +} + +fn decode_aspa_payload_state(explicit_der: &[u8]) -> Result { + let mut outer = DerReader::new(explicit_der); + let mut seq = outer.take_sequence().map_err(CcrDecodeError::Parse)?; + if !outer.is_empty() { + return Err(CcrDecodeError::Parse( + "trailing bytes after ASPAPayloadState".into(), + )); + } + let payload_der = seq.take_tag(0x30).map_err(CcrDecodeError::Parse)?; + let mut reader = DerReader::new(payload_der); + let mut aps = Vec::new(); + while !reader.is_empty() { + let (_tag, full, _value) = reader.take_any_full().map_err(CcrDecodeError::Parse)?; + aps.push(decode_aspa_payload_set(full)?); + } + let hash = seq + .take_octet_string() + .map_err(CcrDecodeError::Parse)? + .to_vec(); + Ok(AspaPayloadState { aps, hash }) +} + +fn decode_aspa_payload_set(der: &[u8]) -> Result { + let mut top = DerReader::new(der); + let mut seq = top.take_sequence().map_err(CcrDecodeError::Parse)?; + if !top.is_empty() { + return Err(CcrDecodeError::Parse( + "trailing bytes after ASPAPayloadSet".into(), + )); + } + let customer_as_id = seq.take_uint_u64().map_err(CcrDecodeError::Parse)? as u32; + let providers_der = seq.take_tag(0x30).map_err(CcrDecodeError::Parse)?; + let mut reader = DerReader::new(providers_der); + let mut providers = Vec::new(); + while !reader.is_empty() { + providers.push(reader.take_uint_u64().map_err(CcrDecodeError::Parse)? as u32); + } + Ok(AspaPayloadSet { + customer_as_id, + providers, + }) +} + +fn decode_trust_anchor_state(explicit_der: &[u8]) -> Result { + let mut outer = DerReader::new(explicit_der); + let mut seq = outer.take_sequence().map_err(CcrDecodeError::Parse)?; + if !outer.is_empty() { + return Err(CcrDecodeError::Parse( + "trailing bytes after TrustAnchorState".into(), + )); + } + let skis_der = seq.take_tag(0x30).map_err(CcrDecodeError::Parse)?; + let mut reader = DerReader::new(skis_der); + let mut skis = Vec::new(); + while !reader.is_empty() { + skis.push( + reader + .take_octet_string() + .map_err(CcrDecodeError::Parse)? + .to_vec(), + ); + } + let hash = seq + .take_octet_string() + .map_err(CcrDecodeError::Parse)? + .to_vec(); + Ok(TrustAnchorState { skis, hash }) +} + +fn decode_router_key_state(explicit_der: &[u8]) -> Result { + let mut outer = DerReader::new(explicit_der); + let mut seq = outer.take_sequence().map_err(CcrDecodeError::Parse)?; + if !outer.is_empty() { + return Err(CcrDecodeError::Parse( + "trailing bytes after RouterKeyState".into(), + )); + } + let sets_der = seq.take_tag(0x30).map_err(CcrDecodeError::Parse)?; + let mut reader = DerReader::new(sets_der); + let mut rksets = Vec::new(); + while !reader.is_empty() { + let (_tag, full, _value) = reader.take_any_full().map_err(CcrDecodeError::Parse)?; + rksets.push(decode_router_key_set(full)?); + } + let hash = seq + .take_octet_string() + .map_err(CcrDecodeError::Parse)? + .to_vec(); + Ok(RouterKeyState { rksets, hash }) +} + +fn decode_router_key_set(der: &[u8]) -> Result { + let mut top = DerReader::new(der); + let mut seq = top.take_sequence().map_err(CcrDecodeError::Parse)?; + if !top.is_empty() { + return Err(CcrDecodeError::Parse( + "trailing bytes after RouterKeySet".into(), + )); + } + let as_id = seq.take_uint_u64().map_err(CcrDecodeError::Parse)? as u32; + let keys_der = seq.take_tag(0x30).map_err(CcrDecodeError::Parse)?; + let mut reader = DerReader::new(keys_der); + let mut router_keys = Vec::new(); + while !reader.is_empty() { + let (_tag, full, _value) = reader.take_any_full().map_err(CcrDecodeError::Parse)?; + router_keys.push(decode_router_key(full)?); + } + Ok(RouterKeySet { as_id, router_keys }) +} + +fn decode_router_key(der: &[u8]) -> Result { + let mut top = DerReader::new(der); + let mut seq = top.take_sequence().map_err(CcrDecodeError::Parse)?; + if !top.is_empty() { + return Err(CcrDecodeError::Parse( + "trailing bytes after RouterKey".into(), + )); + } + let ski = seq + .take_octet_string() + .map_err(CcrDecodeError::Parse)? + .to_vec(); + let (_tag, full, _value) = seq.take_any_full().map_err(CcrDecodeError::Parse)?; + if !seq.is_empty() { + return Err(CcrDecodeError::Parse("trailing fields in RouterKey".into())); + } + Ok(RouterKey { + ski, + spki_der: full.to_vec(), + }) +} + +fn decode_digest_algorithm(mut seq: DerReader<'_>) -> Result { + let oid_raw = seq.take_tag(0x06).map_err(CcrDecodeError::Parse)?; + if oid_raw != OID_SHA256_RAW { + return Err(CcrDecodeError::UnexpectedDigestAlgorithm { + expected: OID_SHA256, + actual: oid_string(oid_raw)?, + }); + } + if !seq.is_empty() { + let tag = seq.peek_tag().map_err(CcrDecodeError::Parse)?; + if tag == 0x05 { + let null = seq.take_tag(0x05).map_err(CcrDecodeError::Parse)?; + if !null.is_empty() { + return Err(CcrDecodeError::Parse( + "AlgorithmIdentifier NULL parameters must be empty".into(), + )); + } + } + } + if !seq.is_empty() { + return Err(CcrDecodeError::Parse( + "trailing fields in DigestAlgorithmIdentifier".into(), + )); + } + Ok(CcrDigestAlgorithm::Sha256) +} + +fn oid_string(raw_body: &[u8]) -> Result { + let der = { + let mut out = Vec::with_capacity(raw_body.len() + 2); + out.push(0x06); + if raw_body.len() < 0x80 { + out.push(raw_body.len() as u8); + } else { + return Err(CcrDecodeError::Parse("OID too long".into())); + } + out.extend_from_slice(raw_body); + out + }; + let (_rem, oid) = parse_der_oid(&der).map_err(|e| CcrDecodeError::Parse(e.to_string()))?; + let oid = oid + .as_oid_val() + .map_err(|e| CcrDecodeError::Parse(e.to_string()))?; + Ok(oid.to_string()) +} + +fn parse_generalized_time(bytes: &[u8]) -> Result { + let s = std::str::from_utf8(bytes).map_err(|e| CcrDecodeError::Parse(e.to_string()))?; + if s.len() != 15 || !s.ends_with('Z') { + return Err(CcrDecodeError::Parse( + "GeneralizedTime must be YYYYMMDDHHMMSSZ".into(), + )); + } + let parse = |range: std::ops::Range| -> Result { + s[range] + .parse::() + .map_err(|e| CcrDecodeError::Parse(e.to_string())) + }; + let year = parse(0..4)? as i32; + let month = parse(4..6)? as u8; + let day = parse(6..8)? as u8; + let hour = parse(8..10)? as u8; + let minute = parse(10..12)? as u8; + let second = parse(12..14)? as u8; + let month = time::Month::try_from(month).map_err(|e| CcrDecodeError::Parse(e.to_string()))?; + let date = time::Date::from_calendar_date(year, month, day) + .map_err(|e| CcrDecodeError::Parse(e.to_string()))?; + let timev = time::Time::from_hms(hour, minute, second) + .map_err(|e| CcrDecodeError::Parse(e.to_string()))?; + Ok(time::PrimitiveDateTime::new(date, timev).assume_utc()) +} + +fn decode_big_unsigned(bytes: &[u8]) -> Result { + if bytes.is_empty() { + return Err(CcrDecodeError::Parse("INTEGER has empty content".into())); + } + if bytes[0] & 0x80 != 0 { + return Err(CcrDecodeError::Parse("INTEGER must be non-negative".into())); + } + if bytes.len() > 1 && bytes[0] == 0x00 && (bytes[1] & 0x80) == 0 { + return Err(CcrDecodeError::Parse( + "INTEGER not minimally encoded".into(), + )); + } + let bytes_be = if bytes.len() > 1 && bytes[0] == 0x00 { + bytes[1..].to_vec() + } else { + bytes.to_vec() + }; + Ok(BigUnsigned { bytes_be }) +} + +#[cfg(all(test, feature = "full"))] +mod tests { + use super::*; + use crate::ccr::encode::{encode_content_info, encode_manifest_state_payload_der}; + use crate::ccr::hash::compute_state_hash; + use crate::ccr::manifest_location::encode_access_description_der; + use crate::ccr::model::{ + CcrContentInfo, CcrDigestAlgorithm, ManifestState, RpkiCanonicalCacheRepresentation, + }; + use crate::data_model::oid::{OID_AD_RPKI_NOTIFY, OID_AD_SIGNED_OBJECT}; + use crate::data_model::rc::AccessDescription; + + #[test] + fn generic_decoder_preserves_multiple_manifest_locations() { + let signed_object = encode_access_description_der(&AccessDescription { + access_method_oid: OID_AD_SIGNED_OBJECT.to_string(), + access_location: "rsync://example.test/repository/current.mft".to_string(), + }) + .expect("encode signedObject"); + let rpki_notify = encode_access_description_der(&AccessDescription { + access_method_oid: OID_AD_RPKI_NOTIFY.to_string(), + access_location: "https://rrdp.example.test/notification.xml".to_string(), + }) + .expect("encode rpkiNotify"); + let manifest = ManifestInstance { + hash: vec![0x11; 32], + size: 1024, + aki: vec![0x22; 20], + manifest_number: BigUnsigned { bytes_be: vec![1] }, + this_update: time::OffsetDateTime::parse( + "2026-07-20T00:00:00Z", + &time::format_description::well_known::Rfc3339, + ) + .expect("parse time"), + locations: vec![signed_object, rpki_notify], + subordinates: Vec::new(), + }; + let manifest_payload = encode_manifest_state_payload_der(&[manifest.clone()]) + .expect("encode manifest payload"); + let content = CcrContentInfo::new(RpkiCanonicalCacheRepresentation { + version: 0, + hash_alg: CcrDigestAlgorithm::Sha256, + produced_at: time::OffsetDateTime::parse( + "2026-07-20T00:00:00Z", + &time::format_description::well_known::Rfc3339, + ) + .expect("parse time"), + mfts: Some(ManifestState { + mis: vec![manifest], + most_recent_update: time::OffsetDateTime::parse( + "2026-07-20T00:00:00Z", + &time::format_description::well_known::Rfc3339, + ) + .expect("parse time"), + hash: compute_state_hash(&manifest_payload), + }), + vrps: None, + vaps: None, + tas: None, + rks: None, + }); + + let encoded = encode_content_info(&content).expect("encode CCR"); + let decoded = decode_content_info(&encoded).expect("decode CCR"); + assert_eq!(decoded.content.mfts.unwrap().mis[0].locations.len(), 2); + } +} diff --git a/crates/panda-rpki-validator/src/ccr/dump.rs b/crates/panda-rpki-validator/src/ccr/dump.rs new file mode 100644 index 0000000..37d8390 --- /dev/null +++ b/crates/panda-rpki-validator/src/ccr/dump.rs @@ -0,0 +1,112 @@ +use crate::ccr::decode::{CcrDecodeError, decode_content_info}; +use serde_json::json; + +#[derive(Debug, thiserror::Error)] +pub enum CcrDumpError { + #[error("CCR decode failed: {0}")] + Decode(#[from] CcrDecodeError), + + #[error("format time failed: {0}")] + FormatTime(String), +} + +pub fn dump_content_info_json_value(der: &[u8]) -> Result { + let content_info = decode_content_info(der)?; + dump_content_info_json(&content_info) +} + +pub fn dump_content_info_json( + content_info: &crate::ccr::model::CcrContentInfo, +) -> Result { + let produced_at = content_info + .content + .produced_at + .to_offset(time::UtcOffset::UTC) + .format(&time::format_description::well_known::Rfc3339) + .map_err(|e| CcrDumpError::FormatTime(e.to_string()))?; + + let mfts = content_info.content.mfts.as_ref().map(|state| { + json!({ + "present": true, + "manifest_instances": state.mis.len(), + "most_recent_update_rfc3339_utc": state.most_recent_update.to_offset(time::UtcOffset::UTC).format(&time::format_description::well_known::Rfc3339).unwrap(), + "hash_hex": hex::encode(&state.hash), + }) + }).unwrap_or_else(|| json!({"present": false})); + + let vrps_total = content_info + .content + .vrps + .as_ref() + .map(|state| { + state + .rps + .iter() + .map(|set| set.ip_addr_blocks.len()) + .sum::() + }) + .unwrap_or(0); + let vrps = content_info + .content + .vrps + .as_ref() + .map(|state| { + json!({ + "present": true, + "payload_sets": state.rps.len(), + "hash_hex": hex::encode(&state.hash), + "ip_addr_block_count": vrps_total, + }) + }) + .unwrap_or_else(|| json!({"present": false})); + + let vaps = content_info + .content + .vaps + .as_ref() + .map(|state| { + json!({ + "present": true, + "payload_sets": state.aps.len(), + "hash_hex": hex::encode(&state.hash), + "provider_count": state.aps.iter().map(|set| set.providers.len()).sum::(), + }) + }) + .unwrap_or_else(|| json!({"present": false})); + + let tas = content_info + .content + .tas + .as_ref() + .map(|state| { + json!({ + "present": true, + "ski_count": state.skis.len(), + "hash_hex": hex::encode(&state.hash), + }) + }) + .unwrap_or_else(|| json!({"present": false})); + + let rks = content_info.content.rks.as_ref().map(|state| { + json!({ + "present": true, + "router_key_sets": state.rksets.len(), + "router_key_count": state.rksets.iter().map(|set| set.router_keys.len()).sum::(), + "hash_hex": hex::encode(&state.hash), + }) + }).unwrap_or_else(|| json!({"present": false})); + + Ok(json!({ + "content_type_oid": content_info.content_type_oid, + "version": content_info.content.version, + "hash_alg": content_info.content.hash_alg.oid(), + "produced_at_rfc3339_utc": produced_at, + "state_aspects": { + "mfts": mfts, + "vrps": vrps, + "vaps": vaps, + "tas": tas, + "rks": rks, + } + })) +} diff --git a/crates/panda-rpki-validator/src/ccr/encode.rs b/crates/panda-rpki-validator/src/ccr/encode.rs new file mode 100644 index 0000000..ad57fed --- /dev/null +++ b/crates/panda-rpki-validator/src/ccr/encode.rs @@ -0,0 +1,303 @@ +use crate::ccr::model::{ + AspaPayloadSet, AspaPayloadState, CCR_VERSION_V0, CcrContentInfo, CcrDigestAlgorithm, + ManifestInstance, ManifestState, RoaPayloadSet, RoaPayloadState, RouterKey, RouterKeySet, + RouterKeyState, RpkiCanonicalCacheRepresentation, TrustAnchorState, +}; +use crate::data_model::common::BigUnsigned; +use crate::data_model::oid::{OID_CT_RPKI_CCR_RAW, OID_SHA256_RAW}; + +#[derive(Debug, thiserror::Error)] +pub enum CcrEncodeError { + #[error("CCR model validation failed: {0}")] + Validate(String), + + #[error("GeneralizedTime formatting failed: {0}")] + ProducedAtFormat(String), +} + +pub fn encode_content_info(content_info: &CcrContentInfo) -> Result, CcrEncodeError> { + content_info.validate().map_err(CcrEncodeError::Validate)?; + let content_der = encode_ccr(&content_info.content)?; + Ok(encode_sequence(&[ + encode_oid(OID_CT_RPKI_CCR_RAW), + encode_explicit(0, &content_der), + ])) +} + +pub fn encode_ccr(ccr: &RpkiCanonicalCacheRepresentation) -> Result, CcrEncodeError> { + ccr.validate().map_err(CcrEncodeError::Validate)?; + let mut fields = Vec::new(); + if ccr.version != CCR_VERSION_V0 { + fields.push(encode_explicit(0, &encode_integer_u32(ccr.version))); + } + fields.push(encode_digest_algorithm(&ccr.hash_alg)); + fields.push(encode_generalized_time(ccr.produced_at)?); + if let Some(mfts) = &ccr.mfts { + fields.push(encode_explicit(1, &encode_manifest_state(mfts)?)); + } + if let Some(vrps) = &ccr.vrps { + fields.push(encode_explicit(2, &encode_roa_payload_state(vrps)?)); + } + if let Some(vaps) = &ccr.vaps { + fields.push(encode_explicit(3, &encode_aspa_payload_state(vaps)?)); + } + if let Some(tas) = &ccr.tas { + fields.push(encode_explicit(4, &encode_trust_anchor_state(tas)?)); + } + if let Some(rks) = &ccr.rks { + fields.push(encode_explicit(5, &encode_router_key_state(rks)?)); + } + Ok(encode_sequence(&fields)) +} + +pub fn encode_manifest_state(state: &ManifestState) -> Result, CcrEncodeError> { + state.validate().map_err(CcrEncodeError::Validate)?; + let mis = encode_manifest_state_payload_der(&state.mis)?; + Ok(encode_sequence(&[ + mis, + encode_generalized_time(state.most_recent_update)?, + encode_octet_string(&state.hash), + ])) +} + +pub fn encode_manifest_state_payload_der( + instances: &[ManifestInstance], +) -> Result, CcrEncodeError> { + Ok(encode_sequence( + &instances + .iter() + .map(encode_manifest_instance) + .collect::, _>>()?, + )) +} + +fn encode_manifest_instance(instance: &ManifestInstance) -> Result, CcrEncodeError> { + instance.validate().map_err(CcrEncodeError::Validate)?; + let mut fields = vec![ + encode_octet_string(&instance.hash), + encode_integer_u64(instance.size), + encode_octet_string(&instance.aki), + encode_integer_bigunsigned(&instance.manifest_number), + encode_generalized_time(instance.this_update)?, + encode_sequence(&instance.locations), + ]; + if !instance.subordinates.is_empty() { + fields.push(encode_sequence( + &instance + .subordinates + .iter() + .map(|ski| encode_octet_string(ski)) + .collect::>(), + )); + } + Ok(encode_sequence(&fields)) +} + +pub fn encode_roa_payload_state(state: &RoaPayloadState) -> Result, CcrEncodeError> { + state.validate().map_err(CcrEncodeError::Validate)?; + let rps = encode_roa_payload_state_payload_der(&state.rps)?; + Ok(encode_sequence(&[rps, encode_octet_string(&state.hash)])) +} + +pub fn encode_roa_payload_state_payload_der( + sets: &[RoaPayloadSet], +) -> Result, CcrEncodeError> { + Ok(encode_sequence( + &sets + .iter() + .map(encode_roa_payload_set) + .collect::, _>>()?, + )) +} + +fn encode_roa_payload_set(set: &RoaPayloadSet) -> Result, CcrEncodeError> { + set.validate().map_err(CcrEncodeError::Validate)?; + Ok(encode_sequence(&[ + encode_integer_u32(set.as_id), + encode_sequence(&set.ip_addr_blocks), + ])) +} + +pub fn encode_aspa_payload_state(state: &AspaPayloadState) -> Result, CcrEncodeError> { + state.validate().map_err(CcrEncodeError::Validate)?; + let aps = encode_aspa_payload_state_payload_der(&state.aps)?; + Ok(encode_sequence(&[aps, encode_octet_string(&state.hash)])) +} + +pub fn encode_aspa_payload_state_payload_der( + sets: &[AspaPayloadSet], +) -> Result, CcrEncodeError> { + Ok(encode_sequence( + &sets + .iter() + .map(encode_aspa_payload_set) + .collect::, _>>()?, + )) +} + +fn encode_aspa_payload_set(set: &AspaPayloadSet) -> Result, CcrEncodeError> { + set.validate().map_err(CcrEncodeError::Validate)?; + Ok(encode_sequence(&[ + encode_integer_u32(set.customer_as_id), + encode_sequence( + &set.providers + .iter() + .map(|provider| encode_integer_u32(*provider)) + .collect::>(), + ), + ])) +} + +pub fn encode_trust_anchor_state(state: &TrustAnchorState) -> Result, CcrEncodeError> { + state.validate().map_err(CcrEncodeError::Validate)?; + let skis = encode_trust_anchor_state_payload_der(&state.skis)?; + Ok(encode_sequence(&[skis, encode_octet_string(&state.hash)])) +} + +pub fn encode_trust_anchor_state_payload_der(skis: &[Vec]) -> Result, CcrEncodeError> { + Ok(encode_sequence( + &skis + .iter() + .map(|ski| encode_octet_string(ski)) + .collect::>(), + )) +} + +pub fn encode_router_key_state(state: &RouterKeyState) -> Result, CcrEncodeError> { + state.validate().map_err(CcrEncodeError::Validate)?; + let rksets = encode_router_key_state_payload_der(&state.rksets)?; + Ok(encode_sequence(&[rksets, encode_octet_string(&state.hash)])) +} + +pub fn encode_router_key_state_payload_der( + sets: &[RouterKeySet], +) -> Result, CcrEncodeError> { + Ok(encode_sequence( + &sets + .iter() + .map(encode_router_key_set) + .collect::, _>>()?, + )) +} + +fn encode_router_key_set(set: &RouterKeySet) -> Result, CcrEncodeError> { + set.validate().map_err(CcrEncodeError::Validate)?; + Ok(encode_sequence(&[ + encode_integer_u32(set.as_id), + encode_sequence( + &set.router_keys + .iter() + .map(encode_router_key) + .collect::, _>>()?, + ), + ])) +} + +fn encode_router_key(key: &RouterKey) -> Result, CcrEncodeError> { + key.validate().map_err(CcrEncodeError::Validate)?; + Ok(encode_sequence(&[ + encode_octet_string(&key.ski), + key.spki_der.clone(), + ])) +} + +fn encode_digest_algorithm(alg: &CcrDigestAlgorithm) -> Vec { + match alg { + CcrDigestAlgorithm::Sha256 => encode_sequence(&[encode_oid(OID_SHA256_RAW)]), + } +} + +fn encode_generalized_time(t: time::OffsetDateTime) -> Result, CcrEncodeError> { + let t = t.to_offset(time::UtcOffset::UTC); + let s = format!( + "{:04}{:02}{:02}{:02}{:02}{:02}Z", + t.year(), + u8::from(t.month()), + t.day(), + t.hour(), + t.minute(), + t.second() + ); + Ok(encode_tlv(0x18, s.into_bytes())) +} + +fn encode_integer_u32(v: u32) -> Vec { + encode_integer_bytes(unsigned_integer_bytes(v as u64)) +} + +fn encode_integer_u64(v: u64) -> Vec { + encode_integer_bytes(unsigned_integer_bytes(v)) +} + +fn encode_integer_bigunsigned(v: &BigUnsigned) -> Vec { + encode_integer_bytes(v.bytes_be.clone()) +} + +fn encode_integer_bytes(mut bytes: Vec) -> Vec { + if bytes.is_empty() { + bytes.push(0); + } + if bytes[0] & 0x80 != 0 { + bytes.insert(0, 0); + } + encode_tlv(0x02, bytes) +} + +fn unsigned_integer_bytes(v: u64) -> Vec { + if v == 0 { + return vec![0]; + } + let mut out = Vec::new(); + let mut n = v; + while n > 0 { + out.push((n & 0xFF) as u8); + n >>= 8; + } + out.reverse(); + out +} + +fn encode_oid(raw_body: &[u8]) -> Vec { + encode_tlv(0x06, raw_body.to_vec()) +} + +fn encode_octet_string(bytes: &[u8]) -> Vec { + encode_tlv(0x04, bytes.to_vec()) +} + +fn encode_explicit(tag_number: u8, inner_der: &[u8]) -> Vec { + encode_tlv(0xA0 + tag_number, inner_der.to_vec()) +} + +fn encode_sequence(elements: &[Vec]) -> Vec { + let total_len: usize = elements.iter().map(Vec::len).sum(); + let mut buf = Vec::with_capacity(total_len); + for element in elements { + buf.extend_from_slice(element); + } + encode_tlv(0x30, buf) +} + +fn encode_tlv(tag: u8, value: Vec) -> Vec { + let mut out = Vec::with_capacity(1 + 9 + value.len()); + out.push(tag); + encode_length(value.len(), &mut out); + out.extend_from_slice(&value); + out +} + +fn encode_length(len: usize, out: &mut Vec) { + if len < 0x80 { + out.push(len as u8); + return; + } + let mut bytes = Vec::new(); + let mut value = len; + while value > 0 { + bytes.push((value & 0xFF) as u8); + value >>= 8; + } + bytes.reverse(); + out.push(0x80 | (bytes.len() as u8)); + out.extend_from_slice(&bytes); +} diff --git a/crates/panda-rpki-validator/src/ccr/export.rs b/crates/panda-rpki-validator/src/ccr/export.rs new file mode 100644 index 0000000..3b76664 --- /dev/null +++ b/crates/panda-rpki-validator/src/ccr/export.rs @@ -0,0 +1,343 @@ +use crate::ccr::build::{ + CcrBuildError, ManifestStateBuildBreakdown, build_aspa_payload_state, + build_manifest_state_from_vcirs_with_breakdown, build_roa_payload_state, + build_router_key_state_from_runtime, build_trust_anchor_state, +}; +use crate::ccr::encode::{CcrEncodeError, encode_content_info}; +use crate::ccr::model::{CcrContentInfo, CcrDigestAlgorithm, RpkiCanonicalCacheRepresentation}; +use crate::data_model::ta::TrustAnchor; +use crate::storage::RocksStore; +use crate::validation::objects::{AspaAttestation, RouterKeyPayload, Vrp}; +use serde::Serialize; +use std::path::Path; + +#[derive(Debug, thiserror::Error)] +pub enum CcrExportError { + #[error("list VCIRs failed: {0}")] + ListVcirs(String), + + #[error("build CCR state failed: {0}")] + Build(#[from] CcrBuildError), + + #[error("encode CCR failed: {0}")] + Encode(#[from] CcrEncodeError), + + #[error("write CCR file failed: {0}: {1}")] + Write(String, String), +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct CcrBuildBreakdown { + pub vcir_count: usize, + pub list_vcirs_ms: u64, + pub manifest_state_ms: u64, + pub manifest_state_breakdown: ManifestStateBuildBreakdown, + pub roa_payload_state_ms: u64, + pub aspa_payload_state_ms: u64, + pub trust_anchor_state_ms: u64, + pub router_key_state_ms: u64, + pub total_ms: u64, +} + +pub fn build_ccr_from_run( + store: &RocksStore, + trust_anchors: &[TrustAnchor], + vrps: &[Vrp], + aspas: &[AspaAttestation], + router_keys: &[RouterKeyPayload], + produced_at: time::OffsetDateTime, +) -> Result { + build_ccr_from_run_with_breakdown(store, trust_anchors, vrps, aspas, router_keys, produced_at) + .map(|(ccr, _)| ccr) +} + +pub fn build_ccr_from_run_with_breakdown( + store: &RocksStore, + trust_anchors: &[TrustAnchor], + vrps: &[Vrp], + aspas: &[AspaAttestation], + router_keys: &[RouterKeyPayload], + produced_at: time::OffsetDateTime, +) -> Result<(RpkiCanonicalCacheRepresentation, CcrBuildBreakdown), CcrExportError> { + let total_started = std::time::Instant::now(); + let mut breakdown = CcrBuildBreakdown::default(); + + let started = std::time::Instant::now(); + let vcirs = store + .list_vcirs() + .map_err(|e| CcrExportError::ListVcirs(e.to_string()))?; + breakdown.list_vcirs_ms = started.elapsed().as_millis() as u64; + breakdown.vcir_count = vcirs.len(); + + let started = std::time::Instant::now(); + let (mfts, manifest_state_breakdown) = + build_manifest_state_from_vcirs_with_breakdown(store, &vcirs)?; + breakdown.manifest_state_ms = started.elapsed().as_millis() as u64; + breakdown.manifest_state_breakdown = manifest_state_breakdown; + + let started = std::time::Instant::now(); + let vrps = build_roa_payload_state(vrps)?; + breakdown.roa_payload_state_ms = started.elapsed().as_millis() as u64; + + let started = std::time::Instant::now(); + let vaps = build_aspa_payload_state(aspas)?; + breakdown.aspa_payload_state_ms = started.elapsed().as_millis() as u64; + + let started = std::time::Instant::now(); + let tas = build_trust_anchor_state(trust_anchors)?; + breakdown.trust_anchor_state_ms = started.elapsed().as_millis() as u64; + + let started = std::time::Instant::now(); + let rks = build_router_key_state_from_runtime(router_keys)?; + breakdown.router_key_state_ms = started.elapsed().as_millis() as u64; + + let ccr = RpkiCanonicalCacheRepresentation { + version: 0, + hash_alg: CcrDigestAlgorithm::Sha256, + produced_at, + mfts: Some(mfts), + vrps: Some(vrps), + vaps: Some(vaps), + tas: Some(tas), + rks: Some(rks), + }; + breakdown.total_ms = total_started.elapsed().as_millis() as u64; + + Ok((ccr, breakdown)) +} + +pub fn write_ccr_file( + path: &Path, + ccr: &RpkiCanonicalCacheRepresentation, +) -> Result<(), CcrExportError> { + let der = encode_content_info(&CcrContentInfo::new(ccr.clone()))?; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| CcrExportError::Write(path.display().to_string(), e.to_string()))?; + } + std::fs::write(path, der) + .map_err(|e| CcrExportError::Write(path.display().to_string(), e.to_string())) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ccr::decode::decode_content_info; + use crate::data_model::manifest::ManifestObject; + use crate::data_model::oid::OID_AD_SIGNED_OBJECT; + use crate::data_model::rc::SubjectInfoAccess; + use crate::data_model::roa::{IpPrefix, RoaAfi}; + use crate::data_model::ta::TrustAnchor; + use crate::data_model::tal::Tal; + use crate::storage::{ + PackTime, RawByHashEntry, RocksStore, ValidatedCaInstanceResult, ValidatedManifestMeta, + VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus, VcirAuditSummary, + VcirCcrManifestProjection, VcirChildEntry, VcirInstanceGate, VcirRelatedArtifact, + VcirSummary, + }; + use crate::validation::objects::{AspaAttestation, RouterKeyPayload, Vrp}; + use sha2::Digest; + + fn sample_trust_anchor() -> TrustAnchor { + let base = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let tal_bytes = std::fs::read(base.join("tests/fixtures/tal/apnic-rfc7730-https.tal")) + .expect("read tal"); + let ta_der = std::fs::read(base.join("tests/fixtures/ta/apnic-ta.cer")).expect("read ta"); + let tal = Tal::decode_bytes(&tal_bytes).expect("decode tal"); + TrustAnchor::bind_der(tal, &ta_der, None).expect("bind ta") + } + + fn sample_vcir_and_raw(store: &RocksStore) -> ValidatedCaInstanceResult { + let base = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let manifest_der = std::fs::read(base.join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft")).expect("read manifest"); + let manifest = ManifestObject::decode_der(&manifest_der).expect("decode manifest"); + let manifest_uri = match manifest.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .extensions + .subject_info_access + .as_ref() + .expect("manifest sia") + { + SubjectInfoAccess::Ee(ee_sia) => ee_sia + .access_descriptions + .iter() + .find(|ad| { + ad.access_method_oid == OID_AD_SIGNED_OBJECT + && ad.access_location.starts_with("rsync://") + }) + .expect("manifest rsync signedObject") + .access_location + .clone(), + SubjectInfoAccess::Ca(_) => panic!("manifest EE SIA should not be CA variant"), + }; + let hash = hex::encode(sha2::Sha256::digest(&manifest_der)); + let mut raw = RawByHashEntry::from_bytes(hash.clone(), manifest_der.clone()); + raw.origin_uris.push(manifest_uri.clone()); + raw.object_type = Some("mft".to_string()); + raw.encoding = Some("der".to_string()); + store.put_raw_by_hash_entry(&raw).expect("put raw"); + let projection = VcirCcrManifestProjection { + manifest_rsync_uri: manifest_uri.clone(), + manifest_sha256: sha2::Sha256::digest(&manifest_der).to_vec(), + manifest_size: manifest_der.len() as u64, + manifest_ee_aki: manifest.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .extensions + .authority_key_identifier + .clone() + .expect("manifest aki"), + manifest_number_be: manifest.manifest.manifest_number.bytes_be.clone(), + manifest_this_update: PackTime::from_utc_offset_datetime(manifest.manifest.this_update), + manifest_sia_locations_der: match manifest.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .extensions + .subject_info_access + .as_ref() + .expect("manifest sia") + { + SubjectInfoAccess::Ee(ee_sia) => vec![ + crate::ccr::manifest_location::select_manifest_signed_object_location( + &manifest_uri, + &ee_sia.access_descriptions, + ) + .expect("select manifest signedObject"), + ], + SubjectInfoAccess::Ca(_) => panic!("manifest EE SIA should not be CA variant"), + }, + subordinate_skis: vec![vec![0x33; 20]], + }; + ValidatedCaInstanceResult { + manifest_rsync_uri: manifest_uri.clone(), + parent_manifest_rsync_uri: None, + tal_id: "apnic".to_string(), + ca_subject_name: "CN=test".to_string(), + ca_ski: "11".repeat(20), + issuer_ski: "22".repeat(20), + last_successful_validation_time: PackTime::from_utc_offset_datetime( + manifest.manifest.this_update, + ), + current_manifest_rsync_uri: manifest_uri.clone(), + current_crl_rsync_uri: "rsync://example.test/repo/current.crl".to_string(), + validated_manifest_meta: ValidatedManifestMeta { + validated_manifest_number: manifest.manifest.manifest_number.bytes_be.clone(), + validated_manifest_this_update: PackTime::from_utc_offset_datetime( + manifest.manifest.this_update, + ), + validated_manifest_next_update: PackTime::from_utc_offset_datetime( + manifest.manifest.next_update, + ), + }, + ccr_manifest_projection: projection, + instance_gate: VcirInstanceGate { + manifest_next_update: PackTime::from_utc_offset_datetime( + manifest.manifest.next_update, + ), + current_crl_next_update: PackTime::from_utc_offset_datetime( + manifest.manifest.next_update, + ), + self_ca_not_after: PackTime::from_utc_offset_datetime( + manifest.manifest.next_update, + ), + instance_effective_until: PackTime::from_utc_offset_datetime( + manifest.manifest.next_update, + ), + }, + child_entries: vec![VcirChildEntry { + child_manifest_rsync_uri: "rsync://example.test/repo/child.mft".to_string(), + child_cert_rsync_uri: "rsync://example.test/repo/child.cer".to_string(), + child_cert_hash: "aa".repeat(32), + child_ski: "33".repeat(20), + child_rsync_base_uri: "rsync://example.test/repo/".to_string(), + child_publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), + child_rrdp_notification_uri: None, + child_effective_ip_resources: None, + child_effective_as_resources: None, + accepted_at_validation_time: PackTime::from_utc_offset_datetime( + manifest.manifest.this_update, + ), + }], + local_outputs: Vec::new(), + related_artifacts: vec![VcirRelatedArtifact { + artifact_role: VcirArtifactRole::Manifest, + artifact_kind: VcirArtifactKind::Mft, + uri: Some(manifest_uri), + sha256: hash, + object_type: Some("mft".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }], + summary: VcirSummary { + local_vrp_count: 0, + local_aspa_count: 0, + local_router_key_count: 0, + child_count: 1, + accepted_object_count: 1, + rejected_object_count: 0, + }, + audit_summary: VcirAuditSummary { + failed_fetch_eligible: true, + last_failed_fetch_reason: None, + warning_count: 0, + audit_flags: Vec::new(), + }, + } + } + + #[test] + fn build_and_write_ccr_from_run_exports_der_content_info() { + let td = tempfile::tempdir().expect("tempdir"); + let db_path = td.path().join("db"); + let store = RocksStore::open(&db_path).expect("open rocksdb"); + let vcir = sample_vcir_and_raw(&store); + store.put_vcir(&vcir).expect("put vcir"); + let trust_anchor = sample_trust_anchor(); + let vrps = vec![Vrp { + asn: 64496, + prefix: IpPrefix { + afi: RoaAfi::Ipv4, + prefix_len: 8, + addr: [10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + }, + max_length: 8, + }]; + let aspas = vec![AspaAttestation { + customer_as_id: 64496, + provider_as_ids: vec![64497], + }]; + let router_keys = vec![RouterKeyPayload { + as_id: 64496, + ski: vec![0x11; 20], + spki_der: vec![0x30, 0x00], + source_object_uri: "rsync://example.test/repo/router.cer".to_string(), + source_object_hash: hex::encode([0x11; 32]), + source_ee_cert_hash: hex::encode([0x11; 32]), + item_effective_until: PackTime::from_utc_offset_datetime( + time::OffsetDateTime::now_utc() + time::Duration::hours(1), + ), + }]; + let ccr = build_ccr_from_run( + &store, + &[trust_anchor], + &vrps, + &aspas, + &router_keys, + time::OffsetDateTime::now_utc(), + ) + .expect("build ccr"); + assert!(ccr.mfts.is_some()); + assert!(ccr.vrps.is_some()); + assert!(ccr.vaps.is_some()); + assert!(ccr.tas.is_some()); + assert!(ccr.rks.is_some()); + assert_eq!(ccr.rks.as_ref().unwrap().rksets.len(), 1); + let out = td.path().join("out/example.ccr"); + write_ccr_file(&out, &ccr).expect("write ccr"); + let der = std::fs::read(&out).expect("read ccr file"); + let decoded = decode_content_info(&der).expect("decode ccr"); + assert_eq!(decoded.content.version, 0); + assert!(decoded.content.mfts.is_some()); + } +} diff --git a/crates/panda-rpki-validator/src/ccr/hash.rs b/crates/panda-rpki-validator/src/ccr/hash.rs new file mode 100644 index 0000000..23b203a --- /dev/null +++ b/crates/panda-rpki-validator/src/ccr/hash.rs @@ -0,0 +1,9 @@ +use sha2::Digest; + +pub fn compute_state_hash(payload_der: &[u8]) -> Vec { + sha2::Sha256::digest(payload_der).to_vec() +} + +pub fn verify_state_hash(expected: &[u8], payload_der: &[u8]) -> bool { + compute_state_hash(payload_der).as_slice() == expected +} diff --git a/crates/panda-rpki-validator/src/ccr/manifest_location.rs b/crates/panda-rpki-validator/src/ccr/manifest_location.rs new file mode 100644 index 0000000..217a0cb --- /dev/null +++ b/crates/panda-rpki-validator/src/ccr/manifest_location.rs @@ -0,0 +1,424 @@ +use crate::data_model::common::DerReader; +use crate::data_model::oid::OID_AD_SIGNED_OBJECT; +use crate::data_model::rc::AccessDescription; + +pub(crate) fn select_manifest_signed_object_location( + manifest_rsync_uri: &str, + access_descriptions: &[AccessDescription], +) -> Result, String> { + let matching = access_descriptions + .iter() + .filter(|access_description| { + access_description.access_method_oid == OID_AD_SIGNED_OBJECT + && access_description.access_location == manifest_rsync_uri + }) + .collect::>(); + let access_description = expect_single_matching_location( + manifest_rsync_uri, + matching.len(), + "parsed Manifest EE SIA", + )?; + encode_access_description_der(matching[access_description]) +} + +pub(crate) fn select_manifest_signed_object_location_from_der( + manifest_rsync_uri: &str, + locations_der: &[Vec], +) -> Result, String> { + let matching = locations_der + .iter() + .filter_map( + |location_der| match decode_access_description_der(location_der) { + Ok(access_description) + if access_description.access_method_oid == OID_AD_SIGNED_OBJECT + && access_description.access_location == manifest_rsync_uri => + { + Some(Ok(location_der.clone())) + } + Ok(_) => None, + Err(detail) => Some(Err(detail)), + }, + ) + .collect::, _>>()?; + let index = expect_single_matching_location( + manifest_rsync_uri, + matching.len(), + "persisted VCIR CCR projection", + )?; + Ok(matching[index].clone()) +} + +pub(crate) fn encode_access_description_der( + access_description: &AccessDescription, +) -> Result, String> { + let oid = encode_oid_der(&access_description.access_method_oid)?; + let uri = encode_tlv(0x86, access_description.access_location.as_bytes().to_vec()); + Ok(encode_sequence(&[oid, uri])) +} + +fn expect_single_matching_location( + manifest_rsync_uri: &str, + matching_count: usize, + source: &str, +) -> Result { + if matching_count == 1 { + return Ok(0); + } + Err(format!( + "{source} contains {matching_count} id-ad-signedObject locations matching manifest URI {manifest_rsync_uri}; expected exactly one" + )) +} + +fn decode_access_description_der(der: &[u8]) -> Result { + let mut top = DerReader::new(der); + let mut sequence = top.take_sequence()?; + if !top.is_empty() { + return Err("trailing bytes after AccessDescription".to_string()); + } + let access_method_oid = decode_oid_der(sequence.take_tag(0x06)?)?; + let access_location = std::str::from_utf8(sequence.take_tag(0x86)?) + .map_err(|error| format!("AccessDescription URI is not UTF-8: {error}"))? + .to_string(); + if !sequence.is_empty() { + return Err("trailing fields in AccessDescription".to_string()); + } + Ok(AccessDescription { + access_method_oid, + access_location, + }) +} + +fn decode_oid_der(value: &[u8]) -> Result { + let mut offset = 0usize; + let first = decode_base128(value, &mut offset)?; + let (first_arc, second_arc) = match first { + 0..=39 => (0, first), + 40..=79 => (1, first - 40), + value => (2, value - 80), + }; + let mut arcs = vec![first_arc, second_arc]; + while offset < value.len() { + arcs.push(decode_base128(value, &mut offset)?); + } + Ok(arcs + .into_iter() + .map(|arc| arc.to_string()) + .collect::>() + .join(".")) +} + +fn decode_base128(value: &[u8], offset: &mut usize) -> Result { + let first = *value + .get(*offset) + .ok_or_else(|| "truncated OBJECT IDENTIFIER".to_string())?; + if first == 0x80 { + return Err("non-minimal OBJECT IDENTIFIER base-128 encoding".to_string()); + } + let mut out = 0u64; + loop { + let byte = *value + .get(*offset) + .ok_or_else(|| "truncated OBJECT IDENTIFIER".to_string())?; + *offset += 1; + out = out + .checked_shl(7) + .ok_or_else(|| "OBJECT IDENTIFIER arc overflows u64".to_string())? + .checked_add((byte & 0x7f) as u64) + .ok_or_else(|| "OBJECT IDENTIFIER arc overflows u64".to_string())?; + if byte & 0x80 == 0 { + return Ok(out); + } + } +} + +fn encode_oid_der(oid: &str) -> Result, String> { + let arcs = oid + .split('.') + .map(|part| { + part.parse::() + .map_err(|_| format!("unsupported accessMethod OID: {oid}")) + }) + .collect::, _>>()?; + if arcs.len() < 2 || arcs[0] > 2 || (arcs[0] < 2 && arcs[1] >= 40) { + return Err(format!("unsupported accessMethod OID: {oid}")); + } + let mut body = Vec::new(); + encode_base128( + arcs[0] + .checked_mul(40) + .and_then(|value| value.checked_add(arcs[1])) + .ok_or_else(|| format!("unsupported accessMethod OID: {oid}"))?, + &mut body, + ); + for arc in &arcs[2..] { + encode_base128(*arc, &mut body); + } + Ok(encode_tlv(0x06, body)) +} + +fn encode_base128(mut value: u64, out: &mut Vec) { + let mut encoded = vec![(value & 0x7f) as u8]; + value >>= 7; + while value > 0 { + encoded.push(((value & 0x7f) as u8) | 0x80); + value >>= 7; + } + encoded.reverse(); + out.extend_from_slice(&encoded); +} + +fn encode_sequence(elements: &[Vec]) -> Vec { + let total_len = elements.iter().map(Vec::len).sum(); + let mut value = Vec::with_capacity(total_len); + for element in elements { + value.extend_from_slice(element); + } + encode_tlv(0x30, value) +} + +fn encode_tlv(tag: u8, value: Vec) -> Vec { + let mut out = Vec::with_capacity(1 + 9 + value.len()); + out.push(tag); + encode_length(value.len(), &mut out); + out.extend_from_slice(&value); + out +} + +fn encode_length(len: usize, out: &mut Vec) { + if len < 0x80 { + out.push(len as u8); + return; + } + let mut bytes = Vec::new(); + let mut value = len; + while value > 0 { + bytes.push((value & 0xff) as u8); + value >>= 8; + } + bytes.reverse(); + out.push(0x80 | bytes.len() as u8); + out.extend_from_slice(&bytes); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::data_model::oid::OID_AD_RPKI_NOTIFY; + + const MANIFEST_URI: &str = "rsync://example.test/repo/manifest.mft"; + + fn access_description(access_method_oid: &str, access_location: &str) -> AccessDescription { + AccessDescription { + access_method_oid: access_method_oid.to_string(), + access_location: access_location.to_string(), + } + } + + #[test] + fn selects_matching_signed_object_and_excludes_rpki_notify() { + let signed_object = access_description(OID_AD_SIGNED_OBJECT, MANIFEST_URI); + let selected = select_manifest_signed_object_location( + MANIFEST_URI, + &[ + signed_object.clone(), + access_description( + OID_AD_RPKI_NOTIFY, + "https://rrdp.example.test/notification.xml", + ), + ], + ) + .expect("select signed object"); + + assert_eq!( + selected, + encode_access_description_der(&signed_object).unwrap() + ); + } + + #[test] + fn selects_only_the_signed_object_matching_manifest_uri() { + let expected = access_description(OID_AD_SIGNED_OBJECT, MANIFEST_URI); + let selected = select_manifest_signed_object_location( + MANIFEST_URI, + &[ + access_description( + OID_AD_SIGNED_OBJECT, + "https://backup.example.test/manifest.mft", + ), + expected.clone(), + ], + ) + .expect("select matching signed object"); + + assert_eq!(selected, encode_access_description_der(&expected).unwrap()); + } + + #[test] + fn rejects_missing_or_duplicate_matching_signed_object() { + let missing = select_manifest_signed_object_location( + MANIFEST_URI, + &[access_description( + OID_AD_RPKI_NOTIFY, + "https://rrdp.example.test/notification.xml", + )], + ) + .expect_err("missing signed object must fail"); + assert!(missing.contains("contains 0"), "{missing}"); + + let duplicate = select_manifest_signed_object_location( + MANIFEST_URI, + &[ + access_description(OID_AD_SIGNED_OBJECT, MANIFEST_URI), + access_description(OID_AD_SIGNED_OBJECT, MANIFEST_URI), + ], + ) + .expect_err("duplicate signed object must fail"); + assert!(duplicate.contains("contains 2"), "{duplicate}"); + } + + #[test] + fn selects_matching_signed_object_from_historical_projection() { + let signed_object = access_description(OID_AD_SIGNED_OBJECT, MANIFEST_URI); + let signed_object_der = encode_access_description_der(&signed_object).unwrap(); + let notify_der = encode_access_description_der(&access_description( + OID_AD_RPKI_NOTIFY, + "https://rrdp.example.test/notification.xml", + )) + .unwrap(); + + let selected = select_manifest_signed_object_location_from_der( + MANIFEST_URI, + &[notify_der, signed_object_der.clone()], + ) + .expect("select historical signed object"); + assert_eq!(selected, signed_object_der); + } + + #[test] + fn rejects_malformed_historical_projection() { + let error = select_manifest_signed_object_location_from_der( + MANIFEST_URI, + &[vec![0x30, 0x01, 0x06]], + ) + .expect_err("malformed historical projection must fail"); + assert!(error.contains("truncated DER"), "{error}"); + } + + #[test] + fn access_description_der_codec_covers_long_and_invalid_forms() { + let long = access_description( + "2.999.200.1", + &format!("rsync://example.test/repo/{}", "x".repeat(160)), + ); + let encoded = encode_access_description_der(&long).expect("encode long access description"); + assert_eq!( + decode_access_description_der(&encoded).expect("decode long access description"), + long + ); + + let mut trailing = encoded.clone(); + trailing.push(0); + assert!( + decode_access_description_der(&trailing) + .expect_err("trailing bytes must fail") + .contains("trailing bytes") + ); + + let oid = encode_oid_der(OID_AD_SIGNED_OBJECT).expect("encode signedObject OID"); + let uri = encode_tlv(0x86, MANIFEST_URI.as_bytes().to_vec()); + assert!( + decode_access_description_der(&encode_sequence(&[ + oid.clone(), + uri.clone(), + encode_tlv(0x05, Vec::new()), + ])) + .expect_err("trailing field must fail") + .contains("trailing fields") + ); + assert!( + decode_access_description_der(&encode_sequence(&[oid, encode_tlv(0x86, vec![0xff]),])) + .expect_err("non-UTF8 URI must fail") + .contains("not UTF-8") + ); + assert!( + decode_access_description_der(&encode_sequence(&[ + encode_tlv(0x06, vec![0x80, 0x00]), + uri.clone(), + ])) + .expect_err("non-minimal OID must fail") + .contains("non-minimal") + ); + assert!( + decode_access_description_der(&encode_sequence(&[encode_tlv(0x06, vec![0x81]), uri,])) + .expect_err("truncated OID must fail") + .contains("truncated OBJECT IDENTIFIER") + ); + assert!( + encode_access_description_der(&access_description("3.1", MANIFEST_URI)) + .expect_err("invalid OID must fail") + .contains("unsupported accessMethod OID") + ); + } + + #[test] + fn access_description_der_codec_reports_malformed_selector_inputs() { + let malformed_sequence = decode_access_description_der(&[0x31, 0x00]) + .expect_err("non-sequence AccessDescription must fail"); + assert!(!malformed_sequence.is_empty()); + + let missing_method = decode_access_description_der(&encode_sequence(&[encode_tlv( + 0x86, + MANIFEST_URI.as_bytes().to_vec(), + )])) + .expect_err("missing accessMethod must fail"); + assert!(!missing_method.is_empty()); + + let oid = encode_oid_der(OID_AD_SIGNED_OBJECT).expect("encode signedObject OID"); + let missing_location = decode_access_description_der(&encode_sequence(&[oid.clone()])) + .expect_err("missing accessLocation must fail"); + assert!(!missing_location.is_empty()); + + let wrong_location_tag = decode_access_description_der(&encode_sequence(&[ + oid.clone(), + encode_tlv(0x04, MANIFEST_URI.as_bytes().to_vec()), + ])) + .expect_err("wrong accessLocation tag must fail"); + assert!(!wrong_location_tag.is_empty()); + + let empty_oid = decode_access_description_der(&encode_sequence(&[ + encode_tlv(0x06, Vec::new()), + encode_tlv(0x86, MANIFEST_URI.as_bytes().to_vec()), + ])) + .expect_err("empty OID must fail"); + assert!(!empty_oid.is_empty()); + + let first_arc_zero = decode_access_description_der(&encode_sequence(&[ + encode_tlv(0x06, vec![0x01, 0x02]), + encode_tlv(0x86, MANIFEST_URI.as_bytes().to_vec()), + ])) + .expect("decode first OID arc zero"); + assert_eq!(first_arc_zero.access_method_oid, "0.1.2"); + assert_eq!(first_arc_zero.access_location, MANIFEST_URI); + + let non_numeric = + encode_access_description_der(&access_description("no.such.oid", MANIFEST_URI)) + .expect_err("non-numeric OID must fail"); + assert!(!non_numeric.is_empty()); + + let too_short = encode_access_description_der(&access_description("1", MANIFEST_URI)) + .expect_err("OID without second arc must fail"); + assert!(!too_short.is_empty()); + + let invalid_second_arc = + encode_access_description_der(&access_description("1.40", MANIFEST_URI)) + .expect_err("invalid second OID arc must fail"); + assert!(!invalid_second_arc.is_empty()); + + let overflowing_first_subidentifier = encode_access_description_der(&access_description( + "2.18446744073709551615", + MANIFEST_URI, + )) + .expect_err("overflowing first OID subidentifier must fail"); + assert!(!overflowing_first_subidentifier.is_empty()); + } +} diff --git a/crates/panda-rpki-validator/src/ccr/mod.rs b/crates/panda-rpki-validator/src/ccr/mod.rs new file mode 100644 index 0000000..54c82ce --- /dev/null +++ b/crates/panda-rpki-validator/src/ccr/mod.rs @@ -0,0 +1,56 @@ +#[cfg(feature = "full")] +pub mod accumulator; +#[cfg(feature = "full")] +pub mod build; +#[cfg(feature = "full")] +pub mod compare_view; +pub mod decode; +pub mod dump; +pub mod encode; +#[cfg(feature = "full")] +pub mod export; +pub mod hash; +#[cfg(feature = "full")] +pub(crate) mod manifest_location; +pub mod model; +pub mod state_digest; +#[cfg(feature = "full")] +pub mod verify; + +#[cfg(feature = "full")] +pub use accumulator::{CcrAccumulator, CcrManifestContribution}; +#[cfg(feature = "full")] +pub use build::{ + CcrBuildError, ManifestStateBuildBreakdown, build_aspa_payload_state, + build_manifest_state_from_vcirs, build_manifest_state_from_vcirs_with_breakdown, + build_roa_payload_state, build_router_key_state_from_runtime, build_trust_anchor_state, +}; +#[cfg(feature = "full")] +pub use compare_view::{ + VapCompareRow, VrpCompareRow, build_vap_compare_rows, build_vrp_compare_rows, + canonical_vrp_prefix, decode_ccr_compare_views, write_vap_csv, write_vrp_csv, +}; +pub use decode::{CcrDecodeError, decode_content_info}; +pub use dump::{CcrDumpError, dump_content_info_json, dump_content_info_json_value}; +pub use encode::{CcrEncodeError, encode_content_info}; +#[cfg(feature = "full")] +pub use export::{ + CcrBuildBreakdown, CcrExportError, build_ccr_from_run, build_ccr_from_run_with_breakdown, + write_ccr_file, +}; +pub use hash::{compute_state_hash, verify_state_hash}; +pub use model::{ + AspaPayloadSet, AspaPayloadState, CcrContentInfo, CcrDigestAlgorithm, ManifestInstance, + ManifestState, RoaPayloadSet, RoaPayloadState, RouterKey, RouterKeySet, RouterKeyState, + RpkiCanonicalCacheRepresentation, TrustAnchorState, +}; +pub use state_digest::{ + CcrStateDigestComparison, CcrStateDigestError, CcrStateDigestStateComparison, + CcrStateDigestSummary, compare_state_digests, decode_state_digest_summary, +}; +#[cfg(feature = "full")] +pub use verify::{ + CcrVerifyError, CcrVerifySummary, extract_vrp_rows, verify_against_report_json_path, + verify_against_vcir_store, verify_against_vcir_store_path, verify_content_info, + verify_content_info_bytes, +}; diff --git a/crates/panda-rpki-validator/src/ccr/model.rs b/crates/panda-rpki-validator/src/ccr/model.rs new file mode 100644 index 0000000..c4e35b1 --- /dev/null +++ b/crates/panda-rpki-validator/src/ccr/model.rs @@ -0,0 +1,398 @@ +use crate::data_model::common::{BigUnsigned, der_take_tlv}; +use crate::data_model::oid::{OID_CT_RPKI_CCR, OID_SHA256}; + +pub const CCR_VERSION_V0: u32 = 0; +pub const DIGEST_LEN_SHA256: usize = 32; +pub const KEY_IDENTIFIER_LEN_SHA1: usize = 20; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CcrDigestAlgorithm { + Sha256, +} + +impl CcrDigestAlgorithm { + pub fn oid(&self) -> &'static str { + match self { + Self::Sha256 => OID_SHA256, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CcrContentInfo { + pub content_type_oid: String, + pub content: RpkiCanonicalCacheRepresentation, +} + +impl CcrContentInfo { + pub fn new(content: RpkiCanonicalCacheRepresentation) -> Self { + Self { + content_type_oid: OID_CT_RPKI_CCR.to_string(), + content, + } + } + + pub fn validate(&self) -> Result<(), String> { + if self.content_type_oid != OID_CT_RPKI_CCR { + return Err(format!( + "contentType must be {OID_CT_RPKI_CCR}, got {}", + self.content_type_oid + )); + } + self.content.validate() + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RpkiCanonicalCacheRepresentation { + pub version: u32, + pub hash_alg: CcrDigestAlgorithm, + pub produced_at: time::OffsetDateTime, + pub mfts: Option, + pub vrps: Option, + pub vaps: Option, + pub tas: Option, + pub rks: Option, +} + +impl RpkiCanonicalCacheRepresentation { + pub fn validate(&self) -> Result<(), String> { + if self.version != CCR_VERSION_V0 { + return Err(format!("CCR version must be 0, got {}", self.version)); + } + if !matches!(self.hash_alg, CcrDigestAlgorithm::Sha256) { + return Err("CCR hashAlg must be SHA-256".into()); + } + if self.mfts.is_none() + && self.vrps.is_none() + && self.vaps.is_none() + && self.tas.is_none() + && self.rks.is_none() + { + return Err("at least one of mfts/vrps/vaps/tas/rks must be present".into()); + } + if let Some(mfts) = &self.mfts { + mfts.validate()?; + } + if let Some(vrps) = &self.vrps { + vrps.validate()?; + } + if let Some(vaps) = &self.vaps { + vaps.validate()?; + } + if let Some(tas) = &self.tas { + tas.validate()?; + } + if let Some(rks) = &self.rks { + rks.validate()?; + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ManifestState { + pub mis: Vec, + pub most_recent_update: time::OffsetDateTime, + pub hash: Vec, +} + +impl ManifestState { + pub fn validate(&self) -> Result<(), String> { + validate_sha256_digest("ManifestState.hash", &self.hash)?; + validate_sorted_unique_by( + &self.mis, + |item| item.hash.as_slice(), + "ManifestState.mis must be sorted by hash and unique", + )?; + for instance in &self.mis { + instance.validate()?; + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ManifestInstance { + pub hash: Vec, + pub size: u64, + pub aki: Vec, + pub manifest_number: BigUnsigned, + pub this_update: time::OffsetDateTime, + pub locations: Vec>, + pub subordinates: Vec>, +} + +impl ManifestInstance { + pub fn validate(&self) -> Result<(), String> { + validate_sha256_digest("ManifestInstance.hash", &self.hash)?; + if self.size < 1000 { + return Err(format!( + "ManifestInstance.size must be >= 1000, got {}", + self.size + )); + } + validate_key_identifier("ManifestInstance.aki", &self.aki)?; + validate_big_unsigned_bytes( + "ManifestInstance.manifest_number", + &self.manifest_number.bytes_be, + )?; + if self.locations.is_empty() { + return Err( + "ManifestInstance.locations must contain at least one AccessDescription".into(), + ); + } + for location in &self.locations { + validate_full_der_with_tag("ManifestInstance.locations[]", location, Some(0x30))?; + } + if !self.subordinates.is_empty() { + validate_sorted_unique_bytes( + &self.subordinates, + KEY_IDENTIFIER_LEN_SHA1, + "ManifestInstance.subordinates must be sorted/unique 20-byte SKIs", + )?; + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RoaPayloadState { + pub rps: Vec, + pub hash: Vec, +} + +impl RoaPayloadState { + pub fn validate(&self) -> Result<(), String> { + validate_sha256_digest("ROAPayloadState.hash", &self.hash)?; + validate_sorted_unique_by( + &self.rps, + |item| &item.as_id, + "ROAPayloadState.rps must be sorted by asID and unique", + )?; + for set in &self.rps { + set.validate()?; + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RoaPayloadSet { + pub as_id: u32, + pub ip_addr_blocks: Vec>, +} + +impl RoaPayloadSet { + pub fn validate(&self) -> Result<(), String> { + if self.ip_addr_blocks.is_empty() || self.ip_addr_blocks.len() > 2 { + return Err(format!( + "ROAPayloadSet.ip_addr_blocks must contain 1..=2 entries, got {}", + self.ip_addr_blocks.len() + )); + } + for block in &self.ip_addr_blocks { + validate_full_der_with_tag("ROAPayloadSet.ip_addr_blocks[]", block, Some(0x30))?; + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AspaPayloadState { + pub aps: Vec, + pub hash: Vec, +} + +impl AspaPayloadState { + pub fn validate(&self) -> Result<(), String> { + validate_sha256_digest("ASPAPayloadState.hash", &self.hash)?; + validate_sorted_unique_by( + &self.aps, + |item| &item.customer_as_id, + "ASPAPayloadState.aps must be sorted by customerASID and unique", + )?; + for set in &self.aps { + set.validate()?; + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AspaPayloadSet { + pub customer_as_id: u32, + pub providers: Vec, +} + +impl AspaPayloadSet { + pub fn validate(&self) -> Result<(), String> { + if self.providers.is_empty() { + return Err("ASPAPayloadSet.providers must be non-empty".into()); + } + validate_sorted_unique_by( + &self.providers, + |provider| provider, + "ASPAPayloadSet.providers must be sorted ascending and unique", + ) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TrustAnchorState { + pub skis: Vec>, + pub hash: Vec, +} + +impl TrustAnchorState { + pub fn validate(&self) -> Result<(), String> { + if self.skis.is_empty() { + return Err("TrustAnchorState.skis must be non-empty".into()); + } + validate_sha256_digest("TrustAnchorState.hash", &self.hash)?; + validate_sorted_unique_bytes( + &self.skis, + KEY_IDENTIFIER_LEN_SHA1, + "TrustAnchorState.skis must be sorted/unique 20-byte SKIs", + ) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RouterKeyState { + pub rksets: Vec, + pub hash: Vec, +} + +impl RouterKeyState { + pub fn validate(&self) -> Result<(), String> { + validate_sha256_digest("RouterKeyState.hash", &self.hash)?; + validate_sorted_unique_by( + &self.rksets, + |item| &item.as_id, + "RouterKeyState.rksets must be sorted by asID and unique", + )?; + for rkset in &self.rksets { + rkset.validate()?; + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RouterKeySet { + pub as_id: u32, + pub router_keys: Vec, +} + +impl RouterKeySet { + pub fn validate(&self) -> Result<(), String> { + if self.router_keys.is_empty() { + return Err("RouterKeySet.router_keys must be non-empty".into()); + } + validate_sorted_unique_by( + &self.router_keys, + |key| key, + "RouterKeySet.router_keys must be sorted by SKI and unique by (SKI, SPKI DER)", + )?; + for key in &self.router_keys { + key.validate()?; + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct RouterKey { + pub ski: Vec, + pub spki_der: Vec, +} + +impl RouterKey { + pub fn validate(&self) -> Result<(), String> { + validate_key_identifier("RouterKey.ski", &self.ski)?; + validate_full_der_with_tag("RouterKey.spki_der", &self.spki_der, Some(0x30)) + } +} + +fn validate_sha256_digest(field: &str, bytes: &[u8]) -> Result<(), String> { + if bytes.len() != DIGEST_LEN_SHA256 { + return Err(format!( + "{field} must be {DIGEST_LEN_SHA256} bytes, got {}", + bytes.len() + )); + } + Ok(()) +} + +fn validate_key_identifier(field: &str, bytes: &[u8]) -> Result<(), String> { + if bytes.len() != KEY_IDENTIFIER_LEN_SHA1 { + return Err(format!( + "{field} must be {KEY_IDENTIFIER_LEN_SHA1} bytes, got {}", + bytes.len() + )); + } + Ok(()) +} + +fn validate_big_unsigned_bytes(field: &str, bytes: &[u8]) -> Result<(), String> { + if bytes.is_empty() { + return Err(format!("{field} must not be empty")); + } + if bytes.len() > 1 && bytes[0] == 0x00 { + return Err(format!( + "{field} must be minimally encoded as an unsigned integer" + )); + } + Ok(()) +} + +fn validate_sorted_unique_by( + values: &[T], + key_fn: impl Fn(&T) -> &K, + message: &str, +) -> Result<(), String> { + for window in values.windows(2) { + if key_fn(&window[0]) >= key_fn(&window[1]) { + return Err(message.to_string()); + } + } + Ok(()) +} + +fn validate_sorted_unique_bytes( + values: &[Vec], + expected_len: usize, + message: &str, +) -> Result<(), String> { + for value in values { + if value.len() != expected_len { + return Err(message.to_string()); + } + } + for window in values.windows(2) { + if window[0] >= window[1] { + return Err(message.to_string()); + } + } + Ok(()) +} + +fn validate_full_der_with_tag( + field: &str, + der: &[u8], + expected_tag: Option, +) -> Result<(), String> { + let (tag, _value, rem) = der_take_tlv(der).map_err(|e| format!("{field}: {e}"))?; + if !rem.is_empty() { + return Err(format!("{field}: trailing bytes after DER object")); + } + if let Some(expected_tag) = expected_tag { + if tag != expected_tag { + return Err(format!( + "{field}: unexpected tag 0x{tag:02X}, expected 0x{expected_tag:02X}" + )); + } + } + Ok(()) +} diff --git a/crates/panda-rpki-validator/src/ccr/state_digest.rs b/crates/panda-rpki-validator/src/ccr/state_digest.rs new file mode 100644 index 0000000..0a60219 --- /dev/null +++ b/crates/panda-rpki-validator/src/ccr/state_digest.rs @@ -0,0 +1,332 @@ +use crate::data_model::common::DerReader; +use crate::data_model::oid::{OID_CT_RPKI_CCR_RAW, OID_SHA256_RAW}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CcrStateDigestSummary { + pub version: u32, + pub hash_alg_oid: String, + pub mfts: Option>, + pub vrps: Option>, + pub vaps: Option>, + pub tas: Option>, + pub rks: Option>, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CcrStateDigestComparison { + pub ours: CcrStateDigestSummary, + pub peer: CcrStateDigestSummary, + pub states: Vec, +} + +impl CcrStateDigestComparison { + pub fn matches(&self) -> bool { + self.ours.version == self.peer.version + && self.ours.hash_alg_oid == self.peer.hash_alg_oid + && self.states.iter().all(|state| state.matches) + } + + pub fn mismatched_state_names(&self) -> Vec<&'static str> { + self.states + .iter() + .filter(|state| !state.matches) + .map(|state| state.name) + .collect() + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CcrStateDigestStateComparison { + pub name: &'static str, + pub ours_present: bool, + pub peer_present: bool, + pub ours_hash_hex: Option, + pub peer_hash_hex: Option, + pub matches: bool, +} + +#[derive(Debug, thiserror::Error)] +pub enum CcrStateDigestError { + #[error("DER parse error: {0}")] + Parse(String), + + #[error("unexpected contentType OID")] + UnexpectedContentType, + + #[error("unexpected digest algorithm OID")] + UnexpectedDigestAlgorithm, + + #[error("unexpected CCR field tag 0x{0:02X}")] + UnexpectedCcrField(u8), +} + +pub fn decode_state_digest_summary( + der: &[u8], +) -> Result { + let mut top = DerReader::new(der); + let mut seq = top.take_sequence().map_err(CcrStateDigestError::Parse)?; + if !top.is_empty() { + return Err(CcrStateDigestError::Parse( + "trailing bytes after ContentInfo".into(), + )); + } + let content_type_raw = seq.take_tag(0x06).map_err(CcrStateDigestError::Parse)?; + if content_type_raw != OID_CT_RPKI_CCR_RAW { + return Err(CcrStateDigestError::UnexpectedContentType); + } + let inner = seq.take_tag(0xA0).map_err(CcrStateDigestError::Parse)?; + if !seq.is_empty() { + return Err(CcrStateDigestError::Parse( + "trailing fields in ContentInfo".into(), + )); + } + decode_ccr_state_digest_summary(inner) +} + +pub fn compare_state_digests( + ours_der: &[u8], + peer_der: &[u8], +) -> Result { + let ours = decode_state_digest_summary(ours_der)?; + let peer = decode_state_digest_summary(peer_der)?; + let states = vec![ + compare_state("mfts", &ours.mfts, &peer.mfts), + compare_state("vrps", &ours.vrps, &peer.vrps), + compare_state("vaps", &ours.vaps, &peer.vaps), + compare_state("tas", &ours.tas, &peer.tas), + compare_state("rks", &ours.rks, &peer.rks), + ]; + Ok(CcrStateDigestComparison { ours, peer, states }) +} + +fn decode_ccr_state_digest_summary( + der: &[u8], +) -> Result { + let mut top = DerReader::new(der); + let mut seq = top.take_sequence().map_err(CcrStateDigestError::Parse)?; + if !top.is_empty() { + return Err(CcrStateDigestError::Parse( + "trailing bytes after CCR".into(), + )); + } + + let version = if !seq.is_empty() && seq.peek_tag().map_err(CcrStateDigestError::Parse)? == 0xA0 + { + let explicit = seq.take_tag(0xA0).map_err(CcrStateDigestError::Parse)?; + let mut inner = DerReader::new(explicit); + let version = inner.take_uint_u64().map_err(CcrStateDigestError::Parse)? as u32; + if !inner.is_empty() { + return Err(CcrStateDigestError::Parse( + "trailing bytes inside CCR version EXPLICIT".into(), + )); + } + version + } else { + crate::ccr::model::CCR_VERSION_V0 + }; + + let hash_alg_oid = + decode_digest_algorithm_oid(seq.take_sequence().map_err(CcrStateDigestError::Parse)?)?; + let _produced_at = seq.take_tag(0x18).map_err(CcrStateDigestError::Parse)?; + + let mut mfts = None; + let mut vrps = None; + let mut vaps = None; + let mut tas = None; + let mut rks = None; + while !seq.is_empty() { + let (tag, value) = seq.take_any().map_err(CcrStateDigestError::Parse)?; + match tag { + 0xA1 => mfts = Some(read_state_hash(value, StateShape::Manifest)?), + 0xA2 => vrps = Some(read_state_hash(value, StateShape::PayloadThenHash)?), + 0xA3 => vaps = Some(read_state_hash(value, StateShape::PayloadThenHash)?), + 0xA4 => tas = Some(read_state_hash(value, StateShape::PayloadThenHash)?), + 0xA5 => rks = Some(read_state_hash(value, StateShape::PayloadThenHash)?), + other => return Err(CcrStateDigestError::UnexpectedCcrField(other)), + } + } + + Ok(CcrStateDigestSummary { + version, + hash_alg_oid, + mfts, + vrps, + vaps, + tas, + rks, + }) +} + +#[derive(Clone, Copy)] +enum StateShape { + Manifest, + PayloadThenHash, +} + +fn read_state_hash(explicit_der: &[u8], shape: StateShape) -> Result, CcrStateDigestError> { + let mut outer = DerReader::new(explicit_der); + let mut seq = outer.take_sequence().map_err(CcrStateDigestError::Parse)?; + if !outer.is_empty() { + return Err(CcrStateDigestError::Parse( + "trailing bytes after CCR state".into(), + )); + } + match shape { + StateShape::Manifest => { + seq.skip_any().map_err(CcrStateDigestError::Parse)?; + seq.take_tag(0x18).map_err(CcrStateDigestError::Parse)?; + } + StateShape::PayloadThenHash => { + seq.skip_any().map_err(CcrStateDigestError::Parse)?; + } + } + let hash = seq + .take_octet_string() + .map_err(CcrStateDigestError::Parse)? + .to_vec(); + if hash.len() != crate::ccr::model::DIGEST_LEN_SHA256 { + return Err(CcrStateDigestError::Parse(format!( + "state hash must be {} bytes, got {}", + crate::ccr::model::DIGEST_LEN_SHA256, + hash.len() + ))); + } + if !seq.is_empty() { + return Err(CcrStateDigestError::Parse( + "trailing fields after CCR state hash".into(), + )); + } + Ok(hash) +} + +fn decode_digest_algorithm_oid(mut seq: DerReader<'_>) -> Result { + let oid_raw = seq.take_tag(0x06).map_err(CcrStateDigestError::Parse)?; + if oid_raw != OID_SHA256_RAW { + return Err(CcrStateDigestError::UnexpectedDigestAlgorithm); + } + if !seq.is_empty() { + let tag = seq.peek_tag().map_err(CcrStateDigestError::Parse)?; + if tag == 0x05 { + let null = seq.take_tag(0x05).map_err(CcrStateDigestError::Parse)?; + if !null.is_empty() { + return Err(CcrStateDigestError::Parse( + "AlgorithmIdentifier NULL parameters must be empty".into(), + )); + } + } + } + if !seq.is_empty() { + return Err(CcrStateDigestError::Parse( + "trailing fields in DigestAlgorithmIdentifier".into(), + )); + } + Ok(crate::data_model::oid::OID_SHA256.to_string()) +} + +fn compare_state( + name: &'static str, + ours: &Option>, + peer: &Option>, +) -> CcrStateDigestStateComparison { + CcrStateDigestStateComparison { + name, + ours_present: ours.is_some(), + peer_present: peer.is_some(), + ours_hash_hex: ours.as_ref().map(hex::encode), + peer_hash_hex: peer.as_ref().map(hex::encode), + matches: ours == peer, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ccr::{ + CcrContentInfo, CcrDigestAlgorithm, RpkiCanonicalCacheRepresentation, + build_aspa_payload_state, build_roa_payload_state, encode_content_info, + }; + use crate::data_model::roa::{IpPrefix, RoaAfi}; + use crate::validation::objects::{AspaAttestation, Vrp}; + + fn sample_content(produced_at: time::OffsetDateTime) -> CcrContentInfo { + let vrps = build_roa_payload_state(&[Vrp { + asn: 64496, + prefix: IpPrefix { + afi: RoaAfi::Ipv4, + prefix_len: 24, + addr: [192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + }, + max_length: 24, + }]) + .expect("build vrps"); + let vaps = build_aspa_payload_state(&[AspaAttestation { + customer_as_id: 64496, + provider_as_ids: vec![64497], + }]) + .expect("build vaps"); + CcrContentInfo::new(RpkiCanonicalCacheRepresentation { + version: 0, + hash_alg: CcrDigestAlgorithm::Sha256, + produced_at, + mfts: None, + vrps: Some(vrps), + vaps: Some(vaps), + tas: None, + rks: None, + }) + } + + #[test] + fn decode_state_digest_summary_extracts_hashes_without_full_model() { + let content = sample_content(time::OffsetDateTime::UNIX_EPOCH); + let der = encode_content_info(&content).expect("encode"); + let summary = decode_state_digest_summary(&der).expect("summary"); + assert_eq!(summary.version, 0); + assert_eq!(summary.hash_alg_oid, crate::data_model::oid::OID_SHA256); + assert_eq!( + summary.vrps.as_deref(), + content + .content + .vrps + .as_ref() + .map(|state| state.hash.as_slice()) + ); + assert_eq!( + summary.vaps.as_deref(), + content + .content + .vaps + .as_ref() + .map(|state| state.hash.as_slice()) + ); + assert!(summary.mfts.is_none()); + assert!(summary.tas.is_none()); + assert!(summary.rks.is_none()); + } + + #[test] + fn compare_state_digests_ignores_produced_at() { + let first = encode_content_info(&sample_content(time::OffsetDateTime::UNIX_EPOCH)) + .expect("encode first"); + let second = encode_content_info(&sample_content( + time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(60), + )) + .expect("encode second"); + let comparison = compare_state_digests(&first, &second).expect("compare"); + assert!(comparison.matches()); + assert!(comparison.mismatched_state_names().is_empty()); + } + + #[test] + fn compare_state_digests_reports_mismatched_state() { + let first = encode_content_info(&sample_content(time::OffsetDateTime::UNIX_EPOCH)) + .expect("encode first"); + let mut second_content = + sample_content(time::OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(60)); + second_content.content.vaps.as_mut().expect("vaps").hash[0] ^= 0xFF; + let second = encode_content_info(&second_content).expect("encode second"); + let comparison = compare_state_digests(&first, &second).expect("compare"); + assert!(!comparison.matches()); + assert_eq!(comparison.mismatched_state_names(), vec!["vaps"]); + } +} diff --git a/crates/panda-rpki-validator/src/ccr/verify.rs b/crates/panda-rpki-validator/src/ccr/verify.rs new file mode 100644 index 0000000..4b883ce --- /dev/null +++ b/crates/panda-rpki-validator/src/ccr/verify.rs @@ -0,0 +1,704 @@ +use crate::ccr::decode::{CcrDecodeError, decode_content_info}; +use crate::ccr::encode::{ + encode_aspa_payload_state_payload_der, encode_manifest_state_payload_der, + encode_roa_payload_state_payload_der, encode_router_key_state_payload_der, + encode_trust_anchor_state_payload_der, +}; +use crate::ccr::hash::verify_state_hash; +use crate::ccr::model::{CcrContentInfo, RouterKeyState, TrustAnchorState}; +use crate::storage::{RocksStore, VcirArtifactRole}; +use serde::Serialize; +use std::collections::BTreeSet; +use std::path::Path; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct CcrVerifySummary { + pub content_type_oid: String, + pub version: u32, + pub produced_at_rfc3339_utc: String, + pub state_hashes_ok: bool, + pub manifest_instances: usize, + pub roa_payload_sets: usize, + pub roa_vrp_count: usize, + pub aspa_payload_sets: usize, + pub trust_anchor_ski_count: usize, + pub router_key_sets: usize, + pub router_key_count: usize, +} + +#[derive(Debug, thiserror::Error)] +pub enum CcrVerifyError { + #[error("CCR decode failed: {0}")] + Decode(#[from] CcrDecodeError), + + #[error("ManifestState hash mismatch")] + ManifestHashMismatch, + + #[error("ROAPayloadState hash mismatch")] + RoaHashMismatch, + + #[error("ASPAPayloadState hash mismatch")] + AspaHashMismatch, + + #[error("TrustAnchorState hash mismatch")] + TrustAnchorHashMismatch, + + #[error("RouterKeyState hash mismatch")] + RouterKeyHashMismatch, + + #[error("read report json failed: {0}: {1}")] + ReportRead(String, String), + + #[error("parse report json failed: {0}")] + ReportParse(String), + + #[error("VRP set mismatch: only_in_ccr={only_in_ccr} only_in_report={only_in_report}")] + ReportVrpMismatch { + only_in_ccr: usize, + only_in_report: usize, + }, + + #[error("ASPA set mismatch: only_in_ccr={only_in_ccr} only_in_report={only_in_report}")] + ReportAspaMismatch { + only_in_ccr: usize, + only_in_report: usize, + }, + + #[error("open RocksDB failed: {0}")] + OpenStore(String), + + #[error("list VCIRs failed: {0}")] + ListVcirs(String), + + #[error("VCIR manifest set mismatch: only_in_ccr={only_in_ccr} only_in_vcir={only_in_vcir}")] + VcirManifestMismatch { + only_in_ccr: usize, + only_in_vcir: usize, + }, +} + +pub fn verify_content_info_bytes(der: &[u8]) -> Result { + let content_info = decode_content_info(der)?; + verify_content_info(&content_info) +} + +pub fn verify_content_info( + content_info: &CcrContentInfo, +) -> Result { + content_info.validate().map_err(CcrDecodeError::Validate)?; + let state_hashes_ok = true; + let mut manifest_instances = 0usize; + let mut roa_payload_sets = 0usize; + let mut roa_vrp_count = 0usize; + let mut aspa_payload_sets = 0usize; + let mut trust_anchor_ski_count = 0usize; + let mut router_key_sets = 0usize; + let mut router_key_count = 0usize; + + if let Some(mfts) = &content_info.content.mfts { + let payload_der = encode_manifest_state_payload_der(&mfts.mis) + .map_err(|e| CcrVerifyError::Decode(CcrDecodeError::Validate(e.to_string())))?; + if !verify_state_hash(&mfts.hash, &payload_der) { + return Err(CcrVerifyError::ManifestHashMismatch); + } + manifest_instances = mfts.mis.len(); + } + if let Some(vrps) = &content_info.content.vrps { + let payload_der = encode_roa_payload_state_payload_der(&vrps.rps) + .map_err(|e| CcrVerifyError::Decode(CcrDecodeError::Validate(e.to_string())))?; + if !verify_state_hash(&vrps.hash, &payload_der) { + return Err(CcrVerifyError::RoaHashMismatch); + } + roa_payload_sets = vrps.rps.len(); + roa_vrp_count = vrps + .rps + .iter() + .map(|set| count_roa_block_entries(&set.ip_addr_blocks)) + .sum(); + } + if let Some(vaps) = &content_info.content.vaps { + let payload_der = encode_aspa_payload_state_payload_der(&vaps.aps) + .map_err(|e| CcrVerifyError::Decode(CcrDecodeError::Validate(e.to_string())))?; + if !verify_state_hash(&vaps.hash, &payload_der) { + return Err(CcrVerifyError::AspaHashMismatch); + } + aspa_payload_sets = vaps.aps.len(); + } + if let Some(tas) = &content_info.content.tas { + verify_trust_anchor_state_hash(tas)?; + trust_anchor_ski_count = tas.skis.len(); + } + if let Some(rks) = &content_info.content.rks { + verify_router_key_state_hash(rks)?; + router_key_sets = rks.rksets.len(); + router_key_count = rks.rksets.iter().map(|set| set.router_keys.len()).sum(); + } + + let produced_at_rfc3339_utc = content_info + .content + .produced_at + .to_offset(time::UtcOffset::UTC) + .format(&time::format_description::well_known::Rfc3339) + .map_err(|e| CcrVerifyError::Decode(CcrDecodeError::Validate(e.to_string())))?; + + Ok(CcrVerifySummary { + content_type_oid: content_info.content_type_oid.clone(), + version: content_info.content.version, + produced_at_rfc3339_utc, + state_hashes_ok, + manifest_instances, + roa_payload_sets, + roa_vrp_count, + aspa_payload_sets, + trust_anchor_ski_count, + router_key_sets, + router_key_count, + }) +} + +pub fn verify_against_report_json_path( + content_info: &CcrContentInfo, + report_json_path: &Path, +) -> Result<(), CcrVerifyError> { + let bytes = std::fs::read(report_json_path).map_err(|e| { + CcrVerifyError::ReportRead(report_json_path.display().to_string(), e.to_string()) + })?; + let json: serde_json::Value = + serde_json::from_slice(&bytes).map_err(|e| CcrVerifyError::ReportParse(e.to_string()))?; + + let report_vrps = report_vrp_keys(&json)?; + let ccr_vrps = extract_vrp_rows(content_info)?; + let only_in_ccr = ccr_vrps.difference(&report_vrps).count(); + let only_in_report = report_vrps.difference(&ccr_vrps).count(); + if only_in_ccr != 0 || only_in_report != 0 { + return Err(CcrVerifyError::ReportVrpMismatch { + only_in_ccr, + only_in_report, + }); + } + + let report_aspas = report_aspa_keys(&json)?; + let ccr_aspas = ccr_aspa_keys(content_info)?; + let only_in_ccr = ccr_aspas.difference(&report_aspas).count(); + let only_in_report = report_aspas.difference(&ccr_aspas).count(); + if only_in_ccr != 0 || only_in_report != 0 { + return Err(CcrVerifyError::ReportAspaMismatch { + only_in_ccr, + only_in_report, + }); + } + Ok(()) +} + +pub fn verify_against_vcir_store_path( + content_info: &CcrContentInfo, + db_path: &Path, +) -> Result<(), CcrVerifyError> { + let store = RocksStore::open(db_path).map_err(|e| CcrVerifyError::OpenStore(e.to_string()))?; + verify_against_vcir_store(content_info, &store) +} + +pub fn verify_against_vcir_store( + content_info: &CcrContentInfo, + store: &RocksStore, +) -> Result<(), CcrVerifyError> { + let Some(mfts) = &content_info.content.mfts else { + return Ok(()); + }; + let vcirs = store + .list_vcirs() + .map_err(|e| CcrVerifyError::ListVcirs(e.to_string()))?; + let mut vcir_hashes = BTreeSet::new(); + for vcir in vcirs { + if let Some(artifact) = vcir.related_artifacts.iter().find(|artifact| { + artifact.artifact_role == VcirArtifactRole::Manifest + && artifact.uri.as_deref() == Some(vcir.current_manifest_rsync_uri.as_str()) + }) { + if let Ok(bytes) = hex::decode(&artifact.sha256) { + vcir_hashes.insert(bytes); + } + } + } + let ccr_hashes = mfts + .mis + .iter() + .map(|mi| mi.hash.clone()) + .collect::>(); + let only_in_ccr = ccr_hashes.difference(&vcir_hashes).count(); + let only_in_vcir = vcir_hashes.difference(&ccr_hashes).count(); + if only_in_ccr != 0 || only_in_vcir != 0 { + return Err(CcrVerifyError::VcirManifestMismatch { + only_in_ccr, + only_in_vcir, + }); + } + Ok(()) +} + +fn verify_trust_anchor_state_hash(state: &TrustAnchorState) -> Result<(), CcrVerifyError> { + let payload_der = encode_trust_anchor_state_payload_der(&state.skis) + .map_err(|e| CcrVerifyError::Decode(CcrDecodeError::Validate(e.to_string())))?; + if !verify_state_hash(&state.hash, &payload_der) { + return Err(CcrVerifyError::TrustAnchorHashMismatch); + } + Ok(()) +} + +fn verify_router_key_state_hash(state: &RouterKeyState) -> Result<(), CcrVerifyError> { + let payload_der = encode_router_key_state_payload_der(&state.rksets) + .map_err(|e| CcrVerifyError::Decode(CcrDecodeError::Validate(e.to_string())))?; + if !verify_state_hash(&state.hash, &payload_der) { + return Err(CcrVerifyError::RouterKeyHashMismatch); + } + Ok(()) +} + +fn report_vrp_keys( + json: &serde_json::Value, +) -> Result, CcrVerifyError> { + let mut out = BTreeSet::new(); + let Some(items) = json.get("vrps").and_then(|v| v.as_array()) else { + return Ok(out); + }; + for item in items { + let asn = item + .get("asn") + .and_then(|v| v.as_u64()) + .ok_or_else(|| CcrVerifyError::ReportParse("vrps[].asn missing".into()))? + as u32; + let prefix = item + .get("prefix") + .and_then(|v| v.as_str()) + .ok_or_else(|| CcrVerifyError::ReportParse("vrps[].prefix missing".into()))? + .to_string(); + let max_length = item + .get("max_length") + .and_then(|v| v.as_u64()) + .ok_or_else(|| CcrVerifyError::ReportParse("vrps[].max_length missing".into()))? + as u16; + out.insert((asn, prefix, max_length)); + } + Ok(out) +} + +fn report_aspa_keys(json: &serde_json::Value) -> Result)>, CcrVerifyError> { + let mut out = BTreeSet::new(); + let Some(items) = json.get("aspas").and_then(|v| v.as_array()) else { + return Ok(out); + }; + for item in items { + let customer = item + .get("customer_as_id") + .and_then(|v| v.as_u64()) + .ok_or_else(|| CcrVerifyError::ReportParse("aspas[].customer_as_id missing".into()))? + as u32; + let mut providers = item + .get("provider_as_ids") + .and_then(|v| v.as_array()) + .ok_or_else(|| CcrVerifyError::ReportParse("aspas[].provider_as_ids missing".into()))? + .iter() + .map(|v| { + v.as_u64() + .ok_or_else(|| CcrVerifyError::ReportParse("provider_as_ids[] invalid".into())) + .map(|v| v as u32) + }) + .collect::, _>>()?; + providers.sort_unstable(); + providers.dedup(); + out.insert((customer, providers)); + } + Ok(out) +} + +pub fn extract_vrp_rows( + content_info: &CcrContentInfo, +) -> Result, CcrVerifyError> { + let mut out = BTreeSet::new(); + let Some(vrps) = &content_info.content.vrps else { + return Ok(out); + }; + for set in &vrps.rps { + for block in &set.ip_addr_blocks { + let (afi, entries) = decode_roa_family_block(block)?; + for (prefix_len, addr_bytes, max_len) in entries { + let prefix = format_prefix(afi, &addr_bytes, prefix_len)?; + out.insert((set.as_id, prefix, max_len.unwrap_or(prefix_len as u16))); + } + } + } + Ok(out) +} + +fn ccr_aspa_keys( + content_info: &CcrContentInfo, +) -> Result)>, CcrVerifyError> { + let mut out = BTreeSet::new(); + let Some(vaps) = &content_info.content.vaps else { + return Ok(out); + }; + for set in &vaps.aps { + out.insert((set.customer_as_id, set.providers.clone())); + } + Ok(out) +} + +fn decode_roa_family_block( + block: &[u8], +) -> Result<(u16, Vec<(u8, Vec, Option)>), CcrVerifyError> { + let mut top = crate::data_model::common::DerReader::new(block); + let mut seq = top + .take_sequence() + .map_err(|e| CcrVerifyError::Decode(CcrDecodeError::Parse(e)))?; + if !top.is_empty() { + return Err(CcrVerifyError::Decode(CcrDecodeError::Parse( + "trailing bytes after ROAIPAddressFamily".into(), + ))); + } + let afi_bytes = seq + .take_octet_string() + .map_err(|e| CcrVerifyError::Decode(CcrDecodeError::Parse(e)))?; + let afi = u16::from_be_bytes([afi_bytes[0], afi_bytes[1]]); + let mut addrs = seq + .take_sequence() + .map_err(|e| CcrVerifyError::Decode(CcrDecodeError::Parse(e)))?; + let mut entries = Vec::new(); + while !addrs.is_empty() { + let mut addr_seq = addrs + .take_sequence() + .map_err(|e| CcrVerifyError::Decode(CcrDecodeError::Parse(e)))?; + let (unused_bits, content) = addr_seq + .take_bit_string() + .map_err(|e| CcrVerifyError::Decode(CcrDecodeError::Parse(e)))?; + let prefix_len = (content.len() * 8) as u8 - unused_bits; + let max_len = if addr_seq.is_empty() { + None + } else { + Some( + addr_seq + .take_uint_u64() + .map_err(|e| CcrVerifyError::Decode(CcrDecodeError::Parse(e)))? + as u16, + ) + }; + entries.push((prefix_len, content.to_vec(), max_len)); + } + Ok((afi, entries)) +} + +fn format_prefix(afi: u16, addr_bytes: &[u8], prefix_len: u8) -> Result { + match afi { + 1 => { + let mut full = [0u8; 4]; + full[..addr_bytes.len()].copy_from_slice(addr_bytes); + Ok(format!("{}/{prefix_len}", std::net::Ipv4Addr::from(full))) + } + 2 => { + let mut full = [0u8; 16]; + full[..addr_bytes.len()].copy_from_slice(addr_bytes); + Ok(format!("{}/{prefix_len}", std::net::Ipv6Addr::from(full))) + } + other => Err(CcrVerifyError::Decode(CcrDecodeError::Parse(format!( + "unsupported AFI {other}" + )))), + } +} + +fn count_roa_block_entries(blocks: &[Vec]) -> usize { + blocks + .iter() + .map(|block| { + decode_roa_family_block(block) + .map(|(_, entries)| entries.len()) + .unwrap_or(0) + }) + .sum() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ccr::build::{build_aspa_payload_state, build_roa_payload_state}; + use crate::ccr::encode::{ + encode_manifest_state_payload_der, encode_roa_payload_state_payload_der, + encode_router_key_state_payload_der, encode_trust_anchor_state_payload_der, + }; + use crate::ccr::model::{ + CcrDigestAlgorithm, ManifestInstance, ManifestState, RouterKey, RouterKeySet, + RpkiCanonicalCacheRepresentation, + }; + use crate::data_model::common::BigUnsigned; + use crate::data_model::roa::{IpPrefix, RoaAfi}; + use crate::storage::{ + PackTime, ValidatedCaInstanceResult, ValidatedManifestMeta, VcirArtifactKind, + VcirArtifactRole, VcirArtifactValidationStatus, VcirAuditSummary, + VcirCcrManifestProjection, VcirChildEntry, VcirInstanceGate, VcirRelatedArtifact, + VcirSummary, + }; + use crate::validation::objects::{AspaAttestation, Vrp}; + + fn sample_time() -> time::OffsetDateTime { + time::OffsetDateTime::parse( + "2026-03-24T00:00:00Z", + &time::format_description::well_known::Rfc3339, + ) + .expect("time") + } + + fn sample_content_info() -> CcrContentInfo { + let mis = vec![ManifestInstance { + hash: vec![0x10; 32], + size: 2048, + aki: vec![0x20; 20], + manifest_number: BigUnsigned { bytes_be: vec![1] }, + this_update: sample_time(), + locations: vec![vec![0x30, 0x00]], + subordinates: vec![vec![0x30; 20]], + }]; + let mfts = ManifestState { + most_recent_update: sample_time(), + hash: crate::ccr::compute_state_hash(&encode_manifest_state_payload_der(&mis).unwrap()), + mis, + }; + let vrps = build_roa_payload_state(&[Vrp { + asn: 64496, + prefix: IpPrefix { + afi: RoaAfi::Ipv4, + prefix_len: 0, + addr: [0; 16], + }, + max_length: 0, + }]) + .expect("build roa state"); + let vaps = build_aspa_payload_state(&[AspaAttestation { + customer_as_id: 64496, + provider_as_ids: vec![64497], + }]) + .expect("build aspa state"); + let skis = vec![vec![0x11; 20]]; + let tas = TrustAnchorState { + hash: crate::ccr::compute_state_hash( + &encode_trust_anchor_state_payload_der(&skis).unwrap(), + ), + skis, + }; + let rksets = vec![RouterKeySet { + as_id: 64496, + router_keys: vec![RouterKey { + ski: vec![0x22; 20], + spki_der: vec![0x30, 0x00], + }], + }]; + let rks = RouterKeyState { + hash: crate::ccr::compute_state_hash( + &encode_router_key_state_payload_der(&rksets).unwrap(), + ), + rksets, + }; + CcrContentInfo::new(RpkiCanonicalCacheRepresentation { + version: 0, + hash_alg: CcrDigestAlgorithm::Sha256, + produced_at: sample_time(), + mfts: Some(mfts), + vrps: Some(vrps), + vaps: Some(vaps), + tas: Some(tas), + rks: Some(rks), + }) + } + + fn sample_ccr_manifest_projection(manifest_rsync_uri: &str) -> VcirCcrManifestProjection { + VcirCcrManifestProjection { + manifest_rsync_uri: manifest_rsync_uri.to_string(), + manifest_sha256: vec![0x44; 32], + manifest_size: 2048, + manifest_ee_aki: vec![0x55; 20], + manifest_number_be: vec![1], + manifest_this_update: PackTime::from_utc_offset_datetime(sample_time()), + manifest_sia_locations_der: vec![vec![ + 0x30, 0x11, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x05, 0x86, 0x05, + b'r', b's', b'y', b'n', b'c', + ]], + subordinate_skis: vec![vec![0x33; 20]], + } + } + + #[test] + fn verify_detects_each_state_hash_mismatch() { + let mut ci = sample_content_info(); + ci.content.vrps.as_mut().unwrap().hash[0] ^= 0x01; + assert!(matches!( + verify_content_info(&ci), + Err(CcrVerifyError::RoaHashMismatch) + )); + + let mut ci = sample_content_info(); + ci.content.vaps.as_mut().unwrap().hash[0] ^= 0x01; + assert!(matches!( + verify_content_info(&ci), + Err(CcrVerifyError::AspaHashMismatch) + )); + + let mut ci = sample_content_info(); + ci.content.tas.as_mut().unwrap().hash[0] ^= 0x01; + assert!(matches!( + verify_content_info(&ci), + Err(CcrVerifyError::TrustAnchorHashMismatch) + )); + + let mut ci = sample_content_info(); + ci.content.rks.as_mut().unwrap().hash[0] ^= 0x01; + assert!(matches!( + verify_content_info(&ci), + Err(CcrVerifyError::RouterKeyHashMismatch) + )); + } + + #[test] + fn verify_against_report_json_accepts_matching_report_and_rejects_parse_errors() { + let td = tempfile::tempdir().expect("tempdir"); + let report = serde_json::json!({ + "vrps": [{"asn": 64496, "prefix": "0.0.0.0/0", "max_length": 0}], + "aspas": [{"customer_as_id": 64496, "provider_as_ids": [64497]}] + }); + let report_path = td.path().join("report.json"); + std::fs::write(&report_path, serde_json::to_vec(&report).unwrap()).unwrap(); + verify_against_report_json_path(&sample_content_info(), &report_path) + .expect("matching report"); + + let bad_path = td.path().join("bad.json"); + std::fs::write(&bad_path, b"not-json").unwrap(); + assert!(matches!( + verify_against_report_json_path(&sample_content_info(), &bad_path), + Err(CcrVerifyError::ReportParse(_)) + )); + } + + #[test] + fn verify_against_report_json_rejects_missing_fields_and_aspa_mismatch() { + let td = tempfile::tempdir().expect("tempdir"); + let missing = + serde_json::json!({"vrps":[{"prefix":"0.0.0.0/0","max_length":0}],"aspas":[]}); + let missing_path = td.path().join("missing.json"); + std::fs::write(&missing_path, serde_json::to_vec(&missing).unwrap()).unwrap(); + assert!(matches!( + verify_against_report_json_path(&sample_content_info(), &missing_path), + Err(CcrVerifyError::ReportParse(_)) + )); + + let mismatch = serde_json::json!({ + "vrps": [{"asn": 64496, "prefix": "0.0.0.0/0", "max_length": 0}], + "aspas": [{"customer_as_id": 64496, "provider_as_ids": [65000]}] + }); + let mismatch_path = td.path().join("mismatch.json"); + std::fs::write(&mismatch_path, serde_json::to_vec(&mismatch).unwrap()).unwrap(); + assert!(matches!( + verify_against_report_json_path(&sample_content_info(), &mismatch_path), + Err(CcrVerifyError::ReportAspaMismatch { .. }) + )); + } + + #[test] + fn verify_against_vcir_store_rejects_mismatched_manifest_hashes() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open db"); + let vcir = ValidatedCaInstanceResult { + manifest_rsync_uri: "rsync://example.test/current.mft".to_string(), + parent_manifest_rsync_uri: None, + tal_id: "apnic".to_string(), + ca_subject_name: "CN=test".to_string(), + ca_ski: "11".repeat(20), + issuer_ski: "22".repeat(20), + last_successful_validation_time: PackTime::from_utc_offset_datetime(sample_time()), + current_manifest_rsync_uri: "rsync://example.test/current.mft".to_string(), + current_crl_rsync_uri: "rsync://example.test/current.crl".to_string(), + validated_manifest_meta: ValidatedManifestMeta { + validated_manifest_number: vec![1], + validated_manifest_this_update: PackTime::from_utc_offset_datetime(sample_time()), + validated_manifest_next_update: PackTime::from_utc_offset_datetime(sample_time()), + }, + ccr_manifest_projection: sample_ccr_manifest_projection( + "rsync://example.test/current.mft", + ), + instance_gate: VcirInstanceGate { + manifest_next_update: PackTime::from_utc_offset_datetime(sample_time()), + current_crl_next_update: PackTime::from_utc_offset_datetime(sample_time()), + self_ca_not_after: PackTime::from_utc_offset_datetime(sample_time()), + instance_effective_until: PackTime::from_utc_offset_datetime(sample_time()), + }, + child_entries: vec![VcirChildEntry { + child_manifest_rsync_uri: "rsync://example.test/child.mft".to_string(), + child_cert_rsync_uri: "rsync://example.test/child.cer".to_string(), + child_cert_hash: "aa".repeat(32), + child_ski: "33".repeat(20), + child_rsync_base_uri: "rsync://example.test/".to_string(), + child_publication_point_rsync_uri: "rsync://example.test/".to_string(), + child_rrdp_notification_uri: None, + child_effective_ip_resources: None, + child_effective_as_resources: None, + accepted_at_validation_time: PackTime::from_utc_offset_datetime(sample_time()), + }], + local_outputs: Vec::new(), + related_artifacts: vec![VcirRelatedArtifact { + artifact_role: VcirArtifactRole::Manifest, + artifact_kind: VcirArtifactKind::Mft, + uri: Some("rsync://example.test/current.mft".to_string()), + sha256: "ff".repeat(32), + object_type: Some("mft".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }], + summary: VcirSummary { + local_vrp_count: 0, + local_aspa_count: 0, + local_router_key_count: 0, + child_count: 1, + accepted_object_count: 1, + rejected_object_count: 0, + }, + audit_summary: VcirAuditSummary { + failed_fetch_eligible: true, + last_failed_fetch_reason: None, + warning_count: 0, + audit_flags: Vec::new(), + }, + }; + store.put_vcir(&vcir).unwrap(); + assert!(matches!( + verify_against_vcir_store(&sample_content_info(), &store), + Err(CcrVerifyError::VcirManifestMismatch { .. }) + )); + } + + #[test] + fn verify_vrp_helpers_reject_bad_afi_and_count_invalid_block_as_zero() { + let block = vec![0x30, 0x08, 0x04, 0x02, 0x00, 0x63, 0x30, 0x02, 0x03, 0x00]; + let ci = CcrContentInfo::new(RpkiCanonicalCacheRepresentation { + version: 0, + hash_alg: CcrDigestAlgorithm::Sha256, + produced_at: sample_time(), + mfts: None, + vrps: Some(crate::ccr::model::RoaPayloadState { + rps: vec![crate::ccr::model::RoaPayloadSet { + as_id: 64496, + ip_addr_blocks: vec![block.clone()], + }], + hash: crate::ccr::compute_state_hash( + &encode_roa_payload_state_payload_der(&[crate::ccr::model::RoaPayloadSet { + as_id: 64496, + ip_addr_blocks: vec![block], + }]) + .unwrap(), + ), + }), + vaps: None, + tas: None, + rks: None, + }); + assert!(matches!( + extract_vrp_rows(&ci), + Err(CcrVerifyError::Decode(_)) + )); + let bad_count = count_roa_block_entries(&[vec![0x04, 0x00]]); + assert_eq!(bad_count, 0); + } +} diff --git a/crates/panda-rpki-validator/src/cir/accumulator.rs b/crates/panda-rpki-validator/src/cir/accumulator.rs new file mode 100644 index 0000000..be89d0a --- /dev/null +++ b/crates/panda-rpki-validator/src/cir/accumulator.rs @@ -0,0 +1,406 @@ +use std::collections::HashMap; + +use crate::audit::{AuditObjectResult, ObjectAuditEntry}; +use crate::cir::export::CirExportError; +use crate::cir::model::{CirObject, CirRejectedObject}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CirInputSection { + Fresh, + Cached, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct CirInputSnapshot { + pub fresh_validated_objects: Vec, + pub cached_validated_objects: Vec, + pub fresh_rejected_objects: Vec, + pub cached_rejected_objects: Vec, +} + +#[derive(Clone, Debug, Default)] +pub struct CirInputAccumulator { + fresh_objects: HashMap, + cached_objects: HashMap, + fresh_rejects: HashMap>, + cached_rejects: HashMap>, +} + +impl CirInputAccumulator { + pub fn submit_audit_entries( + &mut self, + section: CirInputSection, + entries: &[ObjectAuditEntry], + ) -> Result<(), CirExportError> { + for entry in entries { + if !matches!( + entry.result, + AuditObjectResult::Ok | AuditObjectResult::Error + ) { + continue; + } + self.insert_object(section, &entry.rsync_uri, &entry.sha256_hex)?; + if entry.result == AuditObjectResult::Error { + self.insert_reject(section, &entry.rsync_uri, entry.detail.clone()); + } + } + Ok(()) + } + + pub fn submit_snapshot(&mut self, snapshot: CirInputSnapshot) -> Result<(), CirExportError> { + for object in snapshot.fresh_validated_objects { + self.insert_object_bytes(CirInputSection::Fresh, &object.rsync_uri, object.sha256)?; + } + for object in snapshot.cached_validated_objects { + self.insert_object_bytes(CirInputSection::Cached, &object.rsync_uri, object.sha256)?; + } + for rejected in snapshot.fresh_rejected_objects { + self.insert_reject( + CirInputSection::Fresh, + &rejected.object_uri, + rejected.reason, + ); + } + for rejected in snapshot.cached_rejected_objects { + self.insert_reject( + CirInputSection::Cached, + &rejected.object_uri, + rejected.reason, + ); + } + Ok(()) + } + + pub fn insert_object( + &mut self, + section: CirInputSection, + rsync_uri: &str, + sha256_hex: &str, + ) -> Result<(), CirExportError> { + if !rsync_uri.starts_with("rsync://") || !is_sha256_hex(sha256_hex) { + return Ok(()); + } + let mut digest = [0u8; 32]; + hex::decode_to_slice(sha256_hex, &mut digest).expect("validated sha256 hex"); + self.insert_object_digest(section, rsync_uri, digest) + } + + fn insert_object_bytes( + &mut self, + section: CirInputSection, + rsync_uri: &str, + sha256: Vec, + ) -> Result<(), CirExportError> { + if !rsync_uri.starts_with("rsync://") { + return Ok(()); + } + let Ok(digest) = <[u8; 32]>::try_from(sha256) else { + return Ok(()); + }; + self.insert_object_digest(section, rsync_uri, digest) + } + + fn insert_object_digest( + &mut self, + section: CirInputSection, + rsync_uri: &str, + digest: [u8; 32], + ) -> Result<(), CirExportError> { + let objects = match section { + CirInputSection::Fresh => &mut self.fresh_objects, + CirInputSection::Cached => &mut self.cached_objects, + }; + if let Some(existing) = objects.get(rsync_uri) { + if existing != &digest { + return Err(CirExportError::ConflictingObjectHash { + rsync_uri: rsync_uri.to_string(), + first: hex::encode(existing), + second: hex::encode(digest), + }); + } + return Ok(()); + } + objects.insert(rsync_uri.to_string(), digest); + Ok(()) + } + + pub fn insert_reject( + &mut self, + section: CirInputSection, + rsync_uri: &str, + reason: Option, + ) { + if !rsync_uri.starts_with("rsync://") { + return; + } + let rejects = match section { + CirInputSection::Fresh => &mut self.fresh_rejects, + CirInputSection::Cached => &mut self.cached_rejects, + }; + rejects.entry(rsync_uri.to_string()).or_insert(reason); + } + + /// Merge another accumulator into this one. Object digests conflict-check + /// exactly like repeated `insert_object` calls; rejects keep the first + /// reason seen, matching sequential submission order when `other` holds + /// later publication points than `self`. + pub fn merge(&mut self, other: CirInputAccumulator) -> Result<(), CirExportError> { + for (rsync_uri, digest) in other.fresh_objects { + self.insert_object_digest(CirInputSection::Fresh, &rsync_uri, digest)?; + } + for (rsync_uri, digest) in other.cached_objects { + self.insert_object_digest(CirInputSection::Cached, &rsync_uri, digest)?; + } + for (rsync_uri, reason) in other.fresh_rejects { + self.insert_reject(CirInputSection::Fresh, &rsync_uri, reason); + } + for (rsync_uri, reason) in other.cached_rejects { + self.insert_reject(CirInputSection::Cached, &rsync_uri, reason); + } + Ok(()) + } + + pub fn finalize(self) -> CirInputSnapshot { + CirInputSnapshot { + fresh_validated_objects: finalize_objects(self.fresh_objects), + cached_validated_objects: finalize_objects(self.cached_objects), + fresh_rejected_objects: finalize_rejects(self.fresh_rejects), + cached_rejected_objects: finalize_rejects(self.cached_rejects), + } + } +} + +fn finalize_objects(objects: HashMap) -> Vec { + let mut objects = objects + .into_iter() + .map(|(rsync_uri, sha256)| CirObject { + rsync_uri, + sha256: sha256.to_vec(), + }) + .collect::>(); + objects.sort_by(|a, b| a.rsync_uri.cmp(&b.rsync_uri)); + objects +} + +fn finalize_rejects(rejects: HashMap>) -> Vec { + let mut rejects = rejects + .into_iter() + .map(|(object_uri, reason)| CirRejectedObject { object_uri, reason }) + .collect::>(); + rejects.sort_by(|a, b| a.object_uri.cmp(&b.object_uri)); + rejects +} + +fn is_sha256_hex(value: &str) -> bool { + value.len() == 64 && value.as_bytes().iter().all(u8::is_ascii_hexdigit) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::audit::{AuditObjectKind, AuditObjectResult}; + + fn entry(uri: &str, hash_byte: u8, result: AuditObjectResult) -> ObjectAuditEntry { + let is_error = result == AuditObjectResult::Error; + ObjectAuditEntry { + rsync_uri: uri.to_string(), + sha256_hex: format!("{}", hex::encode([hash_byte; 32])), + kind: AuditObjectKind::Roa, + result, + detail: is_error.then(|| "invalid".to_string()), + } + } + + #[test] + fn accumulator_finalizes_sorted_fresh_and_cached_sections() { + let mut acc = CirInputAccumulator::default(); + acc.submit_audit_entries( + CirInputSection::Fresh, + &[ + entry("rsync://example.net/z.roa", 0x22, AuditObjectResult::Ok), + entry("rsync://example.net/a.roa", 0x11, AuditObjectResult::Error), + ], + ) + .unwrap(); + acc.submit_audit_entries( + CirInputSection::Cached, + &[entry( + "rsync://example.net/c.roa", + 0x33, + AuditObjectResult::Ok, + )], + ) + .unwrap(); + let snapshot = acc.finalize(); + assert_eq!( + snapshot.fresh_validated_objects[0].rsync_uri, + "rsync://example.net/a.roa" + ); + assert_eq!( + snapshot.fresh_validated_objects[1].rsync_uri, + "rsync://example.net/z.roa" + ); + assert_eq!( + snapshot.cached_validated_objects[0].rsync_uri, + "rsync://example.net/c.roa" + ); + assert_eq!( + snapshot.fresh_rejected_objects[0].object_uri, + "rsync://example.net/a.roa" + ); + assert!(snapshot.cached_rejected_objects.is_empty()); + } + + #[test] + fn accumulator_rejects_conflicting_hashes_within_same_section() { + let mut acc = CirInputAccumulator::default(); + acc.submit_audit_entries( + CirInputSection::Fresh, + &[entry( + "rsync://example.net/a.roa", + 0x11, + AuditObjectResult::Ok, + )], + ) + .unwrap(); + let err = acc + .submit_audit_entries( + CirInputSection::Fresh, + &[entry( + "rsync://example.net/a.roa", + 0x22, + AuditObjectResult::Ok, + )], + ) + .unwrap_err(); + assert!(matches!(err, CirExportError::ConflictingObjectHash { .. })); + } + + #[test] + fn accumulator_merges_finalized_snapshots() { + let snapshot = CirInputSnapshot { + fresh_validated_objects: vec![CirObject { + rsync_uri: "rsync://example.net/fresh.roa".to_string(), + sha256: vec![0x11; 32], + }], + cached_validated_objects: vec![CirObject { + rsync_uri: "rsync://example.net/cached.roa".to_string(), + sha256: vec![0x22; 32], + }], + fresh_rejected_objects: vec![CirRejectedObject { + object_uri: "rsync://example.net/fresh.roa".to_string(), + reason: Some("fresh invalid".to_string()), + }], + cached_rejected_objects: Vec::new(), + }; + let mut acc = CirInputAccumulator::default(); + acc.submit_snapshot(snapshot).unwrap(); + let merged = acc.finalize(); + assert_eq!(merged.fresh_validated_objects.len(), 1); + assert_eq!(merged.cached_validated_objects.len(), 1); + assert_eq!(merged.fresh_rejected_objects.len(), 1); + } + + #[test] + fn accumulator_merge_matches_sequential_submission() { + let mut sequential = CirInputAccumulator::default(); + sequential + .submit_audit_entries( + CirInputSection::Fresh, + &[ + entry("rsync://example.net/a.roa", 0x11, AuditObjectResult::Ok), + entry("rsync://example.net/b.roa", 0x22, AuditObjectResult::Error), + ], + ) + .unwrap(); + sequential + .submit_audit_entries( + CirInputSection::Cached, + &[entry( + "rsync://example.net/c.roa", + 0x33, + AuditObjectResult::Ok, + )], + ) + .unwrap(); + + let mut first = CirInputAccumulator::default(); + first + .submit_audit_entries( + CirInputSection::Fresh, + &[entry( + "rsync://example.net/a.roa", + 0x11, + AuditObjectResult::Ok, + )], + ) + .unwrap(); + let mut second = CirInputAccumulator::default(); + second + .submit_audit_entries( + CirInputSection::Fresh, + &[entry( + "rsync://example.net/b.roa", + 0x22, + AuditObjectResult::Error, + )], + ) + .unwrap(); + second + .submit_audit_entries( + CirInputSection::Cached, + &[entry( + "rsync://example.net/c.roa", + 0x33, + AuditObjectResult::Ok, + )], + ) + .unwrap(); + first.merge(second).unwrap(); + assert_eq!(sequential.finalize(), first.finalize()); + } + + #[test] + fn accumulator_merge_detects_conflicting_hashes_across_shards() { + let mut first = CirInputAccumulator::default(); + first + .insert_object( + CirInputSection::Fresh, + "rsync://example.net/a.roa", + &"11".repeat(32), + ) + .unwrap(); + let mut second = CirInputAccumulator::default(); + second + .insert_object( + CirInputSection::Fresh, + "rsync://example.net/a.roa", + &"22".repeat(32), + ) + .unwrap(); + let err = first.merge(second).unwrap_err(); + assert!(matches!(err, CirExportError::ConflictingObjectHash { .. })); + } + + #[test] + fn accumulator_merge_keeps_first_reject_reason() { + let mut first = CirInputAccumulator::default(); + first.insert_reject( + CirInputSection::Fresh, + "rsync://example.net/a.roa", + Some("first".to_string()), + ); + let mut second = CirInputAccumulator::default(); + second.insert_reject( + CirInputSection::Fresh, + "rsync://example.net/a.roa", + Some("second".to_string()), + ); + first.merge(second).unwrap(); + let snapshot = first.finalize(); + assert_eq!( + snapshot.fresh_rejected_objects[0].reason, + Some("first".to_string()) + ); + } +} diff --git a/crates/panda-rpki-validator/src/cir/decode.rs b/crates/panda-rpki-validator/src/cir/decode.rs new file mode 100644 index 0000000..b99e6f1 --- /dev/null +++ b/crates/panda-rpki-validator/src/cir/decode.rs @@ -0,0 +1,286 @@ +use crate::cir::model::{ + CIR_VERSION_V4, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirRejectedObject, + CirTrustAnchor, +}; +use crate::data_model::common::DerReader; +use crate::data_model::oid::{OID_SHA256, OID_SHA256_RAW}; +use der_parser::der::parse_der_oid; + +#[derive(Debug, thiserror::Error)] +pub enum CirDecodeError { + #[error("DER parse error: {0}")] + Parse(String), + + #[error("unexpected CIR version: expected {expected}, got {actual}")] + UnexpectedVersion { expected: u32, actual: u32 }, + + #[error("unexpected digest algorithm OID: expected {expected}, got {actual}")] + UnexpectedDigestAlgorithm { + expected: &'static str, + actual: String, + }, + + #[error("CIR model validation failed after decode: {0}")] + Validate(String), +} + +pub fn decode_cir(der: &[u8]) -> Result { + let mut top = DerReader::new(der); + let mut seq = top.take_sequence().map_err(CirDecodeError::Parse)?; + if !top.is_empty() { + return Err(CirDecodeError::Parse("trailing bytes after CIR".into())); + } + + let version = seq.take_uint_u64().map_err(CirDecodeError::Parse)? as u32; + if version != CIR_VERSION_V4 { + return Err(CirDecodeError::UnexpectedVersion { + expected: CIR_VERSION_V4, + actual: version, + }); + } + let hash_alg = decode_hash_alg(seq.take_tag(0x06).map_err(CirDecodeError::Parse)?)?; + let validation_time = + parse_generalized_time(seq.take_tag(0x18).map_err(CirDecodeError::Parse)?)?; + + let fresh_objects_der = seq.take_tag(0x30).map_err(CirDecodeError::Parse)?; + let mut fresh_objects_reader = DerReader::new(fresh_objects_der); + let mut fresh_validated_objects = Vec::new(); + while !fresh_objects_reader.is_empty() { + let (_tag, full, _value) = fresh_objects_reader + .take_any_full() + .map_err(CirDecodeError::Parse)?; + fresh_validated_objects.push(decode_object(full)?); + } + + let cached_objects_der = seq.take_tag(0x30).map_err(CirDecodeError::Parse)?; + let mut cached_objects_reader = DerReader::new(cached_objects_der); + let mut cached_validated_objects = Vec::new(); + while !cached_objects_reader.is_empty() { + let (_tag, full, _value) = cached_objects_reader + .take_any_full() + .map_err(CirDecodeError::Parse)?; + cached_validated_objects.push(decode_object(full)?); + } + + let trust_anchors_der = seq.take_tag(0x30).map_err(CirDecodeError::Parse)?; + let mut trust_anchors_reader = DerReader::new(trust_anchors_der); + let mut trust_anchors = Vec::new(); + while !trust_anchors_reader.is_empty() { + let (_tag, full, _value) = trust_anchors_reader + .take_any_full() + .map_err(CirDecodeError::Parse)?; + trust_anchors.push(decode_trust_anchor(full)?); + } + + let object_list_sha256 = seq + .take_octet_string() + .map_err(CirDecodeError::Parse)? + .to_vec(); + let fresh_object_list_sha256 = seq + .take_octet_string() + .map_err(CirDecodeError::Parse)? + .to_vec(); + let cached_object_list_sha256 = seq + .take_octet_string() + .map_err(CirDecodeError::Parse)? + .to_vec(); + let reject_list_sha256 = seq + .take_octet_string() + .map_err(CirDecodeError::Parse)? + .to_vec(); + let fresh_reject_list_sha256 = seq + .take_octet_string() + .map_err(CirDecodeError::Parse)? + .to_vec(); + let cached_reject_list_sha256 = seq + .take_octet_string() + .map_err(CirDecodeError::Parse)? + .to_vec(); + + let fresh_rejected_der = seq.take_tag(0x30).map_err(CirDecodeError::Parse)?; + let mut fresh_rejected_reader = DerReader::new(fresh_rejected_der); + let mut fresh_rejected_objects = Vec::new(); + while !fresh_rejected_reader.is_empty() { + let (_tag, full, _value) = fresh_rejected_reader + .take_any_full() + .map_err(CirDecodeError::Parse)?; + fresh_rejected_objects.push(decode_rejected_object(full)?); + } + + let cached_rejected_der = seq.take_tag(0x30).map_err(CirDecodeError::Parse)?; + let mut cached_rejected_reader = DerReader::new(cached_rejected_der); + let mut cached_rejected_objects = Vec::new(); + while !cached_rejected_reader.is_empty() { + let (_tag, full, _value) = cached_rejected_reader + .take_any_full() + .map_err(CirDecodeError::Parse)?; + cached_rejected_objects.push(decode_rejected_object(full)?); + } + + if !seq.is_empty() { + return Err(CirDecodeError::Parse("trailing fields in CIR".into())); + } + + let cir = CanonicalInputRepresentation { + version, + hash_alg, + validation_time, + fresh_validated_objects, + cached_validated_objects, + trust_anchors, + object_list_sha256, + fresh_object_list_sha256, + cached_object_list_sha256, + reject_list_sha256, + fresh_reject_list_sha256, + cached_reject_list_sha256, + fresh_rejected_objects, + cached_rejected_objects, + }; + cir.validate().map_err(CirDecodeError::Validate)?; + Ok(cir) +} + +fn decode_hash_alg(raw_body: &[u8]) -> Result { + if raw_body != OID_SHA256_RAW { + return Err(CirDecodeError::UnexpectedDigestAlgorithm { + expected: OID_SHA256, + actual: oid_string(raw_body)?, + }); + } + Ok(CirHashAlgorithm::Sha256) +} + +fn decode_object(der: &[u8]) -> Result { + let mut top = DerReader::new(der); + let mut seq = top.take_sequence().map_err(CirDecodeError::Parse)?; + if !top.is_empty() { + return Err(CirDecodeError::Parse( + "trailing bytes after CirObject".into(), + )); + } + let rsync_uri = std::str::from_utf8(seq.take_tag(0x16).map_err(CirDecodeError::Parse)?) + .map_err(|e| CirDecodeError::Parse(e.to_string()))? + .to_string(); + let sha256 = seq + .take_octet_string() + .map_err(CirDecodeError::Parse)? + .to_vec(); + if !seq.is_empty() { + return Err(CirDecodeError::Parse("trailing fields in CirObject".into())); + } + Ok(CirObject { rsync_uri, sha256 }) +} + +fn decode_trust_anchor(der: &[u8]) -> Result { + let mut top = DerReader::new(der); + let mut seq = top.take_sequence().map_err(CirDecodeError::Parse)?; + if !top.is_empty() { + return Err(CirDecodeError::Parse( + "trailing bytes after CirTrustAnchor".into(), + )); + } + let ta_rsync_uri = std::str::from_utf8(seq.take_tag(0x16).map_err(CirDecodeError::Parse)?) + .map_err(|e| CirDecodeError::Parse(e.to_string()))? + .to_string(); + let tal_uri = std::str::from_utf8(seq.take_tag(0x16).map_err(CirDecodeError::Parse)?) + .map_err(|e| CirDecodeError::Parse(e.to_string()))? + .to_string(); + let tal_bytes = seq + .take_octet_string() + .map_err(CirDecodeError::Parse)? + .to_vec(); + let ta_certificate_der = seq + .take_octet_string() + .map_err(CirDecodeError::Parse)? + .to_vec(); + let ta_certificate_sha256 = seq + .take_octet_string() + .map_err(CirDecodeError::Parse)? + .to_vec(); + if !seq.is_empty() { + return Err(CirDecodeError::Parse( + "trailing fields in CirTrustAnchor".into(), + )); + } + Ok(CirTrustAnchor { + ta_rsync_uri, + tal_uri, + tal_bytes, + ta_certificate_der, + ta_certificate_sha256, + }) +} + +fn decode_rejected_object(der: &[u8]) -> Result { + let mut top = DerReader::new(der); + let mut seq = top.take_sequence().map_err(CirDecodeError::Parse)?; + if !top.is_empty() { + return Err(CirDecodeError::Parse( + "trailing bytes after CirRejectedObject".into(), + )); + } + let object_uri = std::str::from_utf8(seq.take_tag(0x16).map_err(CirDecodeError::Parse)?) + .map_err(|e| CirDecodeError::Parse(e.to_string()))? + .to_string(); + let reason = if seq.is_empty() { + None + } else { + Some( + std::str::from_utf8(seq.take_octet_string().map_err(CirDecodeError::Parse)?) + .map_err(|e| CirDecodeError::Parse(e.to_string()))? + .to_string(), + ) + }; + if !seq.is_empty() { + return Err(CirDecodeError::Parse( + "trailing fields in CirRejectedObject".into(), + )); + } + Ok(CirRejectedObject { object_uri, reason }) +} + +fn oid_string(raw_body: &[u8]) -> Result { + let der = { + let mut out = Vec::with_capacity(raw_body.len() + 2); + out.push(0x06); + if raw_body.len() < 0x80 { + out.push(raw_body.len() as u8); + } else { + return Err(CirDecodeError::Parse("OID too long".into())); + } + out.extend_from_slice(raw_body); + out + }; + let (_rem, oid) = parse_der_oid(&der).map_err(|e| CirDecodeError::Parse(e.to_string()))?; + let oid = oid + .as_oid_val() + .map_err(|e| CirDecodeError::Parse(e.to_string()))?; + Ok(oid.to_string()) +} + +fn parse_generalized_time(bytes: &[u8]) -> Result { + let s = std::str::from_utf8(bytes).map_err(|e| CirDecodeError::Parse(e.to_string()))?; + if s.len() != 15 || !s.ends_with('Z') { + return Err(CirDecodeError::Parse( + "GeneralizedTime must be YYYYMMDDHHMMSSZ".into(), + )); + } + let parse = |range: std::ops::Range| -> Result { + s[range] + .parse::() + .map_err(|e| CirDecodeError::Parse(e.to_string())) + }; + let year = parse(0..4)? as i32; + let month = parse(4..6)? as u8; + let day = parse(6..8)? as u8; + let hour = parse(8..10)? as u8; + let minute = parse(10..12)? as u8; + let second = parse(12..14)? as u8; + let month = time::Month::try_from(month).map_err(|e| CirDecodeError::Parse(e.to_string()))?; + let date = time::Date::from_calendar_date(year, month, day) + .map_err(|e| CirDecodeError::Parse(e.to_string()))?; + let timev = time::Time::from_hms(hour, minute, second) + .map_err(|e| CirDecodeError::Parse(e.to_string()))?; + Ok(time::PrimitiveDateTime::new(date, timev).assume_utc()) +} diff --git a/crates/panda-rpki-validator/src/cir/encode.rs b/crates/panda-rpki-validator/src/cir/encode.rs new file mode 100644 index 0000000..12f97aa --- /dev/null +++ b/crates/panda-rpki-validator/src/cir/encode.rs @@ -0,0 +1,179 @@ +use crate::cir::model::{ + CIR_VERSION_V4, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirRejectedObject, + CirTrustAnchor, +}; +use crate::data_model::oid::OID_SHA256_RAW; + +#[derive(Debug, thiserror::Error)] +pub enum CirEncodeError { + #[error("CIR model validation failed: {0}")] + Validate(String), +} + +pub fn encode_cir(cir: &CanonicalInputRepresentation) -> Result, CirEncodeError> { + cir.validate().map_err(CirEncodeError::Validate)?; + Ok(encode_sequence(&[ + encode_integer_u32(CIR_VERSION_V4), + encode_oid(match cir.hash_alg { + CirHashAlgorithm::Sha256 => OID_SHA256_RAW, + }), + encode_generalized_time(cir.validation_time), + encode_sequence( + &cir.fresh_validated_objects + .iter() + .map(encode_object) + .collect::, _>>()?, + ), + encode_sequence( + &cir.cached_validated_objects + .iter() + .map(encode_object) + .collect::, _>>()?, + ), + encode_sequence( + &cir.trust_anchors + .iter() + .map(encode_trust_anchor) + .collect::, _>>()?, + ), + encode_octet_string(&cir.object_list_sha256), + encode_octet_string(&cir.fresh_object_list_sha256), + encode_octet_string(&cir.cached_object_list_sha256), + encode_octet_string(&cir.reject_list_sha256), + encode_octet_string(&cir.fresh_reject_list_sha256), + encode_octet_string(&cir.cached_reject_list_sha256), + encode_sequence( + &cir.fresh_rejected_objects + .iter() + .map(encode_rejected_object) + .collect::, _>>()?, + ), + encode_sequence( + &cir.cached_rejected_objects + .iter() + .map(encode_rejected_object) + .collect::, _>>()?, + ), + ])) +} + +fn encode_object(object: &CirObject) -> Result, CirEncodeError> { + object.validate().map_err(CirEncodeError::Validate)?; + Ok(encode_sequence(&[ + encode_ia5_string(object.rsync_uri.as_bytes()), + encode_octet_string(&object.sha256), + ])) +} + +fn encode_trust_anchor(trust_anchor: &CirTrustAnchor) -> Result, CirEncodeError> { + trust_anchor.validate().map_err(CirEncodeError::Validate)?; + Ok(encode_sequence(&[ + encode_ia5_string(trust_anchor.ta_rsync_uri.as_bytes()), + encode_ia5_string(trust_anchor.tal_uri.as_bytes()), + encode_octet_string(&trust_anchor.tal_bytes), + encode_octet_string(&trust_anchor.ta_certificate_der), + encode_octet_string(&trust_anchor.ta_certificate_sha256), + ])) +} + +fn encode_rejected_object(item: &CirRejectedObject) -> Result, CirEncodeError> { + item.validate().map_err(CirEncodeError::Validate)?; + let mut fields = vec![encode_ia5_string(item.object_uri.as_bytes())]; + if let Some(reason) = &item.reason { + fields.push(encode_octet_string(reason.as_bytes())); + } + Ok(encode_sequence(&fields)) +} + +fn encode_generalized_time(t: time::OffsetDateTime) -> Vec { + let t = t.to_offset(time::UtcOffset::UTC); + let s = format!( + "{:04}{:02}{:02}{:02}{:02}{:02}Z", + t.year(), + u8::from(t.month()), + t.day(), + t.hour(), + t.minute(), + t.second() + ); + encode_tlv(0x18, s.into_bytes()) +} + +fn encode_integer_u32(v: u32) -> Vec { + encode_integer_bytes(unsigned_integer_bytes(v as u64)) +} + +fn encode_integer_bytes(mut bytes: Vec) -> Vec { + if bytes.is_empty() { + bytes.push(0); + } + if bytes[0] & 0x80 != 0 { + bytes.insert(0, 0); + } + encode_tlv(0x02, bytes) +} + +fn unsigned_integer_bytes(v: u64) -> Vec { + if v == 0 { + return vec![0]; + } + let mut out = Vec::new(); + let mut n = v; + while n > 0 { + out.push((n & 0xFF) as u8); + n >>= 8; + } + out.reverse(); + out +} + +fn encode_oid(raw_body: &[u8]) -> Vec { + encode_tlv(0x06, raw_body.to_vec()) +} + +fn encode_ia5_string(bytes: &[u8]) -> Vec { + encode_tlv(0x16, bytes.to_vec()) +} + +fn encode_octet_string(bytes: &[u8]) -> Vec { + encode_tlv(0x04, bytes.to_vec()) +} + +fn encode_sequence(elements: &[Vec]) -> Vec { + let mut body = Vec::new(); + for element in elements { + body.extend_from_slice(element); + } + encode_tlv(0x30, body) +} + +fn encode_tlv(tag: u8, value: Vec) -> Vec { + let mut out = Vec::with_capacity(1 + encoded_len_len(value.len()) + value.len()); + out.push(tag); + encode_len_into(value.len(), &mut out); + out.extend_from_slice(&value); + out +} + +fn encoded_len_len(len: usize) -> usize { + if len < 0x80 { + 1 + } else { + 1 + len.to_be_bytes().iter().skip_while(|&&b| b == 0).count() + } +} + +fn encode_len_into(len: usize, out: &mut Vec) { + if len < 0x80 { + out.push(len as u8); + return; + } + let bytes = len.to_be_bytes(); + let first_non_zero = bytes + .iter() + .position(|&b| b != 0) + .unwrap_or(bytes.len() - 1); + let len_bytes = &bytes[first_non_zero..]; + out.push(0x80 | (len_bytes.len() as u8)); + out.extend_from_slice(len_bytes); +} diff --git a/crates/panda-rpki-validator/src/cir/export.rs b/crates/panda-rpki-validator/src/cir/export.rs new file mode 100644 index 0000000..e5b6d6c --- /dev/null +++ b/crates/panda-rpki-validator/src/cir/export.rs @@ -0,0 +1,1217 @@ +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::path::Path; + +use crate::audit::{AuditObjectResult, ObjectAuditEntry, PublicationPointAudit}; +use crate::cir::accumulator::CirInputSnapshot; +use crate::cir::encode::{CirEncodeError, encode_cir}; +use crate::cir::model::{ + CanonicalInputRepresentation, CirObject, CirRejectedObject, CirTrustAnchor, +}; +use crate::cir::static_pool::{ + CirStaticPoolError, CirStaticPoolExportSummary, export_hashes_from_store, +}; +use crate::current_repo_index::CurrentRepoObject; +use crate::data_model::ta::TrustAnchor; +use crate::storage::RocksStore; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CirExportTiming { + pub build_cir_ms: u64, + pub write_cir_ms: u64, + pub total_ms: u64, +} + +#[derive(Debug, thiserror::Error)] +pub enum CirExportError { + #[error("CIR TAL URI must be http(s), got: {0}")] + InvalidTalUri(String), + + #[error("TAL does not contain any rsync TA URI; CIR replay scheme A requires one")] + MissingTaRsyncUri, + + #[error("CIR model validation failed: {0}")] + Validate(String), + + #[error("CIR consumed audit has conflicting hashes for {rsync_uri}: {first} vs {second}")] + ConflictingObjectHash { + rsync_uri: String, + first: String, + second: String, + }, + + #[error("encode CIR failed: {0}")] + Encode(#[from] CirEncodeError), + + #[error("static pool export failed: {0}")] + StaticPool(#[from] CirStaticPoolError), + + #[error("write CIR file failed: {0}: {1}")] + Write(String, String), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CirRawStoreExportSummary { + pub unique_hashes: usize, + pub written_entries: usize, + pub reused_entries: usize, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CirExportSummary { + pub object_count: usize, + pub trust_anchor_count: usize, + pub timing: CirExportTiming, +} + +#[derive(Clone, Copy, Debug)] +pub struct CirTrustAnchorBinding<'a> { + pub trust_anchor: &'a TrustAnchor, + pub tal_uri: &'a str, +} + +fn is_sha256_hex(value: &str) -> bool { + value.len() == 64 && value.as_bytes().iter().all(u8::is_ascii_hexdigit) +} + +fn insert_consumed_object_hash( + objects: &mut BTreeMap, + rsync_uri: &str, + sha256_hex: &str, +) -> Result<(), CirExportError> { + if !rsync_uri.starts_with("rsync://") || !is_sha256_hex(sha256_hex) { + return Ok(()); + } + + let normalized = sha256_hex.to_ascii_lowercase(); + if let Some(existing) = objects.get(rsync_uri) { + if existing != &normalized { + return Err(CirExportError::ConflictingObjectHash { + rsync_uri: rsync_uri.to_string(), + first: existing.clone(), + second: normalized, + }); + } + return Ok(()); + } + + objects.insert(rsync_uri.to_string(), normalized); + Ok(()) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CirObjectSection { + Fresh, + Cached, +} + +fn publication_point_is_cached_fallback(pp: &PublicationPointAudit) -> bool { + pp.source == "vcir_current_instance" || pp.repo_terminal_state == "fallback_current_instance" +} + +fn publication_point_cir_entries<'a>( + pp: &'a PublicationPointAudit, + section: CirObjectSection, +) -> &'a [ObjectAuditEntry] { + match (section, publication_point_is_cached_fallback(pp)) { + (CirObjectSection::Fresh, false) => &pp.objects, + (CirObjectSection::Cached, true) => &pp.objects, + _ => &[], + } +} + +fn collect_cir_objects_from_validation_audit( + publication_points: &[PublicationPointAudit], + section: CirObjectSection, +) -> Result, CirExportError> { + let mut objects = BTreeMap::new(); + for pp in publication_points { + for obj in publication_point_cir_entries(pp, section) { + if !matches!(obj.result, AuditObjectResult::Ok | AuditObjectResult::Error) { + continue; + } + insert_consumed_object_hash(&mut objects, &obj.rsync_uri, &obj.sha256_hex)?; + } + } + Ok(objects) +} + +fn collect_rejected_objects_from_validation_audit( + publication_points: &[PublicationPointAudit], + section: CirObjectSection, +) -> Vec { + let mut rejected_objects = publication_points + .iter() + .flat_map(|pp| publication_point_cir_entries(pp, section).iter()) + .filter(|item| item.result == AuditObjectResult::Error) + .filter(|item| item.rsync_uri.starts_with("rsync://")) + .map(|item| CirRejectedObject { + object_uri: item.rsync_uri.clone(), + reason: item.detail.clone(), + }) + .collect::>(); + rejected_objects.sort_by(|a, b| a.object_uri.cmp(&b.object_uri)); + rejected_objects.dedup_by(|a, b| a.object_uri == b.object_uri); + rejected_objects +} + +fn cir_objects_from_hash_map(objects: BTreeMap) -> Vec { + objects + .into_iter() + .map(|(rsync_uri, sha256_hex)| CirObject { + rsync_uri, + sha256: hex::decode(sha256_hex).expect("validated hex"), + }) + .collect() +} + +fn canonical_ta_rsync_uri(trust_anchor: &TrustAnchor) -> Result { + if let Some(uri) = &trust_anchor.resolved_ta_uri + && uri.scheme() == "rsync" + { + return Ok(uri.as_str().to_string()); + } + trust_anchor + .tal + .ta_uris + .iter() + .filter(|uri| uri.scheme() == "rsync") + .map(|uri| uri.as_str().to_string()) + .min() + .ok_or(CirExportError::MissingTaRsyncUri) +} + +fn build_cir_trust_anchors( + tal_bindings: &[CirTrustAnchorBinding<'_>], +) -> Result, CirExportError> { + for binding in tal_bindings { + if !(binding.tal_uri.starts_with("https://") || binding.tal_uri.starts_with("http://")) { + return Err(CirExportError::InvalidTalUri(binding.tal_uri.to_string())); + } + } + + let mut trust_anchors = Vec::with_capacity(tal_bindings.len()); + for binding in tal_bindings { + let ta_rsync_uri = canonical_ta_rsync_uri(binding.trust_anchor)?; + let ta_certificate_der = binding.trust_anchor.ta_certificate.raw_der.clone(); + trust_anchors.push(CirTrustAnchor { + ta_rsync_uri, + tal_uri: binding.tal_uri.to_string(), + tal_bytes: binding.trust_anchor.tal.raw.clone(), + ta_certificate_sha256: crate::cir::model::sha256(&ta_certificate_der), + ta_certificate_der, + }); + } + trust_anchors.sort_by(|a, b| a.ta_rsync_uri.cmp(&b.ta_rsync_uri)); + Ok(trust_anchors) +} + +pub fn build_cir_from_run( + store: &RocksStore, + trust_anchor: &TrustAnchor, + tal_uri: &str, + validation_time: time::OffsetDateTime, + publication_points: &[PublicationPointAudit], +) -> Result { + build_cir_from_run_multi( + store, + &[CirTrustAnchorBinding { + trust_anchor, + tal_uri, + }], + validation_time, + publication_points, + None, + ) +} + +pub fn build_cir_from_run_multi( + _store: &RocksStore, + tal_bindings: &[CirTrustAnchorBinding<'_>], + validation_time: time::OffsetDateTime, + publication_points: &[PublicationPointAudit], + _current_repo_objects: Option<&[CurrentRepoObject]>, +) -> Result { + let fresh_objects = + collect_cir_objects_from_validation_audit(publication_points, CirObjectSection::Fresh)?; + let cached_objects = + collect_cir_objects_from_validation_audit(publication_points, CirObjectSection::Cached)?; + + let trust_anchors = build_cir_trust_anchors(tal_bindings)?; + + let fresh_rejected_objects = + collect_rejected_objects_from_validation_audit(publication_points, CirObjectSection::Fresh); + let cached_rejected_objects = collect_rejected_objects_from_validation_audit( + publication_points, + CirObjectSection::Cached, + ); + + let cir = CanonicalInputRepresentation::new_v4( + validation_time, + cir_objects_from_hash_map(fresh_objects), + cir_objects_from_hash_map(cached_objects), + trust_anchors, + fresh_rejected_objects, + cached_rejected_objects, + ); + cir.validate().map_err(CirExportError::Validate)?; + Ok(cir) +} + +pub fn build_cir_from_input_snapshot_multi( + tal_bindings: &[CirTrustAnchorBinding<'_>], + validation_time: time::OffsetDateTime, + input: CirInputSnapshot, +) -> Result { + let trust_anchors = build_cir_trust_anchors(tal_bindings)?; + Ok(CanonicalInputRepresentation::new_v4( + validation_time, + input.fresh_validated_objects, + input.cached_validated_objects, + trust_anchors, + input.fresh_rejected_objects, + input.cached_rejected_objects, + )) +} + +pub fn write_cir_file( + path: &Path, + cir: &CanonicalInputRepresentation, +) -> Result<(), CirExportError> { + let der = encode_cir(cir)?; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| CirExportError::Write(path.display().to_string(), e.to_string()))?; + } + std::fs::write(path, der) + .map_err(|e| CirExportError::Write(path.display().to_string(), e.to_string())) +} + +pub fn export_cir_static_pool( + store: &RocksStore, + static_root: &Path, + capture_date_utc: time::Date, + cir: &CanonicalInputRepresentation, + trust_anchors: &[&TrustAnchor], +) -> Result { + let _ = trust_anchors; + let hashes = cir + .validated_objects() + .map(|item| hex::encode(&item.sha256)) + .collect::>(); + export_hashes_from_store(store, static_root, capture_date_utc, &hashes).map_err(Into::into) +} + +pub fn export_cir_raw_store( + store: &RocksStore, + raw_store_path: &Path, + cir: &CanonicalInputRepresentation, + trust_anchors: &[&TrustAnchor], +) -> Result { + let _ = trust_anchors; + let unique: BTreeSet = cir + .validated_objects() + .map(|item| hex::encode(&item.sha256)) + .collect(); + + let written_entries = 0usize; + let mut reused_entries = 0usize; + for sha256_hex in &unique { + if store + .get_blob_bytes(sha256_hex) + .map_err(|e| { + CirExportError::Write(raw_store_path.display().to_string(), e.to_string()) + })? + .is_some() + { + reused_entries += 1; + continue; + } + return Err(CirExportError::Write( + raw_store_path.display().to_string(), + format!("raw store missing object for sha256={sha256_hex}"), + )); + } + + Ok(CirRawStoreExportSummary { + unique_hashes: unique.len(), + written_entries, + reused_entries, + }) +} + +pub fn export_cir_from_run( + store: &RocksStore, + trust_anchor: &TrustAnchor, + tal_uri: &str, + validation_time: time::OffsetDateTime, + publication_points: &[PublicationPointAudit], + cir_out: &Path, + capture_date_utc: time::Date, +) -> Result { + export_cir_from_run_multi( + store, + &[CirTrustAnchorBinding { + trust_anchor, + tal_uri, + }], + validation_time, + publication_points, + cir_out, + capture_date_utc, + None, + ) +} + +pub fn export_cir_from_run_multi( + store: &RocksStore, + tal_bindings: &[CirTrustAnchorBinding<'_>], + validation_time: time::OffsetDateTime, + publication_points: &[PublicationPointAudit], + cir_out: &Path, + capture_date_utc: time::Date, + current_repo_objects: Option<&[CurrentRepoObject]>, +) -> Result { + let _ = capture_date_utc; + let total_started = std::time::Instant::now(); + + let started = std::time::Instant::now(); + let cir = build_cir_from_run_multi( + store, + tal_bindings, + validation_time, + publication_points, + current_repo_objects, + )?; + let build_cir_ms = started.elapsed().as_millis() as u64; + + let _ = store; + + let started = std::time::Instant::now(); + write_cir_file(cir_out, &cir)?; + let write_cir_ms = started.elapsed().as_millis() as u64; + + Ok(CirExportSummary { + object_count: cir.validated_object_count(), + trust_anchor_count: cir.trust_anchors.len(), + timing: CirExportTiming { + build_cir_ms, + write_cir_ms, + total_ms: total_started.elapsed().as_millis() as u64, + }, + }) +} + +pub fn export_cir_from_input_snapshot_multi( + tal_bindings: &[CirTrustAnchorBinding<'_>], + validation_time: time::OffsetDateTime, + input: CirInputSnapshot, + cir_out: &Path, +) -> Result { + let total_started = std::time::Instant::now(); + + let started = std::time::Instant::now(); + let cir = build_cir_from_input_snapshot_multi(tal_bindings, validation_time, input)?; + let build_cir_ms = started.elapsed().as_millis() as u64; + + let started = std::time::Instant::now(); + write_cir_file(cir_out, &cir)?; + let write_cir_ms = started.elapsed().as_millis() as u64; + + Ok(CirExportSummary { + object_count: cir.validated_object_count(), + trust_anchor_count: cir.trust_anchors.len(), + timing: CirExportTiming { + build_cir_ms, + write_cir_ms, + total_ms: total_started.elapsed().as_millis() as u64, + }, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cir::decode::decode_cir; + use crate::data_model::ta::TrustAnchor; + use crate::data_model::tal::Tal; + use crate::storage::{RawByHashEntry, RocksStore}; + + fn sample_time() -> time::OffsetDateTime { + time::OffsetDateTime::parse( + "2026-04-07T12:34:56Z", + &time::format_description::well_known::Rfc3339, + ) + .unwrap() + } + + fn sample_date() -> time::Date { + time::Date::from_calendar_date(2026, time::Month::April, 7).unwrap() + } + + fn sample_trust_anchor() -> TrustAnchor { + let base = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let tal_bytes = + std::fs::read(base.join("tests/fixtures/tal/apnic-rfc7730-https.tal")).unwrap(); + let ta_der = std::fs::read(base.join("tests/fixtures/ta/apnic-ta.cer")).unwrap(); + let tal = Tal::decode_bytes(&tal_bytes).unwrap(); + TrustAnchor::bind_der(tal, &ta_der, None).unwrap() + } + + fn sample_arin_trust_anchor() -> TrustAnchor { + let base = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let tal_bytes = std::fs::read(base.join("tests/fixtures/tal/arin.tal")).unwrap(); + let ta_der = std::fs::read(base.join("tests/fixtures/ta/arin-ta.cer")).unwrap(); + let tal = Tal::decode_bytes(&tal_bytes).unwrap(); + TrustAnchor::bind_der(tal, &ta_der, None).unwrap() + } + + fn sample_trust_anchor_without_rsync_uri() -> TrustAnchor { + let mut ta = sample_trust_anchor(); + ta.tal.ta_uris.retain(|uri| uri.scheme() != "rsync"); + ta + } + + fn sha256_hex(bytes: &[u8]) -> String { + use sha2::{Digest, Sha256}; + hex::encode(Sha256::digest(bytes)) + } + + fn audit_entry( + uri: &str, + hash: &str, + kind: crate::audit::AuditObjectKind, + result: crate::audit::AuditObjectResult, + detail: Option<&str>, + ) -> crate::audit::ObjectAuditEntry { + crate::audit::ObjectAuditEntry { + rsync_uri: uri.to_string(), + sha256_hex: hash.to_string(), + kind, + result, + detail: detail.map(ToString::to_string), + } + } + + #[test] + fn build_cir_from_run_collects_consumed_audit_objects_and_tal() { + let td = tempfile::tempdir().unwrap(); + let store = RocksStore::open(td.path()).unwrap(); + let bytes = b"object-a".to_vec(); + let hash = sha256_hex(&bytes); + let publication_points = vec![PublicationPointAudit { + objects: vec![audit_entry( + "rsync://example.test/repo/a.cer", + &hash, + crate::audit::AuditObjectKind::Certificate, + crate::audit::AuditObjectResult::Ok, + None, + )], + ..PublicationPointAudit::default() + }]; + + let ta = sample_trust_anchor(); + let cir = build_cir_from_run( + &store, + &ta, + "https://example.test/root.tal", + sample_time(), + &publication_points, + ) + .expect("build cir"); + assert_eq!(cir.version, crate::cir::model::CIR_VERSION_V4); + assert_eq!(cir.trust_anchors.len(), 1); + assert_eq!( + cir.trust_anchors[0].tal_uri, + "https://example.test/root.tal" + ); + assert!( + cir.fresh_validated_objects + .iter() + .any(|item| item.rsync_uri == "rsync://example.test/repo/a.cer") + ); + assert!( + !cir.fresh_validated_objects + .iter() + .any(|item| item.rsync_uri == cir.trust_anchors[0].ta_rsync_uri) + ); + assert!(!cir.trust_anchors[0].ta_certificate_der.is_empty()); + } + + #[test] + fn export_cir_from_run_writes_der_and_static_pool() { + let td = tempfile::tempdir().unwrap(); + let store_dir = td.path().join("db"); + let out_dir = td.path().join("out"); + let _static_root = td.path().join("static"); + let store = RocksStore::open(&store_dir).unwrap(); + + let bytes = b"object-b".to_vec(); + let hash = sha256_hex(&bytes); + let mut raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone()); + raw.origin_uris + .push("rsync://example.test/repo/b.roa".into()); + store.put_raw_by_hash_entry(&raw).unwrap(); + let publication_points = vec![PublicationPointAudit { + objects: vec![audit_entry( + "rsync://example.test/repo/b.roa", + &hash, + crate::audit::AuditObjectKind::Roa, + crate::audit::AuditObjectResult::Ok, + None, + )], + ..PublicationPointAudit::default() + }]; + + let ta = sample_trust_anchor(); + let cir_path = out_dir.join("example.cir"); + let summary = export_cir_from_run( + &store, + &ta, + "https://example.test/root.tal", + sample_time(), + &publication_points, + &cir_path, + sample_date(), + ) + .expect("export cir"); + assert_eq!(summary.trust_anchor_count, 1); + assert_eq!(summary.object_count, 1); + assert!(summary.timing.total_ms >= summary.timing.build_cir_ms); + + let der = std::fs::read(&cir_path).unwrap(); + let cir = decode_cir(&der).unwrap(); + assert_eq!( + cir.trust_anchors[0].tal_uri, + "https://example.test/root.tal" + ); + } + + #[test] + fn export_cir_from_run_uses_raw_store_backend_without_pool_export() { + let td = tempfile::tempdir().unwrap(); + let store_dir = td.path().join("db"); + let raw_store = td.path().join("raw-store.db"); + let out_dir = td.path().join("out"); + let store = RocksStore::open_with_external_raw_store(&store_dir, &raw_store).unwrap(); + + let bytes = b"object-d".to_vec(); + let hash = sha256_hex(&bytes); + let mut raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone()); + raw.origin_uris + .push("rsync://example.test/repo/d.roa".into()); + store.put_raw_by_hash_entry(&raw).unwrap(); + let publication_points = vec![PublicationPointAudit { + objects: vec![audit_entry( + "rsync://example.test/repo/d.roa", + &hash, + crate::audit::AuditObjectKind::Roa, + crate::audit::AuditObjectResult::Ok, + None, + )], + ..PublicationPointAudit::default() + }]; + + let ta = sample_trust_anchor(); + let cir_path = out_dir.join("example.cir"); + let summary = export_cir_from_run( + &store, + &ta, + "https://example.test/root.tal", + sample_time(), + &publication_points, + &cir_path, + sample_date(), + ) + .expect("export cir"); + assert_eq!(summary.object_count, 1); + assert!(raw_store.exists()); + assert!(cir_path.exists()); + } + + #[test] + fn export_cir_from_run_does_not_write_ta_bytes_to_repo_bytes_store() { + let td = tempfile::tempdir().unwrap(); + let store = RocksStore::open_with_external_repo_bytes( + &td.path().join("db"), + &td.path().join("repo-bytes.db"), + ) + .unwrap(); + + let ta = sample_trust_anchor(); + let ta_hash = sha256_hex(&ta.ta_certificate.raw_der); + let cir_path = td.path().join("out").join("example.cir"); + export_cir_from_run( + &store, + &ta, + "https://example.test/root.tal", + sample_time(), + &[], + &cir_path, + sample_date(), + ) + .expect("export cir"); + + assert_eq!(store.get_blob_bytes(&ta_hash).unwrap(), None); + } + + #[test] + fn build_cir_from_run_includes_consumed_vcir_current_instance_objects_from_audit() { + let td = tempfile::tempdir().unwrap(); + let store = RocksStore::open(td.path()).unwrap(); + let ta = sample_trust_anchor(); + + let mut pp = PublicationPointAudit { + source: "vcir_current_instance".to_string(), + ..PublicationPointAudit::default() + }; + pp.objects.push(crate::audit::ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/fallback.mft".to_string(), + sha256_hex: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + .to_string(), + kind: crate::audit::AuditObjectKind::Manifest, + result: crate::audit::AuditObjectResult::Ok, + detail: None, + }); + pp.objects.push(crate::audit::ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/fallback.roa".to_string(), + sha256_hex: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + .to_string(), + kind: crate::audit::AuditObjectKind::Roa, + result: crate::audit::AuditObjectResult::Ok, + detail: None, + }); + + let cir = build_cir_from_run( + &store, + &ta, + "https://example.test/root.tal", + sample_time(), + &[pp], + ) + .expect("build cir"); + + assert!( + cir.cached_validated_objects + .iter() + .any(|item| item.rsync_uri == "rsync://example.test/repo/fallback.mft") + ); + assert!( + cir.cached_validated_objects + .iter() + .any(|item| item.rsync_uri == "rsync://example.test/repo/fallback.roa") + ); + } + + #[test] + fn build_cir_from_run_multi_ignores_current_repo_superfluous_objects() { + let td = tempfile::tempdir().unwrap(); + let store = RocksStore::open(td.path()).unwrap(); + let ta1 = sample_trust_anchor(); + let ta2 = sample_arin_trust_anchor(); + let current_repo_objects = vec![crate::current_repo_index::CurrentRepoObject { + rsync_uri: "rsync://example.test/repo/superfluous.roa".to_string(), + current_hash_hex: "11".repeat(32), + repository_source: "https://rrdp.example.test/notification.xml".to_string(), + object_type: Some("roa".to_string()), + }]; + let publication_points = vec![PublicationPointAudit { + objects: vec![audit_entry( + "rsync://example.test/repo/consumed.roa", + &"22".repeat(32), + crate::audit::AuditObjectKind::Roa, + crate::audit::AuditObjectResult::Ok, + None, + )], + ..PublicationPointAudit::default() + }]; + + let cir = build_cir_from_run_multi( + &store, + &[ + CirTrustAnchorBinding { + trust_anchor: &ta1, + tal_uri: "https://example.test/apnic.tal", + }, + CirTrustAnchorBinding { + trust_anchor: &ta2, + tal_uri: "https://example.test/arin.tal", + }, + ], + sample_time(), + &publication_points, + Some(¤t_repo_objects), + ) + .expect("build cir from consumed audit objects"); + + assert_eq!(cir.trust_anchors.len(), 2); + assert!( + cir.fresh_validated_objects + .iter() + .any(|item| item.rsync_uri == "rsync://example.test/repo/consumed.roa") + ); + assert!( + !cir.fresh_validated_objects + .iter() + .any(|item| item.rsync_uri == "rsync://example.test/repo/superfluous.roa"), + "current repo objects must not be included unless validation consumed them", + ); + for trust_anchor in &cir.trust_anchors { + assert!( + !cir.fresh_validated_objects + .iter() + .any(|item| item.rsync_uri == trust_anchor.ta_rsync_uri), + "trust anchor rsync objects must not be included in CIR.objects", + ); + } + } + + #[test] + fn build_cir_from_run_multi_sorts_trust_anchors_by_ta_rsync_uri() { + let td = tempfile::tempdir().unwrap(); + let store = RocksStore::open(td.path()).unwrap(); + let apnic = sample_trust_anchor(); + let arin = sample_arin_trust_anchor(); + + let cir = build_cir_from_run_multi( + &store, + &[ + CirTrustAnchorBinding { + trust_anchor: &apnic, + tal_uri: "https://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer", + }, + CirTrustAnchorBinding { + trust_anchor: &arin, + tal_uri: "https://rrdp.arin.net/arin-rpki-ta.cer", + }, + ], + sample_time(), + &[], + Some(&[]), + ) + .expect("build cir with unsorted input bindings"); + + assert_eq!( + cir.trust_anchors + .iter() + .map(|trust_anchor| trust_anchor.ta_rsync_uri.as_str()) + .collect::>(), + vec![ + "rsync://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer", + "rsync://rpki.arin.net/repository/arin-rpki-ta.cer", + ] + ); + } + + #[test] + fn build_cir_from_run_multi_exports_rejected_objects_from_error_audit_only() { + let td = tempfile::tempdir().unwrap(); + let store = RocksStore::open(td.path()).unwrap(); + let ta = sample_trust_anchor(); + let publication_points = vec![PublicationPointAudit { + objects: vec![ + crate::audit::ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/a.roa".to_string(), + sha256_hex: "11".repeat(32), + kind: crate::audit::AuditObjectKind::Roa, + result: crate::audit::AuditObjectResult::Error, + detail: Some("invalid roa".to_string()), + }, + crate::audit::ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/b.asa".to_string(), + sha256_hex: "22".repeat(32), + kind: crate::audit::AuditObjectKind::Aspa, + result: crate::audit::AuditObjectResult::Skipped, + detail: Some("skipped".to_string()), + }, + crate::audit::ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/c.roa".to_string(), + sha256_hex: "33".repeat(32), + kind: crate::audit::AuditObjectKind::Roa, + result: crate::audit::AuditObjectResult::Error, + detail: Some("second rejected roa".to_string()), + }, + ], + ..PublicationPointAudit::default() + }]; + + let cir = build_cir_from_run_multi( + &store, + &[CirTrustAnchorBinding { + trust_anchor: &ta, + tal_uri: "https://example.test/root.tal", + }], + sample_time(), + &publication_points, + None, + ) + .expect("build cir"); + + assert_eq!(cir.rejected_object_count(), 2); + assert_eq!( + cir.fresh_rejected_objects[0].object_uri, + "rsync://example.test/repo/a.roa" + ); + assert_eq!( + cir.fresh_rejected_objects[0].reason.as_deref(), + Some("invalid roa") + ); + assert_eq!( + cir.fresh_rejected_objects[1].object_uri, + "rsync://example.test/repo/c.roa" + ); + assert!( + cir.fresh_validated_objects + .iter() + .any(|item| item.rsync_uri == "rsync://example.test/repo/a.roa"), + "rejected audit objects were still consumed as validation input", + ); + assert!( + cir.fresh_validated_objects + .iter() + .any(|item| item.rsync_uri == "rsync://example.test/repo/c.roa"), + "rejected audit objects were still consumed as validation input", + ); + assert!( + !cir.fresh_validated_objects + .iter() + .any(|item| item.rsync_uri == "rsync://example.test/repo/b.asa"), + "skipped audit objects are not considered consumed input", + ); + } + + #[test] + fn build_cir_from_run_multi_records_crl_expired_manifest_in_objects_and_rejects() { + let td = tempfile::tempdir().unwrap(); + let store = RocksStore::open(td.path()).unwrap(); + let ta = sample_trust_anchor(); + let manifest_uri = "rsync://example.test/repo/expired-crl.mft"; + let reject_reason = "manifest embedded EE certificate path validation failed: CRL not valid at validation_time (RFC 5280 §6.3.3(g); RFC 5280 §5.1.2.4-§5.1.2.5; RFC 6487 §5)"; + let publication_points = vec![PublicationPointAudit { + objects: vec![crate::audit::ObjectAuditEntry { + rsync_uri: manifest_uri.to_string(), + sha256_hex: "44".repeat(32), + kind: crate::audit::AuditObjectKind::Manifest, + result: crate::audit::AuditObjectResult::Error, + detail: Some(reject_reason.to_string()), + }], + ..PublicationPointAudit::default() + }]; + + let cir = build_cir_from_run_multi( + &store, + &[CirTrustAnchorBinding { + trust_anchor: &ta, + tal_uri: "https://example.test/root.tal", + }], + sample_time(), + &publication_points, + None, + ) + .expect("build cir"); + + assert!( + cir.fresh_validated_objects + .iter() + .any(|item| item.rsync_uri == manifest_uri), + "manifest rejected because issuer CRL is expired was still read as validation input", + ); + assert_eq!(cir.rejected_object_count(), 1); + assert_eq!(cir.fresh_rejected_objects[0].object_uri, manifest_uri); + assert_eq!( + cir.fresh_rejected_objects[0].reason.as_deref(), + Some(reject_reason) + ); + assert_eq!( + cir.reject_list_sha256, + crate::cir::model::compute_reject_list_sha256([manifest_uri].into_iter()) + ); + } + + #[test] + fn build_cir_from_run_multi_excludes_manifest_locked_files_when_manifest_is_rejected() { + let td = tempfile::tempdir().unwrap(); + let store = RocksStore::open(td.path()).unwrap(); + let ta = sample_trust_anchor(); + let manifest_uri = "rsync://example.test/repo/rejected.mft"; + let roa_uri = "rsync://example.test/repo/listed.roa"; + let crl_uri = "rsync://example.test/repo/listed.crl"; + let publication_points = vec![PublicationPointAudit { + objects: vec![ + crate::audit::ObjectAuditEntry { + rsync_uri: manifest_uri.to_string(), + sha256_hex: "44".repeat(32), + kind: crate::audit::AuditObjectKind::Manifest, + result: crate::audit::AuditObjectResult::Error, + detail: Some("manifest EE cert path rejected".to_string()), + }, + crate::audit::ObjectAuditEntry { + rsync_uri: roa_uri.to_string(), + sha256_hex: "55".repeat(32), + kind: crate::audit::AuditObjectKind::Roa, + result: crate::audit::AuditObjectResult::Skipped, + detail: Some("manifest rejected before locked object validation".to_string()), + }, + crate::audit::ObjectAuditEntry { + rsync_uri: crl_uri.to_string(), + sha256_hex: "66".repeat(32), + kind: crate::audit::AuditObjectKind::Crl, + result: crate::audit::AuditObjectResult::Skipped, + detail: Some("manifest rejected before locked object validation".to_string()), + }, + ], + ..PublicationPointAudit::default() + }]; + + let cir = build_cir_from_run_multi( + &store, + &[CirTrustAnchorBinding { + trust_anchor: &ta, + tal_uri: "https://example.test/root.tal", + }], + sample_time(), + &publication_points, + None, + ) + .expect("build cir"); + + assert!( + cir.fresh_validated_objects + .iter() + .any(|item| item.rsync_uri == manifest_uri), + "rejected manifest is still a current-run validation input", + ); + assert!( + !cir.fresh_validated_objects + .iter() + .any(|item| item.rsync_uri == roa_uri), + "ROA listed by a rejected manifest must not enter CIR objects", + ); + assert!( + !cir.fresh_validated_objects + .iter() + .any(|item| item.rsync_uri == crl_uri), + "CRL listed by a rejected manifest must not enter CIR objects", + ); + assert_eq!(cir.rejected_object_count(), 1); + assert_eq!(cir.fresh_rejected_objects[0].object_uri, manifest_uri); + } + + #[test] + fn build_cir_from_run_multi_reject_digest_ignores_reason_text() { + let td = tempfile::tempdir().unwrap(); + let store = RocksStore::open(td.path()).unwrap(); + let ta = sample_trust_anchor(); + + let mk_pp = |detail: &str| PublicationPointAudit { + objects: vec![crate::audit::ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/a.roa".to_string(), + sha256_hex: "11".repeat(32), + kind: crate::audit::AuditObjectKind::Roa, + result: crate::audit::AuditObjectResult::Error, + detail: Some(detail.to_string()), + }], + ..PublicationPointAudit::default() + }; + + let cir_a = build_cir_from_run_multi( + &store, + &[CirTrustAnchorBinding { + trust_anchor: &ta, + tal_uri: "https://example.test/root.tal", + }], + sample_time(), + &[mk_pp("reason-a")], + None, + ) + .expect("build cir a"); + let cir_b = build_cir_from_run_multi( + &store, + &[CirTrustAnchorBinding { + trust_anchor: &ta, + tal_uri: "https://example.test/root.tal", + }], + sample_time(), + &[mk_pp("reason-b")], + None, + ) + .expect("build cir b"); + + assert_eq!(cir_a.reject_list_sha256, cir_b.reject_list_sha256); + assert_ne!( + cir_a.fresh_rejected_objects[0].reason, + cir_b.fresh_rejected_objects[0].reason + ); + } + + #[test] + fn build_cir_from_run_multi_rejects_invalid_tal_uri_and_missing_rsync_ta_uri() { + let td = tempfile::tempdir().unwrap(); + let store = RocksStore::open(td.path()).unwrap(); + + let err = build_cir_from_run_multi( + &store, + &[CirTrustAnchorBinding { + trust_anchor: &sample_trust_anchor(), + tal_uri: "file:///not-supported.tal", + }], + sample_time(), + &[], + None, + ) + .expect_err("non-http tal uri must fail"); + assert!(matches!(err, CirExportError::InvalidTalUri(_)), "{err}"); + + let err = build_cir_from_run_multi( + &store, + &[CirTrustAnchorBinding { + trust_anchor: &sample_trust_anchor_without_rsync_uri(), + tal_uri: "https://example.test/root.tal", + }], + sample_time(), + &[], + None, + ) + .expect_err("missing rsync ta uri must fail"); + assert!(matches!(err, CirExportError::MissingTaRsyncUri), "{err}"); + } + + #[test] + fn build_cir_from_input_snapshot_multi_preserves_online_sections() { + let ta = sample_trust_anchor(); + let cir = build_cir_from_input_snapshot_multi( + &[CirTrustAnchorBinding { + trust_anchor: &ta, + tal_uri: "https://example.test/root.tal", + }], + sample_time(), + CirInputSnapshot { + fresh_validated_objects: vec![CirObject { + rsync_uri: "rsync://example.test/repo/fresh.roa".to_string(), + sha256: vec![0x11; 32], + }], + cached_validated_objects: vec![CirObject { + rsync_uri: "rsync://example.test/repo/cached.roa".to_string(), + sha256: vec![0x22; 32], + }], + fresh_rejected_objects: vec![CirRejectedObject { + object_uri: "rsync://example.test/repo/fresh.roa".to_string(), + reason: Some("fresh rejected".to_string()), + }], + cached_rejected_objects: vec![CirRejectedObject { + object_uri: "rsync://example.test/repo/cached.roa".to_string(), + reason: None, + }], + }, + ) + .expect("build CIR from online input"); + + cir.validate().expect("online CIR snapshot is valid"); + assert_eq!(cir.fresh_validated_objects.len(), 1); + assert_eq!(cir.cached_validated_objects.len(), 1); + assert_eq!(cir.fresh_rejected_objects.len(), 1); + assert_eq!(cir.cached_rejected_objects.len(), 1); + } + + #[test] + fn export_cir_static_pool_writes_repository_objects_only() { + let td = tempfile::tempdir().unwrap(); + let store = RocksStore::open(&td.path().join("db")).unwrap(); + let static_root = td.path().join("static"); + let ta1 = sample_trust_anchor(); + let ta2 = sample_arin_trust_anchor(); + + let object_bytes = b"object-z".to_vec(); + let hash = sha256_hex(&object_bytes); + let mut raw = RawByHashEntry::from_bytes(hash.clone(), object_bytes.clone()); + raw.origin_uris + .push("rsync://example.test/repo/z.roa".into()); + store.put_raw_by_hash_entry(&raw).unwrap(); + let publication_points = vec![PublicationPointAudit { + objects: vec![audit_entry( + "rsync://example.test/repo/z.roa", + &hash, + crate::audit::AuditObjectKind::Roa, + crate::audit::AuditObjectResult::Ok, + None, + )], + ..PublicationPointAudit::default() + }]; + + let cir = build_cir_from_run_multi( + &store, + &[ + CirTrustAnchorBinding { + trust_anchor: &ta1, + tal_uri: "https://example.test/apnic.tal", + }, + CirTrustAnchorBinding { + trust_anchor: &ta2, + tal_uri: "https://example.test/arin.tal", + }, + ], + sample_time(), + &publication_points, + None, + ) + .expect("build cir"); + + let summary = + export_cir_static_pool(&store, &static_root, sample_date(), &cir, &[&ta1, &ta2]) + .expect("export static pool"); + assert_eq!(summary.unique_hashes, 1); + assert_eq!(summary.written_files, 1); + for trust_anchor in &cir.trust_anchors { + let ta_hash = hex::encode(&trust_anchor.ta_certificate_sha256); + assert!( + !crate::cir::static_pool::static_pool_path(&static_root, sample_date(), &ta_hash) + .expect("static pool ta path") + .exists() + ); + } + } + + #[test] + fn export_cir_raw_store_reports_missing_non_ta_object_only() { + let td = tempfile::tempdir().unwrap(); + let raw_store_path = td.path().join("raw-store.db"); + let store = + RocksStore::open_with_external_raw_store(&td.path().join("db"), &raw_store_path) + .unwrap(); + let ta1 = sample_trust_anchor(); + let ta2 = sample_arin_trust_anchor(); + + let cir_only_tas = build_cir_from_run_multi( + &store, + &[ + CirTrustAnchorBinding { + trust_anchor: &ta1, + tal_uri: "https://example.test/apnic.tal", + }, + CirTrustAnchorBinding { + trust_anchor: &ta2, + tal_uri: "https://example.test/arin.tal", + }, + ], + sample_time(), + &[], + Some(&[]), + ) + .expect("build cir with tas only"); + + let summary = export_cir_raw_store(&store, &raw_store_path, &cir_only_tas, &[&ta1, &ta2]) + .expect("export raw store"); + assert_eq!(summary.unique_hashes, 0); + assert_eq!(summary.written_entries, 0); + assert_eq!(summary.reused_entries, 0); + + let mut cir_missing_object = cir_only_tas.clone(); + cir_missing_object.fresh_validated_objects.push(CirObject { + rsync_uri: "rsync://example.test/repo/missing.roa".to_string(), + sha256: vec![0x44; 32], + }); + let err = export_cir_raw_store(&store, &raw_store_path, &cir_missing_object, &[&ta1, &ta2]) + .expect_err("missing non-ta object must fail"); + assert!(matches!(err, CirExportError::Write(_, _)), "{err}"); + } +} diff --git a/crates/panda-rpki-validator/src/cir/materialize.rs b/crates/panda-rpki-validator/src/cir/materialize.rs new file mode 100644 index 0000000..f637eed --- /dev/null +++ b/crates/panda-rpki-validator/src/cir/materialize.rs @@ -0,0 +1,867 @@ +use std::collections::BTreeSet; +use std::fs; +use std::path::{Path, PathBuf}; + +use crate::blob_store::{ExternalRawStoreDb, ExternalRepoBytesDb, RawObjectStore}; +use crate::cir::model::CanonicalInputRepresentation; + +#[derive(Debug, thiserror::Error)] +pub enum CirMaterializeError { + #[error("invalid rsync URI: {0}")] + InvalidRsyncUri(String), + + #[error("rsync URI must reference a file object, got directory-like URI: {0}")] + DirectoryLikeRsyncUri(String), + + #[error("create mirror root failed: {path}: {detail}")] + CreateMirrorRoot { path: String, detail: String }, + + #[error("remove mirror root failed: {path}: {detail}")] + RemoveMirrorRoot { path: String, detail: String }, + + #[error("create parent directory failed: {path}: {detail}")] + CreateParent { path: String, detail: String }, + + #[error("remove existing target failed: {path}: {detail}")] + RemoveExistingTarget { path: String, detail: String }, + + #[error("static object not found for sha256={sha256_hex}")] + MissingStaticObject { sha256_hex: String }, + + #[error("link target failed: {src} -> {dst}: {detail}")] + Link { + src: String, + dst: String, + detail: String, + }, + + #[error("copy target failed: {src} -> {dst}: {detail}")] + Copy { + src: String, + dst: String, + detail: String, + }, + + #[error("mirror tree mismatch after materialize: {0}")] + TreeMismatch(String), + + #[error("open raw store failed: {path}: {detail}")] + OpenRawStore { path: String, detail: String }, + + #[error("raw object not found for sha256={sha256_hex}")] + MissingRawStoreObject { sha256_hex: String }, + + #[error("read raw store failed for sha256={sha256_hex}: {detail}")] + ReadRawStore { sha256_hex: String, detail: String }, + + #[error("open repo bytes store failed: {path}: {detail}")] + OpenRepoBytesStore { path: String, detail: String }, + + #[error("repo bytes object not found for sha256={sha256_hex}")] + MissingRepoBytesObject { sha256_hex: String }, + + #[error("read repo bytes store failed for sha256={sha256_hex}: {detail}")] + ReadRepoBytesStore { sha256_hex: String, detail: String }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CirMaterializeSummary { + pub object_count: usize, + pub trust_anchor_count: usize, + pub materialized_file_count: usize, + pub linked_files: usize, + pub copied_files: usize, +} + +pub fn materialize_cir( + cir: &CanonicalInputRepresentation, + static_root: &Path, + mirror_root: &Path, + clean_rebuild: bool, +) -> Result { + cir.validate().map_err(CirMaterializeError::TreeMismatch)?; + + prepare_mirror_root(mirror_root, clean_rebuild)?; + + let mut linked_files = 0usize; + let mut copied_files = 0usize; + + for object in cir.validated_objects() { + let sha256_hex = hex::encode(&object.sha256); + let source = resolve_static_pool_file(static_root, &sha256_hex)?; + let relative = mirror_relative_path_for_rsync_uri(&object.rsync_uri)?; + let target = mirror_root.join(&relative); + + if let Some(parent) = target.parent() { + fs::create_dir_all(parent).map_err(|e| CirMaterializeError::CreateParent { + path: parent.display().to_string(), + detail: e.to_string(), + })?; + } + + if target.exists() { + fs::remove_file(&target).map_err(|e| CirMaterializeError::RemoveExistingTarget { + path: target.display().to_string(), + detail: e.to_string(), + })?; + } + + match fs::hard_link(&source, &target) { + Ok(()) => linked_files += 1, + Err(link_err) => { + fs::copy(&source, &target).map_err(|copy_err| CirMaterializeError::Copy { + src: source.display().to_string(), + dst: target.display().to_string(), + detail: format!("{copy_err}; original link error: {link_err}"), + })?; + copied_files += 1; + } + } + } + + for trust_anchor in &cir.trust_anchors { + write_bytes_to_mirror_uri( + mirror_root, + &trust_anchor.ta_rsync_uri, + &trust_anchor.ta_certificate_der, + "cir trust anchor", + )?; + copied_files += 1; + } + + let actual = collect_materialized_uris(mirror_root)?; + let expected = expected_materialized_uris(cir); + if actual != expected { + return Err(CirMaterializeError::TreeMismatch(format!( + "expected {} files, got {} files", + expected.len(), + actual.len() + ))); + } + + Ok(CirMaterializeSummary { + object_count: cir.validated_object_count(), + trust_anchor_count: cir.trust_anchors.len(), + materialized_file_count: expected.len(), + linked_files, + copied_files, + }) +} + +pub fn materialize_cir_from_raw_store( + cir: &CanonicalInputRepresentation, + raw_store_db: &Path, + mirror_root: &Path, + clean_rebuild: bool, +) -> Result { + cir.validate().map_err(CirMaterializeError::TreeMismatch)?; + + prepare_mirror_root(mirror_root, clean_rebuild)?; + + let raw_store = + ExternalRawStoreDb::open(raw_store_db).map_err(|e| CirMaterializeError::OpenRawStore { + path: raw_store_db.display().to_string(), + detail: e.to_string(), + })?; + + let mut copied_files = 0usize; + for object in cir.validated_objects() { + let sha256_hex = hex::encode(&object.sha256); + let bytes = raw_store + .get_blob_bytes(&sha256_hex) + .map_err(|e| CirMaterializeError::ReadRawStore { + sha256_hex: sha256_hex.clone(), + detail: e.to_string(), + })? + .ok_or_else(|| CirMaterializeError::MissingRawStoreObject { + sha256_hex: sha256_hex.clone(), + })?; + write_bytes_to_mirror_uri( + mirror_root, + &object.rsync_uri, + &bytes, + &raw_store_db.display().to_string(), + )?; + copied_files += 1; + } + + for trust_anchor in &cir.trust_anchors { + write_bytes_to_mirror_uri( + mirror_root, + &trust_anchor.ta_rsync_uri, + &trust_anchor.ta_certificate_der, + "cir trust anchor", + )?; + copied_files += 1; + } + + let actual = collect_materialized_uris(mirror_root)?; + let expected = expected_materialized_uris(cir); + if actual != expected { + return Err(CirMaterializeError::TreeMismatch(format!( + "expected {} files, got {} files", + expected.len(), + actual.len() + ))); + } + + Ok(CirMaterializeSummary { + object_count: cir.validated_object_count(), + trust_anchor_count: cir.trust_anchors.len(), + materialized_file_count: expected.len(), + linked_files: 0, + copied_files, + }) +} + +pub fn materialize_cir_from_repo_bytes( + cir: &CanonicalInputRepresentation, + repo_bytes_db: &Path, + mirror_root: &Path, + clean_rebuild: bool, +) -> Result { + cir.validate().map_err(CirMaterializeError::TreeMismatch)?; + + prepare_mirror_root(mirror_root, clean_rebuild)?; + + let repo_bytes = ExternalRepoBytesDb::open(repo_bytes_db).map_err(|e| { + CirMaterializeError::OpenRepoBytesStore { + path: repo_bytes_db.display().to_string(), + detail: e.to_string(), + } + })?; + + let mut copied_files = 0usize; + for object in cir.validated_objects() { + let sha256_hex = hex::encode(&object.sha256); + let bytes = repo_bytes + .get_blob_bytes(&sha256_hex) + .map_err(|e| CirMaterializeError::ReadRepoBytesStore { + sha256_hex: sha256_hex.clone(), + detail: e.to_string(), + })? + .ok_or_else(|| CirMaterializeError::MissingRepoBytesObject { + sha256_hex: sha256_hex.clone(), + })?; + write_bytes_to_mirror_uri( + mirror_root, + &object.rsync_uri, + &bytes, + &repo_bytes_db.display().to_string(), + )?; + copied_files += 1; + } + + for trust_anchor in &cir.trust_anchors { + write_bytes_to_mirror_uri( + mirror_root, + &trust_anchor.ta_rsync_uri, + &trust_anchor.ta_certificate_der, + "cir trust anchor", + )?; + copied_files += 1; + } + + let actual = collect_materialized_uris(mirror_root)?; + let expected = expected_materialized_uris(cir); + if actual != expected { + return Err(CirMaterializeError::TreeMismatch(format!( + "expected {} files, got {} files", + expected.len(), + actual.len() + ))); + } + + Ok(CirMaterializeSummary { + object_count: cir.validated_object_count(), + trust_anchor_count: cir.trust_anchors.len(), + materialized_file_count: expected.len(), + linked_files: 0, + copied_files, + }) +} + +fn prepare_mirror_root(mirror_root: &Path, clean_rebuild: bool) -> Result<(), CirMaterializeError> { + if clean_rebuild && mirror_root.exists() { + fs::remove_dir_all(mirror_root).map_err(|e| CirMaterializeError::RemoveMirrorRoot { + path: mirror_root.display().to_string(), + detail: e.to_string(), + })?; + } + fs::create_dir_all(mirror_root).map_err(|e| CirMaterializeError::CreateMirrorRoot { + path: mirror_root.display().to_string(), + detail: e.to_string(), + }) +} + +fn write_bytes_to_mirror_uri( + mirror_root: &Path, + rsync_uri: &str, + bytes: &[u8], + src_label: &str, +) -> Result<(), CirMaterializeError> { + let relative = mirror_relative_path_for_rsync_uri(rsync_uri)?; + let target = mirror_root.join(&relative); + + if let Some(parent) = target.parent() { + fs::create_dir_all(parent).map_err(|e| CirMaterializeError::CreateParent { + path: parent.display().to_string(), + detail: e.to_string(), + })?; + } + + if target.exists() { + fs::remove_file(&target).map_err(|e| CirMaterializeError::RemoveExistingTarget { + path: target.display().to_string(), + detail: e.to_string(), + })?; + } + + fs::write(&target, bytes).map_err(|e| CirMaterializeError::Copy { + src: src_label.to_string(), + dst: target.display().to_string(), + detail: e.to_string(), + }) +} + +fn expected_materialized_uris(cir: &CanonicalInputRepresentation) -> BTreeSet { + cir.validated_objects() + .map(|item| item.rsync_uri.clone()) + .chain( + cir.trust_anchors + .iter() + .map(|item| item.ta_rsync_uri.clone()), + ) + .collect() +} + +pub fn mirror_relative_path_for_rsync_uri(rsync_uri: &str) -> Result { + let url = url::Url::parse(rsync_uri) + .map_err(|_| CirMaterializeError::InvalidRsyncUri(rsync_uri.to_string()))?; + if url.scheme() != "rsync" { + return Err(CirMaterializeError::InvalidRsyncUri(rsync_uri.to_string())); + } + let host = url + .host_str() + .ok_or_else(|| CirMaterializeError::InvalidRsyncUri(rsync_uri.to_string()))?; + let segments = url + .path_segments() + .ok_or_else(|| CirMaterializeError::InvalidRsyncUri(rsync_uri.to_string()))? + .collect::>(); + if segments.is_empty() || segments.last().copied().unwrap_or_default().is_empty() { + return Err(CirMaterializeError::DirectoryLikeRsyncUri( + rsync_uri.to_string(), + )); + } + + let mut path = PathBuf::from(host); + for segment in segments { + if !segment.is_empty() { + path.push(segment); + } + } + Ok(path) +} + +pub fn resolve_static_pool_file( + static_root: &Path, + sha256_hex: &str, +) -> Result { + if sha256_hex.len() != 64 || !sha256_hex.as_bytes().iter().all(u8::is_ascii_hexdigit) { + return Err(CirMaterializeError::MissingStaticObject { + sha256_hex: sha256_hex.to_string(), + }); + } + let prefix1 = &sha256_hex[0..2]; + let prefix2 = &sha256_hex[2..4]; + + let entries = + fs::read_dir(static_root).map_err(|_| CirMaterializeError::MissingStaticObject { + sha256_hex: sha256_hex.to_string(), + })?; + let mut dates = entries + .filter_map(Result::ok) + .filter(|entry| entry.path().is_dir()) + .map(|entry| entry.path()) + .collect::>(); + dates.sort(); + + for date_dir in dates { + let candidate = date_dir.join(prefix1).join(prefix2).join(sha256_hex); + if candidate.is_file() { + return Ok(candidate); + } + } + Err(CirMaterializeError::MissingStaticObject { + sha256_hex: sha256_hex.to_string(), + }) +} + +fn collect_materialized_uris(mirror_root: &Path) -> Result, CirMaterializeError> { + let mut out = BTreeSet::new(); + let mut stack = vec![mirror_root.to_path_buf()]; + while let Some(path) = stack.pop() { + for entry in fs::read_dir(&path).map_err(|e| CirMaterializeError::CreateMirrorRoot { + path: path.display().to_string(), + detail: e.to_string(), + })? { + let entry = entry.map_err(|e| CirMaterializeError::CreateMirrorRoot { + path: path.display().to_string(), + detail: e.to_string(), + })?; + let path = entry.path(); + if path.is_dir() { + stack.push(path); + } else { + let rel = path + .strip_prefix(mirror_root) + .expect("materialized path under mirror root") + .to_string_lossy() + .replace('\\', "/"); + let uri = format!("rsync://{rel}"); + out.insert(uri); + } + } + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::{ + CirMaterializeError, materialize_cir, materialize_cir_from_raw_store, + materialize_cir_from_repo_bytes, mirror_relative_path_for_rsync_uri, + resolve_static_pool_file, + }; + use crate::blob_store::{ExternalRawStoreDb, ExternalRepoBytesDb}; + use crate::cir::model::{CanonicalInputRepresentation, CirObject, CirTrustAnchor, sha256}; + use sha2::Digest; + use std::path::{Path, PathBuf}; + + fn sample_time() -> time::OffsetDateTime { + time::OffsetDateTime::parse( + "2026-04-07T12:34:56Z", + &time::format_description::well_known::Rfc3339, + ) + .unwrap() + } + + fn sample_trust_anchor() -> CirTrustAnchor { + let ta_rsync_uri = "rsync://example.net/repo/ta.cer"; + let ta_certificate_der = b"ta-der".to_vec(); + CirTrustAnchor { + ta_rsync_uri: ta_rsync_uri.to_string(), + tal_uri: "https://tal.example.net/root.tal".to_string(), + tal_bytes: format!("{ta_rsync_uri}\n\nAQID\n").into_bytes(), + ta_certificate_sha256: sha256(&ta_certificate_der), + ta_certificate_der, + } + } + + fn sample_cir() -> CanonicalInputRepresentation { + CanonicalInputRepresentation::new_v4( + sample_time(), + vec![ + CirObject { + rsync_uri: "rsync://example.net/repo/a.cer".to_string(), + sha256: hex::decode( + "1111111111111111111111111111111111111111111111111111111111111111", + ) + .unwrap(), + }, + CirObject { + rsync_uri: "rsync://example.net/repo/nested/b.roa".to_string(), + sha256: hex::decode( + "2222222222222222222222222222222222222222222222222222222222222222", + ) + .unwrap(), + }, + ], + Vec::new(), + vec![sample_trust_anchor()], + Vec::new(), + Vec::new(), + ) + } + + fn cir_with_real_hashes(a: &[u8], b: &[u8]) -> CanonicalInputRepresentation { + CanonicalInputRepresentation::new_v4( + sample_time(), + vec![ + CirObject { + rsync_uri: "rsync://example.net/repo/a.cer".to_string(), + sha256: sha2::Sha256::digest(a).to_vec(), + }, + CirObject { + rsync_uri: "rsync://example.net/repo/nested/b.roa".to_string(), + sha256: sha2::Sha256::digest(b).to_vec(), + }, + ], + Vec::new(), + vec![sample_trust_anchor()], + Vec::new(), + Vec::new(), + ) + } + + #[test] + fn mirror_relative_path_for_rsync_uri_maps_host_and_path() { + let path = + mirror_relative_path_for_rsync_uri("rsync://example.net/repo/nested/b.roa").unwrap(); + assert_eq!( + path, + PathBuf::from("example.net") + .join("repo") + .join("nested") + .join("b.roa") + ); + } + + #[test] + fn resolve_static_pool_file_finds_hash_across_dates() { + let td = tempfile::tempdir().unwrap(); + let path = td.path().join("20260407").join("11").join("11"); + std::fs::create_dir_all(&path).unwrap(); + let file = path.join("1111111111111111111111111111111111111111111111111111111111111111"); + std::fs::write(&file, b"x").unwrap(); + + let resolved = resolve_static_pool_file( + td.path(), + "1111111111111111111111111111111111111111111111111111111111111111", + ) + .unwrap(); + assert_eq!(resolved, file); + } + + #[test] + fn resolve_static_pool_file_rejects_invalid_hash_and_missing_hash() { + let td = tempfile::tempdir().unwrap(); + let err = resolve_static_pool_file(td.path(), "not-a-hash") + .expect_err("invalid hash should fail"); + assert!(matches!( + err, + CirMaterializeError::MissingStaticObject { .. } + )); + + let err = resolve_static_pool_file( + td.path(), + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ) + .expect_err("missing hash should fail"); + assert!(matches!( + err, + CirMaterializeError::MissingStaticObject { .. } + )); + } + + #[test] + fn mirror_relative_path_rejects_non_rsync_and_directory_like_uris() { + let err = mirror_relative_path_for_rsync_uri("https://example.net/repo/a.roa") + .expect_err("non-rsync uri must fail"); + assert!(matches!(err, CirMaterializeError::InvalidRsyncUri(_))); + + let err = mirror_relative_path_for_rsync_uri("rsync://example.net/repo/") + .expect_err("directory-like uri must fail"); + assert!(matches!(err, CirMaterializeError::DirectoryLikeRsyncUri(_))); + } + + #[test] + fn materialize_clean_rebuild_creates_exact_tree_and_removes_stale_files() { + let td = tempfile::tempdir().unwrap(); + let static_root = td.path().join("static"); + let mirror_root = td.path().join("mirror"); + + write_static( + &static_root, + "20260407", + "1111111111111111111111111111111111111111111111111111111111111111", + b"a", + ); + write_static( + &static_root, + "20260407", + "2222222222222222222222222222222222222222222222222222222222222222", + b"b", + ); + std::fs::create_dir_all(mirror_root.join("stale")).unwrap(); + std::fs::write(mirror_root.join("stale/old.txt"), b"old").unwrap(); + + let summary = materialize_cir(&sample_cir(), &static_root, &mirror_root, true).unwrap(); + assert_eq!(summary.object_count, 2); + assert_eq!(summary.trust_anchor_count, 1); + assert_eq!(summary.materialized_file_count, 3); + assert_eq!( + std::fs::read(mirror_root.join("example.net/repo/a.cer")).unwrap(), + b"a" + ); + assert_eq!( + std::fs::read(mirror_root.join("example.net/repo/nested/b.roa")).unwrap(), + b"b" + ); + assert_eq!( + std::fs::read(mirror_root.join("example.net/repo/ta.cer")).unwrap(), + b"ta-der" + ); + assert!(!mirror_root.join("stale/old.txt").exists()); + } + + #[test] + fn materialize_fails_when_static_object_missing() { + let td = tempfile::tempdir().unwrap(); + let err = materialize_cir(&sample_cir(), td.path(), &td.path().join("mirror"), true) + .expect_err("missing static object must fail"); + assert!(matches!( + err, + CirMaterializeError::MissingStaticObject { .. } + )); + } + + #[test] + fn materialize_without_clean_rebuild_detects_stale_extra_files() { + let td = tempfile::tempdir().unwrap(); + let static_root = td.path().join("static"); + let mirror_root = td.path().join("mirror"); + + write_static( + &static_root, + "20260407", + "1111111111111111111111111111111111111111111111111111111111111111", + b"a", + ); + write_static( + &static_root, + "20260407", + "2222222222222222222222222222222222222222222222222222222222222222", + b"b", + ); + std::fs::create_dir_all(mirror_root.join("extra")).unwrap(); + std::fs::write(mirror_root.join("extra/stale.txt"), b"stale").unwrap(); + + let err = materialize_cir(&sample_cir(), &static_root, &mirror_root, false) + .expect_err("stale extra files should fail exact tree check"); + assert!(matches!(err, CirMaterializeError::TreeMismatch(_))); + } + + #[test] + fn materialize_from_raw_store_creates_expected_tree() { + let td = tempfile::tempdir().unwrap(); + let raw_store_path = td.path().join("raw-store.db"); + let mirror_root = td.path().join("mirror"); + let a = b"a".to_vec(); + let b = b"b".to_vec(); + let cir = cir_with_real_hashes(&a, &b); + + { + let raw_store = ExternalRawStoreDb::open(&raw_store_path).unwrap(); + raw_store + .put_blob_bytes_batch(&[ + (hex::encode(&cir.fresh_validated_objects[0].sha256), a), + (hex::encode(&cir.fresh_validated_objects[1].sha256), b), + ]) + .unwrap(); + } + + let summary = + materialize_cir_from_raw_store(&cir, &raw_store_path, &mirror_root, true).unwrap(); + assert_eq!(summary.object_count, 2); + assert_eq!(summary.trust_anchor_count, 1); + assert_eq!(summary.materialized_file_count, 3); + assert_eq!(summary.linked_files, 0); + assert_eq!(summary.copied_files, 3); + assert_eq!( + std::fs::read(mirror_root.join("example.net/repo/a.cer")).unwrap(), + b"a" + ); + assert_eq!( + std::fs::read(mirror_root.join("example.net/repo/nested/b.roa")).unwrap(), + b"b" + ); + assert_eq!( + std::fs::read(mirror_root.join("example.net/repo/ta.cer")).unwrap(), + b"ta-der" + ); + } + + #[test] + fn materialize_from_raw_store_fails_when_object_missing() { + let td = tempfile::tempdir().unwrap(); + let raw_store_path = td.path().join("raw-store.db"); + let mirror_root = td.path().join("mirror"); + let cir = cir_with_real_hashes(b"a", b"b"); + { + let raw_store = ExternalRawStoreDb::open(&raw_store_path).unwrap(); + raw_store + .put_blob_bytes_batch(&[( + hex::encode(&cir.fresh_validated_objects[0].sha256), + b"a".to_vec(), + )]) + .unwrap(); + } + + let err = materialize_cir_from_raw_store(&cir, &raw_store_path, &mirror_root, true) + .expect_err("missing second object should fail"); + assert!(matches!( + err, + CirMaterializeError::MissingRawStoreObject { .. } + )); + } + + #[test] + fn materialize_from_raw_store_detects_stale_tree_when_not_clean_rebuild() { + let td = tempfile::tempdir().unwrap(); + let raw_store_path = td.path().join("raw-store.db"); + let mirror_root = td.path().join("mirror"); + let cir = cir_with_real_hashes(b"a", b"b"); + { + let raw_store = ExternalRawStoreDb::open(&raw_store_path).unwrap(); + raw_store + .put_blob_bytes_batch(&[ + ( + hex::encode(&cir.fresh_validated_objects[0].sha256), + b"a".to_vec(), + ), + ( + hex::encode(&cir.fresh_validated_objects[1].sha256), + b"b".to_vec(), + ), + ]) + .unwrap(); + } + std::fs::create_dir_all(mirror_root.join("extra")).unwrap(); + std::fs::write(mirror_root.join("extra/stale.txt"), b"stale").unwrap(); + + let err = materialize_cir_from_raw_store(&cir, &raw_store_path, &mirror_root, false) + .expect_err("stale file should fail exact tree check"); + assert!(matches!(err, CirMaterializeError::TreeMismatch(_))); + } + + #[test] + fn materialize_from_raw_store_overwrites_existing_targets() { + let td = tempfile::tempdir().unwrap(); + let raw_store_path = td.path().join("raw-store.db"); + let mirror_root = td.path().join("mirror"); + let a = b"new-a".to_vec(); + let b = b"new-b".to_vec(); + let cir = cir_with_real_hashes(&a, &b); + { + let raw_store = ExternalRawStoreDb::open(&raw_store_path).unwrap(); + raw_store + .put_blob_bytes_batch(&[ + ( + hex::encode(&cir.fresh_validated_objects[0].sha256), + a.clone(), + ), + ( + hex::encode(&cir.fresh_validated_objects[1].sha256), + b.clone(), + ), + ]) + .unwrap(); + } + let target = mirror_root.join("example.net/repo/a.cer"); + std::fs::create_dir_all(target.parent().unwrap()).unwrap(); + std::fs::write(&target, b"old").unwrap(); + + let summary = + materialize_cir_from_raw_store(&cir, &raw_store_path, &mirror_root, false).unwrap(); + assert_eq!(summary.copied_files, 3); + assert_eq!(std::fs::read(&target).unwrap(), a); + } + + #[test] + fn materialize_from_raw_store_ignores_corrupt_raw_entry_when_blob_exists() { + let td = tempfile::tempdir().unwrap(); + let raw_store_path = td.path().join("raw-store.db"); + let mirror_root = td.path().join("mirror"); + let cir = CanonicalInputRepresentation::new_v4( + sample_time(), + vec![CirObject { + rsync_uri: "rsync://example.net/repo/a.cer".to_string(), + sha256: hex::decode( + "1111111111111111111111111111111111111111111111111111111111111111", + ) + .unwrap(), + }], + Vec::new(), + vec![sample_trust_anchor()], + Vec::new(), + Vec::new(), + ); + { + let raw_store = ExternalRawStoreDb::open(&raw_store_path).unwrap(); + raw_store + .put_blob_bytes_batch(&[( + "1111111111111111111111111111111111111111111111111111111111111111".to_string(), + b"blob-a".to_vec(), + )]) + .unwrap(); + } + { + let db = rocksdb::DB::open_default(&raw_store_path).unwrap(); + db.put( + b"rawbyhash:1111111111111111111111111111111111111111111111111111111111111111", + b"bad-cbor", + ) + .unwrap(); + } + + let summary = + materialize_cir_from_raw_store(&cir, &raw_store_path, &mirror_root, true).unwrap(); + assert_eq!(summary.object_count, 1); + assert_eq!(summary.materialized_file_count, 2); + assert_eq!( + std::fs::read(mirror_root.join("example.net/repo/a.cer")).unwrap(), + b"blob-a" + ); + assert_eq!( + std::fs::read(mirror_root.join("example.net/repo/ta.cer")).unwrap(), + b"ta-der" + ); + } + + #[test] + fn materialize_from_repo_bytes_creates_expected_tree() { + let td = tempfile::tempdir().unwrap(); + let repo_bytes_db = td.path().join("repo-bytes.db"); + let mirror_root = td.path().join("mirror"); + let a = b"a".to_vec(); + let b = b"b".to_vec(); + let cir = cir_with_real_hashes(&a, &b); + + { + let repo_bytes = ExternalRepoBytesDb::open(&repo_bytes_db).unwrap(); + repo_bytes + .put_blob_bytes_batch(&[ + (hex::encode(&cir.fresh_validated_objects[0].sha256), a), + (hex::encode(&cir.fresh_validated_objects[1].sha256), b), + ]) + .unwrap(); + } + + let summary = + materialize_cir_from_repo_bytes(&cir, &repo_bytes_db, &mirror_root, true).unwrap(); + assert_eq!(summary.object_count, 2); + assert_eq!(summary.trust_anchor_count, 1); + assert_eq!(summary.materialized_file_count, 3); + assert_eq!(summary.linked_files, 0); + assert_eq!(summary.copied_files, 3); + assert_eq!( + std::fs::read(mirror_root.join("example.net/repo/a.cer")).unwrap(), + b"a" + ); + assert_eq!( + std::fs::read(mirror_root.join("example.net/repo/nested/b.roa")).unwrap(), + b"b" + ); + assert_eq!( + std::fs::read(mirror_root.join("example.net/repo/ta.cer")).unwrap(), + b"ta-der" + ); + } + + fn write_static(root: &Path, date: &str, hash: &str, bytes: &[u8]) { + let path = root.join(date).join(&hash[0..2]).join(&hash[2..4]); + std::fs::create_dir_all(&path).unwrap(); + std::fs::write(path.join(hash), bytes).unwrap(); + } +} diff --git a/crates/panda-rpki-validator/src/cir/mod.rs b/crates/panda-rpki-validator/src/cir/mod.rs new file mode 100644 index 0000000..2f51e00 --- /dev/null +++ b/crates/panda-rpki-validator/src/cir/mod.rs @@ -0,0 +1,441 @@ +pub mod accumulator; +pub mod decode; +pub mod encode; +#[cfg(feature = "full")] +pub mod export; +#[cfg(feature = "full")] +pub mod materialize; +pub mod model; +pub mod sequence; +#[cfg(feature = "full")] +pub mod static_pool; + +pub use accumulator::{CirInputAccumulator, CirInputSection, CirInputSnapshot}; +pub use decode::{CirDecodeError, decode_cir}; +pub use encode::{CirEncodeError, encode_cir}; +#[cfg(feature = "full")] +pub use export::{ + CirExportError, CirExportSummary, CirTrustAnchorBinding, build_cir_from_input_snapshot_multi, + build_cir_from_run, build_cir_from_run_multi, export_cir_from_input_snapshot_multi, + export_cir_from_run, export_cir_from_run_multi, write_cir_file, +}; +#[cfg(feature = "full")] +pub use materialize::{ + CirMaterializeError, CirMaterializeSummary, materialize_cir, materialize_cir_from_raw_store, + materialize_cir_from_repo_bytes, mirror_relative_path_for_rsync_uri, resolve_static_pool_file, +}; +pub use model::{ + CIR_VERSION_V1, CIR_VERSION_V3, CIR_VERSION_V4, CanonicalInputRepresentation, CirHashAlgorithm, + CirObject, CirRejectedObject, CirTrustAnchor, compute_reject_list_sha256, sha256, +}; +pub use sequence::{CirSequenceManifest, CirSequenceStep, CirSequenceStepKind}; +#[cfg(feature = "full")] +pub use static_pool::{ + CirStaticPoolError, CirStaticPoolExportSummary, CirStaticPoolWriteResult, + export_hashes_from_store, static_pool_path, static_pool_relative_path, + write_bytes_to_static_pool, write_raw_entry_to_static_pool, +}; + +#[cfg(test)] +mod tests { + use super::{ + CIR_VERSION_V4, CanonicalInputRepresentation, CirObject, CirRejectedObject, CirTrustAnchor, + decode_cir, encode_cir, + }; + + fn sample_time() -> time::OffsetDateTime { + time::OffsetDateTime::parse( + "2026-04-07T12:34:56Z", + &time::format_description::well_known::Rfc3339, + ) + .expect("valid rfc3339") + } + + fn sample_trust_anchor(ta_rsync_uri: &str, tal_uri: &str, ta_der: &[u8]) -> CirTrustAnchor { + CirTrustAnchor { + ta_rsync_uri: ta_rsync_uri.to_string(), + tal_uri: tal_uri.to_string(), + tal_bytes: format!("{ta_rsync_uri}\n\nAQID\n").into_bytes(), + ta_certificate_der: ta_der.to_vec(), + ta_certificate_sha256: super::sha256(ta_der), + } + } + + fn sample_cir() -> CanonicalInputRepresentation { + let rejected_objects = vec![ + CirRejectedObject { + object_uri: "rsync://example.net/repo/rejected-a.roa".to_string(), + reason: Some("invalid roa".to_string()), + }, + CirRejectedObject { + object_uri: "rsync://example.net/repo/rejected-b.asa".to_string(), + reason: None, + }, + ]; + CanonicalInputRepresentation::new_v4( + sample_time(), + vec![ + CirObject { + rsync_uri: "rsync://example.net/repo/a.cer".to_string(), + sha256: vec![0x11; 32], + }, + CirObject { + rsync_uri: "rsync://example.net/repo/b.roa".to_string(), + sha256: vec![0x22; 32], + }, + CirObject { + rsync_uri: "rsync://example.net/repo/rejected-a.roa".to_string(), + sha256: vec![0x33; 32], + }, + CirObject { + rsync_uri: "rsync://example.net/repo/rejected-b.asa".to_string(), + sha256: vec![0x44; 32], + }, + ], + Vec::new(), + vec![sample_trust_anchor( + "rsync://example.net/repo/ta.cer", + "https://tal.example.net/root.tal", + b"ta-der", + )], + rejected_objects, + Vec::new(), + ) + } + + fn test_encode_tlv(tag: u8, value: &[u8]) -> Vec { + let mut out = Vec::with_capacity(8 + value.len()); + out.push(tag); + if value.len() < 0x80 { + out.push(value.len() as u8); + } else { + let len = value.len(); + let bytes = len.to_be_bytes(); + let first_non_zero = bytes + .iter() + .position(|&b| b != 0) + .unwrap_or(bytes.len() - 1); + let len_bytes = &bytes[first_non_zero..]; + out.push(0x80 | len_bytes.len() as u8); + out.extend_from_slice(len_bytes); + } + out.extend_from_slice(value); + out + } + + #[test] + fn cir_roundtrip_full_succeeds() { + let cir = sample_cir(); + let der = encode_cir(&cir).expect("encode cir"); + let decoded = decode_cir(&der).expect("decode cir"); + assert_eq!(decoded, cir); + } + + #[test] + fn cir_roundtrip_minimal_succeeds() { + let cir = CanonicalInputRepresentation::new_v4( + sample_time(), + Vec::new(), + Vec::new(), + vec![sample_trust_anchor( + "rsync://example.net/repo/minimal-ta.cer", + "https://tal.example.net/minimal.tal", + b"minimal-ta-der", + )], + Vec::new(), + Vec::new(), + ); + let der = encode_cir(&cir).expect("encode minimal cir"); + let decoded = decode_cir(&der).expect("decode minimal cir"); + assert_eq!(decoded, cir); + } + + #[test] + fn cir_model_rejects_unsorted_duplicate_objects() { + let cir = CanonicalInputRepresentation::new_v4( + sample_time(), + vec![ + CirObject { + rsync_uri: "rsync://example.net/repo/z.roa".to_string(), + sha256: vec![0x11; 32], + }, + CirObject { + rsync_uri: "rsync://example.net/repo/a.roa".to_string(), + sha256: vec![0x22; 32], + }, + ], + Vec::new(), + vec![sample_trust_anchor( + "rsync://example.net/repo/ta.cer", + "https://tal.example.net/root.tal", + b"ta-der", + )], + Vec::new(), + Vec::new(), + ); + let err = encode_cir(&cir).expect_err("unsorted objects must fail"); + assert!( + err.to_string().contains("CIR.freshValidatedObjects"), + "{err}" + ); + } + + #[test] + fn cir_model_rejects_duplicate_tals() { + let cir = CanonicalInputRepresentation::new_v4( + sample_time(), + Vec::new(), + Vec::new(), + vec![ + sample_trust_anchor( + "rsync://example.net/repo/ta.cer", + "https://tal.example.net/root.tal", + b"ta-der-a", + ), + sample_trust_anchor( + "rsync://example.net/repo/ta.cer", + "https://tal.example.net/root.tal", + b"ta-der-b", + ), + ], + Vec::new(), + Vec::new(), + ); + let err = encode_cir(&cir).expect_err("duplicate trust_anchors must fail"); + assert!(err.to_string().contains("CIR.trustAnchors"), "{err}"); + } + + #[test] + fn cir_decode_rejects_wrong_version() { + let mut der = encode_cir(&sample_cir()).expect("encode cir"); + let pos = der + .windows(3) + .position(|window| window == [0x02, 0x01, CIR_VERSION_V4 as u8]) + .or_else(|| { + der.windows(3) + .position(|window| window == [0x02, 0x01, CIR_VERSION_V4 as u8]) + }) + .expect("find version integer"); + der[pos + 2] = 2; + let err = decode_cir(&der).expect_err("wrong version must fail"); + assert!(err.to_string().contains("unexpected CIR version"), "{err}"); + } + + #[test] + fn cir_decode_rejects_wrong_hash_oid() { + let mut der = encode_cir(&sample_cir()).expect("encode cir"); + let sha256_bytes = crate::data_model::oid::OID_SHA256_RAW; + let idx = der + .windows(sha256_bytes.len()) + .position(|window| window == sha256_bytes) + .expect("find sha256 oid"); + der[idx + sha256_bytes.len() - 1] ^= 0x01; + let err = decode_cir(&der).expect_err("wrong oid must fail"); + assert!( + err.to_string().contains(crate::data_model::oid::OID_SHA256), + "{err}" + ); + } + + #[test] + fn cir_decode_rejects_bad_generalized_time() { + let mut der = encode_cir(&sample_cir()).expect("encode cir"); + let pos = der + .windows(15) + .position(|window| window == b"20260407123456Z") + .expect("find generalized time"); + der[pos + 14] = b'X'; + let err = decode_cir(&der).expect_err("bad time must fail"); + assert!(err.to_string().contains("GeneralizedTime"), "{err}"); + } + + #[test] + fn cir_model_rejects_non_rsync_object_uri_and_empty_tals() { + let bad_object = CanonicalInputRepresentation::new_v4( + sample_time(), + vec![CirObject { + rsync_uri: "https://example.net/repo/a.roa".to_string(), + sha256: vec![0x11; 32], + }], + Vec::new(), + vec![sample_trust_anchor( + "rsync://example.net/repo/ta.cer", + "https://tal.example.net/root.tal", + b"ta-der", + )], + Vec::new(), + Vec::new(), + ); + let err = encode_cir(&bad_object).expect_err("non-rsync object uri must fail"); + assert!(err.to_string().contains("rsync://"), "{err}"); + + let no_tals = CanonicalInputRepresentation::new_v4( + sample_time(), + Vec::new(), + Vec::new(), + Vec::new(), + Vec::new(), + Vec::new(), + ); + let err = encode_cir(&no_tals).expect_err("empty trust_anchors must fail"); + assert!( + err.to_string() + .contains("CIR.trustAnchors must be non-empty"), + "{err}" + ); + } + + #[test] + fn cir_model_rejects_non_utc_time_bad_hash_len_and_non_http_tal_uri() { + let bad_time = CanonicalInputRepresentation::new_v4( + sample_time().to_offset(time::UtcOffset::from_hms(8, 0, 0).unwrap()), + Vec::new(), + Vec::new(), + vec![sample_trust_anchor( + "rsync://example.net/repo/ta.cer", + "https://tal.example.net/root.tal", + b"ta-der", + )], + Vec::new(), + Vec::new(), + ); + let bad_time = CanonicalInputRepresentation { + validation_time: sample_time().to_offset(time::UtcOffset::from_hms(8, 0, 0).unwrap()), + ..bad_time + }; + let err = encode_cir(&bad_time).expect_err("non-utc validation time must fail"); + assert!(err.to_string().contains("UTC"), "{err}"); + + let bad_hash = CanonicalInputRepresentation::new_v4( + sample_time(), + vec![CirObject { + rsync_uri: "rsync://example.net/repo/a.roa".to_string(), + sha256: vec![0x11; 31], + }], + Vec::new(), + vec![sample_trust_anchor( + "rsync://example.net/repo/ta.cer", + "https://tal.example.net/root.tal", + b"ta-der", + )], + Vec::new(), + Vec::new(), + ); + let err = encode_cir(&bad_hash).expect_err("bad digest len must fail"); + assert!(err.to_string().contains("32 bytes"), "{err}"); + + let bad_tal_uri = CanonicalInputRepresentation::new_v4( + sample_time(), + Vec::new(), + Vec::new(), + vec![sample_trust_anchor( + "rsync://example.net/repo/ta.cer", + "ftp://tal.example.net/root.tal", + b"ta-der", + )], + Vec::new(), + Vec::new(), + ); + let err = encode_cir(&bad_tal_uri).expect_err("bad tal uri must fail"); + assert!(err.to_string().contains("http:// or https://"), "{err}"); + } + + #[test] + fn cir_decode_rejects_trailing_bytes_and_trailing_fields() { + let cir = sample_cir(); + let mut der = encode_cir(&cir).expect("encode cir"); + der.push(0); + let err = decode_cir(&der).expect_err("trailing bytes after cir must fail"); + assert!( + err.to_string().contains("trailing bytes after CIR"), + "{err}" + ); + + let object = test_encode_tlv( + 0x30, + &[ + test_encode_tlv(0x16, b"rsync://example.net/repo/a.roa"), + test_encode_tlv(0x04, &[0x11; 32]), + test_encode_tlv(0x02, &[0x01]), + ] + .concat(), + ); + let trust_anchor = test_encode_tlv( + 0x30, + &[ + test_encode_tlv(0x16, b"rsync://example.net/repo/ta.cer"), + test_encode_tlv(0x16, b"https://tal.example.net/root.tal"), + test_encode_tlv(0x04, b"rsync://example.net/repo/ta.cer\n\nAQID\n"), + test_encode_tlv(0x04, b"ta-der"), + test_encode_tlv(0x04, &super::sha256(b"ta-der")), + ] + .concat(), + ); + let bad = test_encode_tlv( + 0x30, + &[ + test_encode_tlv(0x02, &[CIR_VERSION_V4 as u8]), + test_encode_tlv(0x06, crate::data_model::oid::OID_SHA256_RAW), + test_encode_tlv(0x18, b"20260407123456Z"), + test_encode_tlv(0x30, &object), + test_encode_tlv(0x30, &[]), + test_encode_tlv(0x30, &trust_anchor), + test_encode_tlv(0x04, &[0x33; 32]), + test_encode_tlv(0x04, &[0x33; 32]), + test_encode_tlv(0x04, &[0x33; 32]), + test_encode_tlv(0x04, &[0x33; 32]), + test_encode_tlv(0x04, &[0x33; 32]), + test_encode_tlv(0x04, &[0x33; 32]), + test_encode_tlv(0x30, &[]), + test_encode_tlv(0x30, &[]), + ] + .concat(), + ); + let err = decode_cir(&bad).expect_err("trailing field in object must fail"); + assert!( + err.to_string().contains("trailing fields in CirObject"), + "{err}" + ); + } + + #[test] + fn cir_decode_rejects_invalid_object_and_tal_shapes() { + let cir = sample_cir(); + let mut der = encode_cir(&cir).expect("encode cir"); + + let rsync_text = b"rsync://example.net/repo/a.cer"; + let idx = der + .windows(rsync_text.len()) + .position(|window| window == rsync_text) + .expect("find object uri"); + der[idx] = 0xFF; + let err = decode_cir(&der).expect_err("invalid utf8 object uri must fail"); + assert!(err.to_string().contains("utf-8"), "{err}"); + + let mut der = encode_cir(&cir).expect("encode cir"); + let tal_text = b"https://tal.example.net/root.tal"; + let idx = der + .windows(tal_text.len()) + .position(|window| window == tal_text) + .expect("find tal uri"); + der[idx] = 0xFF; + let err = decode_cir(&der).expect_err("invalid utf8 tal uri must fail"); + assert!(err.to_string().contains("utf-8"), "{err}"); + } + + #[test] + fn cir_model_rejects_unsorted_rejected_objects_and_bad_digest() { + let mut cir = sample_cir(); + cir.fresh_rejected_objects.swap(0, 1); + let err = encode_cir(&cir).expect_err("unsorted rejected objects must fail"); + assert!( + err.to_string().contains("CIR.freshRejectedObjects"), + "{err}" + ); + + let mut cir = sample_cir(); + cir.reject_list_sha256 = vec![0x55; 32]; + let err = encode_cir(&cir).expect_err("bad reject list digest must fail"); + assert!(err.to_string().contains("rejectListSha256"), "{err}"); + } +} diff --git a/crates/panda-rpki-validator/src/cir/model.rs b/crates/panda-rpki-validator/src/cir/model.rs new file mode 100644 index 0000000..bb35667 --- /dev/null +++ b/crates/panda-rpki-validator/src/cir/model.rs @@ -0,0 +1,441 @@ +use crate::data_model::oid::OID_SHA256; + +pub const CIR_VERSION_V1: u32 = 1; +pub const CIR_VERSION_V2: u32 = 2; +pub const CIR_VERSION_V3: u32 = 3; +pub const CIR_VERSION_V4: u32 = 4; +pub const DIGEST_LEN_SHA256: usize = 32; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CirHashAlgorithm { + Sha256, +} + +impl CirHashAlgorithm { + pub fn oid(&self) -> &'static str { + match self { + Self::Sha256 => OID_SHA256, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CanonicalInputRepresentation { + pub version: u32, + pub hash_alg: CirHashAlgorithm, + pub validation_time: time::OffsetDateTime, + pub fresh_validated_objects: Vec, + pub cached_validated_objects: Vec, + pub trust_anchors: Vec, + pub object_list_sha256: Vec, + pub fresh_object_list_sha256: Vec, + pub cached_object_list_sha256: Vec, + pub reject_list_sha256: Vec, + pub fresh_reject_list_sha256: Vec, + pub cached_reject_list_sha256: Vec, + pub fresh_rejected_objects: Vec, + pub cached_rejected_objects: Vec, +} + +impl CanonicalInputRepresentation { + pub fn new_v4( + validation_time: time::OffsetDateTime, + fresh_validated_objects: Vec, + cached_validated_objects: Vec, + trust_anchors: Vec, + fresh_rejected_objects: Vec, + cached_rejected_objects: Vec, + ) -> Self { + let fresh_object_list_sha256 = compute_object_list_sha256(fresh_validated_objects.iter()); + let cached_object_list_sha256 = compute_object_list_sha256(cached_validated_objects.iter()); + let object_list_sha256 = compute_sectioned_object_list_sha256( + fresh_validated_objects + .iter() + .map(|object| ("fresh", object)) + .chain( + cached_validated_objects + .iter() + .map(|object| ("cached", object)), + ), + ); + let fresh_reject_list_sha256 = compute_reject_list_sha256( + fresh_rejected_objects + .iter() + .map(|item| item.object_uri.as_str()), + ); + let cached_reject_list_sha256 = compute_reject_list_sha256( + cached_rejected_objects + .iter() + .map(|item| item.object_uri.as_str()), + ); + let mut all_rejected_uris = fresh_rejected_objects + .iter() + .chain(cached_rejected_objects.iter()) + .map(|item| item.object_uri.as_str()) + .collect::>(); + all_rejected_uris.sort_unstable(); + all_rejected_uris.dedup(); + let reject_list_sha256 = compute_reject_list_sha256(all_rejected_uris.iter().copied()); + + Self { + version: CIR_VERSION_V4, + hash_alg: CirHashAlgorithm::Sha256, + validation_time: validation_time.to_offset(time::UtcOffset::UTC), + fresh_validated_objects, + cached_validated_objects, + trust_anchors, + object_list_sha256, + fresh_object_list_sha256, + cached_object_list_sha256, + reject_list_sha256, + fresh_reject_list_sha256, + cached_reject_list_sha256, + fresh_rejected_objects, + cached_rejected_objects, + } + } + + pub fn validate(&self) -> Result<(), String> { + if self.version != CIR_VERSION_V4 { + return Err(format!( + "CIR version must be {CIR_VERSION_V4}, got {}", + self.version + )); + } + if !matches!(self.hash_alg, CirHashAlgorithm::Sha256) { + return Err("CIR hashAlg must be SHA-256".into()); + } + if self.validation_time.offset() != time::UtcOffset::UTC { + return Err("CIR validationTime must be UTC".into()); + } + validate_sorted_unique_strings( + self.fresh_validated_objects + .iter() + .map(|item| item.rsync_uri.as_str()), + "CIR.freshValidatedObjects must be sorted by rsyncUri and unique", + )?; + validate_sorted_unique_strings( + self.cached_validated_objects + .iter() + .map(|item| item.rsync_uri.as_str()), + "CIR.cachedValidatedObjects must be sorted by rsyncUri and unique", + )?; + validate_sorted_unique_strings( + self.trust_anchors + .iter() + .map(|item| item.ta_rsync_uri.as_str()), + "CIR.trustAnchors must be sorted by taRsyncUri and unique", + )?; + validate_sorted_unique_strings( + self.fresh_rejected_objects + .iter() + .map(|item| item.object_uri.as_str()), + "CIR.freshRejectedObjects must be sorted by objectUri and unique", + )?; + validate_sorted_unique_strings( + self.cached_rejected_objects + .iter() + .map(|item| item.object_uri.as_str()), + "CIR.cachedRejectedObjects must be sorted by objectUri and unique", + )?; + let object_uris = self + .fresh_validated_objects + .iter() + .chain(self.cached_validated_objects.iter()) + .map(|item| item.rsync_uri.as_str()) + .collect::>(); + for trust_anchor in &self.trust_anchors { + if object_uris.contains(trust_anchor.ta_rsync_uri.as_str()) { + return Err(format!( + "CIR.objects must not include trust anchor URI {}", + trust_anchor.ta_rsync_uri + )); + } + } + let fresh_object_uris = self + .fresh_validated_objects + .iter() + .map(|item| item.rsync_uri.as_str()) + .collect::>(); + for item in &self.fresh_rejected_objects { + if !fresh_object_uris.contains(item.object_uri.as_str()) { + return Err(format!( + "CIR.freshRejectedObjects URI must exist in freshValidatedObjects: {}", + item.object_uri + )); + } + } + let cached_object_uris = self + .cached_validated_objects + .iter() + .map(|item| item.rsync_uri.as_str()) + .collect::>(); + for item in &self.cached_rejected_objects { + if !cached_object_uris.contains(item.object_uri.as_str()) { + return Err(format!( + "CIR.cachedRejectedObjects URI must exist in cachedValidatedObjects: {}", + item.object_uri + )); + } + } + if self.trust_anchors.is_empty() { + return Err("CIR.trustAnchors must be non-empty".into()); + } + for (label, digest) in [ + ("CIR.objectListSha256", &self.object_list_sha256), + ("CIR.freshObjectListSha256", &self.fresh_object_list_sha256), + ( + "CIR.cachedObjectListSha256", + &self.cached_object_list_sha256, + ), + ("CIR.rejectListSha256", &self.reject_list_sha256), + ("CIR.freshRejectListSha256", &self.fresh_reject_list_sha256), + ( + "CIR.cachedRejectListSha256", + &self.cached_reject_list_sha256, + ), + ] { + if digest.len() != DIGEST_LEN_SHA256 { + return Err(format!( + "{label} must be {DIGEST_LEN_SHA256} bytes, got {}", + digest.len() + )); + } + } + for object in self.validated_objects() { + object.validate()?; + } + for trust_anchor in &self.trust_anchors { + trust_anchor.validate()?; + } + for item in self.rejected_objects() { + item.validate()?; + } + let expected_fresh_object_digest = + compute_object_list_sha256(self.fresh_validated_objects.iter()); + if self.fresh_object_list_sha256 != expected_fresh_object_digest { + return Err("CIR.freshObjectListSha256 does not match freshValidatedObjects".into()); + } + let expected_cached_object_digest = + compute_object_list_sha256(self.cached_validated_objects.iter()); + if self.cached_object_list_sha256 != expected_cached_object_digest { + return Err("CIR.cachedObjectListSha256 does not match cachedValidatedObjects".into()); + } + let expected_object_digest = compute_sectioned_object_list_sha256( + self.fresh_validated_objects + .iter() + .map(|object| ("fresh", object)) + .chain( + self.cached_validated_objects + .iter() + .map(|object| ("cached", object)), + ), + ); + if self.object_list_sha256 != expected_object_digest { + return Err("CIR.objectListSha256 does not match fresh/cached objects".into()); + } + let expected_fresh_reject_digest = compute_reject_list_sha256( + self.fresh_rejected_objects + .iter() + .map(|item| item.object_uri.as_str()), + ); + if self.fresh_reject_list_sha256 != expected_fresh_reject_digest { + return Err("CIR.freshRejectListSha256 does not match freshRejectedObjects".into()); + } + let expected_cached_reject_digest = compute_reject_list_sha256( + self.cached_rejected_objects + .iter() + .map(|item| item.object_uri.as_str()), + ); + if self.cached_reject_list_sha256 != expected_cached_reject_digest { + return Err("CIR.cachedRejectListSha256 does not match cachedRejectedObjects".into()); + } + let mut all_rejected_uris = self + .rejected_objects() + .map(|item| item.object_uri.as_str()) + .collect::>(); + all_rejected_uris.sort_unstable(); + all_rejected_uris.dedup(); + let expected_digest = compute_reject_list_sha256(all_rejected_uris.iter().copied()); + if self.reject_list_sha256 != expected_digest { + return Err("CIR.rejectListSha256 does not match fresh/cached rejectedObjects".into()); + } + Ok(()) + } + + pub fn validated_objects(&self) -> impl Iterator { + self.fresh_validated_objects + .iter() + .chain(self.cached_validated_objects.iter()) + } + + pub fn rejected_objects(&self) -> impl Iterator { + self.fresh_rejected_objects + .iter() + .chain(self.cached_rejected_objects.iter()) + } + + pub fn validated_object_count(&self) -> usize { + self.fresh_validated_objects.len() + self.cached_validated_objects.len() + } + + pub fn rejected_object_count(&self) -> usize { + self.fresh_rejected_objects.len() + self.cached_rejected_objects.len() + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CirObject { + pub rsync_uri: String, + pub sha256: Vec, +} + +impl CirObject { + pub fn validate(&self) -> Result<(), String> { + if !self.rsync_uri.starts_with("rsync://") { + return Err(format!( + "CirObject.rsync_uri must start with rsync://, got {}", + self.rsync_uri + )); + } + if self.sha256.len() != DIGEST_LEN_SHA256 { + return Err(format!( + "CirObject.sha256 must be {DIGEST_LEN_SHA256} bytes, got {}", + self.sha256.len() + )); + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CirTrustAnchor { + pub ta_rsync_uri: String, + pub tal_uri: String, + pub tal_bytes: Vec, + pub ta_certificate_der: Vec, + pub ta_certificate_sha256: Vec, +} + +impl CirTrustAnchor { + pub fn validate(&self) -> Result<(), String> { + if !self.ta_rsync_uri.starts_with("rsync://") { + return Err(format!( + "CirTrustAnchor.ta_rsync_uri must start with rsync://, got {}", + self.ta_rsync_uri + )); + } + if !(self.tal_uri.starts_with("https://") || self.tal_uri.starts_with("http://")) { + return Err(format!( + "CirTrustAnchor.tal_uri must start with http:// or https://, got {}", + self.tal_uri + )); + } + if self.tal_bytes.is_empty() { + return Err("CirTrustAnchor.tal_bytes must be non-empty".into()); + } + let tal = crate::data_model::tal::Tal::decode_bytes(&self.tal_bytes) + .map_err(|e| format!("CirTrustAnchor.tal_bytes must decode as TAL: {e}"))?; + if !tal + .ta_uris + .iter() + .any(|uri| uri.as_str() == self.ta_rsync_uri) + { + return Err(format!( + "CirTrustAnchor.ta_rsync_uri must be listed in TAL bytes: {}", + self.ta_rsync_uri + )); + } + if self.ta_certificate_der.is_empty() { + return Err("CirTrustAnchor.ta_certificate_der must be non-empty".into()); + } + if self.ta_certificate_sha256.len() != DIGEST_LEN_SHA256 { + return Err(format!( + "CirTrustAnchor.ta_certificate_sha256 must be {DIGEST_LEN_SHA256} bytes, got {}", + self.ta_certificate_sha256.len() + )); + } + let expected = sha256(&self.ta_certificate_der); + if self.ta_certificate_sha256 != expected { + return Err("CirTrustAnchor.ta_certificate_sha256 does not match DER bytes".into()); + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CirRejectedObject { + pub object_uri: String, + pub reason: Option, +} + +impl CirRejectedObject { + pub fn validate(&self) -> Result<(), String> { + if !self.object_uri.starts_with("rsync://") { + return Err(format!( + "CirRejectedObject.object_uri must start with rsync://, got {}", + self.object_uri + )); + } + Ok(()) + } +} + +pub fn compute_reject_list_sha256<'a>(uris: impl IntoIterator) -> Vec { + let mut body = Vec::new(); + for uri in uris { + let bytes = uri.as_bytes(); + body.extend_from_slice(&(bytes.len() as u32).to_be_bytes()); + body.extend_from_slice(bytes); + } + sha256(&body) +} + +pub fn compute_object_list_sha256<'a>(objects: impl IntoIterator) -> Vec { + let mut body = Vec::new(); + for object in objects { + append_digest_string(&mut body, &object.rsync_uri); + body.extend_from_slice(&object.sha256); + } + sha256(&body) +} + +pub fn compute_sectioned_object_list_sha256<'a>( + objects: impl IntoIterator, +) -> Vec { + let mut body = Vec::new(); + for (section, object) in objects { + append_digest_string(&mut body, section); + append_digest_string(&mut body, &object.rsync_uri); + body.extend_from_slice(&object.sha256); + } + sha256(&body) +} + +pub fn sha256(bytes: &[u8]) -> Vec { + use sha2::Digest; + + sha2::Sha256::digest(bytes).to_vec() +} + +fn append_digest_string(body: &mut Vec, value: &str) { + let bytes = value.as_bytes(); + body.extend_from_slice(&(bytes.len() as u32).to_be_bytes()); + body.extend_from_slice(bytes); +} + +fn validate_sorted_unique_strings<'a>( + items: impl IntoIterator, + message: &str, +) -> Result<(), String> { + let mut prev: Option<&'a str> = None; + for key in items { + if let Some(prev_key) = prev + && key <= prev_key + { + return Err(message.into()); + } + prev = Some(key); + } + Ok(()) +} diff --git a/crates/panda-rpki-validator/src/cir/sequence.rs b/crates/panda-rpki-validator/src/cir/sequence.rs new file mode 100644 index 0000000..2a68b60 --- /dev/null +++ b/crates/panda-rpki-validator/src/cir/sequence.rs @@ -0,0 +1,203 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CirSequenceStepKind { + Full, + Delta, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CirSequenceStep { + pub step_id: String, + pub kind: CirSequenceStepKind, + pub validation_time: String, + pub cir_path: String, + pub ccr_path: String, + pub report_path: String, + #[serde(default)] + pub timing_path: Option, + #[serde(default)] + pub stdout_log_path: Option, + #[serde(default)] + pub stderr_log_path: Option, + #[serde(default)] + pub artifact_prefix: Option, + pub previous_step_id: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CirSequenceManifest { + pub version: u32, + pub repo_bytes_db_path: String, + pub steps: Vec, +} + +impl CirSequenceManifest { + pub fn validate(&self) -> Result<(), String> { + if self.version == 0 { + return Err("sequence.version must be positive".to_string()); + } + if self.repo_bytes_db_path.trim().is_empty() { + return Err("sequence.repo_bytes_db_path must not be empty".to_string()); + } + if self.steps.is_empty() { + return Err("sequence.steps must not be empty".to_string()); + } + let mut previous_ids = std::collections::BTreeSet::new(); + for (idx, step) in self.steps.iter().enumerate() { + if step.step_id.trim().is_empty() { + return Err(format!("sequence.steps[{idx}].step_id must not be empty")); + } + if !previous_ids.insert(step.step_id.clone()) { + return Err(format!("sequence.steps[{idx}].step_id must be unique")); + } + if step.validation_time.trim().is_empty() { + return Err(format!( + "sequence.steps[{idx}].validation_time must not be empty" + )); + } + if let Some(timing_path) = &step.timing_path + && timing_path.trim().is_empty() + { + return Err(format!( + "sequence.steps[{idx}].timing_path must not be empty when set" + )); + } + if let Some(stdout_log_path) = &step.stdout_log_path + && stdout_log_path.trim().is_empty() + { + return Err(format!( + "sequence.steps[{idx}].stdout_log_path must not be empty when set" + )); + } + if let Some(stderr_log_path) = &step.stderr_log_path + && stderr_log_path.trim().is_empty() + { + return Err(format!( + "sequence.steps[{idx}].stderr_log_path must not be empty when set" + )); + } + if let Some(artifact_prefix) = &step.artifact_prefix + && artifact_prefix.trim().is_empty() + { + return Err(format!( + "sequence.steps[{idx}].artifact_prefix must not be empty when set" + )); + } + if step.cir_path.trim().is_empty() + || step.ccr_path.trim().is_empty() + || step.report_path.trim().is_empty() + { + return Err(format!( + "sequence.steps[{idx}] output paths must not be empty" + )); + } + match step.kind { + CirSequenceStepKind::Full => { + if idx != 0 { + return Err("full step must be the first step".to_string()); + } + if step.previous_step_id.is_some() { + return Err("full step must not reference previous_step_id".to_string()); + } + } + CirSequenceStepKind::Delta => { + if idx == 0 { + return Err("delta step cannot be the first step".to_string()); + } + let previous = step.previous_step_id.as_ref().ok_or_else(|| { + format!("sequence.steps[{idx}] delta step must set previous_step_id") + })?; + if !previous_ids.contains(previous) { + return Err(format!( + "sequence.steps[{idx}] previous_step_id must reference an earlier step" + )); + } + } + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::{CirSequenceManifest, CirSequenceStep, CirSequenceStepKind}; + + fn sample_manifest() -> CirSequenceManifest { + CirSequenceManifest { + version: 1, + repo_bytes_db_path: "repo-bytes.db".to_string(), + steps: vec![ + CirSequenceStep { + step_id: "full".to_string(), + kind: CirSequenceStepKind::Full, + validation_time: "2026-04-09T00:00:00Z".to_string(), + cir_path: "full/input.cir".to_string(), + ccr_path: "full/result.ccr".to_string(), + report_path: "full/report.json".to_string(), + timing_path: Some("full/timing.json".to_string()), + stdout_log_path: Some("full/stdout.log".to_string()), + stderr_log_path: Some("full/stderr.log".to_string()), + artifact_prefix: Some("2026-04-09T00:00:00Z-test".to_string()), + previous_step_id: None, + }, + CirSequenceStep { + step_id: "delta-001".to_string(), + kind: CirSequenceStepKind::Delta, + validation_time: "2026-04-09T00:10:00Z".to_string(), + cir_path: "delta-001/input.cir".to_string(), + ccr_path: "delta-001/result.ccr".to_string(), + report_path: "delta-001/report.json".to_string(), + timing_path: Some("delta-001/timing.json".to_string()), + stdout_log_path: Some("delta-001/stdout.log".to_string()), + stderr_log_path: Some("delta-001/stderr.log".to_string()), + artifact_prefix: Some("2026-04-09T00:10:00Z-test".to_string()), + previous_step_id: Some("full".to_string()), + }, + ], + } + } + + #[test] + fn sequence_manifest_validate_accepts_minimal_chain() { + sample_manifest().validate().expect("valid sequence"); + } + + #[test] + fn sequence_manifest_validate_rejects_bad_order_and_duplicates() { + let mut bad = sample_manifest(); + bad.steps.swap(0, 1); + let err = bad.validate().expect_err("full must be first"); + assert!( + err.contains("delta step cannot be the first step") + || err.contains("full step must be the first step") + ); + + let mut dup = sample_manifest(); + dup.steps[1].step_id = "full".to_string(); + let err = dup.validate().expect_err("duplicate id must fail"); + assert!(err.contains("must be unique")); + } + + #[test] + fn sequence_manifest_validate_rejects_missing_previous_reference() { + let mut bad = sample_manifest(); + bad.steps[1].previous_step_id = Some("missing".to_string()); + let err = bad.validate().expect_err("missing previous must fail"); + assert!(err.contains("previous_step_id")); + } + + #[test] + fn sequence_manifest_validate_rejects_empty_repo_bytes_backend() { + let mut manifest = sample_manifest(); + manifest.repo_bytes_db_path = " ".to_string(); + let err = manifest + .validate() + .expect_err("empty repo bytes backend must fail"); + assert!(err.contains("repo_bytes_db_path")); + } +} diff --git a/crates/panda-rpki-validator/src/cir/static_pool.rs b/crates/panda-rpki-validator/src/cir/static_pool.rs new file mode 100644 index 0000000..757a952 --- /dev/null +++ b/crates/panda-rpki-validator/src/cir/static_pool.rs @@ -0,0 +1,376 @@ +use std::collections::BTreeSet; +use std::fs::{self, OpenOptions}; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use crate::storage::{RawByHashEntry, RocksStore}; + +#[derive(Debug, thiserror::Error)] +pub enum CirStaticPoolError { + #[error("invalid sha256 hex: {0}")] + InvalidSha256Hex(String), + + #[error("raw bytes are empty for sha256={sha256_hex}")] + EmptyBytes { sha256_hex: String }, + + #[error("raw bytes do not match sha256 hex: {sha256_hex}")] + HashMismatch { sha256_hex: String }, + + #[error("create directory failed: {path}: {detail}")] + CreateDir { path: String, detail: String }, + + #[error("create temp file failed: {path}: {detail}")] + CreateTemp { path: String, detail: String }, + + #[error("write temp file failed: {path}: {detail}")] + WriteTemp { path: String, detail: String }, + + #[error("sync temp file failed: {path}: {detail}")] + SyncTemp { path: String, detail: String }, + + #[error("publish temp file failed: {temp_path} -> {final_path}: {detail}")] + Publish { + temp_path: String, + final_path: String, + detail: String, + }, + + #[error("remove temp file failed: {path}: {detail}")] + RemoveTemp { path: String, detail: String }, + + #[error("raw_by_hash entry missing for sha256={sha256_hex}")] + MissingRawByHash { sha256_hex: String }, + + #[error("storage error: {0}")] + Storage(String), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CirStaticPoolWriteResult { + pub final_path: PathBuf, + pub written: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CirStaticPoolExportSummary { + pub unique_hashes: usize, + pub written_files: usize, + pub reused_files: usize, +} + +pub fn static_pool_relative_path( + capture_date_utc: time::Date, + sha256_hex: &str, +) -> Result { + validate_sha256_hex(sha256_hex)?; + let date = format_utc_date(capture_date_utc); + Ok(PathBuf::from(date) + .join(&sha256_hex[0..2]) + .join(&sha256_hex[2..4]) + .join(sha256_hex)) +} + +pub fn static_pool_path( + static_root: &Path, + capture_date_utc: time::Date, + sha256_hex: &str, +) -> Result { + Ok(static_root.join(static_pool_relative_path(capture_date_utc, sha256_hex)?)) +} + +pub fn write_bytes_to_static_pool( + static_root: &Path, + capture_date_utc: time::Date, + sha256_hex_value: &str, + bytes: &[u8], +) -> Result { + validate_sha256_hex(sha256_hex_value)?; + if bytes.is_empty() { + return Err(CirStaticPoolError::EmptyBytes { + sha256_hex: sha256_hex_value.to_string(), + }); + } + let computed = compute_sha256_hex(bytes); + if computed != sha256_hex_value.to_ascii_lowercase() { + return Err(CirStaticPoolError::HashMismatch { + sha256_hex: sha256_hex_value.to_string(), + }); + } + + let final_path = static_pool_path(static_root, capture_date_utc, sha256_hex_value)?; + if final_path.exists() { + return Ok(CirStaticPoolWriteResult { + final_path, + written: false, + }); + } + + let parent = final_path.parent().expect("static pool file has parent"); + fs::create_dir_all(parent).map_err(|e| CirStaticPoolError::CreateDir { + path: parent.display().to_string(), + detail: e.to_string(), + })?; + + let temp_path = parent.join(format!("{sha256_hex_value}.tmp.{}", uuid::Uuid::new_v4())); + let mut file = OpenOptions::new() + .create_new(true) + .write(true) + .open(&temp_path) + .map_err(|e| CirStaticPoolError::CreateTemp { + path: temp_path.display().to_string(), + detail: e.to_string(), + })?; + file.write_all(bytes) + .map_err(|e| CirStaticPoolError::WriteTemp { + path: temp_path.display().to_string(), + detail: e.to_string(), + })?; + file.sync_all().map_err(|e| CirStaticPoolError::SyncTemp { + path: temp_path.display().to_string(), + detail: e.to_string(), + })?; + drop(file); + + match fs::hard_link(&temp_path, &final_path) { + Ok(()) => { + fs::remove_file(&temp_path).map_err(|e| CirStaticPoolError::RemoveTemp { + path: temp_path.display().to_string(), + detail: e.to_string(), + })?; + Ok(CirStaticPoolWriteResult { + final_path, + written: true, + }) + } + Err(e) if final_path.exists() => { + fs::remove_file(&temp_path).map_err(|remove_err| CirStaticPoolError::RemoveTemp { + path: temp_path.display().to_string(), + detail: remove_err.to_string(), + })?; + let _ = e; + Ok(CirStaticPoolWriteResult { + final_path, + written: false, + }) + } + Err(e) => Err(CirStaticPoolError::Publish { + temp_path: temp_path.display().to_string(), + final_path: final_path.display().to_string(), + detail: e.to_string(), + }), + } +} + +pub fn write_raw_entry_to_static_pool( + static_root: &Path, + capture_date_utc: time::Date, + entry: &RawByHashEntry, +) -> Result { + write_bytes_to_static_pool( + static_root, + capture_date_utc, + &entry.sha256_hex, + &entry.bytes, + ) +} + +pub fn export_hashes_from_store( + store: &RocksStore, + static_root: &Path, + capture_date_utc: time::Date, + sha256_hexes: &[String], +) -> Result { + let unique: BTreeSet = sha256_hexes + .iter() + .map(|item| item.to_ascii_lowercase()) + .collect(); + + let mut written_files = 0usize; + let mut reused_files = 0usize; + for sha256_hex in &unique { + let bytes = store + .get_blob_bytes(sha256_hex) + .map_err(|e| CirStaticPoolError::Storage(e.to_string()))? + .ok_or_else(|| CirStaticPoolError::MissingRawByHash { + sha256_hex: sha256_hex.clone(), + })?; + let entry = RawByHashEntry::from_bytes(sha256_hex.clone(), bytes); + let result = write_raw_entry_to_static_pool(static_root, capture_date_utc, &entry)?; + if result.written { + written_files += 1; + } else { + reused_files += 1; + } + } + + Ok(CirStaticPoolExportSummary { + unique_hashes: unique.len(), + written_files, + reused_files, + }) +} + +fn format_utc_date(date: time::Date) -> String { + format!( + "{:04}{:02}{:02}", + date.year(), + u8::from(date.month()), + date.day() + ) +} + +fn validate_sha256_hex(sha256_hex: &str) -> Result<(), CirStaticPoolError> { + if sha256_hex.len() != 64 || !sha256_hex.as_bytes().iter().all(u8::is_ascii_hexdigit) { + return Err(CirStaticPoolError::InvalidSha256Hex(sha256_hex.to_string())); + } + Ok(()) +} + +fn compute_sha256_hex(bytes: &[u8]) -> String { + use sha2::{Digest, Sha256}; + hex::encode(Sha256::digest(bytes)) +} + +#[cfg(test)] +mod tests { + use super::{ + CirStaticPoolError, compute_sha256_hex, export_hashes_from_store, + static_pool_relative_path, write_bytes_to_static_pool, + }; + use crate::storage::{RawByHashEntry, RepositoryViewEntry, RepositoryViewState, RocksStore}; + use std::fs; + + fn sample_date() -> time::Date { + time::Date::from_calendar_date(2026, time::Month::April, 7).unwrap() + } + + #[test] + fn static_pool_relative_path_uses_date_and_hash_prefixes() { + let path = static_pool_relative_path( + sample_date(), + "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789", + ) + .expect("relative path"); + assert_eq!( + path, + std::path::PathBuf::from("20260407") + .join("ab") + .join("cd") + .join("abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789") + ); + } + + #[test] + fn write_bytes_to_static_pool_is_idempotent_and_leaves_no_temp_files() { + let td = tempfile::tempdir().expect("tempdir"); + let bytes = b"static-pool-object"; + let sha = compute_sha256_hex(bytes); + + let first = + write_bytes_to_static_pool(td.path(), sample_date(), &sha, bytes).expect("first write"); + let second = write_bytes_to_static_pool(td.path(), sample_date(), &sha, bytes) + .expect("second write"); + + assert!(first.written); + assert!(!second.written); + assert_eq!(fs::read(&first.final_path).expect("read final"), bytes); + + let all_files: Vec<_> = walk_files(td.path()); + assert_eq!(all_files.len(), 1); + assert!( + !all_files[0] + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or_default() + .contains(".tmp.") + ); + } + + #[test] + fn write_bytes_to_static_pool_rejects_bad_hash_and_empty_bytes() { + let td = tempfile::tempdir().expect("tempdir"); + let err = write_bytes_to_static_pool(td.path(), sample_date(), "not-a-hash", b"x") + .expect_err("bad hash must fail"); + assert!(matches!(err, CirStaticPoolError::InvalidSha256Hex(_))); + + let err = write_bytes_to_static_pool( + td.path(), + sample_date(), + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + b"", + ) + .expect_err("empty bytes must fail"); + assert!(matches!(err, CirStaticPoolError::EmptyBytes { .. })); + } + + #[test] + fn export_hashes_from_store_writes_unique_entries_and_fails_when_missing() { + let td = tempfile::tempdir().expect("tempdir"); + let store_dir = td.path().join("db"); + let static_root = td.path().join("static"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + + let bytes = b"store-object".to_vec(); + let sha = compute_sha256_hex(&bytes); + let mut entry = RawByHashEntry::from_bytes(sha.clone(), bytes.clone()); + entry + .origin_uris + .push("rsync://example.test/repo/object.cer".to_string()); + store.put_raw_by_hash_entry(&entry).expect("put raw entry"); + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/object.cer".to_string(), + current_hash: Some(sha.clone()), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("cer".to_string()), + state: RepositoryViewState::Present, + }) + .expect("put repository view"); + + let summary = export_hashes_from_store( + &store, + &static_root, + sample_date(), + &[sha.clone(), sha.clone()], + ) + .expect("export hashes"); + assert_eq!(summary.unique_hashes, 1); + assert_eq!(summary.written_files, 1); + assert_eq!(summary.reused_files, 0); + + let summary = export_hashes_from_store(&store, &static_root, sample_date(), &[sha.clone()]) + .expect("re-export hashes"); + assert_eq!(summary.unique_hashes, 1); + assert_eq!(summary.written_files, 0); + assert_eq!(summary.reused_files, 1); + + let err = export_hashes_from_store( + &store, + &static_root, + sample_date(), + &[String::from( + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + )], + ) + .expect_err("missing raw_by_hash must fail"); + assert!(matches!(err, CirStaticPoolError::MissingRawByHash { .. })); + } + + fn walk_files(root: &std::path::Path) -> Vec { + let mut out = Vec::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(path) = stack.pop() { + for entry in fs::read_dir(path).expect("read_dir") { + let entry = entry.expect("dir entry"); + let path = entry.path(); + if path.is_dir() { + stack.push(path); + } else { + out.push(path); + } + } + } + out.sort(); + out + } +} diff --git a/crates/panda-rpki-validator/src/cli.rs b/crates/panda-rpki-validator/src/cli.rs new file mode 100644 index 0000000..da1f8fd --- /dev/null +++ b/crates/panda-rpki-validator/src/cli.rs @@ -0,0 +1,2897 @@ +mod output; + +use crate::ccr::{ + CcrAccumulator, CcrBuildBreakdown, build_ccr_from_run_with_breakdown, write_ccr_file, +}; +use crate::cir::{CirTrustAnchorBinding, export_cir_from_input_snapshot_multi}; +use std::path::{Path, PathBuf}; + +use crate::analysis::timing::{ + DurationStats, TimingHandle, TimingMeta, TimingMetaUpdate, TopDurationEntry, +}; +use crate::audit::AuditRepoSyncStats; +#[cfg(test)] +use crate::audit::{ + AspaOutput, AuditReportV2, AuditRunMeta, AuditWarning, TreeSummary, VrpOutput, + format_roa_ip_prefix, +}; +use crate::fetch::http::{BlockingHttpFetcher, HttpFetcherConfig}; +use crate::fetch::rsync::LocalDirRsyncFetcher; +use crate::fetch::rsync_system::{RsyncScopePolicy, SystemRsyncConfig, SystemRsyncFetcher}; +use crate::memory_telemetry::{ + MallocTrimProbe, MemoryTelemetryCheckpoint, MemoryTelemetrySummary, ObjectGraphMemoryMetric, + ObjectGraphMemorySection, ObjectGraphMemorySummary, +}; +use crate::parallel::config::{ParallelPhase1Config, ParallelPhase2Config}; +use crate::parallel::types::TalInputSpec; +use crate::policy::{Policy, ResourceValidationMode, StrictPolicy}; +use crate::storage::{RocksStore, VcirStorageSummary}; +use crate::ta_constraints::TaConstraintsByTal; +use crate::validation::run_tree_from_tal::{ + RunTreeFromTalAuditOutput, run_tree_from_multiple_tals_parallel_phase2_audit, + run_tree_from_multiple_tals_parallel_phase2_audit_with_timing, + run_tree_from_tal_and_ta_der_parallel_phase2_audit, + run_tree_from_tal_and_ta_der_parallel_phase2_audit_with_timing, + run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit, + run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit_with_timing, + run_tree_from_tal_and_ta_der_payload_replay_serial_audit, + run_tree_from_tal_and_ta_der_payload_replay_serial_audit_with_timing, + run_tree_from_tal_url_parallel_phase2_audit, + run_tree_from_tal_url_parallel_phase2_audit_with_timing, +}; +use crate::validation::tree::{DEFAULT_MAX_CA_DEPTH, TreeRunConfig}; +#[cfg(test)] +use output::write_json; +use output::{ + ReportJsonFormat, run_compare_view_task, write_report_json_from_shared, write_stage_timing, +}; +use serde::Serialize; +use std::collections::HashMap; +use std::sync::Arc; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +struct RunStageTiming { + validation_ms: u64, + enable_roa_validation_cache: bool, + enable_child_certificate_validation_cache: bool, + publication_point_cache_observe_only: bool, + enable_publication_point_validation_cache: bool, + crypto_signature_cache_observe: Option, + enable_transport_request_prefetch: bool, + report_build_ms: u64, + report_write_ms: Option, + ccr_build_ms: Option, + ccr_build_breakdown: Option, + ccr_write_ms: Option, + compare_view_build_ms: Option, + compare_view_write_ms: Option, + cir_build_cir_ms: Option, + cir_write_cir_ms: Option, + cir_total_ms: Option, + total_ms: u64, + publication_points: usize, + repo_sync_ms_total: u64, + publication_point_repo_sync_ms_total: u64, + download_event_count: u64, + rrdp_download_ms_total: u64, + rsync_download_ms_total: u64, + download_bytes_total: u64, + roa_validation_cache: crate::validation::objects::RoaValidationCacheStats, + analysis_counts: HashMap, + analysis_phases: HashMap, + analysis_top_publication_points: Vec, + analysis_top_publication_point_steps: Vec, + analysis_top_publication_point_cache_steps: Vec, + vcir_storage_summary_ms: Option, + vcir_storage: Option, + publication_point_cache_index_load: Option, + publication_point_cache_index_refresh: Option, + memory_telemetry: Option, +} + +fn record_memory_checkpoint( + checkpoints: &mut Vec, + label: &str, + total_started: &std::time::Instant, + store: &RocksStore, +) { + checkpoints.push(MemoryTelemetryCheckpoint { + label: label.to_string(), + elapsed_ms: total_started.elapsed().as_millis() as u64, + process: crate::memory_telemetry::process_memory_snapshot(label), + rocksdb: store.memory_snapshot(), + }); +} + +fn memory_trim_probe_enabled() -> bool { + std::env::var("RPKI_MEMORY_TRIM_PROBE") + .map(|value| matches!(value.as_str(), "1" | "true" | "TRUE" | "yes" | "YES")) + .unwrap_or(false) +} + +fn vcir_storage_summary_enabled() -> bool { + std::env::var("RPKI_VCIR_STORAGE_SUMMARY") + .map(|value| matches!(value.as_str(), "1" | "true" | "TRUE" | "yes" | "YES")) + .unwrap_or(false) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CliArgs { + pub validation_contract_out_path: Option, + pub tal_urls: Vec, + pub tal_paths: Vec, + pub ta_paths: Vec, + pub tal_url: Option, + pub tal_path: Option, + pub ta_path: Option, + pub parallel_phase1_config: ParallelPhase1Config, + pub parallel_phase2_config: ParallelPhase2Config, + pub tal_inputs: Vec, + pub ta_constraints: TaConstraintsByTal, + + pub db_path: PathBuf, + pub raw_store_db: Option, + pub repo_bytes_db: Option, + pub policy_path: Option, + pub strict_policy: Option, + pub resource_validation_mode: Option, + pub report_json_path: Option, + pub report_json_compact: bool, + pub skip_report_build: bool, + pub skip_vcir_persist: bool, + pub enable_roa_validation_cache: bool, + pub enable_child_certificate_validation_cache: bool, + pub publication_point_cache_observe_only: bool, + pub enable_publication_point_validation_cache: bool, + pub crypto_signature_cache_observe_only: bool, + pub enable_crypto_signature_cache: bool, + pub enable_transport_request_prefetch: bool, + pub ccr_out_path: Option, + pub vrps_csv_out_path: Option, + pub vaps_csv_out_path: Option, + pub compare_view_trust_anchor: Option, + pub cir_enabled: bool, + pub cir_out_path: Option, + pub cir_static_root: Option, + pub cir_tal_uris: Vec, + pub cir_tal_uri: Option, + pub payload_replay_archive: Option, + pub payload_replay_locks: Option, + pub payload_base_archive: Option, + pub payload_base_locks: Option, + pub payload_base_validation_time: Option, + pub payload_delta_archive: Option, + pub payload_delta_locks: Option, + pub memory_trim_after_validation: bool, + + pub rsync_local_dir: Option, + pub disable_rrdp: bool, + pub rsync_command: Option, + + pub http_timeout_secs: u64, + pub http_root_cert_paths: Vec, + pub rsync_timeout_secs: u64, + pub rsync_mirror_root: Option, + pub rsync_scope_policy: RsyncScopePolicy, + + pub max_ca_depth: usize, + pub max_instances: Option, + pub validation_time: Option, + + pub analyze: bool, + pub analysis_out_path: Option, + pub profile_cpu: bool, +} + +fn usage() -> String { + let bin = "panda-rpki-validator"; + format!( + "\ +Usage: + {bin} --db --tal-url [--tal-url ...] [options] + {bin} --db --tal-path --ta-path [--tal-path --ta-path ...] [options] + +Options: + --validation-contract-out + Write the effective normal-run validation contract + --db RocksDB directory path (required) + --raw-store-db External raw-by-hash store DB path (optional) + --repo-bytes-db External repo object bytes DB path (optional) + --policy Policy TOML path (optional) + --ta-constraints = + Apply local EE-resource constraints to one TAL (repeatable); adjacent .constraints files are auto-discovered + --strict [policies] Enable strict policies (default all; comma list: name,cms-der,signed-attrs; none disables) + --resource-validation-mode + Resource certificate validation mode (default: validation-update-03) + --report-json Write full audit report as JSON (optional) + --report-json-compact Write report JSON without pretty-printing (requires --report-json) + --skip-report-build Skip full audit report construction when --report-json is not requested + --skip-vcir-persist Skip VCIR persistence/projection building for compare-only runs + --enable-roa-validation-cache + Reuse accepted ROA validation outputs from previous VCIR records (default: off) + --enable-child-certificate-validation-cache + Experimental: reuse validated child certificate discovery results + --publication-point-cache-observe-only + Evaluate publication-point cache eligibility without changing results + --enable-publication-point-validation-cache + Experimental: reuse complete publication-point validation projections + --crypto-signature-cache-observe-only + Measure crypto signature cache key hit rates and verify durations + without changing validation behavior (default: off) + --enable-crypto-signature-cache + Experimental: skip cryptographic signature verification on cache + hits (positive conclusions only; default: off) + --enable-transport-request-prefetch + Experimental: prefetch previous run transport repo requests before tree traversal + --ccr-out Write CCR DER ContentInfo to this path (optional) + --vrps-csv-out Write VRP compare-view CSV directly from validation output (optional; requires --vaps-csv-out) + --vaps-csv-out Write VAP compare-view CSV directly from validation output (optional; requires --vrps-csv-out) + --compare-view-trust-anchor + Trust-anchor label used by direct compare-view CSV output (default: unknown) + --cir-enable Export CIR after the run completes + --cir-out Write CIR DER to this path (requires --cir-enable) + --cir-static-root Deprecated; CIR export no longer exports object pools + --cir-tal-uri Override TAL URI for CIR export (repeatable in multi-TAL mode) + --payload-replay-archive Use local payload replay archive root (offline replay mode) + --payload-replay-locks Use local payload replay locks.json (offline replay mode) + --payload-base-archive Use local base payload archive root (offline delta replay) + --payload-base-locks Use local base locks.json (offline delta replay) + --payload-base-validation-time Validation time for the base bootstrap inside offline delta replay + --payload-delta-archive Use local delta payload archive root (offline delta replay) + --payload-delta-locks Use local locks-delta.json (offline delta replay) + --memory-trim-after-validation Call malloc_trim(0) after validation/report memory checkpoints (Linux glibc only; default off) + + --tal-url TAL URL (repeatable; URL mode) + --tal-path TAL file path (repeatable; file mode) + --ta-path TA certificate DER file path (repeatable in file mode; pairs with --tal-path by position) + --parallel-max-repo-sync-workers-global + Phase 1 global repo sync worker budget (default: 4) + --parallel-max-inflight-snapshot-bytes-global + Phase 1 inflight snapshot byte budget (default: 512MiB) + --parallel-max-pending-repo-results + Phase 1 pending repo result budget (default: 1024) + --parallel-phase2-object-workers + Phase 2 object worker count (default: 8) + --parallel-phase2-worker-queue-capacity + Phase 2 per-worker object queue capacity (default: 256) + --parallel-phase2-ready-batch-size + Phase 2 ready publication points processed per scheduler turn (default: 256) + --parallel-phase2-ready-batch-wall-time-budget-ms + Phase 2 ready staging wall-time budget per scheduler turn (default: 100) + --parallel-phase2-result-drain-batch-size + Phase 2 object results drained per scheduler turn (default: 2048) + --parallel-phase2-finalize-batch-size + Legacy Phase 2 scheduler finalize budget; dedicated finalize worker ignores it (default: 256) + --parallel-phase2-finalize-batch-wall-time-budget-ms + Legacy Phase 2 scheduler finalize time budget; dedicated finalize worker ignores it (default: 100) + --parallel-phase2-finalize-queue-capacity + Phase 2 dedicated finalize worker queue capacity (default: 32768) + --control-plane-stage-workers + Experimental: Phase 2 ready publication point stage worker count; + 0 disables the stage pool and keeps inline staging (default: 0) + --dead-repo-blacklist + Enable the dead-repo transport blacklist persisted at this JSON + path (default: disabled). Blacklisted rrdp repos skip straight to + rsync; dual-blacklisted repos terminate instantly. + --dead-repo-blacklist-fail-threshold + Consecutive runs with transport-class fetch failure before a + (repo, transport) entry is blacklisted (default: 3) + + --rsync-local-dir Use LocalDirRsyncFetcher rooted at this directory (offline tests) + --disable-rrdp Disable RRDP and synchronize only via rsync + --rsync-command Use this rsync command instead of the default rsync binary + --http-timeout-secs HTTP fetch timeout seconds (default: 20) + --http-root-cert Extra PEM root certificate trusted by HTTPS fetches (repeatable) + --rsync-timeout-secs rsync I/O timeout seconds (default: 60) + --rsync-mirror-root Persist rsync mirrors under this directory (default: disabled) + --rsync-scope rsync scope policy: host, publication-point, or module-root (default: module-root) + --max-ca-depth Maximum CA depth from a trust anchor (root = 0, default: {DEFAULT_MAX_CA_DEPTH}) + --max-depth Deprecated alias for --max-ca-depth + --max-instances Max number of CA instances to process + --validation-time Validation time in RFC3339 (default: now UTC) + --analyze Write timing analysis JSON under target/live/analyze// + --analysis-out Write timing analysis JSON under this directory (implies --analyze) + --profile-cpu (Requires build feature 'profile') Write CPU flamegraph under analyze dir + + --help Show this help +" + ) +} + +pub fn parse_args(argv: &[String]) -> Result { + let mut validation_contract_out_path: Option = None; + let mut tal_urls: Vec = Vec::new(); + let mut tal_paths: Vec = Vec::new(); + let mut ta_paths: Vec = Vec::new(); + let mut ta_constraint_specs: Vec = Vec::new(); + let mut parallel_phase1_cfg = ParallelPhase1Config::default(); + let mut parallel_phase2_cfg = ParallelPhase2Config::default(); + let mut dead_repo_blacklist_path: Option = None; + let mut dead_repo_blacklist_fail_threshold: Option = None; + + let mut db_path: Option = None; + let mut raw_store_db: Option = None; + let mut repo_bytes_db: Option = None; + let mut policy_path: Option = None; + let mut strict_policy: Option = None; + let mut resource_validation_mode: Option = None; + let mut report_json_path: Option = None; + let mut report_json_compact: bool = false; + let mut skip_report_build: bool = false; + let mut skip_vcir_persist: bool = false; + let mut enable_roa_validation_cache: bool = false; + let mut enable_child_certificate_validation_cache: bool = false; + let mut publication_point_cache_observe_only: bool = false; + let mut crypto_signature_cache_observe_only: bool = false; + let mut enable_crypto_signature_cache: bool = false; + let mut enable_publication_point_validation_cache: bool = false; + let mut enable_transport_request_prefetch: bool = false; + let mut ccr_out_path: Option = None; + let mut vrps_csv_out_path: Option = None; + let mut vaps_csv_out_path: Option = None; + let mut compare_view_trust_anchor: Option = None; + let mut cir_enabled: bool = false; + let mut cir_out_path: Option = None; + let mut cir_static_root: Option = None; + let mut cir_tal_uris: Vec = Vec::new(); + let mut cir_tal_uri: Option = None; + let mut payload_replay_archive: Option = None; + let mut payload_replay_locks: Option = None; + let mut payload_base_archive: Option = None; + let mut payload_base_locks: Option = None; + let mut payload_base_validation_time: Option = None; + let mut payload_delta_archive: Option = None; + let mut payload_delta_locks: Option = None; + let mut memory_trim_after_validation = false; + + let mut rsync_local_dir: Option = None; + let mut disable_rrdp: bool = false; + let mut rsync_command: Option = None; + let mut http_timeout_secs: u64 = 30; + let mut http_root_cert_paths: Vec = Vec::new(); + let mut rsync_timeout_secs: u64 = 30; + let mut rsync_mirror_root: Option = None; + let mut rsync_scope_policy = RsyncScopePolicy::default(); + let mut max_ca_depth: Option = None; + let mut max_ca_depth_option: Option<&'static str> = None; + let mut max_instances: Option = None; + let mut validation_time: Option = None; + let mut analyze: bool = false; + let mut analysis_out_path: Option = None; + let mut profile_cpu: bool = false; + + let mut i = 1usize; + while i < argv.len() { + let arg = argv[i].as_str(); + match arg { + "--help" | "-h" => return Err(usage()), + "--validation-contract-out" => { + i += 1; + validation_contract_out_path = Some(PathBuf::from( + argv.get(i) + .ok_or("--validation-contract-out requires a value")?, + )); + } + "--tal-url" => { + i += 1; + let v = argv.get(i).ok_or("--tal-url requires a value")?; + tal_urls.push(v.clone()); + } + "--tal-path" => { + i += 1; + let v = argv.get(i).ok_or("--tal-path requires a value")?; + tal_paths.push(PathBuf::from(v)); + } + "--ta-path" => { + i += 1; + let v = argv.get(i).ok_or("--ta-path requires a value")?; + ta_paths.push(PathBuf::from(v)); + } + "--ta-constraints" => { + i += 1; + let v = argv + .get(i) + .ok_or("--ta-constraints requires =")?; + ta_constraint_specs.push(v.clone()); + } + "--parallel-max-repo-sync-workers-global" => { + i += 1; + let v = argv + .get(i) + .ok_or("--parallel-max-repo-sync-workers-global requires a value")?; + parallel_phase1_cfg.max_repo_sync_workers_global = v + .parse::() + .map_err(|_| format!("invalid --parallel-max-repo-sync-workers-global: {v}"))?; + } + "--parallel-max-inflight-snapshot-bytes-global" => { + i += 1; + let v = argv + .get(i) + .ok_or("--parallel-max-inflight-snapshot-bytes-global requires a value")?; + parallel_phase1_cfg.max_inflight_snapshot_bytes_global = + v.parse::().map_err(|_| { + format!("invalid --parallel-max-inflight-snapshot-bytes-global: {v}") + })?; + } + "--parallel-max-pending-repo-results" => { + i += 1; + let v = argv + .get(i) + .ok_or("--parallel-max-pending-repo-results requires a value")?; + parallel_phase1_cfg.max_pending_repo_results = v + .parse::() + .map_err(|_| format!("invalid --parallel-max-pending-repo-results: {v}"))?; + } + "--parallel-phase2-object-workers" => { + i += 1; + let v = argv + .get(i) + .ok_or("--parallel-phase2-object-workers requires a value")?; + parallel_phase2_cfg.object_workers = v + .parse::() + .map_err(|_| format!("invalid --parallel-phase2-object-workers: {v}"))?; + } + "--parallel-phase2-worker-queue-capacity" => { + i += 1; + let v = argv + .get(i) + .ok_or("--parallel-phase2-worker-queue-capacity requires a value")?; + parallel_phase2_cfg.worker_queue_capacity = v + .parse::() + .map_err(|_| format!("invalid --parallel-phase2-worker-queue-capacity: {v}"))?; + } + "--parallel-phase2-ready-batch-size" => { + i += 1; + let v = argv + .get(i) + .ok_or("--parallel-phase2-ready-batch-size requires a value")?; + parallel_phase2_cfg.ready_batch_size = v + .parse::() + .map_err(|_| format!("invalid --parallel-phase2-ready-batch-size: {v}"))?; + } + "--parallel-phase2-ready-batch-wall-time-budget-ms" => { + i += 1; + let v = argv + .get(i) + .ok_or("--parallel-phase2-ready-batch-wall-time-budget-ms requires a value")?; + parallel_phase2_cfg.ready_batch_wall_time_budget_ms = + v.parse::().map_err(|_| { + format!("invalid --parallel-phase2-ready-batch-wall-time-budget-ms: {v}") + })?; + } + "--parallel-phase2-result-drain-batch-size" => { + i += 1; + let v = argv + .get(i) + .ok_or("--parallel-phase2-result-drain-batch-size requires a value")?; + parallel_phase2_cfg.object_result_drain_batch_size = + v.parse::().map_err(|_| { + format!("invalid --parallel-phase2-result-drain-batch-size: {v}") + })?; + } + "--parallel-phase2-finalize-batch-size" => { + i += 1; + let v = argv + .get(i) + .ok_or("--parallel-phase2-finalize-batch-size requires a value")?; + parallel_phase2_cfg.publication_point_finalize_batch_size = v + .parse::() + .map_err(|_| format!("invalid --parallel-phase2-finalize-batch-size: {v}"))?; + } + "--parallel-phase2-finalize-batch-wall-time-budget-ms" => { + i += 1; + let v = argv.get(i).ok_or( + "--parallel-phase2-finalize-batch-wall-time-budget-ms requires a value", + )?; + parallel_phase2_cfg.publication_point_finalize_wall_time_budget_ms = + v.parse::().map_err(|_| { + format!("invalid --parallel-phase2-finalize-batch-wall-time-budget-ms: {v}") + })?; + } + "--parallel-phase2-finalize-queue-capacity" => { + i += 1; + let v = argv + .get(i) + .ok_or("--parallel-phase2-finalize-queue-capacity requires a value")?; + parallel_phase2_cfg.publication_point_finalize_queue_capacity = + v.parse::().map_err(|_| { + format!("invalid --parallel-phase2-finalize-queue-capacity: {v}") + })?; + } + "--control-plane-stage-workers" => { + i += 1; + let v = argv + .get(i) + .ok_or("--control-plane-stage-workers requires a value")?; + parallel_phase2_cfg.stage_workers = v + .parse::() + .map_err(|_| format!("invalid --control-plane-stage-workers: {v}"))?; + } + "--dead-repo-blacklist" => { + i += 1; + let v = argv + .get(i) + .ok_or("--dead-repo-blacklist requires a value")?; + dead_repo_blacklist_path = Some(PathBuf::from(v)); + } + "--dead-repo-blacklist-fail-threshold" => { + i += 1; + let v = argv + .get(i) + .ok_or("--dead-repo-blacklist-fail-threshold requires a value")?; + dead_repo_blacklist_fail_threshold = + Some(v.parse::().map_err(|_| { + format!("invalid --dead-repo-blacklist-fail-threshold: {v}") + })?); + } + "--db" => { + i += 1; + let v = argv.get(i).ok_or("--db requires a value")?; + db_path = Some(PathBuf::from(v)); + } + "--raw-store-db" => { + i += 1; + let v = argv.get(i).ok_or("--raw-store-db requires a value")?; + raw_store_db = Some(PathBuf::from(v)); + } + "--repo-bytes-db" => { + i += 1; + let v = argv.get(i).ok_or("--repo-bytes-db requires a value")?; + repo_bytes_db = Some(PathBuf::from(v)); + } + "--policy" => { + i += 1; + let v = argv.get(i).ok_or("--policy requires a value")?; + policy_path = Some(PathBuf::from(v)); + } + "--strict" => { + let next = argv.get(i + 1).map(String::as_str); + let spec = next.filter(|v| !v.starts_with("--")); + if spec.is_some() { + i += 1; + } + strict_policy = Some(StrictPolicy::parse_cli_spec(spec)?); + } + _ if arg.starts_with("--strict=") => { + let spec = arg.strip_prefix("--strict=").expect("prefix checked"); + strict_policy = Some(StrictPolicy::parse_cli_spec(Some(spec))?); + } + "--resource-validation-mode" => { + i += 1; + let v = argv + .get(i) + .ok_or("--resource-validation-mode requires a value")?; + resource_validation_mode = Some(ResourceValidationMode::parse_cli_value(v)?); + } + _ if arg.starts_with("--resource-validation-mode=") => { + let spec = arg + .strip_prefix("--resource-validation-mode=") + .expect("prefix checked"); + resource_validation_mode = Some(ResourceValidationMode::parse_cli_value(spec)?); + } + "--report-json" => { + i += 1; + let v = argv.get(i).ok_or("--report-json requires a value")?; + report_json_path = Some(PathBuf::from(v)); + } + "--report-json-compact" => { + report_json_compact = true; + } + "--skip-report-build" => { + skip_report_build = true; + } + "--skip-vcir-persist" => { + skip_vcir_persist = true; + } + "--enable-roa-validation-cache" => { + enable_roa_validation_cache = true; + } + "--enable-child-certificate-validation-cache" => { + enable_child_certificate_validation_cache = true; + } + "--publication-point-cache-observe-only" => { + publication_point_cache_observe_only = true; + } + "--enable-publication-point-validation-cache" => { + enable_publication_point_validation_cache = true; + } + "--crypto-signature-cache-observe-only" => { + crypto_signature_cache_observe_only = true; + } + "--enable-crypto-signature-cache" => { + enable_crypto_signature_cache = true; + } + "--enable-transport-request-prefetch" => { + enable_transport_request_prefetch = true; + } + "--ccr-out" => { + i += 1; + let v = argv.get(i).ok_or("--ccr-out requires a value")?; + ccr_out_path = Some(PathBuf::from(v)); + } + "--vrps-csv-out" => { + i += 1; + let v = argv.get(i).ok_or("--vrps-csv-out requires a value")?; + vrps_csv_out_path = Some(PathBuf::from(v)); + } + "--vaps-csv-out" => { + i += 1; + let v = argv.get(i).ok_or("--vaps-csv-out requires a value")?; + vaps_csv_out_path = Some(PathBuf::from(v)); + } + "--compare-view-trust-anchor" => { + i += 1; + let v = argv + .get(i) + .ok_or("--compare-view-trust-anchor requires a value")?; + compare_view_trust_anchor = Some(v.clone()); + } + "--cir-enable" => { + cir_enabled = true; + } + "--cir-out" => { + i += 1; + let v = argv.get(i).ok_or("--cir-out requires a value")?; + cir_out_path = Some(PathBuf::from(v)); + } + "--cir-static-root" => { + i += 1; + let v = argv.get(i).ok_or("--cir-static-root requires a value")?; + cir_static_root = Some(PathBuf::from(v)); + } + "--cir-tal-uri" => { + i += 1; + let v = argv.get(i).ok_or("--cir-tal-uri requires a value")?; + cir_tal_uris.push(v.clone()); + cir_tal_uri = cir_tal_uris.first().cloned(); + } + "--payload-replay-archive" => { + i += 1; + let v = argv + .get(i) + .ok_or("--payload-replay-archive requires a value")?; + payload_replay_archive = Some(PathBuf::from(v)); + } + "--payload-replay-locks" => { + i += 1; + let v = argv + .get(i) + .ok_or("--payload-replay-locks requires a value")?; + payload_replay_locks = Some(PathBuf::from(v)); + } + "--payload-base-archive" => { + i += 1; + let v = argv + .get(i) + .ok_or("--payload-base-archive requires a value")?; + payload_base_archive = Some(PathBuf::from(v)); + } + "--payload-base-locks" => { + i += 1; + let v = argv.get(i).ok_or("--payload-base-locks requires a value")?; + payload_base_locks = Some(PathBuf::from(v)); + } + "--payload-base-validation-time" => { + i += 1; + let v = argv + .get(i) + .ok_or("--payload-base-validation-time requires a value")?; + use time::format_description::well_known::Rfc3339; + let t = time::OffsetDateTime::parse(v, &Rfc3339).map_err(|e| { + format!("invalid --payload-base-validation-time (RFC3339 expected): {e}") + })?; + payload_base_validation_time = Some(t); + } + "--payload-delta-archive" => { + i += 1; + let v = argv + .get(i) + .ok_or("--payload-delta-archive requires a value")?; + payload_delta_archive = Some(PathBuf::from(v)); + } + "--payload-delta-locks" => { + i += 1; + let v = argv + .get(i) + .ok_or("--payload-delta-locks requires a value")?; + payload_delta_locks = Some(PathBuf::from(v)); + } + "--memory-trim-after-validation" => { + memory_trim_after_validation = true; + } + "--rsync-local-dir" => { + i += 1; + let v = argv.get(i).ok_or("--rsync-local-dir requires a value")?; + rsync_local_dir = Some(PathBuf::from(v)); + } + "--disable-rrdp" => { + disable_rrdp = true; + } + "--rsync-command" => { + i += 1; + let v = argv.get(i).ok_or("--rsync-command requires a value")?; + rsync_command = Some(PathBuf::from(v)); + } + "--http-timeout-secs" => { + i += 1; + let v = argv.get(i).ok_or("--http-timeout-secs requires a value")?; + http_timeout_secs = v + .parse::() + .map_err(|_| format!("invalid --http-timeout-secs: {v}"))?; + } + "--http-root-cert" => { + i += 1; + let v = argv.get(i).ok_or("--http-root-cert requires a value")?; + http_root_cert_paths.push(PathBuf::from(v)); + } + "--rsync-timeout-secs" => { + i += 1; + let v = argv.get(i).ok_or("--rsync-timeout-secs requires a value")?; + rsync_timeout_secs = v + .parse::() + .map_err(|_| format!("invalid --rsync-timeout-secs: {v}"))?; + } + "--rsync-mirror-root" => { + i += 1; + let v = argv.get(i).ok_or("--rsync-mirror-root requires a value")?; + rsync_mirror_root = Some(PathBuf::from(v)); + } + "--rsync-scope" => { + i += 1; + let v = argv.get(i).ok_or("--rsync-scope requires a value")?; + rsync_scope_policy = RsyncScopePolicy::parse_cli_value(v)?; + } + "--max-ca-depth" | "--max-depth" => { + i += 1; + let option = arg; + let v = argv + .get(i) + .ok_or_else(|| format!("{option} requires a value"))?; + if let Some(previous_option) = max_ca_depth_option { + return Err(format!( + "{option} cannot be combined with {previous_option}; use only --max-ca-depth" + )); + } + max_ca_depth = Some( + v.parse::() + .map_err(|_| format!("invalid {option}: {v}"))?, + ); + max_ca_depth_option = Some(match option { + "--max-ca-depth" => "--max-ca-depth", + "--max-depth" => "--max-depth", + _ => unreachable!("matched CA depth option"), + }); + } + "--max-instances" => { + i += 1; + let v = argv.get(i).ok_or("--max-instances requires a value")?; + max_instances = Some( + v.parse::() + .map_err(|_| format!("invalid --max-instances: {v}"))?, + ); + } + "--validation-time" => { + i += 1; + let v = argv.get(i).ok_or("--validation-time requires a value")?; + use time::format_description::well_known::Rfc3339; + let t = time::OffsetDateTime::parse(v, &Rfc3339) + .map_err(|e| format!("invalid --validation-time (RFC3339 expected): {e}"))?; + validation_time = Some(t); + } + "--analyze" => { + analyze = true; + } + "--analysis-out" => { + i += 1; + let v = argv.get(i).ok_or("--analysis-out requires a value")?; + analyze = true; + analysis_out_path = Some(PathBuf::from(v)); + } + "--profile-cpu" => { + profile_cpu = true; + } + _ => return Err(format!("unknown argument: {arg}\n\n{}", usage())), + } + i += 1; + } + + let db_path = db_path.ok_or_else(|| format!("--db is required\n\n{}", usage()))?; + + let tal_mode_count = (!tal_urls.is_empty()) as u8 + (!tal_paths.is_empty()) as u8; + if tal_mode_count != 1 { + return Err(format!( + "must specify either one-or-more --tal-url or one-or-more --tal-path/--ta-path pairs\n\n{}", + usage() + )); + } + if parallel_phase2_cfg.object_workers == 0 { + return Err(format!( + "--parallel-phase2-object-workers must be > 0\n\n{}", + usage() + )); + } + if parallel_phase2_cfg.worker_queue_capacity == 0 { + return Err(format!( + "--parallel-phase2-worker-queue-capacity must be > 0\n\n{}", + usage() + )); + } + if parallel_phase2_cfg.ready_batch_size == 0 { + return Err(format!( + "--parallel-phase2-ready-batch-size must be > 0\n\n{}", + usage() + )); + } + if parallel_phase2_cfg.ready_batch_wall_time_budget_ms == 0 { + return Err(format!( + "--parallel-phase2-ready-batch-wall-time-budget-ms must be > 0\n\n{}", + usage() + )); + } + if parallel_phase2_cfg.object_result_drain_batch_size == 0 { + return Err(format!( + "--parallel-phase2-result-drain-batch-size must be > 0\n\n{}", + usage() + )); + } + if parallel_phase2_cfg.publication_point_finalize_batch_size == 0 { + return Err(format!( + "--parallel-phase2-finalize-batch-size must be > 0\n\n{}", + usage() + )); + } + if parallel_phase2_cfg.publication_point_finalize_wall_time_budget_ms == 0 { + return Err(format!( + "--parallel-phase2-finalize-batch-wall-time-budget-ms must be > 0\n\n{}", + usage() + )); + } + if parallel_phase2_cfg.publication_point_finalize_queue_capacity == 0 { + return Err(format!( + "--parallel-phase2-finalize-queue-capacity must be > 0\n\n{}", + usage() + )); + } + if !tal_urls.is_empty() && !ta_paths.is_empty() { + return Err(format!( + "--ta-path cannot be used with --tal-url mode\n\n{}", + usage() + )); + } + if !tal_paths.is_empty() { + if !ta_paths.is_empty() { + if ta_paths.len() != tal_paths.len() { + return Err(format!( + "--tal-path and --ta-path counts must match in file mode\n\n{}", + usage() + )); + } + } else if ta_paths.is_empty() && !disable_rrdp { + return Err(format!( + "--tal-path requires --ta-path unless --disable-rrdp is set\n\n{}", + usage() + )); + } + } + let tal_url = tal_urls.first().cloned(); + let tal_path = tal_paths.first().cloned(); + let ta_path = ta_paths.first().cloned(); + if cir_enabled && cir_out_path.is_none() { + return Err(format!("--cir-enable requires --cir-out\n\n{}", usage())); + } + if report_json_compact && report_json_path.is_none() { + return Err(format!( + "--report-json-compact requires --report-json\n\n{}", + usage() + )); + } + if skip_report_build && report_json_path.is_some() { + return Err(format!( + "--skip-report-build cannot be combined with --report-json\n\n{}", + usage() + )); + } + if vrps_csv_out_path.is_some() != vaps_csv_out_path.is_some() { + return Err(format!( + "--vrps-csv-out and --vaps-csv-out must be provided together\n\n{}", + usage() + )); + } + if compare_view_trust_anchor.is_some() && vrps_csv_out_path.is_none() { + return Err(format!( + "--compare-view-trust-anchor requires --vrps-csv-out/--vaps-csv-out\n\n{}", + usage() + )); + } + if cir_static_root.is_some() { + return Err(format!( + "--cir-static-root is no longer supported; CIR export now writes only .cir files\n\n{}", + usage() + )); + } + if !cir_enabled && (cir_out_path.is_some() || !cir_tal_uris.is_empty()) { + return Err(format!( + "--cir-out/--cir-tal-uri require --cir-enable\n\n{}", + usage() + )); + } + if cir_enabled && !cir_tal_uris.is_empty() { + let expected = if !tal_paths.is_empty() { + tal_paths.len() + } else { + tal_urls.len() + }; + if cir_tal_uris.len() != expected { + return Err(format!( + "--cir-tal-uri count must match TAL input count when provided\n\n{}", + usage() + )); + } + } + if cir_enabled && !tal_paths.is_empty() && cir_tal_uris.is_empty() { + return Err(format!( + "CIR export in --tal-path mode requires --cir-tal-uri for each TAL\n\n{}", + usage() + )); + } + + let replay_mode_count = + payload_replay_archive.is_some() as u8 + payload_replay_locks.is_some() as u8; + if replay_mode_count == 1 { + return Err(format!( + "--payload-replay-archive and --payload-replay-locks must be provided together + +{}", + usage() + )); + } + + let delta_mode_count = payload_base_archive.is_some() as u8 + + payload_base_locks.is_some() as u8 + + payload_delta_archive.is_some() as u8 + + payload_delta_locks.is_some() as u8; + if delta_mode_count > 0 && delta_mode_count < 4 { + return Err(format!( + "--payload-base-archive, --payload-base-locks, --payload-delta-archive and --payload-delta-locks must be provided together + +{}", + usage() + )); + } + if replay_mode_count == 2 && delta_mode_count == 4 { + return Err(format!( + "snapshot replay mode and delta replay mode are mutually exclusive + +{}", + usage() + )); + } + if replay_mode_count == 2 { + if tal_url.is_some() { + return Err(format!( + "payload replay mode requires --tal-path and --ta-path; --tal-url is not supported + +{}", + usage() + )); + } + if tal_path.is_none() || ta_path.is_none() { + return Err(format!( + "payload replay mode requires --tal-path and --ta-path + +{}", + usage() + )); + } + if rsync_local_dir.is_some() { + return Err(format!( + "payload replay mode cannot be combined with --rsync-local-dir + +{}", + usage() + )); + } + } + if delta_mode_count == 4 { + if tal_url.is_some() { + return Err(format!( + "payload delta replay mode requires --tal-path and --ta-path; --tal-url is not supported + +{}", + usage() + )); + } + if tal_path.is_none() || ta_path.is_none() { + return Err(format!( + "payload delta replay mode requires --tal-path and --ta-path + +{}", + usage() + )); + } + if rsync_local_dir.is_some() { + return Err(format!( + "payload delta replay mode cannot be combined with --rsync-local-dir + +{}", + usage() + )); + } + } + + let mut tal_inputs = Vec::new(); + if !tal_urls.is_empty() { + tal_inputs.extend(tal_urls.iter().cloned().map(TalInputSpec::from_url)); + } else if !tal_paths.is_empty() { + if ta_paths.len() == tal_paths.len() { + tal_inputs.extend(tal_paths.iter().cloned().zip(ta_paths.iter().cloned()).map( + |(tal_path, ta_path)| TalInputSpec::from_file_path_with_ta(tal_path, ta_path), + )); + } else { + tal_inputs.extend(tal_paths.iter().cloned().map(TalInputSpec::from_file_path)); + } + } + let ta_constraints = TaConstraintsByTal::load_for_tals(&tal_inputs, &ta_constraint_specs)?; + + if dead_repo_blacklist_fail_threshold.is_some() && dead_repo_blacklist_path.is_none() { + return Err( + "--dead-repo-blacklist-fail-threshold requires --dead-repo-blacklist".to_string(), + ); + } + if let Some(path) = dead_repo_blacklist_path { + let fail_threshold = dead_repo_blacklist_fail_threshold.unwrap_or(3); + if fail_threshold == 0 { + return Err("--dead-repo-blacklist-fail-threshold must be >= 1".to_string()); + } + parallel_phase1_cfg.dead_repo_blacklist = Some( + crate::parallel::dead_repo_blacklist::DeadRepoBlacklistConfig { + path, + fail_threshold, + capacity: + crate::parallel::dead_repo_blacklist::DEAD_REPO_BLACKLIST_DEFAULT_CAPACITY, + }, + ); + } + + Ok(CliArgs { + validation_contract_out_path, + tal_urls, + tal_paths, + ta_paths, + tal_url, + tal_path, + ta_path, + parallel_phase1_config: parallel_phase1_cfg, + parallel_phase2_config: parallel_phase2_cfg, + tal_inputs, + ta_constraints, + db_path, + raw_store_db, + repo_bytes_db, + policy_path, + strict_policy, + resource_validation_mode, + report_json_path, + report_json_compact, + skip_report_build, + skip_vcir_persist, + enable_roa_validation_cache, + enable_child_certificate_validation_cache, + publication_point_cache_observe_only, + enable_publication_point_validation_cache, + crypto_signature_cache_observe_only, + enable_crypto_signature_cache, + enable_transport_request_prefetch, + ccr_out_path, + vrps_csv_out_path, + vaps_csv_out_path, + compare_view_trust_anchor, + cir_enabled, + cir_out_path, + cir_static_root, + cir_tal_uris, + cir_tal_uri, + payload_replay_archive, + payload_replay_locks, + payload_base_archive, + payload_base_locks, + payload_base_validation_time, + payload_delta_archive, + payload_delta_locks, + memory_trim_after_validation, + rsync_local_dir, + disable_rrdp, + rsync_command, + http_timeout_secs, + http_root_cert_paths, + rsync_timeout_secs, + rsync_mirror_root, + rsync_scope_policy, + max_ca_depth: max_ca_depth.unwrap_or(DEFAULT_MAX_CA_DEPTH), + max_instances, + validation_time, + analyze, + analysis_out_path, + profile_cpu, + }) +} + +fn read_policy(path: Option<&Path>) -> Result { + match path { + None => Ok(Policy::default()), + Some(p) => { + let s = std::fs::read_to_string(p) + .map_err(|e| format!("read policy file failed: {}: {e}", p.display()))?; + Policy::from_toml_str(&s).map_err(|e| e.to_string()) + } + } +} + +fn unique_rrdp_repos_from_publication_points( + publication_points: &[crate::audit::PublicationPointAudit], +) -> usize { + use std::collections::HashSet; + let mut set: HashSet<&str> = HashSet::new(); + for pp in publication_points { + if let Some(u) = pp.rrdp_notification_uri.as_deref() { + set.insert(u); + } + } + set.len() +} + +#[cfg(test)] +fn unique_rrdp_repos(report: &AuditReportV2) -> usize { + unique_rrdp_repos_from_publication_points(&report.publication_points) +} + +#[cfg(test)] +fn print_summary(report: &AuditReportV2) { + let rrdp_repos = unique_rrdp_repos(report); + println!("RPKI stage2 serial run summary"); + println!( + "validation_time={}", + report.meta.validation_time_rfc3339_utc + ); + println!( + "publication_points_processed={} publication_points_failed={}", + report.tree.instances_processed, report.tree.instances_failed + ); + println!("rrdp_repos_unique={rrdp_repos}"); + println!("vrps={}", report.vrps.len()); + println!("aspas={}", report.aspas.len()); + println!( + "audit_publication_points={}", + report.publication_points.len() + ); + println!( + "warnings_total={}", + report.tree.warnings.len() + + report + .publication_points + .iter() + .map(|pp| pp.warnings.len()) + .sum::() + ); +} + +fn print_summary_from_shared(validation_time: time::OffsetDateTime, shared: &PostValidationShared) { + use time::format_description::well_known::Rfc3339; + let validation_time_rfc3339_utc = validation_time + .to_offset(time::UtcOffset::UTC) + .format(&Rfc3339) + .expect("format validation_time"); + let rrdp_repos = unique_rrdp_repos_from_publication_points(shared.publication_points.as_ref()); + println!("RPKI stage2 serial run summary"); + println!("validation_time={validation_time_rfc3339_utc}"); + println!( + "publication_points_processed={} publication_points_failed={}", + shared.instances_processed, shared.instances_failed + ); + println!("rrdp_repos_unique={rrdp_repos}"); + println!("vrps={}", shared.vrps.len()); + println!("aspas={}", shared.aspas.len()); + println!( + "audit_publication_points={}", + shared.publication_points.len() + ); + println!( + "warnings_total={}", + shared.tree_warnings.len() + + shared + .publication_points + .iter() + .map(|pp| pp.warnings.len()) + .sum::() + ); +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct PostValidationShared { + discovery: crate::validation::from_tal::DiscoveredRootCaInstance, + discoveries: Arc<[crate::validation::from_tal::DiscoveredRootCaInstance]>, + successful_tal_inputs: Arc<[TalInputSpec]>, + instances_processed: usize, + instances_failed: usize, + tree_warnings: Arc<[crate::report::Warning]>, + vrps: Arc<[crate::validation::objects::Vrp]>, + aspas: Arc<[crate::validation::objects::AspaAttestation]>, + router_keys: Arc<[crate::validation::objects::RouterKeyPayload]>, + publication_points: Arc<[crate::audit::PublicationPointAudit]>, + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats, + downloads: Arc<[crate::audit::AuditDownloadEvent]>, + download_stats: crate::audit::AuditDownloadStats, + current_repo_objects: Arc<[crate::current_repo_index::CurrentRepoObject]>, + ccr_accumulator: Option, + cir_input: crate::cir::CirInputSnapshot, +} + +impl PostValidationShared { + fn from_run_output(out: RunTreeFromTalAuditOutput) -> Self { + let RunTreeFromTalAuditOutput { + discovery, + discoveries, + successful_tal_inputs, + tree, + publication_points, + roa_cache_stats, + downloads, + download_stats, + current_repo_objects, + ccr_accumulator, + cir_input, + } = out; + let crate::validation::tree::TreeRunOutput { + instances_processed, + instances_failed, + warnings, + vrps, + aspas, + router_keys, + } = tree; + + Self { + discovery, + discoveries: discoveries.into(), + successful_tal_inputs: successful_tal_inputs.into(), + instances_processed, + instances_failed, + tree_warnings: warnings.into(), + vrps: vrps.into(), + aspas: aspas.into(), + router_keys: router_keys.into(), + publication_points: publication_points.into(), + roa_cache_stats, + downloads: downloads.into(), + download_stats, + current_repo_objects: current_repo_objects.into(), + ccr_accumulator, + cir_input, + } + } + + fn trust_anchors(&self) -> Vec { + if self.discoveries.is_empty() { + vec![self.discovery.trust_anchor.clone()] + } else { + self.discoveries + .iter() + .map(|item| item.trust_anchor.clone()) + .collect() + } + } +} + +#[derive(Default)] +struct ObjectGraphSectionBuilder { + name: String, + item_count: u64, + shallow_bytes: u64, + heap_bytes: u64, + string_count: u64, + string_bytes: u64, + string_capacity_bytes: u64, + vec_count: u64, + vec_heap_bytes: u64, + vec_capacity_bytes: u64, + details: Vec, +} + +impl ObjectGraphSectionBuilder { + fn new(name: impl Into) -> Self { + Self { + name: name.into(), + ..Self::default() + } + } + + fn items(&mut self, count: usize, item_size: usize) { + self.item_count += count as u64; + self.shallow_bytes += (count as u64) * (item_size as u64); + } + + fn heap_bytes(&mut self, value: usize) { + self.heap_bytes += value as u64; + } + + fn string(&mut self, value: &str) { + self.string_count += 1; + self.string_bytes += value.len() as u64; + self.string_capacity_bytes += value.len() as u64; + self.heap_bytes += value.len() as u64; + } + + fn owned_string(&mut self, value: &String) { + self.string_count += 1; + self.string_bytes += value.len() as u64; + self.string_capacity_bytes += value.capacity() as u64; + self.heap_bytes += value.capacity() as u64; + } + + fn optional_string(&mut self, value: Option<&String>) { + if let Some(value) = value { + self.owned_string(value); + } + } + + fn vec_header_with_capacity(&mut self, len: usize, capacity: usize, element_size: usize) { + self.vec_count += 1; + let payload_bytes = len * element_size; + let capacity_bytes = capacity * element_size; + self.vec_heap_bytes += payload_bytes as u64; + self.vec_capacity_bytes += capacity_bytes as u64; + self.heap_bytes += capacity_bytes as u64; + } + + fn byte_vec_owned(&mut self, value: &Vec) { + self.vec_header_with_capacity(value.len(), value.capacity(), std::mem::size_of::()); + } + + fn string_vec_owned(&mut self, values: &Vec) { + self.vec_header_with_capacity( + values.len(), + values.capacity(), + std::mem::size_of::(), + ); + for value in values { + self.owned_string(value); + } + } + + fn metric(&mut self, name: impl Into, value: u64) { + self.details.push(ObjectGraphMemoryMetric { + name: name.into(), + value, + }); + } + + fn finish(self) -> ObjectGraphMemorySection { + let estimated_bytes = self.shallow_bytes + self.heap_bytes; + ObjectGraphMemorySection { + name: self.name, + item_count: self.item_count, + shallow_bytes: self.shallow_bytes, + heap_bytes: self.heap_bytes, + estimated_bytes, + string_count: self.string_count, + string_bytes: self.string_bytes, + string_capacity_bytes: self.string_capacity_bytes, + vec_count: self.vec_count, + vec_heap_bytes: self.vec_heap_bytes, + vec_capacity_bytes: self.vec_capacity_bytes, + details: self.details, + } + } +} + +fn estimate_shared_object_graph(shared: &PostValidationShared) -> ObjectGraphMemorySummary { + let mut sections = Vec::new(); + sections.push(estimate_publication_points_graph( + shared.publication_points.as_ref(), + )); + sections.push(estimate_vrps_graph(shared.vrps.as_ref())); + sections.push(estimate_aspas_graph(shared.aspas.as_ref())); + sections.push(estimate_router_keys_graph(shared.router_keys.as_ref())); + sections.push(estimate_warnings_graph( + "tree_warnings", + shared.tree_warnings.as_ref(), + )); + sections.push(estimate_downloads_graph(shared.downloads.as_ref())); + sections.push(estimate_current_repo_objects_graph( + shared.current_repo_objects.as_ref(), + )); + sections.push(estimate_trust_anchor_graph(shared)); + sections.push(estimate_ccr_accumulator_graph( + shared.ccr_accumulator.as_ref(), + )); + + let total_estimated_bytes = sections + .iter() + .map(|section| section.estimated_bytes) + .sum::(); + ObjectGraphMemorySummary { + captured_at_label: "after_validation".to_string(), + total_estimated_bytes, + sections, + notes: vec![ + "Estimated bytes are Rust object graph approximations based on struct sizes and owned String/Vec payload lengths.".to_string(), + "The estimate intentionally excludes allocator metadata, fragmentation, freed-but-retained arenas, RocksDB C++ heap, and transient worker allocations.".to_string(), + "Large RSS minus this estimate points to allocator retention or structures not yet modeled by this telemetry.".to_string(), + ], + } +} + +fn estimate_publication_points_graph( + publication_points: &[crate::audit::PublicationPointAudit], +) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new("publication_points"); + builder.items( + publication_points.len(), + std::mem::size_of::(), + ); + builder.metric("publication_point_count", publication_points.len() as u64); + let mut object_count = 0u64; + let mut pp_warning_count = 0u64; + let mut pp_discovered_from_count = 0u64; + let mut object_detail_count = 0u64; + + for pp in publication_points { + builder.owned_string(&pp.rsync_base_uri); + builder.owned_string(&pp.manifest_rsync_uri); + builder.owned_string(&pp.publication_point_rsync_uri); + builder.optional_string(pp.rrdp_notification_uri.as_ref()); + builder.owned_string(&pp.source); + builder.optional_string(pp.repo_sync_source.as_ref()); + builder.optional_string(pp.repo_sync_phase.as_ref()); + builder.optional_string(pp.repo_sync_error.as_ref()); + builder.owned_string(&pp.repo_terminal_state); + builder.owned_string(&pp.this_update_rfc3339_utc); + builder.owned_string(&pp.next_update_rfc3339_utc); + builder.owned_string(&pp.verified_at_rfc3339_utc); + + if let Some(discovered_from) = &pp.discovered_from { + pp_discovered_from_count += 1; + builder.heap_bytes(std::mem::size_of::()); + builder.owned_string(&discovered_from.parent_manifest_rsync_uri); + builder.owned_string(&discovered_from.child_ca_certificate_rsync_uri); + builder.owned_string(&discovered_from.child_ca_certificate_sha256_hex); + } + + pp_warning_count += pp.warnings.len() as u64; + builder.vec_header_with_capacity( + pp.warnings.len(), + pp.warnings.capacity(), + std::mem::size_of::(), + ); + for warning in &pp.warnings { + builder.owned_string(&warning.message); + builder.string_vec_owned(&warning.rfc_refs); + builder.optional_string(warning.context.as_ref()); + } + + object_count += pp.objects.len() as u64; + builder.vec_header_with_capacity( + pp.objects.len(), + pp.objects.capacity(), + std::mem::size_of::(), + ); + for object in &pp.objects { + builder.owned_string(&object.rsync_uri); + builder.owned_string(&object.sha256_hex); + if object.detail.is_some() { + object_detail_count += 1; + } + builder.optional_string(object.detail.as_ref()); + } + } + + builder.metric("object_audit_entry_count", object_count); + builder.metric("publication_point_warning_count", pp_warning_count); + builder.metric( + "publication_point_discovered_from_count", + pp_discovered_from_count, + ); + builder.metric("object_detail_count", object_detail_count); + builder.finish() +} + +fn estimate_vrps_graph(vrps: &[crate::validation::objects::Vrp]) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new("vrps"); + builder.items( + vrps.len(), + std::mem::size_of::(), + ); + builder.metric("vrp_count", vrps.len() as u64); + builder.finish() +} + +fn estimate_aspas_graph( + aspas: &[crate::validation::objects::AspaAttestation], +) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new("aspas"); + builder.items( + aspas.len(), + std::mem::size_of::(), + ); + let mut providers_total = 0u64; + for aspa in aspas { + providers_total += aspa.provider_as_ids.len() as u64; + builder.vec_header_with_capacity( + aspa.provider_as_ids.len(), + aspa.provider_as_ids.capacity(), + std::mem::size_of::(), + ); + } + builder.metric("aspa_count", aspas.len() as u64); + builder.metric("provider_asn_count", providers_total); + builder.finish() +} + +fn estimate_router_keys_graph( + router_keys: &[crate::validation::objects::RouterKeyPayload], +) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new("router_keys"); + builder.items( + router_keys.len(), + std::mem::size_of::(), + ); + for router_key in router_keys { + builder.byte_vec_owned(&router_key.ski); + builder.byte_vec_owned(&router_key.spki_der); + builder.owned_string(&router_key.source_object_uri); + builder.owned_string(&router_key.source_object_hash); + builder.owned_string(&router_key.source_ee_cert_hash); + } + builder.metric("router_key_count", router_keys.len() as u64); + builder.finish() +} + +fn estimate_warnings_graph( + name: &str, + warnings: &[crate::report::Warning], +) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new(name); + builder.items( + warnings.len(), + std::mem::size_of::(), + ); + for warning in warnings { + builder.owned_string(&warning.message); + builder.vec_header_with_capacity( + warning.rfc_refs.len(), + warning.rfc_refs.capacity(), + std::mem::size_of::(), + ); + builder.optional_string(warning.context.as_ref()); + } + builder.metric("warning_count", warnings.len() as u64); + builder.finish() +} + +fn estimate_downloads_graph( + downloads: &[crate::audit::AuditDownloadEvent], +) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new("downloads"); + builder.items( + downloads.len(), + std::mem::size_of::(), + ); + let mut error_count = 0u64; + let mut bytes_count = 0u64; + let mut objects_stat_count = 0u64; + for event in downloads { + builder.owned_string(&event.uri); + builder.owned_string(&event.started_at_rfc3339_utc); + builder.owned_string(&event.finished_at_rfc3339_utc); + if event.error.is_some() { + error_count += 1; + } + if event.bytes.is_some() { + bytes_count += 1; + } + if event.objects.is_some() { + objects_stat_count += 1; + } + builder.optional_string(event.error.as_ref()); + } + builder.metric("download_event_count", downloads.len() as u64); + builder.metric("download_error_count", error_count); + builder.metric("download_bytes_field_count", bytes_count); + builder.metric("download_objects_stat_count", objects_stat_count); + builder.finish() +} + +fn estimate_current_repo_objects_graph( + objects: &[crate::current_repo_index::CurrentRepoObject], +) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new("current_repo_objects"); + builder.items( + objects.len(), + std::mem::size_of::(), + ); + let mut object_type_count = 0u64; + for object in objects { + builder.owned_string(&object.rsync_uri); + builder.owned_string(&object.current_hash_hex); + builder.owned_string(&object.repository_source); + if object.object_type.is_some() { + object_type_count += 1; + } + builder.optional_string(object.object_type.as_ref()); + } + builder.metric("current_repo_object_count", objects.len() as u64); + builder.metric("current_repo_object_type_count", object_type_count); + builder.finish() +} + +fn estimate_trust_anchor_graph(shared: &PostValidationShared) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new("trust_anchors_and_tal_inputs"); + builder.items( + 1, + std::mem::size_of::(), + ); + estimate_discovered_root(&mut builder, &shared.discovery); + builder.items( + shared.discoveries.len(), + std::mem::size_of::(), + ); + for discovery in shared.discoveries.iter() { + estimate_discovered_root(&mut builder, discovery); + } + builder.items( + shared.successful_tal_inputs.len(), + std::mem::size_of::(), + ); + for tal_input in shared.successful_tal_inputs.iter() { + estimate_tal_input(&mut builder, tal_input); + } + builder.metric("discoveries_count", shared.discoveries.len() as u64); + builder.metric( + "successful_tal_inputs_count", + shared.successful_tal_inputs.len() as u64, + ); + builder.finish() +} + +fn estimate_discovered_root( + builder: &mut ObjectGraphSectionBuilder, + discovery: &crate::validation::from_tal::DiscoveredRootCaInstance, +) { + builder.optional_string(discovery.tal_url.as_ref()); + estimate_trust_anchor(builder, &discovery.trust_anchor); + builder.owned_string(&discovery.ca_instance.rsync_base_uri); + builder.owned_string(&discovery.ca_instance.manifest_rsync_uri); + builder.owned_string(&discovery.ca_instance.publication_point_rsync_uri); + builder.optional_string(discovery.ca_instance.rrdp_notification_uri.as_ref()); +} + +fn estimate_trust_anchor( + builder: &mut ObjectGraphSectionBuilder, + trust_anchor: &crate::data_model::ta::TrustAnchor, +) { + builder.byte_vec_owned(&trust_anchor.tal.raw); + builder.string_vec_owned(&trust_anchor.tal.comments); + builder.vec_header_with_capacity( + trust_anchor.tal.ta_uris.len(), + trust_anchor.tal.ta_uris.capacity(), + std::mem::size_of::(), + ); + for uri in &trust_anchor.tal.ta_uris { + builder.string(uri.as_str()); + } + builder.byte_vec_owned(&trust_anchor.tal.subject_public_key_info_der); + builder.byte_vec_owned(&trust_anchor.ta_certificate.raw_der); + if let Some(uri) = &trust_anchor.resolved_ta_uri { + builder.string(uri.as_str()); + } +} + +fn estimate_tal_input(builder: &mut ObjectGraphSectionBuilder, tal_input: &TalInputSpec) { + builder.owned_string(&tal_input.tal_id); + builder.owned_string(&tal_input.rir_id); + match &tal_input.source { + crate::parallel::types::TalSource::Url(url) => builder.owned_string(url), + crate::parallel::types::TalSource::DerBytes { + tal_url, + tal_bytes, + ta_der, + } => { + builder.owned_string(tal_url); + builder.byte_vec_owned(tal_bytes); + builder.byte_vec_owned(ta_der); + } + crate::parallel::types::TalSource::FilePath(path) => { + builder.string(&path.to_string_lossy()); + } + crate::parallel::types::TalSource::FilePathWithTa { tal_path, ta_path } => { + builder.string(&tal_path.to_string_lossy()); + builder.string(&ta_path.to_string_lossy()); + } + } +} + +fn estimate_ccr_accumulator_graph( + accumulator: Option<&CcrAccumulator>, +) -> ObjectGraphMemorySection { + let mut builder = ObjectGraphSectionBuilder::new("ccr_accumulator"); + if let Some(accumulator) = accumulator { + builder.items(1, std::mem::size_of::()); + let stats = accumulator.memory_stats(); + builder.heap_bytes(stats.estimated_heap_bytes as usize); + builder.metric("trust_anchor_count", stats.trust_anchor_count); + builder.metric("manifest_count", stats.manifest_count); + builder.metric("string_bytes", stats.string_bytes); + builder.metric("string_capacity_bytes", stats.string_capacity_bytes); + builder.metric("vec_payload_bytes", stats.vec_payload_bytes); + builder.metric("vec_capacity_bytes", stats.vec_capacity_bytes); + builder.metric("locations_der_count", stats.locations_der_count); + builder.metric("subordinate_ski_count", stats.subordinate_ski_count); + builder.metric("btree_key_capacity_bytes", stats.btree_key_capacity_bytes); + builder.metric("btree_entry_shallow_bytes", stats.btree_entry_shallow_bytes); + } else { + builder.metric("manifest_count", 0); + } + builder.finish() +} + +#[cfg(test)] +fn build_report( + policy: &Policy, + validation_time: time::OffsetDateTime, + shared: &PostValidationShared, +) -> AuditReportV2 { + use time::format_description::well_known::Rfc3339; + let validation_time_rfc3339_utc = validation_time + .to_offset(time::UtcOffset::UTC) + .format(&Rfc3339) + .expect("format validation_time"); + + let vrps = shared + .vrps + .iter() + .map(|v| VrpOutput { + asn: v.asn, + prefix: format_roa_ip_prefix(&v.prefix), + max_length: v.max_length, + }) + .collect::>(); + + let aspas = shared + .aspas + .iter() + .map(|a| AspaOutput { + customer_as_id: a.customer_as_id, + provider_as_ids: a.provider_as_ids.clone(), + }) + .collect::>(); + + let repo_sync_stats = build_repo_sync_stats(shared.publication_points.as_ref()); + + AuditReportV2 { + format_version: 2, + meta: AuditRunMeta { + validation_time_rfc3339_utc, + }, + policy: policy.clone(), + tree: TreeSummary { + instances_processed: shared.instances_processed, + instances_failed: shared.instances_failed, + warnings: shared + .tree_warnings + .iter() + .map(AuditWarning::from) + .collect(), + }, + publication_points: shared.publication_points.iter().cloned().collect(), + vrps, + aspas, + downloads: shared.downloads.iter().cloned().collect(), + download_stats: shared.download_stats.clone(), + repo_sync_stats, + query_audit: None, + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct ReportTaskOutput { + report_build_ms: u64, + report_write_ms: Option, +} + +impl ReportTaskOutput { + fn skipped() -> Self { + Self { + report_build_ms: 0, + report_write_ms: None, + } + } +} + +fn run_report_task( + policy: &Policy, + validation_time: time::OffsetDateTime, + shared: &PostValidationShared, + report_json_path: Option<&Path>, + report_json_format: ReportJsonFormat, +) -> Result { + if let Some(path) = report_json_path { + let timing = write_report_json_from_shared( + path, + policy, + validation_time, + shared, + report_json_format, + )?; + Ok(ReportTaskOutput { + report_build_ms: timing.build_ms, + report_write_ms: Some(timing.write_ms), + }) + } else { + Ok(ReportTaskOutput::skipped()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct CcrTaskOutput { + ccr_build_ms: Option, + ccr_build_breakdown: Option, + ccr_write_ms: Option, +} + +fn run_ccr_task( + store: &RocksStore, + shared: &PostValidationShared, + ccr_out_path: Option<&Path>, + produced_at: time::OffsetDateTime, +) -> Result { + let mut ccr_build_ms = None; + let mut ccr_build_breakdown = None; + let mut ccr_write_ms = None; + if let Some(path) = ccr_out_path { + let started = std::time::Instant::now(); + let (ccr, build_breakdown) = if let Some(accumulator) = shared.ccr_accumulator.as_ref() { + ( + accumulator + .finish( + produced_at, + shared.vrps.as_ref(), + shared.aspas.as_ref(), + shared.router_keys.as_ref(), + ) + .map_err(|e| e.to_string())?, + None, + ) + } else { + let trust_anchors = shared.trust_anchors(); + let (ccr, build_breakdown) = build_ccr_from_run_with_breakdown( + store, + &trust_anchors, + shared.vrps.as_ref(), + shared.aspas.as_ref(), + shared.router_keys.as_ref(), + produced_at, + ) + .map_err(|e| e.to_string())?; + (ccr, Some(build_breakdown)) + }; + ccr_build_ms = Some(started.elapsed().as_millis() as u64); + ccr_build_breakdown = build_breakdown; + let started = std::time::Instant::now(); + write_ccr_file(path, &ccr).map_err(|e| e.to_string())?; + ccr_write_ms = Some(started.elapsed().as_millis() as u64); + eprintln!("wrote CCR: {}", path.display()); + } + + Ok(CcrTaskOutput { + ccr_build_ms, + ccr_build_breakdown, + ccr_write_ms, + }) +} + +fn resolve_cir_export_tal_uris(args: &CliArgs) -> Result, String> { + if !args.cir_tal_uris.is_empty() { + return Ok(args.cir_tal_uris.clone()); + } + if !args.tal_urls.is_empty() { + return Ok(args.tal_urls.clone()); + } + Err("CIR export requires TAL URI source(s)".to_string()) +} + +fn effective_cir_tal_uris_for_discoveries( + args: &CliArgs, + shared: &PostValidationShared, + cir_tal_uris: Vec, +) -> Result, String> { + if shared.successful_tal_inputs.is_empty() { + return Ok(cir_tal_uris); + } + if cir_tal_uris.len() == shared.discoveries.len() { + return Ok(cir_tal_uris); + } + if cir_tal_uris.len() != args.tal_inputs.len() { + return Ok(cir_tal_uris); + } + + let mut mapped = Vec::with_capacity(shared.successful_tal_inputs.len()); + for successful in shared.successful_tal_inputs.iter() { + let input_index = args + .tal_inputs + .iter() + .position(|candidate| candidate == successful) + .ok_or_else(|| { + format!( + "successful TAL '{}' was not found in original TAL input list", + successful.tal_id + ) + })?; + mapped.push(cir_tal_uris[input_index].clone()); + } + Ok(mapped) +} + +fn build_repo_sync_stats( + publication_points: &[crate::audit::PublicationPointAudit], +) -> AuditRepoSyncStats { + let mut stats = AuditRepoSyncStats { + publication_points_total: publication_points.len() as u64, + ..AuditRepoSyncStats::default() + }; + + for pp in publication_points { + let duration = pp.repo_sync_duration_ms.unwrap_or(0); + if let Some(phase) = pp.repo_sync_phase.as_ref() { + let entry = stats.by_phase.entry(phase.clone()).or_default(); + entry.count += 1; + entry.duration_ms_total += duration; + } + let entry = stats + .by_terminal_state + .entry(pp.repo_terminal_state.clone()) + .or_default(); + entry.count += 1; + entry.duration_ms_total += duration; + } + + stats +} + +fn run_online_validation_with_fetchers( + store: Arc, + policy: &Policy, + args: &CliArgs, + http: &H, + rsync: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + collect_current_repo_objects: bool, + timing: Option<&TimingHandle>, +) -> Result +where + H: crate::sync::rrdp::Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + // The multi-TAL entry point preserves the TAL id supplied by the CLI. + // A single-file TAL may otherwise derive its id from the embedded TA URI, + // which is intentionally different from the local filename used for + // adjacent .constraints discovery. + if args.tal_inputs.len() > 1 || !policy.ta_constraints.is_empty() { + return if let Some(t) = timing { + run_tree_from_multiple_tals_parallel_phase2_audit_with_timing( + store, + policy, + args.tal_inputs.clone(), + http, + rsync, + validation_time, + config, + args.parallel_phase1_config.clone(), + args.parallel_phase2_config.clone(), + collect_current_repo_objects, + t, + ) + } else { + run_tree_from_multiple_tals_parallel_phase2_audit( + store, + policy, + args.tal_inputs.clone(), + http, + rsync, + validation_time, + config, + args.parallel_phase1_config.clone(), + args.parallel_phase2_config.clone(), + collect_current_repo_objects, + ) + } + .map_err(|e| e.to_string()); + } + + match ( + args.tal_url.as_ref(), + args.tal_path.as_ref(), + args.ta_path.as_ref(), + ) { + (Some(url), _, _) => if let Some(t) = timing { + run_tree_from_tal_url_parallel_phase2_audit_with_timing( + store, + policy, + url, + http, + rsync, + validation_time, + config, + args.parallel_phase1_config.clone(), + args.parallel_phase2_config.clone(), + collect_current_repo_objects, + t, + ) + } else { + run_tree_from_tal_url_parallel_phase2_audit( + store, + policy, + url, + http, + rsync, + validation_time, + config, + args.parallel_phase1_config.clone(), + args.parallel_phase2_config.clone(), + collect_current_repo_objects, + ) + } + .map_err(|e| e.to_string()), + (None, Some(tal_path), Some(ta_path)) => { + let tal_bytes = std::fs::read(tal_path) + .map_err(|e| format!("read tal failed: {}: {e}", tal_path.display()))?; + let ta_der = std::fs::read(ta_path) + .map_err(|e| format!("read ta failed: {}: {e}", ta_path.display()))?; + if let Some(t) = timing { + run_tree_from_tal_and_ta_der_parallel_phase2_audit_with_timing( + store, + policy, + &tal_bytes, + &ta_der, + None, + http, + rsync, + validation_time, + config, + args.parallel_phase1_config.clone(), + args.parallel_phase2_config.clone(), + collect_current_repo_objects, + t, + ) + } else { + run_tree_from_tal_and_ta_der_parallel_phase2_audit( + store, + policy, + &tal_bytes, + &ta_der, + None, + http, + rsync, + validation_time, + config, + args.parallel_phase1_config.clone(), + args.parallel_phase2_config.clone(), + collect_current_repo_objects, + ) + } + .map_err(|e| e.to_string()) + } + (None, Some(tal_path), None) => { + let tal_bytes = std::fs::read(tal_path) + .map_err(|e| format!("read tal failed: {}: {e}", tal_path.display()))?; + let tal_uri = args.cir_tal_uri.clone(); + if let Some(t) = timing { + crate::validation::run_tree_from_tal::run_tree_from_tal_bytes_serial_audit_with_timing( + store.as_ref(), + policy, + &tal_bytes, + tal_uri, + http, + rsync, + validation_time, + config, + t, + ) + .map_err(|e| e.to_string()) + } else { + crate::validation::run_tree_from_tal::run_tree_from_tal_bytes_serial_audit( + store.as_ref(), + policy, + &tal_bytes, + tal_uri, + http, + rsync, + validation_time, + config, + ) + .map_err(|e| e.to_string()) + } + } + _ => unreachable!("validated by parse_args"), + } +} + +pub fn run(argv: &[String]) -> Result<(), String> { + let mut args = parse_args(argv)?; + let mut policy = read_policy(args.policy_path.as_deref())?; + if let Some(strict_policy) = args.strict_policy { + policy.strict = strict_policy; + } + if let Some(resource_validation_mode) = args.resource_validation_mode { + policy.resource_validation_mode = resource_validation_mode; + } + if args.disable_rrdp { + policy.sync_preference = crate::policy::SyncPreference::RsyncOnly; + } + policy.ta_constraints = args.ta_constraints.clone(); + for warning in policy.ta_constraints.configuration_warnings() { + eprintln!("warning: {warning}"); + } + let validation_time = args + .validation_time + .unwrap_or_else(time::OffsetDateTime::now_utc); + let validation_time = + time::OffsetDateTime::from_unix_timestamp(validation_time.unix_timestamp()) + .map_err(|error| format!("normalize validation time failed: {error}"))?; + let http_root_certificates_pem = args + .http_root_cert_paths + .iter() + .map(|path| { + std::fs::read(path) + .map_err(|e| format!("read HTTP root certificate failed: {}: {e}", path.display())) + }) + .collect::, _>>()?; + + let store = if args.raw_store_db.is_some() || args.repo_bytes_db.is_some() { + Arc::new( + RocksStore::open_with_external_stores( + &args.db_path, + args.raw_store_db.as_deref(), + args.repo_bytes_db.as_deref(), + ) + .map_err(|e| e.to_string())?, + ) + } else { + Arc::new(RocksStore::open(&args.db_path).map_err(|e| e.to_string())?) + }; + let config = TreeRunConfig { + max_depth: Some(args.max_ca_depth), + max_instances: args.max_instances, + compact_audit: args.skip_report_build + && args.report_json_path.is_none() + && !args.cir_enabled, + persist_vcir: !args.skip_vcir_persist, + build_ccr_accumulator: args.ccr_out_path.is_some(), + enable_roa_validation_cache: args.enable_roa_validation_cache, + enable_child_certificate_validation_cache: args.enable_child_certificate_validation_cache, + publication_point_cache_observe_only: args.publication_point_cache_observe_only, + enable_publication_point_validation_cache: args.enable_publication_point_validation_cache, + enable_transport_request_prefetch: args.enable_transport_request_prefetch, + }; + let replay_mode = args.payload_replay_archive.is_some(); + let delta_replay_mode = args.payload_base_archive.is_some(); + + use time::format_description::well_known::Rfc3339; + let mut timing: Option<(std::path::PathBuf, TimingHandle)> = None; + if args.analyze { + let recorded_at_utc_rfc3339 = time::OffsetDateTime::now_utc() + .to_offset(time::UtcOffset::UTC) + .format(&Rfc3339) + .map_err(|e| format!("format recorded_at_utc failed: {e}"))?; + let validation_time_utc_rfc3339 = validation_time + .to_offset(time::UtcOffset::UTC) + .format(&Rfc3339) + .map_err(|e| format!("format validation_time failed: {e}"))?; + + let ts_compact = { + let fmt = time::format_description::parse("[year][month][day]T[hour][minute][second]Z") + .map_err(|e| format!("format description parse failed: {e}"))?; + time::OffsetDateTime::now_utc() + .format(&fmt) + .map_err(|e| format!("format timestamp failed: {e}"))? + }; + + let out_dir = args.analysis_out_path.clone().unwrap_or_else(|| { + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("target") + .join("live") + .join("analyze") + .join(ts_compact) + }); + std::fs::create_dir_all(&out_dir) + .map_err(|e| format!("create analyze out dir failed: {}: {e}", out_dir.display()))?; + + let handle = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339, + validation_time_utc_rfc3339, + tal_url: None, + db_path: None, + }); + handle.set_meta(TimingMetaUpdate { + tal_url: args.tal_url.as_deref(), + db_path: Some(args.db_path.to_string_lossy().as_ref()), + }); + timing = Some((out_dir, handle)); + } + + if args.profile_cpu && !args.analyze { + return Err("--profile-cpu requires --analyze".to_string()); + } + + #[cfg(not(feature = "profile"))] + if args.profile_cpu { + return Err("CPU profiling requires building with: --features profile".to_string()); + } + + #[cfg(feature = "profile")] + let mut profiler_guard: Option> = if args.profile_cpu { + Some( + pprof::ProfilerGuard::new(100) + .map_err(|e| format!("pprof ProfilerGuard init failed: {e}"))?, + ) + } else { + None + }; + + let total_started = std::time::Instant::now(); + let mut memory_checkpoints: Vec = Vec::new(); + let mut malloc_trim_probes: Vec = Vec::new(); + let enable_memory_trim_probe = memory_trim_probe_enabled() || args.memory_trim_after_validation; + record_memory_checkpoint( + &mut memory_checkpoints, + "after_store_open", + &total_started, + store.as_ref(), + ); + let validation_started = std::time::Instant::now(); + let crypto_sig_cache = + if args.crypto_signature_cache_observe_only || args.enable_crypto_signature_cache { + let cache_file = crate::crypto_sig_cache::default_cache_file_path(&args.db_path); + let cache = Arc::new(crate::crypto_sig_cache::CryptoSigCache::load_or_rebuild( + cache_file, + args.enable_crypto_signature_cache, + )); + crate::crypto_sig_cache::install_global(Arc::clone(&cache)); + Some(cache) + } else { + None + }; + let collect_current_repo_objects = false; + let out = if delta_replay_mode { + let tal_path = args + .tal_path + .as_ref() + .expect("validated by parse_args for delta replay mode"); + let ta_path = args + .ta_path + .as_ref() + .expect("validated by parse_args for delta replay mode"); + let base_archive = args + .payload_base_archive + .as_ref() + .expect("validated by parse_args for delta replay mode"); + let base_locks = args + .payload_base_locks + .as_ref() + .expect("validated by parse_args for delta replay mode"); + let base_validation_time = args.payload_base_validation_time.unwrap_or(validation_time); + let delta_archive = args + .payload_delta_archive + .as_ref() + .expect("validated by parse_args for delta replay mode"); + let delta_locks = args + .payload_delta_locks + .as_ref() + .expect("validated by parse_args for delta replay mode"); + let tal_bytes = std::fs::read(tal_path) + .map_err(|e| format!("read tal failed: {}: {e}", tal_path.display()))?; + let ta_der = std::fs::read(ta_path) + .map_err(|e| format!("read ta failed: {}: {e}", ta_path.display()))?; + if let Some((_, t)) = timing.as_ref() { + run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit_with_timing( + store.as_ref(), + &policy, + &tal_bytes, + &ta_der, + None, + base_archive, + base_locks, + delta_archive, + delta_locks, + base_validation_time, + validation_time, + &config, + t, + ) + .map_err(|e| e.to_string())? + } else { + run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit( + store.as_ref(), + &policy, + &tal_bytes, + &ta_der, + None, + base_archive, + base_locks, + delta_archive, + delta_locks, + base_validation_time, + validation_time, + &config, + ) + .map_err(|e| e.to_string())? + } + } else if replay_mode { + let tal_path = args + .tal_path + .as_ref() + .expect("validated by parse_args for replay mode"); + let ta_path = args + .ta_path + .as_ref() + .expect("validated by parse_args for replay mode"); + let archive_root = args + .payload_replay_archive + .as_ref() + .expect("validated by parse_args for replay mode"); + let locks_path = args + .payload_replay_locks + .as_ref() + .expect("validated by parse_args for replay mode"); + let tal_bytes = std::fs::read(tal_path) + .map_err(|e| format!("read tal failed: {}: {e}", tal_path.display()))?; + let ta_der = std::fs::read(ta_path) + .map_err(|e| format!("read ta failed: {}: {e}", ta_path.display()))?; + if let Some((_, t)) = timing.as_ref() { + run_tree_from_tal_and_ta_der_payload_replay_serial_audit_with_timing( + store.as_ref(), + &policy, + &tal_bytes, + &ta_der, + None, + archive_root, + locks_path, + validation_time, + &config, + t, + ) + .map_err(|e| e.to_string())? + } else { + run_tree_from_tal_and_ta_der_payload_replay_serial_audit( + store.as_ref(), + &policy, + &tal_bytes, + &ta_der, + None, + archive_root, + locks_path, + validation_time, + &config, + ) + .map_err(|e| e.to_string())? + } + } else if let Some(dir) = args.rsync_local_dir.as_ref() { + let http = BlockingHttpFetcher::new(HttpFetcherConfig { + timeout: std::time::Duration::from_secs(args.http_timeout_secs.max(1)), + extra_root_certificates_pem: http_root_certificates_pem.clone(), + ..HttpFetcherConfig::default() + }) + .map_err(|e| e.to_string())?; + let rsync = LocalDirRsyncFetcher::new(dir); + run_online_validation_with_fetchers( + Arc::clone(&store), + &policy, + &args, + &http, + &rsync, + validation_time, + &config, + collect_current_repo_objects, + timing.as_ref().map(|(_, t)| t), + )? + } else { + let http = BlockingHttpFetcher::new(HttpFetcherConfig { + timeout: std::time::Duration::from_secs(args.http_timeout_secs.max(1)), + extra_root_certificates_pem: http_root_certificates_pem.clone(), + ..HttpFetcherConfig::default() + }) + .map_err(|e| e.to_string())?; + let rsync = SystemRsyncFetcher::new(SystemRsyncConfig { + rsync_bin: args + .rsync_command + .clone() + .unwrap_or_else(|| PathBuf::from("rsync")), + timeout: std::time::Duration::from_secs(args.rsync_timeout_secs.max(1)), + mirror_root: args.rsync_mirror_root.clone(), + scope_policy: args.rsync_scope_policy, + ..SystemRsyncConfig::default() + }); + run_online_validation_with_fetchers( + Arc::clone(&store), + &policy, + &args, + &http, + &rsync, + validation_time, + &config, + collect_current_repo_objects, + timing.as_ref().map(|(_, t)| t), + )? + }; + + let validation_ms = validation_started.elapsed().as_millis() as u64; + let mut shared = PostValidationShared::from_run_output(out); + let vcir_storage_summary_enabled = vcir_storage_summary_enabled(); + let vcir_storage_summary_started = std::time::Instant::now(); + let vcir_storage = if config.persist_vcir && vcir_storage_summary_enabled { + Some( + store + .summarize_vcir_storage() + .map_err(|e| format!("summarize VCIR storage failed: {e}"))?, + ) + } else { + None + }; + let vcir_storage_summary_ms = (config.persist_vcir && vcir_storage_summary_enabled) + .then(|| vcir_storage_summary_started.elapsed().as_millis() as u64); + record_memory_checkpoint( + &mut memory_checkpoints, + "after_validation", + &total_started, + store.as_ref(), + ); + if enable_memory_trim_probe { + malloc_trim_probes.push(crate::memory_telemetry::malloc_trim_probe()); + record_memory_checkpoint( + &mut memory_checkpoints, + "after_validation_malloc_trim", + &total_started, + store.as_ref(), + ); + } + + if let Some((_out_dir, t)) = timing.as_ref() { + t.record_count("instances_processed", shared.instances_processed as u64); + t.record_count("instances_failed", shared.instances_failed as u64); + } + + let publication_points = shared.publication_points.len(); + let publication_point_repo_sync_ms_total: u64 = shared + .publication_points + .iter() + .map(|pp| pp.repo_sync_duration_ms.unwrap_or(0)) + .sum(); + let download_event_count = shared.download_stats.events_total; + let rrdp_download_ms_total: u64 = ["rrdp_notification", "rrdp_snapshot", "rrdp_delta"] + .iter() + .map(|key| { + shared + .download_stats + .by_kind + .get(*key) + .map(|item| item.duration_ms_total) + .unwrap_or(0) + }) + .sum(); + let rsync_download_ms_total = shared + .download_stats + .by_kind + .get("rsync") + .map(|item| item.duration_ms_total) + .unwrap_or(0); + let repo_sync_ms_total = rrdp_download_ms_total + rsync_download_ms_total; + let download_bytes_total: u64 = shared + .download_stats + .by_kind + .values() + .map(|item| item.bytes_total.unwrap_or(0)) + .sum(); + + #[cfg(feature = "profile")] + let profiler_report = if let Some(guard) = profiler_guard.take() { + Some( + guard + .report() + .build() + .map_err(|e| format!("pprof report build failed: {e}"))?, + ) + } else { + None + }; + + let report_json_format = if args.report_json_compact { + ReportJsonFormat::Compact + } else { + ReportJsonFormat::Pretty + }; + let ccr_produced_at = time::OffsetDateTime::now_utc(); + let compare_view_trust_anchor = args + .compare_view_trust_anchor + .as_deref() + .unwrap_or("unknown"); + let cir_tal_uris = if args.cir_enabled { + Some(effective_cir_tal_uris_for_discoveries( + &args, + &shared, + resolve_cir_export_tal_uris(&args)?, + )?) + } else { + None + }; + let cir_out_path = if args.cir_enabled { + Some( + args.cir_out_path + .as_deref() + .expect("validated by parse_args for cir"), + ) + } else { + None + }; + // Take the CIR input snapshot before the output stage so the CIR export can + // run inside the same scoped-thread group as report/ccr/compare_view; no + // other output task reads `shared.cir_input`. + let cir_input_owned = args + .cir_enabled + .then(|| std::mem::take(&mut shared.cir_input)); + let (report_result, ccr_result, compare_view_result, cir_result) = + std::thread::scope(|scope| { + // Reborrow `shared` as a plain reference so the scoped output tasks + // (incl. the `move` CIR task) capture the reference instead of + // moving fields out of the owned value. + let shared = &shared; + let report_handle = if args.skip_report_build { + None + } else { + Some(scope.spawn(|| { + run_report_task( + &policy, + validation_time, + shared, + args.report_json_path.as_deref(), + report_json_format, + ) + })) + }; + let ccr_handle = scope.spawn(|| { + run_ccr_task( + store.as_ref(), + shared, + args.ccr_out_path.as_deref(), + ccr_produced_at, + ) + }); + let compare_view_handle = scope.spawn(|| { + run_compare_view_task( + shared, + args.vrps_csv_out_path.as_deref(), + args.vaps_csv_out_path.as_deref(), + compare_view_trust_anchor, + ) + }); + let cir_handle = match (cir_tal_uris.as_ref(), cir_out_path, cir_input_owned) { + (Some(cir_tal_uris), Some(cir_out_path), Some(cir_input)) => { + Some(scope.spawn(move || { + if cir_tal_uris.len() != shared.discoveries.len() { + return Err(format!( + "CIR export TAL URI count ({}) does not match discovery count ({})", + cir_tal_uris.len(), + shared.discoveries.len() + )); + } + let tal_bindings = shared + .discoveries + .iter() + .zip(cir_tal_uris.iter()) + .map(|(discovery, tal_uri)| CirTrustAnchorBinding { + trust_anchor: &discovery.trust_anchor, + tal_uri: tal_uri.as_str(), + }) + .collect::>(); + export_cir_from_input_snapshot_multi( + &tal_bindings, + validation_time, + cir_input, + cir_out_path, + ) + .map_err(|e| e.to_string()) + })) + } + _ => None, + }; + let report_result = match report_handle { + Some(handle) => handle + .join() + .map_err(|_| "report task panicked".to_string()) + .and_then(|result| result), + None => Ok(ReportTaskOutput::skipped()), + }; + let ccr_result = ccr_handle + .join() + .map_err(|_| "ccr task panicked".to_string()) + .and_then(|result| result); + let compare_view_result = compare_view_handle + .join() + .map_err(|_| "compare view task panicked".to_string()) + .and_then(|result| result); + let cir_result = cir_handle.map(|handle| { + handle + .join() + .map_err(|_| "cir task panicked".to_string()) + .and_then(|result| result) + }); + (report_result, ccr_result, compare_view_result, cir_result) + }); + let report_output = report_result?; + let ccr_output = ccr_result?; + let compare_view_output = compare_view_result?; + let cir_summary = match cir_result { + Some(result) => Some(result?), + None => None, + }; + record_memory_checkpoint( + &mut memory_checkpoints, + "after_report_and_ccr", + &total_started, + store.as_ref(), + ); + if enable_memory_trim_probe { + malloc_trim_probes.push(crate::memory_telemetry::malloc_trim_probe()); + record_memory_checkpoint( + &mut memory_checkpoints, + "after_report_and_ccr_malloc_trim", + &total_started, + store.as_ref(), + ); + } + let report_build_ms = report_output.report_build_ms; + let report_write_ms = report_output.report_write_ms; + let ccr_build_ms = ccr_output.ccr_build_ms; + let ccr_build_breakdown = ccr_output.ccr_build_breakdown; + let ccr_write_ms = ccr_output.ccr_write_ms; + let compare_view_build_ms = compare_view_output.build_ms; + let compare_view_write_ms = compare_view_output.write_ms; + record_memory_checkpoint( + &mut memory_checkpoints, + "after_compare_view", + &total_started, + store.as_ref(), + ); + + let mut cir_build_cir_ms = None; + let mut cir_write_cir_ms = None; + let mut cir_total_ms = None; + if let Some(summary) = cir_summary { + cir_build_cir_ms = Some(summary.timing.build_cir_ms); + cir_write_cir_ms = Some(summary.timing.write_cir_ms); + cir_total_ms = Some(summary.timing.total_ms); + eprintln!( + "wrote CIR: {} (objects={}, trust_anchors={}, build_cir_ms={}, write_cir_ms={}, total_ms={})", + cir_out_path + .expect("cir path present when cir enabled") + .display(), + summary.object_count, + summary.trust_anchor_count, + summary.timing.build_cir_ms, + summary.timing.write_cir_ms, + summary.timing.total_ms + ); + record_memory_checkpoint( + &mut memory_checkpoints, + "after_cir", + &total_started, + store.as_ref(), + ); + } + record_memory_checkpoint( + &mut memory_checkpoints, + "before_stage_timing", + &total_started, + store.as_ref(), + ); + let publication_point_cache_index_refresh = if args.enable_publication_point_validation_cache + || args.publication_point_cache_observe_only + { + match store.refresh_publication_point_cache_mmap_index() { + Ok(stats) => stats, + Err(e) => { + crate::progress_log::emit( + "publication_point_cache_mmap_index_refresh", + serde_json::json!({ + "state": "failed", + "error": e.to_string(), + }), + ); + None + } + } + } else { + None + }; + let publication_point_cache_index_load = store.publication_point_cache_mmap_index_load_stats(); + let crypto_signature_cache_observe = crypto_sig_cache.as_ref().map(|cache| { + if let Err(e) = cache.persist() { + crate::progress_log::emit( + "crypto_signature_cache_persist", + serde_json::json!({ + "state": "failed", + "error": e, + }), + ); + } + crate::crypto_sig_cache::clear_global(); + cache.summary() + }); + if let (Some((_, t)), Some(summary)) = + (timing.as_ref(), crypto_signature_cache_observe.as_ref()) + { + let (mut calls, mut would_hit, mut new_keys, mut executed, mut skipped) = + (0u64, 0u64, 0u64, 0u64, 0u64); + for stats in summary.per_point.values() { + calls += stats.calls; + would_hit += stats.would_hit; + new_keys += stats.new_keys; + executed += stats.verify_executed; + skipped += stats.verify_skipped; + } + t.record_count("crypto_signature_cache_observe_calls", calls); + t.record_count("crypto_signature_cache_observe_would_hit", would_hit); + t.record_count("crypto_signature_cache_observe_new_keys", new_keys); + t.record_count("crypto_signature_cache_verify_executed", executed); + t.record_count("crypto_signature_cache_verify_skipped", skipped); + } + let timing_report_snapshot = timing + .as_ref() + .map(|(_, handle)| handle.report_snapshot(50)); + let stage_timing = RunStageTiming { + validation_ms, + enable_roa_validation_cache: args.enable_roa_validation_cache, + enable_child_certificate_validation_cache: args.enable_child_certificate_validation_cache, + publication_point_cache_observe_only: args.publication_point_cache_observe_only, + enable_publication_point_validation_cache: args.enable_publication_point_validation_cache, + crypto_signature_cache_observe, + enable_transport_request_prefetch: args.enable_transport_request_prefetch, + report_build_ms, + report_write_ms, + ccr_build_ms, + ccr_build_breakdown, + ccr_write_ms, + compare_view_build_ms, + compare_view_write_ms, + cir_build_cir_ms, + cir_write_cir_ms, + cir_total_ms, + total_ms: total_started.elapsed().as_millis() as u64, + publication_points, + repo_sync_ms_total, + publication_point_repo_sync_ms_total, + download_event_count, + rrdp_download_ms_total, + rsync_download_ms_total, + download_bytes_total, + roa_validation_cache: shared.roa_cache_stats.clone(), + analysis_counts: timing + .as_ref() + .map(|(_, handle)| handle.counts_snapshot()) + .unwrap_or_default(), + analysis_phases: timing_report_snapshot + .as_ref() + .map(|report| report.phases.clone()) + .unwrap_or_default(), + analysis_top_publication_points: timing_report_snapshot + .as_ref() + .map(|report| report.top_publication_points.clone()) + .unwrap_or_default(), + analysis_top_publication_point_steps: timing_report_snapshot + .as_ref() + .map(|report| report.top_publication_point_steps.clone()) + .unwrap_or_default(), + analysis_top_publication_point_cache_steps: timing_report_snapshot + .as_ref() + .map(|report| { + report + .top_publication_point_steps + .iter() + .filter(|entry| entry.key.contains("::publication_point_cache_")) + .cloned() + .collect() + }) + .unwrap_or_default(), + vcir_storage_summary_ms, + vcir_storage, + publication_point_cache_index_load, + publication_point_cache_index_refresh, + memory_telemetry: Some(MemoryTelemetrySummary { + checkpoints: memory_checkpoints, + object_graph: Some(estimate_shared_object_graph(&shared)), + malloc_trim_probes, + }), + }; + let stage_timing_anchor_path = args + .report_json_path + .as_deref() + .or(args.ccr_out_path.as_deref()) + .or(args.vrps_csv_out_path.as_deref()); + write_stage_timing(stage_timing_anchor_path, &stage_timing)?; + + // Finalize the normal-run contract only after all validation outputs have + // been written successfully. This prevents incomplete runs from leaving a + // contract that looks usable to downstream replay tooling. + if let Some(path) = args.validation_contract_out_path.as_deref() { + let mut contract = crate::contract::ValidationContract::for_current_binary( + validation_time, + policy.clone(), + args.max_ca_depth, + args.max_instances, + crate::contract::ValidationCacheContract { + publication_point: args.enable_publication_point_validation_cache, + roa: args.enable_roa_validation_cache, + child_certificate: args.enable_child_certificate_validation_cache, + transport_prefetch: args.enable_transport_request_prefetch, + crypto_signature: args.enable_crypto_signature_cache, + }, + args.rsync_scope_policy, + )?; + if !policy.ta_constraints.is_empty() { + contract.ta_constraints_fingerprint = + Some(policy.ta_constraints.fingerprint_sha256_hex()); + } + crate::contract::write_validation_contract(path, &contract)?; + } + + if let Some((out_dir, t)) = timing.as_ref() { + t.record_count("vrps", shared.vrps.len() as u64); + t.record_count("aspas", shared.aspas.len() as u64); + t.record_count( + "audit_publication_points", + shared.publication_points.len() as u64, + ); + let timing_json_path = out_dir.join("timing.json"); + t.write_json(&timing_json_path, 20)?; + eprintln!("analysis: wrote {}", timing_json_path.display()); + } + + #[cfg(feature = "profile")] + if let (Some((out_dir, _)), Some(report)) = (timing.as_ref(), profiler_report) { + let svg_path = out_dir.join("flamegraph.svg"); + let svg_file = std::fs::File::create(&svg_path) + .map_err(|e| format!("create flamegraph failed: {}: {e}", svg_path.display()))?; + report + .flamegraph(svg_file) + .map_err(|e| format!("write flamegraph failed: {e}"))?; + eprintln!("analysis: wrote {}", svg_path.display()); + + let pb_path = out_dir.join("pprof.pb.gz"); + let pprof_profile = report + .pprof() + .map_err(|e| format!("pprof export failed: {e}"))?; + use pprof::protos::Message; + let mut body = Vec::with_capacity(pprof_profile.encoded_len()); + pprof_profile + .encode(&mut body) + .map_err(|e| format!("pprof encode failed: {e}"))?; + let gz = flate2::write::GzEncoder::new( + std::fs::File::create(&pb_path) + .map_err(|e| format!("create pprof.pb.gz failed: {}: {e}", pb_path.display()))?, + flate2::Compression::default(), + ); + let mut gz = gz; + use std::io::Write; + gz.write_all(&body) + .map_err(|e| format!("write pprof.pb.gz failed: {e}"))?; + gz.finish() + .map_err(|e| format!("finish pprof.pb.gz failed: {e}"))?; + eprintln!("analysis: wrote {}", pb_path.display()); + } + + print_summary_from_shared(validation_time, &shared); + Ok(()) +} + +#[cfg(test)] +#[path = "cli/tests.rs"] +mod tests; diff --git a/crates/panda-rpki-validator/src/cli/output.rs b/crates/panda-rpki-validator/src/cli/output.rs new file mode 100644 index 0000000..872a2bd --- /dev/null +++ b/crates/panda-rpki-validator/src/cli/output.rs @@ -0,0 +1,419 @@ +use std::io::BufWriter; +use std::path::Path; + +use serde::Serialize; +use serde::ser::SerializeSeq; +use sha2::Digest; + +use crate::audit::{ + AspaOutput, AuditRunMeta, AuditWarning, QueryAuditManifest, ValidationEvent, + ValidationEventCounts, VrpOutput, +}; +use crate::ccr::canonical_vrp_prefix; + +use super::{PostValidationShared, RunStageTiming}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum ReportJsonFormat { + Pretty, + Compact, +} + +pub(super) fn write_json( + path: &Path, + report: &T, + format: ReportJsonFormat, +) -> Result<(), String> { + let f = std::fs::File::create(path) + .map_err(|e| format!("create report file failed: {}: {e}", path.display()))?; + let writer = BufWriter::new(f); + match format { + ReportJsonFormat::Pretty => serde_json::to_writer_pretty(writer, report), + ReportJsonFormat::Compact => serde_json::to_writer(writer, report), + } + .map_err(|e| format!("write report json failed: {e}"))?; + Ok(()) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct ReportJsonWriteTiming { + pub(super) build_ms: u64, + pub(super) write_ms: u64, +} + +#[derive(Serialize)] +struct BorrowedAuditReportV2<'a> { + format_version: u32, + meta: AuditRunMeta, + policy: &'a crate::policy::Policy, + tree: BorrowedTreeSummary<'a>, + publication_points: &'a [crate::audit::PublicationPointAudit], + vrps: VrpReportSequence<'a>, + aspas: AspaReportSequence<'a>, + downloads: &'a [crate::audit::AuditDownloadEvent], + download_stats: &'a crate::audit::AuditDownloadStats, + repo_sync_stats: crate::audit::AuditRepoSyncStats, + #[serde(rename = "queryAudit", skip_serializing_if = "Option::is_none")] + query_audit: Option, +} + +#[derive(Serialize)] +struct BorrowedTreeSummary<'a> { + instances_processed: usize, + instances_failed: usize, + warnings: WarningReportSequence<'a>, +} + +struct WarningReportSequence<'a>(&'a [crate::report::Warning]); + +impl Serialize for WarningReportSequence<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let mut seq = serializer.serialize_seq(Some(self.0.len()))?; + for warning in self.0 { + seq.serialize_element(&AuditWarning::from(warning))?; + } + seq.end() + } +} + +struct VrpReportSequence<'a>(&'a [crate::validation::objects::Vrp]); + +impl Serialize for VrpReportSequence<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let mut seq = serializer.serialize_seq(Some(self.0.len()))?; + for vrp in self.0 { + seq.serialize_element(&VrpOutput { + asn: vrp.asn, + prefix: crate::audit::format_roa_ip_prefix(&vrp.prefix), + max_length: vrp.max_length, + })?; + } + seq.end() + } +} + +struct AspaReportSequence<'a>(&'a [crate::validation::objects::AspaAttestation]); + +impl Serialize for AspaReportSequence<'_> { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let mut seq = serializer.serialize_seq(Some(self.0.len()))?; + for aspa in self.0 { + seq.serialize_element(&AspaOutput { + customer_as_id: aspa.customer_as_id, + provider_as_ids: aspa.provider_as_ids.clone(), + })?; + } + seq.end() + } +} + +pub(super) fn write_report_json_from_shared( + path: &Path, + policy: &crate::policy::Policy, + validation_time: time::OffsetDateTime, + shared: &PostValidationShared, + format: ReportJsonFormat, +) -> Result { + use time::format_description::well_known::Rfc3339; + + let build_started = std::time::Instant::now(); + let validation_time_rfc3339_utc = validation_time + .to_offset(time::UtcOffset::UTC) + .format(&Rfc3339) + .expect("format validation_time"); + let repo_sync_stats = super::build_repo_sync_stats(shared.publication_points.as_ref()); + let query_audit = write_validation_events_sidecar(path, &validation_time_rfc3339_utc, shared)?; + let report = BorrowedAuditReportV2 { + format_version: 2, + meta: AuditRunMeta { + validation_time_rfc3339_utc, + }, + policy, + tree: BorrowedTreeSummary { + instances_processed: shared.instances_processed, + instances_failed: shared.instances_failed, + warnings: WarningReportSequence(shared.tree_warnings.as_ref()), + }, + publication_points: shared.publication_points.as_ref(), + vrps: VrpReportSequence(shared.vrps.as_ref()), + aspas: AspaReportSequence(shared.aspas.as_ref()), + downloads: shared.downloads.as_ref(), + download_stats: &shared.download_stats, + repo_sync_stats, + query_audit: Some(query_audit), + }; + let build_ms = build_started.elapsed().as_millis() as u64; + + let write_started = std::time::Instant::now(); + write_json(path, &report, format)?; + Ok(ReportJsonWriteTiming { + build_ms, + write_ms: write_started.elapsed().as_millis() as u64, + }) +} + +fn write_validation_events_sidecar( + report_path: &Path, + validation_time: &str, + shared: &PostValidationShared, +) -> Result { + let events_path = report_path.with_file_name("validation-events.jsonl"); + if let Some(parent) = events_path.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| format!("create validation events parent failed: {e}"))?; + } + let mut writer = BufWriter::new(std::fs::File::create(&events_path).map_err(|e| { + format!( + "create validation events failed: {}: {e}", + events_path.display() + ) + })?); + let mut seq = 0u64; + let mut hasher = sha2::Sha256::new(); + emit_validation_events(validation_time, shared, &mut seq, &mut |event| { + let mut line = serde_json::to_vec(&event) + .map_err(|e| format!("serialize validation event failed: {e}"))?; + line.push(b'\n'); + std::io::Write::write_all(&mut writer, &line) + .map_err(|e| format!("write validation event failed: {e}"))?; + hasher.update(&line); + Ok(()) + })?; + std::io::Write::flush(&mut writer) + .map_err(|e| format!("flush validation events failed: {e}"))?; + let events_count = seq; + let events_sha256 = hex::encode(hasher.finalize()); + Ok(QueryAuditManifest { + schema_version: 1, + status: "complete".to_string(), + events_path: events_path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("validation-events.jsonl") + .to_string(), + events_count, + events_sha256, + writer_version: 1, + error: None, + }) +} + +fn emit_validation_events( + validation_time: &str, + shared: &PostValidationShared, + seq: &mut u64, + emit: &mut impl FnMut(ValidationEvent) -> Result<(), String>, +) -> Result<(), String> { + emit(next_event(seq, "run_summary", validation_time, |event| { + event.counts = Some(ValidationEventCounts { + objects: Some( + shared + .publication_points + .iter() + .map(|pp| pp.objects.len() as u64) + .sum(), + ), + warnings: Some( + (shared.tree_warnings.len() + + shared + .publication_points + .iter() + .map(|pp| pp.warnings.len()) + .sum::()) as u64, + ), + vrps: Some(shared.vrps.len() as u64), + aspas: Some(shared.aspas.len() as u64), + }); + }))?; + for pp in shared.publication_points.iter() { + emit(next_event( + seq, + "publication_point", + validation_time, + |event| { + event.pp_node_id = pp.node_id; + event.pp_manifest_uri = Some(pp.manifest_rsync_uri.clone()); + event.pp_rsync_base_uri = Some(pp.rsync_base_uri.clone()); + event.repo_sync_phase = pp.repo_sync_phase.clone(); + event.repo_terminal_state = Some(pp.repo_terminal_state.clone()); + event.counts = Some(ValidationEventCounts { + objects: Some(pp.objects.len() as u64), + warnings: Some(pp.warnings.len() as u64), + vrps: None, + aspas: None, + }); + }, + ))?; + for object in &pp.objects { + emit(next_event(seq, "object", validation_time, |event| { + event.pp_node_id = pp.node_id; + event.pp_manifest_uri = Some(pp.manifest_rsync_uri.clone()); + event.object_uri = Some(object.rsync_uri.clone()); + event.sha256 = Some(object.sha256_hex.clone()); + event.object_type = Some(object.kind.clone()); + event.result = Some(object.result.clone()); + event.reason = object.detail.clone(); + }))?; + } + for warning in &pp.warnings { + emit(next_event(seq, "warning", validation_time, |event| { + event.pp_node_id = pp.node_id; + event.pp_manifest_uri = Some(pp.manifest_rsync_uri.clone()); + event.reason = Some(warning.message.clone()); + }))?; + } + } + Ok(()) +} + +fn next_event( + seq: &mut u64, + event_type: &str, + validation_time: &str, + fill: impl FnOnce(&mut ValidationEvent), +) -> ValidationEvent { + *seq += 1; + let mut event = ValidationEvent { + schema_version: 1, + seq: *seq, + event_type: event_type.to_string(), + validation_time: validation_time.to_string(), + pp_node_id: None, + pp_manifest_uri: None, + pp_rsync_base_uri: None, + repo_sync_phase: None, + repo_terminal_state: None, + object_uri: None, + sha256: None, + object_type: None, + result: None, + reason: None, + counts: None, + }; + fill(&mut event); + event +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct CompareViewTaskOutput { + pub(super) build_ms: Option, + pub(super) write_ms: Option, +} + +pub(super) fn run_compare_view_task( + shared: &PostValidationShared, + vrps_csv_out_path: Option<&Path>, + vaps_csv_out_path: Option<&Path>, + trust_anchor: &str, +) -> Result { + let mut build_ms = None; + let mut write_ms = None; + if let (Some(vrps_path), Some(vaps_path)) = (vrps_csv_out_path, vaps_csv_out_path) { + let started = std::time::Instant::now(); + build_ms = Some(0); + write_direct_vrp_csv(vrps_path, shared.vrps.as_ref(), trust_anchor)?; + write_direct_vap_csv(vaps_path, shared.aspas.as_ref(), trust_anchor)?; + write_ms = Some(started.elapsed().as_millis() as u64); + eprintln!( + "wrote compare views: vrps={} vaps={}", + vrps_path.display(), + vaps_path.display() + ); + } + Ok(CompareViewTaskOutput { build_ms, write_ms }) +} + +fn write_direct_vrp_csv( + path: &Path, + vrps: &[crate::validation::objects::Vrp], + trust_anchor: &str, +) -> Result<(), String> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| format!("create parent dirs failed: {}: {e}", parent.display()))?; + } + let file = std::fs::File::create(path) + .map_err(|e| format!("create file failed: {}: {e}", path.display()))?; + let mut writer = BufWriter::new(file); + use std::io::Write; + let trust_anchor = trust_anchor.to_ascii_lowercase(); + writeln!(writer, "ASN,IP Prefix,Max Length,Trust Anchor").map_err(|e| e.to_string())?; + for vrp in vrps { + writeln!( + writer, + "AS{},{},{},{}", + vrp.asn, + canonical_vrp_prefix(&vrp.prefix), + vrp.max_length, + trust_anchor + ) + .map_err(|e| e.to_string())?; + } + Ok(()) +} + +fn write_direct_vap_csv( + path: &Path, + aspas: &[crate::validation::objects::AspaAttestation], + trust_anchor: &str, +) -> Result<(), String> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| format!("create parent dirs failed: {}: {e}", parent.display()))?; + } + let file = std::fs::File::create(path) + .map_err(|e| format!("create file failed: {}: {e}", path.display()))?; + let mut writer = BufWriter::new(file); + use std::io::Write; + let trust_anchor = trust_anchor.to_ascii_lowercase(); + writeln!(writer, "Customer ASN,Providers,Trust Anchor").map_err(|e| e.to_string())?; + for aspa in aspas { + let mut providers = aspa.provider_as_ids.clone(); + providers.sort_unstable(); + providers.dedup(); + let providers = providers + .into_iter() + .map(|asn| format!("AS{asn}")) + .collect::>() + .join(";"); + writeln!( + writer, + "AS{},{},{}", + aspa.customer_as_id, providers, trust_anchor + ) + .map_err(|e| e.to_string())?; + } + Ok(()) +} + +pub(super) fn write_stage_timing( + report_json_path: Option<&Path>, + stage_timing: &RunStageTiming, +) -> Result<(), String> { + if let Some(path) = report_json_path + && let Some(parent) = path.parent() + { + let stage_timing_path = parent.join("stage-timing.json"); + std::fs::write( + &stage_timing_path, + serde_json::to_vec_pretty(stage_timing).map_err(|e| e.to_string())?, + ) + .map_err(|e| { + format!( + "write stage timing failed: {}: {e}", + stage_timing_path.display() + ) + })?; + eprintln!("analysis: wrote {}", stage_timing_path.display()); + } + Ok(()) +} diff --git a/crates/panda-rpki-validator/src/cli/tests.rs b/crates/panda-rpki-validator/src/cli/tests.rs new file mode 100644 index 0000000..5a7f702 --- /dev/null +++ b/crates/panda-rpki-validator/src/cli/tests.rs @@ -0,0 +1,2217 @@ +use super::*; +use crate::data_model::oid::OID_AD_SIGNED_OBJECT; +use crate::data_model::rc::AccessDescription; +use crate::memory_telemetry::{ + MemoryTelemetryCheckpoint, MemoryTelemetrySummary, ProcessMemorySnapshot, +}; +use crate::policy::ResourceValidationMode; +use crate::storage::{ + RocksDbMemorySnapshot, RocksDbMemoryTotals, VcirCcrProjectionSizeBreakdown, + VcirChildResourceSizeBreakdown, VcirCoreFieldSizeBreakdown, VcirFieldSizeBreakdown, + VcirStorageEntrySummary, VcirStorageSummary, +}; + +#[test] +fn parse_help_returns_usage() { + let argv = vec!["rpki".to_string(), "--help".to_string()]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("Usage:"), "{err}"); + assert!(err.contains("--db"), "{err}"); + assert!(err.contains("--rsync-mirror-root"), "{err}"); + assert!(err.contains("--rsync-scope"), "{err}"); + assert!(err.contains("--parallel-phase2-object-workers"), "{err}"); + assert!(err.contains("--memory-trim-after-validation"), "{err}"); + assert!(err.contains("--enable-roa-validation-cache"), "{err}"); + assert!(err.contains("--resource-validation-mode"), "{err}"); + assert!(err.contains("--ta-constraints"), "{err}"); + assert!(err.contains("--max-ca-depth"), "{err}"); + assert!(err.contains("default: 32"), "{err}"); + assert!( + err.contains("--publication-point-cache-observe-only"), + "{err}" + ); + assert!( + err.contains("--enable-publication-point-validation-cache"), + "{err}" + ); + assert!(!err.contains("--parallel-phase1"), "{err}"); + assert!(!err.contains("--parallel-phase2 "), "{err}"); +} + +#[test] +fn parse_accepts_explicit_ta_constraints_for_known_tal() { + let dir = tempfile::tempdir().expect("tmpdir"); + let constraints_path = dir.path().join("example.constraints"); + std::fs::write(&constraints_path, "allow 192.0.2.0/24\n").expect("write constraints"); + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "example.tal".to_string(), + "--ta-path".to_string(), + "example.cer".to_string(), + "--ta-constraints".to_string(), + format!("example={}", constraints_path.display()), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(!args.ta_constraints.is_empty()); +} + +#[test] +fn parse_rejects_unknown_argument() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--nope".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("unknown argument"), "{err}"); +} + +#[test] +fn parse_accepts_normal_validation_contract_output() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--validation-contract-out".to_string(), + "out/validation-contract.json".to_string(), + ]; + let args = parse_args(&argv).expect("parse normal contract output"); + assert_eq!( + args.validation_contract_out_path, + Some(PathBuf::from("out/validation-contract.json")) + ); +} + +#[test] +fn parse_rejects_both_tal_url_and_tal_path() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!( + err.contains("one-or-more --tal-url or one-or-more --tal-path/--ta-path pairs"), + "{err}" + ); +} + +#[test] +fn parse_rejects_invalid_max_depth() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--max-depth".to_string(), + "nope".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("invalid --max-depth"), "{err}"); +} + +#[test] +fn parse_rejects_invalid_max_ca_depth() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--max-ca-depth".to_string(), + "nope".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("invalid --max-ca-depth"), "{err}"); +} + +#[test] +fn parse_rejects_both_ca_depth_options() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--max-ca-depth".to_string(), + "32".to_string(), + "--max-depth".to_string(), + "12".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("cannot be combined"), "{err}"); +} + +#[test] +fn parse_defaults_max_ca_depth_to_32() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!(args.max_ca_depth, 32); +} + +#[test] +fn parse_accepts_ccr_out_path() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--rsync-local-dir".to_string(), + "repo".to_string(), + "--ccr-out".to_string(), + "out/example.ccr".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!( + args.ccr_out_path.as_deref(), + Some(std::path::Path::new("out/example.ccr")) + ); +} + +#[test] +fn parse_accepts_memory_trim_after_validation() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--memory-trim-after-validation".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.memory_trim_after_validation); +} + +#[test] +fn parse_disables_memory_trim_after_validation_by_default() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(!args.memory_trim_after_validation); +} + +#[test] +fn parse_accepts_enable_roa_validation_cache() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--enable-roa-validation-cache".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.enable_roa_validation_cache); +} + +#[test] +fn parse_accepts_enable_child_certificate_validation_cache() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--enable-child-certificate-validation-cache".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.enable_child_certificate_validation_cache); +} + +#[test] +fn parse_accepts_publication_point_cache_flags() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--publication-point-cache-observe-only".to_string(), + "--enable-publication-point-validation-cache".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.publication_point_cache_observe_only); + assert!(args.enable_publication_point_validation_cache); +} + +#[test] +fn parse_disables_crypto_signature_cache_observe_only_by_default() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(!args.crypto_signature_cache_observe_only); + assert!(!args.enable_crypto_signature_cache); +} + +#[test] +fn parse_accepts_crypto_signature_cache_observe_only() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--crypto-signature-cache-observe-only".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.crypto_signature_cache_observe_only); + assert!(!args.enable_crypto_signature_cache); +} + +#[test] +fn parse_accepts_enable_crypto_signature_cache() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--enable-crypto-signature-cache".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.enable_crypto_signature_cache); + assert!(!args.crypto_signature_cache_observe_only); +} + +#[test] +fn parse_accepts_enable_transport_request_prefetch() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--enable-transport-request-prefetch".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.enable_transport_request_prefetch); +} + +#[test] +fn parse_disables_roa_validation_cache_by_default() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(!args.enable_roa_validation_cache); + assert!(!args.enable_child_certificate_validation_cache); + assert!(!args.publication_point_cache_observe_only); + assert!(!args.enable_publication_point_validation_cache); + assert!(!args.enable_transport_request_prefetch); +} + +#[test] +fn parse_accepts_analysis_out_and_implies_analyze() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--analysis-out".to_string(), + "run/analyze".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.analyze); + assert_eq!( + args.analysis_out_path.as_deref(), + Some(std::path::Path::new("run/analyze")) + ); +} + +#[test] +fn parse_accepts_report_json_compact_when_report_json_is_set() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--report-json".to_string(), + "out/report.json".to_string(), + "--report-json-compact".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!( + args.report_json_path.as_deref(), + Some(std::path::Path::new("out/report.json")) + ); + assert!(args.report_json_compact); +} + +#[test] +fn parse_accepts_rsync_scope_policy() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--rsync-scope".to_string(), + "module-root".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!(args.rsync_scope_policy, RsyncScopePolicy::ModuleRoot); +} + +#[test] +fn parse_accepts_host_rsync_scope_policy() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--rsync-scope".to_string(), + "host".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!(args.rsync_scope_policy, RsyncScopePolicy::Host); +} + +#[test] +fn parse_accepts_repeatable_http_root_cert_paths() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--http-root-cert".to_string(), + "local-ca-1.pem".to_string(), + "--http-root-cert".to_string(), + "local-ca-2.pem".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!( + args.http_root_cert_paths, + vec![ + PathBuf::from("local-ca-1.pem"), + PathBuf::from("local-ca-2.pem") + ] + ); +} + +#[test] +fn parse_rejects_invalid_rsync_scope_policy() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--rsync-scope".to_string(), + "wide".to_string(), + ]; + let err = parse_args(&argv).expect_err("invalid rsync scope should fail"); + assert!(err.contains("invalid --rsync-scope"), "{err}"); +} + +#[test] +fn parse_accepts_strict_policy_list() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--strict".to_string(), + "name,cms-der".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!( + args.strict_policy, + Some(StrictPolicy { + name: true, + cms_der: true, + signed_attrs: false, + }) + ); +} + +#[test] +fn parse_accepts_strict_without_value_as_all() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--strict".to_string(), + "--report-json".to_string(), + "out/report.json".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!(args.strict_policy, Some(StrictPolicy::all())); +} + +#[test] +fn parse_accepts_resource_validation_mode() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--resource-validation-mode".to_string(), + "rfc6487".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!( + args.resource_validation_mode, + Some(ResourceValidationMode::Rfc6487) + ); +} + +#[test] +fn parse_rejects_unknown_resource_validation_mode() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--resource-validation-mode".to_string(), + "bogus".to_string(), + ]; + let err = parse_args(&argv).expect_err("unknown mode should fail"); + assert!(err.contains("unknown resource validation mode"), "{err}"); +} + +#[test] +fn parse_rejects_unknown_strict_policy() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--strict=unknown".to_string(), + ]; + let err = parse_args(&argv).expect_err("unknown strict policy should fail"); + assert!(err.contains("unknown strict policy"), "{err}"); +} + +#[test] +fn effective_cir_tal_uris_filters_skipped_multi_tal_inputs() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "afrinic.tal".to_string(), + "--ta-path".to_string(), + "afrinic.cer".to_string(), + "--tal-path".to_string(), + "apnic.tal".to_string(), + "--ta-path".to_string(), + "apnic.cer".to_string(), + "--tal-path".to_string(), + "arin.tal".to_string(), + "--ta-path".to_string(), + "arin.cer".to_string(), + "--cir-enable".to_string(), + "--cir-out".to_string(), + "out.cir".to_string(), + "--cir-tal-uri".to_string(), + "https://example.test/afrinic.cer".to_string(), + "--cir-tal-uri".to_string(), + "https://example.test/apnic.cer".to_string(), + "--cir-tal-uri".to_string(), + "https://example.test/arin.cer".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + let mut shared = synthetic_post_validation_shared(); + shared.discoveries = vec![shared.discovery.clone(), shared.discovery.clone()].into(); + shared.successful_tal_inputs = + vec![args.tal_inputs[0].clone(), args.tal_inputs[2].clone()].into(); + + let effective = effective_cir_tal_uris_for_discoveries( + &args, + &shared, + resolve_cir_export_tal_uris(&args).expect("resolve cir tal uris"), + ) + .expect("map effective cir tal uris"); + + assert_eq!( + effective, + vec![ + "https://example.test/afrinic.cer".to_string(), + "https://example.test/arin.cer".to_string(), + ] + ); +} + +#[test] +fn parse_rejects_report_json_compact_without_report_json() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--report-json-compact".to_string(), + ]; + let err = parse_args(&argv).expect_err("compact flag without report path should fail"); + assert!( + err.contains("--report-json-compact requires --report-json"), + "{err}" + ); +} + +#[test] +fn parse_accepts_skip_report_build_without_report_json() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--ccr-out".to_string(), + "out/result.ccr".to_string(), + "--skip-report-build".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.skip_report_build); + assert_eq!( + args.ccr_out_path.as_deref(), + Some(std::path::Path::new("out/result.ccr")) + ); +} + +#[test] +fn parse_accepts_skip_vcir_persist() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--skip-vcir-persist".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.skip_vcir_persist); +} + +#[test] +fn parse_rejects_skip_report_build_with_report_json() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--report-json".to_string(), + "out/report.json".to_string(), + "--skip-report-build".to_string(), + ]; + let err = parse_args(&argv).expect_err("skip report build with report path should fail"); + assert!( + err.contains("--skip-report-build cannot be combined with --report-json"), + "{err}" + ); +} + +#[test] +fn parse_accepts_direct_compare_view_csv_outputs() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--vrps-csv-out".to_string(), + "out/vrps.csv".to_string(), + "--vaps-csv-out".to_string(), + "out/vaps.csv".to_string(), + "--compare-view-trust-anchor".to_string(), + "unknown".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!( + args.vrps_csv_out_path.as_deref(), + Some(std::path::Path::new("out/vrps.csv")) + ); + assert_eq!( + args.vaps_csv_out_path.as_deref(), + Some(std::path::Path::new("out/vaps.csv")) + ); + assert_eq!(args.compare_view_trust_anchor.as_deref(), Some("unknown")); +} + +#[test] +fn parse_rejects_partial_direct_compare_view_csv_outputs() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--vrps-csv-out".to_string(), + "out/vrps.csv".to_string(), + ]; + let err = parse_args(&argv).expect_err("partial direct compare view output should fail"); + assert!( + err.contains("--vrps-csv-out and --vaps-csv-out must be provided together"), + "{err}" + ); +} + +#[test] +fn parse_accepts_external_raw_store_db() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--raw-store-db".to_string(), + "raw-store.db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!( + args.raw_store_db.as_deref(), + Some(std::path::Path::new("raw-store.db")) + ); +} + +#[test] +fn parse_accepts_external_repo_bytes_db() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--repo-bytes-db".to_string(), + "repo-bytes.db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!( + args.repo_bytes_db.as_deref(), + Some(std::path::Path::new("repo-bytes.db")) + ); +} + +#[test] +fn parse_accepts_cir_enable_with_raw_store_backend() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--raw-store-db".to_string(), + "raw-store.db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--rsync-local-dir".to_string(), + "repo".to_string(), + "--cir-enable".to_string(), + "--cir-out".to_string(), + "out/example.cir".to_string(), + "--cir-tal-uri".to_string(), + "https://example.test/root.tal".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.cir_enabled); + assert_eq!( + args.raw_store_db.as_deref(), + Some(std::path::Path::new("raw-store.db")) + ); + assert_eq!(args.cir_static_root, None); +} + +#[test] +fn parse_accepts_cir_enable_with_required_paths_and_tal_override() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--rsync-local-dir".to_string(), + "repo".to_string(), + "--cir-enable".to_string(), + "--cir-out".to_string(), + "out/example.cir".to_string(), + "--cir-tal-uri".to_string(), + "https://example.test/root.tal".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert!(args.cir_enabled); + assert_eq!( + args.cir_out_path.as_deref(), + Some(std::path::Path::new("out/example.cir")) + ); + assert_eq!( + args.cir_tal_uri.as_deref(), + Some("https://example.test/root.tal") + ); + assert_eq!( + args.cir_tal_uris, + vec!["https://example.test/root.tal".to_string()] + ); +} + +#[test] +fn parse_rejects_deprecated_cir_static_root() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--cir-enable".to_string(), + "--cir-out".to_string(), + "out/example.cir".to_string(), + "--cir-static-root".to_string(), + "out/static".to_string(), + ]; + let err = parse_args(&argv).expect_err("cir-static-root should be rejected"); + assert!(err.contains("no longer supported"), "{err}"); +} + +#[test] +fn parse_accepts_default_parallel_config_and_phase2_overrides() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase2-object-workers".to_string(), + "3".to_string(), + "--parallel-phase2-worker-queue-capacity".to_string(), + "17".to_string(), + "--parallel-phase2-ready-batch-size".to_string(), + "31".to_string(), + "--parallel-phase2-ready-batch-wall-time-budget-ms".to_string(), + "43".to_string(), + "--parallel-phase2-result-drain-batch-size".to_string(), + "37".to_string(), + "--parallel-phase2-finalize-batch-size".to_string(), + "41".to_string(), + "--parallel-phase2-finalize-batch-wall-time-budget-ms".to_string(), + "47".to_string(), + "--parallel-phase2-finalize-queue-capacity".to_string(), + "8192".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!(args.parallel_phase2_config.object_workers, 3); + assert_eq!(args.parallel_phase2_config.worker_queue_capacity, 17); + assert_eq!(args.parallel_phase2_config.ready_batch_size, 31); + assert_eq!( + args.parallel_phase2_config.ready_batch_wall_time_budget_ms, + 43 + ); + assert_eq!( + args.parallel_phase2_config.object_result_drain_batch_size, + 37 + ); + assert_eq!( + args.parallel_phase2_config + .publication_point_finalize_batch_size, + 41 + ); + assert_eq!( + args.parallel_phase2_config + .publication_point_finalize_wall_time_budget_ms, + 47 + ); + assert_eq!( + args.parallel_phase2_config + .publication_point_finalize_queue_capacity, + 8192 + ); + assert_eq!(args.parallel_phase1_config, ParallelPhase1Config::default()); +} + +#[test] +fn parse_rejects_zero_phase2_ready_batch_size() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase2-ready-batch-size".to_string(), + "0".to_string(), + ]; + let err = parse_args(&argv).expect_err("zero ready batch must fail"); + assert!(err.contains("--parallel-phase2-ready-batch-size"), "{err}"); +} + +#[test] +fn parse_rejects_zero_phase2_result_drain_batch_size() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase2-result-drain-batch-size".to_string(), + "0".to_string(), + ]; + let err = parse_args(&argv).expect_err("zero result drain batch must fail"); + assert!( + err.contains("--parallel-phase2-result-drain-batch-size"), + "{err}" + ); +} + +#[test] +fn parse_rejects_zero_phase2_ready_batch_wall_time_budget_ms() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase2-ready-batch-wall-time-budget-ms".to_string(), + "0".to_string(), + ]; + let err = parse_args(&argv).expect_err("zero ready time budget must fail"); + assert!( + err.contains("--parallel-phase2-ready-batch-wall-time-budget-ms"), + "{err}" + ); +} + +#[test] +fn parse_rejects_zero_phase2_finalize_batch_size() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase2-finalize-batch-size".to_string(), + "0".to_string(), + ]; + let err = parse_args(&argv).expect_err("zero finalize batch must fail"); + assert!( + err.contains("--parallel-phase2-finalize-batch-size"), + "{err}" + ); +} + +#[test] +fn parse_rejects_zero_phase2_finalize_batch_wall_time_budget_ms() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase2-finalize-batch-wall-time-budget-ms".to_string(), + "0".to_string(), + ]; + let err = parse_args(&argv).expect_err("zero finalize time budget must fail"); + assert!( + err.contains("--parallel-phase2-finalize-batch-wall-time-budget-ms"), + "{err}" + ); +} + +#[test] +fn parse_rejects_zero_phase2_finalize_queue_capacity() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase2-finalize-queue-capacity".to_string(), + "0".to_string(), + ]; + let err = parse_args(&argv).expect_err("zero finalize queue capacity must fail"); + assert!( + err.contains("--parallel-phase2-finalize-queue-capacity"), + "{err}" + ); +} + +#[test] +fn parse_rejects_removed_parallel_enable_flags() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase1".to_string(), + ]; + let err = parse_args(&argv).expect_err("removed phase flag should fail"); + assert!(err.contains("unknown argument: --parallel-phase1"), "{err}"); + + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--parallel-phase2".to_string(), + ]; + let err = parse_args(&argv).expect_err("removed phase flag should fail"); + assert!(err.contains("unknown argument: --parallel-phase2"), "{err}"); +} + +#[test] +fn parse_accepts_multi_tal_cir_overrides_in_file_mode() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "apnic.tal".to_string(), + "--ta-path".to_string(), + "apnic.cer".to_string(), + "--tal-path".to_string(), + "arin.tal".to_string(), + "--ta-path".to_string(), + "arin.cer".to_string(), + "--rsync-local-dir".to_string(), + "repo".to_string(), + "--cir-enable".to_string(), + "--cir-out".to_string(), + "out/example.cir".to_string(), + "--cir-tal-uri".to_string(), + "https://example.test/apnic.tal".to_string(), + "--cir-tal-uri".to_string(), + "https://example.test/arin.tal".to_string(), + ]; + let args = parse_args(&argv).expect("parse args"); + assert_eq!( + args.cir_tal_uris, + vec![ + "https://example.test/apnic.tal".to_string(), + "https://example.test/arin.tal".to_string() + ] + ); +} + +#[test] +fn parse_rejects_incomplete_or_invalid_cir_flags() { + let argv_missing = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--cir-enable".to_string(), + ]; + let err = parse_args(&argv_missing).unwrap_err(); + assert!(err.contains("--cir-enable requires --cir-out"), "{err}"); + + let argv_needs_enable = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/root.tal".to_string(), + "--cir-out".to_string(), + "out/example.cir".to_string(), + ]; + let err = parse_args(&argv_needs_enable).unwrap_err(); + assert!(err.contains("require --cir-enable"), "{err}"); + + let argv_offline_missing_uri = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "x.tal".to_string(), + "--ta-path".to_string(), + "x.cer".to_string(), + "--rsync-local-dir".to_string(), + "repo".to_string(), + "--cir-enable".to_string(), + "--cir-out".to_string(), + "out/example.cir".to_string(), + ]; + let err = parse_args(&argv_offline_missing_uri).unwrap_err(); + assert!(err.contains("requires --cir-tal-uri"), "{err}"); +} + +#[test] +fn parse_rejects_invalid_validation_time() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--validation-time".to_string(), + "not-a-time".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("invalid --validation-time"), "{err}"); +} + +#[test] +fn parse_rejects_invalid_max_instances() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--max-instances".to_string(), + "nope".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("invalid --max-instances"), "{err}"); +} + +#[test] +fn parse_rejects_missing_value_for_db() { + let argv = vec!["rpki".to_string(), "--db".to_string()]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("--db requires a value"), "{err}"); +} + +#[test] +fn parse_rejects_missing_value_for_tal_url() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("--tal-url requires a value"), "{err}"); +} + +#[test] +fn parse_rejects_missing_db() { + let argv = vec!["rpki".to_string(), "--tal-url".to_string(), "x".to_string()]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("--db is required"), "{err}"); +} + +#[test] +fn parse_rejects_missing_tal_mode() { + let argv = vec!["rpki".to_string(), "--db".to_string(), "db".to_string()]; + let err = parse_args(&argv).unwrap_err(); + assert!( + err.contains("--tal-url") || err.contains("--tal-path"), + "{err}" + ); +} + +#[test] +fn parse_accepts_tal_url_mode() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!(args.tal_url.as_deref(), Some("https://example.test/x.tal")); + assert_eq!( + args.tal_urls, + vec!["https://example.test/x.tal".to_string()] + ); + assert!(args.tal_path.is_none()); + assert!(args.ta_path.is_none()); + assert_eq!(args.tal_inputs.len(), 1); + assert_eq!(args.tal_inputs[0].tal_id, "x"); + assert_eq!(args.parallel_phase1_config, ParallelPhase1Config::default()); + assert_eq!(args.parallel_phase2_config, ParallelPhase2Config::default()); +} + +#[test] +fn parse_accepts_multi_tal_without_parallel_flags() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/arin.tal".to_string(), + "--tal-url".to_string(), + "https://example.test/apnic.tal".to_string(), + "--tal-url".to_string(), + "https://example.test/ripe.tal".to_string(), + "--parallel-max-repo-sync-workers-global".to_string(), + "8".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!(args.tal_urls.len(), 3); + assert_eq!(args.tal_inputs.len(), 3); + assert_eq!(args.tal_inputs[0].tal_id, "arin"); + assert_eq!(args.tal_inputs[1].tal_id, "apnic"); + assert_eq!(args.tal_inputs[2].tal_id, "ripe"); + assert_eq!(args.parallel_phase1_config.max_repo_sync_workers_global, 8); +} + +#[test] +fn parse_accepts_multi_tal_urls_by_default() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/arin.tal".to_string(), + "--tal-url".to_string(), + "https://example.test/apnic.tal".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!(args.tal_urls.len(), 2); + assert_eq!(args.tal_inputs.len(), 2); +} + +#[test] +fn parse_accepts_offline_mode_requires_ta() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--ta-path".to_string(), + "ta.cer".to_string(), + "--max-depth".to_string(), + "0".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!(args.tal_paths, vec![PathBuf::from("a.tal")]); + assert_eq!(args.ta_paths, vec![PathBuf::from("ta.cer")]); + assert_eq!(args.tal_path.as_deref(), Some(Path::new("a.tal"))); + assert_eq!(args.ta_path.as_deref(), Some(Path::new("ta.cer"))); + assert_eq!(args.max_ca_depth, 0); +} + +#[test] +fn parse_accepts_multiple_tal_path_pairs_by_default() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "apnic.tal".to_string(), + "--ta-path".to_string(), + "apnic-ta.cer".to_string(), + "--tal-path".to_string(), + "arin.tal".to_string(), + "--ta-path".to_string(), + "arin-ta.cer".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!(args.tal_paths.len(), 2); + assert_eq!(args.ta_paths.len(), 2); + assert_eq!(args.tal_inputs.len(), 2); +} + +#[test] +fn parse_rejects_mixed_tal_url_and_tal_path_modes() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/arin.tal".to_string(), + "--tal-path".to_string(), + "apnic.tal".to_string(), + "--ta-path".to_string(), + "apnic-ta.cer".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!( + err.contains( + "must specify either one-or-more --tal-url or one-or-more --tal-path/--ta-path pairs" + ), + "{err}" + ); +} + +#[test] +fn parse_rejects_mismatched_tal_path_and_ta_path_counts() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "apnic.tal".to_string(), + "--tal-path".to_string(), + "arin.tal".to_string(), + "--ta-path".to_string(), + "apnic-ta.cer".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!( + err.contains("--tal-path and --ta-path counts must match"), + "{err}" + ); +} + +#[test] +fn parse_accepts_tal_path_without_ta_when_disable_rrdp_is_set() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--disable-rrdp".to_string(), + "--rsync-command".to_string(), + "/tmp/fake-rsync".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!(args.tal_path.as_deref(), Some(Path::new("a.tal"))); + assert!(args.ta_path.is_none()); + assert!(args.disable_rrdp); + assert_eq!( + args.rsync_command.as_deref(), + Some(Path::new("/tmp/fake-rsync")) + ); +} + +#[test] +fn parse_accepts_multiple_tal_paths_without_ta_when_disable_rrdp() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--tal-path".to_string(), + "b.tal".to_string(), + "--disable-rrdp".to_string(), + "--rsync-command".to_string(), + "/tmp/fake-rsync".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!( + args.tal_paths, + vec![PathBuf::from("a.tal"), PathBuf::from("b.tal")] + ); + assert!(args.ta_paths.is_empty()); + assert_eq!(args.tal_inputs.len(), 2); + assert!(args.disable_rrdp); +} + +#[test] +fn parse_accepts_payload_delta_replay_mode_with_offline_tal_and_ta() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--ta-path".to_string(), + "ta.cer".to_string(), + "--payload-base-archive".to_string(), + "base-archive".to_string(), + "--payload-base-locks".to_string(), + "base-locks.json".to_string(), + "--payload-delta-archive".to_string(), + "delta-archive".to_string(), + "--payload-delta-locks".to_string(), + "delta-locks.json".to_string(), + ]; + let args = parse_args(&argv).expect("parse delta replay mode"); + assert_eq!( + args.payload_base_archive.as_deref(), + Some(Path::new("base-archive")) + ); + assert_eq!( + args.payload_base_locks.as_deref(), + Some(Path::new("base-locks.json")) + ); + assert_eq!( + args.payload_delta_archive.as_deref(), + Some(Path::new("delta-archive")) + ); + assert_eq!( + args.payload_delta_locks.as_deref(), + Some(Path::new("delta-locks.json")) + ); +} + +#[test] +fn parse_rejects_partial_payload_delta_arguments_and_mutual_exclusion() { + let argv_partial = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--ta-path".to_string(), + "ta.cer".to_string(), + "--payload-base-archive".to_string(), + "base-archive".to_string(), + ]; + let err = parse_args(&argv_partial).unwrap_err(); + assert!(err.contains("must be provided together"), "{err}"); + + let argv_both = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--ta-path".to_string(), + "ta.cer".to_string(), + "--payload-replay-archive".to_string(), + "archive".to_string(), + "--payload-replay-locks".to_string(), + "locks.json".to_string(), + "--payload-base-archive".to_string(), + "base-archive".to_string(), + "--payload-base-locks".to_string(), + "base-locks.json".to_string(), + "--payload-delta-archive".to_string(), + "delta-archive".to_string(), + "--payload-delta-locks".to_string(), + "delta-locks.json".to_string(), + ]; + let err = parse_args(&argv_both).unwrap_err(); + assert!(err.contains("mutually exclusive"), "{err}"); +} + +#[test] +fn parse_rejects_payload_delta_with_tal_url_or_rsync_local_dir() { + let argv_url = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--payload-base-archive".to_string(), + "base-archive".to_string(), + "--payload-base-locks".to_string(), + "base-locks.json".to_string(), + "--payload-delta-archive".to_string(), + "delta-archive".to_string(), + "--payload-delta-locks".to_string(), + "delta-locks.json".to_string(), + ]; + let err = parse_args(&argv_url).unwrap_err(); + assert!(err.contains("--tal-url is not supported"), "{err}"); + + let argv_rsync = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--ta-path".to_string(), + "ta.cer".to_string(), + "--payload-base-archive".to_string(), + "base-archive".to_string(), + "--payload-base-locks".to_string(), + "base-locks.json".to_string(), + "--payload-delta-archive".to_string(), + "delta-archive".to_string(), + "--payload-delta-locks".to_string(), + "delta-locks.json".to_string(), + "--rsync-local-dir".to_string(), + "repo".to_string(), + ]; + let err = parse_args(&argv_rsync).unwrap_err(); + assert!( + err.contains("payload delta replay mode cannot be combined with --rsync-local-dir"), + "{err}" + ); +} + +#[test] +fn parse_accepts_payload_replay_mode_with_offline_tal_and_ta() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--ta-path".to_string(), + "ta.cer".to_string(), + "--payload-replay-archive".to_string(), + "archive".to_string(), + "--payload-replay-locks".to_string(), + "locks.json".to_string(), + ]; + let args = parse_args(&argv).expect("parse replay mode"); + assert_eq!( + args.payload_replay_archive.as_deref(), + Some(Path::new("archive")) + ); + assert_eq!( + args.payload_replay_locks.as_deref(), + Some(Path::new("locks.json")) + ); +} + +#[test] +fn parse_rejects_partial_payload_replay_arguments() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--ta-path".to_string(), + "ta.cer".to_string(), + "--payload-replay-archive".to_string(), + "archive".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("must be provided together"), "{err}"); +} + +#[test] +fn parse_rejects_payload_replay_with_tal_url_or_rsync_local_dir() { + let argv_url = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--payload-replay-archive".to_string(), + "archive".to_string(), + "--payload-replay-locks".to_string(), + "locks.json".to_string(), + ]; + let err = parse_args(&argv_url).unwrap_err(); + assert!(err.contains("--tal-url is not supported"), "{err}"); + + let argv_rsync = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-path".to_string(), + "a.tal".to_string(), + "--ta-path".to_string(), + "ta.cer".to_string(), + "--payload-replay-archive".to_string(), + "archive".to_string(), + "--payload-replay-locks".to_string(), + "locks.json".to_string(), + "--rsync-local-dir".to_string(), + "repo".to_string(), + ]; + let err = parse_args(&argv_rsync).unwrap_err(); + assert!( + err.contains("cannot be combined with --rsync-local-dir"), + "{err}" + ); +} + +#[test] +fn parse_accepts_validation_time_rfc3339() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--validation-time".to_string(), + "2026-01-01T00:00:00Z".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert!(args.validation_time.is_some()); +} + +#[test] +fn parse_rejects_removed_revalidate_only_flag() { + let argv = vec![ + "rpki".to_string(), + "--db".to_string(), + "db".to_string(), + "--tal-url".to_string(), + "https://example.test/x.tal".to_string(), + "--revalidate-only".to_string(), + ]; + let err = parse_args(&argv).unwrap_err(); + assert!(err.contains("unknown argument: --revalidate-only"), "{err}"); +} + +#[test] +fn read_policy_accepts_valid_toml() { + let dir = tempfile::tempdir().expect("tmpdir"); + let p = dir.path().join("policy.toml"); + std::fs::write( + &p, + "signed_object_failure_policy = \"drop_publication_point\"\n", + ) + .expect("write policy"); + + let policy = read_policy(Some(&p)).expect("parse policy"); + assert_eq!( + policy.signed_object_failure_policy, + crate::policy::SignedObjectFailurePolicy::DropPublicationPoint + ); + assert_eq!( + policy.resource_validation_mode, + ResourceValidationMode::ValidationUpdate03 + ); + assert_eq!(policy.strict, StrictPolicy::default()); +} + +#[test] +fn read_policy_accepts_strict_table() { + let dir = tempfile::tempdir().expect("tmpdir"); + let p = dir.path().join("policy.toml"); + std::fs::write( + &p, + r#" + [strict] + name = true + cms_der = true + "#, + ) + .expect("write policy"); + + let policy = read_policy(Some(&p)).expect("parse policy"); + assert_eq!( + policy.strict, + StrictPolicy { + name: true, + cms_der: true, + signed_attrs: false, + } + ); +} + +#[test] +fn read_policy_accepts_resource_validation_mode() { + let dir = tempfile::tempdir().expect("tmpdir"); + let p = dir.path().join("policy.toml"); + std::fs::write(&p, "resource_validation_mode = \"rfc6487\"\n").expect("write policy"); + + let policy = read_policy(Some(&p)).expect("parse policy"); + assert_eq!( + policy.resource_validation_mode, + ResourceValidationMode::Rfc6487 + ); +} + +#[test] +fn read_policy_reports_missing_file() { + let dir = tempfile::tempdir().expect("tmpdir"); + let p = dir.path().join("missing.toml"); + let err = read_policy(Some(&p)).unwrap_err(); + assert!(err.contains("read policy file failed"), "{err}"); +} + +fn synthetic_post_validation_shared() -> PostValidationShared { + let tal_bytes = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/tal/apnic-rfc7730-https.tal"), + ) + .expect("read tal fixture"); + let ta_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/ta/apnic-ta.cer"), + ) + .expect("read ta fixture"); + + let discovery = crate::validation::from_tal::discover_root_ca_instance_from_tal_and_ta_der( + &tal_bytes, &ta_der, None, + ) + .expect("discover root"); + + let tree = crate::validation::tree::TreeRunOutput { + instances_processed: 1, + instances_failed: 0, + warnings: vec![ + crate::report::Warning::new("synthetic warning") + .with_rfc_refs(&[crate::report::RfcRef("RFC 6487 §4.8.8.1")]) + .with_context("rsync://example.test/repo/pp/"), + ], + vrps: vec![ + crate::validation::objects::Vrp { + asn: 64496, + prefix: crate::data_model::roa::IpPrefix { + afi: crate::data_model::roa::RoaAfi::Ipv4, + prefix_len: 24, + addr: [192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + }, + max_length: 24, + }, + crate::validation::objects::Vrp { + asn: 64497, + prefix: crate::data_model::roa::IpPrefix { + afi: crate::data_model::roa::RoaAfi::Ipv6, + prefix_len: 48, + addr: [0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + }, + max_length: 64, + }, + ], + aspas: vec![crate::validation::objects::AspaAttestation { + customer_as_id: 64496, + provider_as_ids: vec![64497, 64498], + }], + router_keys: Vec::new(), + }; + + let mut pp1 = crate::audit::PublicationPointAudit::default(); + pp1.source = "fresh".to_string(); + pp1.rrdp_notification_uri = Some("https://example.test/n1.xml".to_string()); + pp1.manifest_rsync_uri = "rsync://example.test/repo/pp1/manifest.mft".to_string(); + pp1.objects.push(crate::audit::ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/pp1/a.roa".to_string(), + sha256_hex: "11".repeat(32), + kind: crate::audit::AuditObjectKind::Roa, + result: crate::audit::AuditObjectResult::Ok, + detail: None, + }); + let mut pp2 = crate::audit::PublicationPointAudit::default(); + pp2.source = "fresh".to_string(); + pp2.rrdp_notification_uri = Some("https://example.test/n1.xml".to_string()); + let mut pp3 = crate::audit::PublicationPointAudit::default(); + pp3.source = "fresh".to_string(); + pp3.rrdp_notification_uri = Some("https://example.test/n2.xml".to_string()); + + let out = crate::validation::run_tree_from_tal::RunTreeFromTalAuditOutput { + discovery: discovery.clone(), + discoveries: vec![discovery], + successful_tal_inputs: Vec::new(), + tree, + publication_points: vec![pp1, pp2, pp3], + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + downloads: Vec::new(), + download_stats: crate::audit::AuditDownloadStats::default(), + current_repo_objects: Vec::new(), + ccr_accumulator: None, + cir_input: crate::cir::CirInputSnapshot::default(), + }; + PostValidationShared::from_run_output(out) +} + +fn sample_cli_ccr_accumulator() -> CcrAccumulator { + let tal_bytes = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/tal/apnic-rfc7730-https.tal"), + ) + .expect("read tal fixture"); + let ta_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/ta/apnic-ta.cer"), + ) + .expect("read ta fixture"); + let discovery = crate::validation::from_tal::discover_root_ca_instance_from_tal_and_ta_der( + &tal_bytes, &ta_der, None, + ) + .expect("discover root"); + let mut accumulator = CcrAccumulator::new(vec![discovery.trust_anchor.clone()]); + let manifest_uri = "rsync://example.test/repo/current.mft".to_string(); + let projection = crate::storage::VcirCcrManifestProjection { + manifest_rsync_uri: manifest_uri.clone(), + manifest_sha256: vec![0x44; 32], + manifest_size: 2048, + manifest_ee_aki: vec![0x55; 20], + manifest_number_be: vec![1], + manifest_this_update: crate::storage::PackTime::from_utc_offset_datetime( + time::OffsetDateTime::now_utc(), + ), + manifest_sia_locations_der: vec![ + crate::ccr::manifest_location::encode_access_description_der(&AccessDescription { + access_method_oid: OID_AD_SIGNED_OBJECT.to_string(), + access_location: manifest_uri, + }) + .expect("encode signedObject"), + ], + subordinate_skis: vec![vec![0x33; 20]], + }; + accumulator + .append_manifest_projection(&projection) + .expect("append manifest projection"); + accumulator +} + +#[test] +fn build_report_and_helpers_work_on_synthetic_output() { + let shared = synthetic_post_validation_shared(); + let policy = Policy::default(); + let validation_time = time::OffsetDateTime::now_utc(); + let report = build_report(&policy, validation_time, &shared); + + assert_eq!(unique_rrdp_repos(&report), 2); + assert_eq!(report.vrps.len(), 2); + assert_eq!(report.aspas.len(), 1); + + print_summary(&report); +} + +#[test] +fn run_report_task_and_stage_timing_work() { + let shared = synthetic_post_validation_shared(); + let policy = Policy::default(); + let validation_time = time::OffsetDateTime::now_utc(); + let dir = tempfile::tempdir().expect("tmpdir"); + let report_path = dir.path().join("report.json"); + let report_output = run_report_task( + &policy, + validation_time, + &shared, + Some(&report_path), + ReportJsonFormat::Compact, + ) + .expect("run report task"); + + assert!(report_output.report_write_ms.is_some()); + + let report_json = std::fs::read_to_string(&report_path).expect("read report json"); + assert!(!report_json.contains('\n'), "{report_json}"); + let report: serde_json::Value = + serde_json::from_str(&report_json).expect("parse compact report json"); + assert_eq!(report["vrps"].as_array().unwrap().len(), 2); + assert_eq!(report["aspas"].as_array().unwrap().len(), 1); + assert_eq!(report["queryAudit"]["status"].as_str(), Some("complete")); + assert!(report["queryAudit"]["eventsCount"].as_u64().unwrap() > 0); + let events_path = dir.path().join( + report["queryAudit"]["eventsPath"] + .as_str() + .expect("events path"), + ); + let events = std::fs::read_to_string(events_path).expect("read validation events"); + assert!( + events + .lines() + .any(|line| line.contains("\"eventType\":\"object\"")) + ); + + let stage_timing = RunStageTiming { + validation_ms: 1, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + crypto_signature_cache_observe: None, + enable_transport_request_prefetch: false, + report_build_ms: report_output.report_build_ms, + report_write_ms: report_output.report_write_ms, + ccr_build_ms: Some(2), + ccr_build_breakdown: None, + ccr_write_ms: Some(3), + compare_view_build_ms: Some(4), + compare_view_write_ms: Some(5), + cir_build_cir_ms: Some(6), + cir_write_cir_ms: Some(7), + cir_total_ms: Some(8), + total_ms: 9, + publication_points: shared.publication_points.len(), + repo_sync_ms_total: 10, + publication_point_repo_sync_ms_total: 11, + download_event_count: 12, + rrdp_download_ms_total: 13, + rsync_download_ms_total: 14, + download_bytes_total: 15, + roa_validation_cache: crate::validation::objects::RoaValidationCacheStats::default(), + analysis_counts: std::collections::HashMap::new(), + analysis_phases: std::collections::HashMap::new(), + analysis_top_publication_points: Vec::new(), + analysis_top_publication_point_steps: Vec::new(), + analysis_top_publication_point_cache_steps: Vec::new(), + vcir_storage_summary_ms: Some(16), + vcir_storage: Some(VcirStorageSummary { + entry_count: 2, + vcir_value_bytes: 100, + vcir_value_bytes_max: 60, + vcir_value_bytes_max_manifest_rsync_uri: Some( + "rsync://example.test/repo/max.mft".to_string(), + ), + core_fields: VcirCoreFieldSizeBreakdown { + manifest_rsync_uri_bytes: 10, + ..VcirCoreFieldSizeBreakdown::default() + }, + ccr_projection: VcirCcrProjectionSizeBreakdown { + manifest_sha256_bytes: 32, + ..VcirCcrProjectionSizeBreakdown::default() + }, + child_resources: VcirChildResourceSizeBreakdown { + effective_ip_resource_cbor_bytes: 12, + effective_as_resource_cbor_bytes: 6, + }, + field_sizes: VcirFieldSizeBreakdown { + local_output_count: 1, + local_output_payload_json_bytes: 70, + local_output_payload_typed_body_bytes: 20, + ..VcirFieldSizeBreakdown::default() + }, + local_output_old_projection_bytes: 80, + local_output_typed_projection_bytes: 30, + local_output_projection_saved_bytes: 50, + top_entries_by_vcir_value_bytes: vec![VcirStorageEntrySummary { + manifest_rsync_uri: "rsync://example.test/repo/max.mft".to_string(), + vcir_value_bytes: 60, + local_vrp_count: 1, + local_aspa_count: 0, + local_router_key_count: 0, + accepted_object_count: 1, + rejected_object_count: 0, + child_count: 0, + core_fields: VcirCoreFieldSizeBreakdown::default(), + ccr_projection: VcirCcrProjectionSizeBreakdown::default(), + child_resources: VcirChildResourceSizeBreakdown::default(), + field_sizes: VcirFieldSizeBreakdown::default(), + local_output_old_projection_bytes: 1, + local_output_typed_projection_bytes: 1, + local_output_projection_saved_bytes: 0, + }], + }), + publication_point_cache_index_load: None, + publication_point_cache_index_refresh: None, + memory_telemetry: None, + }; + write_stage_timing(Some(&report_path), &stage_timing).expect("write stage timing"); + let stage_timing_json = + std::fs::read_to_string(dir.path().join("stage-timing.json")).expect("read timing"); + assert!(stage_timing_json.contains("\"validation_ms\"")); + assert!(stage_timing_json.contains("\"ccr_build_ms\"")); + assert!(stage_timing_json.contains("\"vcir_storage\"")); + assert!(stage_timing_json.contains("\"local_output_projection_saved_bytes\"")); + + let ccr_path = dir.path().join("result.ccr"); + write_stage_timing(Some(&ccr_path), &stage_timing).expect("write stage timing via ccr path"); + assert!( + dir.path().join("stage-timing.json").exists(), + "stage timing should use parent directory of the anchor path" + ); + + let skipped = ReportTaskOutput::skipped(); + assert_eq!(skipped.report_build_ms, 0); + assert!(skipped.report_write_ms.is_none()); +} + +#[test] +fn stage_timing_serializes_memory_telemetry() { + let dir = tempfile::tempdir().expect("tmpdir"); + let report_path = dir.path().join("report.json"); + let stage_timing = RunStageTiming { + validation_ms: 1, + enable_roa_validation_cache: true, + enable_child_certificate_validation_cache: true, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + crypto_signature_cache_observe: None, + enable_transport_request_prefetch: true, + report_build_ms: 2, + report_write_ms: None, + ccr_build_ms: None, + ccr_build_breakdown: None, + ccr_write_ms: None, + compare_view_build_ms: None, + compare_view_write_ms: None, + cir_build_cir_ms: None, + cir_write_cir_ms: None, + cir_total_ms: None, + total_ms: 3, + publication_points: 4, + repo_sync_ms_total: 5, + publication_point_repo_sync_ms_total: 6, + download_event_count: 7, + rrdp_download_ms_total: 8, + rsync_download_ms_total: 9, + download_bytes_total: 10, + roa_validation_cache: crate::validation::objects::RoaValidationCacheStats { + hit_roas: 2, + ..crate::validation::objects::RoaValidationCacheStats::default() + }, + analysis_counts: std::collections::HashMap::from([( + "roa_validation_cache_hit_roas".to_string(), + 2, + )]), + analysis_phases: std::collections::HashMap::new(), + analysis_top_publication_points: Vec::new(), + analysis_top_publication_point_steps: Vec::new(), + analysis_top_publication_point_cache_steps: Vec::new(), + vcir_storage_summary_ms: None, + vcir_storage: None, + publication_point_cache_index_load: None, + publication_point_cache_index_refresh: None, + memory_telemetry: Some(MemoryTelemetrySummary { + checkpoints: vec![MemoryTelemetryCheckpoint { + label: "after_validation".to_string(), + elapsed_ms: 11, + process: ProcessMemorySnapshot { + label: "after_validation".to_string(), + vm_rss_kb: Some(12), + vm_size_kb: None, + vm_data_kb: None, + vm_swap_kb: None, + rss_anon_kb: Some(13), + rss_file_kb: None, + rss_shmem_kb: None, + threads: Some(14), + fd_count: Some(15), + smaps_rollup: None, + smaps_mapping_summary: None, + errors: Vec::new(), + }, + rocksdb: RocksDbMemorySnapshot { + databases: Vec::new(), + totals: RocksDbMemoryTotals { + cur_size_all_mem_tables: 16, + size_all_mem_tables: 17, + estimate_table_readers_mem: 18, + block_cache_capacity: 19, + block_cache_usage: 20, + block_cache_pinned_usage: 21, + }, + }, + }], + object_graph: None, + malloc_trim_probes: Vec::new(), + }), + }; + + write_stage_timing(Some(&report_path), &stage_timing).expect("write stage timing"); + let value: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(dir.path().join("stage-timing.json")).unwrap(), + ) + .expect("parse stage timing json"); + let checkpoint = &value["memory_telemetry"]["checkpoints"][0]; + assert_eq!(checkpoint["label"], "after_validation"); + assert_eq!(checkpoint["process"]["vm_rss_kb"], 12); + assert_eq!( + checkpoint["rocksdb"]["totals"]["cur_size_all_mem_tables"], + 16 + ); + assert_eq!(value["analysis_counts"]["roa_validation_cache_hit_roas"], 2); + assert_eq!(value["roa_validation_cache"]["hit_roas"], 2); + assert_eq!(value["publication_point_cache_observe_only"], false); + assert_eq!(value["enable_publication_point_validation_cache"], false); + assert!( + value["memory_telemetry"] + .as_object() + .expect("memory telemetry object") + .get("malloc_trim_probes") + .is_none() + ); +} + +#[test] +fn shared_object_graph_estimate_counts_audit_and_outputs() { + let mut shared = synthetic_post_validation_shared(); + let mut publication_points = shared + .publication_points + .iter() + .cloned() + .collect::>(); + publication_points[0].rsync_base_uri = "rsync://example.test/repo/".to_string(); + publication_points[0].manifest_rsync_uri = "rsync://example.test/repo/a.mft".to_string(); + publication_points[0].publication_point_rsync_uri = "rsync://example.test/repo/".to_string(); + publication_points[0].objects = vec![crate::audit::ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/a.roa".to_string(), + sha256_hex: "11".repeat(32), + kind: crate::audit::AuditObjectKind::Roa, + result: crate::audit::AuditObjectResult::Ok, + detail: None, + }]; + shared.publication_points = publication_points.into(); + + let graph = estimate_shared_object_graph(&shared); + let publication_points_section = graph + .sections + .iter() + .find(|section| section.name == "publication_points") + .expect("publication points section"); + let object_count = publication_points_section + .details + .iter() + .find(|metric| metric.name == "object_audit_entry_count") + .expect("object count metric"); + assert_eq!(object_count.value, 1); + assert!(publication_points_section.estimated_bytes > 0); + + let vrps_section = graph + .sections + .iter() + .find(|section| section.name == "vrps") + .expect("vrps section"); + assert_eq!(vrps_section.item_count, 2); + assert!(graph.total_estimated_bytes >= publication_points_section.estimated_bytes); +} + +#[test] +fn run_compare_view_task_writes_csv_from_shared_output() { + let shared = synthetic_post_validation_shared(); + let dir = tempfile::tempdir().expect("tmpdir"); + let vrps_path = dir.path().join("vrps.csv"); + let vaps_path = dir.path().join("vaps.csv"); + + let output = run_compare_view_task(&shared, Some(&vrps_path), Some(&vaps_path), "unknown") + .expect("write direct compare views"); + + assert!(output.build_ms.is_some()); + assert!(output.write_ms.is_some()); + let vrps_csv = std::fs::read_to_string(vrps_path).expect("read vrps csv"); + let vaps_csv = std::fs::read_to_string(vaps_path).expect("read vaps csv"); + assert!(vrps_csv.contains("ASN,IP Prefix,Max Length,Trust Anchor")); + assert!(vrps_csv.contains("AS64496,192.0.2.0/24,24,unknown")); + assert!(vrps_csv.contains("AS64497,2001:db8::/48,64,unknown")); + assert!(vaps_csv.contains("Customer ASN,Providers,Trust Anchor")); + assert!(vaps_csv.contains("AS64496,AS64497;AS64498,unknown")); +} + +#[test] +fn run_ccr_task_uses_accumulator_when_phase2_output_contains_reuse_sources() { + let mut shared = synthetic_post_validation_shared(); + shared.ccr_accumulator = Some(sample_cli_ccr_accumulator()); + let mut publication_points = shared + .publication_points + .iter() + .cloned() + .collect::>(); + publication_points[1].source = "vcir_current_instance".to_string(); + publication_points[2].source = "failed_no_cache".to_string(); + shared.publication_points = publication_points.into(); + let dir = tempfile::tempdir().expect("tmpdir"); + let ccr_path = dir.path().join("result.ccr"); + let store = RocksStore::open(&dir.path().join("db")).expect("open empty store"); + + let output = run_ccr_task( + &store, + &shared, + Some(&ccr_path), + time::OffsetDateTime::now_utc(), + ) + .expect("run ccr task"); + + assert!(output.ccr_build_ms.is_some()); + assert!(output.ccr_build_breakdown.is_none()); + let der = std::fs::read(&ccr_path).expect("read ccr"); + let ci = crate::ccr::decode_content_info(&der).expect("decode ccr"); + assert_eq!( + ci.content + .mfts + .as_ref() + .map(|manifest_state| manifest_state.mis.len()), + Some(1) + ); +} + +#[test] +fn write_json_writes_report() { + let report = AuditReportV2 { + format_version: 2, + meta: AuditRunMeta { + validation_time_rfc3339_utc: "2026-01-01T00:00:00Z".to_string(), + }, + policy: Policy::default(), + tree: TreeSummary { + instances_processed: 0, + instances_failed: 0, + warnings: Vec::new(), + }, + publication_points: Vec::new(), + vrps: Vec::new(), + aspas: Vec::new(), + downloads: Vec::new(), + download_stats: crate::audit::AuditDownloadStats::default(), + repo_sync_stats: crate::audit::AuditRepoSyncStats::default(), + query_audit: None, + }; + + let dir = tempfile::tempdir().expect("tmpdir"); + let pretty_path = dir.path().join("report-pretty.json"); + write_json(&pretty_path, &report, ReportJsonFormat::Pretty).expect("write pretty json"); + let pretty = std::fs::read_to_string(&pretty_path).expect("read pretty report"); + assert!(pretty.contains("\"format_version\"")); + assert!(pretty.contains("\"policy\"")); + assert!(pretty.contains("\n \"format_version\""), "{pretty}"); + + let compact_path = dir.path().join("report-compact.json"); + write_json(&compact_path, &report, ReportJsonFormat::Compact).expect("write compact json"); + let compact = std::fs::read_to_string(&compact_path).expect("read compact report"); + assert!(compact.contains("\"format_version\"")); + assert!(compact.contains("\"policy\"")); + assert!(!compact.contains('\n'), "{compact}"); +} + +#[test] +fn build_repo_sync_stats_aggregates_phase_and_terminal_state() { + let mut pp1 = crate::audit::PublicationPointAudit::default(); + pp1.repo_sync_phase = Some("rrdp_ok".to_string()); + pp1.repo_sync_duration_ms = Some(10); + pp1.repo_terminal_state = "fresh".to_string(); + + let mut pp2 = crate::audit::PublicationPointAudit::default(); + pp2.repo_sync_phase = Some("rrdp_failed_rsync_failed".to_string()); + pp2.repo_sync_duration_ms = Some(20); + pp2.repo_terminal_state = "failed_no_cache".to_string(); + + let mut pp3 = crate::audit::PublicationPointAudit::default(); + pp3.repo_sync_phase = Some("rrdp_failed_rsync_failed".to_string()); + pp3.repo_sync_duration_ms = Some(30); + pp3.repo_terminal_state = "failed_no_cache".to_string(); + + let stats = build_repo_sync_stats(&[pp1, pp2, pp3]); + assert_eq!(stats.publication_points_total, 3); + assert_eq!(stats.by_phase["rrdp_ok"].count, 1); + assert_eq!(stats.by_phase["rrdp_ok"].duration_ms_total, 10); + assert_eq!(stats.by_phase["rrdp_failed_rsync_failed"].count, 2); + assert_eq!( + stats.by_phase["rrdp_failed_rsync_failed"].duration_ms_total, + 50 + ); + assert_eq!(stats.by_terminal_state["fresh"].count, 1); + assert_eq!(stats.by_terminal_state["failed_no_cache"].count, 2); + assert_eq!( + stats.by_terminal_state["failed_no_cache"].duration_ms_total, + 50 + ); +} diff --git a/crates/panda-rpki-validator/src/contract.rs b/crates/panda-rpki-validator/src/contract.rs new file mode 100644 index 0000000..0fd0d2c --- /dev/null +++ b/crates/panda-rpki-validator/src/contract.rs @@ -0,0 +1,127 @@ +//! Normal-run validation contract. +//! +//! This is deliberately limited to the normal synchronization/validation +//! path. Offline revalidation of a previous run is not part of this package's +//! first public contract. + +use std::path::Path; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use time::format_description::well_known::Rfc3339; + +use crate::fetch::rsync_system::RsyncScopePolicy; +use crate::policy::Policy; + +pub const VALIDATION_CONTRACT_SCHEMA_VERSION: u32 = 1; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ValidationCacheContract { + pub publication_point: bool, + pub roa: bool, + pub child_certificate: bool, + pub transport_prefetch: bool, + #[serde(default)] + pub crypto_signature: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ValidationContract { + pub schema_version: u32, + pub validation_time: String, + pub binary_sha256: String, + pub policy: Policy, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ta_constraints_fingerprint: Option, + pub max_ca_depth: usize, + pub max_instances: Option, + pub cache: ValidationCacheContract, + #[serde(default)] + pub rsync_scope_policy: RsyncScopePolicy, +} + +impl ValidationContract { + pub fn for_current_binary( + validation_time: time::OffsetDateTime, + policy: Policy, + max_ca_depth: usize, + max_instances: Option, + cache: ValidationCacheContract, + rsync_scope_policy: RsyncScopePolicy, + ) -> Result { + Ok(Self { + schema_version: VALIDATION_CONTRACT_SCHEMA_VERSION, + validation_time: format_validation_time(validation_time)?, + binary_sha256: current_binary_sha256()?, + policy, + ta_constraints_fingerprint: None, + max_ca_depth, + max_instances, + cache, + rsync_scope_policy, + }) + } + + pub fn validate(&self) -> Result<(), String> { + if self.schema_version != VALIDATION_CONTRACT_SCHEMA_VERSION { + return Err(format!( + "unsupported validation contract schemaVersion: {}", + self.schema_version + )); + } + time::OffsetDateTime::parse(&self.validation_time, &Rfc3339) + .map_err(|error| format!("invalid validation contract validationTime: {error}"))?; + validate_sha256_hex("validation contract binarySha256", &self.binary_sha256)?; + if let Some(fingerprint) = self.ta_constraints_fingerprint.as_deref() { + validate_sha256_hex("validation contract taConstraintsFingerprint", fingerprint)?; + } + if self.max_ca_depth == 0 { + return Err("validation contract maxCaDepth must be greater than zero".to_string()); + } + Ok(()) + } +} + +pub fn write_validation_contract(path: &Path, contract: &ValidationContract) -> Result<(), String> { + contract.validate()?; + let bytes = serde_json::to_vec_pretty(contract) + .map_err(|error| format!("encode validation contract failed: {error}"))?; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|error| format!("create validation contract directory failed: {error}"))?; + } + std::fs::write(path, bytes).map_err(|error| { + format!( + "write validation contract failed: {}: {error}", + path.display() + ) + }) +} + +pub fn format_validation_time(value: time::OffsetDateTime) -> Result { + value + .to_offset(time::UtcOffset::UTC) + .format(&Rfc3339) + .map_err(|error| format!("format validation time failed: {error}")) +} + +fn current_binary_sha256() -> Result { + let path = std::env::current_exe() + .map_err(|error| format!("resolve current executable failed: {error}"))?; + let bytes = std::fs::read(&path).map_err(|error| { + format!( + "read current executable failed: {}: {error}", + path.display() + ) + })?; + Ok(hex::encode(Sha256::digest(bytes))) +} + +pub(crate) fn validate_sha256_hex(label: &str, value: &str) -> Result<(), String> { + if value.len() != 64 || !value.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(format!("{label} must be a 64-character SHA-256 hex string")); + } + Ok(()) +} diff --git a/crates/panda-rpki-validator/src/crypto_sig_cache.rs b/crates/panda-rpki-validator/src/crypto_sig_cache.rs new file mode 100644 index 0000000..c66f94c --- /dev/null +++ b/crates/panda-rpki-validator/src/crypto_sig_cache.rs @@ -0,0 +1,876 @@ +//! Crypto signature verification cache (feature #126). +//! +//! Caches the context-free cryptographic fact "this signature over these exact bytes verifies +//! under this public key" so that later runs can skip the RSA verification entirely. Only +//! positive conclusions are stored (no negative caching). The cache never replaces any other +//! validation semantics: revocation (CRL), time-window, and policy checks all run unchanged on +//! the hit path — the hook wraps exactly the crypto verify call and nothing else. +//! +//! Modes (CLI): +//! +//! - `--crypto-signature-cache-observe-only` (M1): compute keys, maintain the persistent set, +//! record hit/miss/timing statistics, always run the real verification. +//! - `--enable-crypto-signature-cache` (M2): on a cache hit, skip the real verification and +//! return success; on a miss, run the real verification and store the positive conclusion. +//! Statistics are recorded the same way; `verify_skipped`/`verify_executed` report what +//! actually happened. Both flags may be combined (reuse + statistics). +//! +//! The five choke points (see the development plan for the full caller inventory): +//! +//! - `CmsSignedObject`: `data_model::signed_object` CMS verify (ROA / ASPA / manifest on the +//! main validation path; GBR objects are not signature-verified by the tree runner). +//! - `EeCertFast`: `validation::cert_path::verify_ee_cert_signature_fast` (ring, TBS + signature +//! bytes from the signed-object embedded EE certificate; ROA / ASPA / manifest EE paths). +//! - `EeCertX509`: `validation::cert_path::verify_ee_cert_signature` (x509-parser; router +//! certificates and the `validate_ee_cert_path` family). +//! - `ChildCaCert`: `validation::ca_path::verify_child_signature` (x509-parser; subordinate CA +//! certificates). +//! - `Crl`: `data_model::crl::verify_signature_with_issuer_spki` (x509-parser; issuer CRLs). +//! +//! Trust-anchor self-signature verification is deliberately excluded (a handful of calls per +//! run). +//! +//! Cache key (M1 decision: verify-input bytes, not full object bytes — the key material is in +//! hand inside each choke point, while the repo-bytes content hash is only available at the +//! upper processing layer): +//! +//! ```text +//! key = SHA-256( "rpki-crypto-sig-cache-key-v1" | point-tag | "sha256WithRSAEncryption" +//! | SHA-256(verify input bytes) | SHA-256(signature bytes) +//! | SHA-256(verifier SPKI DER) ) +//! ``` +//! +//! The signature bytes are part of the key so that a different signature over the same input +//! under the same key can never inherit a positive verification fact. +//! +//! Storage: +//! +//! - Single small file next to the work DB (`.crypto-sig-cache`), never inside +//! the work DB, not touched by periodic snapshot reset (#091/#092): the reset paths only +//! remove/replace the work DB directory and the PP/child-cert cache artifacts, not siblings +//! of their own choosing. +//! - Binary format: 8-byte magic `RPKICSC2` | u64 record-count | records of +//! key[32] | verified_at_unix_secs(i64) | crypto_impl_version(u32) | reserved(u32). +//! A missing, corrupt, or magic-mismatched file (including the M1 `RPKICSC1` seen-set +//! format) is silently rebuilt empty. Writes are atomic (write-then-rename). +//! - The cache is persisted once at the end of the validation stage (same point as the M1 +//! seen-set). If a run fails mid-validation, entries added during that run are lost; +//! previously persisted entries are unaffected. +//! - Size cap: approximate上限 `capacity` entries (default 2,000,000; measured M1 APNIC +//! ~148k keys/run, all5 estimated ~750k). When an insert would exceed the cap, the whole +//! cache is cleared and rebuilt from scratch (full-rebuild eviction — the simplest policy; +//! concurrent inserts may briefly overshoot the cap by a few entries). Each eviction is +//! counted in `evictions_total`. +//! - Concurrency: entries live in 64 shards keyed by the first key byte, so parallel +//! validation workers rarely contend; statistics use relaxed atomics. + +use std::collections::{BTreeMap, HashMap}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, RwLock}; + +use serde::Serialize; +use sha2::Digest; + +const CACHE_FILE_MAGIC: &[u8; 8] = b"RPKICSC2"; +const KEY_DOMAIN: &[u8] = b"rpki-crypto-sig-cache-key-v1"; +const ALG_ID: &[u8] = b"sha256WithRSAEncryption"; +const SHARD_COUNT: usize = 64; +const RECORD_BYTES: usize = 32 + 8 + 4 + 4; + +/// The schema version reported in summaries. The on-disk magic already carries the format +/// version; this constant is the semantic version of the key/value definitions. +pub const CACHE_SCHEMA_VERSION: u32 = 2; + +/// Default maximum number of cached positive conclusions (see module docs). +pub const DEFAULT_CAPACITY: usize = 2_000_000; + +/// Crypto implementation identifier stored in every value. Bump when the verification +/// backend changes so old entries can be invalidated wholesale. +/// `1` = ring 0.17 `RSA_PKCS1_2048_8192_SHA256` (direct ring calls and x509-parser's +/// `verify` feature, which is backed by ring in this crate). +pub const CRYPTO_IMPL_VERSION: u32 = 1; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum CryptoSigVerifyPoint { + CmsSignedObject, + EeCertFast, + EeCertX509, + ChildCaCert, + Crl, +} + +impl CryptoSigVerifyPoint { + pub const ALL: [Self; 5] = [ + Self::CmsSignedObject, + Self::EeCertFast, + Self::EeCertX509, + Self::ChildCaCert, + Self::Crl, + ]; + + fn tag(self) -> u8 { + match self { + Self::CmsSignedObject => 1, + Self::EeCertFast => 2, + Self::EeCertX509 => 3, + Self::ChildCaCert => 4, + Self::Crl => 5, + } + } + + pub fn name(self) -> &'static str { + match self { + Self::CmsSignedObject => "cms_signed_object", + Self::EeCertFast => "ee_cert_fast", + Self::EeCertX509 => "ee_cert_x509", + Self::ChildCaCert => "child_ca_cert", + Self::Crl => "crl", + } + } + + fn from_index(index: usize) -> Self { + Self::ALL[index] + } +} + +/// Compute the cache key for one verification fact. +/// +/// `input_bytes` is the exact byte string fed to the verifier (CMS signedAttrs DER for +/// signature, TBSCertificate DER, TBSCertList DER), `signature_bytes` the signature value, +/// and `verifier_spki_der` the DER of the verifying SubjectPublicKeyInfo. +pub fn compute_cache_key( + point: CryptoSigVerifyPoint, + input_bytes: &[u8], + signature_bytes: &[u8], + verifier_spki_der: &[u8], +) -> [u8; 32] { + let mut h = sha2::Sha256::new(); + h.update(KEY_DOMAIN); + h.update([point.tag()]); + h.update(ALG_ID); + for part in [input_bytes, signature_bytes, verifier_spki_der] { + let part_hash = sha2::Sha256::digest(part); + h.update(part_hash); + } + h.finalize().into() +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CacheValue { + pub verified_at_unix_secs: i64, + pub crypto_impl_version: u32, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct CryptoSigVerifyPointStats { + pub calls: u64, + /// Key already present in the cache (positive conclusion available). + pub would_hit: u64, + /// Key not present; after a successful verification it is inserted. + pub new_keys: u64, + /// Real cryptographic verification executed (always equals `calls` in observe-only + /// mode; equals `new_keys` plus hit-but-observe calls otherwise). + pub verify_executed: u64, + /// Real verification skipped because of a cache hit (only > 0 when reuse is enabled). + pub verify_skipped: u64, + pub verify_nanos_on_would_hit: u64, + pub verify_nanos_on_new_key: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct CryptoSigCacheSummary { + pub schema_version: u32, + pub cache_file: String, + pub reuse_enabled: bool, + pub capacity: u64, + pub entries_loaded: u64, + pub entries_total: u64, + pub evictions_total: u64, + pub per_point: BTreeMap, +} + +#[derive(Default)] +struct PointCounters { + calls: AtomicU64, + would_hit: AtomicU64, + new_keys: AtomicU64, + verify_executed: AtomicU64, + verify_skipped: AtomicU64, + verify_nanos_on_would_hit: AtomicU64, + verify_nanos_on_new_key: AtomicU64, +} + +fn shard_index(key: &[u8; 32]) -> usize { + (key[0] as usize) % SHARD_COUNT +} + +/// The persistent positive-conclusion signature cache plus per-run statistics. +/// +/// See the module docs for storage format, eviction, and failure semantics. +pub struct CryptoSigCache { + cache_file: PathBuf, + reuse_enabled: bool, + capacity: usize, + entries_loaded: u64, + shards: Vec>>, + len: AtomicUsize, + evictions: AtomicU64, + eviction_lock: Mutex<()>, + points: [PointCounters; 5], +} + +impl CryptoSigCache { + /// Load the cache from `cache_file`; start empty when the file is missing, corrupt, or + /// has an unknown schema header. `reuse_enabled` selects the M2 hit-skips-verify + /// behavior; when false the cache is observe-only (M1 semantics). + pub fn load_or_rebuild(cache_file: PathBuf, reuse_enabled: bool) -> Self { + Self::load_or_rebuild_with_capacity(cache_file, reuse_enabled, DEFAULT_CAPACITY) + } + + pub fn load_or_rebuild_with_capacity( + cache_file: PathBuf, + reuse_enabled: bool, + capacity: usize, + ) -> Self { + let entries = load_cache_file(&cache_file).unwrap_or_default(); + let mut shards = Vec::with_capacity(SHARD_COUNT); + for _ in 0..SHARD_COUNT { + shards.push(Mutex::new(HashMap::new())); + } + let mut len = 0usize; + for (key, value) in entries { + // Respect the cap even when loading a foreign/oversized file: keep the first + // `capacity` records in file order. + if len >= capacity { + break; + } + shards[shard_index(&key)] + .lock() + .expect("crypto sig shard lock") + .insert(key, value); + len += 1; + } + Self { + cache_file, + reuse_enabled, + capacity, + entries_loaded: len as u64, + shards, + len: AtomicUsize::new(len), + evictions: AtomicU64::new(0), + eviction_lock: Mutex::new(()), + points: Default::default(), + } + } + + pub fn reuse_enabled(&self) -> bool { + self.reuse_enabled + } + + fn lookup(&self, key: &[u8; 32]) -> bool { + self.shards[shard_index(key)] + .lock() + .expect("crypto sig shard lock") + .contains_key(key) + } + + #[cfg(test)] + pub fn contains(&self, key: &[u8; 32]) -> bool { + self.lookup(key) + } + + /// Insert a positive conclusion (used by the verify path and by tests). + pub fn insert_positive(&self, key: [u8; 32]) { + let value = CacheValue { + verified_at_unix_secs: time::OffsetDateTime::now_utc().unix_timestamp(), + crypto_impl_version: CRYPTO_IMPL_VERSION, + }; + if self.len.load(Ordering::Relaxed) >= self.capacity { + self.evict_all(); + } + let mut shard = self.shards[shard_index(&key)] + .lock() + .expect("crypto sig shard lock"); + if shard.insert(key, value).is_none() { + self.len.fetch_add(1, Ordering::Relaxed); + } + } + + /// Full-rebuild eviction: clear every shard. See the module docs for the semantics. + fn evict_all(&self) { + let _guard = self.eviction_lock.lock().expect("crypto sig eviction lock"); + if self.len.load(Ordering::Relaxed) < self.capacity { + // Another thread already evicted. + return; + } + for shard in &self.shards { + shard.lock().expect("crypto sig shard lock").clear(); + } + self.len.store(0, Ordering::Relaxed); + self.evictions.fetch_add(1, Ordering::Relaxed); + } + + /// Cache-aware verification: observe-only or reuse depending on `reuse_enabled`. + /// The `verify` closure must perform exactly the real cryptographic verification. + pub fn verify( + &self, + point: CryptoSigVerifyPoint, + input_bytes: &[u8], + signature_bytes: &[u8], + verifier_spki_der: &[u8], + verify: impl FnOnce() -> Result<(), E>, + ) -> Result<(), E> { + let key = compute_cache_key(point, input_bytes, signature_bytes, verifier_spki_der); + let hit = self.lookup(&key); + let counters = &self.points[(point.tag() - 1) as usize]; + counters.calls.fetch_add(1, Ordering::Relaxed); + if hit { + counters.would_hit.fetch_add(1, Ordering::Relaxed); + if self.reuse_enabled { + counters.verify_skipped.fetch_add(1, Ordering::Relaxed); + return Ok(()); + } + } else { + counters.new_keys.fetch_add(1, Ordering::Relaxed); + } + counters.verify_executed.fetch_add(1, Ordering::Relaxed); + let started = std::time::Instant::now(); + let result = verify(); + let nanos = started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64; + if hit { + counters + .verify_nanos_on_would_hit + .fetch_add(nanos, Ordering::Relaxed); + } else { + counters + .verify_nanos_on_new_key + .fetch_add(nanos, Ordering::Relaxed); + if result.is_ok() { + self.insert_positive(key); + } + } + result + } + + /// Persist the cache atomically (write-then-rename). + pub fn persist(&self) -> Result<(), String> { + persist_cache_file(&self.cache_file, &self.shards) + } + + pub fn summary(&self) -> CryptoSigCacheSummary { + let mut per_point = BTreeMap::new(); + for (index, counters) in self.points.iter().enumerate() { + let stats = CryptoSigVerifyPointStats { + calls: counters.calls.load(Ordering::Relaxed), + would_hit: counters.would_hit.load(Ordering::Relaxed), + new_keys: counters.new_keys.load(Ordering::Relaxed), + verify_executed: counters.verify_executed.load(Ordering::Relaxed), + verify_skipped: counters.verify_skipped.load(Ordering::Relaxed), + verify_nanos_on_would_hit: counters + .verify_nanos_on_would_hit + .load(Ordering::Relaxed), + verify_nanos_on_new_key: counters.verify_nanos_on_new_key.load(Ordering::Relaxed), + }; + if stats.calls > 0 { + per_point.insert( + CryptoSigVerifyPoint::from_index(index).name().to_string(), + stats, + ); + } + } + CryptoSigCacheSummary { + schema_version: CACHE_SCHEMA_VERSION, + cache_file: self.cache_file.to_string_lossy().into_owned(), + reuse_enabled: self.reuse_enabled, + capacity: self.capacity as u64, + entries_loaded: self.entries_loaded, + entries_total: self.len.load(Ordering::Relaxed) as u64, + evictions_total: self.evictions.load(Ordering::Relaxed), + per_point, + } + } +} + +/// Default cache file location: a sibling of the work DB directory, mirroring +/// `default_pp_cache_index_dir` (`.crypto-sig-cache`). +pub fn default_cache_file_path(db_path: &Path) -> PathBuf { + let file_name = db_path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("work-db"); + db_path.with_file_name(format!("{file_name}.crypto-sig-cache")) +} + +fn load_cache_file(path: &Path) -> Option> { + let bytes = std::fs::read(path).ok()?; + let header_len = CACHE_FILE_MAGIC.len() + 8; + if bytes.len() < header_len || &bytes[..CACHE_FILE_MAGIC.len()] != CACHE_FILE_MAGIC { + return None; + } + let mut count_bytes = [0u8; 8]; + count_bytes.copy_from_slice(&bytes[CACHE_FILE_MAGIC.len()..header_len]); + let count = u64::from_le_bytes(count_bytes) as usize; + let body = &bytes[header_len..]; + if body.len() != count.saturating_mul(RECORD_BYTES) { + return None; + } + let mut entries = HashMap::with_capacity(count); + for chunk in body.chunks_exact(RECORD_BYTES) { + let mut key = [0u8; 32]; + key.copy_from_slice(&chunk[..32]); + let mut secs_bytes = [0u8; 8]; + secs_bytes.copy_from_slice(&chunk[32..40]); + let mut version_bytes = [0u8; 4]; + version_bytes.copy_from_slice(&chunk[40..44]); + entries.insert( + key, + CacheValue { + verified_at_unix_secs: i64::from_le_bytes(secs_bytes), + crypto_impl_version: u32::from_le_bytes(version_bytes), + }, + ); + } + Some(entries) +} + +fn persist_cache_file( + path: &Path, + shards: &[Mutex>], +) -> Result<(), String> { + let mut bytes = Vec::new(); + bytes.extend_from_slice(CACHE_FILE_MAGIC); + // Placeholder for the record count; patched after collecting. + bytes.extend_from_slice(&[0u8; 8]); + let mut count = 0u64; + for shard in shards { + let map = shard.lock().expect("crypto sig shard lock"); + for (key, value) in map.iter() { + bytes.extend_from_slice(key); + bytes.extend_from_slice(&value.verified_at_unix_secs.to_le_bytes()); + bytes.extend_from_slice(&value.crypto_impl_version.to_le_bytes()); + bytes.extend_from_slice(&[0u8; 4]); + count += 1; + } + } + bytes[CACHE_FILE_MAGIC.len()..CACHE_FILE_MAGIC.len() + 8].copy_from_slice(&count.to_le_bytes()); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| format!("create cache-file dir failed: {}: {e}", parent.display()))?; + } + let tmp_path = path.with_extension("tmp"); + std::fs::write(&tmp_path, &bytes) + .map_err(|e| format!("write cache file failed: {}: {e}", tmp_path.display()))?; + std::fs::rename(&tmp_path, path).map_err(|e| { + format!( + "rename cache file failed: {} -> {}: {e}", + tmp_path.display(), + path.display() + ) + })?; + Ok(()) +} + +static GLOBAL_CACHE: RwLock>> = RwLock::new(None); + +/// Install the process-global cache (called once from the CLI run path when +/// `--crypto-signature-cache-observe-only` and/or `--enable-crypto-signature-cache` is set). +pub fn install_global(cache: Arc) { + *GLOBAL_CACHE.write().expect("crypto sig cache lock") = Some(cache); +} + +/// Remove the process-global cache. Used by tests and by the CLI after the run finished. +pub fn clear_global() { + *GLOBAL_CACHE.write().expect("crypto sig cache lock") = None; +} + +pub fn global_cache() -> Option> { + GLOBAL_CACHE.read().expect("crypto sig cache lock").clone() +} + +/// Choke-point hook: cache-aware wrapper around a real signature verification. +/// +/// With no cache installed this is a single read-lock check plus the verification itself; +/// verification semantics never change either way (a hit only skips the crypto primitive, +/// never the surrounding revocation/time/policy checks). +pub fn verify_with_cache( + point: CryptoSigVerifyPoint, + input_bytes: &[u8], + signature_bytes: &[u8], + verifier_spki_der: &[u8], + verify: impl FnOnce() -> Result<(), E>, +) -> Result<(), E> { + let Some(cache) = global_cache() else { + return verify(); + }; + cache.verify( + point, + input_bytes, + signature_bytes, + verifier_spki_der, + verify, + ) +} + +#[cfg(test)] +pub(crate) static GLOBAL_TEST_LOCK: Mutex<()> = Mutex::new(()); + +#[cfg(test)] +mod tests { + use super::*; + + fn temp_cache(reuse_enabled: bool) -> (tempfile::TempDir, CryptoSigCache) { + let dir = tempfile::tempdir().expect("tempdir"); + let cache = CryptoSigCache::load_or_rebuild( + dir.path().join("work-db.crypto-sig-cache"), + reuse_enabled, + ); + (dir, cache) + } + + #[test] + fn cache_key_is_deterministic_and_domain_separated() { + let a = compute_cache_key( + CryptoSigVerifyPoint::CmsSignedObject, + b"input", + b"sig", + b"spki", + ); + let b = compute_cache_key( + CryptoSigVerifyPoint::CmsSignedObject, + b"input", + b"sig", + b"spki", + ); + assert_eq!(a, b); + + // Different verify point. + assert_ne!( + a, + compute_cache_key(CryptoSigVerifyPoint::Crl, b"input", b"sig", b"spki") + ); + // Different input bytes. + assert_ne!( + a, + compute_cache_key( + CryptoSigVerifyPoint::CmsSignedObject, + b"input2", + b"sig", + b"spki" + ) + ); + // Different signature bytes over the same input must not share a key. + assert_ne!( + a, + compute_cache_key( + CryptoSigVerifyPoint::CmsSignedObject, + b"input", + b"sig2", + b"spki" + ) + ); + // Different verifier key. + assert_ne!( + a, + compute_cache_key( + CryptoSigVerifyPoint::CmsSignedObject, + b"input", + b"sig", + b"spki2" + ) + ); + } + + #[test] + fn miss_verifies_and_writes_then_hit_skips_verify() { + let (_dir, cache) = temp_cache(true); + let mut executed = 0u64; + // First call: miss -> real verification runs and passes -> entry written. + cache + .verify( + CryptoSigVerifyPoint::EeCertFast, + b"tbs", + b"sig", + b"spki", + || -> Result<(), String> { + executed += 1; + Ok(()) + }, + ) + .expect("verify ok"); + assert_eq!(executed, 1); + // Second call: hit -> verification skipped entirely. + cache + .verify( + CryptoSigVerifyPoint::EeCertFast, + b"tbs", + b"sig", + b"spki", + || -> Result<(), String> { + executed += 1; + Ok(()) + }, + ) + .expect("verify ok"); + assert_eq!(executed, 1, "cache hit must skip the real verification"); + let summary = cache.summary(); + let stats = summary + .per_point + .get("ee_cert_fast") + .expect("stats present"); + assert_eq!(stats.calls, 2); + assert_eq!(stats.new_keys, 1); + assert_eq!(stats.would_hit, 1); + assert_eq!(stats.verify_executed, 1); + assert_eq!(stats.verify_skipped, 1); + assert!(summary.reuse_enabled); + } + + #[test] + fn preset_positive_key_skips_verify_that_would_fail() { + let (_dir, cache) = temp_cache(true); + // Tampered object bytes whose signature can never verify; pre-seed the positive + // conclusion directly (as if a previous run had verified the untampered object). + let key = compute_cache_key( + CryptoSigVerifyPoint::CmsSignedObject, + b"tampered-input", + b"tampered-sig", + b"spki", + ); + cache.insert_positive(key); + let mut executed = 0u64; + let result = cache.verify( + CryptoSigVerifyPoint::CmsSignedObject, + b"tampered-input", + b"tampered-sig", + b"spki", + || -> Result<(), String> { + executed += 1; + Err("signature does not verify".to_string()) + }, + ); + assert_eq!(result, Ok(()), "cache hit must return Ok without verifying"); + assert_eq!(executed, 0, "real verification must not run on a hit"); + } + + #[test] + fn failed_verify_is_not_cached_and_reruns_next_time() { + let (_dir, cache) = temp_cache(true); + let mut executed = 0u64; + for _ in 0..2 { + let result = cache.verify( + CryptoSigVerifyPoint::Crl, + b"tbs", + b"sig", + b"spki", + || -> Result<(), String> { + executed += 1; + Err("bad signature".to_string()) + }, + ); + assert!(result.is_err()); + } + assert_eq!(executed, 2, "no negative caching: every call re-verifies"); + let key = compute_cache_key(CryptoSigVerifyPoint::Crl, b"tbs", b"sig", b"spki"); + assert!(!cache.contains(&key)); + let summary = cache.summary(); + let stats = summary.per_point.get("crl").expect("stats present"); + assert_eq!(stats.new_keys, 2); + assert_eq!(stats.verify_executed, 2); + assert_eq!(stats.verify_skipped, 0); + } + + #[test] + fn observe_only_mode_counts_would_hit_but_still_verifies() { + let (_dir, cache) = temp_cache(false); + let key = compute_cache_key(CryptoSigVerifyPoint::ChildCaCert, b"tbs", b"sig", b"spki"); + cache.insert_positive(key); + let mut executed = 0u64; + cache + .verify( + CryptoSigVerifyPoint::ChildCaCert, + b"tbs", + b"sig", + b"spki", + || -> Result<(), String> { + executed += 1; + Ok(()) + }, + ) + .expect("verify ok"); + assert_eq!( + executed, 1, + "observe-only mode must still run the verification" + ); + let summary = cache.summary(); + assert!(!summary.reuse_enabled); + let stats = summary + .per_point + .get("child_ca_cert") + .expect("stats present"); + assert_eq!(stats.would_hit, 1); + assert_eq!(stats.verify_executed, 1); + assert_eq!(stats.verify_skipped, 0); + } + + #[test] + fn cache_file_roundtrips_entries_across_reload() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("work-db.crypto-sig-cache"); + let cache = CryptoSigCache::load_or_rebuild(path.clone(), true); + let key = compute_cache_key(CryptoSigVerifyPoint::EeCertFast, b"tbs", b"sig", b"spki"); + cache.insert_positive(key); + cache.persist().expect("persist"); + + let reloaded = CryptoSigCache::load_or_rebuild(path, true); + assert!(reloaded.contains(&key)); + let summary = reloaded.summary(); + assert_eq!(summary.entries_loaded, 1); + assert_eq!(summary.entries_total, 1); + assert_eq!(summary.schema_version, CACHE_SCHEMA_VERSION); + + // Reloaded entry drives a skip. + let mut executed = 0u64; + reloaded + .verify( + CryptoSigVerifyPoint::EeCertFast, + b"tbs", + b"sig", + b"spki", + || -> Result<(), String> { + executed += 1; + Ok(()) + }, + ) + .expect("verify ok"); + assert_eq!(executed, 0); + } + + #[test] + fn cache_file_rebuilds_on_missing_corrupt_or_mismatched_schema() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("work-db.crypto-sig-cache"); + + // Missing file. + let cache = CryptoSigCache::load_or_rebuild(path.clone(), true); + assert_eq!(cache.summary().entries_loaded, 0); + + // Garbage content. + std::fs::write(&path, b"not a valid cache file").expect("write garbage"); + let cache = CryptoSigCache::load_or_rebuild(path.clone(), true); + assert_eq!(cache.summary().entries_loaded, 0); + + // M1 seen-set format (RPKICSC1) must not load as a v2 cache: cold start. + let mut bytes = Vec::new(); + bytes.extend_from_slice(b"RPKICSC1"); + bytes.extend_from_slice(&1u64.to_le_bytes()); + bytes.extend_from_slice(&[7u8; 32]); + std::fs::write(&path, &bytes).expect("write v1 file"); + let cache = CryptoSigCache::load_or_rebuild(path.clone(), true); + assert_eq!(cache.summary().entries_loaded, 0); + + // Valid v2 header but truncated body. + let mut bytes = Vec::new(); + bytes.extend_from_slice(CACHE_FILE_MAGIC); + bytes.extend_from_slice(&2u64.to_le_bytes()); + bytes.extend_from_slice(&[9u8; RECORD_BYTES]); + std::fs::write(&path, &bytes).expect("write truncated"); + let cache = CryptoSigCache::load_or_rebuild(path, true); + assert_eq!(cache.summary().entries_loaded, 0); + } + + #[test] + fn size_cap_triggers_full_rebuild_eviction() { + let dir = tempfile::tempdir().expect("tempdir"); + let cache = CryptoSigCache::load_or_rebuild_with_capacity( + dir.path().join("work-db.crypto-sig-cache"), + true, + 2, + ); + let k1 = compute_cache_key(CryptoSigVerifyPoint::Crl, b"a", b"s", b"k"); + let k2 = compute_cache_key(CryptoSigVerifyPoint::Crl, b"b", b"s", b"k"); + let k3 = compute_cache_key(CryptoSigVerifyPoint::Crl, b"c", b"s", b"k"); + cache.insert_positive(k1); + cache.insert_positive(k2); + assert!(cache.contains(&k1) && cache.contains(&k2)); + // Third insert exceeds the cap -> full rebuild, then insert. + cache.insert_positive(k3); + assert!(!cache.contains(&k1)); + assert!(!cache.contains(&k2)); + assert!(cache.contains(&k3)); + let summary = cache.summary(); + assert_eq!(summary.entries_total, 1); + assert_eq!(summary.evictions_total, 1); + } + + #[test] + fn default_cache_file_path_mirrors_pp_cache_index_placement() { + let db = Path::new("/state/db/work-db"); + assert_eq!( + default_cache_file_path(db), + PathBuf::from("/state/db/work-db.crypto-sig-cache") + ); + } + + #[test] + fn verify_with_cache_without_global_cache_runs_verify_and_collects_nothing() { + let _guard = GLOBAL_TEST_LOCK.lock().expect("test lock"); + clear_global(); + assert!(global_cache().is_none()); + let mut executed = 0u64; + let result: Result = (|| { + verify_with_cache( + CryptoSigVerifyPoint::CmsSignedObject, + b"input", + b"sig", + b"spki", + || -> Result<(), String> { + executed += 1; + Ok(()) + }, + )?; + Ok(42) + })(); + assert_eq!(result, Ok(42)); + assert_eq!(executed, 1); + assert!(global_cache().is_none()); + } + + #[test] + fn verify_with_cache_preserves_result_through_global_install() { + let _guard = GLOBAL_TEST_LOCK.lock().expect("test lock"); + clear_global(); + let dir = tempfile::tempdir().expect("tempdir"); + let cache = Arc::new(CryptoSigCache::load_or_rebuild( + dir.path().join("work-db.crypto-sig-cache"), + true, + )); + install_global(Arc::clone(&cache)); + + let ok: Result<(), String> = verify_with_cache( + CryptoSigVerifyPoint::ChildCaCert, + b"tbs", + b"sig", + b"spki", + || Ok(()), + ); + assert_eq!(ok, Ok(())); + let err: Result<(), String> = verify_with_cache( + CryptoSigVerifyPoint::ChildCaCert, + b"tbs2", + b"sig", + b"spki", + || Err("bad signature".to_string()), + ); + assert_eq!(err, Err("bad signature".to_string())); + let summary = cache.summary(); + let stats = summary + .per_point + .get("child_ca_cert") + .expect("stats present"); + assert_eq!(stats.calls, 2); + assert_eq!(stats.new_keys, 2); + clear_global(); + assert!(global_cache().is_none()); + } +} diff --git a/crates/panda-rpki-validator/src/current_repo_index.rs b/crates/panda-rpki-validator/src/current_repo_index.rs new file mode 100644 index 0000000..1dc6e79 --- /dev/null +++ b/crates/panda-rpki-validator/src/current_repo_index.rs @@ -0,0 +1,290 @@ +use std::collections::{HashMap, HashSet}; +use std::sync::{Arc, RwLock}; + +use crate::storage::{RepositoryViewEntry, RepositoryViewState}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CurrentRepoEntry { + pub current_hash: [u8; 32], + pub current_hash_hex: String, + pub repository_source: String, + pub object_type: Option, + pub state: RepositoryViewState, +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct CurrentRepoObject { + pub rsync_uri: String, + pub current_hash_hex: String, + pub repository_source: String, + pub object_type: Option, +} + +#[derive(Default, Debug)] +pub struct CurrentRepoIndex { + by_uri: HashMap, +} + +/// Shared handle to the run-wide current repository index. Readers (phase 2 +/// publication point staging, cache lookups, output snapshots) take +/// `.read()` and no longer exclude each other; writers (repo sync transports +/// applying repository view entries, run-state reset) take `.write()` and +/// stay exclusive. +pub type CurrentRepoIndexHandle = Arc>; + +impl CurrentRepoIndex { + pub fn new() -> Self { + Self::default() + } + + pub fn shared() -> CurrentRepoIndexHandle { + Arc::new(RwLock::new(Self::new())) + } + + pub fn get_by_uri(&self, rsync_uri: &str) -> Option<&CurrentRepoEntry> { + self.by_uri.get(rsync_uri) + } + + pub fn list_scope_uris(&self, repository_source: &str) -> Vec { + let mut out = self + .by_uri + .iter() + .filter_map(|(rsync_uri, entry)| { + (entry.repository_source == repository_source).then(|| rsync_uri.clone()) + }) + .collect::>(); + out.sort(); + out + } + + pub fn active_uri_count(&self) -> usize { + self.by_uri.len() + } + + pub fn scope_count(&self) -> usize { + self.by_uri + .values() + .map(|entry| entry.repository_source.as_str()) + .collect::>() + .len() + } + + pub fn snapshot_objects(&self) -> Vec { + let mut out = self + .by_uri + .iter() + .map(|(rsync_uri, entry)| CurrentRepoObject { + rsync_uri: rsync_uri.clone(), + current_hash_hex: entry.current_hash_hex.clone(), + repository_source: entry.repository_source.clone(), + object_type: entry.object_type.clone(), + }) + .collect::>(); + out.sort(); + out + } + + pub fn clear(&mut self) { + self.by_uri.clear(); + } + + pub fn apply_repository_view_entries( + &mut self, + entries: &[RepositoryViewEntry], + ) -> Result<(), String> { + for entry in entries { + self.apply_repository_view_entry(entry)?; + } + Ok(()) + } + + fn apply_repository_view_entry(&mut self, entry: &RepositoryViewEntry) -> Result<(), String> { + entry.validate_internal().map_err(|e| e.to_string())?; + + match entry.state { + RepositoryViewState::Present | RepositoryViewState::Replaced => { + let repository_source = entry.repository_source.clone().ok_or_else(|| { + format!( + "repository_view entry missing repository_source for current object {}", + entry.rsync_uri + ) + })?; + let current_hash_hex = entry.current_hash.clone().ok_or_else(|| { + format!( + "repository_view entry missing current_hash for current object {}", + entry.rsync_uri + ) + })?; + let current_hash = decode_sha256_hex_32(¤t_hash_hex)?; + self.by_uri.insert( + entry.rsync_uri.clone(), + CurrentRepoEntry { + current_hash, + current_hash_hex: current_hash_hex.to_ascii_lowercase(), + repository_source, + object_type: entry.object_type.clone(), + state: entry.state, + }, + ); + } + RepositoryViewState::Withdrawn => { + self.by_uri.remove(&entry.rsync_uri); + } + } + + Ok(()) + } +} + +fn decode_sha256_hex_32(value: &str) -> Result<[u8; 32], String> { + if value.len() != 64 || !value.as_bytes().iter().all(u8::is_ascii_hexdigit) { + return Err(format!("invalid sha256 hex: {value}")); + } + let mut out = [0u8; 32]; + hex::decode_to_slice(value, &mut out).map_err(|e| format!("hex decode failed: {e}"))?; + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::CurrentRepoIndex; + use crate::storage::{RepositoryViewEntry, RepositoryViewState}; + + fn present(source: &str, uri: &str, hash: &str) -> RepositoryViewEntry { + RepositoryViewEntry { + rsync_uri: uri.to_string(), + current_hash: Some(hash.to_string()), + repository_source: Some(source.to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + } + } + + #[test] + fn current_repo_index_tracks_present_and_withdrawn_entries() { + let mut index = CurrentRepoIndex::new(); + let uri = "rsync://example.test/repo/a.roa"; + let source = "rsync://example.test/repo/"; + let hash = &"11".repeat(32); + index + .apply_repository_view_entries(&[present(source, uri, hash)]) + .expect("apply present"); + let got = index.get_by_uri(uri).expect("current entry"); + assert_eq!(got.current_hash_hex, hash.to_string()); + assert_eq!(index.list_scope_uris(source), vec![uri.to_string()]); + + index + .apply_repository_view_entries(&[RepositoryViewEntry { + rsync_uri: uri.to_string(), + current_hash: Some(hash.to_string()), + repository_source: Some(source.to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Withdrawn, + }]) + .expect("apply withdrawn"); + assert!(index.get_by_uri(uri).is_none()); + assert!(index.list_scope_uris(source).is_empty()); + } + + #[test] + fn current_repo_index_moves_uri_between_scopes() { + let mut index = CurrentRepoIndex::new(); + let uri = "rsync://example.test/repo/a.roa"; + let old_scope = "rsync://example.test/repo/"; + let new_scope = "https://rrdp.example.test/notification.xml"; + index + .apply_repository_view_entries(&[present(old_scope, uri, &"22".repeat(32))]) + .expect("apply old scope"); + index + .apply_repository_view_entries(&[present(new_scope, uri, &"33".repeat(32))]) + .expect("apply new scope"); + + assert!(index.list_scope_uris(old_scope).is_empty()); + assert_eq!(index.list_scope_uris(new_scope), vec![uri.to_string()]); + assert_eq!( + index.get_by_uri(uri).expect("entry").current_hash_hex, + "33".repeat(32) + ); + } + + #[test] + fn current_repo_index_snapshot_objects_and_counts_are_sorted() { + let handle = CurrentRepoIndex::shared(); + let mut index = handle.write().expect("write-lock index"); + index + .apply_repository_view_entries(&[ + present( + "rsync://example.test/repo-b/", + "rsync://example.test/repo-b/b.roa", + &"22".repeat(32), + ), + present( + "rsync://example.test/repo-a/", + "rsync://example.test/repo-a/a.roa", + &"11".repeat(32), + ), + ]) + .expect("apply present entries"); + assert_eq!(index.active_uri_count(), 2); + assert_eq!(index.scope_count(), 2); + + let snapshot = index.snapshot_objects(); + assert_eq!(snapshot.len(), 2); + assert_eq!(snapshot[0].rsync_uri, "rsync://example.test/repo-a/a.roa"); + assert_eq!(snapshot[1].rsync_uri, "rsync://example.test/repo-b/b.roa"); + } + + #[test] + fn current_repo_index_reports_missing_fields_and_invalid_hash() { + let mut index = CurrentRepoIndex::new(); + let err = index + .apply_repository_view_entries(&[RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/a.roa".to_string(), + current_hash: Some("11".repeat(32)), + repository_source: None, + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }]) + .expect_err("missing source should fail"); + assert!(err.contains("missing repository_source"), "{err}"); + + let err = index + .apply_repository_view_entries(&[RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/a.roa".to_string(), + current_hash: Some("not-a-valid-sha256".to_string()), + repository_source: Some("rsync://example.test/repo/".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }]) + .expect_err("invalid hash should fail"); + assert!(err.contains("invalid"), "{err}"); + + index + .apply_repository_view_entries(&[RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/b.roa".to_string(), + current_hash: Some("22".repeat(32)), + repository_source: Some("rsync://example.test/repo/".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }]) + .expect("valid entry"); + let got = index.get_by_uri("rsync://example.test/repo/b.roa").unwrap(); + assert_eq!(got.current_hash_hex, "22".repeat(32)); + } + + #[test] + fn current_repo_index_withdraw_unknown_uri_is_noop() { + let mut index = CurrentRepoIndex::new(); + index + .apply_repository_view_entries(&[RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/missing.roa".to_string(), + current_hash: None, + repository_source: Some("rsync://example.test/repo/".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Withdrawn, + }]) + .expect("withdraw unknown should not fail"); + assert_eq!(index.active_uri_count(), 0); + assert_eq!(index.scope_count(), 0); + } +} diff --git a/crates/panda-rpki-validator/src/daemon.rs b/crates/panda-rpki-validator/src/daemon.rs new file mode 100644 index 0000000..4f9095b --- /dev/null +++ b/crates/panda-rpki-validator/src/daemon.rs @@ -0,0 +1,2216 @@ +use serde::Serialize; +use std::collections::BTreeMap; +use std::fs::{self, File, OpenOptions}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::Duration; + +use crate::parallel::dead_repo_blacklist::DeadRepoBlacklist; +use crate::parallel::types::RepoTransportMode; + +#[derive(Clone, Debug, PartialEq, Eq)] +struct Args { + state_root: PathBuf, + rpki_bin: PathBuf, + interval_secs: u64, + max_runs: Option, + retain_runs: usize, + status_json: Option, + summary_jsonl: Option, + work_db: PathBuf, + repo_bytes_db: Option, + raw_store_db: Option, + db_stats_bin: Option, + db_stats_exact_every: Option, + time_bin: Option, + /// Dead-repo blacklist (#141): enables daemon-side health checks and + /// auto-injects the child `--dead-repo-blacklist` flag when absent. + dead_repo_blacklist: Option, + dead_repo_blacklist_fail_threshold: Option, + dead_repo_health_check_interval_secs: u64, + dead_repo_probe_rsync_bin: PathBuf, + child_args: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct RunContext { + seq: u64, + run_id: String, + run_dir: PathBuf, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +enum DaemonState { + Starting, + Idle, + Running, + Collecting, + Sleeping, + Exited, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +struct DaemonStatus { + state: DaemonState, + updated_at_rfc3339_utc: String, + runs_completed: u64, + max_runs: Option, + current_run_seq: Option, + current_run_id: Option, + last_run_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + dead_repo_blacklist: Option, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +struct DeadRepoBlacklistStatus { + path: String, + entries: usize, + blacklisted: usize, + last_health_check_at_rfc3339_utc: Option, +} + +/// Mutable daemon-side health check bookkeeping (#141). +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct DeadRepoHealthRuntime { + last_health_check_at: Option, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +enum RunStatus { + Success, + Failed, + SpawnFailed, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +struct ArtifactInfo { + path: String, + size_bytes: u64, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct ProcessMetrics { + time_wrapper_used: bool, + time_output_path: Option, + user_seconds: Option, + system_seconds: Option, + cpu_percent: Option, + elapsed_raw: Option, + max_rss_kb: Option, + exit_status_from_time: Option, + parse_error: Option, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +struct ReportCounts { + vrps: usize, + aspas: usize, + publication_points: usize, + rrdp_repos_unique: Option, + tree_instances_processed: Option, + tree_instances_failed: Option, + warnings: usize, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +struct PathFileStats { + label: String, + path: String, + exists: bool, + is_dir: bool, + total_size_bytes: u64, + file_count: u64, + dir_count: u64, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +struct DbStatsSummary { + mode: String, + db_path: String, + output_path: Option, + stderr_path: Option, + status: String, + exit_code: Option, + error: Option, + metrics: BTreeMap, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct RunSummary { + run_seq: u64, + run_id: String, + run_dir: String, + started_at_rfc3339_utc: String, + finished_at_rfc3339_utc: String, + wall_ms: u64, + status: RunStatus, + exit_code: Option, + exit_status: Option, + error: Option, + rpki_bin: String, + child_args: Vec, + stdout_path: String, + stderr_path: String, + process_metrics: Option, + stage_timing: Option, + report_counts: Option, + repo_sync_stats: Option, + path_stats: Vec, + db_stats: Vec, + retention_deleted_runs: Vec, + artifacts: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + dead_repo_blacklist: Option, +} + +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +struct DeadRepoBlacklistRunSummary { + path: String, + entries: usize, + blacklisted: usize, +} + +fn usage() -> String { + let bin = "rpki_daemon"; + format!( + "\ +Usage: + {bin} --state-root --rpki-bin [options] -- + +Options: + --state-root Persistent daemon root containing state/, runs/, status, and JSONL summary + --rpki-bin rpki child binary to execute for each run + --interval-secs Sleep seconds between runs (default: 60) + --max-runs Stop after n runs (default: run forever) + --retain-runs Keep only the latest n run directories (default: 10) + --status-json Override status JSON path (default: /daemon-status.json) + --summary-jsonl Override summary JSONL path (default: /daemon-runs.jsonl) + --work-db Work DB path for metrics (default: /state/work-db) + --repo-bytes-db Repo bytes DB path for file metrics (default: /state/repo-bytes.db) + --raw-store-db Raw store DB path for file metrics (optional) + --db-stats-bin db_stats binary path (default: sibling db_stats next to this executable when present) + --db-stats-exact-every + Run db_stats --exact every n runs (default: disabled) + --time-bin GNU time binary for child process metrics (default: /usr/bin/time when present) + --no-time-wrapper Disable GNU time wrapper + --dead-repo-blacklist + Enable the dead-repo transport blacklist (#141): daemon probes + blacklisted entries between runs and removes revived repos; + auto-injects the child flag when not already present + --dead-repo-blacklist-fail-threshold + Admission threshold forwarded to the child when its args do not + already set it (default: child default 3) + --dead-repo-health-check-interval-secs + Minimum seconds between health check sweeps (default: 600) + --dead-repo-probe-rsync-bin + rsync binary used for health probes (default: rsync) + --help Show this help + +Child argument placeholders: + {{state_root}} Daemon state root + {{run_out}} Current run output directory + {{run_id}} Current run id, e.g. 000001-20260428T090000Z + {{run_seq}} Current run sequence number +" + ) +} + +fn default_time_bin() -> Option { + let path = PathBuf::from("/usr/bin/time"); + if path.is_file() { Some(path) } else { None } +} + +fn default_db_stats_bin() -> Option { + let mut path = std::env::current_exe().ok()?; + path.set_file_name("db_stats"); + if path.is_file() { Some(path) } else { None } +} + +fn parse_args(argv: &[String]) -> Result { + if argv.iter().any(|arg| arg == "--help" || arg == "-h") { + return Err(usage()); + } + + let mut state_root: Option = None; + let mut rpki_bin: Option = None; + let mut interval_secs = 60u64; + let mut max_runs = None; + let mut retain_runs = 10usize; + let mut status_json: Option = None; + let mut summary_jsonl: Option = None; + let mut work_db: Option = None; + let mut repo_bytes_db: Option = None; + let mut raw_store_db: Option = None; + let mut db_stats_bin: Option = None; + let mut db_stats_exact_every = None; + let mut time_bin = default_time_bin(); + let mut no_time_wrapper = false; + let mut dead_repo_blacklist: Option = None; + let mut dead_repo_blacklist_fail_threshold: Option = None; + let mut dead_repo_health_check_interval_secs = 600u64; + let mut dead_repo_probe_rsync_bin = PathBuf::from("rsync"); + + let mut i = 1usize; + while i < argv.len() { + match argv[i].as_str() { + "--" => { + let mut child_args = argv[i + 1..].to_vec(); + let state_root = + state_root.ok_or_else(|| format!("--state-root is required\n\n{}", usage()))?; + let work_db = work_db.unwrap_or_else(|| state_root.join("state").join("work-db")); + let repo_bytes_db = + repo_bytes_db.or_else(|| Some(state_root.join("state").join("repo-bytes.db"))); + if no_time_wrapper { + time_bin = None; + } + // Dead-repo blacklist (#141): forward enablement to the child + // unless its args already carry the flag explicitly. + if let Some(path) = dead_repo_blacklist.as_ref() { + if !child_args.iter().any(|arg| arg == "--dead-repo-blacklist") { + child_args.push("--dead-repo-blacklist".to_string()); + child_args.push(path_string(path)); + } + if let Some(threshold) = dead_repo_blacklist_fail_threshold { + if !child_args + .iter() + .any(|arg| arg == "--dead-repo-blacklist-fail-threshold") + { + child_args.push("--dead-repo-blacklist-fail-threshold".to_string()); + child_args.push(threshold.to_string()); + } + } + } + let args = Args { + state_root, + rpki_bin: rpki_bin + .ok_or_else(|| format!("--rpki-bin is required\n\n{}", usage()))?, + interval_secs, + max_runs, + retain_runs, + status_json, + summary_jsonl, + work_db, + repo_bytes_db, + raw_store_db, + db_stats_bin, + db_stats_exact_every, + time_bin, + dead_repo_blacklist, + dead_repo_blacklist_fail_threshold, + dead_repo_health_check_interval_secs, + dead_repo_probe_rsync_bin, + child_args, + }; + return validate_args(args); + } + "--state-root" => { + i += 1; + state_root = Some(PathBuf::from(value_at(argv, i, "--state-root")?)); + } + "--rpki-bin" => { + i += 1; + rpki_bin = Some(PathBuf::from(value_at(argv, i, "--rpki-bin")?)); + } + "--interval-secs" => { + i += 1; + interval_secs = + parse_u64(value_at(argv, i, "--interval-secs")?, "--interval-secs")?; + } + "--max-runs" => { + i += 1; + let parsed = parse_u64(value_at(argv, i, "--max-runs")?, "--max-runs")?; + if parsed == 0 { + return Err("--max-runs must be > 0".to_string()); + } + max_runs = Some(parsed); + } + "--retain-runs" => { + i += 1; + let parsed = parse_usize(value_at(argv, i, "--retain-runs")?, "--retain-runs")?; + if parsed == 0 { + return Err("--retain-runs must be > 0".to_string()); + } + retain_runs = parsed; + } + "--status-json" => { + i += 1; + status_json = Some(PathBuf::from(value_at(argv, i, "--status-json")?)); + } + "--summary-jsonl" => { + i += 1; + summary_jsonl = Some(PathBuf::from(value_at(argv, i, "--summary-jsonl")?)); + } + "--work-db" => { + i += 1; + work_db = Some(PathBuf::from(value_at(argv, i, "--work-db")?)); + } + "--repo-bytes-db" => { + i += 1; + repo_bytes_db = Some(PathBuf::from(value_at(argv, i, "--repo-bytes-db")?)); + } + "--raw-store-db" => { + i += 1; + raw_store_db = Some(PathBuf::from(value_at(argv, i, "--raw-store-db")?)); + } + "--db-stats-bin" => { + i += 1; + db_stats_bin = Some(PathBuf::from(value_at(argv, i, "--db-stats-bin")?)); + } + "--db-stats-exact-every" => { + i += 1; + let parsed = parse_u64( + value_at(argv, i, "--db-stats-exact-every")?, + "--db-stats-exact-every", + )?; + if parsed == 0 { + return Err("--db-stats-exact-every must be > 0".to_string()); + } + db_stats_exact_every = Some(parsed); + } + "--time-bin" => { + i += 1; + time_bin = Some(PathBuf::from(value_at(argv, i, "--time-bin")?)); + } + "--no-time-wrapper" => { + no_time_wrapper = true; + } + "--dead-repo-blacklist" => { + i += 1; + dead_repo_blacklist = + Some(PathBuf::from(value_at(argv, i, "--dead-repo-blacklist")?)); + } + "--dead-repo-blacklist-fail-threshold" => { + i += 1; + let parsed = parse_u64( + value_at(argv, i, "--dead-repo-blacklist-fail-threshold")?, + "--dead-repo-blacklist-fail-threshold", + )?; + if parsed == 0 || parsed > u32::MAX as u64 { + return Err( + "--dead-repo-blacklist-fail-threshold must be in 1..=u32::MAX".to_string(), + ); + } + dead_repo_blacklist_fail_threshold = Some(parsed as u32); + } + "--dead-repo-health-check-interval-secs" => { + i += 1; + dead_repo_health_check_interval_secs = parse_u64( + value_at(argv, i, "--dead-repo-health-check-interval-secs")?, + "--dead-repo-health-check-interval-secs", + )?; + } + "--dead-repo-probe-rsync-bin" => { + i += 1; + dead_repo_probe_rsync_bin = + PathBuf::from(value_at(argv, i, "--dead-repo-probe-rsync-bin")?); + } + other => return Err(format!("unknown argument: {other}\n\n{}", usage())), + } + i += 1; + } + + Err(format!("missing -- before child rpki args\n\n{}", usage())) +} + +fn validate_args(args: Args) -> Result { + if args.child_args.is_empty() { + return Err(format!( + "child rpki args are required after --\n\n{}", + usage() + )); + } + if args.dead_repo_blacklist_fail_threshold.is_some() && args.dead_repo_blacklist.is_none() { + return Err( + "--dead-repo-blacklist-fail-threshold requires --dead-repo-blacklist".to_string(), + ); + } + Ok(args) +} + +fn value_at<'a>(argv: &'a [String], index: usize, flag: &str) -> Result<&'a str, String> { + argv.get(index) + .map(String::as_str) + .ok_or_else(|| format!("{flag} requires a value")) +} + +fn parse_u64(raw: &str, flag: &str) -> Result { + raw.parse::() + .map_err(|_| format!("invalid {flag}: {raw}")) +} + +fn parse_usize(raw: &str, flag: &str) -> Result { + raw.parse::() + .map_err(|_| format!("invalid {flag}: {raw}")) +} + +fn status_path(args: &Args) -> PathBuf { + args.status_json + .clone() + .unwrap_or_else(|| args.state_root.join("daemon-status.json")) +} + +fn summary_jsonl_path(args: &Args) -> PathBuf { + args.summary_jsonl + .clone() + .unwrap_or_else(|| args.state_root.join("daemon-runs.jsonl")) +} + +fn utc_now() -> time::OffsetDateTime { + time::OffsetDateTime::now_utc().to_offset(time::UtcOffset::UTC) +} + +fn format_rfc3339(t: time::OffsetDateTime) -> Result { + t.format(&time::format_description::well_known::Rfc3339) + .map_err(|e| format!("format RFC3339 failed: {e}")) +} + +fn format_compact_utc(t: time::OffsetDateTime) -> String { + format!( + "{:04}{:02}{:02}T{:02}{:02}{:02}Z", + t.year(), + u8::from(t.month()), + t.day(), + t.hour(), + t.minute(), + t.second() + ) +} + +fn write_json_pretty(path: &Path, value: &T) -> Result<(), String> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .map_err(|e| format!("create parent dir failed: {}: {e}", parent.display()))?; + } + let bytes = serde_json::to_vec_pretty(value) + .map_err(|e| format!("serialize json failed: {}: {e}", path.display()))?; + // Atomic write (tmp + rename) so concurrent readers never see a torn file. + let mut tmp_name = path + .file_name() + .map(|name| name.to_os_string()) + .unwrap_or_default(); + tmp_name.push(".tmp"); + let tmp_path = path.with_file_name(tmp_name); + fs::write(&tmp_path, &bytes) + .map_err(|e| format!("write json tmp failed: {}: {e}", tmp_path.display()))?; + fs::rename(&tmp_path, path).map_err(|e| { + format!( + "rename json tmp failed: {} -> {}: {e}", + tmp_path.display(), + path.display() + ) + }) +} + +fn append_json_line(path: &Path, value: &T) -> Result<(), String> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + .map_err(|e| format!("create parent dir failed: {}: {e}", parent.display()))?; + } + let mut file = OpenOptions::new() + .create(true) + .append(true) + .open(path) + .map_err(|e| format!("open jsonl failed: {}: {e}", path.display()))?; + serde_json::to_writer(&mut file, value) + .map_err(|e| format!("write jsonl failed: {}: {e}", path.display()))?; + file.write_all(b"\n") + .map_err(|e| format!("flush jsonl failed: {}: {e}", path.display())) +} + +fn dead_repo_blacklist_status( + args: &Args, + health: &DeadRepoHealthRuntime, +) -> Option { + let path = args.dead_repo_blacklist.as_ref()?; + let (blacklist, _) = DeadRepoBlacklist::load(path); + let last_health_check_at_rfc3339_utc = health + .last_health_check_at + .and_then(|t| format_rfc3339(t).ok()); + Some(DeadRepoBlacklistStatus { + path: path_string(path), + entries: blacklist.len(), + blacklisted: blacklist.blacklisted_len(), + last_health_check_at_rfc3339_utc, + }) +} + +fn write_status( + args: &Args, + state: DaemonState, + runs_completed: u64, + current: Option<&RunContext>, + last_run_id: Option, + health: &DeadRepoHealthRuntime, +) -> Result<(), String> { + let updated_at_rfc3339_utc = format_rfc3339(utc_now())?; + let status = DaemonStatus { + state, + updated_at_rfc3339_utc, + runs_completed, + max_runs: args.max_runs, + current_run_seq: current.map(|ctx| ctx.seq), + current_run_id: current.map(|ctx| ctx.run_id.clone()), + last_run_id, + dead_repo_blacklist: dead_repo_blacklist_status(args, health), + }; + write_json_pretty(&status_path(args), &status) +} + +/// Short-timeout HTTP GET of the RRDP notification file. Any HTTP status +/// (even 4xx) proves the transport is alive; only transport errors keep the +/// entry blacklisted. +fn probe_rrdp_transport(uri: &str) -> bool { + let config = crate::fetch::http::HttpFetcherConfig { + connect_timeout: Duration::from_secs(3), + timeout: Duration::from_secs(6), + large_body_timeout: Duration::from_secs(6), + ..Default::default() + }; + let fetcher = match crate::fetch::http::BlockingHttpFetcher::new(config) { + Ok(fetcher) => fetcher, + Err(_) => return false, + }; + match fetcher.fetch_bytes(uri) { + Ok(_) => true, + Err(err) => err.starts_with("http status"), + } +} + +/// rsync list-only probe: connecting to the daemon and listing the module +/// root is enough to prove the transport is alive. +fn probe_rsync_transport(rsync_bin: &Path, base_uri: &str) -> bool { + Command::new(rsync_bin) + .arg("--contimeout=5") + .arg("--timeout=8") + .arg(base_uri) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .map(|status| status.success()) + .unwrap_or(false) +} + +/// Probe due blacklist entries between runs (#141). Runs in the supervisor +/// loop while no child is active, preserving the single-writer assumption. +fn maybe_run_dead_repo_health_check(args: &Args, health: &mut DeadRepoHealthRuntime) { + let Some(path) = args.dead_repo_blacklist.clone() else { + return; + }; + let now = utc_now(); + if let Some(last) = health.last_health_check_at { + let elapsed = (now - last).whole_seconds(); + if elapsed >= 0 && (elapsed as u64) < args.dead_repo_health_check_interval_secs { + return; + } + } + let (mut blacklist, warning) = DeadRepoBlacklist::load(&path); + if let Some(warning) = warning { + eprintln!("[dead-repo-health] {warning}"); + } + let now_unix = now.unix_timestamp().max(0) as u64; + let due = blacklist.probe_due_entries(now_unix, args.dead_repo_health_check_interval_secs); + if due.is_empty() { + health.last_health_check_at = Some(now); + return; + } + // Probe in parallel: in the docker soak the daemon runs once per run + // (--max-runs 1), so a serial sweep of ~12 dead entries would add up to a + // minute of latency to every run. + let rsync_bin = args.dead_repo_probe_rsync_bin.clone(); + let probe_results: Vec<(RepoTransportMode, String, bool)> = due + .iter() + .map(|entry| (entry.transport, entry.uri.clone())) + .collect::>() + .into_iter() + .map(|(transport, uri)| { + let rsync_bin = rsync_bin.clone(); + std::thread::spawn(move || { + let alive = match transport { + RepoTransportMode::Rrdp => probe_rrdp_transport(&uri), + RepoTransportMode::Rsync => probe_rsync_transport(&rsync_bin, &uri), + }; + (transport, uri, alive) + }) + }) + .filter_map(|handle| handle.join().ok()) + .collect(); + let mut removed = 0usize; + let mut still_dead = 0usize; + for (transport, uri, alive) in &probe_results { + if *alive { + blacklist.record_probe_success(*transport, uri); + removed += 1; + eprintln!( + "[dead-repo-health] removed {} {} (probe ok)", + transport.as_str(), + uri + ); + crate::progress_log::emit( + "dead_repo_blacklist_remove", + serde_json::json!({ + "transport": transport.as_str(), + "uri": uri, + "reason": "health_probe_ok", + }), + ); + } else { + blacklist.record_probe_failure(*transport, uri, now_unix); + still_dead += 1; + eprintln!( + "[dead-repo-health] still dead {} {} (probe failed)", + transport.as_str(), + uri + ); + } + } + if let Err(err) = blacklist.store_atomic(&path, now_unix) { + eprintln!( + "[dead-repo-health] persist blacklist failed: {}: {err}", + path.display() + ); + } + crate::progress_log::emit( + "dead_repo_health_check", + serde_json::json!({ + "probed": probe_results.len(), + "removed": removed, + "still_dead": still_dead, + }), + ); + health.last_health_check_at = Some(now); +} + +fn render_child_args(args: &[String], daemon_args: &Args, ctx: &RunContext) -> Vec { + args.iter() + .map(|arg| { + arg.replace("{state_root}", &path_string(&daemon_args.state_root)) + .replace("{run_out}", &path_string(&ctx.run_dir)) + .replace("{run_id}", &ctx.run_id) + .replace("{run_seq}", &ctx.seq.to_string()) + }) + .collect() +} + +fn render_path_template(path: &Path, daemon_args: &Args, ctx: &RunContext) -> PathBuf { + PathBuf::from( + path_string(path) + .replace("{state_root}", &path_string(&daemon_args.state_root)) + .replace("{run_out}", &path_string(&ctx.run_dir)) + .replace("{run_id}", &ctx.run_id) + .replace("{run_seq}", &ctx.seq.to_string()), + ) +} + +fn path_string(path: &Path) -> String { + path.to_string_lossy().into_owned() +} + +fn make_run_context(args: &Args, seq: u64, now: time::OffsetDateTime) -> RunContext { + let run_id = format!("{seq:06}-{}", format_compact_utc(now)); + let run_dir = args.state_root.join("runs").join(&run_id); + RunContext { + seq, + run_id, + run_dir, + } +} + +fn collect_artifacts(run_dir: &Path) -> Result, String> { + let mut artifacts = Vec::new(); + for entry in fs::read_dir(run_dir) + .map_err(|e| format!("read run dir failed: {}: {e}", run_dir.display()))? + { + let entry = entry.map_err(|e| format!("read run dir entry failed: {e}"))?; + if !entry + .file_type() + .map_err(|e| format!("read file type failed: {}: {e}", entry.path().display()))? + .is_file() + { + continue; + } + let metadata = entry + .metadata() + .map_err(|e| format!("read metadata failed: {}: {e}", entry.path().display()))?; + artifacts.push(ArtifactInfo { + path: path_string(&entry.path()), + size_bytes: metadata.len(), + }); + } + artifacts.sort_by(|a, b| a.path.cmp(&b.path)); + Ok(artifacts) +} + +fn collect_process_metrics(time_wrapper_used: bool, time_output_path: &Path) -> ProcessMetrics { + if !time_wrapper_used { + return ProcessMetrics { + time_wrapper_used, + time_output_path: None, + user_seconds: None, + system_seconds: None, + cpu_percent: None, + elapsed_raw: None, + max_rss_kb: None, + exit_status_from_time: None, + parse_error: None, + }; + } + + let mut metrics = ProcessMetrics { + time_wrapper_used, + time_output_path: Some(path_string(time_output_path)), + user_seconds: None, + system_seconds: None, + cpu_percent: None, + elapsed_raw: None, + max_rss_kb: None, + exit_status_from_time: None, + parse_error: None, + }; + + let text = match fs::read_to_string(time_output_path) { + Ok(text) => text, + Err(err) => { + metrics.parse_error = Some(format!( + "read process time output failed: {}: {err}", + time_output_path.display() + )); + return metrics; + } + }; + + for line in text.lines() { + let line = line.trim(); + if let Some(value) = line.strip_prefix("User time (seconds):") { + metrics.user_seconds = value.trim().parse::().ok(); + } else if let Some(value) = line.strip_prefix("System time (seconds):") { + metrics.system_seconds = value.trim().parse::().ok(); + } else if let Some(value) = line.strip_prefix("Percent of CPU this job got:") { + metrics.cpu_percent = value.trim().trim_end_matches('%').parse::().ok(); + } else if let Some(value) = + line.strip_prefix("Elapsed (wall clock) time (h:mm:ss or m:ss):") + { + metrics.elapsed_raw = Some(value.trim().to_string()); + } else if let Some(value) = line.strip_prefix("Maximum resident set size (kbytes):") { + metrics.max_rss_kb = value.trim().parse::().ok(); + } else if let Some(value) = line.strip_prefix("Exit status:") { + metrics.exit_status_from_time = value.trim().parse::().ok(); + } + } + metrics +} + +fn run_child_once(args: &Args, ctx: &RunContext) -> Result { + fs::create_dir_all(&ctx.run_dir) + .map_err(|e| format!("create run dir failed: {}: {e}", ctx.run_dir.display()))?; + + let started_at = utc_now(); + let started_at_rfc3339_utc = format_rfc3339(started_at)?; + let stdout_path = ctx.run_dir.join("stdout.log"); + let stderr_path = ctx.run_dir.join("stderr.log"); + let stdout = File::create(&stdout_path) + .map_err(|e| format!("create stdout log failed: {}: {e}", stdout_path.display()))?; + let stderr = File::create(&stderr_path) + .map_err(|e| format!("create stderr log failed: {}: {e}", stderr_path.display()))?; + let child_args = render_child_args(&args.child_args, args, ctx); + let time_output_path = ctx.run_dir.join("process-time.txt"); + + let mut command = if let Some(time_bin) = args.time_bin.as_ref() { + let mut command = Command::new(time_bin); + command + .arg("-v") + .arg("-o") + .arg(&time_output_path) + .arg("--") + .arg(&args.rpki_bin) + .args(&child_args); + command + } else { + let mut command = Command::new(&args.rpki_bin); + command.args(&child_args); + command + }; + command + .stdout(Stdio::from(stdout)) + .stderr(Stdio::from(stderr)); + + let (status, exit_code, exit_status, error) = match command.status() { + Ok(status) if status.success() => ( + RunStatus::Success, + status.code(), + Some(status.to_string()), + None, + ), + Ok(status) => ( + RunStatus::Failed, + status.code(), + Some(status.to_string()), + None, + ), + Err(err) => ( + RunStatus::SpawnFailed, + None, + None, + Some(format!("spawn child failed: {err}")), + ), + }; + + let finished_at = utc_now(); + let finished_at_rfc3339_utc = format_rfc3339(finished_at)?; + let wall_ms = (finished_at - started_at).whole_milliseconds().max(0) as u64; + let process_metrics = Some(collect_process_metrics( + args.time_bin.is_some(), + &time_output_path, + )); + let artifacts = collect_artifacts(&ctx.run_dir)?; + let summary = RunSummary { + run_seq: ctx.seq, + run_id: ctx.run_id.clone(), + run_dir: path_string(&ctx.run_dir), + started_at_rfc3339_utc, + finished_at_rfc3339_utc, + wall_ms, + status, + exit_code, + exit_status, + error, + rpki_bin: path_string(&args.rpki_bin), + child_args, + stdout_path: path_string(&stdout_path), + stderr_path: path_string(&stderr_path), + process_metrics, + stage_timing: None, + report_counts: None, + repo_sync_stats: None, + path_stats: Vec::new(), + db_stats: Vec::new(), + retention_deleted_runs: Vec::new(), + artifacts, + dead_repo_blacklist: None, + }; + Ok(summary) +} + +fn apply_retention(runs_root: &Path, retain_runs: usize) -> Result, String> { + if !runs_root.exists() { + return Ok(Vec::new()); + } + let mut dirs = Vec::new(); + for entry in fs::read_dir(runs_root) + .map_err(|e| format!("read runs dir failed: {}: {e}", runs_root.display()))? + { + let entry = entry.map_err(|e| format!("read runs dir entry failed: {e}"))?; + if entry + .file_type() + .map_err(|e| format!("read file type failed: {}: {e}", entry.path().display()))? + .is_dir() + { + dirs.push(entry.path()); + } + } + dirs.sort(); + let remove_count = dirs.len().saturating_sub(retain_runs); + let mut removed = Vec::new(); + for dir in dirs.into_iter().take(remove_count) { + fs::remove_dir_all(&dir) + .map_err(|e| format!("remove old run dir failed: {}: {e}", dir.display()))?; + removed.push(dir); + } + Ok(removed) +} + +fn find_named_file(root: &Path, name: &str) -> Option { + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + let entries = fs::read_dir(&dir).ok()?; + for entry in entries.flatten() { + let path = entry.path(); + let file_type = entry.file_type().ok()?; + if file_type.is_file() && entry.file_name().to_string_lossy() == name { + return Some(path); + } + if file_type.is_dir() { + stack.push(path); + } + } + } + None +} + +fn read_json_value_if_exists(path: &Path) -> Option { + let bytes = fs::read(path).ok()?; + serde_json::from_slice(&bytes).ok() +} + +fn json_array_len(value: &serde_json::Value, key: &str) -> usize { + value + .get(key) + .and_then(serde_json::Value::as_array) + .map(Vec::len) + .unwrap_or(0) +} + +fn parse_stdout_summary(run_dir: &Path) -> Option { + let stdout_path = run_dir.join("stdout.log"); + let text = fs::read_to_string(stdout_path).ok()?; + let mut vrps = None; + let mut aspas = None; + let mut publication_points = None; + let mut rrdp_repos_unique = None; + let mut tree_instances_processed = None; + let mut tree_instances_failed = None; + let mut warnings = None; + + for line in text.lines() { + if let Some(value) = line.strip_prefix("vrps=") { + vrps = value.trim().parse::().ok(); + } else if let Some(value) = line.strip_prefix("aspas=") { + aspas = value.trim().parse::().ok(); + } else if let Some(value) = line.strip_prefix("audit_publication_points=") { + publication_points = value.trim().parse::().ok(); + } else if let Some(value) = line.strip_prefix("rrdp_repos_unique=") { + rrdp_repos_unique = value.trim().parse::().ok(); + } else if let Some(value) = line.strip_prefix("warnings_total=") { + warnings = value.trim().parse::().ok(); + } else if let Some(rest) = line.strip_prefix("publication_points_processed=") { + for token in rest.split_whitespace() { + if let Some(value) = token.strip_prefix("publication_points_failed=") { + tree_instances_failed = value.parse::().ok(); + } else if tree_instances_processed.is_none() { + tree_instances_processed = token.parse::().ok(); + } + } + } + } + + Some(ReportCounts { + vrps: vrps?, + aspas: aspas?, + publication_points: publication_points?, + rrdp_repos_unique, + tree_instances_processed, + tree_instances_failed, + warnings: warnings.unwrap_or(0), + }) +} + +fn parse_report_counts_fallback(report: &serde_json::Value) -> ReportCounts { + let tree = report.get("tree"); + let tree_warnings = tree + .and_then(|tree| tree.get("warnings")) + .and_then(serde_json::Value::as_array) + .map(Vec::len) + .unwrap_or(0); + let pp_warnings = report + .get("publication_points") + .and_then(serde_json::Value::as_array) + .map(|items| { + items + .iter() + .map(|pp| { + pp.get("warnings") + .and_then(serde_json::Value::as_array) + .map(Vec::len) + .unwrap_or(0) + }) + .sum() + }) + .unwrap_or(0); + ReportCounts { + vrps: json_array_len(report, "vrps"), + aspas: json_array_len(report, "aspas"), + publication_points: json_array_len(report, "publication_points"), + rrdp_repos_unique: None, + tree_instances_processed: tree + .and_then(|tree| tree.get("instances_processed")) + .and_then(serde_json::Value::as_u64), + tree_instances_failed: tree + .and_then(|tree| tree.get("instances_failed")) + .and_then(serde_json::Value::as_u64), + warnings: tree_warnings + pp_warnings, + } +} + +fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) +} + +fn extract_json_object_field(path: &Path, field_name: &str) -> Option { + let bytes = fs::read(path).ok()?; + let needle = format!("\"{field_name}\":"); + let pos = find_subslice(&bytes, needle.as_bytes())?; + let mut i = pos + needle.len(); + while i < bytes.len() && bytes[i].is_ascii_whitespace() { + i += 1; + } + if bytes.get(i).copied()? != b'{' { + return None; + } + let start = i; + let mut depth = 0u32; + let mut in_string = false; + let mut escaped = false; + for (offset, &b) in bytes[start..].iter().enumerate() { + if in_string { + if escaped { + escaped = false; + } else if b == b'\\' { + escaped = true; + } else if b == b'"' { + in_string = false; + } + continue; + } + match b { + b'"' => in_string = true, + b'{' => depth = depth.saturating_add(1), + b'}' => { + depth = depth.saturating_sub(1); + if depth == 0 { + let end = start + offset + 1; + return serde_json::from_slice(&bytes[start..end]).ok(); + } + } + _ => {} + } + } + None +} + +fn parse_report_metadata(run_dir: &Path) -> (Option, Option) { + let Some(report_path) = find_named_file(run_dir, "report.json") else { + return (parse_stdout_summary(run_dir), None); + }; + let counts = parse_stdout_summary(run_dir).or_else(|| { + read_json_value_if_exists(&report_path).map(|report| parse_report_counts_fallback(&report)) + }); + let repo_sync_stats = extract_json_object_field(&report_path, "repo_sync_stats"); + (counts, repo_sync_stats) +} + +fn collect_path_file_stats(label: &str, path: &Path) -> PathFileStats { + let mut stats = PathFileStats { + label: label.to_string(), + path: path_string(path), + exists: path.exists(), + is_dir: path.is_dir(), + total_size_bytes: 0, + file_count: 0, + dir_count: 0, + }; + if !stats.exists { + return stats; + } + if path.is_file() { + if let Ok(metadata) = path.metadata() { + stats.total_size_bytes = metadata.len(); + stats.file_count = 1; + } + return stats; + } + + let mut stack = vec![path.to_path_buf()]; + while let Some(dir) = stack.pop() { + let Ok(entries) = fs::read_dir(&dir) else { + continue; + }; + for entry in entries.flatten() { + let Ok(file_type) = entry.file_type() else { + continue; + }; + if file_type.is_dir() { + stats.dir_count = stats.dir_count.saturating_add(1); + stack.push(entry.path()); + } else if file_type.is_file() { + stats.file_count = stats.file_count.saturating_add(1); + if let Ok(metadata) = entry.metadata() { + stats.total_size_bytes = stats.total_size_bytes.saturating_add(metadata.len()); + } + } + } + } + stats +} + +fn collect_state_path_stats(args: &Args, ctx: &RunContext) -> Vec { + let mut stats = Vec::new(); + stats.push(collect_path_file_stats( + "work_db", + &render_path_template(&args.work_db, args, ctx), + )); + if let Some(path) = args.repo_bytes_db.as_ref() { + stats.push(collect_path_file_stats( + "repo_bytes_db", + &render_path_template(path, args, ctx), + )); + } + if let Some(path) = args.raw_store_db.as_ref() { + stats.push(collect_path_file_stats( + "raw_store_db", + &render_path_template(path, args, ctx), + )); + } + stats +} + +fn parse_key_value_metrics(text: &str) -> BTreeMap { + let mut metrics = BTreeMap::new(); + for line in text.lines() { + let Some((key, value)) = line.split_once('=') else { + continue; + }; + metrics.insert(key.trim().to_string(), value.trim().to_string()); + } + metrics +} + +fn run_db_stats_command( + db_stats_bin: &Path, + db_path: &Path, + run_dir: &Path, + mode: &str, +) -> DbStatsSummary { + let output_path = run_dir.join(format!("db-stats-{mode}.txt")); + let stderr_path = run_dir.join(format!("db-stats-{mode}.stderr.txt")); + let mut summary = DbStatsSummary { + mode: mode.to_string(), + db_path: path_string(db_path), + output_path: Some(path_string(&output_path)), + stderr_path: None, + status: "success".to_string(), + exit_code: None, + error: None, + metrics: BTreeMap::new(), + }; + + if !db_path.exists() { + summary.status = "skipped".to_string(); + summary.error = Some(format!("db path does not exist: {}", db_path.display())); + summary.output_path = None; + return summary; + } + + let mut command = Command::new(db_stats_bin); + command.arg("--db").arg(db_path); + if mode == "exact" { + command.arg("--exact"); + } + match command.output() { + Ok(output) => { + summary.exit_code = output.status.code(); + if !output.status.success() { + summary.status = "failed".to_string(); + } + let stdout_text = String::from_utf8_lossy(&output.stdout).into_owned(); + if let Err(err) = fs::write(&output_path, stdout_text.as_bytes()) { + summary.status = "failed".to_string(); + summary.error = Some(format!( + "write db_stats output failed: {}: {err}", + output_path.display() + )); + } + summary.metrics = parse_key_value_metrics(&stdout_text); + if !output.stderr.is_empty() { + if fs::write(&stderr_path, &output.stderr).is_ok() { + summary.stderr_path = Some(path_string(&stderr_path)); + } + } + if !output.status.success() && summary.error.is_none() { + summary.error = Some(String::from_utf8_lossy(&output.stderr).into_owned()); + } + } + Err(err) => { + summary.status = "spawn_failed".to_string(); + summary.exit_code = None; + summary.output_path = None; + summary.error = Some(format!("spawn db_stats failed: {err}")); + } + } + summary +} + +fn collect_db_stats(args: &Args, ctx: &RunContext) -> Vec { + let work_db = render_path_template(&args.work_db, args, ctx); + let Some(db_stats_bin) = args + .db_stats_bin + .as_ref() + .cloned() + .or_else(default_db_stats_bin) + else { + return vec![DbStatsSummary { + mode: "estimate".to_string(), + db_path: path_string(&work_db), + output_path: None, + stderr_path: None, + status: "skipped".to_string(), + exit_code: None, + error: Some( + "db_stats binary not configured and sibling db_stats was not found".to_string(), + ), + metrics: BTreeMap::new(), + }]; + }; + + let mut stats = Vec::new(); + stats.push(run_db_stats_command( + &db_stats_bin, + &work_db, + &ctx.run_dir, + "estimate", + )); + if args + .db_stats_exact_every + .is_some_and(|every| ctx.seq % every == 0) + { + stats.push(run_db_stats_command( + &db_stats_bin, + &work_db, + &ctx.run_dir, + "exact", + )); + } + stats +} + +fn collect_post_run_metrics(args: &Args, ctx: &RunContext, summary: &mut RunSummary) { + if let Some(path) = find_named_file(&ctx.run_dir, "stage-timing.json") { + summary.stage_timing = read_json_value_if_exists(&path); + } + let (report_counts, repo_sync_stats) = parse_report_metadata(&ctx.run_dir); + summary.report_counts = report_counts; + summary.repo_sync_stats = repo_sync_stats; + summary.path_stats = collect_state_path_stats(args, ctx); + summary.db_stats = collect_db_stats(args, ctx); + summary.artifacts = collect_artifacts(&ctx.run_dir).unwrap_or_default(); + if let Some(path) = args.dead_repo_blacklist.as_ref() { + let (blacklist, _) = DeadRepoBlacklist::load(path); + summary.dead_repo_blacklist = Some(DeadRepoBlacklistRunSummary { + path: path_string(path), + entries: blacklist.len(), + blacklisted: blacklist.blacklisted_len(), + }); + } +} + +fn run_daemon(args: &Args) -> Result<(), String> { + fs::create_dir_all(args.state_root.join("state")).map_err(|e| { + format!( + "create daemon state dir failed: {}: {e}", + args.state_root.join("state").display() + ) + })?; + fs::create_dir_all(args.state_root.join("runs")).map_err(|e| { + format!( + "create daemon runs dir failed: {}: {e}", + args.state_root.join("runs").display() + ) + })?; + + let mut runs_completed = 0u64; + let mut next_seq = 1u64; + let mut last_run_id = None; + let mut health = DeadRepoHealthRuntime::default(); + write_status( + args, + DaemonState::Starting, + runs_completed, + None, + last_run_id.clone(), + &health, + )?; + // Probe once at startup so revived repos are removed before the first run. + maybe_run_dead_repo_health_check(args, &mut health); + + loop { + if args.max_runs.is_some_and(|max| runs_completed >= max) { + break; + } + + write_status( + args, + DaemonState::Idle, + runs_completed, + None, + last_run_id.clone(), + &health, + )?; + let ctx = make_run_context(args, next_seq, utc_now()); + write_status( + args, + DaemonState::Running, + runs_completed, + Some(&ctx), + last_run_id.clone(), + &health, + )?; + let mut summary = run_child_once(args, &ctx)?; + write_status( + args, + DaemonState::Collecting, + runs_completed, + Some(&ctx), + last_run_id.clone(), + &health, + )?; + collect_post_run_metrics(args, &ctx, &mut summary); + let removed = apply_retention(&args.state_root.join("runs"), args.retain_runs)?; + summary.retention_deleted_runs = removed.iter().map(|p| path_string(p)).collect(); + summary.artifacts = collect_artifacts(&ctx.run_dir).unwrap_or_default(); + write_json_pretty(&ctx.run_dir.join("run-summary.json"), &summary)?; + append_json_line(&summary_jsonl_path(args), &summary)?; + runs_completed += 1; + next_seq += 1; + last_run_id = Some(ctx.run_id); + + if args.max_runs.is_some_and(|max| runs_completed >= max) { + break; + } + // Health probes run only while no child is active (single-writer). + maybe_run_dead_repo_health_check(args, &mut health); + write_status( + args, + DaemonState::Sleeping, + runs_completed, + None, + last_run_id.clone(), + &health, + )?; + if args.interval_secs > 0 { + std::thread::sleep(Duration::from_secs(args.interval_secs)); + } + } + + write_status( + args, + DaemonState::Exited, + runs_completed, + None, + last_run_id, + &health, + ) +} + +pub fn main_entry() -> i32 { + let argv: Vec = std::env::args().collect(); + match parse_args(&argv) { + Ok(args) => match run_daemon(&args) { + Ok(()) => 0, + Err(err) => { + eprintln!("{err}"); + 2 + } + }, + Err(err) => { + if argv.iter().any(|a| a == "--help" || a == "-h") { + println!("{err}"); + return 0; + } + eprintln!("{err}"); + 2 + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_args(state_root: PathBuf) -> Args { + Args { + work_db: state_root.join("state/work-db"), + repo_bytes_db: Some(state_root.join("state/repo-bytes.db")), + state_root, + rpki_bin: PathBuf::from("/bin/true"), + interval_secs: 0, + max_runs: Some(1), + retain_runs: 10, + status_json: None, + summary_jsonl: None, + raw_store_db: None, + db_stats_bin: None, + db_stats_exact_every: None, + time_bin: None, + dead_repo_blacklist: None, + dead_repo_blacklist_fail_threshold: None, + dead_repo_health_check_interval_secs: 600, + dead_repo_probe_rsync_bin: PathBuf::from("rsync"), + child_args: vec!["--version".to_string()], + } + } + + #[test] + fn parse_args_accepts_required_flags_and_child_args() { + let argv = vec![ + "rpki_daemon".to_string(), + "--state-root".to_string(), + "/tmp/daemon".to_string(), + "--rpki-bin".to_string(), + "/bin/echo".to_string(), + "--interval-secs".to_string(), + "0".to_string(), + "--max-runs".to_string(), + "2".to_string(), + "--retain-runs".to_string(), + "3".to_string(), + "--".to_string(), + "--db".to_string(), + "{state_root}/state/work-db".to_string(), + "--report-json".to_string(), + "{run_out}/report.json".to_string(), + ]; + + let args = parse_args(&argv).expect("parse args"); + assert_eq!(args.state_root, PathBuf::from("/tmp/daemon")); + assert_eq!(args.rpki_bin, PathBuf::from("/bin/echo")); + assert_eq!(args.interval_secs, 0); + assert_eq!(args.max_runs, Some(2)); + assert_eq!(args.retain_runs, 3); + assert_eq!(args.work_db, PathBuf::from("/tmp/daemon/state/work-db")); + assert_eq!( + args.repo_bytes_db, + Some(PathBuf::from("/tmp/daemon/state/repo-bytes.db")) + ); + assert_eq!( + args.child_args, + vec![ + "--db", + "{state_root}/state/work-db", + "--report-json", + "{run_out}/report.json" + ] + ); + } + + #[test] + fn usage_and_parse_args_cover_optional_flags_and_errors() { + let help = usage(); + assert!(help.contains("--db-stats-exact-every")); + assert!(help.contains("{run_seq}")); + + let argv = vec![ + "rpki_daemon".to_string(), + "--state-root".to_string(), + "/tmp/daemon".to_string(), + "--rpki-bin".to_string(), + "/bin/echo".to_string(), + "--status-json".to_string(), + "/tmp/status.json".to_string(), + "--summary-jsonl".to_string(), + "/tmp/runs.jsonl".to_string(), + "--work-db".to_string(), + "{state_root}/work".to_string(), + "--repo-bytes-db".to_string(), + "{state_root}/repo-bytes".to_string(), + "--raw-store-db".to_string(), + "{state_root}/raw".to_string(), + "--db-stats-bin".to_string(), + "/bin/echo".to_string(), + "--db-stats-exact-every".to_string(), + "2".to_string(), + "--time-bin".to_string(), + "/usr/bin/time".to_string(), + "--no-time-wrapper".to_string(), + "--".to_string(), + "child".to_string(), + ]; + let args = parse_args(&argv).expect("optional args"); + assert_eq!(args.status_json, Some(PathBuf::from("/tmp/status.json"))); + assert_eq!(args.summary_jsonl, Some(PathBuf::from("/tmp/runs.jsonl"))); + assert_eq!(args.work_db, PathBuf::from("{state_root}/work")); + assert_eq!( + args.repo_bytes_db, + Some(PathBuf::from("{state_root}/repo-bytes")) + ); + assert_eq!(args.raw_store_db, Some(PathBuf::from("{state_root}/raw"))); + assert_eq!(args.db_stats_bin, Some(PathBuf::from("/bin/echo"))); + assert_eq!(args.db_stats_exact_every, Some(2)); + assert_eq!(args.time_bin, None); + + for (argv, expected) in [ + (vec!["rpki_daemon", "--help"], "Usage:"), + ( + vec![ + "rpki_daemon", + "--state-root", + "/tmp/x", + "--rpki-bin", + "/bin/echo", + "--max-runs", + "0", + "--", + "child", + ], + "--max-runs must be > 0", + ), + ( + vec![ + "rpki_daemon", + "--state-root", + "/tmp/x", + "--rpki-bin", + "/bin/echo", + "--retain-runs", + "0", + "--", + "child", + ], + "--retain-runs must be > 0", + ), + ( + vec![ + "rpki_daemon", + "--state-root", + "/tmp/x", + "--rpki-bin", + "/bin/echo", + "--db-stats-exact-every", + "0", + "--", + "child", + ], + "--db-stats-exact-every must be > 0", + ), + (vec!["rpki_daemon", "--unknown"], "unknown argument"), + ( + vec!["rpki_daemon", "--state-root"], + "--state-root requires a value", + ), + (vec!["rpki_daemon"], "missing -- before child rpki args"), + ( + vec![ + "rpki_daemon", + "--state-root", + "/tmp/x", + "--rpki-bin", + "/bin/echo", + "--", + ], + "child rpki args are required", + ), + ] { + let owned: Vec = argv.into_iter().map(str::to_string).collect(); + let err = parse_args(&owned).expect_err("parse should fail"); + assert!(err.contains(expected), "{err}"); + } + } + + #[test] + fn render_child_args_replaces_placeholders() { + let args = Args { + state_root: PathBuf::from("/tmp/root"), + rpki_bin: PathBuf::from("/bin/echo"), + interval_secs: 0, + max_runs: Some(1), + retain_runs: 10, + status_json: None, + summary_jsonl: None, + work_db: PathBuf::from("/tmp/root/state/work-db"), + repo_bytes_db: Some(PathBuf::from("/tmp/root/state/repo-bytes.db")), + raw_store_db: None, + db_stats_bin: None, + db_stats_exact_every: None, + time_bin: None, + dead_repo_blacklist: None, + dead_repo_blacklist_fail_threshold: None, + dead_repo_health_check_interval_secs: 600, + dead_repo_probe_rsync_bin: PathBuf::from("rsync"), + child_args: vec![ + "{state_root}/state/work-db".to_string(), + "{run_out}/result.ccr".to_string(), + "{run_id}".to_string(), + "{run_seq}".to_string(), + ], + }; + let ctx = RunContext { + seq: 7, + run_id: "000007-20260428T090000Z".to_string(), + run_dir: PathBuf::from("/tmp/root/runs/000007-20260428T090000Z"), + }; + + assert_eq!( + render_child_args(&args.child_args, &args, &ctx), + vec![ + "/tmp/root/state/work-db", + "/tmp/root/runs/000007-20260428T090000Z/result.ccr", + "000007-20260428T090000Z", + "7", + ] + ); + } + + #[test] + fn path_json_and_report_helpers_cover_fallbacks_and_nested_stats() { + let td = tempfile::tempdir().expect("tempdir"); + let state_root = td.path().join("daemon"); + let mut args = test_args(state_root.clone()); + args.work_db = PathBuf::from("{state_root}/state/work-db"); + args.repo_bytes_db = Some(PathBuf::from("{state_root}/state/repo-bytes.db")); + args.raw_store_db = Some(PathBuf::from("{state_root}/state/raw-store.db")); + + let now = time::Date::from_calendar_date(2026, time::Month::April, 28) + .expect("date") + .with_hms(9, 0, 0) + .expect("time") + .assume_utc(); + let ctx = make_run_context(&args, 42, now); + assert_eq!(ctx.run_id, "000042-20260428T090000Z"); + + let rendered = render_path_template(Path::new("{run_out}/{run_id}/{run_seq}"), &args, &ctx); + assert!(rendered.ends_with("000042-20260428T090000Z/000042-20260428T090000Z/42")); + + let nested_json = td.path().join("nested/out/status.json"); + write_json_pretty(&nested_json, &serde_json::json!({"ok": true})).expect("write json"); + append_json_line( + &td.path().join("nested/out/runs.jsonl"), + &serde_json::json!({"n": 1}), + ) + .expect("append jsonl"); + + fs::create_dir_all(ctx.run_dir.join("subdir")).expect("subdir"); + fs::write(ctx.run_dir.join("a.txt"), "aaa").expect("file"); + fs::write(ctx.run_dir.join("subdir/ignored.txt"), "bbb").expect("nested file"); + let artifacts = collect_artifacts(&ctx.run_dir).expect("artifacts"); + assert_eq!(artifacts.len(), 1); + assert!(artifacts[0].path.ends_with("a.txt")); + + fs::create_dir_all(state_root.join("state/work-db/nested")).expect("work db"); + fs::write(state_root.join("state/work-db/file.sst"), "abc").expect("sst"); + fs::write(state_root.join("state/work-db/nested/inner.sst"), "def").expect("inner"); + fs::write(state_root.join("state/raw-store.db"), "raw").expect("raw file"); + let file_stats = + collect_path_file_stats("raw_store_db", &state_root.join("state/raw-store.db")); + assert!(file_stats.exists); + assert!(!file_stats.is_dir); + assert_eq!(file_stats.file_count, 1); + let missing_stats = collect_path_file_stats("missing", &state_root.join("missing")); + assert!(!missing_stats.exists); + let state_stats = collect_state_path_stats(&args, &ctx); + assert!( + state_stats + .iter() + .any(|s| s.label == "raw_store_db" && s.exists) + ); + assert!( + state_stats + .iter() + .any(|s| s.label == "work_db" && s.file_count == 2) + ); + + let report_path = td.path().join("report.json"); + fs::write( + &report_path, + r#"{"repo_sync_stats": { "nested": {"text": "a\"b"} }, "after": 1}"#, + ) + .expect("report"); + assert_eq!( + extract_json_object_field(&report_path, "repo_sync_stats").expect("repo stats")["nested"] + ["text"], + "a\"b" + ); + fs::write(&report_path, r#"{"repo_sync_stats": []}"#).expect("report"); + assert!(extract_json_object_field(&report_path, "repo_sync_stats").is_none()); + assert!(extract_json_object_field(&report_path, "missing").is_none()); + + let counts = parse_report_counts_fallback(&serde_json::json!({ + "vrps": [{}, {}], + "aspas": [{}], + "publication_points": [{"warnings": [{}, {}]}, {"warnings": [{}]}], + "tree": {"instances_processed": 2, "instances_failed": 1, "warnings": [{}]} + })); + assert_eq!(counts.vrps, 2); + assert_eq!(counts.aspas, 1); + assert_eq!(counts.publication_points, 2); + assert_eq!(counts.warnings, 4); + assert_eq!(counts.tree_instances_processed, Some(2)); + assert_eq!(counts.tree_instances_failed, Some(1)); + } + + #[test] + fn retention_removes_oldest_run_directories() { + let td = tempfile::tempdir().expect("tempdir"); + let runs = td.path().join("runs"); + fs::create_dir_all(&runs).expect("runs dir"); + for name in [ + "000001-20260428T000001Z", + "000002-20260428T000002Z", + "000003-20260428T000003Z", + ] { + fs::create_dir_all(runs.join(name)).expect("run dir"); + } + + let removed = apply_retention(&runs, 2).expect("retention"); + assert_eq!(removed.len(), 1); + assert!(!runs.join("000001-20260428T000001Z").exists()); + assert!(runs.join("000002-20260428T000002Z").exists()); + assert!(runs.join("000003-20260428T000003Z").exists()); + } + + #[test] + fn retention_empty_root_and_parse_metrics_error_paths_are_reported() { + let td = tempfile::tempdir().expect("tempdir"); + let missing_runs = td.path().join("missing-runs"); + assert!( + apply_retention(&missing_runs, 2) + .expect("empty retention") + .is_empty() + ); + + let disabled = collect_process_metrics(false, &td.path().join("missing-time.txt")); + assert!(!disabled.time_wrapper_used); + assert!(disabled.time_output_path.is_none()); + + let missing = collect_process_metrics(true, &td.path().join("missing-time.txt")); + assert!(missing.time_wrapper_used); + assert!( + missing + .parse_error + .expect("parse error") + .contains("read process time output failed") + ); + } + + #[test] + fn process_metrics_parses_gnu_time_elapsed_line() { + let td = tempfile::tempdir().expect("tempdir"); + let path = td.path().join("time.txt"); + fs::write( + &path, + "User time (seconds): 1.25\nSystem time (seconds): 0.50\nPercent of CPU this job got: 175%\nElapsed (wall clock) time (h:mm:ss or m:ss): 0:01.00\nMaximum resident set size (kbytes): 12345\nExit status: 0\n", + ) + .expect("write time"); + + let metrics = collect_process_metrics(true, &path); + assert_eq!(metrics.user_seconds, Some(1.25)); + assert_eq!(metrics.system_seconds, Some(0.50)); + assert_eq!(metrics.cpu_percent, Some(175.0)); + assert_eq!(metrics.elapsed_raw.as_deref(), Some("0:01.00")); + assert_eq!(metrics.max_rss_kb, Some(12345)); + assert_eq!(metrics.exit_status_from_time, Some(0)); + } + + #[test] + #[cfg(unix)] + fn child_and_db_stats_error_paths_are_reported() { + use std::os::unix::fs::PermissionsExt; + + let td = tempfile::tempdir().expect("tempdir"); + let run_dir = td.path().join("run"); + fs::create_dir_all(&run_dir).expect("run dir"); + let missing_db = td.path().join("missing-db"); + let skipped = + run_db_stats_command(Path::new("/bin/echo"), &missing_db, &run_dir, "estimate"); + assert_eq!(skipped.status, "skipped"); + assert!(skipped.output_path.is_none()); + + let db_path = td.path().join("db"); + fs::create_dir_all(&db_path).expect("db"); + let failing_bin = td.path().join("failing_db_stats.sh"); + fs::write( + &failing_bin, + "#!/bin/sh\necho partial=1\necho db-stats failed >&2\nexit 7\n", + ) + .expect("script"); + let mut permissions = fs::metadata(&failing_bin).expect("metadata").permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&failing_bin, permissions).expect("chmod"); + let failed = run_db_stats_command(&failing_bin, &db_path, &run_dir, "exact"); + assert_eq!(failed.status, "failed"); + assert_eq!(failed.exit_code, Some(7)); + assert_eq!(failed.metrics.get("partial").map(String::as_str), Some("1")); + assert!(failed.stderr_path.is_some()); + assert!(failed.error.expect("stderr").contains("db-stats failed")); + + let spawn_failed = run_db_stats_command( + Path::new("/definitely/not/db_stats"), + &db_path, + &run_dir, + "estimate", + ); + assert_eq!(spawn_failed.status, "spawn_failed"); + assert!( + spawn_failed + .error + .expect("spawn err") + .contains("spawn db_stats failed") + ); + + let metrics = parse_key_value_metrics("alpha=1\nnot-a-pair\nbeta = two\n"); + assert_eq!(metrics.get("alpha").map(String::as_str), Some("1")); + assert_eq!(metrics.get("beta").map(String::as_str), Some("two")); + + let mut args = test_args(td.path().join("daemon")); + args.rpki_bin = PathBuf::from("/bin/sh"); + args.time_bin = Some(PathBuf::from("/usr/bin/time")); + args.child_args = vec![ + "-c".to_string(), + "echo child-out; echo child-err >&2; exit 7".to_string(), + ]; + let ctx = RunContext { + seq: 1, + run_id: "000001-20260428T000000Z".to_string(), + run_dir: args.state_root.join("runs/000001-20260428T000000Z"), + }; + let summary = run_child_once(&args, &ctx).expect("run failing child"); + assert_eq!(summary.status, RunStatus::Failed); + assert_eq!(summary.exit_code, Some(7)); + let process_metrics = summary.process_metrics.expect("process metrics"); + assert!(process_metrics.time_wrapper_used); + assert_eq!(process_metrics.exit_status_from_time, Some(7)); + + let mut spawn_args = test_args(td.path().join("spawn")); + spawn_args.rpki_bin = PathBuf::from("/definitely/not/rpki"); + spawn_args.child_args = vec!["--help".to_string()]; + let spawn_ctx = RunContext { + seq: 1, + run_id: "000001-20260428T000001Z".to_string(), + run_dir: spawn_args.state_root.join("runs/000001-20260428T000001Z"), + }; + let spawn_summary = run_child_once(&spawn_args, &spawn_ctx).expect("spawn summary"); + assert_eq!(spawn_summary.status, RunStatus::SpawnFailed); + assert!( + spawn_summary + .error + .expect("spawn error") + .contains("spawn child failed") + ); + } + + #[test] + fn report_metadata_prefers_stdout_summary_and_extracts_repo_sync_stats() { + let td = tempfile::tempdir().expect("tempdir"); + fs::write( + td.path().join("stdout.log"), + "RPKI stage2 serial run summary\npublication_points_processed=7 publication_points_failed=1\nrrdp_repos_unique=3\nvrps=11\naspas=2\naudit_publication_points=7\nwarnings_total=5\n", + ) + .expect("stdout"); + fs::write( + td.path().join("report.json"), + "{\"large\":[{\"ignored\":\"{}\"}],\"repo_sync_stats\":{\"by_phase\":{\"rrdp_ok\":{\"count\":7}},\"by_terminal_state\":{},\"publication_points_total\":7}}", + ) + .expect("report"); + + let (counts, repo_sync_stats) = parse_report_metadata(td.path()); + let counts = counts.expect("counts"); + assert_eq!(counts.vrps, 11); + assert_eq!(counts.aspas, 2); + assert_eq!(counts.publication_points, 7); + assert_eq!(counts.rrdp_repos_unique, Some(3)); + assert_eq!(counts.tree_instances_processed, Some(7)); + assert_eq!(counts.tree_instances_failed, Some(1)); + assert_eq!(counts.warnings, 5); + assert_eq!( + repo_sync_stats.expect("repo sync")["by_phase"]["rrdp_ok"]["count"].as_u64(), + Some(7) + ); + } + + #[test] + fn daemon_exits_immediately_when_max_runs_already_reached() { + let td = tempfile::tempdir().expect("tempdir"); + let mut args = test_args(td.path().join("daemon")); + args.max_runs = Some(0); + + run_daemon(&args).expect("run daemon"); + + let status_text = + fs::read_to_string(args.state_root.join("daemon-status.json")).expect("status"); + assert!(status_text.contains("\"state\": \"exited\"")); + assert!(status_text.contains("\"runsCompleted\": 0")); + assert!(args.state_root.join("runs").exists()); + } + + #[test] + fn daemon_runs_fake_child_twice_and_writes_summaries() { + let td = tempfile::tempdir().expect("tempdir"); + let args = Args { + state_root: td.path().join("daemon"), + rpki_bin: PathBuf::from("/bin/sh"), + interval_secs: 0, + max_runs: Some(2), + retain_runs: 10, + status_json: None, + summary_jsonl: None, + work_db: td.path().join("daemon/state/work-db"), + repo_bytes_db: Some(td.path().join("daemon/state/repo-bytes.db")), + raw_store_db: None, + db_stats_bin: None, + db_stats_exact_every: None, + time_bin: None, + dead_repo_blacklist: None, + dead_repo_blacklist_fail_threshold: None, + dead_repo_health_check_interval_secs: 600, + dead_repo_probe_rsync_bin: PathBuf::from("rsync"), + child_args: vec![ + "-c".to_string(), + "echo stdout-{run_seq}; echo stderr-{run_seq} >&2; echo marker > {run_out}/marker.txt".to_string(), + ], + }; + + run_daemon(&args).expect("run daemon"); + + let status_text = + fs::read_to_string(args.state_root.join("daemon-status.json")).expect("status"); + assert!(status_text.contains("\"state\": \"exited\"")); + assert!(status_text.contains("\"runsCompleted\": 2")); + + let jsonl = + fs::read_to_string(args.state_root.join("daemon-runs.jsonl")).expect("summary jsonl"); + assert_eq!(jsonl.lines().count(), 2); + + let run_dirs = fs::read_dir(args.state_root.join("runs")) + .expect("runs dir") + .collect::, _>>() + .expect("entries"); + assert_eq!(run_dirs.len(), 2); + for entry in run_dirs { + assert!(entry.path().join("run-summary.json").exists()); + assert!(entry.path().join("marker.txt").exists()); + } + } + + #[test] + fn daemon_sleep_and_retention_deletion_are_recorded() { + let td = tempfile::tempdir().expect("tempdir"); + let mut args = test_args(td.path().join("daemon")); + args.rpki_bin = PathBuf::from("/bin/sh"); + args.interval_secs = 1; + args.max_runs = Some(2); + args.retain_runs = 1; + args.child_args = vec![ + "-c".to_string(), + "echo run-{run_seq}; printf marker > {run_out}/marker.txt".to_string(), + ]; + + run_daemon(&args).expect("run daemon"); + + let jsonl = fs::read_to_string(args.state_root.join("daemon-runs.jsonl")).expect("jsonl"); + assert_eq!(jsonl.lines().count(), 2); + let last: serde_json::Value = + serde_json::from_str(jsonl.lines().last().expect("last line")).expect("summary"); + assert_eq!( + last["retentionDeletedRuns"] + .as_array() + .expect("deleted") + .len(), + 1 + ); + let run_dirs = fs::read_dir(args.state_root.join("runs")) + .expect("runs") + .collect::, _>>() + .expect("entries"); + assert_eq!(run_dirs.len(), 1); + } + + #[test] + #[cfg(unix)] + fn daemon_collects_stage_report_db_and_file_metrics() { + use std::os::unix::fs::PermissionsExt; + + let td = tempfile::tempdir().expect("tempdir"); + let db_stats_bin = td.path().join("fake_db_stats.sh"); + fs::write( + &db_stats_bin, + "#!/bin/sh\nif [ \"$3\" = \"--exact\" ]; then echo mode=exact; else echo mode=estimate; fi\necho total=42\necho db.files.total_size_bytes=123\n", + ) + .expect("fake db_stats"); + let mut permissions = fs::metadata(&db_stats_bin).expect("metadata").permissions(); + permissions.set_mode(0o755); + fs::set_permissions(&db_stats_bin, permissions).expect("chmod"); + + let args = Args { + state_root: td.path().join("daemon"), + rpki_bin: PathBuf::from("/bin/sh"), + interval_secs: 0, + max_runs: Some(1), + retain_runs: 10, + status_json: None, + summary_jsonl: None, + work_db: PathBuf::from("{state_root}/state/work-db"), + repo_bytes_db: Some(PathBuf::from("{state_root}/state/repo-bytes.db")), + raw_store_db: None, + db_stats_bin: Some(db_stats_bin), + db_stats_exact_every: Some(1), + time_bin: None, + dead_repo_blacklist: None, + dead_repo_blacklist_fail_threshold: None, + dead_repo_health_check_interval_secs: 600, + dead_repo_probe_rsync_bin: PathBuf::from("rsync"), + child_args: vec![ + "-c".to_string(), + "mkdir -p {state_root}/state/work-db {state_root}/state/repo-bytes.db; \ + printf x > {state_root}/state/work-db/000001.sst; \ + printf '{\"validation_ms\":7,\"download_event_count\":2}' > {run_out}/stage-timing.json; \ + printf '{\"tree\":{\"instances_processed\":3,\"instances_failed\":1,\"warnings\":[{}]},\"publication_points\":[{},{}],\"vrps\":[{},{}],\"aspas\":[{}],\"repo_sync_stats\":{\"by_phase\":{\"snapshot\":{\"count\":1}}}}' > {run_out}/report.json" + .to_string(), + ], + }; + + run_daemon(&args).expect("run daemon"); + + let run_summary = find_named_file(&args.state_root.join("runs"), "run-summary.json") + .expect("run summary"); + let summary: serde_json::Value = + serde_json::from_slice(&fs::read(run_summary).expect("read run summary")) + .expect("parse run summary"); + + assert_eq!(summary["stageTiming"]["validation_ms"].as_u64(), Some(7)); + assert_eq!(summary["reportCounts"]["vrps"].as_u64(), Some(2)); + assert_eq!(summary["reportCounts"]["aspas"].as_u64(), Some(1)); + assert_eq!( + summary["reportCounts"]["publicationPoints"].as_u64(), + Some(2) + ); + assert_eq!( + summary["repoSyncStats"]["by_phase"]["snapshot"]["count"].as_u64(), + Some(1) + ); + assert_eq!(summary["dbStats"].as_array().expect("db stats").len(), 2); + assert!( + summary["pathStats"] + .as_array() + .expect("path stats") + .iter() + .any(|item| item["label"] == "work_db" && item["exists"] == true) + ); + } + + #[test] + fn dead_repo_blacklist_flag_is_injected_into_child_args() { + let argv = vec![ + "rpki_daemon".to_string(), + "--state-root".to_string(), + "/tmp/daemon".to_string(), + "--rpki-bin".to_string(), + "/bin/echo".to_string(), + "--dead-repo-blacklist".to_string(), + "/tmp/daemon/state/dead-repo-blacklist.json".to_string(), + "--dead-repo-blacklist-fail-threshold".to_string(), + "2".to_string(), + "--".to_string(), + "--db".to_string(), + "{state_root}/state/work-db".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!( + args.child_args, + vec![ + "--db".to_string(), + "{state_root}/state/work-db".to_string(), + "--dead-repo-blacklist".to_string(), + "/tmp/daemon/state/dead-repo-blacklist.json".to_string(), + "--dead-repo-blacklist-fail-threshold".to_string(), + "2".to_string(), + ] + ); + } + + #[test] + fn dead_repo_blacklist_injection_respects_existing_child_flags() { + let argv = vec![ + "rpki_daemon".to_string(), + "--state-root".to_string(), + "/tmp/daemon".to_string(), + "--rpki-bin".to_string(), + "/bin/echo".to_string(), + "--dead-repo-blacklist".to_string(), + "/tmp/daemon/state/dead-repo-blacklist.json".to_string(), + "--".to_string(), + "--dead-repo-blacklist".to_string(), + "/custom/path.json".to_string(), + ]; + let args = parse_args(&argv).expect("parse"); + assert_eq!( + args.child_args, + vec![ + "--dead-repo-blacklist".to_string(), + "/custom/path.json".to_string(), + ] + ); + } + + #[test] + fn dead_repo_threshold_without_blacklist_is_rejected() { + let argv = vec![ + "rpki_daemon".to_string(), + "--state-root".to_string(), + "/tmp/daemon".to_string(), + "--rpki-bin".to_string(), + "/bin/echo".to_string(), + "--dead-repo-blacklist-fail-threshold".to_string(), + "2".to_string(), + "--".to_string(), + "--version".to_string(), + ]; + let err = parse_args(&argv).expect_err("parse should fail"); + assert!(err.contains("requires --dead-repo-blacklist"), "{err}"); + } + + #[test] + fn health_check_keeps_dead_entry_and_removes_revived_entry() { + use std::io::{Read as _, Write as _}; + use std::net::TcpListener; + + // Revived RRDP host: one-shot HTTP server answering 200. + let listener = TcpListener::bind("127.0.0.1:0").expect("bind"); + let port = listener.local_addr().expect("local addr").port(); + std::thread::spawn(move || { + if let Ok((mut stream, _)) = listener.accept() { + let mut buf = [0u8; 1024]; + let _ = stream.read(&mut buf); + let body = b""; + let _ = stream.write_all( + format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .as_bytes(), + ); + let _ = stream.write_all(body); + } + }); + + let td = tempfile::tempdir().expect("tempdir"); + let blacklist_path = td.path().join("dead-repo-blacklist.json"); + let revived_uri = format!("http://127.0.0.1:{port}/notification.xml"); + let dead_uri = "http://127.0.0.1:1/notification.xml".to_string(); + let mut blacklist = DeadRepoBlacklist::new(); + for uri in [&revived_uri, &dead_uri] { + blacklist.record_transport_failure(RepoTransportMode::Rrdp, uri, 100, 1, 256); + } + blacklist + .store_atomic(&blacklist_path, 100) + .expect("store blacklist"); + + let mut args = test_args(td.path().join("daemon")); + args.dead_repo_blacklist = Some(blacklist_path.clone()); + args.dead_repo_health_check_interval_secs = 0; + let mut health = DeadRepoHealthRuntime::default(); + maybe_run_dead_repo_health_check(&args, &mut health); + + let (after, warning) = DeadRepoBlacklist::load(&blacklist_path); + assert!(warning.is_none()); + assert!(!after.is_blacklisted(RepoTransportMode::Rrdp, &revived_uri)); + assert!(after.is_blacklisted(RepoTransportMode::Rrdp, &dead_uri)); + assert!(health.last_health_check_at.is_some()); + + // Status JSON exposes the blacklist section. + let status = dead_repo_blacklist_status(&args, &health).expect("status"); + assert_eq!(status.entries, 1); + assert_eq!(status.blacklisted, 1); + assert!(status.last_health_check_at_rfc3339_utc.is_some()); + } +} diff --git a/crates/panda-rpki-validator/src/data_model/aspa.rs b/crates/panda-rpki-validator/src/data_model/aspa.rs new file mode 100644 index 0000000..e2dbcab --- /dev/null +++ b/crates/panda-rpki-validator/src/data_model/aspa.rs @@ -0,0 +1,414 @@ +use crate::data_model::common::{DerReader, der_take_tlv}; +use crate::data_model::oid::OID_CT_ASPA; +use crate::data_model::rc::ResourceCertificate; +use crate::data_model::signed_object::{ + RpkiSignedObject, RpkiSignedObjectParsed, SignedObjectDecodeError, SignedObjectParseError, + SignedObjectValidateError, +}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AspaObject { + pub signed_object: RpkiSignedObject, + pub econtent_type: String, + pub aspa: AspaEContent, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AspaObjectParsed { + pub signed_object: RpkiSignedObjectParsed, + pub econtent_type: String, + pub aspa: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AspaEContent { + pub version: u32, + pub customer_as_id: u32, + pub provider_as_ids: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AspaEContentParsed { + der: Vec, +} + +#[derive(Debug, thiserror::Error)] +pub enum AspaParseError { + #[error("signed object parse error: {0} (RFC 6488 §2-§3; RFC 9589 §4)")] + SignedObject(#[from] SignedObjectParseError), + #[error("ASPA parse error: {0} (draft-ietf-sidrops-aspa-profile-21 §3; DER)")] + Parse(String), + + #[error("ASPA trailing bytes: {0} bytes (draft-ietf-sidrops-aspa-profile-21 §3; DER)")] + TrailingBytes(usize), +} + +#[derive(Debug, thiserror::Error)] +pub enum AspaProfileError { + #[error("signed object profile error: {0} (RFC 6488 §2-§3; RFC 9589 §4)")] + SignedObject(#[from] SignedObjectValidateError), + + #[error( + "ASPA eContentType must be {OID_CT_ASPA}, got {0} (draft-ietf-sidrops-aspa-profile-21 §2)" + )] + InvalidEContentType(String), + + #[error("ASPA profile decode error: {0} (draft-ietf-sidrops-aspa-profile-21 §3; DER)")] + ProfileDecode(String), + + #[error( + "ASProviderAttestation must be a SEQUENCE of 3 elements (draft-ietf-sidrops-aspa-profile-21 §3)" + )] + InvalidAttestationSequence, + + #[error( + "ASPA version must be 1 and MUST be explicitly encoded (draft-ietf-sidrops-aspa-profile-21 §3.1)" + )] + VersionMustBeExplicitOne, + + #[error( + "ASPA customerASID out of range (0..=4294967295), got {0} (draft-ietf-sidrops-aspa-profile-21 §3.2)" + )] + CustomerAsIdOutOfRange(u64), + + #[error( + "ASPA providers must contain at least one ASID (draft-ietf-sidrops-aspa-profile-21 §3.3)" + )] + EmptyProviders, + + #[error( + "ASPA provider ASID out of range (0..=4294967295), got {0} (draft-ietf-sidrops-aspa-profile-21 §3.3)" + )] + ProviderAsIdOutOfRange(u64), + + #[error( + "ASPA providers must be in strictly increasing order (draft-ietf-sidrops-aspa-profile-21 §3.3)" + )] + ProvidersNotStrictlyIncreasing, + + #[error( + "ASPA providers contains the customerASID ({0}) which is not allowed (draft-ietf-sidrops-aspa-profile-21 §3.3)" + )] + ProvidersContainCustomer(u32), +} + +impl From for AspaProfileError { + fn from(value: SignedObjectDecodeError) -> Self { + match value { + SignedObjectDecodeError::Parse(e) => AspaProfileError::ProfileDecode(e.to_string()), + SignedObjectDecodeError::Validate(e) => AspaProfileError::SignedObject(e), + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum AspaDecodeError { + #[error("{0}")] + Parse(#[from] AspaParseError), + + #[error("{0}")] + Validate(#[from] AspaProfileError), +} + +#[derive(Debug, thiserror::Error)] +pub enum AspaValidateError { + #[error( + "ASPA EE certificate must contain AS resources extension (draft-ietf-sidrops-aspa-profile-21 §4; RFC 3779 §3.2)" + )] + EeAsResourcesMissing, + + #[error( + "ASPA EE certificate AS resources must not use inherit (draft-ietf-sidrops-aspa-profile-21 §4; RFC 3779 §3.2.3.3)" + )] + EeAsResourcesInherit, + + #[error( + "ASPA EE certificate AS resources must not include ranges (draft-ietf-sidrops-aspa-profile-21 §4; RFC 3779 §3.2.3.6-§3.2.3.7)" + )] + EeAsResourcesRangePresent, + + #[error( + "ASPA EE certificate AS resources must not include RDI (draft-ietf-sidrops-aspa-profile-21 §4; RFC 3779 §3.2.3.5; RFC 6487 §4.8.11)" + )] + EeAsResourcesRdiPresent, + + #[error( + "ASPA EE certificate AS resources must contain exactly one ASID (id element) (draft-ietf-sidrops-aspa-profile-21 §4)" + )] + EeAsResourcesNotSingleId, + + #[error( + "ASPA customerASID ({customer_as_id}) does not match EE AS resources ({ee_as_id}) (draft-ietf-sidrops-aspa-profile-21 §4)" + )] + CustomerAsIdMismatch { customer_as_id: u32, ee_as_id: u32 }, + + #[error( + "ASPA EE certificate must not contain IP resources extension (draft-ietf-sidrops-aspa-profile-21 §4; RFC 3779 §2.2)" + )] + EeIpResourcesPresent, +} + +impl AspaObject { + /// Parse step of scheme A (`parse → validate → verify`). + pub fn parse_der(der: &[u8]) -> Result { + let signed_object = RpkiSignedObject::parse_der(der)?; + let econtent_type = signed_object + .signed_data + .encap_content_info + .econtent_type + .clone(); + let aspa = signed_object + .signed_data + .encap_content_info + .econtent + .as_deref() + .map(AspaEContent::parse_der) + .transpose()?; + Ok(AspaObjectParsed { + signed_object, + econtent_type, + aspa, + }) + } + + /// Profile validate step of scheme A (`parse → validate → verify`). + /// + /// `AspaObject` is already profile-validated when constructed via `decode_der()` / + /// `AspaObjectParsed::validate_profile()`. + pub fn validate_profile(&self) -> Result<(), AspaProfileError> { + Ok(()) + } + + pub fn decode_der(der: &[u8]) -> Result { + Ok(Self::parse_der(der)?.validate_profile()?) + } + + pub fn decode_der_with_strict_options( + der: &[u8], + strict_cms_der: bool, + strict_name: bool, + ) -> Result { + let signed_object = + RpkiSignedObject::decode_der_with_strict_options(der, strict_cms_der, strict_name) + .map_err(AspaProfileError::from)?; + Self::from_signed_object(signed_object) + } + + pub fn from_signed_object(signed_object: RpkiSignedObject) -> Result { + let econtent_type = signed_object + .signed_data + .encap_content_info + .econtent_type + .clone(); + if econtent_type != OID_CT_ASPA { + return Err(AspaProfileError::InvalidEContentType(econtent_type).into()); + } + + let aspa = + AspaEContent::decode_der(&signed_object.signed_data.encap_content_info.econtent)?; + Ok(Self { + aspa, + signed_object, + econtent_type: OID_CT_ASPA.to_string(), + }) + } + + /// Validate this ASPA's embedded EE certificate resources. + pub fn validate_embedded_ee_cert(&self) -> Result<(), AspaValidateError> { + let ee = &self.signed_object.signed_data.certificates[0].resource_cert; + self.aspa.validate_against_ee_cert(ee) + } +} + +impl AspaEContent { + /// Parse step of scheme A (`parse → validate → verify`). + pub fn parse_der(der: &[u8]) -> Result { + let (_tag, _value, rem) = der_take_tlv(der).map_err(AspaParseError::Parse)?; + if !rem.is_empty() { + return Err(AspaParseError::TrailingBytes(rem.len())); + } + Ok(AspaEContentParsed { der: der.to_vec() }) + } + + /// Profile validate step of scheme A (`parse → validate → verify`). + /// + /// `AspaEContent` is already profile-validated when constructed via `decode_der()` / + /// `AspaEContentParsed::validate_profile()`. + pub fn validate_profile(&self) -> Result<(), AspaProfileError> { + Ok(()) + } + + /// Decode the DER-encoded ASProviderAttestation defined in + /// draft-ietf-sidrops-aspa-profile-21 §3 (`parse + validate`). + pub fn decode_der(der: &[u8]) -> Result { + Ok(Self::parse_der(der)?.validate_profile()?) + } + + /// Validate ASPA payload against the embedded EE resource certificate. + /// + /// This implements the EE/payload semantic checks described in + /// `draft-ietf-sidrops-aspa-profile-21` §4 (as summarized in `rpki/specs/08_aspa.md`). + pub fn validate_against_ee_cert( + &self, + ee: &ResourceCertificate, + ) -> Result<(), AspaValidateError> { + if ee.tbs.extensions.ip_resources.is_some() { + return Err(AspaValidateError::EeIpResourcesPresent); + } + + let asn = ee + .tbs + .extensions + .as_resources + .as_ref() + .ok_or(AspaValidateError::EeAsResourcesMissing)?; + + if asn.rdi.is_some() { + return Err(AspaValidateError::EeAsResourcesRdiPresent); + } + if asn.is_asnum_inherit() { + return Err(AspaValidateError::EeAsResourcesInherit); + } + if asn.has_any_range() { + return Err(AspaValidateError::EeAsResourcesRangePresent); + } + + let ee_as_id = asn + .asnum_single_id() + .ok_or(AspaValidateError::EeAsResourcesNotSingleId)?; + if ee_as_id != self.customer_as_id { + return Err(AspaValidateError::CustomerAsIdMismatch { + customer_as_id: self.customer_as_id, + ee_as_id, + }); + } + + Ok(()) + } +} + +impl AspaObjectParsed { + pub fn validate_profile(self) -> Result { + let signed_object = self.signed_object.validate_profile()?; + let econtent_type = signed_object + .signed_data + .encap_content_info + .econtent_type + .clone(); + if econtent_type != OID_CT_ASPA { + return Err(AspaProfileError::InvalidEContentType(econtent_type)); + } + let aspa = self + .aspa + .ok_or_else(|| AspaProfileError::ProfileDecode("ASPA.eContent missing".into()))? + .validate_profile()?; + Ok(AspaObject { + signed_object, + econtent_type: OID_CT_ASPA.to_string(), + aspa, + }) + } +} + +impl AspaEContentParsed { + pub fn validate_profile(self) -> Result { + fn count_elements(mut r: DerReader<'_>) -> Result { + let mut n = 0usize; + while !r.is_empty() { + r.skip_any()?; + n += 1; + } + Ok(n) + } + + let mut r = DerReader::new(&self.der); + let mut seq = r + .take_sequence() + .map_err(|e| AspaProfileError::ProfileDecode(e))?; + if !r.is_empty() { + return Err(AspaProfileError::ProfileDecode( + "trailing bytes after ASProviderAttestation".into(), + )); + } + + let elem_count = + count_elements(seq).map_err(|e| AspaProfileError::ProfileDecode(e.to_string()))?; + if elem_count != 3 { + return Err(AspaProfileError::InvalidAttestationSequence); + } + + // version [0] EXPLICIT INTEGER MUST be present and MUST be 1. + if seq + .peek_tag() + .map_err(|e| AspaProfileError::ProfileDecode(e.to_string()))? + != 0xA0 + { + return Err(AspaProfileError::VersionMustBeExplicitOne); + } + let (inner_tag, inner_val) = seq + .take_explicit(0xA0) + .map_err(|e| AspaProfileError::ProfileDecode(e.to_string()))?; + if inner_tag != 0x02 { + return Err(AspaProfileError::VersionMustBeExplicitOne); + } + let v = crate::data_model::common::der_uint_from_bytes(inner_val) + .map_err(|e| AspaProfileError::ProfileDecode(e.to_string()))?; + if v != 1 { + return Err(AspaProfileError::VersionMustBeExplicitOne); + } + + let customer_u64 = seq + .take_uint_u64() + .map_err(|e| AspaProfileError::ProfileDecode(e.to_string()))?; + if customer_u64 > u32::MAX as u64 { + return Err(AspaProfileError::CustomerAsIdOutOfRange(customer_u64)); + } + let customer_as_id = customer_u64 as u32; + + let providers_seq = seq + .take_sequence() + .map_err(|e| AspaProfileError::ProfileDecode(e.to_string()))?; + if !seq.is_empty() { + return Err(AspaProfileError::InvalidAttestationSequence); + } + let providers = parse_providers_cursor(providers_seq, customer_as_id)?; + + Ok(AspaEContent { + version: 1, + customer_as_id, + provider_as_ids: providers, + }) + } +} + +fn parse_providers_cursor( + mut seq: DerReader<'_>, + customer_as_id: u32, +) -> Result, AspaProfileError> { + if seq.is_empty() { + return Err(AspaProfileError::EmptyProviders); + } + + let mut out: Vec = Vec::new(); + let mut prev: Option = None; + while !seq.is_empty() { + let v = seq + .take_uint_u64() + .map_err(|e| AspaProfileError::ProfileDecode(e.to_string()))?; + if v > u32::MAX as u64 { + return Err(AspaProfileError::ProviderAsIdOutOfRange(v)); + } + let asn = v as u32; + if asn == customer_as_id { + return Err(AspaProfileError::ProvidersContainCustomer(customer_as_id)); + } + if let Some(p) = prev { + if asn <= p { + return Err(AspaProfileError::ProvidersNotStrictlyIncreasing); + } + } + prev = Some(asn); + out.push(asn); + } + Ok(out) +} diff --git a/crates/panda-rpki-validator/src/data_model/common.rs b/crates/panda-rpki-validator/src/data_model/common.rs new file mode 100644 index 0000000..9db580e --- /dev/null +++ b/crates/panda-rpki-validator/src/data_model/common.rs @@ -0,0 +1,302 @@ +use x509_parser::asn1_rs::Tag; +use x509_parser::prelude::FromDer; +use x509_parser::x509::AlgorithmIdentifier; + +pub type UtcTime = time::OffsetDateTime; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Asn1TimeEncoding { + UtcTime, + GeneralizedTime, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Asn1TimeUtc { + pub utc: UtcTime, + pub encoding: Asn1TimeEncoding, +} + +impl Asn1TimeUtc { + /// Validate Time encoding rules (RFC 5280): years 1950-2049 use UTCTime, + /// other years use GeneralizedTime. + pub fn validate_encoding_rfc5280( + &self, + field: &'static str, + ) -> Result<(), InvalidTimeEncodingError> { + let year = self.utc.year(); + let expected = if year <= 2049 { + Asn1TimeEncoding::UtcTime + } else { + Asn1TimeEncoding::GeneralizedTime + }; + if self.encoding != expected { + return Err(InvalidTimeEncodingError { + field, + year, + encoding: self.encoding, + }); + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BigUnsigned { + /// Minimal big-endian bytes. For zero, this is `[0]`. + pub bytes_be: Vec, +} + +impl BigUnsigned { + pub fn from_biguint(n: &der_parser::num_bigint::BigUint) -> Self { + let mut bytes = n.to_bytes_be(); + if bytes.is_empty() { + bytes.push(0); + } + Self { bytes_be: bytes } + } + + pub fn to_hex_upper(&self) -> String { + hex::encode_upper(&self.bytes_be) + } + + pub fn to_u64(&self) -> Option { + if self.bytes_be.len() > 8 { + return None; + } + let mut value: u64 = 0; + for &b in &self.bytes_be { + value = (value << 8) | (b as u64); + } + Some(value) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +#[error( + "{field} time encoding invalid for year {year}: got {encoding:?} (RFC 5280 §4.1.2.5; RFC 5280 §5.1.2.4-§5.1.2.6)" +)] +pub struct InvalidTimeEncodingError { + pub field: &'static str, + pub year: i32, + pub encoding: Asn1TimeEncoding, +} + +pub fn asn1_time_to_model(t: x509_parser::time::ASN1Time) -> Asn1TimeUtc { + let encoding = if t.is_utctime() { + Asn1TimeEncoding::UtcTime + } else { + Asn1TimeEncoding::GeneralizedTime + }; + Asn1TimeUtc { + utc: t.to_datetime(), + encoding, + } +} + +pub fn algorithm_params_absent_or_null(sig: &AlgorithmIdentifier<'_>) -> bool { + match sig.parameters.as_ref() { + None => true, + Some(p) if p.tag() == Tag::Null => true, + Some(_p) => false, + } +} + +/// Take a single DER TLV (Tag-Length-Value) from the start of `input`. +/// +/// This helper supports: +/// - short- and long-form lengths (up to 8 length bytes) +/// - only low-tag-number form tags (no high-tag-number form) +/// - definite length only (DER forbids indefinite length) +/// +/// Returns: `(tag_byte, value_bytes, remaining_bytes)`. +pub(crate) fn der_take_tlv(input: &[u8]) -> Result<(u8, &[u8], &[u8]), String> { + if input.len() < 2 { + return Err("truncated DER (need tag+len)".into()); + } + let tag = input[0]; + if (tag & 0x1F) == 0x1F { + return Err("high-tag-number form not supported".into()); + } + let len0 = input[1]; + if len0 == 0x80 { + return Err("indefinite length not allowed in DER".into()); + } + let (len, hdr_len) = if len0 & 0x80 == 0 { + (len0 as usize, 2usize) + } else { + let n = (len0 & 0x7F) as usize; + if n == 0 || n > 8 { + return Err("invalid DER length".into()); + } + if input.len() < 2 + n { + return Err("truncated DER (length bytes)".into()); + } + let mut l: usize = 0; + for &b in &input[2..2 + n] { + l = (l << 8) | (b as usize); + } + (l, 2 + n) + }; + if input.len() < hdr_len + len { + return Err("truncated DER (value bytes)".into()); + } + let value = &input[hdr_len..hdr_len + len]; + let rem = &input[hdr_len + len..]; + Ok((tag, value, rem)) +} + +/// Minimal streaming DER reader built on `der_take_tlv`. +/// +/// This is intentionally small and only supports the subset of DER needed by +/// RPKI object eContent decoders (ROA/ASPA), to avoid constructing a generic AST +/// (which is expensive on large objects such as ROAs with thousands of prefixes). +#[derive(Clone, Copy)] +pub(crate) struct DerReader<'a> { + buf: &'a [u8], +} + +impl<'a> DerReader<'a> { + pub(crate) fn new(buf: &'a [u8]) -> Self { + Self { buf } + } + + pub(crate) fn is_empty(&self) -> bool { + self.buf.is_empty() + } + + pub(crate) fn remaining_len(&self) -> usize { + self.buf.len() + } + + pub(crate) fn peek_tag(&self) -> Result { + self.buf + .first() + .copied() + .ok_or_else(|| "truncated DER".into()) + } + + pub(crate) fn take_any(&mut self) -> Result<(u8, &'a [u8]), String> { + let (tag, value, rem) = der_take_tlv(self.buf)?; + self.buf = rem; + Ok((tag, value)) + } + + pub(crate) fn take_any_full(&mut self) -> Result<(u8, &'a [u8], &'a [u8]), String> { + let (tag, value, rem) = der_take_tlv(self.buf)?; + let consumed = self.buf.len() - rem.len(); + let full = &self.buf[..consumed]; + self.buf = rem; + Ok((tag, full, value)) + } + + pub(crate) fn skip_any(&mut self) -> Result<(), String> { + let _ = self.take_any()?; + Ok(()) + } + + pub(crate) fn take_tag(&mut self, expected_tag: u8) -> Result<&'a [u8], String> { + let (tag, value) = self.take_any()?; + if tag != expected_tag { + return Err(format!( + "unexpected tag: got 0x{tag:02X}, expected 0x{expected_tag:02X}" + )); + } + Ok(value) + } + + pub(crate) fn take_sequence(&mut self) -> Result, String> { + let value = self.take_tag(0x30)?; + Ok(DerReader::new(value)) + } + + pub(crate) fn take_octet_string(&mut self) -> Result<&'a [u8], String> { + self.take_tag(0x04) + } + + pub(crate) fn take_bit_string(&mut self) -> Result<(u8, &'a [u8]), String> { + let v = self.take_tag(0x03)?; + if v.is_empty() { + return Err("BIT STRING content is empty".into()); + } + Ok((v[0], &v[1..])) + } + + pub(crate) fn take_uint_u64(&mut self) -> Result { + let v = self.take_tag(0x02)?; + der_uint_from_bytes(v) + } + + pub(crate) fn take_explicit( + &mut self, + expected_outer_tag: u8, + ) -> Result<(u8, &'a [u8]), String> { + let inner_der = self.take_tag(expected_outer_tag)?; + let (tag, value, rem) = der_take_tlv(inner_der)?; + if !rem.is_empty() { + return Err("trailing bytes inside EXPLICIT value".into()); + } + Ok((tag, value)) + } + + pub(crate) fn take_explicit_der(&mut self, expected_outer_tag: u8) -> Result<&'a [u8], String> { + let inner_der = self.take_tag(expected_outer_tag)?; + let (_tag, _value, rem) = der_take_tlv(inner_der)?; + if !rem.is_empty() { + return Err("trailing bytes inside EXPLICIT value".into()); + } + Ok(inner_der) + } +} + +pub(crate) fn der_uint_from_bytes(bytes: &[u8]) -> Result { + if bytes.is_empty() { + return Err("INTEGER has empty content".into()); + } + // Disallow negative values. + if (bytes[0] & 0x80) != 0 { + return Err("INTEGER is negative".into()); + } + // DER requires minimal encoding for INTEGER. + if bytes.len() > 1 && bytes[0] == 0x00 && (bytes[1] & 0x80) == 0 { + return Err("INTEGER not minimally encoded".into()); + } + if bytes.len() > 8 { + return Err("INTEGER does not fit u64".into()); + } + let mut v: u64 = 0; + for &b in bytes { + v = (v << 8) | (b as u64); + } + Ok(v) +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct X509NameDer(pub Vec); + +impl X509NameDer { + pub fn as_raw(&self) -> &[u8] { + &self.0 + } +} + +impl std::fmt::Display for X509NameDer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let Ok((rem, name)) = x509_parser::x509::X509Name::from_der(&self.0) else { + return write!(f, ""); + }; + if !rem.is_empty() { + return write!(f, ""); + } + write!(f, "{name}") + } +} + +/// Filename extensions registered in IANA "RPKI Repository Name Schemes". +/// +/// Source: +/// Snapshot date: 2026-01-28. +/// +/// Notes: +/// - Includes entries marked TEMPORARY/DEPRECATED by IANA (e.g., `asa`, `gbr`). +pub const IANA_RPKI_REPOSITORY_FILENAME_EXTENSIONS: &[&str] = + &["asa", "cer", "crl", "gbr", "mft", "roa", "sig", "tak"]; diff --git a/crates/panda-rpki-validator/src/data_model/crl.rs b/crates/panda-rpki-validator/src/data_model/crl.rs new file mode 100644 index 0000000..4396565 --- /dev/null +++ b/crates/panda-rpki-validator/src/data_model/crl.rs @@ -0,0 +1,546 @@ +pub use crate::data_model::common::{Asn1TimeEncoding, Asn1TimeUtc, BigUnsigned}; +use crate::data_model::oid::{ + OID_AUTHORITY_KEY_IDENTIFIER, OID_AUTHORITY_KEY_IDENTIFIER_RAW, OID_CRL_NUMBER, + OID_CRL_NUMBER_RAW, OID_SHA256_WITH_RSA_ENCRYPTION, OID_SHA256_WITH_RSA_ENCRYPTION_RAW, + OID_SUBJECT_KEY_IDENTIFIER_RAW, +}; +use x509_parser::certificate::X509Certificate; +use x509_parser::extensions::{ParsedExtension, X509Extension}; +use x509_parser::prelude::{FromDer, X509Version}; +use x509_parser::revocation_list::CertificateRevocationList; +use x509_parser::x509::{AlgorithmIdentifier, SubjectPublicKeyInfo}; +use x509_parser::{asn1_rs::Class as Asn1Class, asn1_rs::Tag as Asn1Tag}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RevokedCert { + pub serial_number: BigUnsigned, + pub revocation_date: Asn1TimeUtc, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CrlExtensions { + pub authority_key_identifier: Vec, + pub crl_number: BigUnsigned, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RpkixCrl { + pub raw_der: Vec, + pub version: u32, + pub issuer_dn: String, + pub signature_algorithm_oid: String, + pub this_update: Asn1TimeUtc, + pub next_update: Asn1TimeUtc, + pub revoked_certs: Vec, + pub extensions: CrlExtensions, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RpkixCrlParsed { + pub raw_der: Vec, + pub version: Option, + pub issuer_dn: String, + pub signature_algorithm: AlgorithmIdentifierValue, + pub tbs_signature_algorithm: AlgorithmIdentifierValue, + pub this_update: Asn1TimeUtc, + pub next_update: Option, + pub revoked_certs: Vec, + pub extensions: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RevokedCertParsed { + pub serial_number: BigUnsigned, + pub revocation_date: Asn1TimeUtc, + pub has_extensions: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum CrlExtensionParsed { + AuthorityKeyIdentifier { + key_identifier: Option>, + has_other_fields: bool, + critical: bool, + }, + CrlNumber { + number: der_parser::num_bigint::BigUint, + critical: bool, + }, + Other { + oid: String, + critical: bool, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AlgorithmIdentifierValue { + pub oid: String, + pub parameters: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AlgorithmParametersValue { + pub class: Asn1Class, + pub tag: Asn1Tag, + pub data: Vec, +} + +impl AlgorithmIdentifierValue { + pub fn params_absent_or_null(&self) -> bool { + match &self.parameters { + None => true, + Some(p) if p.class == Asn1Class::Universal && p.tag == Asn1Tag::Null => true, + Some(_p) => false, + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum CrlParseError { + #[error("X.509 CRL parse error: {0} (RFC 5280 §5.1; RFC 6487 §5)")] + Parse(String), + + #[error("trailing bytes after CRL DER: {0} bytes (DER; RFC 5280 §5.1)")] + TrailingBytes(usize), +} + +#[derive(Debug, thiserror::Error)] +pub enum CrlProfileError { + #[error("CRL version must be v2, got {0:?} (RFC 5280 §5.1; RFC 6487 §5)")] + InvalidVersion(Option), + + #[error("CRL signatureAlgorithm must match TBSCertList.signature (RFC 5280 §5.1)")] + SignatureAlgorithmMismatch, + + #[error( + "CRL signatureAlgorithm must be sha256WithRSAEncryption ({OID_SHA256_WITH_RSA_ENCRYPTION}), got {0} (RFC 6487 §5; RFC 7935 §2)" + )] + InvalidSignatureAlgorithm(String), + + #[error( + "CRL signature algorithm parameters must be absent or NULL (RFC 5280 §4.1.1.2; RFC 7935 §2)" + )] + InvalidSignatureAlgorithmParameters, + + #[error("CRL extensions must be exactly two (AKI + CRLNumber), got {0} (RFC 9829 §3.1)")] + InvalidExtensionsCount(usize), + + #[error("unsupported CRL extension OID {0} (RFC 9829 §3.1)")] + UnsupportedExtension(String), + + #[error("duplicate CRL extension OID {0} (RFC 5280 §4.2; RFC 9829 §3.1)")] + DuplicateExtension(String), + + #[error("AuthorityKeyIdentifier CRL extension missing (RFC 9829 §3.1; RFC 5280 §5.2.1)")] + AkiMissing, + + #[error("AuthorityKeyIdentifier must contain keyIdentifier (RFC 5280 §5.2.1; RFC 9829 §3.1)")] + AkiMissingKeyIdentifier, + + #[error( + "AuthorityKeyIdentifier must not contain authorityCertIssuer or authorityCertSerialNumber (RFC 5280 §5.2.1; RFC 9829 §3.1)" + )] + AkiHasOtherFields, + + #[error("CRLNumber CRL extension missing (RFC 9829 §3.1; RFC 5280 §5.2.3)")] + CrlNumberMissing, + + #[error("CRLNumber must be non-critical (RFC 9829 §3.1; RFC 5280 §5.2.3)")] + CrlNumberCritical, + + #[error("CRLNumber out of range (must fit in 0..2^159-1) (RFC 9829 §3.1)")] + CrlNumberOutOfRange, + + #[error("CRL entry extensions must not be present (RFC 6487 §5; RFC 5280 §5.1)")] + EntryExtensionsNotAllowed, + + #[error("CRL nextUpdate must be present (RFC 5280 §5.1.2.5; RFC 6487 §5)")] + NextUpdateMissing, + + #[error( + "{field} time encoding invalid for year {year}: got {encoding:?} (RFC 5280 §5.1.2.4-§5.1.2.6)" + )] + InvalidTimeEncoding { + field: &'static str, + year: i32, + encoding: Asn1TimeEncoding, + }, +} + +#[derive(Debug, thiserror::Error)] +pub enum CrlDecodeError { + #[error("{0}")] + Parse(#[from] CrlParseError), + + #[error("{0}")] + Validate(#[from] CrlProfileError), +} + +impl RpkixCrl { + /// Parse step of scheme A (`parse → validate → verify`). + pub fn parse_der(der: &[u8]) -> Result { + let (rem, crl) = CertificateRevocationList::from_der(der) + .map_err(|e| CrlParseError::Parse(e.to_string()))?; + if !rem.is_empty() { + return Err(CrlParseError::TrailingBytes(rem.len())); + } + + let revoked_certs = crl + .iter_revoked_certificates() + .map(|rc| RevokedCertParsed { + serial_number: BigUnsigned::from_biguint(rc.serial()), + revocation_date: crate::data_model::common::asn1_time_to_model(rc.revocation_date), + has_extensions: !rc.extensions().is_empty(), + }) + .collect::>(); + + let this_update = crate::data_model::common::asn1_time_to_model(crl.last_update()); + let next_update = crl + .next_update() + .map(crate::data_model::common::asn1_time_to_model); + + let extensions = + parse_extensions_parse(crl.extensions()).map_err(|e| CrlParseError::Parse(e))?; + + Ok(RpkixCrlParsed { + raw_der: der.to_vec(), + version: crl.version(), + issuer_dn: crl.issuer().to_string(), + signature_algorithm: algorithm_identifier_value(&crl.signature_algorithm), + tbs_signature_algorithm: algorithm_identifier_value(&crl.tbs_cert_list.signature), + this_update, + next_update, + revoked_certs, + extensions, + }) + } + + /// Decode a DER-encoded X.509 v2 CRL and enforce the RPKI profile constraints from + /// `specs/prepare/data_models/04_crl.md` (RFC 6487 §5; RFC 9829 §3.1; RFC 5280 §5.1). + pub fn decode_der(der: &[u8]) -> Result { + Ok(Self::parse_der(der)?.validate_profile()?) + } + + /// Profile validate step of scheme A (`parse → validate → verify`). + /// + /// `RpkixCrl` is already profile-validated when constructed via `decode_der()` / + /// `RpkixCrlParsed::validate_profile()`. + pub fn validate_profile(&self) -> Result<(), CrlProfileError> { + Ok(()) + } + + /// Verify the cryptographic signature on this CRL using the issuer certificate. + /// + /// Signature verification needs the issuer public key (RFC 5280 §6.3.3 (f)-(g)). + /// In RPKI practice, this public key is obtained from the CRL issuer CA certificate + /// (and that certificate must already be validated up to the same trust anchor). + /// + /// This helper also performs common binding checks: + /// - CRL `issuer_dn` must equal issuer certificate `subject` + /// - if issuer KeyUsage is present, require `cRLSign` + /// - if issuer SKI is present, require it matches CRL AKI.keyIdentifier + pub fn verify_signature_with_issuer_certificate_der( + &self, + issuer_cert_der: &[u8], + ) -> Result<(), CrlVerifyError> { + let (rem, issuer_cert) = X509Certificate::from_der(issuer_cert_der) + .map_err(|e| CrlVerifyError::IssuerCertificateParse(e.to_string()))?; + if !rem.is_empty() { + return Err(CrlVerifyError::IssuerCertificateTrailingBytes(rem.len())); + } + + let subject_dn = issuer_cert.subject().to_string(); + if subject_dn != self.issuer_dn { + return Err(CrlVerifyError::IssuerSubjectMismatch { + crl_issuer_dn: self.issuer_dn.clone(), + issuer_subject_dn: subject_dn, + }); + } + + if let Some(ku) = issuer_cert + .key_usage() + .map_err(|e| CrlVerifyError::IssuerCertificateParse(e.to_string()))? + { + if !ku.value.crl_sign() { + return Err(CrlVerifyError::IssuerKeyUsageMissingCrlSign); + } + } + + if let Some(issuer_ski) = get_subject_key_identifier(&issuer_cert) { + if issuer_ski != self.extensions.authority_key_identifier { + return Err(CrlVerifyError::AkiSkiMismatch); + } + } + + self.verify_signature_with_issuer_spki(issuer_cert.public_key()) + } + + /// Verify the cryptographic signature on this CRL using the issuer SubjectPublicKeyInfo. + pub fn verify_signature_with_issuer_spki( + &self, + issuer_spki: &SubjectPublicKeyInfo<'_>, + ) -> Result<(), CrlVerifyError> { + let (rem, crl) = CertificateRevocationList::from_der(&self.raw_der) + .map_err(|e| CrlVerifyError::CrlParse(e.to_string()))?; + if !rem.is_empty() { + return Err(CrlVerifyError::CrlTrailingBytes(rem.len())); + } + crate::crypto_sig_cache::verify_with_cache( + crate::crypto_sig_cache::CryptoSigVerifyPoint::Crl, + crl.tbs_cert_list.as_ref(), + crl.signature_value.data.as_ref(), + issuer_spki.raw, + || { + crl.verify_signature(issuer_spki) + .map_err(|e| CrlVerifyError::InvalidSignature(e.to_string())) + }, + ) + } + + /// Verify the cryptographic signature on this CRL using a DER-encoded SubjectPublicKeyInfo. + pub fn verify_signature_with_issuer_spki_der( + &self, + issuer_spki_der: &[u8], + ) -> Result<(), CrlVerifyError> { + let (rem, spki) = SubjectPublicKeyInfo::from_der(issuer_spki_der) + .map_err(|e| CrlVerifyError::IssuerSpkiParse(e.to_string()))?; + if !rem.is_empty() { + return Err(CrlVerifyError::IssuerSpkiTrailingBytes(rem.len())); + } + self.verify_signature_with_issuer_spki(&spki) + } +} + +impl RpkixCrlParsed { + /// Profile validate step of scheme A (`parse → validate → verify`). + pub fn validate_profile(self) -> Result { + let version = match self.version { + Some(X509Version::V2) => 2, + Some(v) => return Err(CrlProfileError::InvalidVersion(Some(v.0))), + None => return Err(CrlProfileError::InvalidVersion(None)), + }; + + // signatureAlgorithm must match tbsCertList.signature + if self.signature_algorithm != self.tbs_signature_algorithm { + return Err(CrlProfileError::SignatureAlgorithmMismatch); + } + let sig_oid = self.signature_algorithm.oid.clone(); + if sig_oid != OID_SHA256_WITH_RSA_ENCRYPTION { + return Err(CrlProfileError::InvalidSignatureAlgorithm(sig_oid)); + } + if !self.signature_algorithm.params_absent_or_null() { + return Err(CrlProfileError::InvalidSignatureAlgorithmParameters); + } + + let extensions = validate_extensions_profile(&self.extensions)?; + + let mut revoked_out = Vec::with_capacity(self.revoked_certs.len()); + for rc in self.revoked_certs { + if rc.has_extensions { + return Err(CrlProfileError::EntryExtensionsNotAllowed); + } + validate_time_encoding_rfc5280("revocationDate", &rc.revocation_date)?; + revoked_out.push(RevokedCert { + serial_number: rc.serial_number, + revocation_date: rc.revocation_date, + }); + } + + validate_time_encoding_rfc5280("thisUpdate", &self.this_update)?; + + let next_update = self.next_update.ok_or(CrlProfileError::NextUpdateMissing)?; + validate_time_encoding_rfc5280("nextUpdate", &next_update)?; + + Ok(RpkixCrl { + raw_der: self.raw_der, + version, + issuer_dn: self.issuer_dn, + signature_algorithm_oid: OID_SHA256_WITH_RSA_ENCRYPTION.to_string(), + this_update: self.this_update, + next_update, + revoked_certs: revoked_out, + extensions, + }) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum CrlVerifyError { + #[error("issuer certificate parse error: {0} (RFC 5280 §4.1; RFC 6487 §4)")] + IssuerCertificateParse(String), + + #[error("trailing bytes after issuer certificate DER: {0} bytes (DER; RFC 5280 §4.1)")] + IssuerCertificateTrailingBytes(usize), + + #[error("issuer SubjectPublicKeyInfo parse error: {0} (RFC 5280 §4.1.2.7)")] + IssuerSpkiParse(String), + + #[error( + "trailing bytes after issuer SubjectPublicKeyInfo DER: {0} bytes (DER; RFC 5280 §4.1.2.7)" + )] + IssuerSpkiTrailingBytes(usize), + + #[error("CRL parse error: {0} (RFC 5280 §5.1; RFC 6487 §5)")] + CrlParse(String), + + #[error("trailing bytes after CRL DER: {0} bytes (DER; RFC 5280 §5.1)")] + CrlTrailingBytes(usize), + + #[error( + "CRL issuer DN does not match issuer certificate subject (RFC 5280 §5.1; RFC 5280 §6.3.3(b))" + )] + IssuerSubjectMismatch { + crl_issuer_dn: String, + issuer_subject_dn: String, + }, + + #[error( + "issuer certificate keyUsage present but missing cRLSign (RFC 5280 §4.2.1.3; RFC 5280 §6.3.3(f))" + )] + IssuerKeyUsageMissingCrlSign, + + #[error( + "CRL AKI.keyIdentifier does not match issuer certificate SKI (RFC 5280 §4.2.1.1; RFC 5280 §4.2.1.2; RFC 5280 §6.3.3(c)/(f))" + )] + AkiSkiMismatch, + + #[error("CRL signature verification failed: {0} (RFC 5280 §6.3.3(g); RFC 7935 §2)")] + InvalidSignature(String), +} + +fn validate_time_encoding_rfc5280( + field: &'static str, + t: &Asn1TimeUtc, +) -> Result<(), CrlProfileError> { + let year = t.utc.year(); + let expected = if year <= 2049 { + Asn1TimeEncoding::UtcTime + } else { + Asn1TimeEncoding::GeneralizedTime + }; + if t.encoding != expected { + return Err(CrlProfileError::InvalidTimeEncoding { + field, + year, + encoding: t.encoding, + }); + } + Ok(()) +} + +fn algorithm_identifier_value(ai: &AlgorithmIdentifier<'_>) -> AlgorithmIdentifierValue { + let parameters = ai.parameters.as_ref().map(|p| AlgorithmParametersValue { + class: p.class(), + tag: p.tag(), + data: p.as_bytes().to_vec(), + }); + // NOTE(perf): Avoid `to_id_string()` allocations for the signature algorithms we expect + // in RPKI CRLs. Fall back to `to_id_string()` for unexpected algorithms (mostly error paths). + let oid = if ai.algorithm.as_bytes() == OID_SHA256_WITH_RSA_ENCRYPTION_RAW { + OID_SHA256_WITH_RSA_ENCRYPTION.to_string() + } else { + ai.algorithm.to_id_string() + }; + AlgorithmIdentifierValue { oid, parameters } +} + +fn parse_extensions_parse(exts: &[X509Extension<'_>]) -> Result, String> { + let mut out = Vec::with_capacity(exts.len()); + for ext in exts { + let oid = ext.oid.as_bytes(); + if oid == OID_AUTHORITY_KEY_IDENTIFIER_RAW { + let ParsedExtension::AuthorityKeyIdentifier(aki) = ext.parsed_extension() else { + return Err("AKI extension parse failed".to_string()); + }; + out.push(CrlExtensionParsed::AuthorityKeyIdentifier { + key_identifier: aki.key_identifier.as_ref().map(|k| k.0.to_vec()), + has_other_fields: aki.authority_cert_issuer.is_some() + || aki.authority_cert_serial.is_some(), + critical: ext.critical, + }); + } else if oid == OID_CRL_NUMBER_RAW { + match ext.parsed_extension() { + ParsedExtension::CRLNumber(n) => out.push(CrlExtensionParsed::CrlNumber { + number: n.clone(), + critical: ext.critical, + }), + _ => return Err("CRLNumber extension parse failed".to_string()), + } + } else { + out.push(CrlExtensionParsed::Other { + oid: ext.oid.to_id_string(), + critical: ext.critical, + }) + } + } + Ok(out) +} + +fn validate_extensions_profile( + exts: &[CrlExtensionParsed], +) -> Result { + if exts.len() != 2 { + return Err(CrlProfileError::InvalidExtensionsCount(exts.len())); + } + + let mut seen: Vec = Vec::new(); + let mut authority_key_identifier: Option> = None; + let mut crl_number: Option = None; + + for ext in exts { + match ext { + CrlExtensionParsed::AuthorityKeyIdentifier { + key_identifier, + has_other_fields, + critical: _, + } => { + let oid = OID_AUTHORITY_KEY_IDENTIFIER.to_string(); + if seen.iter().any(|s| s == &oid) { + return Err(CrlProfileError::DuplicateExtension(oid)); + } + seen.push(oid.clone()); + + if *has_other_fields { + return Err(CrlProfileError::AkiHasOtherFields); + } + let ki = key_identifier + .as_ref() + .ok_or(CrlProfileError::AkiMissingKeyIdentifier)?; + authority_key_identifier = Some(ki.clone()); + } + CrlExtensionParsed::CrlNumber { number, critical } => { + let oid = OID_CRL_NUMBER.to_string(); + if seen.iter().any(|s| s == &oid) { + return Err(CrlProfileError::DuplicateExtension(oid)); + } + seen.push(oid.clone()); + + if *critical { + return Err(CrlProfileError::CrlNumberCritical); + } + if number.bits() > 159 { + return Err(CrlProfileError::CrlNumberOutOfRange); + } + crl_number = Some(BigUnsigned::from_biguint(number)); + } + CrlExtensionParsed::Other { oid, .. } => { + return Err(CrlProfileError::UnsupportedExtension(oid.clone())); + } + } + } + + Ok(CrlExtensions { + authority_key_identifier: authority_key_identifier.ok_or(CrlProfileError::AkiMissing)?, + crl_number: crl_number.ok_or(CrlProfileError::CrlNumberMissing)?, + }) +} + +fn get_subject_key_identifier(cert: &X509Certificate<'_>) -> Option> { + cert.extensions() + .iter() + .find(|ext| ext.oid.as_bytes() == OID_SUBJECT_KEY_IDENTIFIER_RAW) + .and_then(|ext| match ext.parsed_extension() { + ParsedExtension::SubjectKeyIdentifier(ki) => Some(ki.0.to_vec()), + _ => None, + }) +} diff --git a/crates/panda-rpki-validator/src/data_model/manifest.rs b/crates/panda-rpki-validator/src/data_model/manifest.rs new file mode 100644 index 0000000..bbd02db --- /dev/null +++ b/crates/panda-rpki-validator/src/data_model/manifest.rs @@ -0,0 +1,975 @@ +use crate::data_model::common::der_take_tlv; +use crate::data_model::common::{BigUnsigned, UtcTime}; +use crate::data_model::oid::{OID_CT_RPKI_MANIFEST, OID_SHA256}; +use crate::data_model::rc::ResourceCertificate; +use crate::data_model::signed_object::{ + RpkiSignedObject, RpkiSignedObjectParsed, SignedObjectDecodeError, SignedObjectParseError, + SignedObjectValidateError, +}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ManifestObject { + pub signed_object: RpkiSignedObject, + pub econtent_type: String, + pub manifest: ManifestEContent, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ManifestObjectParsed { + pub signed_object: RpkiSignedObjectParsed, + pub econtent_type: String, + pub manifest: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ManifestEContent { + pub version: u32, + pub manifest_number: BigUnsigned, + pub this_update: UtcTime, + pub next_update: UtcTime, + pub file_hash_alg: String, + /// DER-encoded content bytes of `Manifest.fileList` (SEQUENCE OF FileAndHash). + pub file_list_der: Vec, + /// Count of FileAndHash entries in `fileList`. + pub file_count: usize, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ManifestEContentParsed { + der: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FileAndHash { + pub file_name: String, + pub hash_bytes: [u8; 32], +} + +#[derive(Debug, thiserror::Error)] +pub enum ManifestParseError { + #[error("signed object parse error: {0} (RFC 6488 §2-§3; RFC 9589 §4)")] + SignedObject(#[from] SignedObjectParseError), + #[error("DER parse error: {0} (RFC 9286 §4.2; DER)")] + Parse(String), + + #[error("trailing bytes after DER object: {0} bytes (RFC 9286 §4.2; DER)")] + TrailingBytes(usize), +} + +#[derive(Debug, thiserror::Error)] +pub enum ManifestProfileError { + #[error("signed object profile error: {0} (RFC 6488 §2-§3; RFC 9589 §4)")] + SignedObject(#[from] SignedObjectValidateError), + + #[error( + "eContentType must be id-ct-rpkiManifest ({OID_CT_RPKI_MANIFEST}), got {0} (RFC 9286 §4.1; RFC 9286 §4.4(1))" + )] + InvalidEContentType(String), + + #[error("manifest profile decode error: {0} (RFC 9286 §4.2; DER)")] + ProfileDecode(String), + + #[error("Manifest must be a SEQUENCE of 5 or 6 elements, got {0} (RFC 9286 §4.2)")] + InvalidManifestSequenceLen(usize), + + #[error("Manifest.version must be 0, got {0} (RFC 9286 §4.2.1)")] + InvalidManifestVersion(u64), + + #[error( + "Manifest.manifestNumber must be non-negative INTEGER (RFC 9286 §4.2; RFC 9286 §4.2.1)" + )] + InvalidManifestNumber, + + #[error("Manifest.manifestNumber longer than 20 octets (RFC 9286 §4.2.1)")] + ManifestNumberTooLong, + + #[error("Manifest.thisUpdate must be GeneralizedTime (RFC 9286 §4.2)")] + InvalidThisUpdate, + + #[error("Manifest.nextUpdate must be GeneralizedTime (RFC 9286 §4.2)")] + InvalidNextUpdate, + + #[error("Manifest.nextUpdate must be later than thisUpdate (RFC 9286 §4.2.1)")] + NextUpdateNotLater, + + #[error( + "Manifest.fileHashAlg must be id-sha256 ({OID_SHA256}), got {0} (RFC 9286 §4.2.1; RFC 7935 §2)" + )] + InvalidFileHashAlg(String), + + #[error("Manifest.fileList must be a SEQUENCE (RFC 9286 §4.2)")] + InvalidFileList, + + #[error("FileAndHash must be SEQUENCE of 2 (RFC 9286 §4.2)")] + InvalidFileAndHash, + + #[error("fileList file name invalid: {0} (RFC 9286 §4.2.2)")] + InvalidFileName(String), + + #[error("fileList hash must be BIT STRING (RFC 9286 §4.2)")] + InvalidHashType, + + #[error( + "fileList hash BIT STRING must be octet-aligned (unused bits=0) (RFC 9286 §4.2.1; DER BIT STRING)" + )] + HashNotOctetAligned, + + #[error( + "fileList hash length invalid for sha256: got {0} bytes (RFC 9286 §4.2.1; RFC 7935 §2)" + )] + InvalidHashLength(usize), +} + +impl From for ManifestProfileError { + fn from(value: SignedObjectDecodeError) -> Self { + match value { + SignedObjectDecodeError::Parse(e) => ManifestProfileError::ProfileDecode(e.to_string()), + SignedObjectDecodeError::Validate(e) => ManifestProfileError::SignedObject(e), + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum ManifestDecodeError { + #[error("{0}")] + Parse(#[from] ManifestParseError), + + #[error("{0}")] + Validate(#[from] ManifestProfileError), +} + +#[derive(Debug, thiserror::Error)] +pub enum ManifestValidateError { + #[error( + "Manifest EE certificate MUST include at least one RFC 3779 resource extension (IP or AS) (RFC 6487 §4.8.10-§4.8.11; RFC 3779; RFC 9286 §5.1)" + )] + EeResourcesMissing, + + #[error( + "Manifest EE certificate IP resources MUST use inherit only (RFC 9286 §5.1; RFC 3779 §2.2.3.5)" + )] + EeIpResourcesNotInherit, + + #[error( + "Manifest EE certificate AS resources MUST use inherit only (RFC 9286 §5.1; RFC 3779 §3.2.3.3)" + )] + EeAsResourcesNotInherit, + + #[error( + "Manifest EE certificate AS resources rdi MUST be absent (RFC 6487 §4.8.11; RFC 3779 §3.2.3.5)" + )] + EeAsResourcesRdiPresent, +} + +impl ManifestObject { + /// Parse step of scheme A (`parse → validate → verify`). + pub fn parse_der(der: &[u8]) -> Result { + let signed_object = RpkiSignedObject::parse_der(der)?; + let econtent_type = signed_object + .signed_data + .encap_content_info + .econtent_type + .clone(); + let manifest = signed_object + .signed_data + .encap_content_info + .econtent + .as_deref() + .map(ManifestEContent::parse_der) + .transpose()?; + Ok(ManifestObjectParsed { + signed_object, + econtent_type, + manifest, + }) + } + + /// Profile validate step of scheme A (`parse → validate → verify`). + /// + /// `ManifestObject` is already profile-validated when constructed via `decode_der()` / + /// `ManifestObjectParsed::validate_profile()`. + pub fn validate_profile(&self) -> Result<(), ManifestProfileError> { + Ok(()) + } + + pub fn decode_der(der: &[u8]) -> Result { + Ok(Self::parse_der(der)?.validate_profile()?) + } + + pub fn decode_der_with_strict_options( + der: &[u8], + strict_cms_der: bool, + strict_name: bool, + ) -> Result { + let signed_object = + RpkiSignedObject::decode_der_with_strict_options(der, strict_cms_der, strict_name) + .map_err(ManifestProfileError::from)?; + Self::from_signed_object(signed_object) + } + + pub fn from_signed_object( + signed_object: RpkiSignedObject, + ) -> Result { + let econtent_type = signed_object + .signed_data + .encap_content_info + .econtent_type + .clone(); + if econtent_type != OID_CT_RPKI_MANIFEST { + return Err(ManifestProfileError::InvalidEContentType(econtent_type).into()); + } + let manifest = + ManifestEContent::decode_der(&signed_object.signed_data.encap_content_info.econtent)?; + Ok(Self { + signed_object, + econtent_type: OID_CT_RPKI_MANIFEST.to_string(), + manifest, + }) + } + + /// Validate the embedded EE certificate resources against RFC 9286 §5.1. + /// + /// This does **not** perform certificate path validation. It assumes `ee` is a parsed and + /// profile-validated RPKI EE resource certificate. + pub fn validate_against_ee_cert( + &self, + ee: &ResourceCertificate, + ) -> Result<(), ManifestValidateError> { + let ip = ee.tbs.extensions.ip_resources.as_ref(); + let asn = ee.tbs.extensions.as_resources.as_ref(); + if ip.is_none() && asn.is_none() { + return Err(ManifestValidateError::EeResourcesMissing); + } + + if let Some(ip) = ip { + if !ip.is_all_inherit() { + return Err(ManifestValidateError::EeIpResourcesNotInherit); + } + } + + if let Some(asn) = asn { + if asn.rdi.is_some() { + return Err(ManifestValidateError::EeAsResourcesRdiPresent); + } + if !asn.is_asnum_inherit() { + return Err(ManifestValidateError::EeAsResourcesNotInherit); + } + } + + Ok(()) + } + + /// Validate this manifest's embedded EE certificate resources. + pub fn validate_embedded_ee_cert(&self) -> Result<(), ManifestValidateError> { + let ee = &self.signed_object.signed_data.certificates[0].resource_cert; + self.validate_against_ee_cert(ee) + } +} + +impl ManifestEContent { + /// Parse step of scheme A (`parse → validate → verify`). + pub fn parse_der(der: &[u8]) -> Result { + let (_tag, _value, rem) = der_take_tlv(der).map_err(|e| ManifestParseError::Parse(e))?; + if !rem.is_empty() { + return Err(ManifestParseError::TrailingBytes(rem.len())); + } + Ok(ManifestEContentParsed { der: der.to_vec() }) + } + + /// Profile validate step of scheme A (`parse → validate → verify`). + /// + /// `ManifestEContent` is already profile-validated when constructed via `decode_der()` / + /// `ManifestEContentParsed::validate_profile()`. + pub fn validate_profile(&self) -> Result<(), ManifestProfileError> { + Ok(()) + } + + /// Decode the DER-encoded Manifest eContent defined in RFC 9286 §4.2 (`parse + validate`). + pub fn decode_der(der: &[u8]) -> Result { + Ok(Self::parse_der(der)?.validate_profile()?) + } + + /// Parse and return the manifest fileList. + /// + /// Note: `ManifestEContent` is profile-validated when produced via `decode_der()`, so this + /// should only fail due to internal inconsistencies (or if constructed manually). + pub fn parse_files(&self) -> Result, ManifestProfileError> { + parse_file_list_sha256_fast(&self.file_list_der) + } + + pub fn file_count(&self) -> usize { + self.file_count + } +} + +impl ManifestObjectParsed { + pub fn validate_profile(self) -> Result { + let signed_object = self.signed_object.validate_profile()?; + let econtent_type = signed_object + .signed_data + .encap_content_info + .econtent_type + .clone(); + if econtent_type != OID_CT_RPKI_MANIFEST { + return Err(ManifestProfileError::InvalidEContentType(econtent_type)); + } + let manifest = self + .manifest + .ok_or_else(|| ManifestProfileError::ProfileDecode("Manifest.eContent missing".into()))? + .validate_profile()?; + Ok(ManifestObject { + signed_object, + econtent_type: OID_CT_RPKI_MANIFEST.to_string(), + manifest, + }) + } +} + +impl ManifestEContentParsed { + pub fn validate_profile(self) -> Result { + decode_manifest_econtent_fast(&self.der) + } +} + +fn validate_file_name_bytes(bytes: &[u8]) -> Result<(), ManifestProfileError> { + // RFC 9286 §4.2.2: + // 1+ chars from a-zA-Z0-9-_ , then '.', then 3-letter extension. + if bytes.len() < 5 { + return Err(ManifestProfileError::InvalidFileName( + String::from_utf8_lossy(bytes).into_owned(), + )); + }; + + // "followed by a single . (DOT), followed by a three letter extension" + // -> the dot must be exactly 4 bytes from the end. + let dot_pos = bytes.len() - 4; + if bytes[dot_pos] != b'.' { + return Err(ManifestProfileError::InvalidFileName( + String::from_utf8_lossy(bytes).into_owned(), + )); + } + + #[inline(always)] + fn valid_base_char(b: u8) -> bool { + // RFC 9286 allowed set: a-zA-Z0-9-_ + (b'0'..=b'9').contains(&b) + || (b'a'..=b'z').contains(&b) + || (b'A'..=b'Z').contains(&b) + || b == b'-' + || b == b'_' + } + + for &b in &bytes[..dot_pos] { + if (b & 0x80) != 0 || !valid_base_char(b) { + return Err(ManifestProfileError::InvalidFileName( + String::from_utf8_lossy(bytes).into_owned(), + )); + } + } + + let e0 = bytes[dot_pos + 1]; + let e1 = bytes[dot_pos + 2]; + let e2 = bytes[dot_pos + 3]; + if (e0 & 0x80) != 0 || (e1 & 0x80) != 0 || (e2 & 0x80) != 0 { + return Err(ManifestProfileError::InvalidFileName( + String::from_utf8_lossy(bytes).into_owned(), + )); + } + + #[inline(always)] + fn lower_if_alpha(b: u8) -> Option { + match b { + b'a'..=b'z' => Some(b), + b'A'..=b'Z' => Some(b + 32), + _ => None, + } + } + + let Some(l0) = lower_if_alpha(e0) else { + return Err(ManifestProfileError::InvalidFileName( + String::from_utf8_lossy(bytes).into_owned(), + )); + }; + let Some(l1) = lower_if_alpha(e1) else { + return Err(ManifestProfileError::InvalidFileName( + String::from_utf8_lossy(bytes).into_owned(), + )); + }; + let Some(l2) = lower_if_alpha(e2) else { + return Err(ManifestProfileError::InvalidFileName( + String::from_utf8_lossy(bytes).into_owned(), + )); + }; + + match [l0, l1, l2] { + // Full IANA list (see `common.rs`). + [b'a', b's', b'a'] + | [b'c', b'e', b'r'] + | [b'c', b'r', b'l'] + | [b'g', b'b', b'r'] + | [b'm', b'f', b't'] + | [b'r', b'o', b'a'] + | [b's', b'i', b'g'] + | [b't', b'a', b'k'] => Ok(()), + _ => Err(ManifestProfileError::InvalidFileName( + String::from_utf8_lossy(bytes).into_owned(), + )), + } +} + +fn decode_manifest_econtent_fast(der: &[u8]) -> Result { + let (tag, mut seq_content, rem) = der_take_tlv(der) + .map_err(|e| ManifestProfileError::ProfileDecode(format!("DER decode error: {e}")))?; + if !rem.is_empty() { + return Err(ManifestProfileError::ProfileDecode(format!( + "trailing bytes after DER object: {} bytes", + rem.len() + ))); + } + if tag != 0x30 { + return Err(ManifestProfileError::ProfileDecode( + "Manifest eContent must be SEQUENCE".into(), + )); + } + + let seq_len = + der_count_elements(seq_content).map_err(|e| ManifestProfileError::ProfileDecode(e))?; + if seq_len != 5 && seq_len != 6 { + return Err(ManifestProfileError::InvalidManifestSequenceLen(seq_len)); + } + + let mut version: u32 = 0; + if seq_len == 6 { + let Some(&first_tag) = seq_content.first() else { + return Err(ManifestProfileError::InvalidManifestSequenceLen(0)); + }; + if first_tag != 0xA0 { + return Err(ManifestProfileError::ProfileDecode( + "Manifest.version must be [0] EXPLICIT INTEGER".into(), + )); + } + + let (_cs_tag, cs_value, after) = der_take_tlv(seq_content).map_err(|e| { + ManifestProfileError::ProfileDecode(format!("Manifest.version decode error: {e}")) + })?; + seq_content = after; + + let (inner_tag, inner_value, inner_rem) = der_take_tlv(cs_value).map_err(|e| { + ManifestProfileError::ProfileDecode(format!("Manifest.version inner decode error: {e}")) + })?; + if !inner_rem.is_empty() { + return Err(ManifestProfileError::ProfileDecode( + "trailing bytes inside Manifest.version".into(), + )); + } + if inner_tag != 0x02 { + return Err(ManifestProfileError::ProfileDecode( + "Manifest.version must be [0] EXPLICIT INTEGER".into(), + )); + } + let v = der_integer_to_u64(inner_value).map_err(|e| { + ManifestProfileError::ProfileDecode(format!("Manifest.version decode error: {e}")) + })?; + if v != 0 { + return Err(ManifestProfileError::InvalidManifestVersion(v)); + } + version = 0; + } + + let (mn_tag, mn_value, after) = der_take_tlv(seq_content).map_err(|e| { + ManifestProfileError::ProfileDecode(format!("Manifest.manifestNumber decode error: {e}")) + })?; + seq_content = after; + if mn_tag != 0x02 { + return Err(ManifestProfileError::InvalidManifestNumber); + } + let manifest_number = der_integer_to_bigunsigned(mn_value)?; + + let (tu_tag, tu_value, after) = der_take_tlv(seq_content).map_err(|e| { + ManifestProfileError::ProfileDecode(format!("Manifest.thisUpdate decode error: {e}")) + })?; + seq_content = after; + if tu_tag != 0x18 { + return Err(ManifestProfileError::InvalidThisUpdate); + } + let this_update = parse_generalized_time_bytes(tu_value) + .map_err(|e| ManifestProfileError::ProfileDecode(e))?; + + let (nu_tag, nu_value, after) = der_take_tlv(seq_content).map_err(|e| { + ManifestProfileError::ProfileDecode(format!("Manifest.nextUpdate decode error: {e}")) + })?; + seq_content = after; + if nu_tag != 0x18 { + return Err(ManifestProfileError::InvalidNextUpdate); + } + let next_update = parse_generalized_time_bytes(nu_value) + .map_err(|e| ManifestProfileError::ProfileDecode(e))?; + if next_update <= this_update { + return Err(ManifestProfileError::NextUpdateNotLater); + } + + let (oid_tag, oid_value, after) = der_take_tlv(seq_content).map_err(|e| { + ManifestProfileError::ProfileDecode(format!("Manifest.fileHashAlg decode error: {e}")) + })?; + seq_content = after; + if oid_tag != 0x06 { + return Err(ManifestProfileError::ProfileDecode( + "Manifest.fileHashAlg must be OBJECT IDENTIFIER".into(), + )); + } + if !oid_content_is_sha256(oid_value) { + return Err(ManifestProfileError::InvalidFileHashAlg( + oid_content_to_string(oid_value), + )); + } + + let (fl_tag, fl_value, after) = der_take_tlv(seq_content).map_err(|e| { + ManifestProfileError::ProfileDecode(format!("Manifest.fileList decode error: {e}")) + })?; + seq_content = after; + if fl_tag != 0x30 { + return Err(ManifestProfileError::InvalidFileList); + } + let file_count = validate_file_list_sha256_fast(fl_value)?; + let file_list_der = fl_value.to_vec(); + + if !seq_content.is_empty() { + return Err(ManifestProfileError::InvalidManifestSequenceLen(seq_len)); + } + + Ok(ManifestEContent { + version, + manifest_number, + this_update, + next_update, + file_hash_alg: OID_SHA256.to_string(), + file_list_der, + file_count, + }) +} + +fn validate_file_list_sha256_fast(content: &[u8]) -> Result { + let mut cur = content; + let mut count: usize = 0; + while !cur.is_empty() { + let (tag, value, rem) = der_take_tlv(cur).map_err(|e| { + ManifestProfileError::ProfileDecode(format!("fileList entry decode error: {e}")) + })?; + cur = rem; + if tag != 0x30 { + return Err(ManifestProfileError::InvalidFileAndHash); + } + + let mut entry = value; + let (fn_tag, fn_value, entry_rem) = der_take_tlv(entry).map_err(|e| { + ManifestProfileError::ProfileDecode(format!("fileList fileName decode error: {e}")) + })?; + entry = entry_rem; + if fn_tag != 0x16 { + return Err(ManifestProfileError::InvalidFileAndHash); + } + if entry.is_empty() { + return Err(ManifestProfileError::InvalidFileAndHash); + } + validate_file_name_bytes(fn_value)?; + + let (hash_tag, hash_value, entry_rem) = der_take_tlv(entry).map_err(|_e| { + // Missing second element should map to "SEQUENCE of 2" shape error. + ManifestProfileError::InvalidFileAndHash + })?; + entry = entry_rem; + if !entry.is_empty() { + return Err(ManifestProfileError::InvalidFileAndHash); + } + if hash_tag != 0x03 { + return Err(ManifestProfileError::InvalidHashType); + } + if hash_value.is_empty() { + return Err(ManifestProfileError::InvalidHashLength(0)); + } + let unused_bits = hash_value[0]; + if unused_bits != 0 { + return Err(ManifestProfileError::HashNotOctetAligned); + } + let bits = &hash_value[1..]; + if bits.len() != 32 { + return Err(ManifestProfileError::InvalidHashLength(bits.len())); + } + count += 1; + } + Ok(count) +} + +fn parse_file_list_sha256_fast(content: &[u8]) -> Result, ManifestProfileError> { + // Heuristic initial capacity (avoid a full pre-scan, which is expensive for xlarge manifests). + // Each FileAndHash entry is typically tens of bytes; 80 is a conservative average. + let est = (content.len() / 80).clamp(16, 4096); + let mut cur = content; + let mut out: Vec = Vec::with_capacity(est); + while !cur.is_empty() { + let (tag, value, rem) = der_take_tlv(cur).map_err(|e| { + ManifestProfileError::ProfileDecode(format!("fileList entry decode error: {e}")) + })?; + cur = rem; + if tag != 0x30 { + return Err(ManifestProfileError::InvalidFileAndHash); + } + let mut entry = value; + let (fn_tag, fn_value, entry_rem) = der_take_tlv(entry).map_err(|e| { + ManifestProfileError::ProfileDecode(format!("fileList fileName decode error: {e}")) + })?; + entry = entry_rem; + if fn_tag != 0x16 { + return Err(ManifestProfileError::InvalidFileAndHash); + } + if entry.is_empty() { + return Err(ManifestProfileError::InvalidFileAndHash); + } + let file_name = validate_and_copy_file_name(fn_value)?; + + let (hash_tag, hash_value, entry_rem) = der_take_tlv(entry).map_err(|_e| { + // Missing second element should map to "SEQUENCE of 2" shape error. + ManifestProfileError::InvalidFileAndHash + })?; + entry = entry_rem; + if !entry.is_empty() { + return Err(ManifestProfileError::InvalidFileAndHash); + } + if hash_tag != 0x03 { + return Err(ManifestProfileError::InvalidHashType); + } + if hash_value.is_empty() { + return Err(ManifestProfileError::InvalidHashLength(0)); + } + let unused_bits = hash_value[0]; + if unused_bits != 0 { + return Err(ManifestProfileError::HashNotOctetAligned); + } + let bits = &hash_value[1..]; + if bits.len() != 32 { + return Err(ManifestProfileError::InvalidHashLength(bits.len())); + } + let mut hash_bytes = [0u8; 32]; + hash_bytes.copy_from_slice(bits); + out.push(FileAndHash { + file_name, + hash_bytes, + }); + } + Ok(out) +} + +fn validate_and_copy_file_name(bytes: &[u8]) -> Result { + validate_file_name_bytes(bytes)?; + Ok(unsafe { String::from_utf8_unchecked(bytes.to_vec()) }) +} + +fn der_count_elements(mut input: &[u8]) -> Result { + let mut count: usize = 0; + while !input.is_empty() { + let (_tag, _value, rem) = der_take_tlv(input)?; + input = rem; + count += 1; + } + Ok(count) +} + +fn der_integer_to_u64(bytes: &[u8]) -> Result { + if bytes.is_empty() { + return Err("INTEGER empty".into()); + } + // Reject negative (two's complement). + if bytes[0] & 0x80 != 0 { + return Err("INTEGER is negative".into()); + } + if bytes.len() > 8 { + return Err("INTEGER too large".into()); + } + let mut v: u64 = 0; + for &b in bytes { + v = (v << 8) | (b as u64); + } + Ok(v) +} + +fn der_integer_to_bigunsigned(bytes: &[u8]) -> Result { + if bytes.is_empty() { + return Err(ManifestProfileError::InvalidManifestNumber); + } + // Two's complement: for non-negative values, a leading 0x00 may be present. + if bytes[0] & 0x80 != 0 { + return Err(ManifestProfileError::InvalidManifestNumber); + } + let mut start = 0usize; + while start + 1 < bytes.len() && bytes[start] == 0 { + start += 1; + } + let mut minimal = bytes[start..].to_vec(); + if minimal.is_empty() { + minimal.push(0); + } + if minimal.len() > 20 { + return Err(ManifestProfileError::ManifestNumberTooLong); + } + Ok(BigUnsigned { bytes_be: minimal }) +} + +fn parse_generalized_time_bytes(bytes: &[u8]) -> Result { + // Accept "YYYYMMDDHHMMSSZ" and also allow optional fractional seconds (".fff...Z"). + if !bytes.is_ascii() { + return Err("GeneralizedTime not ASCII".into()); + } + let s = std::str::from_utf8(bytes).map_err(|e| e.to_string())?; + if !s.ends_with('Z') { + return Err("GeneralizedTime must end with 'Z'".into()); + } + let core = &s[..s.len() - 1]; + let (main, frac) = core + .split_once('.') + .map_or((core, None), |(a, b)| (a, Some(b))); + if main.len() != 14 || !main.bytes().all(|b| b.is_ascii_digit()) { + return Err("GeneralizedTime must be YYYYMMDDHHMMSS[.fff]Z".into()); + } + let year: i32 = main[0..4].parse().map_err(|_| "bad year")?; + let month: u8 = main[4..6].parse().map_err(|_| "bad month")?; + let day: u8 = main[6..8].parse().map_err(|_| "bad day")?; + let hour: u8 = main[8..10].parse().map_err(|_| "bad hour")?; + let minute: u8 = main[10..12].parse().map_err(|_| "bad minute")?; + let second: u8 = main[12..14].parse().map_err(|_| "bad second")?; + + let nanosecond: u32 = if let Some(frac) = frac { + if frac.is_empty() || !frac.bytes().all(|b| b.is_ascii_digit()) { + return Err("bad fractional seconds".into()); + } + let mut ns: u32 = 0; + let mut scale: u32 = 1_000_000_000; + for (i, ch) in frac.bytes().enumerate() { + if i >= 9 { + break; + } + scale /= 10; + ns += ((ch - b'0') as u32) * scale; + } + ns + } else { + 0 + }; + + let date = time::Date::from_calendar_date( + year, + time::Month::try_from(month).map_err(|_| "bad month")?, + day, + ) + .map_err(|e| e.to_string())?; + let t = + time::Time::from_hms_nano(hour, minute, second, nanosecond).map_err(|e| e.to_string())?; + Ok(date.with_time(t).assume_utc()) +} + +fn oid_content_is_sha256(bytes: &[u8]) -> bool { + // 2.16.840.1.101.3.4.2.1 + let mut arcs = oid_content_iter(bytes); + const EXPECTED: &[u64] = &[2, 16, 840, 1, 101, 3, 4, 2, 1]; + for &e in EXPECTED { + match arcs.next() { + Some(v) if v == e => {} + _ => return false, + } + } + arcs.next().is_none() +} + +fn oid_content_to_string(bytes: &[u8]) -> String { + let arcs: Vec = oid_content_iter(bytes).collect(); + if arcs.is_empty() { + return "".to_string(); + } + let mut s = String::new(); + for (i, a) in arcs.iter().enumerate() { + if i > 0 { + s.push('.'); + } + s.push_str(&a.to_string()); + } + s +} + +fn oid_content_iter(bytes: &[u8]) -> impl Iterator + '_ { + struct It<'a> { + bytes: &'a [u8], + pos: usize, + first_done: bool, + first_a0: u64, + first_a1: u64, + emit_first_idx: u8, + } + impl<'a> Iterator for It<'a> { + type Item = u64; + fn next(&mut self) -> Option { + if !self.first_done { + if self.bytes.is_empty() { + self.first_done = true; + return None; + } + let first = self.bytes[0] as u64; + self.first_a0 = first / 40; + self.first_a1 = first % 40; + self.pos = 1; + self.first_done = true; + self.emit_first_idx = 0; + } + if self.emit_first_idx == 0 { + self.emit_first_idx = 1; + return Some(self.first_a0); + } + if self.emit_first_idx == 1 { + self.emit_first_idx = 2; + return Some(self.first_a1); + } + if self.pos >= self.bytes.len() { + return None; + } + let mut v: u64 = 0; + while self.pos < self.bytes.len() { + let b = self.bytes[self.pos]; + self.pos += 1; + v = (v << 7) | ((b & 0x7F) as u64); + if b & 0x80 == 0 { + return Some(v); + } + } + None + } + } + It { + bytes, + pos: 0, + first_done: false, + first_a0: 0, + first_a1: 0, + emit_first_idx: 0, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tlv(tag: u8, value: &[u8]) -> Vec { + assert!(value.len() < 128); + let mut out = Vec::with_capacity(2 + value.len()); + out.push(tag); + out.push(value.len() as u8); + out.extend_from_slice(value); + out + } + + fn tlv_long_len(tag: u8, len_bytes: &[u8], value: &[u8]) -> Vec { + let mut out = Vec::with_capacity(2 + len_bytes.len() + value.len()); + out.push(tag); + out.push(0x80 | (len_bytes.len() as u8)); + out.extend_from_slice(len_bytes); + out.extend_from_slice(value); + out + } + + #[test] + fn der_take_tlv_supports_short_and_long_form_lengths_and_errors() { + let v = b"abc"; + let der = tlv(0x04, v); + let (tag, val, rem) = der_take_tlv(&der).expect("short len"); + assert_eq!(tag, 0x04); + assert_eq!(val, v); + assert!(rem.is_empty()); + + // Long-form length with 1 length byte (130). + let v = vec![b'x'; 130]; + let der = tlv_long_len(0x04, &[0x82], &v); + let (tag, val, rem) = der_take_tlv(&der).expect("long len 1"); + assert_eq!(tag, 0x04); + assert_eq!(val.len(), 130); + assert!(rem.is_empty()); + + // Long-form length with 2 length bytes (256). + let v = vec![b'y'; 256]; + let der = tlv_long_len(0x04, &[0x01, 0x00], &v); + let (tag, val, rem) = der_take_tlv(&der).expect("long len 2"); + assert_eq!(tag, 0x04); + assert_eq!(val.len(), 256); + assert!(rem.is_empty()); + + assert!(der_take_tlv(&[]).is_err()); + assert!(der_take_tlv(&[0x04]).is_err()); + + // High-tag-number form not supported. + assert!(der_take_tlv(&[0x1F, 0x01, 0x00]).is_err()); + + // Indefinite length is not allowed in DER. + assert!(der_take_tlv(&[0x04, 0x80]).is_err()); + + // Invalid long-form length encoding. + assert!(der_take_tlv(&[0x04, 0x81]).is_err()); + assert!(der_take_tlv(&[0x04, 0x89]).is_err()); + } + + #[test] + fn parse_generalized_time_bytes_accepts_fraction_and_rejects_invalid() { + let t = parse_generalized_time_bytes(b"20260101000000Z").expect("basic time"); + assert_eq!(t.year(), 2026); + + let t = parse_generalized_time_bytes(b"20260101000000.1Z").expect("fractional"); + assert_eq!(t.nanosecond(), 100_000_000); + + assert!(parse_generalized_time_bytes(b"20260101000000").is_err()); + assert!(parse_generalized_time_bytes(b"20260101000000+00").is_err()); + assert!(parse_generalized_time_bytes(b"2026010100000Z").is_err()); + assert!(parse_generalized_time_bytes(b"20261301000000Z").is_err()); + assert!(parse_generalized_time_bytes(b"20260132000000Z").is_err()); + assert!(parse_generalized_time_bytes(&[0xFF]).is_err()); + } + + #[test] + fn oid_helpers_accept_sha256_and_format_invalid() { + // 2.16.840.1.101.3.4.2.1 + let sha256_oid_content = [0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01]; + assert!(oid_content_is_sha256(&sha256_oid_content)); + assert!(!oid_content_is_sha256(&[0x55, 0x04, 0x03])); // 2.5.4.3 + + assert_eq!(oid_content_to_string(&[]), "".to_string()); + } + + #[test] + fn validate_file_list_sha256_fast_counts_and_rejects_bad_hash() { + fn file_and_hash(file: &str, digest: u8) -> Vec { + let mut hash = vec![0u8; 33]; + hash[0] = 0; // unused bits + for b in &mut hash[1..] { + *b = digest; + } + let ia5 = tlv(0x16, file.as_bytes()); + let bit = tlv(0x03, &hash); + let mut entry = Vec::new(); + entry.extend_from_slice(&ia5); + entry.extend_from_slice(&bit); + tlv(0x30, &entry) + } + + let mut list = Vec::new(); + list.extend_from_slice(&file_and_hash("A.cer", 0xAA)); + list.extend_from_slice(&file_and_hash("B.roa", 0xBB)); + assert_eq!(validate_file_list_sha256_fast(&list).expect("count"), 2); + + // Wrong hash length. + let mut bad = Vec::new(); + let ia5 = tlv(0x16, b"A.cer"); + let bit = tlv(0x03, &[0u8; 2]); // too short + let mut entry = Vec::new(); + entry.extend_from_slice(&ia5); + entry.extend_from_slice(&bit); + bad.extend_from_slice(&tlv(0x30, &entry)); + assert!(matches!( + validate_file_list_sha256_fast(&bad), + Err(ManifestProfileError::InvalidHashLength(_)) + )); + } +} diff --git a/crates/panda-rpki-validator/src/data_model/mod.rs b/crates/panda-rpki-validator/src/data_model/mod.rs new file mode 100644 index 0000000..011570d --- /dev/null +++ b/crates/panda-rpki-validator/src/data_model/mod.rs @@ -0,0 +1,13 @@ +pub mod aspa; +pub mod common; +pub mod crl; +pub mod manifest; +pub mod oid; +pub mod rc; +pub mod roa; +pub mod signed_object; +pub mod ta; +pub mod tal; + +#[cfg(feature = "full")] +pub mod router_cert; diff --git a/crates/panda-rpki-validator/src/data_model/oid.rs b/crates/panda-rpki-validator/src/data_model/oid.rs new file mode 100644 index 0000000..a5504cb --- /dev/null +++ b/crates/panda-rpki-validator/src/data_model/oid.rs @@ -0,0 +1,78 @@ +pub const OID_SHA256: &str = "2.16.840.1.101.3.4.2.1"; +pub const OID_SHA256_RAW: &[u8] = &asn1_rs::oid!(raw 2.16.840.1.101.3.4.2.1); + +pub const OID_SIGNED_DATA: &str = "1.2.840.113549.1.7.2"; +pub const OID_SIGNED_DATA_RAW: &[u8] = &asn1_rs::oid!(raw 1.2.840.113549.1.7.2); + +pub const OID_CMS_ATTR_CONTENT_TYPE: &str = "1.2.840.113549.1.9.3"; +pub const OID_CMS_ATTR_CONTENT_TYPE_RAW: &[u8] = &asn1_rs::oid!(raw 1.2.840.113549.1.9.3); +pub const OID_CMS_ATTR_MESSAGE_DIGEST: &str = "1.2.840.113549.1.9.4"; +pub const OID_CMS_ATTR_MESSAGE_DIGEST_RAW: &[u8] = &asn1_rs::oid!(raw 1.2.840.113549.1.9.4); +pub const OID_CMS_ATTR_SIGNING_TIME: &str = "1.2.840.113549.1.9.5"; +pub const OID_CMS_ATTR_SIGNING_TIME_RAW: &[u8] = &asn1_rs::oid!(raw 1.2.840.113549.1.9.5); + +pub const OID_RSA_ENCRYPTION: &str = "1.2.840.113549.1.1.1"; +pub const OID_RSA_ENCRYPTION_RAW: &[u8] = &asn1_rs::oid!(raw 1.2.840.113549.1.1.1); +pub const OID_SHA256_WITH_RSA_ENCRYPTION: &str = "1.2.840.113549.1.1.11"; +pub const OID_SHA256_WITH_RSA_ENCRYPTION_RAW: &[u8] = &asn1_rs::oid!(raw 1.2.840.113549.1.1.11); + +// X.509 extensions (RFC 5280 / RFC 6487) +pub const OID_BASIC_CONSTRAINTS: &str = "2.5.29.19"; +pub const OID_BASIC_CONSTRAINTS_RAW: &[u8] = &asn1_rs::oid!(raw 2.5.29.19); +pub const OID_KEY_USAGE: &str = "2.5.29.15"; +pub const OID_KEY_USAGE_RAW: &[u8] = &asn1_rs::oid!(raw 2.5.29.15); +pub const OID_EXTENDED_KEY_USAGE: &str = "2.5.29.37"; +pub const OID_EXTENDED_KEY_USAGE_RAW: &[u8] = &asn1_rs::oid!(raw 2.5.29.37); +pub const OID_CRL_DISTRIBUTION_POINTS: &str = "2.5.29.31"; +pub const OID_CRL_DISTRIBUTION_POINTS_RAW: &[u8] = &asn1_rs::oid!(raw 2.5.29.31); +pub const OID_AUTHORITY_INFO_ACCESS: &str = "1.3.6.1.5.5.7.1.1"; +pub const OID_AUTHORITY_INFO_ACCESS_RAW: &[u8] = &asn1_rs::oid!(raw 1.3.6.1.5.5.7.1.1); +pub const OID_CERTIFICATE_POLICIES: &str = "2.5.29.32"; +pub const OID_CERTIFICATE_POLICIES_RAW: &[u8] = &asn1_rs::oid!(raw 2.5.29.32); +pub const OID_QT_CPS: &str = "1.3.6.1.5.5.7.2.1"; + +pub const OID_AUTHORITY_KEY_IDENTIFIER: &str = "2.5.29.35"; +pub const OID_AUTHORITY_KEY_IDENTIFIER_RAW: &[u8] = &asn1_rs::oid!(raw 2.5.29.35); +pub const OID_CRL_NUMBER: &str = "2.5.29.20"; +pub const OID_CRL_NUMBER_RAW: &[u8] = &asn1_rs::oid!(raw 2.5.29.20); +pub const OID_SUBJECT_KEY_IDENTIFIER: &str = "2.5.29.14"; +pub const OID_SUBJECT_KEY_IDENTIFIER_RAW: &[u8] = &asn1_rs::oid!(raw 2.5.29.14); + +pub const OID_CT_RPKI_MANIFEST: &str = "1.2.840.113549.1.9.16.1.26"; +pub const OID_CT_RPKI_MANIFEST_RAW: &[u8] = &asn1_rs::oid!(raw 1.2.840.113549.1.9.16.1.26); +pub const OID_CT_ROUTE_ORIGIN_AUTHZ: &str = "1.2.840.113549.1.9.16.1.24"; +pub const OID_CT_ROUTE_ORIGIN_AUTHZ_RAW: &[u8] = &asn1_rs::oid!(raw 1.2.840.113549.1.9.16.1.24); +pub const OID_CT_ASPA: &str = "1.2.840.113549.1.9.16.1.49"; +pub const OID_CT_ASPA_RAW: &[u8] = &asn1_rs::oid!(raw 1.2.840.113549.1.9.16.1.49); + +// X.509 extensions / access methods (RFC 5280 / RFC 6487) +pub const OID_SUBJECT_INFO_ACCESS: &str = "1.3.6.1.5.5.7.1.11"; +pub const OID_SUBJECT_INFO_ACCESS_RAW: &[u8] = &asn1_rs::oid!(raw 1.3.6.1.5.5.7.1.11); +pub const OID_AD_SIGNED_OBJECT: &str = "1.3.6.1.5.5.7.48.11"; +pub const OID_AD_SIGNED_OBJECT_RAW: &[u8] = &asn1_rs::oid!(raw 1.3.6.1.5.5.7.48.11); + +pub const OID_AD_CA_ISSUERS: &str = "1.3.6.1.5.5.7.48.2"; +pub const OID_AD_CA_ISSUERS_RAW: &[u8] = &asn1_rs::oid!(raw 1.3.6.1.5.5.7.48.2); +pub const OID_AD_CA_REPOSITORY: &str = "1.3.6.1.5.5.7.48.5"; +pub const OID_AD_CA_REPOSITORY_RAW: &[u8] = &asn1_rs::oid!(raw 1.3.6.1.5.5.7.48.5); +pub const OID_AD_RPKI_MANIFEST: &str = "1.3.6.1.5.5.7.48.10"; +pub const OID_AD_RPKI_MANIFEST_RAW: &[u8] = &asn1_rs::oid!(raw 1.3.6.1.5.5.7.48.10); +pub const OID_AD_RPKI_NOTIFY: &str = "1.3.6.1.5.5.7.48.13"; +pub const OID_AD_RPKI_NOTIFY_RAW: &[u8] = &asn1_rs::oid!(raw 1.3.6.1.5.5.7.48.13); + +// RFC 3779 resource extensions (RFC 6487 profile) +pub const OID_IP_ADDR_BLOCKS: &str = "1.3.6.1.5.5.7.1.7"; +pub const OID_IP_ADDR_BLOCKS_RAW: &[u8] = &asn1_rs::oid!(raw 1.3.6.1.5.5.7.1.7); +pub const OID_AUTONOMOUS_SYS_IDS: &str = "1.3.6.1.5.5.7.1.8"; +pub const OID_AUTONOMOUS_SYS_IDS_RAW: &[u8] = &asn1_rs::oid!(raw 1.3.6.1.5.5.7.1.8); + +// RPKI CP (RFC 6484 / RFC 6487) +pub const OID_CP_IPADDR_ASNUMBER: &str = "1.3.6.1.5.5.7.14.2"; +pub const OID_CP_IPADDR_ASNUMBER_RAW: &[u8] = &asn1_rs::oid!(raw 1.3.6.1.5.5.7.14.2); + +pub const OID_CT_RPKI_CCR: &str = "1.2.840.113549.1.9.16.1.54"; +pub const OID_CT_RPKI_CCR_RAW: &[u8] = &asn1_rs::oid!(raw 1.2.840.113549.1.9.16.1.54); + +pub const OID_KP_BGPSEC_ROUTER: &str = "1.3.6.1.5.5.7.3.30"; +pub const OID_EC_PUBLIC_KEY: &str = "1.2.840.10045.2.1"; +pub const OID_SECP256R1: &str = "1.2.840.10045.3.1.7"; diff --git a/crates/panda-rpki-validator/src/data_model/rc.rs b/crates/panda-rpki-validator/src/data_model/rc.rs new file mode 100644 index 0000000..701ea91 --- /dev/null +++ b/crates/panda-rpki-validator/src/data_model/rc.rs @@ -0,0 +1,1868 @@ +use der_parser::ber::{BerObjectContent, Class}; +use der_parser::der::{DerObject, Tag, parse_der}; +use der_parser::num_bigint::BigUint; +use x509_parser::asn1_rs::{Class as Asn1Class, Tag as Asn1Tag}; +use x509_parser::extensions::ParsedExtension; +use x509_parser::prelude::{FromDer, X509Certificate, X509Extension, X509Version}; + +use serde::{Deserialize, Serialize}; + +use crate::data_model::common::{ + Asn1TimeUtc, DerReader, InvalidTimeEncodingError, UtcTime, X509NameDer, asn1_time_to_model, +}; +use crate::data_model::oid::{ + OID_AD_CA_ISSUERS_RAW, OID_AD_CA_REPOSITORY, OID_AD_CA_REPOSITORY_RAW, OID_AD_RPKI_MANIFEST, + OID_AD_RPKI_MANIFEST_RAW, OID_AD_RPKI_NOTIFY, OID_AD_RPKI_NOTIFY_RAW, OID_AD_SIGNED_OBJECT, + OID_AD_SIGNED_OBJECT_RAW, OID_AUTHORITY_INFO_ACCESS, OID_AUTHORITY_INFO_ACCESS_RAW, + OID_AUTHORITY_KEY_IDENTIFIER, OID_AUTHORITY_KEY_IDENTIFIER_RAW, OID_AUTONOMOUS_SYS_IDS, + OID_AUTONOMOUS_SYS_IDS_RAW, OID_BASIC_CONSTRAINTS, OID_BASIC_CONSTRAINTS_RAW, + OID_CERTIFICATE_POLICIES, OID_CERTIFICATE_POLICIES_RAW, OID_CP_IPADDR_ASNUMBER, + OID_CP_IPADDR_ASNUMBER_RAW, OID_CRL_DISTRIBUTION_POINTS, OID_CRL_DISTRIBUTION_POINTS_RAW, + OID_EXTENDED_KEY_USAGE, OID_IP_ADDR_BLOCKS, OID_IP_ADDR_BLOCKS_RAW, OID_KEY_USAGE, OID_QT_CPS, + OID_SHA256_WITH_RSA_ENCRYPTION, OID_SHA256_WITH_RSA_ENCRYPTION_RAW, OID_SUBJECT_INFO_ACCESS, + OID_SUBJECT_INFO_ACCESS_RAW, OID_SUBJECT_KEY_IDENTIFIER, OID_SUBJECT_KEY_IDENTIFIER_RAW, +}; + +/// Resource Certificate kind (semantic classification). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ResourceCertKind { + Ca, + Ee, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ResourceCertificateRole { + TrustAnchor, + Ca, + SignedObjectEe, + RouterEe, +} + +/// A parsed RPKI Resource Certificate (RFC 6487) data model. +/// +/// This module intentionally focuses on the semantics needed by Signed Object validation and +/// object-specific EE certificate checks (MFT/ROA/ASPA), as described in +/// `rpki/specs/03_resource_certificate_rc.md`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ResourceCertificate { + pub raw_der: Vec, + pub tbs: RpkixTbsCertificate, + pub kind: ResourceCertKind, +} + +pub type ResourceCaCertificate = ResourceCertificate; +pub type ResourceEeCertificate = ResourceCertificate; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RpkixTbsCertificate { + pub version: u32, + pub serial_number: BigUint, + pub signature_algorithm: String, + pub issuer_name: X509NameDer, + pub subject_name: X509NameDer, + pub validity_not_before: UtcTime, + pub validity_not_after: UtcTime, + /// DER encoding of SubjectPublicKeyInfo. + pub subject_public_key_info: Vec, + pub extensions: RcExtensions, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RcExtensions { + pub basic_constraints_ca: bool, + pub basic_constraints: Option, + pub subject_key_identifier: Option>, + /// Authority Key Identifier (AKI) keyIdentifier value. + pub authority_key_identifier: Option>, + /// CRL Distribution Points URIs (fullName). + pub crl_distribution_points_uris: Option>, + /// Authority Information Access (AIA) caIssuers URIs. + pub ca_issuers_uris: Option>, + pub subject_info_access: Option, + pub certificate_policies_oid: Option, + pub certificate_policies: Option, + pub extension_oids: Vec, + + pub ip_resources: Option, + pub as_resources: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BasicConstraintsProfile { + pub ca: bool, + pub critical: bool, + pub path_len_constraint: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CertificatePoliciesProfile { + pub policy_oid: String, + pub qualifier_oids: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ResourceCertificateParsed { + pub raw_der: Vec, + pub version: X509Version, + pub serial_number: BigUint, + pub signature_algorithm: AlgorithmIdentifierValue, + pub tbs_signature_algorithm: AlgorithmIdentifierValue, + pub issuer_name: X509NameDer, + pub subject_name: X509NameDer, + pub validity_not_before: Asn1TimeUtc, + pub validity_not_after: Asn1TimeUtc, + /// DER encoding of SubjectPublicKeyInfo. + pub subject_public_key_info: Vec, + pub extensions: RcExtensionsParsed, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AlgorithmIdentifierValue { + pub oid: String, + pub parameters: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AlgorithmParametersValue { + pub class: Asn1Class, + pub tag: Asn1Tag, + pub data: Vec, +} + +impl AlgorithmIdentifierValue { + pub fn params_absent_or_null(&self) -> bool { + match &self.parameters { + None => true, + Some(p) if p.class == Asn1Class::Universal && p.tag == Asn1Tag::Null => true, + Some(_p) => false, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RcExtensionsParsed { + pub basic_constraints: Vec, + pub subject_key_identifier: Vec<(Vec, bool)>, + pub authority_key_identifier: Vec<(AuthorityKeyIdentifierParsed, bool)>, + pub crl_distribution_points: Vec<(CrlDistributionPointsParsed, bool)>, + pub authority_info_access: Vec<(AuthorityInfoAccessParsed, bool)>, + pub subject_info_access: Vec<(SubjectInfoAccessParsed, bool)>, + pub certificate_policies: Vec<(Vec, bool)>, + pub extension_oids: Vec, + pub ip_resources: Vec<(IpResourceSet, bool)>, + pub as_resources: Vec<(AsResourceSet, bool)>, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AuthorityKeyIdentifierParsed { + pub key_identifier: Option>, + pub has_authority_cert_issuer: bool, + pub has_authority_cert_serial: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AuthorityInfoAccessParsed { + pub ca_issuers_uris: Vec, + pub ca_issuers_access_location_not_uri: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CrlDistributionPointsParsed { + pub distribution_points: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CrlDistributionPointParsed { + pub distribution_point_present: bool, + pub reasons_present: bool, + pub crl_issuer_present: bool, + pub name_relative_to_crl_issuer_present: bool, + pub full_name_uris: Vec, + pub full_name_not_uri: bool, + pub full_name_present: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SubjectInfoAccessParsed { + pub access_descriptions: Vec, + pub signed_object_uris: Vec, + pub signed_object_access_location_not_uri: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SubjectInfoAccess { + Ca(SubjectInfoAccessCa), + Ee(SubjectInfoAccessEe), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SubjectInfoAccessCa { + pub access_descriptions: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SubjectInfoAccessEe { + pub signed_object_uris: Vec, + /// The full list of access descriptions as carried in the SIA extension. + pub access_descriptions: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AccessDescription { + pub access_method_oid: String, + pub access_location: String, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +pub enum Afi { + Ipv4, + Ipv6, +} + +impl Afi { + pub fn ub(self) -> u16 { + match self { + Afi::Ipv4 => 32, + Afi::Ipv6 => 128, + } + } + + pub fn octets_len(self) -> usize { + match self { + Afi::Ipv4 => 4, + Afi::Ipv6 => 16, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct IpResourceSet { + pub families: Vec, +} + +impl IpResourceSet { + /// Decode the DER bytes carried inside the X.509 `extnValue` OCTET STRING for + /// `id-pe-ipAddrBlocks` (RFC 3779 / RFC 6487). + pub fn decode_extn_value(extn_value: &[u8]) -> Result { + parse_ip_addr_blocks(extn_value).map_err(|_| IpResourceSetDecodeError::InvalidEncoding) + } + + pub fn is_all_inherit(&self) -> bool { + self.families + .iter() + .all(|f| matches!(f.choice, IpAddressChoice::Inherit)) + } + + pub fn has_any_inherit(&self) -> bool { + self.families + .iter() + .any(|f| matches!(f.choice, IpAddressChoice::Inherit)) + } + + pub fn contains_prefix(&self, prefix: &IpPrefix) -> bool { + self.families.iter().any(|fam| fam.contains_prefix(prefix)) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum IpResourceSetDecodeError { + #[error("invalid ipAddrBlocks encoding (RFC 3779 §2.2.3; RFC 6487 §4.8.10)")] + InvalidEncoding, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct IpAddressFamily { + pub afi: Afi, + pub choice: IpAddressChoice, +} + +impl IpAddressFamily { + pub fn contains_prefix(&self, prefix: &IpPrefix) -> bool { + if self.afi != prefix.afi { + return false; + } + match &self.choice { + IpAddressChoice::Inherit => true, + IpAddressChoice::AddressesOrRanges(items) => items.iter().any(|item| match item { + IpAddressOrRange::Prefix(p) => prefix_covers(p, prefix), + IpAddressOrRange::Range(r) => range_covers_prefix(self.afi, r, prefix), + }), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum IpAddressChoice { + Inherit, + AddressesOrRanges(Vec), +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum IpAddressOrRange { + Prefix(IpPrefix), + Range(IpAddressRange), +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct IpAddressRange { + pub min: Vec, + pub max: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub struct IpPrefix { + pub afi: Afi, + pub prefix_len: u16, + pub addr: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct AsResourceSet { + pub asnum: Option, + pub rdi: Option, +} + +impl AsResourceSet { + /// Decode the DER bytes carried inside the X.509 `extnValue` OCTET STRING for + /// `id-pe-autonomousSysIds` (RFC 3779 / RFC 6487). + pub fn decode_extn_value(extn_value: &[u8]) -> Result { + parse_as_identifiers(extn_value).map_err(|_| AsResourceSetDecodeError::InvalidEncoding) + } + + pub fn is_asnum_inherit(&self) -> bool { + matches!(self.asnum, Some(AsIdentifierChoice::Inherit)) + } + + pub fn has_any_range(&self) -> bool { + self.asnum.as_ref().map(|c| c.has_range()).unwrap_or(false) + || self.rdi.as_ref().map(|c| c.has_range()).unwrap_or(false) + } + + pub fn asnum_single_id(&self) -> Option { + match self.asnum.as_ref()? { + AsIdentifierChoice::Inherit => None, + AsIdentifierChoice::AsIdsOrRanges(items) => { + if items.len() != 1 { + return None; + } + match &items[0] { + AsIdOrRange::Id(v) => Some(*v), + AsIdOrRange::Range { .. } => None, + } + } + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum AsResourceSetDecodeError { + #[error("invalid autonomousSysIds encoding (RFC 3779 §3.2.3; RFC 6487 §4.8.11)")] + InvalidEncoding, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum AsIdentifierChoice { + Inherit, + AsIdsOrRanges(Vec), +} + +impl AsIdentifierChoice { + pub fn has_range(&self) -> bool { + match self { + AsIdentifierChoice::Inherit => false, + AsIdentifierChoice::AsIdsOrRanges(items) => { + items.iter().any(|i| matches!(i, AsIdOrRange::Range { .. })) + } + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum AsIdOrRange { + Id(u32), + Range { min: u32, max: u32 }, +} + +#[derive(Debug, thiserror::Error)] +pub enum ResourceCertificateParseError { + #[error("X.509 parse error: {0} (RFC 5280 §4.1; RFC 6487 §4)")] + Parse(String), + + #[error("trailing bytes after certificate DER: {0} bytes (DER; RFC 5280 §4.1)")] + TrailingBytes(usize), + + #[error("invalid RFC 3779 IP resources extension encoding (RFC 6487 §4.8.10; RFC 3779 §2.2)")] + InvalidIpResourcesEncoding, + + #[error("invalid RFC 3779 AS resources extension encoding (RFC 6487 §4.8.11; RFC 3779 §3.2)")] + InvalidAsResourcesEncoding, +} + +#[derive(Debug, thiserror::Error)] +pub enum ResourceCertificateProfileError { + #[error("{0}")] + InvalidTimeEncoding(#[from] InvalidTimeEncodingError), + + #[error("certificate version must be v3 (RFC 5280 §4.1; RFC 6487 §4)")] + InvalidVersion, + + #[error("signatureAlgorithm does not match tbsCertificate.signature (RFC 5280 §4.1)")] + SignatureAlgorithmMismatch, + + #[error( + "unsupported signature algorithm (expected sha256WithRSAEncryption {OID_SHA256_WITH_RSA_ENCRYPTION}) (RFC 7935 §2; RFC 6487 §4)" + )] + UnsupportedSignatureAlgorithm, + + #[error("invalid signature algorithm parameters (RFC 5280 §4.1.1.2)")] + InvalidSignatureAlgorithmParameters, + + #[error( + "{role} Name strict validation failed: {detail} (RFC 6487 §4.4; RFC 5280 §4.1.2.4/§4.1.2.6)" + )] + StrictName { role: &'static str, detail: String }, + + #[error("duplicate extension: {0} (RFC 5280 §4.2; RFC 6487 §4.8)")] + DuplicateExtension(&'static str), + + #[error("SubjectKeyIdentifier criticality must be non-critical (RFC 6487 §4.8.2)")] + SkiCriticality, + + #[error("SubjectInfoAccess criticality must be non-critical (RFC 6487 §4.8.8)")] + SiaCriticality, + + #[error("certificatePolicies criticality must be critical (RFC 6487 §4.8.9)")] + CertificatePoliciesCriticality, + + #[error("certificatePolicies must be present (RFC 6487 §4.8.9)")] + CertificatePoliciesMissing, + + #[error( + "certificatePolicies must contain RPKI policy OID {OID_CP_IPADDR_ASNUMBER}, got {0} (RFC 6487 §4.8.9)" + )] + InvalidCertificatePolicy(String), + + #[error("certificatePolicies may contain at most one CPS qualifier (RFC 6487 §4.8.9)")] + CertificatePoliciesTooManyQualifiers, + + #[error( + "certificatePolicies qualifier must be id-qt-cps ({OID_QT_CPS}), got {0} (RFC 6487 §4.8.9)" + )] + CertificatePoliciesInvalidQualifier(String), + + #[error("basicConstraints must be present in CA certificates (RFC 6487 §4.8.1)")] + BasicConstraintsMissing, + + #[error("basicConstraints criticality must be critical in CA certificates (RFC 6487 §4.8.1)")] + BasicConstraintsCriticality, + + #[error("basicConstraints cA must be TRUE in CA certificates (RFC 6487 §4.8.1)")] + BasicConstraintsCaFalse, + + #[error( + "basicConstraints pathLenConstraint must be absent in CA certificates (RFC 6487 §4.8.1)" + )] + BasicConstraintsPathLenPresent, + + #[error("basicConstraints must be absent in EE certificates (RFC 6487 §4.8.1)")] + BasicConstraintsEeMustOmit, + + #[error("extension {oid} is not permitted for {role} resource certificates (RFC 6487 §4.8)")] + DisallowedExtension { role: &'static str, oid: String }, + + #[error("autonomousSysIds RDI field must be absent (RFC 6487 §4.8.11; RFC 3779 §3.2.3)")] + AsResourcesRdiPresent, + + #[error( + "SIA id-ad-signedObject accessLocation must be URI (RFC 6487 §4.8.8.2; RFC 5280 §4.2.2.2)" + )] + SignedObjectSiaNotUri, + + #[error("SIA id-ad-signedObject must include at least one rsync:// URI (RFC 6487 §4.8.8.2)")] + SignedObjectSiaNoRsync, + + #[error("ipAddrBlocks criticality must be critical when present (RFC 6487 §4.8.10)")] + IpResourcesCriticality, + + #[error("autonomousSysIds criticality must be critical when present (RFC 6487 §4.8.11)")] + AsResourcesCriticality, + + #[error( + "authorityKeyIdentifier must be present in non-self-signed certificates (RFC 6487 §4.8.3; RFC 5280 §4.2.1.1)" + )] + AkiMissing, + + #[error( + "authorityKeyIdentifier criticality must be non-critical (RFC 6487 §4.8.3; RFC 5280 §4.2.1.1)" + )] + AkiCriticality, + + #[error( + "authorityKeyIdentifier authorityCertIssuer MUST NOT be present (RFC 6487 §4.8.3; RFC 5280 §4.2.1.1)" + )] + AkiAuthorityCertIssuerPresent, + + #[error( + "authorityKeyIdentifier authorityCertSerialNumber MUST NOT be present (RFC 6487 §4.8.3; RFC 5280 §4.2.1.1)" + )] + AkiAuthorityCertSerialPresent, + + #[error( + "self-signed certificate authorityKeyIdentifier must equal subjectKeyIdentifier when present (RFC 6487 §4.8.3)" + )] + AkiSelfSignedNotEqualSki, + + #[error( + "CRLDistributionPoints must be present in non-self-signed certificates (RFC 6487 §4.8.6; RFC 5280 §4.2.1.13)" + )] + CrlDistributionPointsMissing, + + #[error( + "CRLDistributionPoints criticality must be non-critical (RFC 6487 §4.8.6; RFC 5280 §4.2.1.13)" + )] + CrlDistributionPointsCriticality, + + #[error("CRLDistributionPoints MUST be omitted in self-signed certificates (RFC 6487 §4.8.6)")] + CrlDistributionPointsSelfSignedMustOmit, + + #[error("CRLDistributionPoints must contain exactly one DistributionPoint (RFC 6487 §4.8.6)")] + CrlDistributionPointsNotSingle, + + #[error("CRLDistributionPoints distributionPoint field MUST be present (RFC 6487 §4.8.6)")] + CrlDistributionPointsNoDistributionPoint, + + #[error("CRLDistributionPoints reasons field MUST be omitted (RFC 6487 §4.8.6)")] + CrlDistributionPointsHasReasons, + + #[error("CRLDistributionPoints cRLIssuer field MUST be omitted (RFC 6487 §4.8.6)")] + CrlDistributionPointsHasCrlIssuer, + + #[error( + "CRLDistributionPoints distributionPoint MUST contain fullName and MUST NOT contain nameRelativeToCRLIssuer (RFC 6487 §4.8.6)" + )] + CrlDistributionPointsInvalidName, + + #[error( + "CRLDistributionPoints fullName must contain only URI GeneralNames (RFC 6487 §4.8.6; RFC 5280 §4.2.1.6)" + )] + CrlDistributionPointsFullNameNotUri, + + #[error("CRLDistributionPoints must include at least one rsync:// URI (RFC 6487 §4.8.6)")] + CrlDistributionPointsNoRsync, + + #[error( + "authorityInfoAccess must be present in non-self-signed certificates (RFC 6487 §4.8.7; RFC 5280 §4.2.2.1)" + )] + AuthorityInfoAccessMissing, + + #[error( + "authorityInfoAccess criticality must be non-critical (RFC 6487 §4.8.7; RFC 5280 §4.2.2.1)" + )] + AuthorityInfoAccessCriticality, + + #[error("authorityInfoAccess MUST be omitted in self-signed certificates (RFC 6487 §4.8.7)")] + AuthorityInfoAccessSelfSignedMustOmit, + + #[error( + "authorityInfoAccess id-ad-caIssuers accessLocation must be URI (RFC 6487 §4.8.7; RFC 5280 §4.2.2.1)" + )] + AuthorityInfoAccessCaIssuersNotUri, + + #[error("authorityInfoAccess must include at least one id-ad-caIssuers URI (RFC 6487 §4.8.7)")] + AuthorityInfoAccessMissingCaIssuers, + + #[error("authorityInfoAccess must include at least one rsync:// URI (RFC 6487 §4.8.7)")] + AuthorityInfoAccessNoRsync, +} + +#[derive(Debug, thiserror::Error)] +pub enum ResourceCertificateDecodeError { + #[error("{0}")] + Parse(#[from] ResourceCertificateParseError), + + #[error("{0}")] + Validate(#[from] ResourceCertificateProfileError), +} + +pub type ResourceCertificateError = ResourceCertificateDecodeError; + +impl ResourceCertificate { + /// Parse step of scheme A (`parse → validate → verify`). + pub fn parse_der( + der: &[u8], + ) -> Result { + let (rem, cert) = X509Certificate::from_der(der) + .map_err(|e| ResourceCertificateParseError::Parse(e.to_string()))?; + if !rem.is_empty() { + return Err(ResourceCertificateParseError::TrailingBytes(rem.len())); + } + + let validity_not_before = asn1_time_to_model(cert.validity().not_before); + let validity_not_after = asn1_time_to_model(cert.validity().not_after); + + let subject_public_key_info = cert.tbs_certificate.subject_pki.raw.to_vec(); + + let signature_algorithm = algorithm_identifier_value(&cert.signature_algorithm); + let tbs_signature_algorithm = algorithm_identifier_value(&cert.tbs_certificate.signature); + let extensions = parse_extensions_parse(cert.extensions())?; + + Ok(ResourceCertificateParsed { + raw_der: der.to_vec(), + version: cert.version(), + serial_number: cert.tbs_certificate.serial.clone(), + signature_algorithm, + tbs_signature_algorithm, + issuer_name: X509NameDer(cert.issuer().as_raw().to_vec()), + subject_name: X509NameDer(cert.subject().as_raw().to_vec()), + validity_not_before, + validity_not_after, + subject_public_key_info, + extensions, + }) + } + + /// Profile validate step of scheme A (`parse → validate → verify`). + /// + /// `ResourceCertificate` is already profile-validated when constructed via `decode_der()` / + /// `ResourceCertificateParsed::validate_profile()`. + pub fn validate_profile(&self) -> Result<(), ResourceCertificateProfileError> { + Ok(()) + } + + pub fn validate_rfc6487_profile( + &self, + role: ResourceCertificateRole, + ) -> Result<(), ResourceCertificateProfileError> { + let role_name = match role { + ResourceCertificateRole::TrustAnchor => "trust anchor CA", + ResourceCertificateRole::Ca => "CA", + ResourceCertificateRole::SignedObjectEe => "signed-object EE", + ResourceCertificateRole::RouterEe => "router EE", + }; + let ca_role = matches!( + role, + ResourceCertificateRole::TrustAnchor | ResourceCertificateRole::Ca + ); + + if ca_role { + let constraints = self + .tbs + .extensions + .basic_constraints + .as_ref() + .ok_or(ResourceCertificateProfileError::BasicConstraintsMissing)?; + if !constraints.critical { + return Err(ResourceCertificateProfileError::BasicConstraintsCriticality); + } + if !constraints.ca { + return Err(ResourceCertificateProfileError::BasicConstraintsCaFalse); + } + if constraints.path_len_constraint.is_some() { + return Err(ResourceCertificateProfileError::BasicConstraintsPathLenPresent); + } + } else if self.tbs.extensions.basic_constraints.is_some() { + return Err(ResourceCertificateProfileError::BasicConstraintsEeMustOmit); + } + + for oid in &self.tbs.extensions.extension_oids { + if !is_permitted_extension(oid, role) { + return Err(ResourceCertificateProfileError::DisallowedExtension { + role: role_name, + oid: oid.clone(), + }); + } + } + + let policies = self + .tbs + .extensions + .certificate_policies + .as_ref() + .ok_or(ResourceCertificateProfileError::CertificatePoliciesMissing)?; + if policies.policy_oid != OID_CP_IPADDR_ASNUMBER { + return Err(ResourceCertificateProfileError::InvalidCertificatePolicy( + policies.policy_oid.clone(), + )); + } + if policies.qualifier_oids.len() > 1 { + return Err(ResourceCertificateProfileError::CertificatePoliciesTooManyQualifiers); + } + if let Some(qualifier_oid) = policies.qualifier_oids.first() { + if qualifier_oid != OID_QT_CPS { + return Err( + ResourceCertificateProfileError::CertificatePoliciesInvalidQualifier( + qualifier_oid.clone(), + ), + ); + } + } + + if self + .tbs + .extensions + .as_resources + .as_ref() + .is_some_and(|resources| resources.rdi.is_some()) + { + return Err(ResourceCertificateProfileError::AsResourcesRdiPresent); + } + + Ok(()) + } + + pub fn validate_strict_name_profile(&self) -> Result<(), ResourceCertificateProfileError> { + validate_strict_rpki_name(&self.tbs.issuer_name, "issuer")?; + validate_strict_rpki_name(&self.tbs.subject_name, "subject")?; + Ok(()) + } + + /// Decode a resource certificate (`parse + validate`). + pub fn decode_der(der: &[u8]) -> Result { + Ok(Self::parse_der(der)?.validate_profile()?) + } + + pub fn decode_der_with_strict_name(der: &[u8]) -> Result { + let cert = Self::decode_der(der)?; + cert.validate_strict_name_profile()?; + Ok(cert) + } + + /// Backwards-compatible helper (historical name). + pub fn from_der(der: &[u8]) -> Result { + Self::decode_der(der) + } +} + +fn is_permitted_extension(oid: &str, role: ResourceCertificateRole) -> bool { + matches!( + oid, + OID_BASIC_CONSTRAINTS + | OID_KEY_USAGE + | OID_SUBJECT_KEY_IDENTIFIER + | OID_AUTHORITY_KEY_IDENTIFIER + | OID_CRL_DISTRIBUTION_POINTS + | OID_AUTHORITY_INFO_ACCESS + | OID_SUBJECT_INFO_ACCESS + | OID_CERTIFICATE_POLICIES + | OID_IP_ADDR_BLOCKS + | OID_AUTONOMOUS_SYS_IDS + ) || (role == ResourceCertificateRole::RouterEe && oid == OID_EXTENDED_KEY_USAGE) +} + +fn validate_strict_rpki_name( + name: &X509NameDer, + role: &'static str, +) -> Result<(), ResourceCertificateProfileError> { + let mut name_seq = DerReader::new(name.as_raw()) + .take_sequence() + .map_err(|e| ResourceCertificateProfileError::StrictName { role, detail: e })?; + + let mut common_name_count = 0usize; + let mut serial_number_count = 0usize; + + while !name_seq.is_empty() { + let set_bytes = name_seq + .take_tag(0x31) + .map_err(|e| ResourceCertificateProfileError::StrictName { role, detail: e })?; + let mut rdn_set = DerReader::new(set_bytes); + if rdn_set.is_empty() { + return Err(ResourceCertificateProfileError::StrictName { + role, + detail: "RelativeDistinguishedName SET is empty".to_string(), + }); + } + + while !rdn_set.is_empty() { + let mut attr = rdn_set + .take_sequence() + .map_err(|e| ResourceCertificateProfileError::StrictName { role, detail: e })?; + let oid = attr + .take_tag(0x06) + .map_err(|e| ResourceCertificateProfileError::StrictName { role, detail: e })?; + let (value_tag, _value) = attr + .take_any() + .map_err(|e| ResourceCertificateProfileError::StrictName { role, detail: e })?; + if !attr.is_empty() { + return Err(ResourceCertificateProfileError::StrictName { + role, + detail: "AttributeTypeAndValue must be SEQUENCE of 2".to_string(), + }); + } + + match oid { + // 2.5.4.3 commonName + &[0x55, 0x04, 0x03] => { + common_name_count += 1; + if value_tag != 0x13 { + return Err(ResourceCertificateProfileError::StrictName { + role, + detail: format!( + "commonName must be PrintableString, got tag 0x{value_tag:02X}" + ), + }); + } + } + // 2.5.4.5 serialNumber + &[0x55, 0x04, 0x05] => { + serial_number_count += 1; + if value_tag != 0x13 { + return Err(ResourceCertificateProfileError::StrictName { + role, + detail: format!( + "serialNumber must be PrintableString, got tag 0x{value_tag:02X}" + ), + }); + } + } + _ => {} + } + } + } + + if common_name_count != 1 { + return Err(ResourceCertificateProfileError::StrictName { + role, + detail: format!("commonName must appear exactly once, got {common_name_count}"), + }); + } + if serial_number_count > 1 { + return Err(ResourceCertificateProfileError::StrictName { + role, + detail: format!("serialNumber must appear at most once, got {serial_number_count}"), + }); + } + Ok(()) +} + +#[cfg(test)] +mod strict_name_tests { + use super::*; + + fn name_with_attrs(attrs: &[(&[u8], u8, &[u8])]) -> X509NameDer { + let mut rdns = Vec::new(); + for (oid, tag, value) in attrs { + let mut attr = Vec::new(); + attr.extend(der_tlv(0x06, oid)); + attr.extend(der_tlv(*tag, value)); + let attr = der_tlv(0x30, &attr); + let rdn = der_tlv(0x31, &attr); + rdns.extend(rdn); + } + X509NameDer(der_tlv(0x30, &rdns)) + } + + fn der_tlv(tag: u8, value: &[u8]) -> Vec { + let mut out = vec![tag]; + encode_len(value.len(), &mut out); + out.extend_from_slice(value); + out + } + + fn encode_len(len: usize, out: &mut Vec) { + if len < 0x80 { + out.push(len as u8); + return; + } + let mut bytes = Vec::new(); + let mut value = len; + while value > 0 { + bytes.push((value & 0xFF) as u8); + value >>= 8; + } + bytes.reverse(); + out.push(0x80 | bytes.len() as u8); + out.extend(bytes); + } + + #[test] + fn strict_name_accepts_printable_common_name_and_serial_number() { + let name = name_with_attrs(&[ + (&[0x55, 0x04, 0x03], 0x13, b"CN1"), + (&[0x55, 0x04, 0x05], 0x13, b"SN1"), + ]); + validate_strict_rpki_name(&name, "subject").expect("strict name"); + } + + #[test] + fn strict_name_rejects_utf8_common_name() { + let name = name_with_attrs(&[(&[0x55, 0x04, 0x03], 0x0C, b"CN1")]); + let err = validate_strict_rpki_name(&name, "subject").expect_err("strict name fails"); + assert!(err.to_string().contains("PrintableString"), "{err}"); + } + + #[test] + fn strict_name_rejects_duplicate_common_name() { + let name = name_with_attrs(&[ + (&[0x55, 0x04, 0x03], 0x13, b"CN1"), + (&[0x55, 0x04, 0x03], 0x13, b"CN2"), + ]); + let err = validate_strict_rpki_name(&name, "subject").expect_err("strict name fails"); + assert!(err.to_string().contains("exactly once"), "{err}"); + } + + #[test] + fn profile_rejects_rfc8360_v2_policy_oid() { + let extensions = RcExtensionsParsed { + basic_constraints: vec![BasicConstraintsProfile { + ca: true, + critical: true, + path_len_constraint: None, + }], + subject_key_identifier: Vec::new(), + authority_key_identifier: Vec::new(), + crl_distribution_points: Vec::new(), + authority_info_access: Vec::new(), + subject_info_access: Vec::new(), + certificate_policies: vec![( + vec![CertificatePoliciesProfile { + policy_oid: "1.3.6.1.5.5.7.14.3".to_string(), + qualifier_oids: Vec::new(), + }], + true, + )], + extension_oids: Vec::new(), + ip_resources: Vec::new(), + as_resources: Vec::new(), + }; + + let err = extensions + .validate_profile(true) + .expect_err("v2 policy OID must remain invalid"); + assert!( + matches!(&err, ResourceCertificateProfileError::InvalidCertificatePolicy(oid) if oid == "1.3.6.1.5.5.7.14.3"), + "{err}" + ); + } +} + +impl ResourceCertificateParsed { + pub fn validate_profile(self) -> Result { + let version = match self.version { + X509Version::V3 => 2u32, + _ => return Err(ResourceCertificateProfileError::InvalidVersion), + }; + + self.validity_not_before + .validate_encoding_rfc5280("notBefore")?; + self.validity_not_after + .validate_encoding_rfc5280("notAfter")?; + + if self.signature_algorithm != self.tbs_signature_algorithm { + return Err(ResourceCertificateProfileError::SignatureAlgorithmMismatch); + } + if self.signature_algorithm.oid != OID_SHA256_WITH_RSA_ENCRYPTION { + return Err(ResourceCertificateProfileError::UnsupportedSignatureAlgorithm); + } + if !self.signature_algorithm.params_absent_or_null() { + return Err(ResourceCertificateProfileError::InvalidSignatureAlgorithmParameters); + } + + let is_self_signed = self.issuer_name == self.subject_name; + let extensions = self.extensions.validate_profile(is_self_signed)?; + let kind = if extensions.basic_constraints_ca { + ResourceCertKind::Ca + } else { + ResourceCertKind::Ee + }; + + Ok(ResourceCertificate { + raw_der: self.raw_der, + tbs: RpkixTbsCertificate { + version, + serial_number: self.serial_number, + signature_algorithm: self.signature_algorithm.oid, + issuer_name: self.issuer_name, + subject_name: self.subject_name, + validity_not_before: self.validity_not_before.utc, + validity_not_after: self.validity_not_after.utc, + subject_public_key_info: self.subject_public_key_info, + extensions, + }, + kind, + }) + } +} + +impl RcExtensionsParsed { + pub fn validate_profile( + self, + is_self_signed: bool, + ) -> Result { + // NOTE(perf): `self` is consumed. Prefer moving decoded fields out rather than cloning, + // especially for large resource sets and URI lists. + let RcExtensionsParsed { + basic_constraints, + subject_key_identifier, + authority_key_identifier, + crl_distribution_points, + authority_info_access, + subject_info_access, + certificate_policies, + extension_oids, + ip_resources, + as_resources, + } = self; + + if basic_constraints.len() > 1 { + return Err(ResourceCertificateProfileError::DuplicateExtension( + "basicConstraints", + )); + } + let basic_constraints = basic_constraints.into_iter().next(); + let basic_constraints_ca = basic_constraints.as_ref().is_some_and(|bc| bc.ca); + + let subject_key_identifier = match subject_key_identifier.len() { + 0 => None, + 1 => { + let (ski, critical) = subject_key_identifier.into_iter().next().expect("len==1"); + if critical { + return Err(ResourceCertificateProfileError::SkiCriticality); + } + Some(ski) + } + _ => { + return Err(ResourceCertificateProfileError::DuplicateExtension( + "subjectKeyIdentifier", + )); + } + }; + + let authority_key_identifier = match authority_key_identifier.len() { + 0 => { + if is_self_signed { + None + } else { + return Err(ResourceCertificateProfileError::AkiMissing); + } + } + 1 => { + let (aki, critical) = authority_key_identifier.into_iter().next().expect("len==1"); + if critical { + return Err(ResourceCertificateProfileError::AkiCriticality); + } + if aki.has_authority_cert_issuer { + return Err(ResourceCertificateProfileError::AkiAuthorityCertIssuerPresent); + } + if aki.has_authority_cert_serial { + return Err(ResourceCertificateProfileError::AkiAuthorityCertSerialPresent); + } + let keyid = aki.key_identifier; + if is_self_signed { + if let (Some(keyid), Some(ski)) = + (keyid.as_ref(), subject_key_identifier.as_ref()) + { + if keyid != ski { + return Err(ResourceCertificateProfileError::AkiSelfSignedNotEqualSki); + } + } + } else if keyid.is_none() { + return Err(ResourceCertificateProfileError::AkiMissing); + } + keyid + } + _ => { + return Err(ResourceCertificateProfileError::DuplicateExtension( + "authorityKeyIdentifier", + )); + } + }; + + let crl_distribution_points_uris = match crl_distribution_points.len() { + 0 => { + if is_self_signed { + None + } else { + return Err(ResourceCertificateProfileError::CrlDistributionPointsMissing); + } + } + 1 => { + let (crldp, critical) = crl_distribution_points.into_iter().next().expect("len==1"); + if critical { + return Err(ResourceCertificateProfileError::CrlDistributionPointsCriticality); + } + if is_self_signed { + return Err( + ResourceCertificateProfileError::CrlDistributionPointsSelfSignedMustOmit, + ); + } + if crldp.distribution_points.len() != 1 { + return Err(ResourceCertificateProfileError::CrlDistributionPointsNotSingle); + } + let dp = crldp + .distribution_points + .into_iter() + .next() + .expect("len==1"); + if dp.reasons_present { + return Err(ResourceCertificateProfileError::CrlDistributionPointsHasReasons); + } + if dp.crl_issuer_present { + return Err(ResourceCertificateProfileError::CrlDistributionPointsHasCrlIssuer); + } + if !dp.distribution_point_present { + return Err( + ResourceCertificateProfileError::CrlDistributionPointsNoDistributionPoint, + ); + } + if dp.name_relative_to_crl_issuer_present || !dp.full_name_present { + return Err(ResourceCertificateProfileError::CrlDistributionPointsInvalidName); + } + if dp.full_name_not_uri { + return Err( + ResourceCertificateProfileError::CrlDistributionPointsFullNameNotUri, + ); + } + if !dp.full_name_uris.iter().any(|u| u.starts_with("rsync://")) { + return Err(ResourceCertificateProfileError::CrlDistributionPointsNoRsync); + } + Some(dp.full_name_uris) + } + _ => { + return Err(ResourceCertificateProfileError::DuplicateExtension( + "cRLDistributionPoints", + )); + } + }; + + let ca_issuers_uris = match authority_info_access.len() { + 0 => { + if is_self_signed { + None + } else { + return Err(ResourceCertificateProfileError::AuthorityInfoAccessMissing); + } + } + 1 => { + let (aia, critical) = authority_info_access.into_iter().next().expect("len==1"); + if critical { + return Err(ResourceCertificateProfileError::AuthorityInfoAccessCriticality); + } + if is_self_signed { + return Err( + ResourceCertificateProfileError::AuthorityInfoAccessSelfSignedMustOmit, + ); + } + if aia.ca_issuers_access_location_not_uri { + return Err( + ResourceCertificateProfileError::AuthorityInfoAccessCaIssuersNotUri, + ); + } + if aia.ca_issuers_uris.is_empty() { + return Err( + ResourceCertificateProfileError::AuthorityInfoAccessMissingCaIssuers, + ); + } + if !aia + .ca_issuers_uris + .iter() + .any(|u| u.starts_with("rsync://")) + { + return Err(ResourceCertificateProfileError::AuthorityInfoAccessNoRsync); + } + Some(aia.ca_issuers_uris) + } + _ => { + return Err(ResourceCertificateProfileError::DuplicateExtension( + "authorityInfoAccess", + )); + } + }; + + let subject_info_access = match subject_info_access.len() { + 0 => None, + 1 => { + let (sia, critical) = subject_info_access.into_iter().next().expect("len==1"); + if critical { + return Err(ResourceCertificateProfileError::SiaCriticality); + } + if sia.signed_object_access_location_not_uri { + return Err(ResourceCertificateProfileError::SignedObjectSiaNotUri); + } + if !sia.signed_object_uris.is_empty() + && !sia + .signed_object_uris + .iter() + .any(|u| u.starts_with("rsync://")) + { + return Err(ResourceCertificateProfileError::SignedObjectSiaNoRsync); + } + if sia.signed_object_uris.is_empty() { + Some(SubjectInfoAccess::Ca(SubjectInfoAccessCa { + access_descriptions: sia.access_descriptions, + })) + } else { + Some(SubjectInfoAccess::Ee(SubjectInfoAccessEe { + signed_object_uris: sia.signed_object_uris, + access_descriptions: sia.access_descriptions, + })) + } + } + _ => { + return Err(ResourceCertificateProfileError::DuplicateExtension( + "subjectInfoAccess", + )); + } + }; + + let certificate_policies = match certificate_policies.len() { + 0 => None, + 1 => { + let (policies, critical) = certificate_policies.into_iter().next().expect("len==1"); + if !critical { + return Err(ResourceCertificateProfileError::CertificatePoliciesCriticality); + } + if policies.len() != 1 { + return Err(ResourceCertificateProfileError::InvalidCertificatePolicy( + "expected exactly one policy".into(), + )); + } + let policy = policies.into_iter().next().expect("len==1"); + if policy.policy_oid != OID_CP_IPADDR_ASNUMBER { + return Err(ResourceCertificateProfileError::InvalidCertificatePolicy( + policy.policy_oid, + )); + } + Some(policy) + } + _ => { + return Err(ResourceCertificateProfileError::DuplicateExtension( + "certificatePolicies", + )); + } + }; + + let ip_resources = match ip_resources.len() { + 0 => None, + 1 => { + let (ip, critical) = ip_resources.into_iter().next().expect("len==1"); + if !critical { + return Err(ResourceCertificateProfileError::IpResourcesCriticality); + } + Some(ip) + } + _ => { + return Err(ResourceCertificateProfileError::DuplicateExtension( + "ipAddrBlocks", + )); + } + }; + + let as_resources = match as_resources.len() { + 0 => None, + 1 => { + let (asn, critical) = as_resources.into_iter().next().expect("len==1"); + if !critical { + return Err(ResourceCertificateProfileError::AsResourcesCriticality); + } + Some(asn) + } + _ => { + return Err(ResourceCertificateProfileError::DuplicateExtension( + "autonomousSysIds", + )); + } + }; + + Ok(RcExtensions { + basic_constraints_ca, + basic_constraints, + subject_key_identifier, + authority_key_identifier, + crl_distribution_points_uris, + ca_issuers_uris, + subject_info_access, + certificate_policies_oid: certificate_policies + .as_ref() + .map(|_| OID_CP_IPADDR_ASNUMBER.to_string()), + certificate_policies, + extension_oids, + ip_resources, + as_resources, + }) + } +} + +fn algorithm_identifier_value( + ai: &x509_parser::x509::AlgorithmIdentifier<'_>, +) -> AlgorithmIdentifierValue { + let parameters = ai.parameters.as_ref().map(|p| AlgorithmParametersValue { + class: p.class(), + tag: p.tag(), + data: p.as_bytes().to_vec(), + }); + // NOTE(perf): Avoid `to_id_string()` allocations for the algorithms we expect + // in RPKI resource certificates. Fall back to `to_id_string()` for unexpected + // algorithms (mostly error paths). + let oid = if ai.algorithm.as_bytes() == OID_SHA256_WITH_RSA_ENCRYPTION_RAW { + OID_SHA256_WITH_RSA_ENCRYPTION.to_string() + } else { + ai.algorithm.to_id_string() + }; + AlgorithmIdentifierValue { oid, parameters } +} + +fn parse_extensions_parse( + exts: &[X509Extension<'_>], +) -> Result { + let mut basic_constraints: Vec = Vec::new(); + let mut ski: Vec<(Vec, bool)> = Vec::new(); + let mut aki: Vec<(AuthorityKeyIdentifierParsed, bool)> = Vec::new(); + let mut crldp: Vec<(CrlDistributionPointsParsed, bool)> = Vec::new(); + let mut aia: Vec<(AuthorityInfoAccessParsed, bool)> = Vec::new(); + let mut sia: Vec<(SubjectInfoAccessParsed, bool)> = Vec::new(); + let mut cert_policies: Vec<(Vec, bool)> = Vec::new(); + let mut extension_oids: Vec = Vec::with_capacity(exts.len()); + + let mut ip_resources: Vec<(IpResourceSet, bool)> = Vec::new(); + let mut as_resources: Vec<(AsResourceSet, bool)> = Vec::new(); + + for ext in exts { + let oid = ext.oid.as_bytes(); + extension_oids.push(ext.oid.to_id_string()); + if oid == OID_BASIC_CONSTRAINTS_RAW { + let ParsedExtension::BasicConstraints(bc) = ext.parsed_extension() else { + return Err(ResourceCertificateParseError::Parse( + "basicConstraints parse failed".into(), + )); + }; + basic_constraints.push(BasicConstraintsProfile { + ca: bc.ca, + critical: ext.critical, + path_len_constraint: bc.path_len_constraint, + }); + } else if oid == OID_SUBJECT_KEY_IDENTIFIER_RAW { + let ParsedExtension::SubjectKeyIdentifier(s) = ext.parsed_extension() else { + return Err(ResourceCertificateParseError::Parse( + "subjectKeyIdentifier parse failed".into(), + )); + }; + ski.push((s.0.to_vec(), ext.critical)); + } else if oid == OID_AUTHORITY_KEY_IDENTIFIER_RAW { + let ParsedExtension::AuthorityKeyIdentifier(a) = ext.parsed_extension() else { + return Err(ResourceCertificateParseError::Parse( + "authorityKeyIdentifier parse failed".into(), + )); + }; + aki.push(( + AuthorityKeyIdentifierParsed { + key_identifier: a.key_identifier.as_ref().map(|k| k.0.to_vec()), + has_authority_cert_issuer: a.authority_cert_issuer.is_some(), + has_authority_cert_serial: a.authority_cert_serial.is_some(), + }, + ext.critical, + )); + } else if oid == OID_CRL_DISTRIBUTION_POINTS_RAW { + let ParsedExtension::CRLDistributionPoints(p) = ext.parsed_extension() else { + return Err(ResourceCertificateParseError::Parse( + "cRLDistributionPoints parse failed".into(), + )); + }; + crldp.push((parse_crldp_parse(p)?, ext.critical)); + } else if oid == OID_AUTHORITY_INFO_ACCESS_RAW { + let ParsedExtension::AuthorityInfoAccess(p) = ext.parsed_extension() else { + return Err(ResourceCertificateParseError::Parse( + "authorityInfoAccess parse failed".into(), + )); + }; + aia.push((parse_aia_parse(p.accessdescs.as_slice())?, ext.critical)); + } else if oid == OID_SUBJECT_INFO_ACCESS_RAW { + let ParsedExtension::SubjectInfoAccess(s) = ext.parsed_extension() else { + return Err(ResourceCertificateParseError::Parse( + "subjectInfoAccess parse failed".into(), + )); + }; + sia.push((parse_sia_parse(s.accessdescs.as_slice())?, ext.critical)); + } else if oid == OID_CERTIFICATE_POLICIES_RAW { + let ParsedExtension::CertificatePolicies(cp) = ext.parsed_extension() else { + return Err(ResourceCertificateParseError::Parse( + "certificatePolicies parse failed".into(), + )); + }; + let mut policies: Vec = Vec::with_capacity(cp.len()); + for p in cp.iter() { + let b = p.policy_id.as_bytes(); + let policy_oid = if b == OID_CP_IPADDR_ASNUMBER_RAW { + OID_CP_IPADDR_ASNUMBER.to_string() + } else { + p.policy_id.to_id_string() + }; + let qualifier_oids = p + .policy_qualifiers + .as_ref() + .map(|qualifiers| { + qualifiers + .iter() + .map(|qualifier| qualifier.policy_qualifier_id.to_id_string()) + .collect() + }) + .unwrap_or_default(); + policies.push(CertificatePoliciesProfile { + policy_oid, + qualifier_oids, + }); + } + cert_policies.push((policies, ext.critical)); + } else if oid == OID_IP_ADDR_BLOCKS_RAW { + let parsed = IpResourceSet::decode_extn_value(ext.value) + .map_err(|_e| ResourceCertificateParseError::InvalidIpResourcesEncoding)?; + ip_resources.push((parsed, ext.critical)); + } else if oid == OID_AUTONOMOUS_SYS_IDS_RAW { + let parsed = AsResourceSet::decode_extn_value(ext.value) + .map_err(|_e| ResourceCertificateParseError::InvalidAsResourcesEncoding)?; + as_resources.push((parsed, ext.critical)); + } + } + + Ok(RcExtensionsParsed { + basic_constraints, + subject_key_identifier: ski, + authority_key_identifier: aki, + crl_distribution_points: crldp, + authority_info_access: aia, + subject_info_access: sia, + certificate_policies: cert_policies, + extension_oids, + ip_resources, + as_resources, + }) +} + +fn parse_aia_parse( + access: &[x509_parser::extensions::AccessDescription<'_>], +) -> Result { + let mut ca_issuers_uris: Vec = Vec::new(); + let mut ca_issuers_access_location_not_uri = false; + + for ad in access { + if ad.access_method.as_bytes() != OID_AD_CA_ISSUERS_RAW { + continue; + } + let uri = match &ad.access_location { + x509_parser::extensions::GeneralName::URI(u) => u, + _ => { + ca_issuers_access_location_not_uri = true; + continue; + } + }; + ca_issuers_uris.push(uri.to_string()); + } + + Ok(AuthorityInfoAccessParsed { + ca_issuers_uris, + ca_issuers_access_location_not_uri, + }) +} + +fn parse_crldp_parse( + crldp: &x509_parser::extensions::CRLDistributionPoints<'_>, +) -> Result { + let mut out: Vec = Vec::new(); + for p in crldp.iter() { + let mut full_name_uris: Vec = Vec::new(); + let mut full_name_not_uri = false; + let mut full_name_present = false; + let mut name_relative_to_crl_issuer_present = false; + let mut distribution_point_present = false; + + if let Some(dp) = &p.distribution_point { + distribution_point_present = true; + match dp { + x509_parser::extensions::DistributionPointName::FullName(names) => { + full_name_present = true; + for n in names { + match n { + x509_parser::extensions::GeneralName::URI(u) => { + full_name_uris.push(u.to_string()); + } + _ => { + full_name_not_uri = true; + } + } + } + } + x509_parser::extensions::DistributionPointName::NameRelativeToCRLIssuer(_) => { + name_relative_to_crl_issuer_present = true; + } + } + } + + out.push(CrlDistributionPointParsed { + distribution_point_present, + reasons_present: p.reasons.is_some(), + crl_issuer_present: p.crl_issuer.is_some(), + name_relative_to_crl_issuer_present, + full_name_uris, + full_name_not_uri, + full_name_present, + }); + } + Ok(CrlDistributionPointsParsed { + distribution_points: out, + }) +} + +fn parse_sia_parse( + access: &[x509_parser::extensions::AccessDescription<'_>], +) -> Result { + let mut all = Vec::with_capacity(access.len()); + let mut signed_object_uris: Vec = Vec::new(); + let mut signed_object_access_location_not_uri = false; + + for ad in access { + let access_method_oid = if ad.access_method.as_bytes() == OID_AD_CA_REPOSITORY_RAW { + OID_AD_CA_REPOSITORY.to_string() + } else if ad.access_method.as_bytes() == OID_AD_RPKI_MANIFEST_RAW { + OID_AD_RPKI_MANIFEST.to_string() + } else if ad.access_method.as_bytes() == OID_AD_RPKI_NOTIFY_RAW { + OID_AD_RPKI_NOTIFY.to_string() + } else if ad.access_method.as_bytes() == OID_AD_SIGNED_OBJECT_RAW { + OID_AD_SIGNED_OBJECT.to_string() + } else { + ad.access_method.to_id_string() + }; + let is_signed_object = access_method_oid == OID_AD_SIGNED_OBJECT; + let uri = match &ad.access_location { + x509_parser::extensions::GeneralName::URI(u) => u, + _ => { + if is_signed_object { + signed_object_access_location_not_uri = true; + } + continue; + } + }; + if is_signed_object { + signed_object_uris.push(uri.to_string()); + } + all.push(AccessDescription { + access_method_oid, + access_location: uri.to_string(), + }); + } + + Ok(SubjectInfoAccessParsed { + access_descriptions: all, + signed_object_uris, + signed_object_access_location_not_uri, + }) +} + +fn parse_ip_addr_blocks(ext_value: &[u8]) -> Result { + let (rem, obj) = parse_der(ext_value).map_err(|_| ())?; + if !rem.is_empty() { + return Err(()); + } + let seq = obj.as_sequence().map_err(|_| ())?; + let mut families = Vec::with_capacity(seq.len()); + for fam in seq { + let fam_seq = fam.as_sequence().map_err(|_| ())?; + if fam_seq.len() != 2 { + return Err(()); + } + let af_bytes = fam_seq[0].as_slice().map_err(|_| ())?; + if af_bytes.len() != 2 { + return Err(()); + } + let afi = match af_bytes { + [0x00, 0x01] => Afi::Ipv4, + [0x00, 0x02] => Afi::Ipv6, + _ => return Err(()), + }; + + let choice = match &fam_seq[1].content { + BerObjectContent::Null => IpAddressChoice::Inherit, + BerObjectContent::Sequence(_) => { + let items_seq = fam_seq[1].as_sequence().map_err(|_| ())?; + let mut items = Vec::with_capacity(items_seq.len()); + for item in items_seq { + items.push(parse_ip_address_or_range(afi, item)?); + } + IpAddressChoice::AddressesOrRanges(items) + } + _ => return Err(()), + }; + families.push(IpAddressFamily { afi, choice }); + } + Ok(IpResourceSet { families }) +} + +fn parse_ip_address_or_range(afi: Afi, obj: &DerObject<'_>) -> Result { + match &obj.content { + BerObjectContent::BitString(_, _) => { + Ok(IpAddressOrRange::Prefix(parse_ip_prefix(afi, obj)?)) + } + BerObjectContent::Sequence(_) => { + let seq = obj.as_sequence().map_err(|_| ())?; + if seq.len() != 2 { + return Err(()); + } + let min = parse_ip_address_bound(afi, &seq[0], false)?; + let max = parse_ip_address_bound(afi, &seq[1], true)?; + Ok(IpAddressOrRange::Range(IpAddressRange { min, max })) + } + _ => Err(()), + } +} + +fn parse_ip_prefix(afi: Afi, obj: &DerObject<'_>) -> Result { + let (unused_bits, bytes) = match &obj.content { + BerObjectContent::BitString(unused, bso) => (*unused, bso.data.to_vec()), + _ => return Err(()), + }; + if unused_bits > 7 { + return Err(()); + } + if !bytes.is_empty() && unused_bits != 0 { + let mask = (1u8 << unused_bits) - 1; + if (bytes[bytes.len() - 1] & mask) != 0 { + return Err(()); + } + } else if bytes.is_empty() && unused_bits != 0 { + return Err(()); + } + let prefix_len = (bytes.len() * 8) + .checked_sub(unused_bits as usize) + .ok_or(())? as u16; + if prefix_len > afi.ub() { + return Err(()); + } + let addr = canonicalize_prefix_addr(afi, prefix_len, &bytes); + Ok(IpPrefix { + afi, + prefix_len, + addr, + }) +} + +/// Parse an RFC 3779 `IPAddress` BIT STRING into an address-like byte array. +/// +/// When used as an `IPAddressRange` endpoint, RFC 3779 allows endpoints to be encoded with +/// fewer than `ub` bits. In that case, the missing bits are interpreted as 0s for the lower +/// bound and 1s for the upper bound. This is essential to correctly interpret ranges that +/// are expressed on non-octet boundaries. +fn parse_ip_address_bound( + afi: Afi, + obj: &DerObject<'_>, + fill_remaining_ones: bool, +) -> Result, ()> { + let (unused_bits, bytes) = match &obj.content { + BerObjectContent::BitString(unused, bso) => (*unused, bso.data.to_vec()), + _ => return Err(()), + }; + if unused_bits > 7 { + return Err(()); + } + if !bytes.is_empty() && unused_bits != 0 { + let mask = (1u8 << unused_bits) - 1; + if (bytes[bytes.len() - 1] & mask) != 0 { + return Err(()); + } + } else if bytes.is_empty() && unused_bits != 0 { + return Err(()); + } + + let bit_len: u16 = (bytes.len() * 8) + .checked_sub(unused_bits as usize) + .ok_or(())? + .try_into() + .map_err(|_| ())?; + if bit_len > afi.ub() { + return Err(()); + } + + let mut out = vec![0u8; afi.octets_len()]; + let copy_len = bytes.len().min(out.len()); + out[..copy_len].copy_from_slice(&bytes[..copy_len]); + + if fill_remaining_ones { + if bit_len == 0 { + for b in &mut out { + *b = 0xFF; + } + return Ok(out); + } + + let last_bit = (bit_len - 1) as usize; + let last_byte = last_bit / 8; + let rem = (bit_len % 8) as u8; + + if rem != 0 && last_byte < out.len() { + // Set the (8-rem) trailing bits in the last byte to 1. + let mask: u8 = (1u8 << (8 - rem)) - 1; + out[last_byte] |= mask; + } + for b in out.iter_mut().skip(last_byte + 1) { + *b = 0xFF; + } + } + + Ok(out) +} + +fn parse_as_identifiers(ext_value: &[u8]) -> Result { + let (rem, obj) = parse_der(ext_value).map_err(|_| ())?; + if !rem.is_empty() { + return Err(()); + } + let seq = obj.as_sequence().map_err(|_| ())?; + let mut asnum: Option = None; + let mut rdi: Option = None; + for item in seq { + if item.class() != Class::ContextSpecific { + return Err(()); + } + match item.tag() { + Tag(0) => { + if asnum.is_some() { + return Err(()); + } + let inner = parse_explicit_inner(item)?; + asnum = Some(parse_as_identifier_choice(&inner)?); + } + Tag(1) => { + if rdi.is_some() { + return Err(()); + } + let inner = parse_explicit_inner(item)?; + rdi = Some(parse_as_identifier_choice(&inner)?); + } + _ => return Err(()), + } + } + Ok(AsResourceSet { asnum, rdi }) +} + +fn parse_explicit_inner<'a>(obj: &'a DerObject<'a>) -> Result, ()> { + let inner_der = obj.as_slice().map_err(|_| ())?; + let (rem, inner) = parse_der(inner_der).map_err(|_| ())?; + if !rem.is_empty() { + return Err(()); + } + Ok(inner) +} + +fn parse_as_identifier_choice(obj: &DerObject<'_>) -> Result { + match &obj.content { + BerObjectContent::Null => Ok(AsIdentifierChoice::Inherit), + BerObjectContent::Sequence(_) => { + let seq = obj.as_sequence().map_err(|_| ())?; + let mut items = Vec::with_capacity(seq.len()); + for item in seq { + items.push(parse_as_id_or_range(item)?); + } + Ok(AsIdentifierChoice::AsIdsOrRanges(items)) + } + _ => Err(()), + } +} + +fn parse_as_id_or_range(obj: &DerObject<'_>) -> Result { + match &obj.content { + BerObjectContent::Integer(_) => { + let v = obj.as_u64().map_err(|_| ())?; + if v > u32::MAX as u64 { + return Err(()); + } + Ok(AsIdOrRange::Id(v as u32)) + } + BerObjectContent::Sequence(_) => { + let seq = obj.as_sequence().map_err(|_| ())?; + if seq.len() != 2 { + return Err(()); + } + let min = seq[0].as_u64().map_err(|_| ())?; + let max = seq[1].as_u64().map_err(|_| ())?; + if min > u32::MAX as u64 || max > u32::MAX as u64 || min > max { + return Err(()); + } + Ok(AsIdOrRange::Range { + min: min as u32, + max: max as u32, + }) + } + _ => Err(()), + } +} + +fn canonicalize_prefix_addr(afi: Afi, prefix_len: u16, bytes: &[u8]) -> Vec { + let full_len = afi.octets_len(); + let mut addr = vec![0u8; full_len]; + let copy_len = bytes.len().min(full_len); + addr[..copy_len].copy_from_slice(&bytes[..copy_len]); + + if prefix_len == 0 { + return addr; + } + + let last_prefix_bit = (prefix_len - 1) as usize; + let last_prefix_byte = last_prefix_bit / 8; + let rem = (prefix_len % 8) as u8; + if rem != 0 && last_prefix_byte < addr.len() { + let mask: u8 = 0xFF << (8 - rem); + addr[last_prefix_byte] &= mask; + } + addr +} + +fn prefix_covers(resource: &IpPrefix, subject: &IpPrefix) -> bool { + if resource.afi != subject.afi { + return false; + } + if resource.prefix_len > subject.prefix_len { + return false; + } + let n = resource.prefix_len as usize; + let whole = n / 8; + let rem = (n % 8) as u8; + if resource.addr.len() != subject.addr.len() { + return false; + } + if resource.addr[..whole] != subject.addr[..whole] { + return false; + } + if rem == 0 { + return true; + } + let mask = 0xFFu8 << (8 - rem); + (resource.addr[whole] & mask) == (subject.addr[whole] & mask) +} + +fn prefix_range(afi: Afi, p: &IpPrefix) -> (u128, u128) { + let mut base_bytes = [0u8; 16]; + match afi { + Afi::Ipv4 => { + base_bytes[12..].copy_from_slice(&p.addr[..4]); + } + Afi::Ipv6 => { + base_bytes.copy_from_slice(&p.addr[..16]); + } + } + let base = u128::from_be_bytes(base_bytes); + let host_bits = (afi.ub() - p.prefix_len) as u32; + if host_bits == 0 { + return (base, base); + } + let mask = (1u128 << host_bits) - 1; + (base, base | mask) +} + +fn range_covers_prefix(afi: Afi, r: &IpAddressRange, p: &IpPrefix) -> bool { + let (p_min, p_max) = prefix_range(afi, p); + let r_min = bytes_to_u128(afi, &r.min); + let r_max = bytes_to_u128(afi, &r.max); + r_min <= p_min && p_max <= r_max +} + +fn bytes_to_u128(afi: Afi, bytes: &[u8]) -> u128 { + let mut out = [0u8; 16]; + match afi { + Afi::Ipv4 => { + let copy_len = bytes.len().min(4); + out[12..12 + copy_len].copy_from_slice(&bytes[..copy_len]); + } + Afi::Ipv6 => { + let copy_len = bytes.len().min(16); + out[..copy_len].copy_from_slice(&bytes[..copy_len]); + } + } + u128::from_be_bytes(out) +} diff --git a/crates/panda-rpki-validator/src/data_model/roa.rs b/crates/panda-rpki-validator/src/data_model/roa.rs new file mode 100644 index 0000000..ef35002 --- /dev/null +++ b/crates/panda-rpki-validator/src/data_model/roa.rs @@ -0,0 +1,633 @@ +use crate::data_model::common::{DerReader, der_take_tlv}; +use crate::data_model::oid::OID_CT_ROUTE_ORIGIN_AUTHZ; +use crate::data_model::rc::{Afi as RcAfi, IpPrefix as RcIpPrefix, ResourceCertificate}; +use crate::data_model::signed_object::{ + RpkiSignedObject, RpkiSignedObjectParsed, SignedObjectDecodeError, SignedObjectParseError, + SignedObjectValidateError, +}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RoaObject { + pub signed_object: RpkiSignedObject, + pub econtent_type: String, + pub roa: RoaEContent, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RoaObjectParsed { + pub signed_object: RpkiSignedObjectParsed, + pub econtent_type: String, + pub roa: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RoaEContent { + pub version: u32, + pub as_id: u32, + pub ip_addr_blocks: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RoaEContentParsed { + der: Vec, +} + +#[derive(Debug, thiserror::Error)] +pub enum RoaParseError { + #[error("signed object parse error: {0} (RFC 6488 §2-§3; RFC 9589 §4)")] + SignedObject(#[from] SignedObjectParseError), + #[error("ROA parse error: {0} (RFC 9582 §4; DER)")] + Parse(String), + + #[error("ROA trailing bytes: {0} bytes (RFC 9582 §4; DER)")] + TrailingBytes(usize), +} + +#[derive(Debug, thiserror::Error)] +pub enum RoaProfileError { + #[error("signed object profile error: {0} (RFC 6488 §2-§3; RFC 9589 §4)")] + SignedObject(#[from] SignedObjectValidateError), + + #[error("ROA eContentType must be {OID_CT_ROUTE_ORIGIN_AUTHZ}, got {0} (RFC 9582 §3)")] + InvalidEContentType(String), + + #[error("ROA profile decode error: {0} (RFC 9582 §4; DER)")] + ProfileDecode(String), + + #[error("RouteOriginAttestation must be a SEQUENCE of 2 or 3 elements, got {0} (RFC 9582 §4)")] + InvalidAttestationSequenceLen(usize), + + #[error("ROA version must be 0, got {0} (RFC 9582 §4.1)")] + InvalidVersion(u64), + + #[error("ROA asID out of range (0..=4294967295), got {0} (RFC 9582 §4.2)")] + AsIdOutOfRange(u64), + + #[error("ROA ipAddrBlocks must have length 1..2, got {0} (RFC 9582 §4; RFC 9582 §4.3.1)")] + InvalidIpAddrBlocksLen(usize), + + #[error("ROAIPAddressFamily must be a SEQUENCE of 2 elements (RFC 9582 §4.3.1)")] + InvalidIpAddressFamily, + + #[error("ROA addressFamily must be an OCTET STRING of 2 bytes (RFC 9582 §4.3.1)")] + InvalidAddressFamily, + + #[error("ROA addressFamily AFI not supported: {0:02X?} (RFC 9582 §4.3.1)")] + UnsupportedAfi(Vec), + + #[error("ROA contains duplicate AFI {0:?} (RFC 9582 §4.3.1)")] + DuplicateAfi(RoaAfi), + + #[error("ROAAddresses must have at least one entry (RFC 9582 §4.3.2)")] + EmptyAddressList, + + #[error("ROAIPAddress must be a SEQUENCE of 1..2 elements (RFC 9582 §4.3.2)")] + InvalidRoaIpAddress, + + #[error("ROAIPAddress.address must be a BIT STRING (RFC 9582 §4.3.2.1; RFC 3779 §2.2.3.8)")] + InvalidPrefixBitString, + + #[error( + "ROAIPAddress.address has invalid unused bits encoding (RFC 9582 §4.3.2.1; RFC 3779 §2.2.3.8)" + )] + InvalidPrefixUnusedBits, + + #[error( + "ROAIPAddress.address prefix length {prefix_len} out of range for {afi:?} (RFC 9582 §4.3.2.1)" + )] + PrefixLenOutOfRange { afi: RoaAfi, prefix_len: u16 }, + + #[error( + "ROAIPAddress.maxLength out of range for {afi:?}: prefix_len={prefix_len}, max_len={max_len} (RFC 9582 §4.3.2.2)" + )] + InvalidMaxLength { + afi: RoaAfi, + prefix_len: u16, + max_len: u16, + }, +} + +impl From for RoaProfileError { + fn from(value: SignedObjectDecodeError) -> Self { + match value { + SignedObjectDecodeError::Parse(e) => RoaProfileError::ProfileDecode(e.to_string()), + SignedObjectDecodeError::Validate(e) => RoaProfileError::SignedObject(e), + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum RoaDecodeError { + #[error("{0}")] + Parse(#[from] RoaParseError), + + #[error("{0}")] + Validate(#[from] RoaProfileError), +} + +#[derive(Debug, thiserror::Error)] +pub enum RoaValidateError { + #[error("ROA EE certificate must not contain AS resources extension (RFC 9582 §5)")] + EeAsResourcesPresent, + + #[error("ROA EE certificate must contain IP resources extension (RFC 9582 §5)")] + EeIpResourcesMissing, + + #[error("ROA EE certificate IP resources must not use inherit (RFC 9582 §5)")] + EeIpResourcesInherit, + + #[error( + "ROA prefix not covered by EE certificate IP resources: {afi:?} {addr:?}/{prefix_len} (RFC 9582 §5; RFC 3779 §2.3)" + )] + PrefixNotInEeResources { + afi: RoaAfi, + addr: Vec, + prefix_len: u16, + }, +} + +impl RoaObject { + /// Parse step of scheme A (`parse → validate → verify`). + pub fn parse_der(der: &[u8]) -> Result { + let signed_object = RpkiSignedObject::parse_der(der)?; + let econtent_type = signed_object + .signed_data + .encap_content_info + .econtent_type + .clone(); + let roa = signed_object + .signed_data + .encap_content_info + .econtent + .as_deref() + .map(RoaEContent::parse_der) + .transpose()?; + Ok(RoaObjectParsed { + signed_object, + econtent_type, + roa, + }) + } + + /// Profile validate step of scheme A (`parse → validate → verify`). + /// + /// `RoaObject` is already profile-validated when constructed via `decode_der()` / + /// `RoaObjectParsed::validate_profile()`. + pub fn validate_profile(&self) -> Result<(), RoaProfileError> { + Ok(()) + } + + pub fn decode_der(der: &[u8]) -> Result { + Ok(Self::parse_der(der)?.validate_profile()?) + } + + pub fn decode_der_with_strict_options( + der: &[u8], + strict_cms_der: bool, + strict_name: bool, + ) -> Result { + let signed_object = + RpkiSignedObject::decode_der_with_strict_options(der, strict_cms_der, strict_name) + .map_err(RoaProfileError::from)?; + Self::from_signed_object(signed_object) + } + + pub fn from_signed_object(signed_object: RpkiSignedObject) -> Result { + let econtent_type = signed_object + .signed_data + .encap_content_info + .econtent_type + .clone(); + if econtent_type != OID_CT_ROUTE_ORIGIN_AUTHZ { + return Err(RoaProfileError::InvalidEContentType(econtent_type).into()); + } + + let roa = RoaEContent::decode_der(&signed_object.signed_data.encap_content_info.econtent)?; + Ok(Self { + roa, + signed_object, + econtent_type: OID_CT_ROUTE_ORIGIN_AUTHZ.to_string(), + }) + } + + /// Validate this ROA's embedded EE certificate resources. + pub fn validate_embedded_ee_cert(&self) -> Result<(), RoaValidateError> { + let ee = &self.signed_object.signed_data.certificates[0].resource_cert; + self.roa.validate_against_ee_cert(ee) + } +} + +#[derive( + Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize, +)] +pub enum RoaAfi { + Ipv4, + Ipv6, +} + +impl RoaAfi { + fn ub(self) -> u16 { + match self { + RoaAfi::Ipv4 => 32, + RoaAfi::Ipv6 => 128, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RoaIpAddressFamily { + pub afi: RoaAfi, + pub addresses: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct RoaIpAddress { + pub prefix: IpPrefix, + pub max_length: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct IpPrefix { + pub afi: RoaAfi, + /// Prefix length in bits. + pub prefix_len: u16, + /// Network order address bytes (always 16 bytes), with host bits cleared. + /// + /// For IPv4 prefixes, only the first 4 bytes are used and the remaining 12 bytes are zero. + pub addr: [u8; 16], +} + +impl IpPrefix { + pub fn addr_bytes(&self) -> &[u8] { + match self.afi { + RoaAfi::Ipv4 => &self.addr[..4], + RoaAfi::Ipv6 => &self.addr[..16], + } + } +} + +impl RoaEContent { + /// Parse step of scheme A (`parse → validate → verify`). + pub fn parse_der(der: &[u8]) -> Result { + let (_tag, _value, rem) = der_take_tlv(der).map_err(RoaParseError::Parse)?; + if !rem.is_empty() { + return Err(RoaParseError::TrailingBytes(rem.len())); + } + Ok(RoaEContentParsed { der: der.to_vec() }) + } + + /// Profile validate step of scheme A (`parse → validate → verify`). + /// + /// `RoaEContent` is already profile-validated when constructed via `decode_der()` / + /// `RoaEContentParsed::validate_profile()`. + pub fn validate_profile(&self) -> Result<(), RoaProfileError> { + Ok(()) + } + + /// Decode the DER-encoded RouteOriginAttestation defined in RFC 9582 §4 (`parse + validate`). + pub fn decode_der(der: &[u8]) -> Result { + Ok(Self::parse_der(der)?.validate_profile()?) + } + + pub fn canonicalize(&mut self) { + self.ip_addr_blocks.sort_by_key(|f| f.afi); + for fam in &mut self.ip_addr_blocks { + fam.addresses.sort(); + fam.addresses.dedup(); + } + } + + /// Validate ROA payload against the embedded EE resource certificate (RFC 9582 §5). + /// + /// This performs the EE/payload semantic checks that do not require certificate path + /// validation. + pub fn validate_against_ee_cert( + &self, + ee: &ResourceCertificate, + ) -> Result<(), RoaValidateError> { + if ee.tbs.extensions.as_resources.is_some() { + return Err(RoaValidateError::EeAsResourcesPresent); + } + + let ip = ee + .tbs + .extensions + .ip_resources + .as_ref() + .ok_or(RoaValidateError::EeIpResourcesMissing)?; + + if ip.has_any_inherit() { + return Err(RoaValidateError::EeIpResourcesInherit); + } + + for fam in &self.ip_addr_blocks { + for entry in &fam.addresses { + let rc_prefix = roa_prefix_to_rc(&entry.prefix); + if !ip.contains_prefix(&rc_prefix) { + return Err(RoaValidateError::PrefixNotInEeResources { + afi: entry.prefix.afi, + addr: entry.prefix.addr_bytes().to_vec(), + prefix_len: entry.prefix.prefix_len, + }); + } + } + } + Ok(()) + } +} + +impl RoaObjectParsed { + pub fn validate_profile(self) -> Result { + let signed_object = self.signed_object.validate_profile()?; + let econtent_type = signed_object + .signed_data + .encap_content_info + .econtent_type + .clone(); + if econtent_type != OID_CT_ROUTE_ORIGIN_AUTHZ { + return Err(RoaProfileError::InvalidEContentType(econtent_type)); + } + let roa = self + .roa + .ok_or_else(|| RoaProfileError::ProfileDecode("ROA.eContent missing".into()))? + .validate_profile()?; + Ok(RoaObject { + signed_object, + econtent_type: OID_CT_ROUTE_ORIGIN_AUTHZ.to_string(), + roa, + }) + } +} + +impl RoaEContentParsed { + pub fn validate_profile(self) -> Result { + fn count_elements(mut r: DerReader<'_>) -> Result { + let mut n = 0usize; + while !r.is_empty() { + r.skip_any()?; + n += 1; + } + Ok(n) + } + + let mut r = DerReader::new(&self.der); + let mut seq = r + .take_sequence() + .map_err(|e| RoaProfileError::ProfileDecode(e))?; + if !r.is_empty() { + return Err(RoaProfileError::ProfileDecode( + "trailing bytes after RouteOriginAttestation".into(), + )); + } + + let elem_count = + count_elements(seq).map_err(|e| RoaProfileError::ProfileDecode(e.to_string()))?; + if elem_count != 2 && elem_count != 3 { + return Err(RoaProfileError::InvalidAttestationSequenceLen(elem_count)); + } + + let mut version: u32 = 0; + if elem_count == 3 { + if seq + .peek_tag() + .map_err(|e| RoaProfileError::ProfileDecode(e.to_string()))? + != 0xA0 + { + return Err(RoaProfileError::ProfileDecode( + "RouteOriginAttestation.version must be [0] EXPLICIT INTEGER".into(), + )); + } + let (inner_tag, inner_val) = seq + .take_explicit(0xA0) + .map_err(|e| RoaProfileError::ProfileDecode(e.to_string()))?; + if inner_tag != 0x02 { + return Err(RoaProfileError::ProfileDecode( + "RouteOriginAttestation.version must be [0] EXPLICIT INTEGER".into(), + )); + } + let v = crate::data_model::common::der_uint_from_bytes(inner_val) + .map_err(|e| RoaProfileError::ProfileDecode(e.to_string()))?; + if v != 0 { + return Err(RoaProfileError::InvalidVersion(v)); + } + version = 0; + } + + let as_id_u64 = seq + .take_uint_u64() + .map_err(|e| RoaProfileError::ProfileDecode(e.to_string()))?; + if as_id_u64 > u32::MAX as u64 { + return Err(RoaProfileError::AsIdOutOfRange(as_id_u64)); + } + let as_id = as_id_u64 as u32; + let ip_addr_blocks = parse_ip_addr_blocks_cursor( + seq.take_sequence() + .map_err(|e| RoaProfileError::ProfileDecode(format!("ipAddrBlocks: {e}")))?, + )?; + + if !seq.is_empty() { + // Extra elements beyond the expected 2..3. + let extra = + count_elements(seq).map_err(|e| RoaProfileError::ProfileDecode(e.to_string()))?; + return Err(RoaProfileError::InvalidAttestationSequenceLen( + elem_count + extra, + )); + } + + let mut out = RoaEContent { + version, + as_id, + ip_addr_blocks, + }; + out.canonicalize(); + Ok(out) + } +} + +fn roa_prefix_to_rc(p: &IpPrefix) -> RcIpPrefix { + let afi = match p.afi { + RoaAfi::Ipv4 => RcAfi::Ipv4, + RoaAfi::Ipv6 => RcAfi::Ipv6, + }; + RcIpPrefix { + afi, + prefix_len: p.prefix_len, + addr: p.addr_bytes().to_vec(), + } +} + +fn parse_ip_addr_blocks_cursor( + mut seq: DerReader<'_>, +) -> Result, RoaProfileError> { + fn count_elements(mut r: DerReader<'_>) -> Result { + let mut n = 0usize; + while !r.is_empty() { + r.skip_any()?; + n += 1; + } + Ok(n) + } + + let fam_count = + count_elements(seq).map_err(|e| RoaProfileError::ProfileDecode(e.to_string()))?; + if fam_count == 0 || fam_count > 2 { + return Err(RoaProfileError::InvalidIpAddrBlocksLen(fam_count)); + } + + let mut out: Vec = Vec::with_capacity(fam_count); + while !seq.is_empty() { + let family = parse_ip_address_family_cursor( + seq.take_sequence() + .map_err(|e| RoaProfileError::ProfileDecode(e.to_string()))?, + )?; + if out.iter().any(|f| f.afi == family.afi) { + return Err(RoaProfileError::DuplicateAfi(family.afi)); + } + out.push(family); + } + Ok(out) +} + +fn parse_ip_address_family_cursor( + mut fam: DerReader<'_>, +) -> Result { + let afi = { + let bytes = fam + .take_octet_string() + .map_err(|_e| RoaProfileError::InvalidAddressFamily)?; + if bytes.len() != 2 { + return Err(RoaProfileError::InvalidAddressFamily); + } + match bytes { + [0x00, 0x01] => RoaAfi::Ipv4, + [0x00, 0x02] => RoaAfi::Ipv6, + _ => return Err(RoaProfileError::UnsupportedAfi(bytes.to_vec())), + } + }; + + let mut addrs = fam + .take_sequence() + .map_err(|_e| RoaProfileError::InvalidIpAddressFamily)?; + if !fam.is_empty() { + return Err(RoaProfileError::InvalidIpAddressFamily); + } + + if addrs.is_empty() { + return Err(RoaProfileError::EmptyAddressList); + } + let mut addresses: Vec = Vec::new(); + while !addrs.is_empty() { + let entry = addrs + .take_sequence() + .map_err(|e| RoaProfileError::ProfileDecode(e.to_string()))?; + addresses.push(parse_roa_ip_address_cursor(afi, entry)?); + } + + Ok(RoaIpAddressFamily { afi, addresses }) +} + +fn parse_roa_ip_address_cursor( + afi: RoaAfi, + mut seq: DerReader<'_>, +) -> Result { + if seq.is_empty() { + return Err(RoaProfileError::InvalidRoaIpAddress); + } + + let (unused_bits, bytes) = seq + .take_bit_string() + .map_err(|_e| RoaProfileError::InvalidPrefixBitString)?; + let prefix = parse_prefix_bits_bytes(afi, unused_bits, bytes)?; + + let max_length = if !seq.is_empty() { + let v = seq + .take_uint_u64() + .map_err(|e| RoaProfileError::ProfileDecode(e.to_string()))?; + let max_len: u16 = v + .try_into() + .map_err(|_e| RoaProfileError::InvalidMaxLength { + afi, + prefix_len: prefix.prefix_len, + max_len: u16::MAX, + })?; + Some(max_len) + } else { + None + }; + + if !seq.is_empty() { + return Err(RoaProfileError::InvalidRoaIpAddress); + } + + if let Some(max_len) = max_length { + let ub = afi.ub(); + if max_len > ub || max_len < prefix.prefix_len { + return Err(RoaProfileError::InvalidMaxLength { + afi, + prefix_len: prefix.prefix_len, + max_len, + }); + } + } + + Ok(RoaIpAddress { prefix, max_length }) +} + +fn parse_prefix_bits_bytes( + afi: RoaAfi, + unused_bits: u8, + bytes: &[u8], +) -> Result { + if unused_bits > 7 { + return Err(RoaProfileError::InvalidPrefixUnusedBits); + } + if bytes.is_empty() { + if unused_bits != 0 { + return Err(RoaProfileError::InvalidPrefixUnusedBits); + } + } else if unused_bits != 0 { + let mask = (1u8 << unused_bits) - 1; + if (bytes[bytes.len() - 1] & mask) != 0 { + return Err(RoaProfileError::InvalidPrefixUnusedBits); + } + } + + let prefix_len = (bytes.len() * 8) + .checked_sub(unused_bits as usize) + .ok_or(RoaProfileError::InvalidPrefixUnusedBits)? as u16; + if prefix_len > afi.ub() { + return Err(RoaProfileError::PrefixLenOutOfRange { afi, prefix_len }); + } + + let addr = canonicalize_prefix_addr(afi, prefix_len, bytes); + Ok(IpPrefix { + afi, + prefix_len, + addr, + }) +} + +fn canonicalize_prefix_addr(afi: RoaAfi, prefix_len: u16, bytes: &[u8]) -> [u8; 16] { + let full_len = match afi { + RoaAfi::Ipv4 => 4, + RoaAfi::Ipv6 => 16, + }; + let mut addr = [0u8; 16]; + let copy_len = bytes.len().min(full_len); + addr[..copy_len].copy_from_slice(&bytes[..copy_len]); + + if prefix_len == 0 { + return addr; + } + + let last_prefix_bit = (prefix_len - 1) as usize; + let last_prefix_byte = last_prefix_bit / 8; + let rem = (prefix_len % 8) as u8; + if rem != 0 { + let mask: u8 = 0xFF << (8 - rem); + if last_prefix_byte < full_len { + addr[last_prefix_byte] &= mask; + } + } + addr +} diff --git a/crates/panda-rpki-validator/src/data_model/router_cert.rs b/crates/panda-rpki-validator/src/data_model/router_cert.rs new file mode 100644 index 0000000..279706c --- /dev/null +++ b/crates/panda-rpki-validator/src/data_model/router_cert.rs @@ -0,0 +1,363 @@ +use crate::data_model::oid::{ + OID_EC_PUBLIC_KEY, OID_EXTENDED_KEY_USAGE_RAW, OID_KP_BGPSEC_ROUTER, OID_SECP256R1, +}; +use crate::data_model::rc::{ + AsIdOrRange, AsIdentifierChoice, ResourceCertKind, ResourceCertificate, + ResourceCertificateParseError, ResourceCertificateParsed, ResourceCertificateProfileError, + ResourceCertificateRole, +}; +use crate::validation::cert_path::{CertPathError, validate_ee_cert_path_with_predecoded_ee}; +use x509_parser::extensions::ParsedExtension; +use x509_parser::prelude::{FromDer, X509Certificate}; +use x509_parser::public_key::PublicKey; +use x509_parser::x509::SubjectPublicKeyInfo; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BgpsecRouterCertificateParsed { + pub rc_parsed: ResourceCertificateParsed, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BgpsecRouterCertificate { + pub raw_der: Vec, + pub resource_cert: ResourceCertificate, + pub subject_key_identifier: Vec, + pub spki_der: Vec, + pub asns: Vec, +} + +#[derive(Debug, thiserror::Error)] +pub enum BgpsecRouterCertificateParseError { + #[error("resource certificate parse error: {0} (RFC 5280 §4.1; RFC 6487 §4; RFC 8209 §3.1)")] + ResourceCertificate(#[from] ResourceCertificateParseError), + + #[error("X.509 parse error: {0} (RFC 5280 §4.1; RFC 8209 §3.1)")] + X509(String), + + #[error("trailing bytes after router certificate DER: {0} bytes (DER; RFC 5280 §4.1)")] + TrailingBytes(usize), + + #[error("router SubjectPublicKeyInfo parse error: {0} (RFC 5280 §4.1.2.7; RFC 8208 §3.1)")] + SpkiParse(String), + + #[error("trailing bytes after router SubjectPublicKeyInfo DER: {0} bytes (DER; RFC 8208 §3.1)")] + SpkiTrailingBytes(usize), +} + +#[derive(Debug, thiserror::Error)] +pub enum BgpsecRouterCertificateProfileError { + #[error("resource certificate profile error: {0} (RFC 6487 §4; RFC 8209 §3.1)")] + ResourceCertificate(#[from] ResourceCertificateProfileError), + + #[error("BGPsec router certificate must be an EE certificate (RFC 8209 §3.1)")] + NotEe, + + #[error( + "BGPsec router certificate must contain SubjectKeyIdentifier (RFC 6487 §4.8.2; RFC 8209 §3.3)" + )] + MissingSki, + + #[error( + "BGPsec router certificate must include ExtendedKeyUsage (RFC 8209 §3.1.3.2; RFC 8209 §3.3)" + )] + MissingExtendedKeyUsage, + + #[error( + "BGPsec router certificate ExtendedKeyUsage must be non-critical (RFC 6487 §4.8.4; RFC 8209 §3.1.3.2)" + )] + ExtendedKeyUsageCriticality, + + #[error( + "BGPsec router certificate ExtendedKeyUsage must contain id-kp-bgpsec-router ({OID_KP_BGPSEC_ROUTER}) (RFC 8209 §3.1.3.2; RFC 8209 §3.3)" + )] + MissingBgpsecRouterEku, + + #[error( + "BGPsec router certificate MUST NOT include Subject Information Access (RFC 8209 §3.1.3.3; RFC 8209 §3.3)" + )] + SubjectInfoAccessPresent, + + #[error( + "BGPsec router certificate MUST NOT include IP resources extension (RFC 8209 §3.1.3.4; RFC 8209 §3.3)" + )] + IpResourcesPresent, + + #[error( + "BGPsec router certificate MUST include AS resources extension (RFC 8209 §3.1.3.5; RFC 8209 §3.3)" + )] + AsResourcesMissing, + + #[error( + "BGPsec router certificate AS resources MUST include one or more ASNs (RFC 8209 §3.1.3.5)" + )] + AsResourcesAsnumMissing, + + #[error("BGPsec router certificate AS resources MUST NOT use inherit (RFC 8209 §3.1.3.5)")] + AsResourcesInherit, + + #[error( + "BGPsec router certificate AS resources MUST contain explicit ASNs, not ranges (RFC 8209 §3.1.3.5)" + )] + AsResourcesRangeNotAllowed, + + #[error( + "BGPsec router certificate subjectPublicKeyInfo.algorithm must be id-ecPublicKey ({OID_EC_PUBLIC_KEY}) (RFC 8208 §3.1)" + )] + SpkiAlgorithmNotEcPublicKey, + + #[error( + "BGPsec router certificate subjectPublicKeyInfo.parameters must be secp256r1 ({OID_SECP256R1}) (RFC 8208 §3.1)" + )] + SpkiWrongCurve, + + #[error( + "BGPsec router certificate subjectPublicKeyInfo.parameters missing or invalid (RFC 8208 §3.1)" + )] + SpkiParametersMissingOrInvalid, + + #[error( + "BGPsec router certificate subjectPublicKey MUST be uncompressed P-256 ECPoint (RFC 8208 §3.1)" + )] + SpkiEcPointNotUncompressedP256, +} + +#[derive(Debug, thiserror::Error)] +pub enum BgpsecRouterCertificateDecodeError { + #[error("{0}")] + Parse(#[from] BgpsecRouterCertificateParseError), + + #[error("{0}")] + Validate(#[from] BgpsecRouterCertificateProfileError), +} + +#[derive(Debug, thiserror::Error)] +pub enum BgpsecRouterCertificatePathError { + #[error("{0}")] + Decode(#[from] BgpsecRouterCertificateDecodeError), + + #[error("{0}")] + CertPath(#[from] CertPathError), +} + +impl BgpsecRouterCertificate { + pub fn parse_der( + der: &[u8], + ) -> Result { + let (rem, cert) = X509Certificate::from_der(der) + .map_err(|e| BgpsecRouterCertificateParseError::X509(e.to_string()))?; + if !rem.is_empty() { + return Err(BgpsecRouterCertificateParseError::TrailingBytes(rem.len())); + } + let (spki_rem, _spki) = + SubjectPublicKeyInfo::from_der(cert.tbs_certificate.subject_pki.raw) + .map_err(|e| BgpsecRouterCertificateParseError::SpkiParse(e.to_string()))?; + if !spki_rem.is_empty() { + return Err(BgpsecRouterCertificateParseError::SpkiTrailingBytes( + spki_rem.len(), + )); + } + let rc_parsed = ResourceCertificate::parse_der(der)?; + Ok(BgpsecRouterCertificateParsed { rc_parsed }) + } + + pub fn validate_profile(&self) -> Result<(), BgpsecRouterCertificateProfileError> { + Ok(()) + } + + pub fn decode_der(der: &[u8]) -> Result { + Ok(Self::parse_der(der)?.validate_profile()?) + } + + pub fn from_der(der: &[u8]) -> Result { + Self::decode_der(der) + } + + pub fn validate_path_with_prevalidated_issuer( + der: &[u8], + issuer_ca: &ResourceCertificate, + issuer_spki: &SubjectPublicKeyInfo<'_>, + issuer_crl: &crate::data_model::crl::RpkixCrl, + issuer_crl_revoked_serials: &std::collections::HashSet>, + issuer_ca_rsync_uri: Option<&str>, + issuer_crl_rsync_uri: Option<&str>, + validation_time: time::OffsetDateTime, + ) -> Result { + let cert = Self::decode_der(der)?; + validate_ee_cert_path_with_predecoded_ee( + &cert.resource_cert, + der, + issuer_ca, + issuer_spki, + issuer_crl, + issuer_crl_revoked_serials, + issuer_ca_rsync_uri, + issuer_crl_rsync_uri, + validation_time, + )?; + Ok(cert) + } +} + +impl BgpsecRouterCertificateParsed { + pub fn validate_profile( + self, + ) -> Result { + let rc = self.rc_parsed.validate_profile()?; + rc.validate_rfc6487_profile(ResourceCertificateRole::RouterEe)?; + if rc.kind != ResourceCertKind::Ee { + return Err(BgpsecRouterCertificateProfileError::NotEe); + } + let ski = rc + .tbs + .extensions + .subject_key_identifier + .clone() + .ok_or(BgpsecRouterCertificateProfileError::MissingSki)?; + + if rc.tbs.extensions.subject_info_access.is_some() { + return Err(BgpsecRouterCertificateProfileError::SubjectInfoAccessPresent); + } + if rc.tbs.extensions.ip_resources.is_some() { + return Err(BgpsecRouterCertificateProfileError::IpResourcesPresent); + } + let as_resources = rc + .tbs + .extensions + .as_resources + .as_ref() + .ok_or(BgpsecRouterCertificateProfileError::AsResourcesMissing)?; + let asns = extract_router_asns(as_resources)?; + + let (rem, cert) = X509Certificate::from_der(&rc.raw_der).map_err(|e| { + BgpsecRouterCertificateProfileError::ResourceCertificate( + ResourceCertificateProfileError::InvalidCertificatePolicy(e.to_string()), + ) + })?; + if !rem.is_empty() { + return Err(BgpsecRouterCertificateProfileError::ResourceCertificate( + ResourceCertificateProfileError::InvalidCertificatePolicy(format!( + "trailing bytes after router certificate DER: {}", + rem.len() + )), + )); + } + validate_router_eku(&cert)?; + validate_router_spki(&rc.tbs.subject_public_key_info)?; + + Ok(BgpsecRouterCertificate { + raw_der: rc.raw_der.clone(), + resource_cert: rc.clone(), + subject_key_identifier: ski, + spki_der: rc.tbs.subject_public_key_info.clone(), + asns, + }) + } +} + +fn extract_router_asns( + as_resources: &crate::data_model::rc::AsResourceSet, +) -> Result, BgpsecRouterCertificateProfileError> { + let asnum = as_resources + .asnum + .as_ref() + .ok_or(BgpsecRouterCertificateProfileError::AsResourcesAsnumMissing)?; + if matches!(asnum, AsIdentifierChoice::Inherit) + || matches!(as_resources.rdi.as_ref(), Some(AsIdentifierChoice::Inherit)) + { + return Err(BgpsecRouterCertificateProfileError::AsResourcesInherit); + } + let AsIdentifierChoice::AsIdsOrRanges(items) = asnum else { + return Err(BgpsecRouterCertificateProfileError::AsResourcesInherit); + }; + if items.is_empty() { + return Err(BgpsecRouterCertificateProfileError::AsResourcesAsnumMissing); + } + let mut asns = Vec::with_capacity(items.len()); + for item in items { + match item { + AsIdOrRange::Id(v) => asns.push(*v), + AsIdOrRange::Range { .. } => { + return Err(BgpsecRouterCertificateProfileError::AsResourcesRangeNotAllowed); + } + } + } + asns.sort_unstable(); + asns.dedup(); + Ok(asns) +} + +fn validate_router_eku( + cert: &X509Certificate<'_>, +) -> Result<(), BgpsecRouterCertificateProfileError> { + let mut matches = cert + .tbs_certificate + .extensions() + .iter() + .filter(|ext| ext.oid.as_bytes() == OID_EXTENDED_KEY_USAGE_RAW); + let Some(ext) = matches.next() else { + return Err(BgpsecRouterCertificateProfileError::MissingExtendedKeyUsage); + }; + if matches.next().is_some() { + return Err(BgpsecRouterCertificateProfileError::MissingExtendedKeyUsage); + } + if ext.critical { + return Err(BgpsecRouterCertificateProfileError::ExtendedKeyUsageCriticality); + } + let ParsedExtension::ExtendedKeyUsage(eku) = ext.parsed_extension() else { + return Err(BgpsecRouterCertificateProfileError::MissingExtendedKeyUsage); + }; + let found = eku + .other + .iter() + .any(|oid| oid.to_id_string() == OID_KP_BGPSEC_ROUTER); + if !found { + return Err(BgpsecRouterCertificateProfileError::MissingBgpsecRouterEku); + } + Ok(()) +} + +fn validate_router_spki(spki_der: &[u8]) -> Result<(), BgpsecRouterCertificateProfileError> { + let (rem, spki) = SubjectPublicKeyInfo::from_der(spki_der) + .map_err(|_| BgpsecRouterCertificateProfileError::SpkiParametersMissingOrInvalid)?; + if !rem.is_empty() { + return Err(BgpsecRouterCertificateProfileError::SpkiParametersMissingOrInvalid); + } + if spki.algorithm.algorithm.to_id_string() != OID_EC_PUBLIC_KEY { + return Err(BgpsecRouterCertificateProfileError::SpkiAlgorithmNotEcPublicKey); + } + let Some(params) = spki.algorithm.parameters.as_ref() else { + return Err(BgpsecRouterCertificateProfileError::SpkiParametersMissingOrInvalid); + }; + if params.header.tag().0 != 0x06 { + return Err(BgpsecRouterCertificateProfileError::SpkiParametersMissingOrInvalid); + } + let mut der = Vec::with_capacity(params.data.len() + 2); + der.push(0x06); + if params.data.len() >= 0x80 { + return Err(BgpsecRouterCertificateProfileError::SpkiParametersMissingOrInvalid); + } + der.push(params.data.len() as u8); + der.extend_from_slice(params.data); + let (prem, oid) = der_parser::der::parse_der_oid(&der) + .map_err(|_| BgpsecRouterCertificateProfileError::SpkiParametersMissingOrInvalid)?; + if !prem.is_empty() { + return Err(BgpsecRouterCertificateProfileError::SpkiParametersMissingOrInvalid); + } + let curve = oid + .as_oid_val() + .map_err(|_| BgpsecRouterCertificateProfileError::SpkiParametersMissingOrInvalid)? + .to_string(); + if curve != OID_SECP256R1 { + return Err(BgpsecRouterCertificateProfileError::SpkiWrongCurve); + } + let parsed = spki + .parsed() + .map_err(|_| BgpsecRouterCertificateProfileError::SpkiEcPointNotUncompressedP256)?; + let PublicKey::EC(ec) = parsed else { + return Err(BgpsecRouterCertificateProfileError::SpkiAlgorithmNotEcPublicKey); + }; + if ec.data().len() != 65 || ec.data().first() != Some(&0x04) { + return Err(BgpsecRouterCertificateProfileError::SpkiEcPointNotUncompressedP256); + } + Ok(()) +} diff --git a/crates/panda-rpki-validator/src/data_model/signed_object.rs b/crates/panda-rpki-validator/src/data_model/signed_object.rs new file mode 100644 index 0000000..524f0c0 --- /dev/null +++ b/crates/panda-rpki-validator/src/data_model/signed_object.rs @@ -0,0 +1,1629 @@ +use crate::data_model::common::{Asn1TimeEncoding, Asn1TimeUtc, DerReader, der_uint_from_bytes}; +use crate::data_model::oid::{ + OID_AD_SIGNED_OBJECT, OID_CMS_ATTR_CONTENT_TYPE, OID_CMS_ATTR_CONTENT_TYPE_RAW, + OID_CMS_ATTR_MESSAGE_DIGEST, OID_CMS_ATTR_MESSAGE_DIGEST_RAW, OID_CMS_ATTR_SIGNING_TIME, + OID_CMS_ATTR_SIGNING_TIME_RAW, OID_CT_ASPA, OID_CT_ASPA_RAW, OID_CT_ROUTE_ORIGIN_AUTHZ, + OID_CT_ROUTE_ORIGIN_AUTHZ_RAW, OID_CT_RPKI_MANIFEST, OID_CT_RPKI_MANIFEST_RAW, + OID_KEY_USAGE_RAW, OID_RSA_ENCRYPTION, OID_RSA_ENCRYPTION_RAW, OID_SHA256, OID_SHA256_RAW, + OID_SHA256_WITH_RSA_ENCRYPTION, OID_SHA256_WITH_RSA_ENCRYPTION_RAW, OID_SIGNED_DATA, + OID_SIGNED_DATA_RAW, OID_SUBJECT_INFO_ACCESS, +}; +use crate::data_model::rc::{ResourceCertificate, ResourceCertificateRole, SubjectInfoAccess}; +use asn1_rs::{Any, Class, FromBer, FromDer as Asn1FromDer, Header, Tag}; +use ring::digest; +use x509_parser::extensions::ParsedExtension; +use x509_parser::prelude::X509Certificate; +use x509_parser::public_key::PublicKey; +use x509_parser::x509::SubjectPublicKeyInfo; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum EeKeyUsageSummary { + DigitalSignatureOnly, + Missing, + NotCritical, + InvalidBits, + ParseError(String), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ResourceEeCertificate { + pub raw_der: Vec, + pub subject_key_identifier: Vec, + pub spki_der: Vec, + pub rsa_public_modulus: Vec, + pub rsa_public_exponent: Vec, + pub tbs_certificate_der: Vec, + pub signature_bytes: Vec, + pub key_usage_summary: EeKeyUsageSummary, + pub sia_signed_object_uris: Vec, + pub resource_cert: ResourceCertificate, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RpkiSignedObject { + pub raw_der: Vec, + pub content_info_content_type: String, + pub signed_data: SignedDataProfiled, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SignedDataProfiled { + pub version: u32, + pub digest_algorithms: Vec, + pub encap_content_info: EncapsulatedContentInfo, + pub certificates: Vec, + pub crls_present: bool, + pub signer_infos: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EncapsulatedContentInfo { + pub econtent_type: String, + pub econtent: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SignerInfoProfiled { + pub version: u32, + pub sid_ski: Vec, + pub digest_algorithm: String, + pub signature_algorithm: String, + pub signed_attrs: SignedAttrsProfiled, + pub unsigned_attrs_present: bool, + pub signature: Vec, + pub signed_attrs_der_for_signature: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SignedAttrsProfiled { + pub content_type: String, + pub message_digest: Vec, + pub signing_time: Asn1TimeUtc, + pub other_attrs_present: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RpkiSignedObjectParsed { + pub raw_der: Vec, + pub content_info_content_type: String, + pub signed_data: SignedDataParsed, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SignedDataParsed { + pub version: u64, + pub digest_algorithms: Vec, + pub encap_content_info: EncapsulatedContentInfoParsed, + pub certificates: Option>>, + pub crls_present: bool, + pub signer_infos: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AlgorithmIdentifierParsed { + pub oid: String, + pub params_ok: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EncapsulatedContentInfoParsed { + pub econtent_type: String, + pub econtent: Option>, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SignerInfoParsed { + pub version: u64, + pub sid: SignerIdentifierParsed, + pub digest_algorithm: AlgorithmIdentifierParsed, + pub signature_algorithm: AlgorithmIdentifierParsed, + pub signed_attrs_content: Option>, + pub signed_attrs_der_for_signature: Option>, + pub unsigned_attrs_present: bool, + pub signature: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum SignerIdentifierParsed { + SubjectKeyIdentifier(Vec), + Other, +} + +#[derive(Debug, thiserror::Error)] +pub enum SignedObjectParseError { + #[error("DER parse error: {0} (RFC 6488 §2; RFC 6488 §3(1l); RFC 5652 §3/§5)")] + Parse(String), + + #[error("trailing bytes after DER object: {0} bytes (DER; RFC 6488 §3(1l))")] + TrailingBytes(usize), +} + +#[derive(Debug, thiserror::Error)] +pub enum SignedObjectValidateError { + #[error( + "ContentInfo.contentType must be SignedData ({OID_SIGNED_DATA}), got {0} (RFC 6488 §3(1a); RFC 5652 §3)" + )] + InvalidContentInfoContentType(String), + + #[error( + "SignedData.version must be 3, got {0} (RFC 6488 §2.1.1; RFC 6488 §3(1b); RFC 5652 §5.1)" + )] + InvalidSignedDataVersion(u64), + + #[error( + "SignedData.digestAlgorithms must contain exactly one AlgorithmIdentifier, got {0} (RFC 6488 §2.1.2; RFC 6488 §3(1b); RFC 5652 §5.1)" + )] + InvalidDigestAlgorithmsCount(usize), + + #[error( + "digest algorithm must be id-sha256 ({OID_SHA256}), got {0} (RFC 6488 §2.1.2; RFC 6488 §3(1b); RFC 7935 §2)" + )] + InvalidDigestAlgorithm(String), + + #[error("SignedData.certificates MUST be present (RFC 6488 §3(1c); RFC 5652 §5.1)")] + CertificatesMissing, + + #[error( + "SignedData.certificates must contain exactly one EE certificate, got {0} (RFC 6488 §3(1c))" + )] + InvalidCertificatesCount(usize), + + #[error("SignedData.crls MUST be omitted (RFC 6488 §3(1d))")] + CrlsPresent, + + #[error( + "SignedData.signerInfos must contain exactly one SignerInfo, got {0} (RFC 6488 §2.1; RFC 6488 §3(1e); RFC 5652 §5.1)" + )] + InvalidSignerInfosCount(usize), + + #[error("SignerInfo.version must be 3, got {0} (RFC 6488 §3(1e); RFC 5652 §5.3)")] + InvalidSignerInfoVersion(u64), + + #[error("SignerInfo.sid must be subjectKeyIdentifier [0] (RFC 6488 §3(1c); RFC 5652 §5.3)")] + InvalidSignerIdentifier, + + #[error( + "SignerInfo.digestAlgorithm must be id-sha256 ({OID_SHA256}), got {0} (RFC 6488 §3(1j); RFC 7935 §2)" + )] + InvalidSignerInfoDigestAlgorithm(String), + + #[error("SignerInfo.signedAttrs MUST be present (RFC 9589 §4; RFC 6488 §3(1f))")] + SignedAttrsMissing, + + #[error("SignerInfo.unsignedAttrs MUST be omitted (RFC 6488 §3(1i))")] + UnsignedAttrsPresent, + + #[error( + "SignerInfo.signatureAlgorithm must be rsaEncryption ({OID_RSA_ENCRYPTION}) or \ +sha256WithRSAEncryption ({OID_SHA256_WITH_RSA_ENCRYPTION}), got {0} (RFC 6488 §3(1k); RFC 7935 §2)" + )] + InvalidSignatureAlgorithm(String), + + #[error( + "SignerInfo.signatureAlgorithm parameters must be absent or NULL (RFC 5280 §4.1.1.2; RFC 7935 §2)" + )] + InvalidSignatureAlgorithmParameters, + + #[error("signedAttrs contains unsupported attribute OID {0} (RFC 9589 §4; RFC 6488 §2.1.6.4)")] + UnsupportedSignedAttribute(String), + + #[error("signedAttrs contains duplicate attribute OID {0} (RFC 6488 §2.1.6.4; RFC 9589 §4)")] + DuplicateSignedAttribute(String), + + #[error("signedAttrs parse error: {0} (RFC 5652 §5.3; RFC 6488 §3(1f); RFC 9589 §4)")] + SignedAttrsParse(String), + + #[error( + "signedAttrs attribute {oid} attrValues must contain exactly one value, got {count} (RFC 6488 §2.1.6.4; RFC 5652 §5.3)" + )] + InvalidSignedAttributeValuesCount { oid: String, count: usize }, + + #[error( + "signedAttrs missing content-type attribute (RFC 9589 §4; RFC 5652 §11.1; RFC 6488 §2.1.6.4)" + )] + SignedAttrsContentTypeMissing, + + #[error( + "signedAttrs missing message-digest attribute (RFC 9589 §4; RFC 5652 §11.2; RFC 6488 §2.1.6.4)" + )] + SignedAttrsMessageDigestMissing, + + #[error( + "signedAttrs missing signing-time attribute (RFC 9589 §4; RFC 5652 §11.3; RFC 6488 §2.1.6.4)" + )] + SignedAttrsSigningTimeMissing, + + #[error( + "signedAttrs.content-type attrValues must equal eContentType ({econtent_type}), got {attr_content_type} (RFC 6488 §3(1h); RFC 9589 §4)" + )] + ContentTypeAttrMismatch { + econtent_type: String, + attr_content_type: String, + }, + + #[error("EncapsulatedContentInfo.eContent MUST be present (RFC 6488 §2.1.3; RFC 5652 §5.2)")] + EContentMissing, + + #[error( + "signedAttrs.message-digest does not match SHA-256(eContent) (RFC 6488 §3(1f); RFC 5652 §11.2)" + )] + MessageDigestMismatch, + + #[error("EE certificate parse error: {0} (RFC 6488 §3(1c); RFC 6487 §4)")] + EeCertificateParse(String), + + #[error( + "EE certificate missing SubjectKeyIdentifier extension (RFC 6488 §3(1c); RFC 6487 §4.8.2)" + )] + EeCertificateMissingSki, + + #[error( + "EE certificate missing SubjectInfoAccess extension ({OID_SUBJECT_INFO_ACCESS}) (RFC 6487 §4.8.8.2)" + )] + EeCertificateMissingSia, + + #[error( + "EE certificate SIA missing id-ad-signedObject access method ({OID_AD_SIGNED_OBJECT}) (RFC 6487 §4.8.8.2)" + )] + EeCertificateMissingSignedObjectSia, + + #[error( + "EE certificate SIA id-ad-signedObject accessLocation must be a URI (RFC 6487 §4.8.8.2; RFC 5280 §4.2.2.2)" + )] + EeCertificateSignedObjectSiaNotUri, + + #[error( + "EE certificate SIA id-ad-signedObject must include at least one rsync:// URI (RFC 6487 §4.8.8.2)" + )] + EeCertificateSignedObjectSiaNoRsync, + + #[error( + "SignerInfo.sid SKI does not match EE certificate SKI (RFC 6488 §3(1c); RFC 5652 §5.3)" + )] + SidSkiMismatch, + + #[error( + "invalid signing-time attribute value (expected UTCTime or GeneralizedTime) (RFC 5652 §11.3; RFC 9589 §4)" + )] + InvalidSigningTimeValue, +} + +#[derive(Debug, thiserror::Error)] +pub enum SignedObjectDecodeError { + #[error("SignedObject parse error: {0}")] + Parse(#[from] SignedObjectParseError), + + #[error("SignedObject validate error: {0}")] + Validate(#[from] SignedObjectValidateError), +} + +#[derive(Debug, thiserror::Error)] +pub enum SignedObjectVerifyError { + #[error("EE SubjectPublicKeyInfo parse error: {0} (RFC 5280 §4.1.2.7)")] + EeSpkiParse(String), + + #[error("trailing bytes after EE SubjectPublicKeyInfo DER: {0} bytes (DER; RFC 5280 §4.1.2.7)")] + EeSpkiTrailingBytes(usize), + + #[error("unsupported EE public key algorithm (only RSA supported in M3) (RFC 7935 §2)")] + UnsupportedEePublicKeyAlgorithm, + + #[error("EE RSA public exponent invalid (RFC 8017 §A.1.1; RFC 7935 §2)")] + InvalidEeRsaExponent, + + #[error("signature verification failed (RFC 6488 §3(2)-(3); RFC 5652 §5.3; RFC 7935 §2)")] + InvalidSignature, +} + +impl RpkiSignedObject { + /// Parse a DER-encoded RPKI Signed Object (CMS ContentInfo wrapping SignedData). + /// + /// This performs encoding/structure parsing only. Profile constraints are enforced by + /// `RpkiSignedObjectParsed::validate_profile`. + pub fn parse_der(der: &[u8]) -> Result { + parse_signed_object_content_info(der, der, CmsParseMode::BerCompatible) + } + + pub fn parse_der_strict_cms( + der: &[u8], + ) -> Result { + parse_signed_object_content_info(der, der, CmsParseMode::DerStrict) + } + + /// Return the strict-DER CMS parse error for an object that was otherwise + /// accepted through the normal BER-compatible CMS parser. + /// + /// Callers must only surface this as a compatibility warning after normal + /// decoding and validation have succeeded; a strict parse failure alone + /// does not prove that an arbitrary byte string is an RPKI signed object. + pub fn strict_cms_der_error(der: &[u8]) -> Option { + Self::parse_der_strict_cms(der).err() + } + + /// Decode a DER-encoded RPKI Signed Object (CMS ContentInfo wrapping SignedData) and enforce + /// the profile constraints from RFC 6488 §2-§3 and RFC 9589 §4. + pub fn decode_der(der: &[u8]) -> Result { + let parsed = Self::parse_der(der)?; + Ok(parsed.validate_profile()?) + } + + pub fn decode_der_with_strict_options( + der: &[u8], + strict_cms_der: bool, + strict_name: bool, + ) -> Result { + let parsed = if strict_cms_der { + Self::parse_der_strict_cms(der)? + } else { + Self::parse_der(der)? + }; + Ok(parsed.validate_profile_with_strict_name(strict_name)?) + } + + /// Scheme-A naming for signature verification. + pub fn verify(&self) -> Result<(), SignedObjectVerifyError> { + self.verify_signature() + } + + /// Verify the CMS signature using the embedded EE certificate public key. + pub fn verify_signature(&self) -> Result<(), SignedObjectVerifyError> { + let ee = &self.signed_data.certificates[0]; + let signer = &self.signed_data.signer_infos[0]; + crate::crypto_sig_cache::verify_with_cache( + crate::crypto_sig_cache::CryptoSigVerifyPoint::CmsSignedObject, + &signer.signed_attrs_der_for_signature, + &signer.signature, + &ee.spki_der, + || { + self.verify_signature_with_rsa_components( + &ee.rsa_public_modulus, + &ee.rsa_public_exponent, + ) + }, + ) + } + + /// Verify the CMS signature using a DER-encoded SubjectPublicKeyInfo. + pub fn verify_signature_with_ee_spki_der( + &self, + ee_spki_der: &[u8], + ) -> Result<(), SignedObjectVerifyError> { + let (rem, spki) = SubjectPublicKeyInfo::from_der(ee_spki_der) + .map_err(|e| SignedObjectVerifyError::EeSpkiParse(e.to_string()))?; + if !rem.is_empty() { + return Err(SignedObjectVerifyError::EeSpkiTrailingBytes(rem.len())); + } + self.verify_signature_with_ee_spki(&spki) + } + + /// Verify the CMS signature using a parsed SubjectPublicKeyInfo. + pub fn verify_signature_with_ee_spki( + &self, + ee_spki: &SubjectPublicKeyInfo<'_>, + ) -> Result<(), SignedObjectVerifyError> { + let pk = ee_spki + .parsed() + .map_err(|_e| SignedObjectVerifyError::UnsupportedEePublicKeyAlgorithm)?; + let (n, e) = match pk { + PublicKey::RSA(rsa) => { + let n = strip_leading_zeros(rsa.modulus).to_vec(); + let e = strip_leading_zeros(rsa.exponent).to_vec(); + let _exp = rsa + .try_exponent() + .map_err(|_e| SignedObjectVerifyError::InvalidEeRsaExponent)?; + (n, e) + } + _ => return Err(SignedObjectVerifyError::UnsupportedEePublicKeyAlgorithm), + }; + + let signer = &self.signed_data.signer_infos[0]; + crate::crypto_sig_cache::verify_with_cache( + crate::crypto_sig_cache::CryptoSigVerifyPoint::CmsSignedObject, + &signer.signed_attrs_der_for_signature, + &signer.signature, + ee_spki.raw, + || self.verify_signature_with_rsa_components(n.as_slice(), e.as_slice()), + ) + } + + fn verify_signature_with_rsa_components( + &self, + modulus: &[u8], + exponent: &[u8], + ) -> Result<(), SignedObjectVerifyError> { + let signer = &self.signed_data.signer_infos[0]; + let msg = &signer.signed_attrs_der_for_signature; + + let pk = ring::signature::RsaPublicKeyComponents { + n: modulus, + e: exponent, + }; + pk.verify( + &ring::signature::RSA_PKCS1_2048_8192_SHA256, + msg, + &signer.signature, + ) + .map_err(|_e| SignedObjectVerifyError::InvalidSignature) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CmsParseMode { + BerCompatible, + DerStrict, +} + +struct CmsReader<'a> { + buf: &'a [u8], + mode: CmsParseMode, +} + +impl<'a> CmsReader<'a> { + fn new(buf: &'a [u8], mode: CmsParseMode) -> Self { + Self { buf, mode } + } + + fn is_empty(&self) -> bool { + self.buf.is_empty() + } + + fn remaining_len(&self) -> usize { + self.buf.len() + } + + fn peek_tag(&self) -> Result { + let (_rem, any) = parse_any(self.buf, self.mode)?; + header_to_single_byte_tag(&any.header) + } + + fn take_any(&mut self) -> Result<(u8, &'a [u8]), String> { + let (rem, any) = parse_any(self.buf, self.mode)?; + let tag = header_to_single_byte_tag(&any.header)?; + self.buf = rem; + Ok((tag, any.data)) + } + + fn take_any_full(&mut self) -> Result<(u8, &'a [u8], &'a [u8]), String> { + let (rem, any) = parse_any(self.buf, self.mode)?; + let consumed = self.buf.len() - rem.len(); + let full = &self.buf[..consumed]; + let tag = header_to_single_byte_tag(&any.header)?; + self.buf = rem; + Ok((tag, full, any.data)) + } + + fn skip_any(&mut self) -> Result<(), String> { + let _ = self.take_any()?; + Ok(()) + } + + fn take_tag(&mut self, expected_tag: u8) -> Result<&'a [u8], String> { + let (tag, value) = self.take_any()?; + if tag != expected_tag { + return Err(format!( + "unexpected tag: got 0x{tag:02X}, expected 0x{expected_tag:02X}" + )); + } + Ok(value) + } + + fn take_sequence(&mut self) -> Result, String> { + let value = self.take_tag(0x30)?; + Ok(CmsReader::new(value, self.mode)) + } + + fn take_octet_string(&mut self) -> Result, String> { + let (rem, any) = parse_any(self.buf, self.mode)?; + let tag = header_to_single_byte_tag(&any.header)?; + if self.mode == CmsParseMode::DerStrict && tag != 0x04 { + return Err(format!( + "unexpected tag in DER strict mode: got 0x{tag:02X}, expected 0x04" + )); + } + if tag != 0x04 && tag != 0x24 { + return Err(format!("unexpected tag: got 0x{tag:02X}, expected 0x04")); + } + let octets = flatten_octet_string(any, self.mode)?; + self.buf = rem; + Ok(octets) + } + + fn take_uint_u64(&mut self) -> Result { + let value = self.take_tag(0x02)?; + der_uint_from_bytes(value) + } + + fn take_explicit(&mut self, expected_outer_tag: u8) -> Result<(u8, &'a [u8]), String> { + let inner_der = self.take_tag(expected_outer_tag)?; + let (tag, value, rem) = cms_take_tlv(inner_der, self.mode)?; + if !rem.is_empty() { + return Err("trailing bytes inside EXPLICIT value".into()); + } + Ok((tag, value)) + } + + fn take_explicit_der(&mut self, expected_outer_tag: u8) -> Result<&'a [u8], String> { + let inner_der = self.take_tag(expected_outer_tag)?; + let (_tag, _value, rem) = cms_take_tlv(inner_der, self.mode)?; + if !rem.is_empty() { + return Err("trailing bytes inside EXPLICIT value".into()); + } + Ok(inner_der) + } +} + +fn parse_signed_object_content_info( + raw_der: &[u8], + parse_der: &[u8], + mode: CmsParseMode, +) -> Result { + let mut r = CmsReader::new(parse_der, mode); + let mut content_info_seq = r + .take_sequence() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + if !r.is_empty() { + return Err(SignedObjectParseError::TrailingBytes(r.remaining_len())); + } + + let content_type = take_oid_string(&mut content_info_seq)?; + let signed_data = parse_signed_data_from_contentinfo_cursor(&mut content_info_seq)?; + if !content_info_seq.is_empty() { + return Err(SignedObjectParseError::Parse( + "ContentInfo must be a SEQUENCE of 2 elements".into(), + )); + } + + Ok(RpkiSignedObjectParsed { + raw_der: raw_der.to_vec(), + content_info_content_type: content_type, + signed_data, + }) +} + +fn parse_any<'a>(input: &'a [u8], mode: CmsParseMode) -> Result<(&'a [u8], Any<'a>), String> { + match mode { + CmsParseMode::BerCompatible => { + Any::from_ber(input).map_err(|e| format!("BER parse error: {e}")) + } + CmsParseMode::DerStrict => { + Any::from_der(input).map_err(|e| format!("DER parse error: {e}")) + } + } +} + +fn header_to_single_byte_tag(header: &Header<'_>) -> Result { + let tag_no = header.tag().0; + if tag_no > 30 { + return Err(format!("high-tag-number form not supported: {tag_no}")); + } + Ok(((header.class() as u8) << 6) + | if header.constructed() { 0x20 } else { 0x00 } + | tag_no as u8) +} + +fn cms_take_tlv(input: &[u8], mode: CmsParseMode) -> Result<(u8, &[u8], &[u8]), String> { + let (rem, any) = parse_any(input, mode)?; + let tag = header_to_single_byte_tag(&any.header)?; + Ok((tag, any.data, rem)) +} + +fn flatten_octet_string(any: Any<'_>, mode: CmsParseMode) -> Result, String> { + if any.class() != Class::Universal || any.tag() != Tag::OctetString { + return Err("expected OCTET STRING".into()); + } + if !any.header.constructed() { + return Ok(any.data.to_vec()); + } + if mode == CmsParseMode::DerStrict { + return Err("constructed OCTET STRING is not allowed in DER strict mode".into()); + } + let mut out = Vec::new(); + let mut input = any.data; + while !input.is_empty() { + let (rem, child) = Any::from_ber(input).map_err(|e| format!("BER parse error: {e}"))?; + out.extend(flatten_octet_string(child, mode)?); + input = rem; + } + Ok(out) +} + +impl RpkiSignedObjectParsed { + pub fn validate_profile(self) -> Result { + self.validate_profile_with_strict_name(false) + } + + pub fn validate_profile_with_strict_name( + self, + strict_name: bool, + ) -> Result { + if self.content_info_content_type != OID_SIGNED_DATA { + return Err(SignedObjectValidateError::InvalidContentInfoContentType( + self.content_info_content_type, + )); + } + + let signed_data = validate_signed_data_profile(self.signed_data, strict_name)?; + + Ok(RpkiSignedObject { + raw_der: self.raw_der, + content_info_content_type: OID_SIGNED_DATA.to_string(), + signed_data, + }) + } +} + +fn parse_signed_data_from_contentinfo_cursor( + seq: &mut CmsReader<'_>, +) -> Result { + let inner_der = seq.take_explicit_der(0xA0).map_err(|_e| { + SignedObjectParseError::Parse("ContentInfo.content must be [0] EXPLICIT".into()) + })?; + let mut r = CmsReader::new(inner_der, seq.mode); + let signed_data_seq = r + .take_sequence() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + if !r.is_empty() { + return Err(SignedObjectParseError::Parse( + "trailing bytes inside ContentInfo.content".into(), + )); + } + parse_signed_data_cursor(signed_data_seq) +} + +fn parse_signed_data_cursor( + mut seq: CmsReader<'_>, +) -> Result { + let version = seq + .take_uint_u64() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + + let digest_set_bytes = seq + .take_tag(0x31) + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + let mut digest_set = CmsReader::new(digest_set_bytes, seq.mode); + let mut digest_algorithms: Vec = Vec::new(); + while !digest_set.is_empty() { + let alg = digest_set + .take_sequence() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + let (oid, params_ok) = parse_algorithm_identifier_cursor(alg)?; + digest_algorithms.push(AlgorithmIdentifierParsed { oid, params_ok }); + } + + let encap_content_info = parse_encapsulated_content_info_cursor( + seq.take_sequence() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?, + )?; + + let mut certificates: Option>> = None; + let mut crls_present = false; + let mut signer_infos: Option> = None; + + while !seq.is_empty() { + let tag = seq + .peek_tag() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + match tag { + 0xA0 => { + if certificates.is_some() { + return Err(SignedObjectParseError::Parse( + "SignedData.certificates appears more than once".into(), + )); + } + let content = seq + .take_tag(0xA0) + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + certificates = Some(split_der_objects(content, seq.mode)?); + } + 0xA1 => { + crls_present = true; + seq.skip_any() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + } + 0x31 => { + if signer_infos.is_some() { + return Err(SignedObjectParseError::Parse( + "SignedData.signerInfos appears more than once".into(), + )); + } + let set_bytes = seq + .take_tag(0x31) + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + signer_infos = Some(parse_signer_infos_set_cursor(set_bytes, seq.mode)?); + } + _ => { + return Err(SignedObjectParseError::Parse( + "unexpected field in SignedData".into(), + )); + } + } + } + + let signer_infos = signer_infos + .ok_or_else(|| SignedObjectParseError::Parse("SignedData.signerInfos missing".into()))?; + + Ok(SignedDataParsed { + version, + digest_algorithms, + encap_content_info, + certificates, + crls_present, + signer_infos, + }) +} + +fn parse_encapsulated_content_info_cursor( + mut seq: CmsReader<'_>, +) -> Result { + if seq.is_empty() { + return Err(SignedObjectParseError::Parse( + "EncapsulatedContentInfo must be SEQUENCE of 1..2".into(), + )); + } + + let econtent_type = take_oid_string(&mut seq)?; + + let econtent = if seq.is_empty() { + None + } else { + let inner_der = seq.take_explicit_der(0xA0).map_err(|_e| { + SignedObjectParseError::Parse( + "EncapsulatedContentInfo.eContent must be [0] EXPLICIT".into(), + ) + })?; + let mut inner = CmsReader::new(inner_der, seq.mode); + let octets = inner + .take_octet_string() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + if !inner.is_empty() { + return Err(SignedObjectParseError::Parse( + "trailing bytes inside EncapsulatedContentInfo.eContent".into(), + )); + } + Some(octets) + }; + if !seq.is_empty() { + return Err(SignedObjectParseError::Parse( + "EncapsulatedContentInfo must be SEQUENCE of 1..2".into(), + )); + } + + Ok(EncapsulatedContentInfoParsed { + econtent_type, + econtent, + }) +} + +fn split_der_objects( + mut input: &[u8], + mode: CmsParseMode, +) -> Result>, SignedObjectParseError> { + let mut out: Vec> = Vec::new(); + while !input.is_empty() { + let (_tag, _value, rem) = + cms_take_tlv(input, mode).map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + let consumed = input.len() - rem.len(); + out.push(input[..consumed].to_vec()); + input = rem; + } + Ok(out) +} + +fn parse_signer_infos_set_cursor( + set_bytes: &[u8], + mode: CmsParseMode, +) -> Result, SignedObjectParseError> { + let mut set = CmsReader::new(set_bytes, mode); + let mut out: Vec = Vec::new(); + while !set.is_empty() { + let si = set + .take_sequence() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + out.push(parse_signer_info_cursor(si)?); + } + Ok(out) +} + +fn validate_ee_certificate( + der: &[u8], + strict_name: bool, +) -> Result { + let (rem, cert) = X509Certificate::from_der(der) + .map_err(|e| SignedObjectValidateError::EeCertificateParse(e.to_string()))?; + if !rem.is_empty() { + return Err(SignedObjectValidateError::EeCertificateParse(format!( + "trailing bytes after EE certificate DER: {}", + rem.len() + ))); + } + + let rc = match ResourceCertificate::from_der(der) { + Ok(v) => v, + Err(e) => { + return match e { + crate::data_model::rc::ResourceCertificateDecodeError::Validate( + crate::data_model::rc::ResourceCertificateProfileError::SignedObjectSiaNotUri, + ) => Err(SignedObjectValidateError::EeCertificateSignedObjectSiaNotUri), + crate::data_model::rc::ResourceCertificateDecodeError::Validate( + crate::data_model::rc::ResourceCertificateProfileError::SignedObjectSiaNoRsync, + ) => Err(SignedObjectValidateError::EeCertificateSignedObjectSiaNoRsync), + _ => Err(SignedObjectValidateError::EeCertificateParse(e.to_string())), + }; + } + }; + rc.validate_rfc6487_profile(ResourceCertificateRole::SignedObjectEe) + .map_err(|e| SignedObjectValidateError::EeCertificateParse(e.to_string()))?; + if strict_name { + rc.validate_strict_name_profile() + .map_err(|e| SignedObjectValidateError::EeCertificateParse(e.to_string()))?; + } + + let ski = rc + .tbs + .extensions + .subject_key_identifier + .clone() + .ok_or(SignedObjectValidateError::EeCertificateMissingSki)?; + + let spki_der = rc.tbs.subject_public_key_info.clone(); + let (rem, spki) = SubjectPublicKeyInfo::from_der(&spki_der) + .map_err(|e| SignedObjectValidateError::EeCertificateParse(e.to_string()))?; + if !rem.is_empty() { + return Err(SignedObjectValidateError::EeCertificateParse( + "trailing bytes after EE SubjectPublicKeyInfo DER".to_string(), + )); + } + let parsed_pk = spki.parsed().map_err(|_e| { + SignedObjectValidateError::EeCertificateParse( + "unsupported EE public key algorithm".to_string(), + ) + })?; + let (rsa_public_modulus, rsa_public_exponent) = match parsed_pk { + PublicKey::RSA(rsa) => { + let modulus = strip_leading_zeros(rsa.modulus).to_vec(); + let exponent = strip_leading_zeros(rsa.exponent).to_vec(); + let _ = rsa.try_exponent().map_err(|_e| { + SignedObjectValidateError::EeCertificateParse("invalid EE RSA exponent".to_string()) + })?; + (modulus, exponent) + } + _ => { + return Err(SignedObjectValidateError::EeCertificateParse( + "unsupported EE public key algorithm".to_string(), + )); + } + }; + + let sia = rc + .tbs + .extensions + .subject_info_access + .as_ref() + .ok_or(SignedObjectValidateError::EeCertificateMissingSia)?; + let signed_object_uris: Vec = match sia { + SubjectInfoAccess::Ee(ee) => ee.signed_object_uris.clone(), + SubjectInfoAccess::Ca(_ca) => Vec::new(), + }; + if signed_object_uris.is_empty() { + return Err(SignedObjectValidateError::EeCertificateMissingSignedObjectSia); + } + if !signed_object_uris.iter().any(|u| u.starts_with("rsync://")) { + return Err(SignedObjectValidateError::EeCertificateSignedObjectSiaNoRsync); + } + + Ok(ResourceEeCertificate { + raw_der: der.to_vec(), + subject_key_identifier: ski, + spki_der, + rsa_public_modulus, + rsa_public_exponent, + tbs_certificate_der: cert.tbs_certificate.as_ref().to_vec(), + signature_bytes: cert.signature_value.data.to_vec(), + key_usage_summary: summarize_ee_key_usage(&cert), + sia_signed_object_uris: signed_object_uris, + resource_cert: rc, + }) +} + +fn summarize_ee_key_usage(cert: &X509Certificate<'_>) -> EeKeyUsageSummary { + for ext in cert.extensions() { + if ext.oid.as_bytes() == OID_KEY_USAGE_RAW { + match ext.parsed_extension() { + ParsedExtension::KeyUsage(ku) => { + if !ext.critical { + return EeKeyUsageSummary::NotCritical; + } + let ok = ku.digital_signature() + && !ku.key_cert_sign() + && !ku.crl_sign() + && !ku.non_repudiation() + && !ku.key_encipherment() + && !ku.data_encipherment() + && !ku.key_agreement() + && !ku.encipher_only() + && !ku.decipher_only(); + return if ok { + EeKeyUsageSummary::DigitalSignatureOnly + } else { + EeKeyUsageSummary::InvalidBits + }; + } + other => { + return EeKeyUsageSummary::ParseError(format!( + "unexpected parsed keyUsage extension: {other:?}" + )); + } + } + } + } + + EeKeyUsageSummary::Missing +} + +fn parse_signer_info_cursor( + mut seq: CmsReader<'_>, +) -> Result { + let version = seq + .take_uint_u64() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + + let (sid_tag, sid_bytes) = seq + .take_any() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + let sid = if (sid_tag & 0xC0) == 0x80 && (sid_tag & 0x1F) == 0 { + SignerIdentifierParsed::SubjectKeyIdentifier(sid_bytes.to_vec()) + } else { + SignerIdentifierParsed::Other + }; + + let digest_alg_seq = seq + .take_sequence() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + let (digest_oid, digest_params_ok) = parse_algorithm_identifier_cursor(digest_alg_seq)?; + let digest_algorithm = AlgorithmIdentifierParsed { + oid: digest_oid, + params_ok: digest_params_ok, + }; + + let mut signed_attrs_content: Option> = None; + let mut signed_attrs_der_for_signature: Option> = None; + + if seq + .peek_tag() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))? + == 0xA0 + { + let (tag, full_tlv, value) = seq + .take_any_full() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + if tag != 0xA0 { + return Err(SignedObjectParseError::Parse( + "SignerInfo.signedAttrs must be [0] IMPLICIT".into(), + )); + } + signed_attrs_content = Some(value.to_vec()); + signed_attrs_der_for_signature = Some(make_signed_attrs_der_for_signature(full_tlv)?); + } + + let sig_alg_seq = seq + .take_sequence() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + let (signature_oid, signature_params_ok) = parse_algorithm_identifier_cursor(sig_alg_seq)?; + let signature_algorithm = AlgorithmIdentifierParsed { + oid: signature_oid, + params_ok: signature_params_ok, + }; + + let signature = seq + .take_octet_string() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))? + .to_vec(); + + let unsigned_attrs_present = !seq.is_empty(); + + Ok(SignerInfoParsed { + version, + sid, + digest_algorithm, + signature_algorithm, + signed_attrs_content, + signed_attrs_der_for_signature, + unsigned_attrs_present, + signature, + }) +} + +fn validate_signed_data_profile( + signed_data: SignedDataParsed, + strict_name: bool, +) -> Result { + if signed_data.version != 3 { + return Err(SignedObjectValidateError::InvalidSignedDataVersion( + signed_data.version, + )); + } + + if signed_data.digest_algorithms.len() != 1 { + return Err(SignedObjectValidateError::InvalidDigestAlgorithmsCount( + signed_data.digest_algorithms.len(), + )); + } + let digest_alg = &signed_data.digest_algorithms[0]; + if digest_alg.oid != OID_SHA256 { + return Err(SignedObjectValidateError::InvalidDigestAlgorithm( + digest_alg.oid.clone(), + )); + } + + if signed_data.crls_present { + return Err(SignedObjectValidateError::CrlsPresent); + } + + let econtent = signed_data + .encap_content_info + .econtent + .clone() + .ok_or(SignedObjectValidateError::EContentMissing)?; + if econtent.is_empty() { + return Err(SignedObjectValidateError::EContentMissing); + } + let encap_content_info = EncapsulatedContentInfo { + econtent_type: signed_data.encap_content_info.econtent_type.clone(), + econtent: econtent.clone(), + }; + + let certs = signed_data + .certificates + .as_ref() + .ok_or(SignedObjectValidateError::CertificatesMissing)?; + if certs.len() != 1 { + return Err(SignedObjectValidateError::InvalidCertificatesCount( + certs.len(), + )); + } + let ee = validate_ee_certificate(&certs[0], strict_name)?; + + if signed_data.signer_infos.len() != 1 { + return Err(SignedObjectValidateError::InvalidSignerInfosCount( + signed_data.signer_infos.len(), + )); + } + let signer = &signed_data.signer_infos[0]; + + if signer.version != 3 { + return Err(SignedObjectValidateError::InvalidSignerInfoVersion( + signer.version, + )); + } + let sid_ski = match &signer.sid { + SignerIdentifierParsed::SubjectKeyIdentifier(ski) => ski.clone(), + SignerIdentifierParsed::Other => { + return Err(SignedObjectValidateError::InvalidSignerIdentifier); + } + }; + + if signer.digest_algorithm.oid != OID_SHA256 { + return Err(SignedObjectValidateError::InvalidSignerInfoDigestAlgorithm( + signer.digest_algorithm.oid.clone(), + )); + } + + let signed_attrs_content = signer + .signed_attrs_content + .as_deref() + .ok_or(SignedObjectValidateError::SignedAttrsMissing)?; + let signed_attrs_der_for_signature = signer + .signed_attrs_der_for_signature + .clone() + .ok_or(SignedObjectValidateError::SignedAttrsMissing)?; + let signed_attrs = parse_signed_attrs_implicit(signed_attrs_content)?; + + if signer.unsigned_attrs_present { + return Err(SignedObjectValidateError::UnsignedAttrsPresent); + } + + if !signer.signature_algorithm.params_ok { + return Err(SignedObjectValidateError::InvalidSignatureAlgorithmParameters); + } + let signature_algorithm = signer.signature_algorithm.oid.clone(); + if signature_algorithm != OID_RSA_ENCRYPTION + && signature_algorithm != OID_SHA256_WITH_RSA_ENCRYPTION + { + return Err(SignedObjectValidateError::InvalidSignatureAlgorithm( + signature_algorithm, + )); + } + + if sid_ski != ee.subject_key_identifier { + return Err(SignedObjectValidateError::SidSkiMismatch); + } + if signed_attrs.content_type != encap_content_info.econtent_type { + return Err(SignedObjectValidateError::ContentTypeAttrMismatch { + econtent_type: encap_content_info.econtent_type.clone(), + attr_content_type: signed_attrs.content_type.clone(), + }); + } + + let computed = digest::digest(&digest::SHA256, &encap_content_info.econtent); + if computed.as_ref() != signed_attrs.message_digest.as_slice() { + return Err(SignedObjectValidateError::MessageDigestMismatch); + } + + Ok(SignedDataProfiled { + version: 3, + digest_algorithms: vec![OID_SHA256.to_string()], + encap_content_info, + certificates: vec![ee.clone()], + crls_present: false, + signer_infos: vec![SignerInfoProfiled { + version: 3, + sid_ski, + digest_algorithm: OID_SHA256.to_string(), + signature_algorithm: signer.signature_algorithm.oid.clone(), + signed_attrs, + unsigned_attrs_present: false, + signature: signer.signature.clone(), + signed_attrs_der_for_signature, + }], + }) +} + +fn parse_signed_attrs_implicit( + input: &[u8], +) -> Result { + let mut content_type: Option = None; + let mut message_digest: Option> = None; + let mut signing_time: Option = None; + + fn count_elements(mut r: DerReader<'_>) -> Result { + let mut n = 0usize; + while !r.is_empty() { + r.skip_any()?; + n += 1; + } + Ok(n) + } + + let mut remaining = DerReader::new(input); + while !remaining.is_empty() { + let mut attr = remaining + .take_sequence() + .map_err(|e| SignedObjectValidateError::SignedAttrsParse(e.to_string()))?; + + let oid_bytes = attr + .take_tag(0x06) + .map_err(|e| SignedObjectValidateError::SignedAttrsParse(e.to_string()))?; + let oid = oid_value_bytes_to_string(oid_bytes); + + let values_bytes = attr + .take_tag(0x31) + .map_err(|e| SignedObjectValidateError::SignedAttrsParse(e.to_string()))?; + if !attr.is_empty() { + return Err(SignedObjectValidateError::SignedAttrsParse( + "Attribute must be SEQUENCE of 2".into(), + )); + } + + let mut values = DerReader::new(values_bytes); + let count = if values.is_empty() { + 0 + } else { + values + .skip_any() + .map_err(|e| SignedObjectValidateError::SignedAttrsParse(e.to_string()))?; + if values.is_empty() { + 1 + } else { + 1 + count_elements(values) + .map_err(|e| SignedObjectValidateError::SignedAttrsParse(e.to_string()))? + } + }; + if count != 1 { + return Err( + SignedObjectValidateError::InvalidSignedAttributeValuesCount { oid, count }, + ); + } + + // Re-parse the sole value. + let mut values = DerReader::new(values_bytes); + let (val_tag, val_bytes) = values + .take_any() + .map_err(|e| SignedObjectValidateError::SignedAttrsParse(e.to_string()))?; + + match oid.as_str() { + OID_CMS_ATTR_CONTENT_TYPE => { + if content_type.is_some() { + return Err(SignedObjectValidateError::DuplicateSignedAttribute(oid)); + } + if val_tag != 0x06 { + return Err(SignedObjectValidateError::SignedAttrsParse( + "content-type attr value must be OBJECT IDENTIFIER".into(), + )); + } + content_type = Some(oid_value_bytes_to_string(val_bytes)); + } + OID_CMS_ATTR_MESSAGE_DIGEST => { + if message_digest.is_some() { + return Err(SignedObjectValidateError::DuplicateSignedAttribute(oid)); + } + if val_tag != 0x04 { + return Err(SignedObjectValidateError::SignedAttrsParse( + "message-digest attr value must be OCTET STRING".into(), + )); + } + message_digest = Some(val_bytes.to_vec()); + } + OID_CMS_ATTR_SIGNING_TIME => { + if signing_time.is_some() { + return Err(SignedObjectValidateError::DuplicateSignedAttribute(oid)); + } + signing_time = Some(parse_signing_time_value_tlv(val_tag, val_bytes)?); + } + _ => { + return Err(SignedObjectValidateError::UnsupportedSignedAttribute(oid)); + } + } + } + + Ok(SignedAttrsProfiled { + content_type: content_type + .ok_or(SignedObjectValidateError::SignedAttrsContentTypeMissing)?, + message_digest: message_digest + .ok_or(SignedObjectValidateError::SignedAttrsMessageDigestMissing)?, + signing_time: signing_time + .ok_or(SignedObjectValidateError::SignedAttrsSigningTimeMissing)?, + other_attrs_present: false, + }) +} + +fn parse_signing_time_value_tlv( + tag: u8, + value: &[u8], +) -> Result { + match tag { + 0x17 => Ok(Asn1TimeUtc { + utc: parse_utctime(value)?, + encoding: Asn1TimeEncoding::UtcTime, + }), + 0x18 => Ok(Asn1TimeUtc { + utc: parse_generalized_time(value)?, + encoding: Asn1TimeEncoding::GeneralizedTime, + }), + _ => Err(SignedObjectValidateError::InvalidSigningTimeValue), + } +} + +fn make_signed_attrs_der_for_signature(full_tlv: &[u8]) -> Result, SignedObjectParseError> { + // We need the DER encoding of SignedAttributes (SET OF Attribute) as signature input. + // The SignedAttributes field in SignerInfo is `[0] IMPLICIT`, so the on-wire bytes start with + // a context-specific constructed tag (0xA0 for tag 0). For signature verification, this tag + // is replaced with the universal SET tag (0x31), leaving length+content unchanged. + // + let mut cs_der = full_tlv.to_vec(); + if cs_der.is_empty() { + return Err(SignedObjectParseError::Parse( + "signedAttrs encoding is empty".into(), + )); + } + // The first byte should be the context-specific tag (0xA0) for [0] constructed. + // Replace it with universal SET (0x31) for signature input. + cs_der[0] = 0x31; + Ok(cs_der) +} + +fn take_oid_string(seq: &mut CmsReader<'_>) -> Result { + let oid = seq + .take_tag(0x06) + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + Ok(oid_value_bytes_to_string(oid)) +} + +fn oid_value_bytes_to_string(oid_value: &[u8]) -> String { + if oid_value == OID_SHA256_RAW { + return OID_SHA256.to_string(); + } + if oid_value == OID_SIGNED_DATA_RAW { + return OID_SIGNED_DATA.to_string(); + } + if oid_value == OID_CMS_ATTR_CONTENT_TYPE_RAW { + return OID_CMS_ATTR_CONTENT_TYPE.to_string(); + } + if oid_value == OID_CMS_ATTR_MESSAGE_DIGEST_RAW { + return OID_CMS_ATTR_MESSAGE_DIGEST.to_string(); + } + if oid_value == OID_CMS_ATTR_SIGNING_TIME_RAW { + return OID_CMS_ATTR_SIGNING_TIME.to_string(); + } + if oid_value == OID_RSA_ENCRYPTION_RAW { + return OID_RSA_ENCRYPTION.to_string(); + } + if oid_value == OID_SHA256_WITH_RSA_ENCRYPTION_RAW { + return OID_SHA256_WITH_RSA_ENCRYPTION.to_string(); + } + if oid_value == OID_CT_RPKI_MANIFEST_RAW { + return OID_CT_RPKI_MANIFEST.to_string(); + } + if oid_value == OID_CT_ROUTE_ORIGIN_AUTHZ_RAW { + return OID_CT_ROUTE_ORIGIN_AUTHZ.to_string(); + } + if oid_value == OID_CT_ASPA_RAW { + return OID_CT_ASPA.to_string(); + } + decode_oid_to_dotted_string(oid_value) +} + +fn decode_oid_to_dotted_string(value: &[u8]) -> String { + if value.is_empty() { + return "".into(); + } + let first = value[0]; + let a = (first / 40) as u32; + let b = (first % 40) as u32; + let mut out = String::new(); + out.push_str(&a.to_string()); + out.push('.'); + out.push_str(&b.to_string()); + + let mut idx = 1usize; + while idx < value.len() { + let mut v: u32 = 0; + loop { + if idx >= value.len() { + out.push_str("."); + return out; + } + let byte = value[idx]; + idx += 1; + v = (v << 7) | (byte as u32 & 0x7F); + if (byte & 0x80) == 0 { + break; + } + } + out.push('.'); + out.push_str(&v.to_string()); + } + out +} + +fn parse_algorithm_identifier_cursor( + mut seq: CmsReader<'_>, +) -> Result<(String, bool), SignedObjectParseError> { + if seq.is_empty() { + return Err(SignedObjectParseError::Parse( + "AlgorithmIdentifier must be SEQUENCE of 1..2".into(), + )); + } + let oid = take_oid_string(&mut seq)?; + let params_ok = if seq.is_empty() { + true + } else { + let (tag, value) = seq + .take_any() + .map_err(|e| SignedObjectParseError::Parse(e.to_string()))?; + tag == 0x05 && value.is_empty() + }; + if !seq.is_empty() { + return Err(SignedObjectParseError::Parse( + "AlgorithmIdentifier must be SEQUENCE of 1..2".into(), + )); + } + Ok((oid, params_ok)) +} + +fn parse_utctime(value: &[u8]) -> Result { + let s = std::str::from_utf8(value) + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + if !s.ends_with('Z') { + return Err(SignedObjectValidateError::InvalidSigningTimeValue); + } + let digits = &s[..s.len() - 1]; + if digits.len() != 10 && digits.len() != 12 { + return Err(SignedObjectValidateError::InvalidSigningTimeValue); + } + if !digits.as_bytes().iter().all(|b| b.is_ascii_digit()) { + return Err(SignedObjectValidateError::InvalidSigningTimeValue); + } + let yy: i32 = digits[0..2] + .parse() + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let year = if yy <= 49 { 2000 + yy } else { 1900 + yy }; + let mon: u8 = digits[2..4] + .parse() + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let day: u8 = digits[4..6] + .parse() + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let hour: u8 = digits[6..8] + .parse() + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let min: u8 = digits[8..10] + .parse() + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let sec: u8 = if digits.len() == 12 { + digits[10..12] + .parse() + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)? + } else { + 0 + }; + let month = time::Month::try_from(mon) + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let date = time::Date::from_calendar_date(year, month, day) + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let time = time::Time::from_hms(hour, min, sec) + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + Ok(time::OffsetDateTime::new_utc(date, time)) +} + +fn parse_generalized_time(value: &[u8]) -> Result { + let s = std::str::from_utf8(value) + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + if !s.ends_with('Z') { + return Err(SignedObjectValidateError::InvalidSigningTimeValue); + } + let digits = &s[..s.len() - 1]; + if digits.len() != 12 && digits.len() != 14 { + return Err(SignedObjectValidateError::InvalidSigningTimeValue); + } + if !digits.as_bytes().iter().all(|b| b.is_ascii_digit()) { + return Err(SignedObjectValidateError::InvalidSigningTimeValue); + } + let year: i32 = digits[0..4] + .parse() + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let mon: u8 = digits[4..6] + .parse() + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let day: u8 = digits[6..8] + .parse() + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let hour: u8 = digits[8..10] + .parse() + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let min: u8 = digits[10..12] + .parse() + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let sec: u8 = if digits.len() == 14 { + digits[12..14] + .parse() + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)? + } else { + 0 + }; + let month = time::Month::try_from(mon) + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let date = time::Date::from_calendar_date(year, month, day) + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + let time = time::Time::from_hms(hour, min, sec) + .map_err(|_| SignedObjectValidateError::InvalidSigningTimeValue)?; + Ok(time::OffsetDateTime::new_utc(date, time)) +} + +fn strip_leading_zeros(bytes: &[u8]) -> &[u8] { + let mut idx = 0; + while idx < bytes.len() && bytes[idx] == 0 { + idx += 1; + } + if idx == bytes.len() { + &bytes[bytes.len() - 1..] + } else { + &bytes[idx..] + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn strict_cms_der_rejects_constructed_octet_string_fixture() { + let der = std::fs::read(std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.mft", + )) + .expect("read manifest fixture"); + let parsed = RpkiSignedObject::parse_der(&der).expect("parse fixture"); + let econtent = parsed + .signed_data + .encap_content_info + .econtent + .clone() + .expect("fixture eContent"); + assert_eq!( + parsed.signed_data.encap_content_info.econtent.as_deref(), + Some(econtent.as_slice()) + ); + + let primitive_octets = der_tlv(0x04, &econtent); + let constructed_octets = same_size_constructed_octet_string(&primitive_octets, &econtent); + let mutated = replace_first_subslice(&der, &primitive_octets, &constructed_octets) + .expect("replace eContent OCTET STRING"); + let compatible = RpkiSignedObject::parse_der(&mutated).expect("BER-compatible parse"); + assert!( + econtent.starts_with( + compatible + .signed_data + .encap_content_info + .econtent + .as_deref() + .expect("compatible eContent") + ) + ); + let err = RpkiSignedObject::parse_der_strict_cms(&mutated) + .expect_err("DER strict rejects constructed OCTET STRING"); + assert!(err.to_string().contains("DER"), "{err}"); + let compatibility_error = RpkiSignedObject::strict_cms_der_error(&mutated) + .expect("BER-compatible fixture must report strict-DER incompatibility"); + assert_eq!(compatibility_error.to_string(), err.to_string()); + assert!(RpkiSignedObject::strict_cms_der_error(&der).is_none()); + } + + fn replace_first_subslice(input: &[u8], from: &[u8], to: &[u8]) -> Option> { + let pos = input + .windows(from.len()) + .position(|candidate| candidate == from)?; + let mut out = Vec::with_capacity(input.len() - from.len() + to.len()); + out.extend_from_slice(&input[..pos]); + out.extend_from_slice(to); + out.extend_from_slice(&input[pos + from.len()..]); + Some(out) + } + + fn same_size_constructed_octet_string(primitive: &[u8], content: &[u8]) -> Vec { + assert_eq!(primitive[0], 0x04); + let header_len = tlv_header_len(primitive); + let outer_value_len = primitive.len() - header_len; + let child_len = (0..=outer_value_len) + .rev() + .find(|candidate| 1 + len_len(*candidate) + *candidate == outer_value_len) + .expect("find child length"); + let mut out = primitive[..header_len].to_vec(); + out[0] = 0x24; + out.extend(der_tlv(0x04, &content[..child_len])); + assert_eq!(out.len(), primitive.len()); + out + } + + fn tlv_header_len(tlv: &[u8]) -> usize { + if tlv[1] & 0x80 == 0 { + 2 + } else { + 2 + (tlv[1] & 0x7F) as usize + } + } + + fn len_len(len: usize) -> usize { + if len < 0x80 { + return 1; + } + let mut value = len; + let mut n = 0usize; + while value > 0 { + n += 1; + value >>= 8; + } + 1 + n + } + + fn der_tlv(tag: u8, value: &[u8]) -> Vec { + let mut out = vec![tag]; + encode_len(value.len(), &mut out); + out.extend_from_slice(value); + out + } + + fn encode_len(len: usize, out: &mut Vec) { + if len < 0x80 { + out.push(len as u8); + return; + } + let mut bytes = Vec::new(); + let mut value = len; + while value > 0 { + bytes.push((value & 0xFF) as u8); + value >>= 8; + } + bytes.reverse(); + out.push(0x80 | bytes.len() as u8); + out.extend(bytes); + } +} diff --git a/crates/panda-rpki-validator/src/data_model/ta.rs b/crates/panda-rpki-validator/src/data_model/ta.rs new file mode 100644 index 0000000..cd2c77a --- /dev/null +++ b/crates/panda-rpki-validator/src/data_model/ta.rs @@ -0,0 +1,312 @@ +use url::Url; +use x509_parser::prelude::{FromDer, X509Certificate}; + +use crate::data_model::oid::OID_CP_IPADDR_ASNUMBER; +use crate::data_model::rc::{ + AsIdentifierChoice, IpAddressChoice, ResourceCertKind, ResourceCertificate, + ResourceCertificateParseError, ResourceCertificateParsed, ResourceCertificateProfileError, + ResourceCertificateRole, +}; +use crate::data_model::tal::Tal; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TaCertificate { + pub raw_der: Vec, + pub rc_ca: ResourceCertificate, +} + +#[derive(Debug, thiserror::Error)] +pub enum TaCertificateParseError { + #[error("TA certificate parse error: {0} (RFC 5280 §4.1; RFC 6487 §4; RFC 8630 §2.3)")] + ResourceCertificate(#[from] ResourceCertificateParseError), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TaCertificateParsed { + pub rc_parsed: ResourceCertificateParsed, +} + +#[derive(Debug, thiserror::Error)] +pub enum TaCertificateProfileError { + #[error("resource certificate profile error: {0} (RFC 5280 §4; RFC 6487 §4)")] + ResourceCertificate(#[from] ResourceCertificateProfileError), + + #[error("TA certificate must be a CA certificate (RFC 8630 §2.3; RFC 6487 §4.8.1)")] + NotCa, + + #[error( + "TA certificate must be self-signed (issuer DN must equal subject DN) (RFC 8630 §2.3; RFC 5280 §4.1.2.4)" + )] + NotSelfSignedIssuerSubject, + + #[error( + "TA certificate must contain certificatePolicies ipAddr-asNumber ({OID_CP_IPADDR_ASNUMBER}) (RFC 6487 §4.8.9; RFC 8630 §2.3)" + )] + MissingOrInvalidCertificatePolicies, + + #[error("TA certificate must contain SubjectKeyIdentifier (RFC 6487 §4.8.2; RFC 8630 §2.3)")] + MissingSubjectKeyIdentifier, + + #[error( + "TA certificate must contain at least one RFC 3779 resource extension (IP or AS) (RFC 6487 §4.8.10-§4.8.11; RFC 8630 §2.3)" + )] + ResourcesMissing, + + #[error("TA certificate resources must be non-empty (RFC 8630 §2.3)")] + ResourcesEmpty, + + #[error( + "TA certificate MUST NOT use inherit in IP resources (RFC 8630 §2.3; RFC 3779 §2.2.3.5)" + )] + IpResourcesInherit, + + #[error( + "TA certificate MUST NOT use inherit in AS resources (RFC 8630 §2.3; RFC 3779 §3.2.3.3)" + )] + AsResourcesInherit, +} + +#[derive(Debug, thiserror::Error)] +pub enum TaCertificateDecodeError { + #[error("{0}")] + Parse(#[from] TaCertificateParseError), + + #[error("{0}")] + Validate(#[from] TaCertificateProfileError), +} + +/// Backwards-compatible name: TA certificate errors from parse+validate. +pub type TaCertificateError = TaCertificateDecodeError; + +#[derive(Debug, thiserror::Error)] +pub enum TaCertificateVerifyError { + #[error("TA certificate parse error: {0} (RFC 5280 §4.1; RFC 8630 §2.3)")] + Parse(String), + + #[error("trailing bytes after TA certificate DER: {0} bytes (DER; RFC 5280 §4.1)")] + TrailingBytes(usize), + + #[error( + "TA certificate self-signature verification failed: {0} (RFC 8630 §2.3; RFC 5280 §6.1)" + )] + InvalidSelfSignature(String), +} + +impl TaCertificate { + /// Parse step of scheme A (`parse → validate → verify`). + pub fn parse_der(der: &[u8]) -> Result { + Ok(TaCertificateParsed { + rc_parsed: ResourceCertificate::parse_der(der)?, + }) + } + + /// Profile validate step of scheme A (`parse → validate → verify`). + /// + /// `TaCertificate` is already profile-validated when constructed via `decode_der()` / + /// `TaCertificateParsed::validate_profile()`. + pub fn validate_profile(&self) -> Result<(), TaCertificateProfileError> { + Ok(()) + } + + /// Decode a TA certificate (`parse + validate`). + pub fn decode_der(der: &[u8]) -> Result { + Ok(Self::parse_der(der)?.validate_profile()?) + } + + pub fn decode_der_with_strict_name(der: &[u8]) -> Result { + let ta = Self::decode_der(der)?; + ta.rc_ca + .validate_strict_name_profile() + .map_err(TaCertificateProfileError::from)?; + Ok(ta) + } + + /// Backwards-compatible helper (historical name). + pub fn from_der(der: &[u8]) -> Result { + Self::decode_der(der) + } + + pub fn spki_der(&self) -> &[u8] { + &self.rc_ca.tbs.subject_public_key_info + } + + /// Verify step of scheme A (`parse → validate → verify`). + pub fn verify_self_signature(&self) -> Result<(), TaCertificateVerifyError> { + let (rem, cert) = X509Certificate::from_der(&self.raw_der) + .map_err(|e| TaCertificateVerifyError::Parse(e.to_string()))?; + if !rem.is_empty() { + return Err(TaCertificateVerifyError::TrailingBytes(rem.len())); + } + cert.verify_signature(None) + .map_err(|e| TaCertificateVerifyError::InvalidSelfSignature(e.to_string()))?; + Ok(()) + } + + /// Validate TA-specific semantic constraints on a parsed Resource Certificate. + /// + /// Note: this does not verify the X.509 signature; it is intended for higher-level logic and + /// for unit tests that exercise individual constraint branches. + pub fn validate_rc_constraints( + rc_ca: &ResourceCertificate, + ) -> Result<(), TaCertificateProfileError> { + if rc_ca.kind != ResourceCertKind::Ca { + return Err(TaCertificateProfileError::NotCa); + } + + if rc_ca.tbs.extensions.certificate_policies_oid.as_deref() != Some(OID_CP_IPADDR_ASNUMBER) + { + return Err(TaCertificateProfileError::MissingOrInvalidCertificatePolicies); + } + if rc_ca.tbs.extensions.subject_key_identifier.is_none() { + return Err(TaCertificateProfileError::MissingSubjectKeyIdentifier); + } + + let ip = rc_ca.tbs.extensions.ip_resources.as_ref(); + let asn = rc_ca.tbs.extensions.as_resources.as_ref(); + if ip.is_none() && asn.is_none() { + return Err(TaCertificateProfileError::ResourcesMissing); + } + + let mut has_any_resource = false; + + if let Some(ip) = ip { + if ip.has_any_inherit() { + return Err(TaCertificateProfileError::IpResourcesInherit); + } + for fam in &ip.families { + match &fam.choice { + IpAddressChoice::Inherit => { + return Err(TaCertificateProfileError::IpResourcesInherit); + } + IpAddressChoice::AddressesOrRanges(items) => { + if !items.is_empty() { + has_any_resource = true; + } + } + } + } + } + + if let Some(asn) = asn { + if matches!(asn.asnum, Some(AsIdentifierChoice::Inherit)) + || matches!(asn.rdi, Some(AsIdentifierChoice::Inherit)) + { + return Err(TaCertificateProfileError::AsResourcesInherit); + } + if let Some(AsIdentifierChoice::AsIdsOrRanges(items)) = asn.asnum.as_ref() { + if !items.is_empty() { + has_any_resource = true; + } + } + if let Some(AsIdentifierChoice::AsIdsOrRanges(items)) = asn.rdi.as_ref() { + if !items.is_empty() { + has_any_resource = true; + } + } + } + + if !has_any_resource { + return Err(TaCertificateProfileError::ResourcesEmpty); + } + + Ok(()) + } +} + +impl TaCertificateParsed { + pub fn validate_profile(self) -> Result { + let rc_ca = self.rc_parsed.validate_profile()?; + if rc_ca.kind != ResourceCertKind::Ca { + return Err(TaCertificateProfileError::NotCa); + } + rc_ca.validate_rfc6487_profile(ResourceCertificateRole::TrustAnchor)?; + + if rc_ca.tbs.issuer_name != rc_ca.tbs.subject_name { + return Err(TaCertificateProfileError::NotSelfSignedIssuerSubject); + } + + TaCertificate::validate_rc_constraints(&rc_ca)?; + + Ok(TaCertificate { + raw_der: rc_ca.raw_der.clone(), + rc_ca, + }) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TrustAnchor { + pub tal: Tal, + pub ta_certificate: TaCertificate, + pub resolved_ta_uri: Option, +} + +#[derive(Debug, thiserror::Error)] +pub enum TrustAnchorError { + #[error("TA certificate error: {0} (RFC 8630 §2.3)")] + TaCertificate(#[from] TaCertificateDecodeError), + + #[error("TA certificate self-signature error: {0} (RFC 8630 §2.3)")] + TaSelfSignature(#[from] TaCertificateVerifyError), + + #[error("{0}")] + Bind(#[from] TrustAnchorBindError), +} + +#[derive(Debug, thiserror::Error)] +pub enum TrustAnchorBindError { + #[error("resolved TA URI not listed in TAL: {0} (RFC 8630 §2.2-§2.3)")] + ResolvedUriNotInTal(String), + + #[error( + "TAL SPKI does not match TA certificate SubjectPublicKeyInfo (RFC 8630 §2.3; RFC 5280 §4.1.2.7)" + )] + TalSpkiMismatch, +} + +impl TrustAnchor { + /// Bind a TAL and a downloaded TA certificate. + /// + /// This does not download anything; it only validates the binding rules from RFC 8630 §2.3. + pub fn bind_der( + tal: Tal, + ta_der: &[u8], + resolved_uri: Option<&Url>, + ) -> Result { + let ta_certificate = TaCertificate::decode_der(ta_der)?; + ta_certificate.verify_self_signature()?; + Ok(Self::bind(tal, ta_certificate, resolved_uri)?) + } + + pub fn bind_der_with_strict_name( + tal: Tal, + ta_der: &[u8], + resolved_uri: Option<&Url>, + ) -> Result { + let ta_certificate = TaCertificate::decode_der_with_strict_name(ta_der)?; + ta_certificate.verify_self_signature()?; + Ok(Self::bind(tal, ta_certificate, resolved_uri)?) + } + + pub fn bind( + tal: Tal, + ta_certificate: TaCertificate, + resolved_uri: Option<&Url>, + ) -> Result { + if let Some(u) = resolved_uri { + if !tal.ta_uris.iter().any(|x| x == u) { + return Err(TrustAnchorBindError::ResolvedUriNotInTal(u.to_string())); + } + } + + if tal.subject_public_key_info_der != ta_certificate.spki_der() { + return Err(TrustAnchorBindError::TalSpkiMismatch); + } + + Ok(Self { + tal, + ta_certificate, + resolved_ta_uri: resolved_uri.cloned(), + }) + } +} diff --git a/crates/panda-rpki-validator/src/data_model/tal.rs b/crates/panda-rpki-validator/src/data_model/tal.rs new file mode 100644 index 0000000..53683a9 --- /dev/null +++ b/crates/panda-rpki-validator/src/data_model/tal.rs @@ -0,0 +1,183 @@ +use base64::Engine; +use url::Url; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TalParsed { + pub raw: Vec, + /// Lines split by '\n' and normalized by stripping a trailing '\r' per line. + pub lines: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Tal { + pub raw: Vec, + pub comments: Vec, + pub ta_uris: Vec, + pub subject_public_key_info_der: Vec, +} + +#[derive(Debug, thiserror::Error)] +pub enum TalParseError { + #[error("TAL must be valid UTF-8 (RFC 8630 §2.2)")] + InvalidUtf8, +} + +#[derive(Debug, thiserror::Error)] +pub enum TalProfileError { + #[error("TAL comments must appear only at the beginning (RFC 8630 §2.2)")] + CommentAfterHeader, + + #[error("TAL must contain at least one TA URI line (RFC 8630 §2.2)")] + MissingTaUris, + + #[error( + "TAL must contain an empty line separator between URI list and SPKI base64 (RFC 8630 §2.2)" + )] + MissingSeparatorEmptyLine, + + #[error("TAL TA URI invalid: {0} (RFC 8630 §2.2)")] + InvalidUri(String), + + #[error("TAL TA URI scheme must be rsync or https, got {0} (RFC 8630 §2.2)")] + UnsupportedUriScheme(String), + + #[error( + "TAL TA URI must reference a single object (must not end with '/'): {0} (RFC 8630 §2.3)" + )] + UriIsDirectory(String), + + #[error( + "TAL must contain base64-encoded SubjectPublicKeyInfo after the separator (RFC 8630 §2.2)" + )] + MissingSpki, + + #[error("TAL SPKI base64 decode failed (RFC 8630 §2.2)")] + SpkiBase64Decode, + + #[error("TAL SPKI DER is empty (RFC 8630 §2.2)")] + SpkiDerEmpty, +} + +#[derive(Debug, thiserror::Error)] +pub enum TalDecodeError { + #[error("{0}")] + Parse(#[from] TalParseError), + + #[error("{0}")] + Validate(#[from] TalProfileError), +} + +impl Tal { + /// Parse step of scheme A (`parse → validate → verify`). + pub fn parse_bytes(input: &[u8]) -> Result { + let raw = input.to_vec(); + let text = std::str::from_utf8(input).map_err(|_| TalParseError::InvalidUtf8)?; + + let lines: Vec = text + .split('\n') + .map(|l| l.strip_suffix('\r').unwrap_or(l).to_string()) + .collect(); + + Ok(TalParsed { raw, lines }) + } + + /// Validate step of scheme A (`parse → validate → verify`). + /// + /// `Tal` is already profile-validated when constructed via `decode_bytes()` / + /// `TalParsed::validate_profile()`. + pub fn validate_profile(&self) -> Result<(), TalProfileError> { + Ok(()) + } + + pub fn decode_bytes(input: &[u8]) -> Result { + Ok(Self::parse_bytes(input)?.validate_profile()?) + } +} + +impl TalParsed { + pub fn validate_profile(self) -> Result { + let mut idx = 0usize; + + // 1) Leading comments. + let mut comments: Vec = Vec::new(); + while idx < self.lines.len() && self.lines[idx].starts_with('#') { + comments.push(self.lines[idx][1..].to_string()); + idx += 1; + } + + // 2) URI list (one or more non-empty lines). + let mut ta_uris: Vec = Vec::new(); + while idx < self.lines.len() { + let line = self.lines[idx].trim(); + if line.is_empty() { + break; + } + if line.starts_with('#') { + return Err(TalProfileError::CommentAfterHeader); + } + let url = match Url::parse(line) { + Ok(u) => u, + Err(_) => { + if !ta_uris.is_empty() { + return Err(TalProfileError::MissingSeparatorEmptyLine); + } + return Err(TalProfileError::InvalidUri(line.to_string())); + } + }; + match url.scheme() { + "rsync" | "https" => {} + s => return Err(TalProfileError::UnsupportedUriScheme(s.to_string())), + } + if url.path().ends_with('/') { + return Err(TalProfileError::UriIsDirectory(line.to_string())); + } + if url + .path_segments() + .and_then(|mut s| s.next_back()) + .unwrap_or("") + .is_empty() + { + return Err(TalProfileError::UriIsDirectory(line.to_string())); + } + ta_uris.push(url); + idx += 1; + } + + if ta_uris.is_empty() { + return Err(TalProfileError::MissingTaUris); + } + + // 3) Empty line separator (must exist). + if idx >= self.lines.len() || !self.lines[idx].trim().is_empty() { + return Err(TalProfileError::MissingSeparatorEmptyLine); + } + idx += 1; + + // 4) Base64(SPKI DER) remainder; allow line wrapping. + let mut b64 = String::new(); + while idx < self.lines.len() { + let line = self.lines[idx].trim(); + if !line.is_empty() { + b64.push_str(line); + } + idx += 1; + } + if b64.is_empty() { + return Err(TalProfileError::MissingSpki); + } + + let spki_der = base64::engine::general_purpose::STANDARD + .decode(b64.as_bytes()) + .map_err(|_| TalProfileError::SpkiBase64Decode)?; + if spki_der.is_empty() { + return Err(TalProfileError::SpkiDerEmpty); + } + + Ok(Tal { + raw: self.raw, + comments, + ta_uris, + subject_public_key_info_der: spki_der, + }) + } +} diff --git a/crates/panda-rpki-validator/src/fetch/current_repository.rs b/crates/panda-rpki-validator/src/fetch/current_repository.rs new file mode 100644 index 0000000..55034a1 --- /dev/null +++ b/crates/panda-rpki-validator/src/fetch/current_repository.rs @@ -0,0 +1,234 @@ +use std::sync::Arc; + +use crate::fetch::rsync::{RsyncFetchError, RsyncFetchResult, RsyncFetcher}; +use crate::fetch::rsync_system::{ + RsyncScopePolicy, scoped_rsync_failure_dedup_key, scoped_rsync_fetch_uri, +}; +use crate::storage::{RepositoryViewState, RocksStore}; + +#[derive(Clone)] +pub struct CurrentRepositoryViewRsyncFetcher { + store: Arc, + scope_policy: RsyncScopePolicy, +} + +impl CurrentRepositoryViewRsyncFetcher { + pub fn new(store: Arc, scope_policy: RsyncScopePolicy) -> Self { + Self { + store, + scope_policy, + } + } +} + +impl RsyncFetcher for CurrentRepositoryViewRsyncFetcher { + fn fetch_objects(&self, rsync_base_uri: &str) -> RsyncFetchResult)>> { + // The source run's live fetcher may have fetched a module root rather + // than the publication-point URI that triggered it. Read that same + // frozen-view prefix here so transport prefetch request identities and + // their materialized object sets stay equivalent. + let base = scoped_rsync_fetch_uri(self.scope_policy, rsync_base_uri); + let entries = self + .store + .list_repository_view_entries_with_prefix(&base) + .map_err(|error| { + RsyncFetchError::Fetch(format!( + "list frozen repository view failed for {base}: {error}" + )) + })?; + let mut objects = Vec::with_capacity(entries.len()); + for entry in entries { + if !matches!( + entry.state, + RepositoryViewState::Present | RepositoryViewState::Replaced + ) { + continue; + } + let bytes = self + .store + .load_current_object_bytes_by_uri(&entry.rsync_uri) + .map_err(|error| { + RsyncFetchError::Fetch(format!( + "load frozen repository object failed for {}: {error}", + entry.rsync_uri + )) + })? + .ok_or_else(|| { + RsyncFetchError::Fetch(format!( + "frozen repository object missing for {}", + entry.rsync_uri + )) + })?; + objects.push((entry.rsync_uri, bytes)); + } + objects.sort_by(|left, right| left.0.cmp(&right.0)); + if objects.is_empty() { + return Err(RsyncFetchError::Fetch(format!( + "frozen repository view contains no current objects under {base}" + ))); + } + Ok(objects) + } + + fn fetch_object(&self, rsync_uri: &str) -> RsyncFetchResult> { + self.store + .load_current_object_bytes_by_uri(rsync_uri) + .map_err(|error| { + RsyncFetchError::Fetch(format!( + "load frozen repository object failed for {rsync_uri}: {error}" + )) + })? + .ok_or_else(|| { + RsyncFetchError::Fetch(format!("frozen repository object not found: {rsync_uri}")) + }) + } + + fn dedup_key(&self, rsync_base_uri: &str) -> String { + scoped_rsync_fetch_uri(self.scope_policy, rsync_base_uri) + } + + fn failure_dedup_key(&self, rsync_base_uri: &str) -> Option { + scoped_rsync_failure_dedup_key(self.scope_policy, rsync_base_uri) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::storage::{RepositoryViewEntry, RocksStore}; + + #[test] + fn current_repository_fetcher_returns_sorted_present_objects() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = Arc::new(RocksStore::open(dir.path()).expect("open store")); + let entries = [ + ("rsync://example.test/repo/b.roa", b"b".as_slice()), + ("rsync://example.test/repo/a.mft", b"a".as_slice()), + ]; + for (uri, bytes) in entries { + let hash = hex::encode(crate::cir::sha256(bytes)); + store + .put_blob_bytes_batch(&[(hash.clone(), bytes.to_vec())]) + .expect("put blob"); + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: uri.to_string(), + repository_source: Some("fixture".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + current_hash: Some(hash), + }) + .expect("put view"); + } + let fetcher = CurrentRepositoryViewRsyncFetcher::new(store, RsyncScopePolicy::default()); + let objects = fetcher + .fetch_objects("rsync://example.test/repo/") + .expect("fetch objects"); + assert_eq!( + objects + .iter() + .map(|(uri, _)| uri.as_str()) + .collect::>(), + vec![ + "rsync://example.test/repo/a.mft", + "rsync://example.test/repo/b.roa" + ] + ); + assert_eq!( + fetcher + .fetch_object("rsync://example.test/repo/a.mft") + .expect("fetch one"), + b"a" + ); + assert!( + fetcher + .fetch_object("rsync://example.test/repo/missing.roa") + .is_err() + ); + assert_eq!( + fetcher.dedup_key("rsync://example.test/repo"), + "rsync://example.test/repo/" + ); + assert!( + fetcher + .fetch_objects("rsync://empty.example/repo/") + .unwrap_err() + .to_string() + .contains("no current objects") + ); + } + + #[test] + fn current_repository_fetcher_ignores_withdrawn_and_reports_missing_blob() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = Arc::new(RocksStore::open(dir.path()).expect("open store")); + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/withdrawn.roa".to_string(), + repository_source: Some("fixture".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Withdrawn, + current_hash: None, + }) + .expect("put withdrawn view"); + let missing_hash = "ab".repeat(32); + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/missing.roa".to_string(), + repository_source: Some("fixture".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + current_hash: Some(missing_hash), + }) + .expect("put missing view"); + let fetcher = CurrentRepositoryViewRsyncFetcher::new(store, RsyncScopePolicy::default()); + let error = fetcher + .fetch_objects("rsync://example.test/repo/") + .unwrap_err() + .to_string(); + assert!(error.contains("blob bytes missing")); + } + + #[test] + fn current_repository_fetcher_replays_module_scope_and_failure_dedup() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = Arc::new(RocksStore::open(dir.path()).expect("open store")); + for (uri, bytes) in [ + ("rsync://example.test/repo/ca/a.mft", b"a".as_slice()), + ("rsync://example.test/repo/other/b.roa", b"b".as_slice()), + ] { + let hash = hex::encode(crate::cir::sha256(bytes)); + store + .put_blob_bytes_batch(&[(hash.clone(), bytes.to_vec())]) + .expect("put blob"); + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: uri.to_string(), + repository_source: Some("fixture".to_string()), + object_type: Some("fixture".to_string()), + state: RepositoryViewState::Present, + current_hash: Some(hash), + }) + .expect("put view"); + } + + let module_fetcher = CurrentRepositoryViewRsyncFetcher::new( + Arc::clone(&store), + RsyncScopePolicy::ModuleRoot, + ); + let objects = module_fetcher + .fetch_objects("rsync://example.test/repo/ca/") + .expect("fetch module scope"); + assert_eq!(objects.len(), 2); + assert_eq!( + module_fetcher.dedup_key("rsync://example.test/repo/ca/"), + "rsync://example.test/repo/" + ); + + let host_fetcher = CurrentRepositoryViewRsyncFetcher::new(store, RsyncScopePolicy::Host); + assert_eq!( + host_fetcher.failure_dedup_key("rsync://example.test/repo/ca/"), + Some("rsync://example.test/".to_string()) + ); + } +} diff --git a/crates/panda-rpki-validator/src/fetch/http.rs b/crates/panda-rpki-validator/src/fetch/http.rs new file mode 100644 index 0000000..7286277 --- /dev/null +++ b/crates/panda-rpki-validator/src/fetch/http.rs @@ -0,0 +1,652 @@ +use std::cell::RefCell; +use std::io::Write; +use std::time::Duration; + +use reqwest::blocking::Client; +use reqwest::header::HeaderMap; + +use crate::sync::rrdp::Fetcher; + +thread_local! { + static HTTP_TIMEOUT_OVERRIDE: RefCell> = const { RefCell::new(None) }; +} + +pub fn with_scoped_http_timeout_override(timeout: Duration, f: impl FnOnce() -> R) -> R { + HTTP_TIMEOUT_OVERRIDE.with(|cell| { + let previous = cell.replace(Some(timeout)); + let result = f(); + let _ = cell.replace(previous); + result + }) +} + +/// Default User-Agent sent with every outgoing HTTP request (RRDP +/// notification/snapshot/delta, TAL/TA downloads, daemon probes). +pub const DEFAULT_HTTP_USER_AGENT: &str = "panda-rpki/0.2"; + +/// Environment variable that overrides the HTTP User-Agent. Unset, empty, or +/// values containing characters that are illegal in a header value fall back +/// to [`DEFAULT_HTTP_USER_AGENT`]. +pub const HTTP_USER_AGENT_ENV: &str = "RPKI_HTTP_USER_AGENT"; + +fn resolve_http_user_agent(env_value: Option) -> String { + let Some(value) = env_value else { + return DEFAULT_HTTP_USER_AGENT.to_string(); + }; + let trimmed = value.trim(); + let valid = !trimmed.is_empty() && trimmed.bytes().all(|b| (0x20..=0x7e).contains(&b)); + if valid { + trimmed.to_string() + } else { + DEFAULT_HTTP_USER_AGENT.to_string() + } +} + +#[derive(Clone, Debug)] +pub struct HttpFetcherConfig { + /// Connection-establishment timeout for HTTP requests. + pub connect_timeout: Duration, + /// Short timeout used for connection establishment and small metadata objects. + pub timeout: Duration, + /// Larger timeout used for RRDP snapshot / delta bodies. + pub large_body_timeout: Duration, + pub user_agent: String, + /// Extra PEM trust anchors for HTTPS transport tests or private RRDP endpoints. + pub extra_root_certificates_pem: Vec>, +} + +impl Default for HttpFetcherConfig { + fn default() -> Self { + Self { + connect_timeout: Duration::from_secs(15), + timeout: Duration::from_secs(30), + large_body_timeout: Duration::from_secs(180), + user_agent: resolve_http_user_agent(std::env::var(HTTP_USER_AGENT_ENV).ok()), + extra_root_certificates_pem: Vec::new(), + } + } +} + +/// Minimal blocking HTTP(S) fetcher for stage2. +/// +/// This is used for: +/// - downloading TAL / TA certificates (RFC 8630 §2) +/// - fetching RRDP notification/snapshot files (RFC 8182 §3.4) +#[derive(Clone, Debug)] +pub struct BlockingHttpFetcher { + short_client: Client, + large_body_client: Client, + retry_short_client: Client, + short_timeout: Duration, + large_body_timeout: Duration, +} + +impl BlockingHttpFetcher { + pub fn new(config: HttpFetcherConfig) -> Result { + let short_timeout = config.timeout; + let large_body_timeout = std::cmp::max(config.large_body_timeout, config.timeout); + let connect_timeout = std::cmp::min(config.connect_timeout, config.timeout); + let short_client = Self::client_builder( + &config, + connect_timeout, + config.timeout, + config.user_agent.clone(), + )? + .build() + .map_err(|e| e.to_string())?; + let large_body_client = Self::client_builder( + &config, + connect_timeout, + large_body_timeout, + config.user_agent.clone(), + )? + .build() + .map_err(|e| e.to_string())?; + let retry_short_client = Self::client_builder( + &config, + Duration::from_secs(1), + Duration::from_secs(1), + config.user_agent.clone(), + )? + .build() + .map_err(|e| e.to_string())?; + Ok(Self { + short_client, + large_body_client, + retry_short_client, + short_timeout, + large_body_timeout, + }) + } + + fn client_builder( + config: &HttpFetcherConfig, + connect_timeout: Duration, + timeout: Duration, + user_agent: String, + ) -> Result { + let mut builder = Client::builder() + .connect_timeout(connect_timeout) + .timeout(timeout) + .user_agent(user_agent); + for (idx, pem) in config.extra_root_certificates_pem.iter().enumerate() { + let certificate = reqwest::Certificate::from_pem(pem) + .map_err(|e| format!("parse HTTP root certificate #{idx} failed: {e}"))?; + builder = builder.add_root_certificate(certificate); + } + Ok(builder) + } + + pub fn fetch_bytes(&self, uri: &str) -> Result, String> { + let started = std::time::Instant::now(); + let (client, timeout_profile, timeout_value) = self.client_for_uri(uri); + let resp = client.get(uri).send().map_err(|e| { + let msg = format!("http request failed: {e:?}"); + crate::progress_log::emit( + "http_fetch_failed", + serde_json::json!({ + "uri": uri, + "stage": "request", + "timeout_profile": timeout_profile, + "request_timeout_ms": timeout_value.as_millis() as u64, + "duration_ms": started.elapsed().as_millis() as u64, + "error": msg, + }), + ); + msg + })?; + + let status = resp.status(); + let headers = resp.headers().clone(); + if !status.is_success() { + let body_preview = resp + .text() + .ok() + .map(|text| text.chars().take(160).collect::()); + let body_prefix = body_preview + .clone() + .unwrap_or_else(|| "".to_string()); + let msg = format!( + "http status {status}; content_type={}; content_encoding={}; content_length={}; transfer_encoding={}; body_prefix={}", + header_value(&headers, "content-type"), + header_value(&headers, "content-encoding"), + header_value(&headers, "content-length"), + header_value(&headers, "transfer-encoding"), + body_prefix, + ); + crate::progress_log::emit( + "http_fetch_failed", + serde_json::json!({ + "uri": uri, + "stage": "status", + "timeout_profile": timeout_profile, + "request_timeout_ms": timeout_value.as_millis() as u64, + "duration_ms": started.elapsed().as_millis() as u64, + "status": status.as_u16(), + "content_type": header_value_opt(&headers, "content-type"), + "content_encoding": header_value_opt(&headers, "content-encoding"), + "content_length": header_value_opt(&headers, "content-length"), + "transfer_encoding": header_value_opt(&headers, "transfer-encoding"), + "body_prefix": body_preview, + "error": msg, + }), + ); + return Err(msg); + } + + match resp.bytes() { + Ok(bytes) => { + let duration_ms = started.elapsed().as_millis() as u64; + if (duration_ms as f64) / 1000.0 >= crate::progress_log::slow_threshold_secs() { + crate::progress_log::emit( + "http_fetch_slow", + serde_json::json!({ + "uri": uri, + "status": status.as_u16(), + "timeout_profile": timeout_profile, + "request_timeout_ms": timeout_value.as_millis() as u64, + "duration_ms": duration_ms, + "bytes": bytes.len(), + "content_type": header_value_opt(&headers, "content-type"), + "content_encoding": header_value_opt(&headers, "content-encoding"), + "content_length": header_value_opt(&headers, "content-length"), + "transfer_encoding": header_value_opt(&headers, "transfer-encoding"), + }), + ); + } + Ok(bytes.to_vec()) + } + Err(e) => { + let msg = format!( + "http read body failed: {e}; status={}; content_type={}; content_encoding={}; content_length={}; transfer_encoding={}", + status, + header_value(&headers, "content-type"), + header_value(&headers, "content-encoding"), + header_value(&headers, "content-length"), + header_value(&headers, "transfer-encoding"), + ); + crate::progress_log::emit( + "http_fetch_failed", + serde_json::json!({ + "uri": uri, + "stage": "read_body", + "timeout_profile": timeout_profile, + "request_timeout_ms": timeout_value.as_millis() as u64, + "duration_ms": started.elapsed().as_millis() as u64, + "status": status.as_u16(), + "content_type": header_value_opt(&headers, "content-type"), + "content_encoding": header_value_opt(&headers, "content-encoding"), + "content_length": header_value_opt(&headers, "content-length"), + "transfer_encoding": header_value_opt(&headers, "transfer-encoding"), + "error": msg, + }), + ); + Err(msg) + } + } + } + + fn client_for_uri(&self, uri: &str) -> (&Client, &'static str, Duration) { + if let Some(timeout) = HTTP_TIMEOUT_OVERRIDE.with(|cell| *cell.borrow()) { + return (&self.retry_short_client, "retry_short", timeout); + } + if uses_large_body_timeout(uri) { + ( + &self.large_body_client, + "large_body", + self.large_body_timeout, + ) + } else { + (&self.short_client, "short", self.short_timeout) + } + } +} + +impl Fetcher for BlockingHttpFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + self.fetch_bytes(uri) + } + + fn fetch_to_writer(&self, uri: &str, out: &mut dyn Write) -> Result { + let started = std::time::Instant::now(); + let (client, timeout_profile, timeout_value) = self.client_for_uri(uri); + let resp = client.get(uri).send().map_err(|e| { + let msg = format!("http request failed: {e:?}"); + crate::progress_log::emit( + "http_fetch_failed", + serde_json::json!({ + "uri": uri, + "stage": "request", + "timeout_profile": timeout_profile, + "request_timeout_ms": timeout_value.as_millis() as u64, + "duration_ms": started.elapsed().as_millis() as u64, + "error": msg, + }), + ); + msg + })?; + + let status = resp.status(); + let headers = resp.headers().clone(); + if !status.is_success() { + let body_preview = resp + .text() + .ok() + .map(|text| text.chars().take(160).collect::()); + let body_prefix = body_preview + .clone() + .unwrap_or_else(|| "".to_string()); + let msg = format!( + "http status {status}; content_type={}; content_encoding={}; content_length={}; transfer_encoding={}; body_prefix={}", + header_value(&headers, "content-type"), + header_value(&headers, "content-encoding"), + header_value(&headers, "content-length"), + header_value(&headers, "transfer-encoding"), + body_prefix, + ); + crate::progress_log::emit( + "http_fetch_failed", + serde_json::json!({ + "uri": uri, + "stage": "status", + "timeout_profile": timeout_profile, + "request_timeout_ms": timeout_value.as_millis() as u64, + "duration_ms": started.elapsed().as_millis() as u64, + "status": status.as_u16(), + "content_type": header_value_opt(&headers, "content-type"), + "content_encoding": header_value_opt(&headers, "content-encoding"), + "content_length": header_value_opt(&headers, "content-length"), + "transfer_encoding": header_value_opt(&headers, "transfer-encoding"), + "body_prefix": body_preview, + "error": msg, + }), + ); + return Err(msg); + } + + let mut resp = resp; + match resp.copy_to(out) { + Ok(bytes) => { + let duration_ms = started.elapsed().as_millis() as u64; + if (duration_ms as f64) / 1000.0 >= crate::progress_log::slow_threshold_secs() { + crate::progress_log::emit( + "http_fetch_slow", + serde_json::json!({ + "uri": uri, + "status": status.as_u16(), + "timeout_profile": timeout_profile, + "request_timeout_ms": timeout_value.as_millis() as u64, + "duration_ms": duration_ms, + "bytes": bytes, + "content_type": header_value_opt(&headers, "content-type"), + "content_encoding": header_value_opt(&headers, "content-encoding"), + "content_length": header_value_opt(&headers, "content-length"), + "transfer_encoding": header_value_opt(&headers, "transfer-encoding"), + }), + ); + } + Ok(bytes) + } + Err(e) => { + let msg = format!( + "http stream body failed: {e}; status={}; content_type={}; content_encoding={}; content_length={}; transfer_encoding={}", + status, + header_value(&headers, "content-type"), + header_value(&headers, "content-encoding"), + header_value(&headers, "content-length"), + header_value(&headers, "transfer-encoding"), + ); + crate::progress_log::emit( + "http_fetch_failed", + serde_json::json!({ + "uri": uri, + "stage": "stream_body", + "timeout_profile": timeout_profile, + "request_timeout_ms": timeout_value.as_millis() as u64, + "duration_ms": started.elapsed().as_millis() as u64, + "status": status.as_u16(), + "content_type": header_value_opt(&headers, "content-type"), + "content_encoding": header_value_opt(&headers, "content-encoding"), + "content_length": header_value_opt(&headers, "content-length"), + "transfer_encoding": header_value_opt(&headers, "transfer-encoding"), + "error": msg, + }), + ); + Err(msg) + } + } + } +} + +fn header_value(headers: &HeaderMap, name: &str) -> String { + header_value_opt(headers, name).unwrap_or_else(|| "".to_string()) +} + +fn header_value_opt(headers: &HeaderMap, name: &str) -> Option { + headers + .get(name) + .and_then(|v| v.to_str().ok()) + .map(|v| v.to_string()) +} + +fn uses_large_body_timeout(uri: &str) -> bool { + uri.starts_with("https://") && uri.ends_with(".xml") && !uri.ends_with("notification.xml") +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::thread; + use std::time::Duration as StdDuration; + + fn spawn_one_shot_http_server(status_line: &'static str, body: &'static [u8]) -> String { + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind"); + let addr = listener.local_addr().expect("addr"); + thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("accept"); + let mut buf = [0u8; 1024]; + let _ = stream.read(&mut buf); + let hdr = format!( + "{status_line}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + stream.write_all(hdr.as_bytes()).expect("write hdr"); + stream.write_all(body).expect("write body"); + }); + format!("http://{}/", addr) + } + + #[test] + fn fetch_bytes_returns_body_on_success() { + let url = spawn_one_shot_http_server("HTTP/1.1 200 OK", b"hello"); + let http = BlockingHttpFetcher::new(HttpFetcherConfig { + timeout: Duration::from_secs(2), + ..HttpFetcherConfig::default() + }) + .expect("http"); + let got = http.fetch_bytes(&url).expect("fetch"); + assert_eq!(got, b"hello"); + } + + #[test] + fn fetch_bytes_rejects_non_success_status() { + let url = spawn_one_shot_http_server("HTTP/1.1 404 Not Found", b""); + let http = BlockingHttpFetcher::new(HttpFetcherConfig { + timeout: Duration::from_secs(2), + ..HttpFetcherConfig::default() + }) + .expect("http"); + let err = http.fetch_bytes(&url).unwrap_err(); + assert!(err.contains("http status"), "{err}"); + } + + #[test] + fn fetch_bytes_times_out_on_idle_body_read() { + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind"); + let addr = listener.local_addr().expect("addr"); + thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("accept"); + let mut buf = [0u8; 1024]; + let _ = stream.read(&mut buf); + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\nConnection: close\r\n\r\nh") + .expect("write partial body"); + std::thread::sleep(StdDuration::from_secs(2)); + let _ = stream.write_all(b"ello"); + }); + let url = format!("http://{}/", addr); + let http = BlockingHttpFetcher::new(HttpFetcherConfig { + timeout: Duration::from_secs(1), + ..HttpFetcherConfig::default() + }) + .expect("http"); + let err = http.fetch_bytes(&url).unwrap_err(); + assert!(err.contains("http read body failed"), "{err}"); + } + + #[test] + fn fetch_to_writer_streams_body_on_success() { + let url = spawn_one_shot_http_server("HTTP/1.1 200 OK", b"writer-body"); + let http = BlockingHttpFetcher::new(HttpFetcherConfig { + timeout: Duration::from_secs(2), + ..HttpFetcherConfig::default() + }) + .expect("http"); + let mut out = Vec::new(); + let bytes = http.fetch_to_writer(&url, &mut out).expect("stream"); + assert_eq!(bytes, 11); + assert_eq!(out, b"writer-body"); + } + + #[test] + fn fetch_to_writer_rejects_non_success_status() { + let url = spawn_one_shot_http_server("HTTP/1.1 500 Internal Server Error", b"boom"); + let http = BlockingHttpFetcher::new(HttpFetcherConfig { + timeout: Duration::from_secs(2), + ..HttpFetcherConfig::default() + }) + .expect("http"); + let mut out = Vec::new(); + let err = http.fetch_to_writer(&url, &mut out).unwrap_err(); + assert!(err.contains("http status"), "{err}"); + assert!(out.is_empty()); + } + + #[test] + fn fetch_to_writer_times_out_on_idle_stream_read() { + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind"); + let addr = listener.local_addr().expect("addr"); + thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("accept"); + let mut buf = [0u8; 1024]; + let _ = stream.read(&mut buf); + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 5\r\nConnection: close\r\n\r\nh") + .expect("write partial body"); + std::thread::sleep(StdDuration::from_secs(2)); + let _ = stream.write_all(b"ello"); + }); + let url = format!("http://{}/", addr); + let http = BlockingHttpFetcher::new(HttpFetcherConfig { + timeout: Duration::from_secs(1), + ..HttpFetcherConfig::default() + }) + .expect("http"); + let mut out = Vec::new(); + let err = http.fetch_to_writer(&url, &mut out).unwrap_err(); + assert!(err.contains("http stream body failed"), "{err}"); + } + + #[test] + fn uses_large_body_timeout_selects_rrdp_snapshot_and_delta_not_notification() { + assert!(!uses_large_body_timeout( + "https://rrdp.example.test/notification.xml" + )); + assert!(uses_large_body_timeout( + "https://rrdp.example.test/session/123/snapshot.xml" + )); + assert!(uses_large_body_timeout( + "https://rrdp.example.test/session/123/delta-42.xml" + )); + assert!(!uses_large_body_timeout( + "https://tal.example.test/example.tal" + )); + } + + #[test] + fn client_for_uri_selects_expected_timeout_profile() { + let http = BlockingHttpFetcher::new(HttpFetcherConfig { + timeout: Duration::from_secs(3), + large_body_timeout: Duration::from_secs(9), + ..HttpFetcherConfig::default() + }) + .expect("http"); + + let (_, profile_short, timeout_short) = + http.client_for_uri("https://example.test/root.tal"); + assert_eq!(profile_short, "short"); + assert_eq!(timeout_short, Duration::from_secs(3)); + + let (_, profile_large, timeout_large) = + http.client_for_uri("https://rrdp.example.test/session/1/snapshot.xml"); + assert_eq!(profile_large, "large_body"); + assert_eq!(timeout_large, Duration::from_secs(9)); + } + + #[test] + fn resolve_http_user_agent_falls_back_to_default() { + assert_eq!(resolve_http_user_agent(None), DEFAULT_HTTP_USER_AGENT); + assert_eq!( + resolve_http_user_agent(Some(String::new())), + DEFAULT_HTTP_USER_AGENT + ); + assert_eq!( + resolve_http_user_agent(Some(" ".to_string())), + DEFAULT_HTTP_USER_AGENT + ); + // Control characters are illegal in header values; fall back instead + // of letting reqwest fail to build the client. + assert_eq!( + resolve_http_user_agent(Some("bad\u{1}ua".to_string())), + DEFAULT_HTTP_USER_AGENT + ); + assert_eq!( + resolve_http_user_agent(Some("non-ascii-ua-中".to_string())), + DEFAULT_HTTP_USER_AGENT + ); + } + + #[test] + fn resolve_http_user_agent_accepts_custom_value() { + assert_eq!( + resolve_http_user_agent(Some("panda-rpki/9.9 (test)".to_string())), + "panda-rpki/9.9 (test)" + ); + // Surrounding whitespace is trimmed. + assert_eq!( + resolve_http_user_agent(Some(" custom/1.0 ".to_string())), + "custom/1.0" + ); + } + + fn spawn_user_agent_capture_server() -> (String, std::sync::mpsc::Receiver) { + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind"); + let addr = listener.local_addr().expect("addr"); + let (tx, rx) = std::sync::mpsc::channel(); + thread::spawn(move || { + let (mut stream, _) = listener.accept().expect("accept"); + let mut buf = [0u8; 4096]; + let read = stream.read(&mut buf).expect("read request"); + let request = String::from_utf8_lossy(&buf[..read]).to_string(); + let body = b"ok"; + let hdr = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + stream.write_all(hdr.as_bytes()).expect("write hdr"); + stream.write_all(body).expect("write body"); + let _ = tx.send(request); + }); + (format!("http://{}/", addr), rx) + } + + #[test] + fn fetch_bytes_sends_configured_user_agent_header() { + let (url, rx) = spawn_user_agent_capture_server(); + let http = BlockingHttpFetcher::new(HttpFetcherConfig { + timeout: Duration::from_secs(2), + user_agent: "panda-rpki/0.2".to_string(), + ..HttpFetcherConfig::default() + }) + .expect("http"); + http.fetch_bytes(&url).expect("fetch"); + let request = rx.recv().expect("request captured"); + assert!( + request + .to_ascii_lowercase() + .contains("user-agent: panda-rpki/0.2"), + "{request}" + ); + } + + #[test] + fn fetch_bytes_sends_custom_user_agent_header() { + let (url, rx) = spawn_user_agent_capture_server(); + let http = BlockingHttpFetcher::new(HttpFetcherConfig { + timeout: Duration::from_secs(2), + user_agent: "custom-marker/7.7".to_string(), + ..HttpFetcherConfig::default() + }) + .expect("http"); + http.fetch_bytes(&url).expect("fetch"); + let request = rx.recv().expect("request captured"); + assert!( + request + .to_ascii_lowercase() + .contains("user-agent: custom-marker/7.7"), + "{request}" + ); + } +} diff --git a/crates/panda-rpki-validator/src/fetch/mod.rs b/crates/panda-rpki-validator/src/fetch/mod.rs new file mode 100644 index 0000000..5f92b39 --- /dev/null +++ b/crates/panda-rpki-validator/src/fetch/mod.rs @@ -0,0 +1,4 @@ +pub mod current_repository; +pub mod http; +pub mod rsync; +pub mod rsync_system; diff --git a/crates/panda-rpki-validator/src/fetch/rsync.rs b/crates/panda-rpki-validator/src/fetch/rsync.rs new file mode 100644 index 0000000..a18ec2d --- /dev/null +++ b/crates/panda-rpki-validator/src/fetch/rsync.rs @@ -0,0 +1,206 @@ +use std::path::{Path, PathBuf}; + +#[derive(Debug, thiserror::Error)] +pub enum RsyncFetchError { + #[error("rsync fetch error: {0}")] + Fetch(String), +} + +pub type RsyncFetchResult = Result; + +pub fn normalize_rsync_base_uri(s: &str) -> String { + if s.ends_with('/') { + s.to_string() + } else { + format!("{s}/") + } +} + +/// Fetch repository objects from a publication point. +/// +/// v1: this is intentionally abstract so unit tests can use a mock, and later we can +/// back it by calling the system `rsync` binary (RFC 6481 §5; RFC 8182 §3.4.5). +pub trait RsyncFetcher: Send + Sync { + /// Return a list of objects as `(rsync_uri, bytes)` pairs. + fn fetch_objects(&self, rsync_base_uri: &str) -> RsyncFetchResult)>>; + + /// Fetch one object by exact rsync URI. + /// + /// The default implementation fetches the parent directory and filters the exact + /// object. Live fetchers should override this to avoid widening one-object TAL + /// bootstrap fetches into whole publication point or module synchronizations. + fn fetch_object(&self, rsync_uri: &str) -> RsyncFetchResult> { + let base = parent_rsync_uri(rsync_uri).map_err(RsyncFetchError::Fetch)?; + self.fetch_objects(&base)? + .into_iter() + .find(|(uri, _)| uri == rsync_uri) + .map(|(_, bytes)| bytes) + .ok_or_else(|| RsyncFetchError::Fetch(format!("rsync object not found: {rsync_uri}"))) + } + + /// Stream fetched objects to a visitor without requiring callers to materialize the + /// full result vector in memory. + fn visit_objects( + &self, + rsync_base_uri: &str, + visitor: &mut dyn FnMut(String, Vec) -> Result<(), String>, + ) -> RsyncFetchResult<(usize, u64)> { + let objects = self.fetch_objects(rsync_base_uri)?; + let mut count = 0usize; + let mut bytes_total = 0u64; + for (uri, bytes) in objects { + bytes_total += bytes.len() as u64; + count += 1; + visitor(uri, bytes).map_err(RsyncFetchError::Fetch)?; + } + Ok((count, bytes_total)) + } + + /// Return the deduplication key used by orchestration layers. + /// + /// By default this is the normalized publication point base URI. Fetchers that + /// intentionally widen their fetch scope (for example to a full rsync module) + /// should override this so callers can safely deduplicate at the same scope. + fn dedup_key(&self, rsync_base_uri: &str) -> String { + normalize_rsync_base_uri(rsync_base_uri) + } + + /// Return an optional failure-only deduplication key. + /// + /// This key is only used to short-circuit repeated failed rsync fallbacks. It must + /// not be used to reuse successful fetch results unless it is identical to + /// `dedup_key`, because a successful fetch for one publication point does not imply + /// that another publication point under the same host has been fetched. + fn failure_dedup_key(&self, _rsync_base_uri: &str) -> Option { + None + } +} + +fn parent_rsync_uri(rsync_uri: &str) -> Result { + let parsed = url::Url::parse(rsync_uri).map_err(|e| e.to_string())?; + if parsed.scheme() != "rsync" { + return Err(format!("not an rsync URI: {rsync_uri}")); + } + let host = parsed + .host_str() + .ok_or_else(|| format!("missing host in rsync URI: {rsync_uri}"))?; + let segments = parsed + .path_segments() + .ok_or_else(|| format!("missing path in rsync URI: {rsync_uri}"))? + .collect::>(); + if segments.is_empty() || segments.last().copied().unwrap_or_default().is_empty() { + return Err(format!( + "rsync URI must reference a file object: {rsync_uri}" + )); + } + let parent_segments = &segments[..segments.len() - 1]; + let mut parent = format!("rsync://{host}/"); + if !parent_segments.is_empty() { + parent.push_str(&parent_segments.join("/")); + parent.push('/'); + } + Ok(parent) +} + +/// A simple "rsync" implementation backed by a local directory. +/// +/// This is primarily meant for offline tests and fixtures. The key generation mimics rsync URIs: +/// `rsync_base_uri` + relative path (with `/` separators). +#[derive(Clone, Debug)] +pub struct LocalDirRsyncFetcher { + pub root_dir: PathBuf, +} + +impl LocalDirRsyncFetcher { + pub fn new(root_dir: impl Into) -> Self { + Self { + root_dir: root_dir.into(), + } + } +} + +impl RsyncFetcher for LocalDirRsyncFetcher { + fn fetch_objects(&self, rsync_base_uri: &str) -> RsyncFetchResult)>> { + let base = normalize_rsync_base_uri(rsync_base_uri); + let mut out = Vec::new(); + walk_dir_collect(&self.root_dir, &self.root_dir, &base, &mut out) + .map_err(|e| RsyncFetchError::Fetch(e))?; + Ok(out) + } +} + +fn walk_dir_collect( + root: &Path, + current: &Path, + rsync_base_uri: &str, + out: &mut Vec<(String, Vec)>, +) -> Result<(), String> { + let rd = std::fs::read_dir(current).map_err(|e| e.to_string())?; + for entry in rd { + let entry = entry.map_err(|e| e.to_string())?; + let path = entry.path(); + let meta = entry.metadata().map_err(|e| e.to_string())?; + if meta.is_dir() { + walk_dir_collect(root, &path, rsync_base_uri, out)?; + continue; + } + if !meta.is_file() { + continue; + } + let rel = path + .strip_prefix(root) + .map_err(|e| e.to_string())? + .to_string_lossy() + .replace('\\', "/"); + let uri = format!("{rsync_base_uri}{rel}"); + let bytes = std::fs::read(&path).map_err(|e| e.to_string())?; + out.push((uri, bytes)); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn local_dir_rsync_fetcher_collects_files_and_normalizes_base_uri() { + let tmp = tempfile::tempdir().expect("tempdir"); + std::fs::create_dir_all(tmp.path().join("nested")).expect("mkdir"); + std::fs::write(tmp.path().join("a.mft"), b"a").expect("write"); + std::fs::write(tmp.path().join("nested").join("b.roa"), b"b").expect("write"); + + let f = LocalDirRsyncFetcher::new(tmp.path()); + let mut objects = f + .fetch_objects("rsync://example.net/repo") + .expect("fetch_objects"); + objects.sort_by(|(a, _), (b, _)| a.cmp(b)); + + assert_eq!(objects.len(), 2); + assert_eq!(objects[0].0, "rsync://example.net/repo/a.mft"); + assert_eq!(objects[0].1, b"a"); + assert_eq!(objects[1].0, "rsync://example.net/repo/nested/b.roa"); + assert_eq!(objects[1].1, b"b"); + } + + #[test] + fn local_dir_rsync_fetcher_reports_read_dir_errors() { + let tmp = tempfile::tempdir().expect("tempdir"); + let missing = tmp.path().join("missing"); + let f = LocalDirRsyncFetcher::new(missing); + let err = f.fetch_objects("rsync://example.net/repo").unwrap_err(); + match err { + RsyncFetchError::Fetch(msg) => assert!(!msg.is_empty()), + } + } + + #[test] + fn default_dedup_key_is_normalized_base_uri() { + let tmp = tempfile::tempdir().expect("tempdir"); + let fetcher = LocalDirRsyncFetcher::new(tmp.path()); + assert_eq!( + fetcher.dedup_key("rsync://example.net/repo"), + "rsync://example.net/repo/" + ); + } +} diff --git a/crates/panda-rpki-validator/src/fetch/rsync_system.rs b/crates/panda-rpki-validator/src/fetch/rsync_system.rs new file mode 100644 index 0000000..20e44cf --- /dev/null +++ b/crates/panda-rpki-validator/src/fetch/rsync_system.rs @@ -0,0 +1,944 @@ +use std::cell::RefCell; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::process::Stdio; +use std::thread; +use std::time::Duration; +use std::time::Instant; + +use sha2::Digest; +use uuid::Uuid; + +use crate::fetch::rsync::{ + RsyncFetchError, RsyncFetchResult, RsyncFetcher, normalize_rsync_base_uri, +}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum RsyncScopePolicy { + Host, + PublicationPoint, + ModuleRoot, +} + +impl Default for RsyncScopePolicy { + fn default() -> Self { + Self::ModuleRoot + } +} + +impl RsyncScopePolicy { + pub fn parse_cli_value(value: &str) -> Result { + match value { + "host" => Ok(Self::Host), + "publication-point" => Ok(Self::PublicationPoint), + "module-root" => Ok(Self::ModuleRoot), + _ => Err(format!( + "invalid --rsync-scope: {value}; expected host, publication-point, or module-root" + )), + } + } +} + +#[derive(Clone, Debug)] +pub struct SystemRsyncConfig { + pub rsync_bin: PathBuf, + pub connect_timeout: Duration, + pub timeout: Duration, + pub extra_args: Vec, + /// Optional root directory for persistent rsync mirrors. + /// + /// When set, callers may choose to sync into stable subdirectories under this + /// root (instead of a temporary directory) to benefit from rsync's incremental + /// behavior across runs. + /// + /// Note: actual mirror behavior is implemented separately from config wiring. + pub mirror_root: Option, + pub scope_policy: RsyncScopePolicy, +} + +impl Default for SystemRsyncConfig { + fn default() -> Self { + Self { + rsync_bin: PathBuf::from("rsync"), + connect_timeout: Duration::from_secs(15), + timeout: Duration::from_secs(30), + extra_args: Vec::new(), + mirror_root: None, + scope_policy: RsyncScopePolicy::default(), + } + } +} + +/// A `RsyncFetcher` implementation backed by the system `rsync` binary. +/// +/// This is intended for live stage2 runs. For unit tests and offline fixtures, +/// prefer `LocalDirRsyncFetcher`. +#[derive(Clone, Debug)] +pub struct SystemRsyncFetcher { + config: SystemRsyncConfig, +} + +thread_local! { + static RSYNC_TIMEOUT_OVERRIDE: RefCell> = const { RefCell::new(None) }; + static RSYNC_FAIL_FAST_PROFILE: RefCell> = const { RefCell::new(None) }; +} + +pub fn with_scoped_rsync_timeout_override(timeout: Duration, f: impl FnOnce() -> R) -> R { + RSYNC_TIMEOUT_OVERRIDE.with(|cell| { + let previous = cell.replace(Some(timeout)); + let result = f(); + let _ = cell.replace(previous); + result + }) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RsyncFailFastProfile { + pub initial_wall_clock_timeout: Duration, + pub max_wall_clock_timeout: Duration, + pub max_attempts: usize, +} + +pub fn with_scoped_rsync_fail_fast_profile( + profile: RsyncFailFastProfile, + f: impl FnOnce() -> R, +) -> R { + RSYNC_FAIL_FAST_PROFILE.with(|cell| { + let previous = cell.replace(Some(profile)); + let result = f(); + let _ = cell.replace(previous); + result + }) +} + +impl SystemRsyncFetcher { + pub fn new(config: SystemRsyncConfig) -> Self { + Self { config } + } + + fn mirror_dst_dir(&self, normalized_rsync_base_uri: &str) -> Result, String> { + let Some(root) = self.config.mirror_root.as_ref() else { + return Ok(None); + }; + + std::fs::create_dir_all(root) + .map_err(|e| format!("create rsync mirror root failed: {}: {e}", root.display()))?; + + let hash = hex::encode(sha2::Sha256::digest(normalized_rsync_base_uri.as_bytes())); + let dir = root.join(hash); + std::fs::create_dir_all(&dir).map_err(|e| { + format!( + "create rsync mirror directory failed: {}: {e}", + dir.display() + ) + })?; + Ok(Some(dir)) + } + + fn run_rsync(&self, src: &str, dst: &Path) -> Result<(), String> { + let fail_fast = RSYNC_FAIL_FAST_PROFILE.with(|cell| *cell.borrow()); + if let Some(profile) = fail_fast { + return self.run_rsync_fail_fast(src, dst, profile); + } + self.run_rsync_once(src, dst, None, false) + } + + fn run_rsync_once( + &self, + src: &str, + dst: &Path, + wall_clock_timeout: Option, + keep_partial: bool, + ) -> Result<(), String> { + // `--timeout` is I/O timeout in seconds (applies to network reads/writes). + let timeout = + RSYNC_TIMEOUT_OVERRIDE.with(|cell| cell.borrow().unwrap_or(self.config.timeout)); + let connect_timeout_secs = self.config.connect_timeout.as_secs().max(1).to_string(); + let timeout_secs = timeout.as_secs().max(1).to_string(); + let is_remote_rsync = src.starts_with("rsync://"); + + let mut cmd = Command::new(&self.config.rsync_bin); + cmd.arg("-rt") + .arg("--delete") + .arg("--timeout") + .arg(timeout_secs) + .args(&self.config.extra_args); + if is_remote_rsync { + cmd.arg("--contimeout").arg(connect_timeout_secs); + } + if keep_partial { + cmd.arg("--partial"); + } + cmd.arg(src) + .arg(dst) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let mut child = cmd + .spawn() + .map_err(|e| format!("rsync spawn failed: {e}"))?; + if let Some(limit) = wall_clock_timeout { + let started = Instant::now(); + loop { + match child + .try_wait() + .map_err(|e| format!("rsync wait failed: {e}"))? + { + Some(_status) => { + let out = child + .wait_with_output() + .map_err(|e| format!("rsync wait_with_output failed: {e}"))?; + if out.status.success() { + return Ok(()); + } + let stderr = String::from_utf8_lossy(&out.stderr); + let stdout = String::from_utf8_lossy(&out.stdout); + return Err(format!( + "rsync failed: status={} stdout={} stderr={}", + out.status, + stdout.trim(), + stderr.trim() + )); + } + None => { + if started.elapsed() >= limit { + let _ = child.kill(); + let out = child + .wait_with_output() + .map_err(|e| format!("rsync wait_with_output failed: {e}"))?; + let stderr = String::from_utf8_lossy(&out.stderr); + let stdout = String::from_utf8_lossy(&out.stdout); + return Err(format!( + "rsync wall-clock timeout after {}s: stdout={} stderr={}", + limit.as_secs(), + stdout.trim(), + stderr.trim() + )); + } + thread::sleep(Duration::from_millis(100)); + } + } + } + } + let out = child + .wait_with_output() + .map_err(|e| format!("rsync wait_with_output failed: {e}"))?; + if out.status.success() { + return Ok(()); + } + let stderr = String::from_utf8_lossy(&out.stderr); + let stdout = String::from_utf8_lossy(&out.stdout); + Err(format!( + "rsync failed: status={} stdout={} stderr={}", + out.status, + stdout.trim(), + stderr.trim() + )) + } + + fn run_rsync_fail_fast( + &self, + src: &str, + dst: &Path, + profile: RsyncFailFastProfile, + ) -> Result<(), String> { + let mut attempt = 0usize; + let mut timeout = profile.initial_wall_clock_timeout; + let mut previous_progress = (0usize, 0u64); + let mut zero_progress_attempts = 0usize; + let max_timeout = std::cmp::max( + profile.max_wall_clock_timeout, + profile.initial_wall_clock_timeout, + ); + + loop { + attempt += 1; + match self.run_rsync_once(src, dst, Some(timeout), true) { + Ok(()) => return Ok(()), + Err(err) => { + if is_hard_fail_rsync_error(&err) { + return Err(format!( + "rsync fail-fast hard-fail on attempt {}: {}", + attempt, err + )); + } + if !err.contains("wall-clock timeout") { + return Err(err); + } + let progress = dir_progress(dst) + .map_err(|e| format!("rsync fail-fast progress stat failed: {e}"))?; + if progress == (0, 0) { + zero_progress_attempts += 1; + if zero_progress_attempts >= 2 || attempt >= profile.max_attempts { + return Err(format!( + "rsync fail-fast gave up after {} attempts with no progress: {}", + attempt, err + )); + } + } else if progress == previous_progress { + return Err(format!( + "rsync fail-fast gave up after {} attempts with no additional progress: {}", + attempt, err + )); + } else { + previous_progress = progress; + } + + if attempt >= profile.max_attempts { + return Err(format!( + "rsync fail-fast exhausted {} attempts: {}", + profile.max_attempts, err + )); + } + timeout = std::cmp::min(timeout.saturating_mul(2), max_timeout); + } + } + } + } + + fn scope_fetch_uri(&self, rsync_base_uri: &str) -> String { + scoped_rsync_fetch_uri(self.config.scope_policy, rsync_base_uri) + } +} + +impl RsyncFetcher for SystemRsyncFetcher { + fn fetch_objects(&self, rsync_base_uri: &str) -> RsyncFetchResult)>> { + let mut out = Vec::new(); + self.visit_objects(rsync_base_uri, &mut |uri, bytes| { + out.push((uri, bytes)); + Ok(()) + })?; + Ok(out) + } + + fn fetch_object(&self, rsync_uri: &str) -> RsyncFetchResult> { + let parsed = + url::Url::parse(rsync_uri).map_err(|e| RsyncFetchError::Fetch(e.to_string()))?; + if parsed.scheme() != "rsync" { + return Err(RsyncFetchError::Fetch(format!( + "not an rsync URI: {rsync_uri}" + ))); + } + let file_name = parsed + .path_segments() + .and_then(|segments| segments.filter(|segment| !segment.is_empty()).next_back()) + .ok_or_else(|| { + RsyncFetchError::Fetch(format!( + "rsync URI must reference a file object: {rsync_uri}" + )) + })?; + let tmp = TempDir::new().map_err(RsyncFetchError::Fetch)?; + self.run_rsync(rsync_uri, tmp.path()) + .map_err(RsyncFetchError::Fetch)?; + let object_path = tmp.path().join(file_name); + std::fs::read(&object_path).map_err(|e| { + RsyncFetchError::Fetch(format!( + "read fetched rsync object failed: {}: {e}", + object_path.display() + )) + }) + } + + fn visit_objects( + &self, + rsync_base_uri: &str, + visitor: &mut dyn FnMut(String, Vec) -> Result<(), String>, + ) -> RsyncFetchResult<(usize, u64)> { + let base = self.scope_fetch_uri(rsync_base_uri); + let mut count = 0usize; + let mut bytes_total = 0u64; + let mut wrapped = |uri: String, bytes: Vec| -> Result<(), String> { + bytes_total += bytes.len() as u64; + count += 1; + visitor(uri, bytes) + }; + + if let Some(dst) = self + .mirror_dst_dir(&base) + .map_err(|e| RsyncFetchError::Fetch(e.to_string()))? + { + self.run_rsync(&base, &dst) + .map_err(RsyncFetchError::Fetch)?; + walk_dir_visit(&dst, &dst, &base, &mut wrapped).map_err(RsyncFetchError::Fetch)?; + return Ok((count, bytes_total)); + } + + let tmp = TempDir::new().map_err(|e| RsyncFetchError::Fetch(e.to_string()))?; + self.run_rsync(&base, tmp.path()) + .map_err(RsyncFetchError::Fetch)?; + walk_dir_visit(tmp.path(), tmp.path(), &base, &mut wrapped) + .map_err(RsyncFetchError::Fetch)?; + Ok((count, bytes_total)) + } + + fn dedup_key(&self, rsync_base_uri: &str) -> String { + self.scope_fetch_uri(rsync_base_uri) + } + + fn failure_dedup_key(&self, rsync_base_uri: &str) -> Option { + scoped_rsync_failure_dedup_key(self.config.scope_policy, rsync_base_uri) + } +} + +/// Return the exact successful-fetch scope used by the live rsync fetcher. +/// +/// The request scope is shared by live and replay transport paths so their +/// request keys and object projection scope remain equivalent. +pub fn scoped_rsync_fetch_uri(scope_policy: RsyncScopePolicy, rsync_base_uri: &str) -> String { + match scope_policy { + RsyncScopePolicy::Host | RsyncScopePolicy::PublicationPoint => { + normalize_rsync_base_uri(rsync_base_uri) + } + RsyncScopePolicy::ModuleRoot => rsync_module_root_uri(rsync_base_uri) + .unwrap_or_else(|| normalize_rsync_base_uri(rsync_base_uri)), + } +} + +/// Return the live fetcher's failure-deduplication key for the configured +/// scope. This is part of the transport request identity, even though host +/// scope deliberately does not widen the successful fetch URI. +pub fn scoped_rsync_failure_dedup_key( + scope_policy: RsyncScopePolicy, + rsync_base_uri: &str, +) -> Option { + match scope_policy { + RsyncScopePolicy::Host => rsync_host_scope_uri(rsync_base_uri), + RsyncScopePolicy::PublicationPoint | RsyncScopePolicy::ModuleRoot => None, + } +} + +fn rsync_host_scope_uri(rsync_base_uri: &str) -> Option { + let parsed = url::Url::parse(rsync_base_uri).ok()?; + if parsed.scheme() != "rsync" { + return None; + } + Some(format!("rsync://{}/", parsed.host_str()?)) +} + +struct TempDir { + path: PathBuf, +} + +impl TempDir { + fn new() -> Result { + let mut p = std::env::temp_dir(); + p.push(format!("rpki-system-rsync-{}", Uuid::new_v4())); + std::fs::create_dir_all(&p).map_err(|e| e.to_string())?; + Ok(Self { path: p }) + } + + fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.path); + } +} + +fn rsync_module_root_uri(s: &str) -> Option { + let normalized = normalize_rsync_base_uri(s); + let rest = normalized.strip_prefix("rsync://")?; + let mut host_and_path = rest.splitn(2, '/'); + let authority = host_and_path.next()?; + let path = host_and_path.next()?; + let mut segments: Vec<&str> = path + .split('/') + .filter(|segment| !segment.is_empty()) + .collect(); + if segments.is_empty() { + return None; + } + let module = segments.remove(0); + Some(format!("rsync://{authority}/{module}/")) +} + +fn walk_dir_collect( + root: &Path, + current: &Path, + rsync_base_uri: &str, + out: &mut Vec<(String, Vec)>, +) -> Result<(), String> { + let rd = std::fs::read_dir(current).map_err(|e| e.to_string())?; + for entry in rd { + let entry = entry.map_err(|e| e.to_string())?; + let path = entry.path(); + let meta = entry.metadata().map_err(|e| e.to_string())?; + if meta.is_dir() { + walk_dir_collect(root, &path, rsync_base_uri, out)?; + continue; + } + if !meta.is_file() { + continue; + } + let rel = path + .strip_prefix(root) + .map_err(|e| e.to_string())? + .to_string_lossy() + .replace('\\', "/"); + let uri = format!("{rsync_base_uri}{rel}"); + let bytes = std::fs::read(&path).map_err(|e| e.to_string())?; + out.push((uri, bytes)); + } + Ok(()) +} + +fn dir_progress(root: &Path) -> Result<(usize, u64), String> { + if !root.exists() { + return Ok((0, 0)); + } + let mut files = 0usize; + let mut bytes = 0u64; + let mut stack = vec![root.to_path_buf()]; + while let Some(path) = stack.pop() { + let rd = std::fs::read_dir(&path).map_err(|e| e.to_string())?; + for entry in rd { + let entry = entry.map_err(|e| e.to_string())?; + let path = entry.path(); + let meta = entry.metadata().map_err(|e| e.to_string())?; + if meta.is_dir() { + stack.push(path); + } else if meta.is_file() { + files += 1; + bytes += meta.len(); + } + } + } + Ok((files, bytes)) +} + +fn is_hard_fail_rsync_error(msg: &str) -> bool { + let lower = msg.to_ascii_lowercase(); + lower.contains("no route to host") + || lower.contains("network is unreachable") + || lower.contains("connection refused") + || lower.contains("name or service not known") +} + +fn walk_dir_visit( + root: &Path, + current: &Path, + rsync_base_uri: &str, + visitor: &mut dyn FnMut(String, Vec) -> Result<(), String>, +) -> Result<(), String> { + let rd = std::fs::read_dir(current).map_err(|e| e.to_string())?; + for entry in rd { + let entry = entry.map_err(|e| e.to_string())?; + let path = entry.path(); + let meta = entry.metadata().map_err(|e| e.to_string())?; + if meta.is_dir() { + walk_dir_visit(root, &path, rsync_base_uri, visitor)?; + continue; + } + if !meta.is_file() { + continue; + } + let rel = path + .strip_prefix(root) + .map_err(|e| e.to_string())? + .to_string_lossy() + .replace('\\', "/"); + let uri = format!("{rsync_base_uri}{rel}"); + let bytes = std::fs::read(&path).map_err(|e| e.to_string())?; + visitor(uri, bytes)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalize_rsync_base_uri_appends_slash_when_missing() { + assert_eq!( + normalize_rsync_base_uri("rsync://example.net/repo"), + "rsync://example.net/repo/".to_string() + ); + assert_eq!( + normalize_rsync_base_uri("rsync://example.net/repo/"), + "rsync://example.net/repo/".to_string() + ); + } + + #[test] + fn walk_dir_collect_collects_files_and_normalizes_backslashes_in_uri() { + let temp = tempfile::tempdir().expect("tempdir"); + let root = temp.path(); + std::fs::create_dir_all(root.join("sub")).expect("mkdir"); + std::fs::write(root.join("sub").join("a.cer"), b"x").expect("write"); + std::fs::write(root.join("b\\c.mft"), b"y").expect("write backslash file"); + + let mut out: Vec<(String, Vec)> = Vec::new(); + walk_dir_collect(root, root, "rsync://example.net/repo/", &mut out).expect("walk"); + out.sort_by(|a, b| a.0.cmp(&b.0)); + + assert_eq!(out.len(), 2); + assert_eq!(out[0].0, "rsync://example.net/repo/b/c.mft"); + assert_eq!(out[0].1, b"y"); + assert_eq!(out[1].0, "rsync://example.net/repo/sub/a.cer"); + assert_eq!(out[1].1, b"x"); + } + + #[test] + fn rsync_module_root_uri_returns_host_and_module_only() { + assert_eq!( + rsync_module_root_uri("rsync://example.net/repo/ta/ca/publication-point/"), + Some("rsync://example.net/repo/".to_string()) + ); + assert_eq!( + rsync_module_root_uri("rsync://example.net/repo/ta/"), + Some("rsync://example.net/repo/".to_string()) + ); + assert_eq!( + rsync_module_root_uri("rsync://example.net/repo/"), + Some("rsync://example.net/repo/".to_string()) + ); + assert_eq!(rsync_module_root_uri("https://example.net/repo"), None); + } + + #[test] + fn rsync_host_scope_uri_returns_host_only() { + assert_eq!( + rsync_host_scope_uri("rsync://example.net/repo/ta/ca/publication-point/"), + Some("rsync://example.net/".to_string()) + ); + assert_eq!(rsync_host_scope_uri("https://example.net/repo"), None); + } + + #[test] + fn system_rsync_dedup_key_uses_module_root_by_default() { + let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig::default()); + assert_eq!( + fetcher.dedup_key("rsync://example.net/repo/ta/ca/publication-point/"), + "rsync://example.net/repo/" + ); + assert_eq!( + fetcher.failure_dedup_key("rsync://example.net/repo/ta/ca/publication-point/"), + None + ); + } + + #[test] + fn system_rsync_host_scope_does_not_widen_success_fetch_scope() { + let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig { + scope_policy: RsyncScopePolicy::Host, + ..SystemRsyncConfig::default() + }); + assert_eq!( + fetcher.dedup_key("rsync://example.net/repo/ta/ca/publication-point/"), + "rsync://example.net/repo/ta/ca/publication-point/" + ); + assert_eq!( + fetcher.scope_fetch_uri("rsync://example.net/repo/ta/ca/publication-point/"), + "rsync://example.net/repo/ta/ca/publication-point/" + ); + assert_eq!( + fetcher.failure_dedup_key("rsync://example.net/repo/ta/ca/publication-point/"), + Some("rsync://example.net/".to_string()) + ); + } + + #[test] + fn system_rsync_dedup_key_uses_publication_point_scope_when_configured() { + let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig { + scope_policy: RsyncScopePolicy::PublicationPoint, + ..SystemRsyncConfig::default() + }); + assert_eq!( + fetcher.dedup_key("rsync://example.net/repo/ta/ca/publication-point/"), + "rsync://example.net/repo/ta/ca/publication-point/" + ); + } + + #[test] + fn system_rsync_dedup_key_uses_module_root_when_configured() { + let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig { + scope_policy: RsyncScopePolicy::ModuleRoot, + ..SystemRsyncConfig::default() + }); + assert_eq!( + fetcher.dedup_key("rsync://example.net/repo/ta/ca/publication-point/"), + "rsync://example.net/repo/" + ); + } + + #[test] + fn system_rsync_fetcher_reports_spawn_and_exit_errors() { + let dst = tempfile::tempdir().expect("tempdir"); + + // 1) Spawn error. + let f = SystemRsyncFetcher::new(SystemRsyncConfig { + rsync_bin: PathBuf::from("/this/does/not/exist/rsync"), + connect_timeout: Duration::from_secs(1), + timeout: Duration::from_secs(1), + extra_args: Vec::new(), + mirror_root: None, + scope_policy: RsyncScopePolicy::default(), + }); + let e = f + .run_rsync("rsync://example.net/repo/", dst.path()) + .expect_err("spawn must fail"); + assert!(e.contains("rsync spawn failed:"), "{e}"); + + // 2) Non-zero exit status. + let f = SystemRsyncFetcher::new(SystemRsyncConfig { + rsync_bin: PathBuf::from("false"), + connect_timeout: Duration::from_secs(1), + timeout: Duration::from_secs(1), + extra_args: Vec::new(), + mirror_root: None, + scope_policy: RsyncScopePolicy::default(), + }); + let e = f + .run_rsync("rsync://example.net/repo/", dst.path()) + .expect_err("false must fail"); + assert!(e.contains("rsync failed:"), "{e}"); + assert!(e.contains("status="), "{e}"); + } + + #[test] + fn mirror_dst_dir_reports_root_creation_error() { + let temp = tempfile::tempdir().expect("tempdir"); + let root_file = temp.path().join("mirror-root-file"); + std::fs::write(&root_file, b"not a directory").expect("write root file"); + + let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig { + rsync_bin: PathBuf::from("rsync"), + connect_timeout: Duration::from_secs(1), + timeout: Duration::from_secs(1), + extra_args: Vec::new(), + mirror_root: Some(root_file.clone()), + scope_policy: RsyncScopePolicy::default(), + }); + + let err = fetcher + .mirror_dst_dir("rsync://example.net/repo/") + .expect_err("file mirror root must fail"); + assert!(err.contains("create rsync mirror root failed"), "{err}"); + assert!(err.contains(&root_file.display().to_string()), "{err}"); + } + + #[cfg(unix)] + #[test] + fn mirror_dst_dir_reports_directory_creation_error_inside_root() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().expect("tempdir"); + let root = temp.path().join("mirror"); + std::fs::create_dir_all(&root).expect("mkdir root"); + let mut perms = std::fs::metadata(&root).expect("metadata").permissions(); + perms.set_mode(0o555); + std::fs::set_permissions(&root, perms).expect("chmod root readonly"); + + let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig { + rsync_bin: PathBuf::from("rsync"), + connect_timeout: Duration::from_secs(1), + timeout: Duration::from_secs(1), + extra_args: Vec::new(), + mirror_root: Some(root.clone()), + scope_policy: RsyncScopePolicy::default(), + }); + + let err = fetcher + .mirror_dst_dir("rsync://example.net/repo/") + .expect_err("readonly mirror root must fail"); + assert!( + err.contains("create rsync mirror directory failed"), + "{err}" + ); + + let mut perms = std::fs::metadata(&root).expect("metadata").permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&root, perms).expect("restore perms"); + } + + #[cfg(unix)] + #[test] + fn walk_dir_collect_ignores_non_file_entries() { + use std::os::unix::net::UnixListener; + + let temp = tempfile::tempdir().expect("tempdir"); + let root = temp.path(); + std::fs::write(root.join("a.cer"), b"x").expect("write file"); + let socket_path = root.join("skip.sock"); + let _listener = UnixListener::bind(&socket_path).expect("bind socket"); + + let mut out: Vec<(String, Vec)> = Vec::new(); + walk_dir_collect(root, root, "rsync://example.net/repo/", &mut out).expect("walk"); + + assert_eq!(out.len(), 1); + assert_eq!(out[0].0, "rsync://example.net/repo/a.cer"); + } + + #[cfg(unix)] + #[test] + fn rsync_fail_fast_retries_when_progress_is_made() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().expect("tempdir"); + let script = temp.path().join("fake-rsync.sh"); + let state = temp.path().join("state.txt"); + std::fs::write( + &script, + format!( + "#!/usr/bin/env bash\nset -euo pipefail\nSTATE=\"{}\"\nDST=\"${{@: -1}}\"\nCOUNT=0\nif [[ -f \"$STATE\" ]]; then COUNT=$(cat \"$STATE\"); fi\nCOUNT=$((COUNT+1))\necho \"$COUNT\" > \"$STATE\"\nmkdir -p \"$DST\"\nif [[ \"$COUNT\" -eq 1 ]]; then\n echo first > \"$DST/part1\"\n sleep 2\nelse\n echo second > \"$DST/part2\"\nfi\n", + state.display() + ), + ) + .expect("write script"); + let mut perms = std::fs::metadata(&script).unwrap().permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&script, perms).unwrap(); + + let dst = temp.path().join("dst"); + let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig { + rsync_bin: script, + connect_timeout: Duration::from_secs(15), + timeout: Duration::from_secs(60), + extra_args: Vec::new(), + mirror_root: None, + scope_policy: RsyncScopePolicy::default(), + }); + + fetcher + .run_rsync_fail_fast( + "rsync://example.net/repo/", + &dst, + RsyncFailFastProfile { + initial_wall_clock_timeout: Duration::from_secs(1), + max_wall_clock_timeout: Duration::from_secs(4), + max_attempts: 3, + }, + ) + .expect("eventual success"); + + assert!(dst.join("part1").exists()); + assert!(dst.join("part2").exists()); + } + + #[cfg(unix)] + #[test] + fn rsync_fail_fast_gives_up_after_two_zero_progress_timeouts() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().expect("tempdir"); + let script = temp.path().join("fake-rsync.sh"); + std::fs::write(&script, "#!/usr/bin/env bash\nset -euo pipefail\nsleep 5\n") + .expect("write script"); + let mut perms = std::fs::metadata(&script).unwrap().permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&script, perms).unwrap(); + + let dst = temp.path().join("dst"); + let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig { + rsync_bin: script, + connect_timeout: Duration::from_secs(15), + timeout: Duration::from_secs(60), + extra_args: Vec::new(), + mirror_root: None, + scope_policy: RsyncScopePolicy::default(), + }); + + let err = fetcher + .run_rsync_fail_fast( + "rsync://example.net/repo/", + &dst, + RsyncFailFastProfile { + initial_wall_clock_timeout: Duration::from_secs(1), + max_wall_clock_timeout: Duration::from_secs(2), + max_attempts: 4, + }, + ) + .expect_err("must fail"); + assert!(err.contains("no progress"), "{err}"); + } + + #[cfg(unix)] + #[test] + fn rsync_fail_fast_hard_fail_stops_after_first_attempt() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().expect("tempdir"); + let script = temp.path().join("fake-rsync.sh"); + let state = temp.path().join("state.txt"); + std::fs::write( + &script, + format!( + "#!/usr/bin/env bash\nset -euo pipefail\nSTATE=\"{}\"\nCOUNT=0\nif [[ -f \"$STATE\" ]]; then COUNT=$(cat \"$STATE\"); fi\nCOUNT=$((COUNT+1))\necho \"$COUNT\" > \"$STATE\"\necho 'rsync: [Receiver] failed to connect to host (1.2.3.4): Connection refused (111)' >&2\nexit 10\n", + state.display() + ), + ) + .expect("write script"); + let mut perms = std::fs::metadata(&script).unwrap().permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&script, perms).unwrap(); + + let dst = temp.path().join("dst"); + let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig { + rsync_bin: script, + connect_timeout: Duration::from_secs(15), + timeout: Duration::from_secs(60), + extra_args: Vec::new(), + mirror_root: None, + scope_policy: RsyncScopePolicy::default(), + }); + + let err = fetcher + .run_rsync_fail_fast( + "rsync://example.net/repo/", + &dst, + RsyncFailFastProfile { + initial_wall_clock_timeout: Duration::from_secs(10), + max_wall_clock_timeout: Duration::from_secs(80), + max_attempts: 4, + }, + ) + .expect_err("must hard fail"); + assert!(err.contains("hard-fail"), "{err}"); + let count = std::fs::read_to_string(&state).unwrap(); + assert_eq!(count.trim(), "1"); + } + + #[cfg(unix)] + #[test] + fn run_rsync_once_passes_contimeout_and_timeout_args() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().expect("tempdir"); + let script = temp.path().join("capture-rsync.sh"); + let args_file = temp.path().join("args.txt"); + std::fs::write( + &script, + format!( + "#!/usr/bin/env bash\nset -euo pipefail\nprintf '%s\\n' \"$@\" > \"{}\"\nDST=\"${{@: -1}}\"\nmkdir -p \"$DST\"\n", + args_file.display() + ), + ) + .expect("write script"); + let mut perms = std::fs::metadata(&script).unwrap().permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&script, perms).unwrap(); + + let dst = temp.path().join("dst"); + let fetcher = SystemRsyncFetcher::new(SystemRsyncConfig { + rsync_bin: script, + connect_timeout: Duration::from_secs(15), + timeout: Duration::from_secs(30), + extra_args: Vec::new(), + mirror_root: None, + scope_policy: RsyncScopePolicy::default(), + }); + + fetcher + .run_rsync_once("rsync://example.net/repo/", &dst, None, false) + .expect("rsync"); + + let args = std::fs::read_to_string(&args_file).expect("read args"); + assert!(args.contains("--contimeout\n15\n"), "{args}"); + assert!(args.contains("--timeout\n30\n"), "{args}"); + } +} diff --git a/crates/panda-rpki-validator/src/lib.rs b/crates/panda-rpki-validator/src/lib.rs new file mode 100644 index 0000000..fd9ed65 --- /dev/null +++ b/crates/panda-rpki-validator/src/lib.rs @@ -0,0 +1,54 @@ +//! Public package boundary for the Panda RPKI synchronization validator. +//! +//! The package intentionally keeps the audited implementation in one crate +//! while exposing only the normal synchronization/validation CLI. A separate +//! offline revalidation workflow remains in the original private repository +//! until a follow-up backlog item defines its public contract. +//! +//! The implementation is extracted from the private `rpki` tree. During the +//! parity window we intentionally keep the upstream implementation's lint +//! profile unchanged; functional and output-equivalence gates take precedence +//! over behavior-changing lint cleanup. +#![allow(clippy::all)] +#![allow(clippy::pedantic, clippy::nursery, clippy::restriction)] +#![allow(dead_code, deprecated, unused_mut)] + +pub mod analysis; +pub mod audit; +pub mod audit_downloads; +pub mod blob_store; +pub mod ccr; +pub mod cir; +pub mod cli; +pub mod contract; +pub mod crypto_sig_cache; +pub mod current_repo_index; +pub mod daemon; +pub mod data_model; +pub mod fetch; +pub mod memory_telemetry; +pub mod object_projection; +pub mod parallel; +pub mod policy; +pub mod progress_log; +pub mod replay; +pub mod report; +pub mod storage; +pub mod sync; +pub mod ta_constraints; +pub mod validation; + +pub const COMPONENT_NAME: &str = "panda-rpki-validator"; +pub const COMPONENT_VERSION: &str = env!("CARGO_PKG_VERSION"); + +/// Runs the normal synchronization/validation CLI. +pub fn run(args: I) -> Result<(), String> +where + I: IntoIterator, + S: Into, +{ + let mut argv = Vec::with_capacity(1); + argv.push(COMPONENT_NAME.to_string()); + argv.extend(args.into_iter().map(Into::into)); + cli::run(&argv) +} diff --git a/crates/panda-rpki-validator/src/main.rs b/crates/panda-rpki-validator/src/main.rs new file mode 100644 index 0000000..7c07a6e --- /dev/null +++ b/crates/panda-rpki-validator/src/main.rs @@ -0,0 +1,6 @@ +fn main() { + if let Err(error) = panda_rpki_validator::run(std::env::args().skip(1)) { + eprintln!("panda-rpki-validator: {error}"); + std::process::exit(2); + } +} diff --git a/crates/panda-rpki-validator/src/memory_telemetry.rs b/crates/panda-rpki-validator/src/memory_telemetry.rs new file mode 100644 index 0000000..598e1ee --- /dev/null +++ b/crates/panda-rpki-validator/src/memory_telemetry.rs @@ -0,0 +1,348 @@ +use serde::Serialize; + +use crate::storage::RocksDbMemorySnapshot; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct MallocTrimProbe { + pub supported: bool, + pub return_value: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct ProcessMemorySnapshot { + pub label: String, + pub vm_rss_kb: Option, + pub vm_size_kb: Option, + pub vm_data_kb: Option, + pub vm_swap_kb: Option, + pub rss_anon_kb: Option, + pub rss_file_kb: Option, + pub rss_shmem_kb: Option, + pub threads: Option, + pub fd_count: Option, + pub smaps_rollup: Option, + pub smaps_mapping_summary: Option, + pub errors: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct SmapsRollupSnapshot { + pub rss_kb: Option, + pub pss_kb: Option, + pub shared_clean_kb: Option, + pub shared_dirty_kb: Option, + pub private_clean_kb: Option, + pub private_dirty_kb: Option, + pub anonymous_kb: Option, + pub swap_kb: Option, + pub swap_pss_kb: Option, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct SmapsMappingSummary { + pub heap: SmapsMappingCategory, + pub anonymous_mmap: SmapsMappingCategory, + pub file_backed: SmapsMappingCategory, + pub stack: SmapsMappingCategory, + pub special: SmapsMappingCategory, + pub total: SmapsMappingCategory, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct SmapsMappingCategory { + pub mappings: u64, + pub size_kb: u64, + pub rss_kb: u64, + pub pss_kb: u64, + pub private_clean_kb: u64, + pub private_dirty_kb: u64, + pub anonymous_kb: u64, + pub largest_mapping_rss_kb: u64, + pub large_mapping_count_64m: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct MemoryTelemetryCheckpoint { + pub label: String, + pub elapsed_ms: u64, + pub process: ProcessMemorySnapshot, + pub rocksdb: RocksDbMemorySnapshot, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct MemoryTelemetrySummary { + pub checkpoints: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub object_graph: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub malloc_trim_probes: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct ObjectGraphMemorySummary { + pub captured_at_label: String, + pub total_estimated_bytes: u64, + pub sections: Vec, + pub notes: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct ObjectGraphMemorySection { + pub name: String, + pub item_count: u64, + pub shallow_bytes: u64, + pub heap_bytes: u64, + pub estimated_bytes: u64, + pub string_count: u64, + pub string_bytes: u64, + pub string_capacity_bytes: u64, + pub vec_count: u64, + pub vec_heap_bytes: u64, + pub vec_capacity_bytes: u64, + pub details: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct ObjectGraphMemoryMetric { + pub name: String, + pub value: u64, +} + +pub fn process_memory_snapshot(label: impl Into) -> ProcessMemorySnapshot { + let label = label.into(); + let mut snapshot = ProcessMemorySnapshot { + label, + vm_rss_kb: None, + vm_size_kb: None, + vm_data_kb: None, + vm_swap_kb: None, + rss_anon_kb: None, + rss_file_kb: None, + rss_shmem_kb: None, + threads: None, + fd_count: current_fd_count(), + smaps_rollup: None, + smaps_mapping_summary: None, + errors: Vec::new(), + }; + + match std::fs::read_to_string("/proc/self/status") { + Ok(status) => parse_status(&status, &mut snapshot), + Err(err) => snapshot + .errors + .push(format!("read /proc/self/status failed: {err}")), + } + + match std::fs::read_to_string("/proc/self/smaps_rollup") { + Ok(smaps) => snapshot.smaps_rollup = Some(parse_smaps_rollup(&smaps)), + Err(err) => snapshot + .errors + .push(format!("read /proc/self/smaps_rollup failed: {err}")), + } + + match std::fs::read_to_string("/proc/self/smaps") { + Ok(smaps) => snapshot.smaps_mapping_summary = Some(parse_smaps_mapping_summary(&smaps)), + Err(err) => snapshot + .errors + .push(format!("read /proc/self/smaps failed: {err}")), + } + + snapshot +} + +pub fn malloc_trim_probe() -> MallocTrimProbe { + #[cfg(all(target_os = "linux", target_env = "gnu"))] + { + MallocTrimProbe { + supported: true, + return_value: Some(unsafe { malloc_trim(0) }), + } + } + #[cfg(not(all(target_os = "linux", target_env = "gnu")))] + { + MallocTrimProbe { + supported: false, + return_value: None, + } + } +} + +#[cfg(all(target_os = "linux", target_env = "gnu"))] +unsafe extern "C" { + fn malloc_trim(pad: usize) -> i32; +} + +fn current_fd_count() -> Option { + std::fs::read_dir("/proc/self/fd") + .ok() + .map(|entries| entries.filter_map(Result::ok).count() as u64) +} + +fn parse_status(status: &str, snapshot: &mut ProcessMemorySnapshot) { + for line in status.lines() { + let Some((key, value)) = line.split_once(':') else { + continue; + }; + let parsed = parse_kb_or_plain_u64(value); + match key { + "VmRSS" => snapshot.vm_rss_kb = parsed, + "VmSize" => snapshot.vm_size_kb = parsed, + "VmData" => snapshot.vm_data_kb = parsed, + "VmSwap" => snapshot.vm_swap_kb = parsed, + "RssAnon" => snapshot.rss_anon_kb = parsed, + "RssFile" => snapshot.rss_file_kb = parsed, + "RssShmem" => snapshot.rss_shmem_kb = parsed, + "Threads" => snapshot.threads = parsed, + _ => {} + } + } +} + +fn parse_smaps_rollup(smaps: &str) -> SmapsRollupSnapshot { + let mut snapshot = SmapsRollupSnapshot::default(); + for line in smaps.lines() { + let Some((key, value)) = line.split_once(':') else { + continue; + }; + let parsed = parse_kb_or_plain_u64(value); + match key { + "Rss" => snapshot.rss_kb = parsed, + "Pss" => snapshot.pss_kb = parsed, + "Shared_Clean" => snapshot.shared_clean_kb = parsed, + "Shared_Dirty" => snapshot.shared_dirty_kb = parsed, + "Private_Clean" => snapshot.private_clean_kb = parsed, + "Private_Dirty" => snapshot.private_dirty_kb = parsed, + "Anonymous" => snapshot.anonymous_kb = parsed, + "Swap" => snapshot.swap_kb = parsed, + "SwapPss" => snapshot.swap_pss_kb = parsed, + _ => {} + } + } + snapshot +} + +fn parse_smaps_mapping_summary(smaps: &str) -> SmapsMappingSummary { + let mut summary = SmapsMappingSummary::default(); + let mut current_path = String::new(); + let mut current = SmapsMappingCategory::default(); + let mut have_mapping = false; + + for line in smaps.lines() { + if is_smaps_mapping_header(line) { + if have_mapping { + add_mapping(&mut summary, ¤t_path, ¤t); + } + current_path = smaps_header_path(line); + current = SmapsMappingCategory { + mappings: 1, + ..SmapsMappingCategory::default() + }; + have_mapping = true; + continue; + } + + if !have_mapping { + continue; + } + + let Some((key, value)) = line.split_once(':') else { + continue; + }; + let parsed = parse_kb_or_plain_u64(value).unwrap_or(0); + match key { + "Size" => current.size_kb = parsed, + "Rss" => current.rss_kb = parsed, + "Pss" => current.pss_kb = parsed, + "Private_Clean" => current.private_clean_kb = parsed, + "Private_Dirty" => current.private_dirty_kb = parsed, + "Anonymous" => current.anonymous_kb = parsed, + _ => {} + } + } + + if have_mapping { + add_mapping(&mut summary, ¤t_path, ¤t); + } + + summary +} + +fn is_smaps_mapping_header(line: &str) -> bool { + let mut parts = line.split_whitespace(); + let Some(range) = parts.next() else { + return false; + }; + let Some(perms) = parts.next() else { + return false; + }; + let Some((start, end)) = range.split_once('-') else { + return false; + }; + !start.is_empty() + && !end.is_empty() + && start.as_bytes().iter().all(u8::is_ascii_hexdigit) + && end.as_bytes().iter().all(u8::is_ascii_hexdigit) + && perms.len() == 4 + && perms + .as_bytes() + .iter() + .all(|b| matches!(b, b'r' | b'w' | b'x' | b's' | b'p' | b'-')) +} + +fn smaps_header_path(line: &str) -> String { + line.split_whitespace() + .skip(5) + .collect::>() + .join(" ") +} + +fn add_mapping(summary: &mut SmapsMappingSummary, path: &str, mapping: &SmapsMappingCategory) { + add_category(&mut summary.total, mapping); + match mapping_category(path) { + MappingCategory::Heap => add_category(&mut summary.heap, mapping), + MappingCategory::AnonymousMmap => add_category(&mut summary.anonymous_mmap, mapping), + MappingCategory::FileBacked => add_category(&mut summary.file_backed, mapping), + MappingCategory::Stack => add_category(&mut summary.stack, mapping), + MappingCategory::Special => add_category(&mut summary.special, mapping), + } +} + +fn add_category(target: &mut SmapsMappingCategory, source: &SmapsMappingCategory) { + target.mappings += source.mappings; + target.size_kb += source.size_kb; + target.rss_kb += source.rss_kb; + target.pss_kb += source.pss_kb; + target.private_clean_kb += source.private_clean_kb; + target.private_dirty_kb += source.private_dirty_kb; + target.anonymous_kb += source.anonymous_kb; + target.largest_mapping_rss_kb = target.largest_mapping_rss_kb.max(source.rss_kb); + if source.rss_kb >= 64 * 1024 { + target.large_mapping_count_64m += source.mappings; + } +} + +enum MappingCategory { + Heap, + AnonymousMmap, + FileBacked, + Stack, + Special, +} + +fn mapping_category(path: &str) -> MappingCategory { + if path == "[heap]" { + MappingCategory::Heap + } else if path.starts_with("[stack") { + MappingCategory::Stack + } else if path.is_empty() { + MappingCategory::AnonymousMmap + } else if path.starts_with('/') { + MappingCategory::FileBacked + } else { + MappingCategory::Special + } +} + +fn parse_kb_or_plain_u64(value: &str) -> Option { + value.split_whitespace().next()?.parse::().ok() +} diff --git a/crates/panda-rpki-validator/src/object_projection.rs b/crates/panda-rpki-validator/src/object_projection.rs new file mode 100644 index 0000000..da39843 --- /dev/null +++ b/crates/panda-rpki-validator/src/object_projection.rs @@ -0,0 +1,668 @@ +use std::net::{Ipv4Addr, Ipv6Addr}; +use std::path::Path; + +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use sha2::{Digest, Sha256}; + +use crate::data_model::aspa::AspaObject; +use crate::data_model::crl::RpkixCrl; +use crate::data_model::manifest::ManifestObject; +use crate::data_model::rc::{ + AccessDescription, RcExtensions, ResourceCertificate, SubjectInfoAccess, +}; +use crate::data_model::roa::{IpPrefix as RoaIpPrefix, RoaAfi, RoaObject}; +use crate::data_model::signed_object::{ + ResourceEeCertificate, RpkiSignedObject, SignedAttrsProfiled, SignerInfoProfiled, +}; +use crate::data_model::ta::TaCertificate; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ObjectType { + Auto, + Cer, + Mft, + Crl, + Roa, + Aspa, +} + +impl ObjectType { + pub fn parse(value: &str) -> Result { + match value.to_ascii_lowercase().as_str() { + "auto" => Ok(Self::Auto), + "cer" | ".cer" | "cert" | "certificate" => Ok(Self::Cer), + "mft" | ".mft" | "manifest" => Ok(Self::Mft), + "crl" | ".crl" => Ok(Self::Crl), + "roa" | ".roa" => Ok(Self::Roa), + "asa" | ".asa" | "aspa" => Ok(Self::Aspa), + _ => Err(format!("unsupported object type: {value}")), + } + } + + pub fn label(self) -> &'static str { + object_type_label(self) + } +} + +pub fn resolve_object_type(object_type: ObjectType, path: &Path) -> Result { + if object_type != ObjectType::Auto { + return Ok(object_type); + } + match path + .extension() + .and_then(|v| v.to_str()) + .map(|v| v.to_ascii_lowercase()) + .as_deref() + { + Some("cer") => Ok(ObjectType::Cer), + Some("mft") => Ok(ObjectType::Mft), + Some("crl") => Ok(ObjectType::Crl), + Some("roa") => Ok(ObjectType::Roa), + Some("asa") | Some("aspa") => Ok(ObjectType::Aspa), + _ => Err(format!( + "cannot infer object type from path: {}", + path.display() + )), + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ObjectProjectionRecord { + pub schema_version: u32, + pub sha256: String, + pub object_type: String, + pub parse_status: String, + pub error_summary: Option, + pub projection: Value, +} + +pub fn build_object_projection( + object_type: ObjectType, + input_path: &Path, + bytes: &[u8], + entry_limit: usize, +) -> ObjectProjectionRecord { + let resolved = match resolve_object_type(object_type, input_path) { + Ok(value) => value, + Err(err) => { + return ObjectProjectionRecord { + schema_version: 1, + sha256: sha256_hex(bytes), + object_type: "unknown".to_string(), + parse_status: "error".to_string(), + error_summary: Some(err), + projection: json!({"decode": {"profileValid": false}}), + }; + } + }; + let projection = parse_object_json(resolved, input_path, bytes, entry_limit); + let parse_status = if projection + .get("object") + .and_then(|v| v.get("decode")) + .and_then(|v| v.get("profileValid")) + .and_then(Value::as_bool) + .unwrap_or(false) + { + "ok" + } else { + "error" + }; + let error_summary = projection + .get("object") + .and_then(|v| v.get("decode")) + .and_then(|v| v.get("error")) + .and_then(Value::as_str) + .map(str::to_string); + ObjectProjectionRecord { + schema_version: 1, + sha256: sha256_hex(bytes), + object_type: resolved.label().to_string(), + parse_status: parse_status.to_string(), + error_summary, + projection, + } +} + +pub fn parse_object_json( + object_type: ObjectType, + input_path: &Path, + bytes: &[u8], + entry_limit: usize, +) -> Value { + let object = match object_type { + ObjectType::Auto => unreachable!("auto must be resolved"), + ObjectType::Cer => parse_cer_json(bytes), + ObjectType::Mft => parse_mft_json(bytes, entry_limit), + ObjectType::Crl => parse_crl_json(bytes, entry_limit), + ObjectType::Roa => parse_roa_json(bytes, entry_limit), + ObjectType::Aspa => parse_aspa_json(bytes, entry_limit), + }; + json!({ + "tool": "rpki_object_parse", + "schemaVersion": 1, + "input": { + "path": input_path.display().to_string(), + "type": object_type_label(object_type), + "bytes": bytes_summary(bytes), + }, + "object": object, + }) +} + +pub fn parse_cer_json(bytes: &[u8]) -> Value { + match ResourceCertificate::decode_der(bytes) { + Ok(cert) => { + let ta_profile = match TaCertificate::decode_der(bytes) { + Ok(ta) => json!({ + "valid": true, + "selfSignature": result_json(ta.verify_self_signature().map_err(|e| e.to_string())), + }), + Err(err) => json!({ + "valid": false, + "error": err.to_string(), + }), + }; + json!({ + "type": "cer", + "decode": {"profileValid": true}, + "resourceCertificate": resource_certificate_json(&cert), + "trustAnchorProfile": ta_profile, + }) + } + Err(err) => json!({ + "type": "cer", + "decode": {"profileValid": false, "error": err.to_string()}, + }), + } +} + +pub fn parse_mft_json(bytes: &[u8], entry_limit: usize) -> Value { + match ManifestObject::decode_der(bytes) { + Ok(mft) => { + let files = mft.manifest.parse_files(); + let (file_sample, file_list_error) = match files { + Ok(entries) => ( + json!({ + "count": entries.len(), + "truncated": entries.len() > entry_limit, + "entries": entries.iter().take(entry_limit).map(|item| { + json!({"fileName": item.file_name, "hashHex": hex::encode(item.hash_bytes)}) + }).collect::>(), + }), + Value::Null, + ), + Err(err) => (Value::Null, json!(err.to_string())), + }; + json!({ + "type": "mft", + "decode": {"profileValid": true}, + "eContentType": mft.econtent_type, + "signedObject": signed_object_json(&mft.signed_object), + "manifest": { + "version": mft.manifest.version, + "manifestNumberHex": mft.manifest.manifest_number.to_hex_upper(), + "thisUpdate": format_time(mft.manifest.this_update), + "nextUpdate": format_time(mft.manifest.next_update), + "fileHashAlg": mft.manifest.file_hash_alg, + "fileCount": mft.manifest.file_count(), + "fileList": file_sample, + "fileListError": file_list_error, + }, + "embeddedEeProfile": result_json(mft.validate_embedded_ee_cert().map_err(|e| e.to_string())), + "cmsSignature": result_json(mft.signed_object.verify_signature().map_err(|e| e.to_string())), + }) + } + Err(err) => json!({ + "type": "mft", + "decode": {"profileValid": false, "error": err.to_string()}, + }), + } +} + +pub fn parse_crl_json(bytes: &[u8], entry_limit: usize) -> Value { + match RpkixCrl::decode_der(bytes) { + Ok(crl) => json!({ + "type": "crl", + "decode": {"profileValid": true}, + "rawDer": bytes_summary(&crl.raw_der), + "version": crl.version, + "issuer": crl.issuer_dn, + "signatureAlgorithm": crl.signature_algorithm_oid, + "thisUpdate": format_time(crl.this_update.utc), + "nextUpdate": format_time(crl.next_update.utc), + "extensions": { + "authorityKeyIdentifier": hex::encode(&crl.extensions.authority_key_identifier), + "crlNumberHex": crl.extensions.crl_number.to_hex_upper(), + "crlNumber": crl.extensions.crl_number.to_u64(), + }, + "revokedCertificates": { + "count": crl.revoked_certs.len(), + "truncated": crl.revoked_certs.len() > entry_limit, + "entries": crl.revoked_certs.iter().take(entry_limit).map(|item| { + json!({ + "serialNumberHex": item.serial_number.to_hex_upper(), + "serialNumber": item.serial_number.to_u64(), + "revocationDate": format_time(item.revocation_date.utc), + }) + }).collect::>(), + }, + }), + Err(err) => json!({ + "type": "crl", + "decode": {"profileValid": false, "error": err.to_string()}, + }), + } +} + +pub fn manifest_file_entries_page( + bytes: &[u8], + offset: usize, + limit: usize, +) -> Result<(usize, Vec), String> { + let mft = ManifestObject::decode_der(bytes).map_err(|err| err.to_string())?; + let entries = mft.manifest.parse_files().map_err(|err| err.to_string())?; + let total = entries.len(); + let end = (offset + limit).min(total); + let page = entries[offset.min(total)..end] + .iter() + .map(|item| json!({"fileName": item.file_name, "hashHex": hex::encode(&item.hash_bytes)})) + .collect::>(); + Ok((total, page)) +} + +pub fn crl_revoked_entries_page( + bytes: &[u8], + offset: usize, + limit: usize, +) -> Result<(usize, Vec), String> { + let crl = RpkixCrl::decode_der(bytes).map_err(|err| err.to_string())?; + let total = crl.revoked_certs.len(); + let end = (offset + limit).min(total); + let page = crl.revoked_certs[offset.min(total)..end] + .iter() + .map(|item| { + json!({ + "serialNumberHex": item.serial_number.to_hex_upper(), + "serialNumber": item.serial_number.to_u64(), + "revocationDate": format_time(item.revocation_date.utc), + }) + }) + .collect::>(); + Ok((total, page)) +} + +pub fn parse_roa_json(bytes: &[u8], entry_limit: usize) -> Value { + match RoaObject::decode_der(bytes) { + Ok(roa) => json!({ + "type": "roa", + "decode": {"profileValid": true}, + "eContentType": roa.econtent_type, + "signedObject": signed_object_json(&roa.signed_object), + "roa": { + "version": roa.roa.version, + "asId": roa.roa.as_id, + "ipAddressFamilies": roa.roa.ip_addr_blocks.iter().map(|family| { + json!({ + "afi": format!("{:?}", family.afi), + "addressCount": family.addresses.len(), + "truncated": family.addresses.len() > entry_limit, + "addresses": family.addresses.iter().take(entry_limit).map(|entry| { + json!({ + "prefix": roa_prefix_string(&entry.prefix), + "maxLength": entry.max_length, + }) + }).collect::>(), + }) + }).collect::>(), + }, + "embeddedEeProfile": result_json(roa.validate_embedded_ee_cert().map_err(|e| e.to_string())), + "cmsSignature": result_json(roa.signed_object.verify_signature().map_err(|e| e.to_string())), + }), + Err(err) => json!({ + "type": "roa", + "decode": {"profileValid": false, "error": err.to_string()}, + }), + } +} + +pub fn parse_aspa_json(bytes: &[u8], entry_limit: usize) -> Value { + match AspaObject::decode_der(bytes) { + Ok(aspa) => json!({ + "type": "aspa", + "decode": {"profileValid": true}, + "eContentType": aspa.econtent_type, + "signedObject": signed_object_json(&aspa.signed_object), + "aspa": { + "version": aspa.aspa.version, + "customerAsId": aspa.aspa.customer_as_id, + "providerCount": aspa.aspa.provider_as_ids.len(), + "providersTruncated": aspa.aspa.provider_as_ids.len() > entry_limit, + "providerAsIds": aspa.aspa.provider_as_ids.iter().take(entry_limit).copied().collect::>(), + }, + "embeddedEeProfile": result_json(aspa.validate_embedded_ee_cert().map_err(|e| e.to_string())), + "cmsSignature": result_json(aspa.signed_object.verify_signature().map_err(|e| e.to_string())), + }), + Err(err) => json!({ + "type": "aspa", + "decode": {"profileValid": false, "error": err.to_string()}, + }), + } +} + +fn resource_certificate_json(cert: &ResourceCertificate) -> Value { + let tbs = &cert.tbs; + json!({ + "rawDer": bytes_summary(&cert.raw_der), + "kind": format!("{:?}", cert.kind), + "version": tbs.version, + "serialNumberHex": hex::encode(tbs.serial_number.to_bytes_be()), + "signatureAlgorithm": tbs.signature_algorithm, + "issuer": tbs.issuer_name.to_string(), + "subject": tbs.subject_name.to_string(), + "validity": { + "notBefore": format_time(tbs.validity_not_before), + "notAfter": format_time(tbs.validity_not_after), + }, + "subjectPublicKeyInfo": bytes_summary(&tbs.subject_public_key_info), + "extensions": rc_extensions_json(&tbs.extensions), + }) +} + +fn rc_extensions_json(ext: &RcExtensions) -> Value { + json!({ + "basicConstraintsCa": ext.basic_constraints_ca, + "subjectKeyIdentifier": ext.subject_key_identifier.as_ref().map(|v| hex::encode(v)), + "authorityKeyIdentifier": ext.authority_key_identifier.as_ref().map(|v| hex::encode(v)), + "crlDistributionPointsUris": ext.crl_distribution_points_uris, + "caIssuersUris": ext.ca_issuers_uris, + "subjectInfoAccess": subject_info_access_json(ext.subject_info_access.as_ref()), + "certificatePoliciesOid": ext.certificate_policies_oid, + "ipResources": serde_json::to_value(&ext.ip_resources).unwrap_or(Value::Null), + "asResources": serde_json::to_value(&ext.as_resources).unwrap_or(Value::Null), + }) +} + +fn subject_info_access_json(value: Option<&SubjectInfoAccess>) -> Value { + match value { + None => Value::Null, + Some(SubjectInfoAccess::Ca(ca)) => json!({ + "kind": "ca", + "accessDescriptions": ca.access_descriptions.iter().map(access_description_json).collect::>(), + }), + Some(SubjectInfoAccess::Ee(ee)) => json!({ + "kind": "ee", + "signedObjectUris": ee.signed_object_uris, + "accessDescriptions": ee.access_descriptions.iter().map(access_description_json).collect::>(), + }), + } +} + +fn access_description_json(value: &AccessDescription) -> Value { + json!({ + "accessMethodOid": value.access_method_oid, + "accessLocation": value.access_location, + }) +} + +fn signed_object_json(signed_object: &RpkiSignedObject) -> Value { + let signed_data = &signed_object.signed_data; + json!({ + "rawDer": bytes_summary(&signed_object.raw_der), + "contentInfoContentType": signed_object.content_info_content_type, + "signedData": { + "version": signed_data.version, + "digestAlgorithms": signed_data.digest_algorithms, + "encapContentInfo": { + "eContentType": signed_data.encap_content_info.econtent_type, + "eContent": bytes_summary(&signed_data.encap_content_info.econtent), + }, + "certificates": signed_data.certificates.iter().map(ee_certificate_json).collect::>(), + "crlsPresent": signed_data.crls_present, + "signerInfos": signed_data.signer_infos.iter().map(signer_info_json).collect::>(), + }, + }) +} + +fn ee_certificate_json(cert: &ResourceEeCertificate) -> Value { + json!({ + "rawDer": bytes_summary(&cert.raw_der), + "subjectKeyIdentifier": hex::encode(&cert.subject_key_identifier), + "spkiDer": bytes_summary(&cert.spki_der), + "rsaPublicKey": { + "modulus": bytes_summary(&cert.rsa_public_modulus), + "exponent": bytes_summary(&cert.rsa_public_exponent), + }, + "tbsCertificate": bytes_summary(&cert.tbs_certificate_der), + "certificateSignature": bytes_summary(&cert.signature_bytes), + "keyUsageSummary": format!("{:?}", cert.key_usage_summary), + "siaSignedObjectUris": cert.sia_signed_object_uris, + "resourceCertificate": resource_certificate_json(&cert.resource_cert), + }) +} + +fn signer_info_json(info: &SignerInfoProfiled) -> Value { + json!({ + "version": info.version, + "sidSki": hex::encode(&info.sid_ski), + "digestAlgorithm": info.digest_algorithm, + "signatureAlgorithm": info.signature_algorithm, + "signedAttrs": signed_attrs_json(&info.signed_attrs), + "unsignedAttrsPresent": info.unsigned_attrs_present, + "signature": bytes_summary(&info.signature), + "signedAttrsDerForSignature": bytes_summary(&info.signed_attrs_der_for_signature), + }) +} + +fn signed_attrs_json(attrs: &SignedAttrsProfiled) -> Value { + json!({ + "contentType": attrs.content_type, + "messageDigest": hex::encode(&attrs.message_digest), + "signingTime": { + "utc": format_time(attrs.signing_time.utc), + "encoding": format!("{:?}", attrs.signing_time.encoding), + }, + "otherAttrsPresent": attrs.other_attrs_present, + }) +} + +fn result_json(result: Result<(), String>) -> Value { + match result { + Ok(()) => json!({"valid": true}), + Err(err) => json!({"valid": false, "error": err}), + } +} + +fn object_type_label(object_type: ObjectType) -> &'static str { + match object_type { + ObjectType::Auto => "auto", + ObjectType::Cer => "cer", + ObjectType::Mft => "mft", + ObjectType::Crl => "crl", + ObjectType::Roa => "roa", + ObjectType::Aspa => "aspa", + } +} + +fn bytes_summary(bytes: &[u8]) -> Value { + let head_len = bytes.len().min(16); + let tail_len = bytes.len().min(16); + json!({ + "len": bytes.len(), + "sha256": sha256_hex(bytes), + "headHex": hex::encode(&bytes[..head_len]), + "tailHex": hex::encode(&bytes[bytes.len().saturating_sub(tail_len)..]), + }) +} + +fn sha256_hex(bytes: &[u8]) -> String { + hex::encode(Sha256::digest(bytes)) +} + +fn format_time(value: time::OffsetDateTime) -> String { + value + .to_offset(time::UtcOffset::UTC) + .format(&time::format_description::well_known::Rfc3339) + .unwrap_or_else(|_| value.unix_timestamp().to_string()) +} + +fn roa_prefix_string(prefix: &RoaIpPrefix) -> String { + let bytes = prefix.addr_bytes(); + match prefix.afi { + RoaAfi::Ipv4 => { + let octets = [bytes[0], bytes[1], bytes[2], bytes[3]]; + format!("{}/{}", Ipv4Addr::from(octets), prefix.prefix_len) + } + RoaAfi::Ipv6 => { + let mut octets = [0u8; 16]; + octets.copy_from_slice(bytes); + format!("{}/{}", Ipv6Addr::from(octets), prefix.prefix_len) + } + } +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use super::*; + + #[test] + fn object_type_parser_and_resolver_cover_aliases() { + assert_eq!(ObjectType::parse("auto").unwrap(), ObjectType::Auto); + assert_eq!(ObjectType::parse(".cer").unwrap(), ObjectType::Cer); + assert_eq!(ObjectType::parse("certificate").unwrap(), ObjectType::Cer); + assert_eq!(ObjectType::parse("manifest").unwrap(), ObjectType::Mft); + assert_eq!(ObjectType::parse(".crl").unwrap(), ObjectType::Crl); + assert_eq!(ObjectType::parse("roa").unwrap(), ObjectType::Roa); + assert_eq!(ObjectType::parse("aspa").unwrap(), ObjectType::Aspa); + assert_eq!(ObjectType::parse(".asa").unwrap(), ObjectType::Aspa); + assert!(ObjectType::parse("unknown").is_err()); + assert_eq!(ObjectType::Aspa.label(), "aspa"); + + assert_eq!( + resolve_object_type(ObjectType::Auto, Path::new("repo/a.cer")).unwrap(), + ObjectType::Cer + ); + assert_eq!( + resolve_object_type(ObjectType::Auto, Path::new("repo/a.mft")).unwrap(), + ObjectType::Mft + ); + assert_eq!( + resolve_object_type(ObjectType::Auto, Path::new("repo/a.crl")).unwrap(), + ObjectType::Crl + ); + assert_eq!( + resolve_object_type(ObjectType::Auto, Path::new("repo/a.roa")).unwrap(), + ObjectType::Roa + ); + assert_eq!( + resolve_object_type(ObjectType::Auto, Path::new("repo/a.asa")).unwrap(), + ObjectType::Aspa + ); + assert_eq!( + resolve_object_type(ObjectType::Roa, Path::new("repo/a.bin")).unwrap(), + ObjectType::Roa + ); + assert!(resolve_object_type(ObjectType::Auto, Path::new("repo/a.bin")).is_err()); + } + + #[test] + fn invalid_der_returns_error_projection_for_all_object_types() { + let bytes = b"not der"; + for object_type in [ + ObjectType::Cer, + ObjectType::Mft, + ObjectType::Crl, + ObjectType::Roa, + ObjectType::Aspa, + ] { + let value = parse_object_json(object_type, Path::new("bad.der"), bytes, 1); + assert_eq!( + value["object"]["decode"]["profileValid"].as_bool(), + Some(false) + ); + assert!(value["object"]["decode"]["error"].as_str().is_some()); + } + + let record = build_object_projection(ObjectType::Auto, Path::new("bad.bin"), bytes, 1); + assert_eq!(record.object_type, "unknown"); + assert_eq!(record.parse_status, "error"); + assert!(record.error_summary.is_some()); + } + + #[test] + fn parses_fixture_objects_into_human_readable_projection() { + let cases = [ + ( + ObjectType::Cer, + "tests/fixtures/ta/apnic-ta.cer", + "cer", + "resourceCertificate", + ), + ( + ObjectType::Mft, + "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft", + "mft", + "manifest", + ), + ( + ObjectType::Crl, + "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.crl", + "crl", + "revokedCertificates", + ), + ( + ObjectType::Roa, + "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa", + "roa", + "roa", + ), + ( + ObjectType::Aspa, + "tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/5m80fwYws_3FiFD7JiQjAqZ1RYQ.asa", + "aspa", + "aspa", + ), + ]; + + for (object_type, path, expected_type, expected_section) in cases { + let bytes = std::fs::read(path).expect("fixture"); + let record = build_object_projection(object_type, Path::new(path), &bytes, 1); + assert_eq!(record.object_type, expected_type); + assert_eq!(record.parse_status, "ok"); + assert_eq!( + record.projection["object"]["decode"]["profileValid"].as_bool(), + Some(true) + ); + assert!(record.projection["object"][expected_section].is_object()); + } + } + + #[test] + fn large_projection_lists_are_paged_from_raw_bytes() { + let mft_bytes = std::fs::read( + "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft", + ) + .expect("mft"); + let (total, page) = manifest_file_entries_page(&mft_bytes, 1, 3).expect("mft page"); + assert!(total >= 3); + assert_eq!(page.len(), 3); + assert!(page[0]["fileName"].as_str().is_some()); + let (_, empty_page) = + manifest_file_entries_page(&mft_bytes, total + 10, 3).expect("empty page"); + assert!(empty_page.is_empty()); + + let crl_bytes = std::fs::read( + "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.crl", + ) + .expect("crl"); + let (total, page) = crl_revoked_entries_page(&crl_bytes, 0, 5).expect("crl page"); + assert!(page.len() <= total); + let (_, empty_page) = + crl_revoked_entries_page(&crl_bytes, total + 10, 5).expect("empty crl page"); + assert!(empty_page.is_empty()); + } +} diff --git a/crates/panda-rpki-validator/src/parallel/config.rs b/crates/panda-rpki-validator/src/parallel/config.rs new file mode 100644 index 0000000..a4d8ff1 --- /dev/null +++ b/crates/panda-rpki-validator/src/parallel/config.rs @@ -0,0 +1,79 @@ +use crate::parallel::dead_repo_blacklist::DeadRepoBlacklistConfig; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ParallelPhase1Config { + pub max_repo_sync_workers_global: usize, + pub max_inflight_snapshot_bytes_global: usize, + pub max_pending_repo_results: usize, + /// Dead-repo transport blacklist (#141). `None` disables the feature + /// entirely (default, behavior unchanged). + pub dead_repo_blacklist: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ParallelPhase2Config { + pub object_workers: usize, + pub worker_queue_capacity: usize, + pub ready_batch_size: usize, + pub ready_batch_wall_time_budget_ms: u64, + pub object_result_drain_batch_size: usize, + pub publication_point_finalize_batch_size: usize, + pub publication_point_finalize_wall_time_budget_ms: u64, + pub publication_point_finalize_queue_capacity: usize, + /// Experimental: workers of the ready publication point stage pool. `0` + /// disables the pool and keeps the inline compute+apply staging path. + pub stage_workers: usize, +} + +impl Default for ParallelPhase2Config { + fn default() -> Self { + Self { + object_workers: 8, + worker_queue_capacity: 256, + ready_batch_size: 256, + ready_batch_wall_time_budget_ms: 100, + object_result_drain_batch_size: 2048, + publication_point_finalize_batch_size: 256, + publication_point_finalize_wall_time_budget_ms: 100, + publication_point_finalize_queue_capacity: 32768, + stage_workers: 0, + } + } +} + +impl Default for ParallelPhase1Config { + fn default() -> Self { + Self { + max_repo_sync_workers_global: 4, + max_inflight_snapshot_bytes_global: 512 * 1024 * 1024, + max_pending_repo_results: 1024, + dead_repo_blacklist: None, + } + } +} + +#[cfg(test)] +mod tests { + use super::{ParallelPhase1Config, ParallelPhase2Config}; + + #[test] + fn default_parallel_phase1_config_is_bounded() { + let cfg = ParallelPhase1Config::default(); + assert!(cfg.max_repo_sync_workers_global > 0); + assert!(cfg.max_inflight_snapshot_bytes_global > 0); + assert!(cfg.max_pending_repo_results > 0); + } + + #[test] + fn default_parallel_phase2_config_is_bounded() { + let cfg = ParallelPhase2Config::default(); + assert!(cfg.object_workers > 0); + assert!(cfg.worker_queue_capacity > 0); + assert!(cfg.ready_batch_size > 0); + assert!(cfg.ready_batch_wall_time_budget_ms > 0); + assert!(cfg.object_result_drain_batch_size > 0); + assert!(cfg.publication_point_finalize_batch_size > 0); + assert!(cfg.publication_point_finalize_wall_time_budget_ms > 0); + assert!(cfg.publication_point_finalize_queue_capacity > 0); + } +} diff --git a/crates/panda-rpki-validator/src/parallel/dead_repo_blacklist.rs b/crates/panda-rpki-validator/src/parallel/dead_repo_blacklist.rs new file mode 100644 index 0000000..3b8d641 --- /dev/null +++ b/crates/panda-rpki-validator/src/parallel/dead_repo_blacklist.rs @@ -0,0 +1,391 @@ +//! Persistent dead-repo transport blacklist (#141). +//! +//! Tracks per-(repo, transport) consecutive transport-layer fetch failures +//! across runs. Once an entry reaches the configured failure threshold it is +//! admitted to the blacklist; the live transport scheduler then skips the +//! dead transport (rrdp-blacklisted -> straight to rsync, dual-blacklisted -> +//! immediate terminal failure). Entries are removed either by a successful +//! fetch (self-heal) or by the daemon-side health probe. + +use std::collections::HashMap; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +use crate::parallel::types::{RepoTransportErrorClass, RepoTransportMode}; + +pub const DEAD_REPO_BLACKLIST_SCHEMA_VERSION: u32 = 1; +pub const DEAD_REPO_BLACKLIST_DEFAULT_CAPACITY: usize = 256; +/// Probe backoff is capped at interval * 2^PROBE_BACKOFF_CAP_SHIFT. +pub const PROBE_BACKOFF_CAP_SHIFT: u32 = 4; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DeadRepoBlacklistConfig { + pub path: PathBuf, + pub fail_threshold: u32, + pub capacity: usize, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DeadRepoBlacklistEntry { + pub transport: RepoTransportMode, + pub uri: String, + /// Consecutive runs with transport-class fetch failure. Reset on success. + pub consecutive_failures: u32, + /// True once `consecutive_failures` reached the admission threshold. + #[serde(default)] + pub blacklisted: bool, + #[serde(default)] + pub first_failure_at_unix: u64, + #[serde(default)] + pub last_failure_at_unix: u64, + /// When the entry was admitted to the blacklist. + #[serde(default)] + pub added_at_unix: u64, + #[serde(default)] + pub last_probe_at_unix: Option, + #[serde(default)] + pub probe_failures: u32, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DeadRepoFailureOutcome { + /// Failure counted, entry not (or not yet) blacklisted. + Counted { consecutive_failures: u32 }, + /// Entry just reached the threshold and was admitted. + Admitted, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DeadRepoSuccessOutcome { + /// A counting (not yet admitted) entry was reset. + CounterReset, + /// A blacklisted entry was removed (repo revived). + BlacklistRemoved, +} + +#[derive(Serialize, Deserialize)] +struct DeadRepoBlacklistFile { + schema_version: u32, + updated_at_unix: u64, + #[serde(default)] + entries: Vec, +} + +/// In-memory blacklist state for one writer at a time (child run, or daemon +/// supervisor between runs). Lookups go through a HashMap keyed by +/// (transport, uri). +#[derive(Clone, Debug, Default)] +pub struct DeadRepoBlacklist { + entries: HashMap<(RepoTransportMode, String), DeadRepoBlacklistEntry>, +} + +impl DeadRepoBlacklist { + pub fn new() -> Self { + Self::default() + } + + /// Load from disk. A missing file or an unreadable/incompatible file + /// degrades to an empty blacklist (logged by the caller via `warning`). + pub fn load(path: &Path) -> (Self, Option) { + let bytes = match fs::read(path) { + Ok(bytes) => bytes, + Err(err) if err.kind() == io::ErrorKind::NotFound => return (Self::new(), None), + Err(err) => { + return ( + Self::new(), + Some(format!( + "dead repo blacklist {} unreadable ({err}); starting empty", + path.display() + )), + ); + } + }; + match serde_json::from_slice::(&bytes) { + Ok(file) if file.schema_version == DEAD_REPO_BLACKLIST_SCHEMA_VERSION => { + let entries = file + .entries + .into_iter() + .map(|entry| ((entry.transport, entry.uri.clone()), entry)) + .collect(); + (Self { entries }, None) + } + Ok(file) => ( + Self::new(), + Some(format!( + "dead repo blacklist {} schema_version {} != {}; starting empty", + path.display(), + file.schema_version, + DEAD_REPO_BLACKLIST_SCHEMA_VERSION + )), + ), + Err(err) => ( + Self::new(), + Some(format!( + "dead repo blacklist {} unparsable ({err}); starting empty", + path.display() + )), + ), + } + } + + /// Persist atomically (tmp file + rename). + pub fn store_atomic(&self, path: &Path, now_unix: u64) -> io::Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + let mut entries: Vec<&DeadRepoBlacklistEntry> = self.entries.values().collect(); + entries.sort_by(|a, b| a.uri.cmp(&b.uri).then(a.transport.cmp(&b.transport))); + let file = DeadRepoBlacklistFile { + schema_version: DEAD_REPO_BLACKLIST_SCHEMA_VERSION, + updated_at_unix: now_unix, + entries: entries.into_iter().cloned().collect(), + }; + let bytes = serde_json::to_vec_pretty(&file) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err))?; + let mut tmp_name = path + .file_name() + .map(|name| name.to_os_string()) + .unwrap_or_default(); + tmp_name.push(".tmp"); + let tmp_path = path.with_file_name(tmp_name); + fs::write(&tmp_path, &bytes)?; + // Best-effort durability before the rename. + if let Ok(file) = fs::File::open(&tmp_path) { + let _ = file.sync_all(); + } + fs::rename(&tmp_path, path)?; + Ok(()) + } + + pub fn is_blacklisted(&self, transport: RepoTransportMode, uri: &str) -> bool { + self.entries + .get(&(transport, uri.to_string())) + .map(|entry| entry.blacklisted) + .unwrap_or(false) + } + + /// Record a transport-class fetch failure. Returns whether the failure + /// newly admitted the entry to the blacklist. + pub fn record_transport_failure( + &mut self, + transport: RepoTransportMode, + uri: &str, + now_unix: u64, + fail_threshold: u32, + capacity: usize, + ) -> DeadRepoFailureOutcome { + let key = (transport, uri.to_string()); + if !self.entries.contains_key(&key) { + self.evict_if_full(capacity); + } + let threshold = fail_threshold.max(1); + let entry = self + .entries + .entry(key) + .or_insert_with(|| DeadRepoBlacklistEntry { + transport, + uri: uri.to_string(), + consecutive_failures: 0, + blacklisted: false, + first_failure_at_unix: now_unix, + last_failure_at_unix: now_unix, + added_at_unix: 0, + last_probe_at_unix: None, + probe_failures: 0, + }); + entry.consecutive_failures = entry.consecutive_failures.saturating_add(1); + entry.last_failure_at_unix = now_unix; + if entry.first_failure_at_unix == 0 { + entry.first_failure_at_unix = now_unix; + } + if !entry.blacklisted && entry.consecutive_failures >= threshold { + entry.blacklisted = true; + entry.added_at_unix = now_unix; + // A fresh admission restarts probe bookkeeping. + entry.last_probe_at_unix = None; + entry.probe_failures = 0; + return DeadRepoFailureOutcome::Admitted; + } + DeadRepoFailureOutcome::Counted { + consecutive_failures: entry.consecutive_failures, + } + } + + /// Record a successful fetch: reset a counting entry, or remove a + /// blacklisted entry entirely (self-heal). + pub fn record_transport_success( + &mut self, + transport: RepoTransportMode, + uri: &str, + ) -> Option { + match self.entries.remove(&(transport, uri.to_string())) { + Some(entry) if entry.blacklisted => Some(DeadRepoSuccessOutcome::BlacklistRemoved), + Some(_) => Some(DeadRepoSuccessOutcome::CounterReset), + None => None, + } + } + + /// Blacklisted entries whose next probe is due at `now_unix`. + pub fn probe_due_entries( + &self, + now_unix: u64, + probe_interval_secs: u64, + ) -> Vec { + let mut due: Vec = self + .entries + .values() + .filter(|entry| entry.blacklisted) + .filter(|entry| { + let interval = probe_backoff_secs(probe_interval_secs, entry.probe_failures); + match entry.last_probe_at_unix { + None => true, + Some(last) => now_unix.saturating_sub(last) >= interval, + } + }) + .cloned() + .collect(); + due.sort_by(|a, b| a.uri.cmp(&b.uri)); + due + } + + /// Probe succeeded: remove the entry. Returns true if it was present. + pub fn record_probe_success(&mut self, transport: RepoTransportMode, uri: &str) -> bool { + self.entries.remove(&(transport, uri.to_string())).is_some() + } + + pub fn record_probe_failure(&mut self, transport: RepoTransportMode, uri: &str, now_unix: u64) { + if let Some(entry) = self.entries.get_mut(&(transport, uri.to_string())) { + entry.last_probe_at_unix = Some(now_unix); + entry.probe_failures = entry.probe_failures.saturating_add(1); + } + } + + pub fn len(&self) -> usize { + self.entries.len() + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + pub fn blacklisted_len(&self) -> usize { + self.entries + .values() + .filter(|entry| entry.blacklisted) + .count() + } + + /// Deterministic snapshot of all entries (for status/observability). + pub fn entries_sorted(&self) -> Vec { + let mut entries: Vec = self.entries.values().cloned().collect(); + entries.sort_by(|a, b| a.uri.cmp(&b.uri).then(a.transport.cmp(&b.transport))); + entries + } + + fn evict_if_full(&mut self, capacity: usize) { + let capacity = capacity.max(1); + if self.entries.len() < capacity { + return; + } + // Prefer evicting the stalest counting entry; fall back to the oldest + // blacklisted entry. + let victim = self + .entries + .values() + .filter(|entry| !entry.blacklisted) + .min_by_key(|entry| entry.last_failure_at_unix) + .map(|entry| (entry.transport, entry.uri.clone())) + .or_else(|| { + self.entries + .values() + .min_by_key(|entry| entry.added_at_unix) + .map(|entry| (entry.transport, entry.uri.clone())) + }); + if let Some(key) = victim { + self.entries.remove(&key); + } + } +} + +pub fn probe_backoff_secs(base_interval_secs: u64, probe_failures: u32) -> u64 { + // First retry after a probe failure still uses the base interval; + // exponential backoff kicks in from the second consecutive failure. + let shift = probe_failures + .saturating_sub(1) + .min(PROBE_BACKOFF_CAP_SHIFT); + base_interval_secs.saturating_mul(1u64 << shift) +} + +/// Classify an HTTP fetch error string (formats from +/// `fetch/http.rs::BlockingHttpFetcher::fetch_bytes`). +pub fn classify_http_fetch_error(detail: &str) -> RepoTransportErrorClass { + if detail.starts_with("http status") { + // The peer answered: transport is fine, the failure is protocol-level. + RepoTransportErrorClass::Protocol + } else if detail.starts_with("http request failed") + || detail.starts_with("http read body failed") + { + RepoTransportErrorClass::TransportFetch + } else { + RepoTransportErrorClass::Unknown + } +} + +/// Classify an rsync fetch error string using the same host-level substring +/// set as the run-local rsync failure scope short-circuit. +pub fn classify_rsync_fetch_error(detail: &str) -> RepoTransportErrorClass { + if crate::parallel::repo_scheduler::is_host_level_rsync_failure(detail) + // Stall failures are transport death too: the fail-fast mechanism + // gives up when wall-clock windows pass with no (additional) progress + // (#141 M9). Content-level errors fail immediately and never carry + // this marker, so they still do not count. + || detail.contains("rsync fail-fast gave up") + { + RepoTransportErrorClass::TransportFetch + } else { + // Content/protocol-level rsync failures (digest mismatch, vanished + // files, partial transfers) never count toward the blacklist. + RepoTransportErrorClass::Unknown + } +} + +/// Map a structured RRDP sync error to a transport error class. +pub fn classify_rrdp_sync_error(err: &crate::sync::rrdp::RrdpSyncError) -> RepoTransportErrorClass { + match err { + crate::sync::rrdp::RrdpSyncError::Fetch(detail) => classify_http_fetch_error(detail), + crate::sync::rrdp::RrdpSyncError::Rrdp(_) => RepoTransportErrorClass::Protocol, + crate::sync::rrdp::RrdpSyncError::Storage(_) => RepoTransportErrorClass::Storage, + } +} + +/// Map a structured rsync-side repo sync error to a transport error class. +pub fn classify_rsync_repo_sync_error( + err: &crate::sync::repo::RepoSyncError, +) -> RepoTransportErrorClass { + match err { + crate::sync::repo::RepoSyncError::Rsync(crate::fetch::rsync::RsyncFetchError::Fetch( + detail, + )) => classify_rsync_fetch_error(detail), + crate::sync::repo::RepoSyncError::Storage(_) => RepoTransportErrorClass::Storage, + _ => RepoTransportErrorClass::Protocol, + } +} + +impl RepoTransportMode { + pub fn as_str(self) -> &'static str { + match self { + RepoTransportMode::Rrdp => "rrdp", + RepoTransportMode::Rsync => "rsync", + } + } +} + +// Keep the public surface free of internal key types; the (transport, uri) +// tuple key is an implementation detail of the lookup map. + +#[cfg(test)] +#[path = "dead_repo_blacklist_tests.rs"] +mod tests; diff --git a/crates/panda-rpki-validator/src/parallel/dead_repo_blacklist_tests.rs b/crates/panda-rpki-validator/src/parallel/dead_repo_blacklist_tests.rs new file mode 100644 index 0000000..3cb95d4 --- /dev/null +++ b/crates/panda-rpki-validator/src/parallel/dead_repo_blacklist_tests.rs @@ -0,0 +1,430 @@ +use super::*; +use crate::parallel::types::RepoTransportMode; + +const RRDP_URI: &str = "https://dead.example.com/notification.xml"; +const RSYNC_URI: &str = "rsync://dead.example.com/repo/"; + +fn now() -> u64 { + 1_700_000_000 +} + +#[test] +fn admission_requires_threshold_consecutive_failures() { + let mut bl = DeadRepoBlacklist::new(); + assert!(!bl.is_blacklisted(RepoTransportMode::Rrdp, RRDP_URI)); + + let out1 = bl.record_transport_failure(RepoTransportMode::Rrdp, RRDP_URI, now(), 3, 256); + assert_eq!( + out1, + DeadRepoFailureOutcome::Counted { + consecutive_failures: 1 + } + ); + assert!(!bl.is_blacklisted(RepoTransportMode::Rrdp, RRDP_URI)); + + let out2 = bl.record_transport_failure(RepoTransportMode::Rrdp, RRDP_URI, now() + 1, 3, 256); + assert_eq!( + out2, + DeadRepoFailureOutcome::Counted { + consecutive_failures: 2 + } + ); + assert!(!bl.is_blacklisted(RepoTransportMode::Rrdp, RRDP_URI)); + + let out3 = bl.record_transport_failure(RepoTransportMode::Rrdp, RRDP_URI, now() + 2, 3, 256); + assert_eq!(out3, DeadRepoFailureOutcome::Admitted); + assert!(bl.is_blacklisted(RepoTransportMode::Rrdp, RRDP_URI)); + + // Further failures keep counting but do not re-admit. + let out4 = bl.record_transport_failure(RepoTransportMode::Rrdp, RRDP_URI, now() + 3, 3, 256); + assert!(matches!( + out4, + DeadRepoFailureOutcome::Counted { + consecutive_failures: 4 + } + )); +} + +#[test] +fn threshold_of_one_admits_immediately() { + let mut bl = DeadRepoBlacklist::new(); + let out = bl.record_transport_failure(RepoTransportMode::Rsync, RSYNC_URI, now(), 1, 256); + assert_eq!(out, DeadRepoFailureOutcome::Admitted); +} + +#[test] +fn success_resets_counting_entry() { + let mut bl = DeadRepoBlacklist::new(); + bl.record_transport_failure(RepoTransportMode::Rrdp, RRDP_URI, now(), 3, 256); + assert_eq!(bl.len(), 1); + let out = bl.record_transport_success(RepoTransportMode::Rrdp, RRDP_URI); + assert_eq!(out, Some(DeadRepoSuccessOutcome::CounterReset)); + assert!(bl.is_empty()); + // Next failure starts from zero again. + let out = bl.record_transport_failure(RepoTransportMode::Rrdp, RRDP_URI, now(), 3, 256); + assert_eq!( + out, + DeadRepoFailureOutcome::Counted { + consecutive_failures: 1 + } + ); +} + +#[test] +fn success_on_blacklisted_entry_self_heals() { + let mut bl = DeadRepoBlacklist::new(); + for i in 0..3 { + bl.record_transport_failure(RepoTransportMode::Rsync, RSYNC_URI, now() + i, 3, 256); + } + assert!(bl.is_blacklisted(RepoTransportMode::Rsync, RSYNC_URI)); + let out = bl.record_transport_success(RepoTransportMode::Rsync, RSYNC_URI); + assert_eq!(out, Some(DeadRepoSuccessOutcome::BlacklistRemoved)); + assert!(!bl.is_blacklisted(RepoTransportMode::Rsync, RSYNC_URI)); + assert!(bl.is_empty()); +} + +#[test] +fn success_on_unknown_entry_is_none() { + let mut bl = DeadRepoBlacklist::new(); + assert_eq!( + bl.record_transport_success(RepoTransportMode::Rrdp, RRDP_URI), + None + ); +} + +#[test] +fn entries_are_per_transport() { + let mut bl = DeadRepoBlacklist::new(); + for i in 0..3 { + bl.record_transport_failure(RepoTransportMode::Rrdp, RRDP_URI, now() + i, 3, 256); + } + assert!(bl.is_blacklisted(RepoTransportMode::Rrdp, RRDP_URI)); + assert!(!bl.is_blacklisted(RepoTransportMode::Rsync, RRDP_URI)); +} + +#[test] +fn capacity_evicts_stale_counting_entries_first() { + let mut bl = DeadRepoBlacklist::new(); + // One blacklisted entry (old) and two counting entries. + for i in 0..3 { + bl.record_transport_failure( + RepoTransportMode::Rrdp, + "https://a.example/n.xml", + 100 + i, + 1, + 3, + ); + } + bl.record_transport_failure( + RepoTransportMode::Rrdp, + "https://b.example/n.xml", + 200, + 3, + 3, + ); + bl.record_transport_failure( + RepoTransportMode::Rrdp, + "https://c.example/n.xml", + 300, + 3, + 3, + ); + assert_eq!(bl.len(), 3); + assert_eq!(bl.blacklisted_len(), 1); + + // Inserting a fourth entry evicts the stalest counting entry (b). + bl.record_transport_failure( + RepoTransportMode::Rrdp, + "https://d.example/n.xml", + 400, + 3, + 3, + ); + assert_eq!(bl.len(), 3); + assert!( + bl.entries_sorted() + .iter() + .any(|e| e.uri == "https://a.example/n.xml") + ); + assert!( + !bl.entries_sorted() + .iter() + .any(|e| e.uri == "https://b.example/n.xml") + ); + assert!( + bl.entries_sorted() + .iter() + .any(|e| e.uri == "https://c.example/n.xml") + ); + assert!( + bl.entries_sorted() + .iter() + .any(|e| e.uri == "https://d.example/n.xml") + ); +} + +#[test] +fn capacity_evicts_oldest_blacklisted_when_no_counting_entries() { + let mut bl = DeadRepoBlacklist::new(); + bl.record_transport_failure( + RepoTransportMode::Rrdp, + "https://a.example/n.xml", + 100, + 1, + 2, + ); + bl.record_transport_failure( + RepoTransportMode::Rrdp, + "https://b.example/n.xml", + 200, + 1, + 2, + ); + assert_eq!(bl.blacklisted_len(), 2); + bl.record_transport_failure( + RepoTransportMode::Rrdp, + "https://c.example/n.xml", + 300, + 1, + 2, + ); + assert_eq!(bl.len(), 2); + assert!( + !bl.entries_sorted() + .iter() + .any(|e| e.uri == "https://a.example/n.xml") + ); + assert!( + bl.entries_sorted() + .iter() + .any(|e| e.uri == "https://b.example/n.xml") + ); + assert!( + bl.entries_sorted() + .iter() + .any(|e| e.uri == "https://c.example/n.xml") + ); +} + +#[test] +fn store_load_roundtrip() { + let dir = std::env::temp_dir().join(format!( + "dead_repo_blacklist_test_roundtrip_{}_{}", + std::process::id(), + now() + )); + let path = dir.join("state").join("dead-repo-blacklist.json"); + let mut bl = DeadRepoBlacklist::new(); + for i in 0..3 { + bl.record_transport_failure(RepoTransportMode::Rrdp, RRDP_URI, now() + i, 3, 256); + } + bl.record_transport_failure(RepoTransportMode::Rsync, RSYNC_URI, now(), 3, 256); + bl.store_atomic(&path, now() + 10).unwrap(); + + let (loaded, warning) = DeadRepoBlacklist::load(&path); + assert!(warning.is_none()); + assert!(loaded.is_blacklisted(RepoTransportMode::Rrdp, RRDP_URI)); + assert!(!loaded.is_blacklisted(RepoTransportMode::Rsync, RSYNC_URI)); + assert_eq!(loaded.len(), 2); + let entry = loaded + .entries_sorted() + .into_iter() + .find(|e| e.uri == RRDP_URI) + .unwrap(); + assert_eq!(entry.consecutive_failures, 3); + assert!(entry.blacklisted); + assert_eq!(entry.first_failure_at_unix, now()); + + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn load_missing_file_is_empty_without_warning() { + let path = std::env::temp_dir().join(format!( + "dead_repo_blacklist_test_missing_{}_{}.json", + std::process::id(), + now() + )); + let (bl, warning) = DeadRepoBlacklist::load(&path); + assert!(bl.is_empty()); + assert!(warning.is_none()); +} + +#[test] +fn load_corrupted_file_degrades_to_empty_with_warning() { + let path = std::env::temp_dir().join(format!( + "dead_repo_blacklist_test_corrupt_{}_{}.json", + std::process::id(), + now() + )); + std::fs::write(&path, b"{not json").unwrap(); + let (bl, warning) = DeadRepoBlacklist::load(&path); + assert!(bl.is_empty()); + assert!(warning.is_some()); + let _ = std::fs::remove_file(&path); +} + +#[test] +fn load_wrong_schema_version_degrades_to_empty() { + let path = std::env::temp_dir().join(format!( + "dead_repo_blacklist_test_schema_{}_{}.json", + std::process::id(), + now() + )); + std::fs::write( + &path, + r#"{"schema_version": 99, "updated_at_unix": 0, "entries": []}"#, + ) + .unwrap(); + let (bl, warning) = DeadRepoBlacklist::load(&path); + assert!(bl.is_empty()); + assert!(warning.unwrap().contains("schema_version")); + let _ = std::fs::remove_file(&path); +} + +#[test] +fn probe_due_uses_exponential_backoff() { + let mut bl = DeadRepoBlacklist::new(); + for i in 0..3 { + bl.record_transport_failure(RepoTransportMode::Rrdp, RRDP_URI, now() + i, 3, 256); + } + // Never probed: due immediately. + let due = bl.probe_due_entries(now() + 100, 600); + assert_eq!(due.len(), 1); + + // Probe failure: not due until base interval has passed. + bl.record_probe_failure(RepoTransportMode::Rrdp, RRDP_URI, now() + 100); + assert!(bl.probe_due_entries(now() + 100 + 599, 600).is_empty()); + assert_eq!(bl.probe_due_entries(now() + 100 + 600, 600).len(), 1); + + // Second probe failure: backoff doubles to 1200s. + bl.record_probe_failure(RepoTransportMode::Rrdp, RRDP_URI, now() + 700); + assert!(bl.probe_due_entries(now() + 700 + 1199, 600).is_empty()); + assert_eq!(bl.probe_due_entries(now() + 700 + 1200, 600).len(), 1); +} + +#[test] +fn probe_success_removes_entry() { + let mut bl = DeadRepoBlacklist::new(); + for i in 0..3 { + bl.record_transport_failure(RepoTransportMode::Rsync, RSYNC_URI, now() + i, 3, 256); + } + assert!(bl.record_probe_success(RepoTransportMode::Rsync, RSYNC_URI)); + assert!(bl.is_empty()); + assert!(!bl.record_probe_success(RepoTransportMode::Rsync, RSYNC_URI)); +} + +#[test] +fn counting_entries_are_not_probed() { + let mut bl = DeadRepoBlacklist::new(); + bl.record_transport_failure(RepoTransportMode::Rrdp, RRDP_URI, now(), 3, 256); + assert!(bl.probe_due_entries(now() + 10_000, 600).is_empty()); +} + +#[test] +fn probe_backoff_caps_at_max_shift() { + assert_eq!(probe_backoff_secs(600, 0), 600); + assert_eq!(probe_backoff_secs(600, 1), 600); + assert_eq!(probe_backoff_secs(600, 2), 1200); + assert_eq!(probe_backoff_secs(600, 5), 9600); + assert_eq!(probe_backoff_secs(600, 100), 9600); +} + +#[test] +fn http_fetch_error_classification() { + use crate::parallel::types::RepoTransportErrorClass; + assert_eq!( + classify_http_fetch_error("http request failed: reqwest::Error { kind: Connect, .. }"), + RepoTransportErrorClass::TransportFetch + ); + assert_eq!( + classify_http_fetch_error("http read body failed: connection closed; status=200"), + RepoTransportErrorClass::TransportFetch + ); + assert_eq!( + classify_http_fetch_error("http status 404 Not Found; content_type=; .."), + RepoTransportErrorClass::Protocol + ); + assert_eq!( + classify_http_fetch_error("something unexpected"), + RepoTransportErrorClass::Unknown + ); +} + +#[test] +fn rsync_fetch_error_classification() { + use crate::parallel::types::RepoTransportErrorClass; + assert_eq!( + classify_rsync_fetch_error("rsync error: timeout waiting for daemon connection"), + RepoTransportErrorClass::TransportFetch + ); + assert_eq!( + classify_rsync_fetch_error("rsync: failed to connect to host: Connection refused"), + RepoTransportErrorClass::TransportFetch + ); + assert_eq!( + classify_rsync_fetch_error("temporary failure in name resolution"), + RepoTransportErrorClass::TransportFetch + ); + // Content-level failures must not count. + assert_eq!( + classify_rsync_fetch_error("rsync file digest mismatch after download"), + RepoTransportErrorClass::Unknown + ); + // Stall failures (#141 M9): fail-fast give-up means the host delivered no + // bytes inside the wall-clock window — transport death, must count. + assert_eq!( + classify_rsync_fetch_error( + "rsync fail-fast gave up after 1 attempts with no progress: rsync wall-clock window expired" + ), + RepoTransportErrorClass::TransportFetch + ); + assert_eq!( + classify_rsync_fetch_error( + "rsync fail-fast gave up after 3 attempts with no additional progress: rsync wall-clock window expired" + ), + RepoTransportErrorClass::TransportFetch + ); + // Content-level give-up text without the fail-fast marker stays unknown. + assert_eq!( + classify_rsync_fetch_error("rsync error: some files vanished before transfer"), + RepoTransportErrorClass::Unknown + ); +} + +#[test] +fn rrdp_sync_error_classification() { + use crate::parallel::types::RepoTransportErrorClass; + use crate::sync::rrdp::RrdpSyncError; + assert_eq!( + classify_rrdp_sync_error(&RrdpSyncError::Fetch( + "http request failed: connect timeout".to_string() + )), + RepoTransportErrorClass::TransportFetch + ); + assert_eq!( + classify_rrdp_sync_error(&RrdpSyncError::Fetch("http status 500".to_string())), + RepoTransportErrorClass::Protocol + ); + assert_eq!( + classify_rrdp_sync_error(&RrdpSyncError::Storage("db full".to_string())), + RepoTransportErrorClass::Storage + ); +} + +#[test] +fn readmission_after_probe_removal_starts_fresh() { + let mut bl = DeadRepoBlacklist::new(); + for i in 0..3 { + bl.record_transport_failure(RepoTransportMode::Rrdp, RRDP_URI, now() + i, 3, 256); + } + assert!(bl.record_probe_success(RepoTransportMode::Rrdp, RRDP_URI)); + // No half-open: needs the full threshold again. + let out = bl.record_transport_failure(RepoTransportMode::Rrdp, RRDP_URI, now() + 10, 3, 256); + assert_eq!( + out, + DeadRepoFailureOutcome::Counted { + consecutive_failures: 1 + } + ); +} diff --git a/crates/panda-rpki-validator/src/parallel/mod.rs b/crates/panda-rpki-validator/src/parallel/mod.rs new file mode 100644 index 0000000..06174cd --- /dev/null +++ b/crates/panda-rpki-validator/src/parallel/mod.rs @@ -0,0 +1,11 @@ +pub mod config; +pub mod dead_repo_blacklist; +pub mod object_worker; +pub mod phase2_scheduler; +pub mod repo_runtime; +pub mod repo_scheduler; +pub mod repo_worker; +pub mod run_coordinator; +pub mod stats; +pub mod transport_prefetch; +pub mod types; diff --git a/crates/panda-rpki-validator/src/parallel/object_worker.rs b/crates/panda-rpki-validator/src/parallel/object_worker.rs new file mode 100644 index 0000000..c32980e --- /dev/null +++ b/crates/panda-rpki-validator/src/parallel/object_worker.rs @@ -0,0 +1,485 @@ +use std::marker::PhantomData; +use std::sync::Arc; +use std::sync::mpsc::{self, Receiver, RecvTimeoutError, SyncSender, TrySendError}; +use std::thread::{self, JoinHandle, Scope}; +use std::time::Duration; + +/// Executor shared by every worker of a pool. The `'static` bound required to +/// move the executor into detached threads is expressed on the pool types +/// instead of this trait so scoped pools can borrow their environment. +pub trait ObjectTaskExecutor: Send + Sync { + fn execute(&self, worker_index: usize, task: T) -> R; +} + +enum ObjectWorkerMessage { + Task(T), + Shutdown, +} + +#[derive(Debug)] +pub enum ObjectWorkerSubmitError { + QueueFull { worker_index: usize, task: T }, + Disconnected { worker_index: usize, task: T }, +} + +pub struct ObjectWorkerPool +where + T: Send + 'static, + R: Send + 'static, + E: ObjectTaskExecutor + 'static, +{ + task_txs: Vec>>, + result_rx: Receiver, + workers: Vec>, + next_worker_idx: usize, + _executor: Arc, +} + +impl ObjectWorkerPool +where + T: Send + 'static, + R: Send + 'static, + E: ObjectTaskExecutor + 'static, +{ + pub fn new(worker_count: usize, queue_capacity: usize, executor: E) -> Result { + if worker_count == 0 { + return Err("ObjectWorkerPool requires at least one worker".to_string()); + } + if queue_capacity == 0 { + return Err("ObjectWorkerPool requires queue_capacity > 0".to_string()); + } + + let executor = Arc::new(executor); + let (result_tx, result_rx) = mpsc::channel::(); + let mut task_txs = Vec::with_capacity(worker_count); + let mut workers = Vec::with_capacity(worker_count); + + for worker_index in 0..worker_count { + let (task_tx, task_rx) = mpsc::sync_channel::>(queue_capacity); + let result_tx = result_tx.clone(); + let executor = Arc::clone(&executor); + let handle = thread::Builder::new() + .name(format!("object-validation-worker-{worker_index}")) + .spawn(move || object_worker_loop(worker_index, task_rx, result_tx, executor)) + .map_err(|e| format!("spawn object worker failed: {e}"))?; + task_txs.push(task_tx); + workers.push(handle); + } + + Ok(Self { + task_txs, + result_rx, + workers, + next_worker_idx: 0, + _executor: executor, + }) + } + + pub fn worker_count(&self) -> usize { + self.task_txs.len() + } + + pub fn next_worker_index(&self) -> usize { + self.next_worker_idx + } + + pub fn try_submit_round_robin(&mut self, task: T) -> Result> { + let worker_index = self.next_worker_idx % self.task_txs.len(); + match self.task_txs[worker_index].try_send(ObjectWorkerMessage::Task(task)) { + Ok(()) => { + self.next_worker_idx = (worker_index + 1) % self.task_txs.len(); + Ok(worker_index) + } + Err(TrySendError::Full(ObjectWorkerMessage::Task(task))) => { + Err(ObjectWorkerSubmitError::QueueFull { worker_index, task }) + } + Err(TrySendError::Disconnected(ObjectWorkerMessage::Task(task))) => { + Err(ObjectWorkerSubmitError::Disconnected { worker_index, task }) + } + Err(TrySendError::Full(ObjectWorkerMessage::Shutdown)) + | Err(TrySendError::Disconnected(ObjectWorkerMessage::Shutdown)) => { + unreachable!("shutdown is never submitted via try_submit_round_robin") + } + } + } + + pub fn recv_result_timeout(&self, timeout: Duration) -> Result, String> { + match self.result_rx.recv_timeout(timeout) { + Ok(result) => Ok(Some(result)), + Err(RecvTimeoutError::Timeout) => Ok(None), + Err(RecvTimeoutError::Disconnected) => { + Err("object worker result channel disconnected".to_string()) + } + } + } + + pub fn shutdown(mut self) -> Result<(), String> { + self.shutdown_inner() + } + + fn shutdown_inner(&mut self) -> Result<(), String> { + if self.workers.is_empty() { + return Ok(()); + } + for tx in &self.task_txs { + tx.send(ObjectWorkerMessage::Shutdown) + .map_err(|e| format!("send shutdown to object worker failed: {e}"))?; + } + let mut first_err = None; + for handle in self.workers.drain(..) { + if let Err(e) = handle.join() { + if first_err.is_none() { + first_err = Some(format!("join object worker failed: {e:?}")); + } + } + } + if let Some(err) = first_err { + return Err(err); + } + Ok(()) + } +} + +impl Drop for ObjectWorkerPool +where + T: Send + 'static, + R: Send + 'static, + E: ObjectTaskExecutor + 'static, +{ + fn drop(&mut self) { + let _ = self.shutdown_inner(); + } +} + +fn object_worker_loop( + worker_index: usize, + task_rx: Receiver>, + result_tx: mpsc::Sender, + executor: Arc, +) where + T: Send, + R: Send, + E: ObjectTaskExecutor, +{ + loop { + match task_rx.recv() { + Ok(ObjectWorkerMessage::Task(task)) => { + let result = executor.execute(worker_index, task); + if result_tx.send(result).is_err() { + break; + } + } + Ok(ObjectWorkerMessage::Shutdown) | Err(_) => break, + } + } +} + +/// Scoped variant of `ObjectWorkerPool`: workers are spawned on a +/// `std::thread::Scope`, so tasks, results and the executor may borrow their +/// environment (`'env`) instead of being `'static`. The pool never sends +/// `Shutdown`; workers exit when every task sender is dropped, which happens +/// when the pool itself is dropped ahead of the scope join. +pub struct ScopedObjectWorkerPool<'scope, 'env, T, R, E> +where + T: Send + 'env, + R: Send + 'env, + E: ObjectTaskExecutor + 'env, +{ + task_txs: Vec>>, + result_rx: Receiver, + next_worker_idx: usize, + _executor: Arc, + // Join handles are intentionally not stored: dropping a `ScopedJoinHandle` + // detaches the worker and the enclosing scope joins it on exit, after the + // dropped task senders have made every worker return. + _marker: PhantomData<(&'scope (), &'env ())>, +} + +impl<'scope, 'env, T, R, E> ScopedObjectWorkerPool<'scope, 'env, T, R, E> +where + T: Send + 'env, + R: Send + 'env, + E: ObjectTaskExecutor + 'env, +{ + pub fn new( + scope: &'scope Scope<'scope, 'env>, + worker_count: usize, + queue_capacity: usize, + executor: E, + ) -> Result { + if worker_count == 0 { + return Err("ScopedObjectWorkerPool requires at least one worker".to_string()); + } + if queue_capacity == 0 { + return Err("ScopedObjectWorkerPool requires queue_capacity > 0".to_string()); + } + + let executor = Arc::new(executor); + let (result_tx, result_rx) = mpsc::channel::(); + let mut task_txs = Vec::with_capacity(worker_count); + + for worker_index in 0..worker_count { + let (task_tx, task_rx) = mpsc::sync_channel::>(queue_capacity); + let result_tx = result_tx.clone(); + let executor = Arc::clone(&executor); + thread::Builder::new() + .name(format!("object-validation-worker-{worker_index}")) + .spawn_scoped(scope, move || { + object_worker_loop(worker_index, task_rx, result_tx, executor) + }) + .map_err(|e| format!("spawn scoped object worker failed: {e}"))?; + task_txs.push(task_tx); + } + + Ok(Self { + task_txs, + result_rx, + next_worker_idx: 0, + _executor: executor, + _marker: PhantomData, + }) + } + + pub fn worker_count(&self) -> usize { + self.task_txs.len() + } + + pub fn try_submit_round_robin(&mut self, task: T) -> Result> { + let worker_index = self.next_worker_idx % self.task_txs.len(); + match self.task_txs[worker_index].try_send(ObjectWorkerMessage::Task(task)) { + Ok(()) => { + self.next_worker_idx = (worker_index + 1) % self.task_txs.len(); + Ok(worker_index) + } + Err(TrySendError::Full(ObjectWorkerMessage::Task(task))) => { + Err(ObjectWorkerSubmitError::QueueFull { worker_index, task }) + } + Err(TrySendError::Disconnected(ObjectWorkerMessage::Task(task))) => { + Err(ObjectWorkerSubmitError::Disconnected { worker_index, task }) + } + Err(TrySendError::Full(ObjectWorkerMessage::Shutdown)) + | Err(TrySendError::Disconnected(ObjectWorkerMessage::Shutdown)) => { + unreachable!("shutdown is never submitted via try_submit_round_robin") + } + } + } + + pub fn recv_result_timeout(&self, timeout: Duration) -> Result, String> { + match self.result_rx.recv_timeout(timeout) { + Ok(result) => Ok(Some(result)), + Err(RecvTimeoutError::Timeout) => Ok(None), + Err(RecvTimeoutError::Disconnected) => { + Err("scoped object worker result channel disconnected".to_string()) + } + } + } +} + +impl<'scope, 'env, T, R, E> Drop for ScopedObjectWorkerPool<'scope, 'env, T, R, E> +where + T: Send + 'env, + R: Send + 'env, + E: ObjectTaskExecutor + 'env, +{ + fn drop(&mut self) { + // Close every worker input queue so blocked `recv` calls return and + // the scoped workers exit before the enclosing scope joins them. + self.task_txs.clear(); + } +} + +#[cfg(test)] +mod tests { + use super::{ObjectTaskExecutor, ObjectWorkerPool, ObjectWorkerSubmitError}; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Barrier}; + use std::time::Duration; + + #[derive(Clone)] + struct EchoExecutor; + + impl ObjectTaskExecutor for EchoExecutor { + fn execute(&self, worker_index: usize, task: u32) -> (usize, u32) { + (worker_index, task) + } + } + + #[test] + fn object_worker_pool_rejects_invalid_config_and_shutdowns_explicitly() { + let err = match ObjectWorkerPool::new(0, 1, EchoExecutor) { + Ok(_) => panic!("zero workers should be rejected"), + Err(err) => err, + }; + assert!(err.contains("at least one worker")); + let err = match ObjectWorkerPool::new(1, 0, EchoExecutor) { + Ok(_) => panic!("zero queue should be rejected"), + Err(err) => err, + }; + assert!(err.contains("queue_capacity > 0")); + + let pool = ObjectWorkerPool::new(2, 1, EchoExecutor).expect("pool"); + assert_eq!(pool.worker_count(), 2); + assert_eq!(pool.next_worker_index(), 0); + pool.shutdown().expect("shutdown"); + } + + #[test] + fn object_worker_pool_round_robin_submits_to_worker_queues() { + let mut pool = ObjectWorkerPool::new(3, 4, EchoExecutor).expect("pool"); + assert_eq!(pool.try_submit_round_robin(10).expect("submit 10"), 0); + assert_eq!(pool.try_submit_round_robin(11).expect("submit 11"), 1); + assert_eq!(pool.try_submit_round_robin(12).expect("submit 12"), 2); + assert_eq!(pool.try_submit_round_robin(13).expect("submit 13"), 0); + + let mut results = Vec::new(); + for _ in 0..4 { + results.push( + pool.recv_result_timeout(Duration::from_secs(1)) + .expect("result channel") + .expect("result"), + ); + } + results.sort_by_key(|(_, task)| *task); + assert_eq!(results, vec![(0, 10), (1, 11), (2, 12), (0, 13)]); + } + + struct BlockingExecutor { + barrier: Arc, + started: Arc, + } + + impl ObjectTaskExecutor for BlockingExecutor { + fn execute(&self, _worker_index: usize, task: u32) -> u32 { + self.started.store(true, Ordering::SeqCst); + self.barrier.wait(); + task + } + } + + #[test] + fn object_worker_pool_reports_full_worker_queue_without_advancing_round_robin() { + let barrier = Arc::new(Barrier::new(2)); + let started = Arc::new(AtomicBool::new(false)); + let mut pool = ObjectWorkerPool::new( + 1, + 1, + BlockingExecutor { + barrier: Arc::clone(&barrier), + started: Arc::clone(&started), + }, + ) + .expect("pool"); + + assert_eq!(pool.try_submit_round_robin(1).expect("first task"), 0); + let deadline = std::time::Instant::now() + Duration::from_secs(1); + while !started.load(Ordering::SeqCst) { + assert!( + std::time::Instant::now() < deadline, + "worker did not start first task" + ); + std::thread::sleep(Duration::from_millis(1)); + } + assert_eq!(pool.try_submit_round_robin(2).expect("queued task"), 0); + match pool.try_submit_round_robin(3) { + Err(ObjectWorkerSubmitError::QueueFull { worker_index, task }) => { + assert_eq!(worker_index, 0); + assert_eq!(task, 3); + } + other => panic!("expected queue full, got {other:?}"), + } + assert_eq!(pool.next_worker_index(), 0); + + barrier.wait(); + assert_eq!( + pool.recv_result_timeout(Duration::from_secs(1)) + .expect("result channel"), + Some(1) + ); + barrier.wait(); + assert_eq!( + pool.recv_result_timeout(Duration::from_secs(1)) + .expect("result channel"), + Some(2) + ); + } + + struct BorrowingEchoExecutor<'a> { + base: &'a u32, + } + + impl<'a> ObjectTaskExecutor for BorrowingEchoExecutor<'a> { + fn execute(&self, _worker_index: usize, task: u32) -> u32 { + task + *self.base + } + } + + #[test] + fn scoped_object_worker_pool_borrows_environment_and_processes_tasks() { + let base = 100u32; + std::thread::scope(|scope| { + let mut pool = super::ScopedObjectWorkerPool::new( + scope, + 2, + 2, + BorrowingEchoExecutor { base: &base }, + ) + .expect("scoped pool"); + assert_eq!(pool.worker_count(), 2); + pool.try_submit_round_robin(1).expect("submit 1"); + pool.try_submit_round_robin(2).expect("submit 2"); + let mut results = Vec::new(); + for _ in 0..2 { + results.push( + pool.recv_result_timeout(Duration::from_secs(1)) + .expect("result channel") + .expect("result"), + ); + } + results.sort(); + assert_eq!(results, vec![101, 102]); + // Dropping the pool inside the scope closes the task queues; the + // workers exit on their own and the scope join below must not hang. + drop(pool); + }); + } + + #[test] + fn scoped_object_worker_pool_reports_full_queue() { + std::thread::scope(|scope| { + let barrier = Arc::new(Barrier::new(2)); + let started = Arc::new(AtomicBool::new(false)); + let mut pool = super::ScopedObjectWorkerPool::new( + scope, + 1, + 1, + BlockingExecutor { + barrier: Arc::clone(&barrier), + started: Arc::clone(&started), + }, + ) + .expect("scoped pool"); + pool.try_submit_round_robin(1).expect("first task"); + let deadline = std::time::Instant::now() + Duration::from_secs(1); + while !started.load(Ordering::SeqCst) { + assert!( + std::time::Instant::now() < deadline, + "scoped worker did not start first task" + ); + std::thread::sleep(Duration::from_millis(1)); + } + pool.try_submit_round_robin(2).expect("queued task"); + match pool.try_submit_round_robin(3) { + Err(ObjectWorkerSubmitError::QueueFull { worker_index, task }) => { + assert_eq!(worker_index, 0); + assert_eq!(task, 3); + } + other => panic!("expected queue full, got {other:?}"), + } + // Dropping the pool closes the task queues; releasing the barrier + // afterwards lets the blocked worker finish task 1, observe the + // closed result channel, and exit before the scope join. + drop(pool); + barrier.wait(); + }); + } +} diff --git a/crates/panda-rpki-validator/src/parallel/phase2_scheduler.rs b/crates/panda-rpki-validator/src/parallel/phase2_scheduler.rs new file mode 100644 index 0000000..8ce2896 --- /dev/null +++ b/crates/panda-rpki-validator/src/parallel/phase2_scheduler.rs @@ -0,0 +1,332 @@ +use std::collections::{HashMap, VecDeque}; + +use crate::parallel::types::RepoIdentity; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct CaInstanceId(pub u64); + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct PublicationPointId(pub u64); + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PublicationPointState { + pub ca_instance_id: CaInstanceId, + pub pending_roa_tasks: usize, + pub child_discovery_released: bool, + pub finalized: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Phase2CompletionSnapshot { + pub ca_ready_queue_empty: bool, + pub ca_waiting_repo_empty: bool, + pub repo_tasks_idle: bool, + pub pending_roa_dispatch_empty: bool, + pub worker_queues_empty: bool, + pub object_result_queue_empty: bool, + pub object_workers_idle: bool, + pub inflight_publication_points_empty: bool, +} + +impl Phase2CompletionSnapshot { + pub fn is_complete(&self) -> bool { + self.ca_ready_queue_empty + && self.ca_waiting_repo_empty + && self.repo_tasks_idle + && self.pending_roa_dispatch_empty + && self.worker_queues_empty + && self.object_result_queue_empty + && self.object_workers_idle + && self.inflight_publication_points_empty + } +} + +#[derive(Default)] +pub struct Phase2SchedulerState { + ca_waiting_repo_by_identity: HashMap>, + ca_ready_queue: VecDeque, + inflight_publication_points: HashMap, + pending_roa_dispatch: VecDeque, +} + +impl Phase2SchedulerState { + pub fn new() -> Self { + Self { + ca_waiting_repo_by_identity: HashMap::new(), + ca_ready_queue: VecDeque::new(), + inflight_publication_points: HashMap::new(), + pending_roa_dispatch: VecDeque::new(), + } + } + + pub fn wait_for_repo(&mut self, identity: RepoIdentity, ca_id: CaInstanceId) { + self.ca_waiting_repo_by_identity + .entry(identity) + .or_default() + .push(ca_id); + } + + pub fn release_repo_waiters(&mut self, identity: &RepoIdentity) -> Vec { + let released = self + .ca_waiting_repo_by_identity + .remove(identity) + .unwrap_or_default(); + for ca_id in &released { + self.ca_ready_queue.push_back(*ca_id); + } + released + } + + pub fn push_ready_ca(&mut self, ca_id: CaInstanceId) { + self.ca_ready_queue.push_back(ca_id); + } + + pub fn pop_ready_ca(&mut self) -> Option { + self.ca_ready_queue.pop_front() + } + + pub fn start_publication_point( + &mut self, + pp_id: PublicationPointId, + ca_id: CaInstanceId, + pending_roa_tasks: usize, + ) { + self.inflight_publication_points.insert( + pp_id, + PublicationPointState { + ca_instance_id: ca_id, + pending_roa_tasks, + child_discovery_released: false, + finalized: false, + }, + ); + } + + pub fn mark_child_discovery_released(&mut self, pp_id: PublicationPointId) { + if let Some(state) = self.inflight_publication_points.get_mut(&pp_id) { + state.child_discovery_released = true; + } + } + + pub fn enqueue_roa_task(&mut self, task: T) { + self.pending_roa_dispatch.push_back(task); + } + + pub fn pop_pending_roa_dispatch(&mut self) -> Option { + self.pending_roa_dispatch.pop_front() + } + + pub fn push_front_pending_roa_dispatch(&mut self, task: T) { + self.pending_roa_dispatch.push_front(task); + } + + pub fn record_roa_result(&mut self, pp_id: PublicationPointId) -> Option { + let state = self.inflight_publication_points.get_mut(&pp_id)?; + state.pending_roa_tasks = state.pending_roa_tasks.saturating_sub(1); + if state.pending_roa_tasks == 0 { + state.finalized = true; + self.inflight_publication_points.remove(&pp_id); + Some(pp_id) + } else { + None + } + } + + pub fn waiting_repo_len(&self) -> usize { + self.ca_waiting_repo_by_identity + .values() + .map(Vec::len) + .sum() + } + + pub fn ready_queue_len(&self) -> usize { + self.ca_ready_queue.len() + } + + pub fn inflight_len(&self) -> usize { + self.inflight_publication_points.len() + } + + pub fn pending_roa_dispatch_len(&self) -> usize { + self.pending_roa_dispatch.len() + } + + pub fn publication_point_state( + &self, + pp_id: PublicationPointId, + ) -> Option<&PublicationPointState> { + self.inflight_publication_points.get(&pp_id) + } + + pub fn completion_snapshot( + &self, + repo_tasks_idle: bool, + worker_queues_empty: bool, + object_result_queue_empty: bool, + object_workers_idle: bool, + ) -> Phase2CompletionSnapshot { + Phase2CompletionSnapshot { + ca_ready_queue_empty: self.ca_ready_queue.is_empty(), + ca_waiting_repo_empty: self.ca_waiting_repo_by_identity.is_empty(), + repo_tasks_idle, + pending_roa_dispatch_empty: self.pending_roa_dispatch.is_empty(), + worker_queues_empty, + object_result_queue_empty, + object_workers_idle, + inflight_publication_points_empty: self.inflight_publication_points.is_empty(), + } + } +} + +#[cfg(test)] +mod tests { + use super::{CaInstanceId, Phase2SchedulerState, PublicationPointId}; + use crate::parallel::object_worker::{ObjectTaskExecutor, ObjectWorkerPool}; + use crate::parallel::types::RepoIdentity; + use std::time::Duration; + + fn identity(name: &str) -> RepoIdentity { + RepoIdentity::new( + Some(format!("https://example.test/{name}/notification.xml")), + format!("rsync://example.test/{name}/"), + ) + } + + #[test] + fn scheduler_repo_ready_moves_waiting_ca_to_ready_queue() { + let mut state = Phase2SchedulerState::::new(); + let repo = identity("arin"); + state.wait_for_repo(repo.clone(), CaInstanceId(1)); + state.wait_for_repo(repo.clone(), CaInstanceId(2)); + assert_eq!(state.waiting_repo_len(), 2); + + let released = state.release_repo_waiters(&repo); + assert_eq!(released, vec![CaInstanceId(1), CaInstanceId(2)]); + assert_eq!(state.waiting_repo_len(), 0); + assert_eq!(state.ready_queue_len(), 2); + assert_eq!(state.pop_ready_ca(), Some(CaInstanceId(1))); + assert_eq!(state.pop_ready_ca(), Some(CaInstanceId(2))); + } + + #[test] + fn scheduler_releases_child_before_roa_results_finalize_parent() { + let mut state = Phase2SchedulerState::::new(); + let pp = PublicationPointId(10); + state.start_publication_point(pp, CaInstanceId(1), 2); + state.mark_child_discovery_released(pp); + state.push_ready_ca(CaInstanceId(2)); + + let pp_state = state.publication_point_state(pp).expect("inflight pp"); + assert!(pp_state.child_discovery_released); + assert_eq!(pp_state.pending_roa_tasks, 2); + assert_eq!(state.pop_ready_ca(), Some(CaInstanceId(2))); + assert_eq!(state.record_roa_result(pp), None); + assert_eq!(state.inflight_len(), 1); + assert_eq!(state.record_roa_result(pp), Some(pp)); + assert_eq!(state.inflight_len(), 0); + } + + #[test] + fn scheduler_completion_requires_all_queues_and_inflight_to_be_empty() { + let mut state = Phase2SchedulerState::new(); + assert!( + state + .completion_snapshot(true, true, true, true) + .is_complete() + ); + + state.enqueue_roa_task(1u64); + assert!( + !state + .completion_snapshot(true, true, true, true) + .is_complete() + ); + assert_eq!(state.pop_pending_roa_dispatch(), Some(1)); + assert!( + state + .completion_snapshot(true, true, true, true) + .is_complete() + ); + } + + #[test] + fn scheduler_can_retry_pending_roa_task_at_front() { + let mut state = Phase2SchedulerState::new(); + state.enqueue_roa_task(1u64); + state.enqueue_roa_task(2u64); + + assert_eq!(state.pop_pending_roa_dispatch(), Some(1)); + state.push_front_pending_roa_dispatch(3); + assert_eq!(state.pop_pending_roa_dispatch(), Some(3)); + assert_eq!(state.pop_pending_roa_dispatch(), Some(2)); + assert_eq!(state.pop_pending_roa_dispatch(), None); + } + + #[derive(Clone, Debug, PartialEq, Eq)] + struct TestRoaTask { + pp_id: PublicationPointId, + value: u64, + } + + #[derive(Clone, Debug, PartialEq, Eq)] + struct TestRoaResult { + worker_index: usize, + pp_id: PublicationPointId, + value: u64, + } + + #[derive(Clone)] + struct TestRoaExecutor; + + impl ObjectTaskExecutor for TestRoaExecutor { + fn execute(&self, worker_index: usize, task: TestRoaTask) -> TestRoaResult { + TestRoaResult { + worker_index, + pp_id: task.pp_id, + value: task.value, + } + } + } + + #[test] + fn scheduler_dispatches_pending_roa_tasks_to_workers_and_finalizes_on_results() { + let pp = PublicationPointId(42); + let mut state = Phase2SchedulerState::new(); + state.start_publication_point(pp, CaInstanceId(7), 3); + state.mark_child_discovery_released(pp); + for value in 0..3 { + state.enqueue_roa_task(TestRoaTask { pp_id: pp, value }); + } + + let mut pool = ObjectWorkerPool::new(2, 4, TestRoaExecutor).expect("object pool"); + while let Some(task) = state.pop_pending_roa_dispatch() { + pool.try_submit_round_robin(task).expect("submit task"); + } + assert_eq!(state.pending_roa_dispatch_len(), 0); + + let mut results = Vec::new(); + for _ in 0..3 { + let result = pool + .recv_result_timeout(Duration::from_secs(1)) + .expect("result channel") + .expect("result"); + let finalized = state.record_roa_result(result.pp_id); + results.push(result); + if results.len() < 3 { + assert_eq!(finalized, None); + } else { + assert_eq!(finalized, Some(pp)); + } + } + results.sort_by_key(|result| result.value); + assert_eq!(results[0].worker_index, 0); + assert_eq!(results[1].worker_index, 1); + assert_eq!(results[2].worker_index, 0); + assert_eq!(state.inflight_len(), 0); + assert!( + state + .completion_snapshot(true, true, true, true) + .is_complete() + ); + } +} diff --git a/crates/panda-rpki-validator/src/parallel/repo_runtime.rs b/crates/panda-rpki-validator/src/parallel/repo_runtime.rs new file mode 100644 index 0000000..ac7a4a2 --- /dev/null +++ b/crates/panda-rpki-validator/src/parallel/repo_runtime.rs @@ -0,0 +1,1514 @@ +use std::collections::HashSet; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use crate::parallel::repo_scheduler::TransportRequestAction; +use crate::parallel::repo_worker::{RepoTransportExecutor, RepoTransportWorkerPool}; +use crate::parallel::run_coordinator::GlobalRunCoordinator; +use crate::parallel::transport_prefetch::{ + TransportPrefetchDispatchStats, TransportPrefetchRecorder, TransportPrefetchSnapshot, +}; +use crate::parallel::types::{ + RepoIdentity, RepoRequester, RepoRuntimeState, RepoTransportMode, RepoTransportResultEnvelope, + RepoTransportResultKind, +}; +use crate::policy::SyncPreference; +use crate::report::Warning; +use crate::validation::tree::{CaInstanceHandle, DiscoveredChildCaInstance}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RepoSyncRuntimeOutcome { + pub repo_sync_ok: bool, + pub repo_sync_err: Option, + pub repo_sync_source: Option, + pub repo_sync_phase: Option, + pub repo_sync_duration_ms: u64, + pub warnings: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RepoSyncRequestStatus { + Ready { + identity: RepoIdentity, + outcome: RepoSyncRuntimeOutcome, + }, + Pending { + identity: RepoIdentity, + state: RepoRuntimeState, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RepoSyncRuntimeCompletion { + pub identity: RepoIdentity, + pub state: RepoRuntimeState, + pub outcome: RepoSyncRuntimeOutcome, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RepoSyncRuntimeEvent { + pub transport_identity: RepoIdentity, + pub completions: Vec, +} + +pub trait RepoSyncRuntime: Send + Sync { + fn sync_publication_point_repo( + &self, + ca: &CaInstanceHandle, + ) -> Result; + + fn request_publication_point_repo( + &self, + ca: &CaInstanceHandle, + priority: u8, + ) -> Result; + + fn recv_repo_result_timeout( + &self, + timeout: Duration, + ) -> Result, String>; + + fn drain_repo_results_timeout( + &self, + timeout: Duration, + max_events: usize, + ) -> Result, String> { + let max_events = max_events.max(1); + let mut events = Vec::new(); + for index in 0..max_events { + let poll_timeout = if index == 0 { + timeout + } else { + Duration::from_millis(0) + }; + let Some(event) = self.recv_repo_result_timeout(poll_timeout)? else { + break; + }; + events.push(event); + } + Ok(events) + } + + fn reset_run_state(&self) -> Result<(), String>; + + fn prefetch_discovered_children( + &self, + children: &[DiscoveredChildCaInstance], + ) -> Result<(), String>; + + fn prefetch_transport_requests( + &self, + snapshot: &TransportPrefetchSnapshot, + validation_time: time::OffsetDateTime, + ) -> Result; + + fn transport_prefetch_snapshot(&self) -> TransportPrefetchSnapshot; + + /// Dead-repo blacklist working copy (#141) for run-end persistence. + /// `None` when the feature is disabled. + fn dead_repo_blacklist_state( + &self, + ) -> Option<( + crate::parallel::dead_repo_blacklist::DeadRepoBlacklistConfig, + crate::parallel::dead_repo_blacklist::DeadRepoBlacklist, + )> { + None + } +} + +pub struct Phase1RepoSyncRuntime { + coordinator: Mutex, + worker_pool: Mutex>, + transport_prefetch_recorder: Option>, + retry_short_rsync_scopes: Mutex>, + rsync_scope_resolver: Arc String + Send + Sync>, + rsync_failure_scope_resolver: Arc Option + Send + Sync>, + sync_preference: SyncPreference, +} + +impl Phase1RepoSyncRuntime { + pub fn new( + coordinator: GlobalRunCoordinator, + worker_pool: RepoTransportWorkerPool, + rsync_scope_resolver: Arc String + Send + Sync>, + sync_preference: SyncPreference, + ) -> Self { + Self::new_with_failure_scope( + coordinator, + worker_pool, + rsync_scope_resolver, + Arc::new(|_base: &str| None), + sync_preference, + ) + } + + pub fn new_with_failure_scope( + coordinator: GlobalRunCoordinator, + worker_pool: RepoTransportWorkerPool, + rsync_scope_resolver: Arc String + Send + Sync>, + rsync_failure_scope_resolver: Arc Option + Send + Sync>, + sync_preference: SyncPreference, + ) -> Self { + Self { + coordinator: Mutex::new(coordinator), + worker_pool: Mutex::new(worker_pool), + transport_prefetch_recorder: None, + retry_short_rsync_scopes: Mutex::new(HashSet::new()), + rsync_scope_resolver, + rsync_failure_scope_resolver, + sync_preference, + } + } + + pub fn new_with_failure_scope_and_prefetch_recording( + coordinator: GlobalRunCoordinator, + worker_pool: RepoTransportWorkerPool, + rsync_scope_resolver: Arc String + Send + Sync>, + rsync_failure_scope_resolver: Arc Option + Send + Sync>, + sync_preference: SyncPreference, + record_transport_prefetch_requests: bool, + ) -> Self { + Self { + coordinator: Mutex::new(coordinator), + worker_pool: Mutex::new(worker_pool), + transport_prefetch_recorder: record_transport_prefetch_requests + .then(|| Mutex::new(TransportPrefetchRecorder::default())), + retry_short_rsync_scopes: Mutex::new(HashSet::new()), + rsync_scope_resolver, + rsync_failure_scope_resolver, + sync_preference, + } + } + + fn build_requester(ca: &CaInstanceHandle) -> RepoRequester { + RepoRequester { + tal_id: ca.tal_id.clone(), + rir_id: ca.tal_id.clone(), + parent_node_id: None, + ca_instance_handle_id: format!("{}:{}", ca.tal_id, ca.manifest_rsync_uri), + publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(), + manifest_rsync_uri: ca.manifest_rsync_uri.clone(), + } + } + + fn build_identity(ca: &CaInstanceHandle) -> RepoIdentity { + RepoIdentity::new(ca.rrdp_notification_uri.clone(), ca.rsync_base_uri.clone()) + } + + fn request_transport_for_ca( + &self, + ca: &CaInstanceHandle, + priority: u8, + ) -> Result { + let identity = Self::build_identity(ca); + let requester = Self::build_requester(ca); + let rsync_scope_uri = (self.rsync_scope_resolver)(&identity.rsync_base_uri); + let rsync_failure_scope_uri = (self.rsync_failure_scope_resolver)(&identity.rsync_base_uri); + if let Some(recorder) = self.transport_prefetch_recorder.as_ref() { + let mut recorder = recorder + .lock() + .expect("transport prefetch recorder lock poisoned"); + recorder.record_registered_request( + &identity, + &requester, + priority, + rsync_scope_uri.clone(), + rsync_failure_scope_uri.clone(), + self.sync_preference, + ); + } + let action = { + let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); + coordinator.register_transport_request( + identity.clone(), + requester, + time::OffsetDateTime::now_utc(), + priority, + rsync_scope_uri, + rsync_failure_scope_uri, + self.sync_preference, + false, + ) + }; + + match action { + TransportRequestAction::Enqueue(task) => { + crate::progress_log::emit( + "phase1_repo_task_enqueued", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "repo_key_rsync_base_uri": task.repo_identity.rsync_base_uri, + "rsync_failure_scope_uri": task.rsync_failure_scope_uri, + "repo_key_notification_uri": task.repo_identity.notification_uri, + "priority": priority, + "transport_mode": match task.mode { + RepoTransportMode::Rrdp => "rrdp", + RepoTransportMode::Rsync => "rsync", + }, + }), + ); + self.drain_pending_transport_tasks()?; + Ok(RepoSyncRequestStatus::Pending { + identity, + state: self + .runtime_state_for_identity(&task.repo_identity) + .unwrap_or(RepoRuntimeState::WaitingRrdp), + }) + } + TransportRequestAction::Waiting { state } => { + crate::progress_log::emit( + "phase1_repo_task_waiting", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "repo_key_rsync_base_uri": identity.rsync_base_uri, + "rsync_failure_scope_uri": (self.rsync_failure_scope_resolver)(&identity.rsync_base_uri), + "repo_key_notification_uri": identity.notification_uri, + "priority": priority, + "runtime_state": format!("{state:?}"), + }), + ); + Ok(RepoSyncRequestStatus::Pending { identity, state }) + } + TransportRequestAction::ReusedSuccess(result) + | TransportRequestAction::ReusedTerminalFailure(result) => { + crate::progress_log::emit( + "phase1_repo_task_reused", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "repo_key_rsync_base_uri": identity.rsync_base_uri, + "rsync_failure_scope_uri": result.rsync_failure_scope_uri, + "repo_key_notification_uri": identity.notification_uri, + "priority": priority, + "transport_mode": match result.mode { + RepoTransportMode::Rrdp => "rrdp", + RepoTransportMode::Rsync => "rsync", + }, + }), + ); + Ok(RepoSyncRequestStatus::Ready { + outcome: outcome_from_transport_result( + &result, + self.runtime_state_for_identity(&identity) + .unwrap_or(RepoRuntimeState::Init), + ), + identity, + }) + } + } + } + + fn drain_pending_transport_tasks(&self) -> Result<(), String> { + loop { + let maybe_task = { + let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); + coordinator.pop_next_transport_task() + }; + let Some(task) = maybe_task else { + break; + }; + { + let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); + coordinator + .mark_transport_running(&task.dedup_key, time::OffsetDateTime::now_utc())?; + } + crate::progress_log::emit( + "phase1_repo_task_dispatched", + serde_json::json!({ + "repo_key_rsync_base_uri": task.repo_identity.rsync_base_uri, + "rsync_failure_scope_uri": task.rsync_failure_scope_uri, + "repo_key_notification_uri": task.repo_identity.notification_uri, + "requester_count": task.requesters.len(), + "priority": task.priority, + "transport_mode": match task.mode { + RepoTransportMode::Rrdp => "rrdp", + RepoTransportMode::Rsync => "rsync", + }, + }), + ); + let pool = self.worker_pool.lock().expect("worker pool lock poisoned"); + pool.submit(task)?; + } + Ok(()) + } + + fn pump_one_transport_result( + &self, + timeout: Duration, + ) -> Result, String> { + let envelope = { + let pool = self.worker_pool.lock().expect("worker pool lock poisoned"); + pool.recv_result_timeout(timeout)? + }; + let Some(envelope) = envelope else { + return Ok(None); + }; + let transport_identity = envelope.repo_identity.clone(); + let completed_envelope = envelope.clone(); + if let Some(recorder) = self.transport_prefetch_recorder.as_ref() { + recorder + .lock() + .expect("transport prefetch recorder lock poisoned") + .record_result(&envelope); + } + crate::progress_log::emit( + "phase1_repo_task_result", + serde_json::json!({ + "repo_key_rsync_base_uri": envelope.repo_identity.rsync_base_uri, + "rsync_failure_scope_uri": envelope.rsync_failure_scope_uri, + "repo_key_notification_uri": envelope.repo_identity.notification_uri, + "timing_ms": envelope.timing_ms, + "transport_mode": match envelope.mode { + RepoTransportMode::Rrdp => "rrdp", + RepoTransportMode::Rsync => "rsync", + }, + "result": match &envelope.result { + RepoTransportResultKind::Success { .. } => "success", + RepoTransportResultKind::Failed { .. } => "failed", + }, + }), + ); + let finished_at = time::OffsetDateTime::now_utc(); + let completion = { + let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); + coordinator.complete_transport_result(envelope, finished_at)? + }; + if !completion.follow_up_tasks.is_empty() { + let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); + for mut task in completion.follow_up_tasks { + if let crate::parallel::types::RepoDedupKey::RsyncScope { rsync_scope_uri } = + &task.dedup_key + { + if self + .retry_short_rsync_scopes + .lock() + .expect("retry short rsync scopes lock poisoned") + .contains(rsync_scope_uri) + { + task.retry_short_timeout = true; + } + } + crate::progress_log::emit( + "phase1_repo_task_enqueued", + serde_json::json!({ + "manifest_rsync_uri": serde_json::Value::Null, + "publication_point_rsync_uri": task.requesters.first().map(|r| r.publication_point_rsync_uri.clone()), + "repo_key_rsync_base_uri": task.repo_identity.rsync_base_uri, + "repo_key_notification_uri": task.repo_identity.notification_uri, + "priority": task.priority, + "transport_mode": "rsync", + }), + ); + coordinator.push_transport_task(task); + } + } + self.drain_pending_transport_tasks()?; + let completions = { + let coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); + coordinator + .finalized_runtime_records_for_transport_result(&completed_envelope) + .into_iter() + .filter_map(|record| { + let outcome = match record.state { + RepoRuntimeState::RrdpOk | RepoRuntimeState::RsyncOk => record + .last_success + .as_ref() + .map(|result| outcome_from_transport_result(result, record.state)), + RepoRuntimeState::FailedTerminal => record + .terminal_failure + .as_ref() + .map(|result| outcome_from_transport_result(result, record.state)), + _ => None, + }?; + Some(RepoSyncRuntimeCompletion { + identity: record.identity, + state: record.state, + outcome, + }) + }) + .collect::>() + }; + if completions.is_empty() { + return Ok(None); + } + Ok(Some(RepoSyncRuntimeEvent { + transport_identity, + completions, + })) + } + + fn pump_transport_results( + &self, + timeout: Duration, + max_events: usize, + ) -> Result, String> { + let max_events = max_events.max(1); + let mut events = Vec::new(); + for index in 0..max_events { + let poll_timeout = if index == 0 { + timeout + } else { + Duration::from_millis(0) + }; + let Some(event) = self.pump_one_transport_result(poll_timeout)? else { + break; + }; + events.push(event); + } + Ok(events) + } + + fn runtime_state_for_identity(&self, identity: &RepoIdentity) -> Option { + let coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); + coordinator + .runtime_record(identity) + .map(|record| record.state) + } + + fn resolved_outcome_for_identity( + &self, + identity: &RepoIdentity, + ) -> Option { + let coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); + let record = coordinator.runtime_record(identity)?; + match record.state { + RepoRuntimeState::RrdpOk | RepoRuntimeState::RsyncOk => record + .last_success + .as_ref() + .map(|result| outcome_from_transport_result(result, record.state)), + RepoRuntimeState::FailedTerminal => record + .terminal_failure + .as_ref() + .map(|result| outcome_from_transport_result(result, record.state)), + _ => None, + } + } +} + +impl RepoSyncRuntime for Phase1RepoSyncRuntime { + fn sync_publication_point_repo( + &self, + ca: &CaInstanceHandle, + ) -> Result { + if let RepoSyncRequestStatus::Ready { outcome, .. } = + self.request_publication_point_repo(ca, 0)? + { + return Ok(outcome); + } + let identity = Self::build_identity(ca); + loop { + if let Some(done) = self.resolved_outcome_for_identity(&identity) { + return Ok(done); + } + let _ = self.recv_repo_result_timeout(Duration::from_millis(50))?; + } + } + + fn request_publication_point_repo( + &self, + ca: &CaInstanceHandle, + priority: u8, + ) -> Result { + self.request_transport_for_ca(ca, priority) + } + + fn recv_repo_result_timeout( + &self, + timeout: Duration, + ) -> Result, String> { + self.pump_one_transport_result(timeout) + } + + fn drain_repo_results_timeout( + &self, + timeout: Duration, + max_events: usize, + ) -> Result, String> { + self.pump_transport_results(timeout, max_events) + } + + fn reset_run_state(&self) -> Result<(), String> { + { + let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); + if coordinator.stats.repo_tasks_running != 0 { + return Err(format!( + "cannot reset repo runtime with {} repo task(s) still running", + coordinator.stats.repo_tasks_running + )); + } + coordinator.reset_run_state(); + } + loop { + let maybe_result = { + let pool = self.worker_pool.lock().expect("worker pool lock poisoned"); + pool.recv_result_timeout(Duration::from_millis(0))? + }; + if maybe_result.is_none() { + break; + } + } + Ok(()) + } + + fn prefetch_discovered_children( + &self, + children: &[DiscoveredChildCaInstance], + ) -> Result<(), String> { + for child in children { + let _ = self.request_publication_point_repo(&child.handle, 1)?; + } + Ok(()) + } + + fn prefetch_transport_requests( + &self, + snapshot: &TransportPrefetchSnapshot, + validation_time: time::OffsetDateTime, + ) -> Result { + let mut stats = TransportPrefetchDispatchStats { + loaded_requests: snapshot.requests.len() as u64, + ..TransportPrefetchDispatchStats::default() + }; + + for request in &snapshot.requests { + let identity = request.to_identity(); + let current_rsync_scope_uri = (self.rsync_scope_resolver)(&identity.rsync_base_uri); + let current_rsync_failure_scope_uri = + (self.rsync_failure_scope_resolver)(&identity.rsync_base_uri); + if current_rsync_scope_uri != request.rsync_scope_uri + || current_rsync_failure_scope_uri != request.rsync_failure_scope_uri + { + stats.skipped_incompatible += 1; + continue; + } + if request.retry_short_rsync_timeout() { + self.retry_short_rsync_scopes + .lock() + .expect("retry short rsync scopes lock poisoned") + .insert(current_rsync_scope_uri.clone()); + } + let action = { + let mut coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); + coordinator.register_transport_request( + identity, + request.to_requester(), + validation_time, + request.priority, + current_rsync_scope_uri, + current_rsync_failure_scope_uri, + self.sync_preference, + request.retry_short_timeout(), + ) + }; + + match action { + TransportRequestAction::Enqueue(task) => { + stats.enqueued_tasks += 1; + crate::progress_log::emit( + "phase1_repo_prefetch_enqueued", + serde_json::json!({ + "repo_key_rsync_base_uri": task.repo_identity.rsync_base_uri, + "rsync_failure_scope_uri": task.rsync_failure_scope_uri, + "repo_key_notification_uri": task.repo_identity.notification_uri, + "priority": task.priority, + "transport_mode": match task.mode { + RepoTransportMode::Rrdp => "rrdp", + RepoTransportMode::Rsync => "rsync", + }, + }), + ); + } + TransportRequestAction::Waiting { .. } => { + stats.waiting_requests += 1; + } + TransportRequestAction::ReusedSuccess(_) + | TransportRequestAction::ReusedTerminalFailure(_) => { + stats.reused_results += 1; + } + } + } + + self.drain_pending_transport_tasks()?; + Ok(stats) + } + + fn transport_prefetch_snapshot(&self) -> TransportPrefetchSnapshot { + self.transport_prefetch_recorder + .as_ref() + .map(|recorder| { + recorder + .lock() + .expect("transport prefetch recorder lock poisoned") + .snapshot(self.sync_preference) + }) + .unwrap_or_else(|| TransportPrefetchSnapshot::new(self.sync_preference, Vec::new())) + } + + fn dead_repo_blacklist_state( + &self, + ) -> Option<( + crate::parallel::dead_repo_blacklist::DeadRepoBlacklistConfig, + crate::parallel::dead_repo_blacklist::DeadRepoBlacklist, + )> { + let coordinator = self.coordinator.lock().expect("coordinator lock poisoned"); + let config = coordinator.config.dead_repo_blacklist.clone()?; + let blacklist = coordinator.dead_repo_blacklist().cloned()?; + Some((config, blacklist)) + } +} + +fn outcome_from_transport_result( + envelope: &RepoTransportResultEnvelope, + state: RepoRuntimeState, +) -> RepoSyncRuntimeOutcome { + match (&envelope.result, state) { + (RepoTransportResultKind::Success { source, warnings }, RepoRuntimeState::RrdpOk) => { + RepoSyncRuntimeOutcome { + repo_sync_ok: true, + repo_sync_err: None, + repo_sync_source: Some(source.clone()), + repo_sync_phase: Some("rrdp_ok".to_string()), + repo_sync_duration_ms: envelope.timing_ms, + warnings: warnings.clone(), + } + } + (RepoTransportResultKind::Success { source, warnings }, RepoRuntimeState::RsyncOk) => { + RepoSyncRuntimeOutcome { + repo_sync_ok: true, + repo_sync_err: None, + repo_sync_source: Some(source.clone()), + repo_sync_phase: Some(if envelope.repo_identity.notification_uri.is_some() { + "rrdp_failed_rsync_ok".to_string() + } else { + "rsync_only_ok".to_string() + }), + repo_sync_duration_ms: envelope.timing_ms, + warnings: warnings.clone(), + } + } + ( + RepoTransportResultKind::Failed { + detail, warnings, .. + }, + RepoRuntimeState::FailedTerminal, + ) => RepoSyncRuntimeOutcome { + repo_sync_ok: false, + repo_sync_err: Some(detail.clone()), + repo_sync_source: None, + repo_sync_phase: Some(if envelope.repo_identity.notification_uri.is_some() { + "rrdp_failed_rsync_failed".to_string() + } else { + "rsync_failed".to_string() + }), + repo_sync_duration_ms: envelope.timing_ms, + warnings: warnings.clone(), + }, + _ => RepoSyncRuntimeOutcome { + repo_sync_ok: false, + repo_sync_err: Some("repo runtime state unresolved".to_string()), + repo_sync_source: None, + repo_sync_phase: Some("repo_runtime_unresolved".to_string()), + repo_sync_duration_ms: envelope.timing_ms, + warnings: Vec::new(), + }, + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::{Duration, Instant}; + + use crate::parallel::config::ParallelPhase1Config; + use crate::parallel::repo_runtime::{Phase1RepoSyncRuntime, RepoSyncRuntime}; + use crate::parallel::repo_worker::{ + RepoTransportExecutor, RepoTransportWorkerPool, RepoWorkerPoolConfig, + }; + use crate::parallel::run_coordinator::GlobalRunCoordinator; + use crate::parallel::transport_prefetch::TransportPrefetchSnapshot; + use crate::parallel::types::{ + RepoRuntimeState, RepoTransportMode, RepoTransportResultEnvelope, RepoTransportResultKind, + RepoTransportTask, TalInputSpec, + }; + use crate::policy::SyncPreference; + use crate::report::Warning; + use crate::validation::tree::{CaCertificateRef, CaInstanceHandle, DiscoveredChildCaInstance}; + + fn sample_ca(manifest: &str) -> CaInstanceHandle { + CaInstanceHandle { + depth: 0, + tal_id: "arin".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(vec![1, 2, 3]), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/".to_string(), + manifest_rsync_uri: manifest.to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), + rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), + } + } + + struct SuccessTransportExecutor; + + impl RepoTransportExecutor for SuccessTransportExecutor { + fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { + RepoTransportResultEnvelope { + dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(), + repo_identity: task.repo_identity, + mode: task.mode, + tal_id: task.tal_id, + rir_id: task.rir_id, + timing_ms: 7, + result: RepoTransportResultKind::Success { + source: match task.mode { + RepoTransportMode::Rrdp => "rrdp".to_string(), + RepoTransportMode::Rsync => "rsync".to_string(), + }, + warnings: vec![Warning::new("transport ok")], + }, + } + } + } + + struct CountingSuccessTransportExecutor { + count: Arc, + } + + impl RepoTransportExecutor for CountingSuccessTransportExecutor { + fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { + self.count.fetch_add(1, Ordering::SeqCst); + RepoTransportResultEnvelope { + dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(), + repo_identity: task.repo_identity, + mode: task.mode, + tal_id: task.tal_id, + rir_id: task.rir_id, + timing_ms: 7, + result: RepoTransportResultKind::Success { + source: match task.mode { + RepoTransportMode::Rrdp => "rrdp".to_string(), + RepoTransportMode::Rsync => "rsync".to_string(), + }, + warnings: vec![Warning::new("transport ok")], + }, + } + } + } + + struct FailRrdpThenSucceedRsyncExecutor { + rrdp_count: Arc, + rsync_count: Arc, + } + + impl RepoTransportExecutor for FailRrdpThenSucceedRsyncExecutor { + fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { + match task.mode { + RepoTransportMode::Rrdp => { + self.rrdp_count.fetch_add(1, Ordering::SeqCst); + RepoTransportResultEnvelope { + dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(), + repo_identity: task.repo_identity, + mode: RepoTransportMode::Rrdp, + tal_id: task.tal_id, + rir_id: task.rir_id, + timing_ms: 10, + result: RepoTransportResultKind::Failed { + detail: "rrdp failed".to_string(), + warnings: vec![Warning::new("rrdp failed")], + error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, + }, + } + } + RepoTransportMode::Rsync => { + self.rsync_count.fetch_add(1, Ordering::SeqCst); + RepoTransportResultEnvelope { + dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(), + repo_identity: task.repo_identity, + mode: RepoTransportMode::Rsync, + tal_id: task.tal_id, + rir_id: task.rir_id, + timing_ms: 12, + result: RepoTransportResultKind::Success { + source: "rsync".to_string(), + warnings: vec![Warning::new("rsync ok")], + }, + } + } + } + } + } + + struct FailRrdpThenFailRsyncExecutor { + rrdp_count: Arc, + rsync_count: Arc, + } + + impl RepoTransportExecutor for FailRrdpThenFailRsyncExecutor { + fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { + match task.mode { + RepoTransportMode::Rrdp => { + self.rrdp_count.fetch_add(1, Ordering::SeqCst); + RepoTransportResultEnvelope { + dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(), + repo_identity: task.repo_identity, + mode: RepoTransportMode::Rrdp, + tal_id: task.tal_id, + rir_id: task.rir_id, + timing_ms: 10, + result: RepoTransportResultKind::Failed { + detail: "rrdp failed".to_string(), + warnings: vec![Warning::new("rrdp failed")], + error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, + }, + } + } + RepoTransportMode::Rsync => { + self.rsync_count.fetch_add(1, Ordering::SeqCst); + RepoTransportResultEnvelope { + dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(), + repo_identity: task.repo_identity, + mode: RepoTransportMode::Rsync, + tal_id: task.tal_id, + rir_id: task.rir_id, + timing_ms: 12, + result: RepoTransportResultKind::Failed { + detail: "rsync failed".to_string(), + warnings: vec![Warning::new("rsync failed")], + error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, + }, + } + } + } + } + } + + #[test] + fn phase1_runtime_waits_for_rrdp_transport_and_returns_rrdp_outcome() { + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + SuccessTransportExecutor, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new( + coordinator, + pool, + Arc::new(|base: &str| base.to_string()), + SyncPreference::RrdpThenRsync, + ); + + let outcome = runtime + .sync_publication_point_repo(&sample_ca("rsync://example.test/repo/root.mft")) + .expect("sync repo"); + assert!(outcome.repo_sync_ok); + assert_eq!(outcome.repo_sync_source.as_deref(), Some("rrdp")); + assert_eq!(outcome.repo_sync_phase.as_deref(), Some("rrdp_ok")); + } + + #[test] + fn phase1_runtime_request_repo_returns_pending_then_repo_ready_event() { + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + SuccessTransportExecutor, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new( + coordinator, + pool, + Arc::new(|base: &str| base.to_string()), + SyncPreference::RrdpThenRsync, + ); + let ca = sample_ca("rsync://example.test/repo/root.mft"); + + let status = runtime + .request_publication_point_repo(&ca, 0) + .expect("request repo"); + let identity = match status { + super::RepoSyncRequestStatus::Pending { identity, state } => { + assert_eq!(state, RepoRuntimeState::WaitingRrdp); + identity + } + other => panic!("expected pending, got {other:?}"), + }; + + let event = runtime + .recv_repo_result_timeout(Duration::from_secs(1)) + .expect("repo event") + .expect("event"); + assert_eq!(event.transport_identity, identity); + assert_eq!(event.completions.len(), 1); + assert_eq!(event.completions[0].identity, identity); + assert_eq!(event.completions[0].state, RepoRuntimeState::RrdpOk); + assert!(event.completions[0].outcome.repo_sync_ok); + assert_eq!( + event.completions[0].outcome.repo_sync_source.as_deref(), + Some("rrdp") + ); + } + + #[test] + fn phase1_runtime_request_repo_reuses_ready_event_result() { + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + SuccessTransportExecutor, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new( + coordinator, + pool, + Arc::new(|base: &str| base.to_string()), + SyncPreference::RrdpThenRsync, + ); + let ca = sample_ca("rsync://example.test/repo/root.mft"); + let first = runtime + .request_publication_point_repo(&ca, 0) + .expect("request repo"); + assert!(matches!( + first, + super::RepoSyncRequestStatus::Pending { .. } + )); + let _ = runtime + .recv_repo_result_timeout(Duration::from_secs(1)) + .expect("repo event") + .expect("event"); + + let second = runtime + .request_publication_point_repo(&ca, 0) + .expect("request repo reused"); + match second { + super::RepoSyncRequestStatus::Ready { outcome, .. } => { + assert!(outcome.repo_sync_ok); + assert_eq!(outcome.repo_sync_phase.as_deref(), Some("rrdp_ok")); + } + other => panic!("expected ready reuse, got {other:?}"), + } + } + + #[test] + fn phase1_runtime_repo_event_reports_all_finalized_identities_for_shared_rrdp() { + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + SuccessTransportExecutor, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new( + coordinator, + pool, + Arc::new(|base: &str| base.to_string()), + SyncPreference::RrdpThenRsync, + ); + let ca1 = sample_ca("rsync://example.test/repo/root.mft"); + let mut ca2 = sample_ca("rsync://example.test/other/root.mft"); + ca2.rsync_base_uri = "rsync://example.test/other/".to_string(); + ca2.publication_point_rsync_uri = "rsync://example.test/other/".to_string(); + + let id1 = match runtime + .request_publication_point_repo(&ca1, 0) + .expect("request first") + { + super::RepoSyncRequestStatus::Pending { identity, .. } => identity, + other => panic!("expected first pending, got {other:?}"), + }; + let id2 = match runtime + .request_publication_point_repo(&ca2, 0) + .expect("request second") + { + super::RepoSyncRequestStatus::Pending { identity, .. } => identity, + other => panic!("expected second pending, got {other:?}"), + }; + assert_ne!(id1, id2); + + let event = runtime + .recv_repo_result_timeout(Duration::from_secs(1)) + .expect("repo event") + .expect("event"); + let mut identities = event + .completions + .iter() + .map(|completion| completion.identity.clone()) + .collect::>(); + identities.sort_by(|a, b| a.rsync_base_uri.cmp(&b.rsync_base_uri)); + let mut expected = vec![id1, id2]; + expected.sort_by(|a, b| a.rsync_base_uri.cmp(&b.rsync_base_uri)); + assert_eq!(identities, expected); + assert!( + event + .completions + .iter() + .all(|completion| completion.state == RepoRuntimeState::RrdpOk) + ); + } + + #[test] + fn phase1_runtime_drains_multiple_ready_transport_events() { + let count = Arc::new(AtomicUsize::new(0)); + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 2 }, + CountingSuccessTransportExecutor { + count: Arc::clone(&count), + }, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new( + coordinator, + pool, + Arc::new(|base: &str| base.to_string()), + SyncPreference::RrdpThenRsync, + ); + let ca1 = sample_ca("rsync://example.test/repo/root.mft"); + let mut ca2 = sample_ca("rsync://example.net/repo/root.mft"); + ca2.rsync_base_uri = "rsync://example.net/repo/".to_string(); + ca2.publication_point_rsync_uri = "rsync://example.net/repo/".to_string(); + ca2.rrdp_notification_uri = Some("https://example.net/notify.xml".to_string()); + + assert!(matches!( + runtime + .request_publication_point_repo(&ca1, 0) + .expect("request ca1"), + super::RepoSyncRequestStatus::Pending { .. } + )); + assert!(matches!( + runtime + .request_publication_point_repo(&ca2, 0) + .expect("request ca2"), + super::RepoSyncRequestStatus::Pending { .. } + )); + + let started = Instant::now(); + while count.load(Ordering::SeqCst) < 2 && started.elapsed() < Duration::from_secs(1) { + std::thread::sleep(Duration::from_millis(5)); + } + assert_eq!(count.load(Ordering::SeqCst), 2); + + let events = runtime + .drain_repo_results_timeout(Duration::from_millis(0), 8) + .expect("drain events"); + assert_eq!(events.len(), 2); + assert_eq!( + events + .iter() + .map(|event| event.completions.len()) + .sum::(), + 2 + ); + } + + #[test] + fn phase1_runtime_reset_run_state_clears_completed_transport_reuse() { + let count = Arc::new(AtomicUsize::new(0)); + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + CountingSuccessTransportExecutor { + count: Arc::clone(&count), + }, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new( + coordinator, + pool, + Arc::new(|base: &str| base.to_string()), + SyncPreference::RrdpThenRsync, + ); + let ca = sample_ca("rsync://example.test/repo/root.mft"); + + assert!(matches!( + runtime + .request_publication_point_repo(&ca, 0) + .expect("first request"), + super::RepoSyncRequestStatus::Pending { .. } + )); + let _ = runtime + .recv_repo_result_timeout(Duration::from_secs(1)) + .expect("first event") + .expect("event"); + assert_eq!(count.load(Ordering::SeqCst), 1); + assert!(matches!( + runtime + .request_publication_point_repo(&ca, 0) + .expect("ready reuse before reset"), + super::RepoSyncRequestStatus::Ready { .. } + )); + + runtime.reset_run_state().expect("reset"); + + assert!(matches!( + runtime + .request_publication_point_repo(&ca, 0) + .expect("second request after reset"), + super::RepoSyncRequestStatus::Pending { .. } + )); + let _ = runtime + .recv_repo_result_timeout(Duration::from_secs(1)) + .expect("second event") + .expect("event"); + assert_eq!(count.load(Ordering::SeqCst), 2); + } + + #[test] + fn phase1_runtime_records_prefetch_snapshot_only_when_enabled() { + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + SuccessTransportExecutor, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new_with_failure_scope_and_prefetch_recording( + coordinator, + pool, + Arc::new(|base: &str| base.to_string()), + Arc::new(|_base: &str| Some("rsync://example.test/".to_string())), + SyncPreference::RrdpThenRsync, + true, + ); + let ca = sample_ca("rsync://example.test/repo/root.mft"); + + let _ = runtime + .request_publication_point_repo(&ca, 0) + .expect("request repo"); + let snapshot = runtime.transport_prefetch_snapshot(); + assert_eq!(snapshot.requests.len(), 1); + assert_eq!( + snapshot.requests[0].repo_identity.rsync_base_uri, + "rsync://example.test/repo/" + ); + assert_eq!( + snapshot.requests[0].rsync_failure_scope_uri.as_deref(), + Some("rsync://example.test/") + ); + + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + SuccessTransportExecutor, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new( + coordinator, + pool, + Arc::new(|base: &str| base.to_string()), + SyncPreference::RrdpThenRsync, + ); + let _ = runtime + .request_publication_point_repo(&ca, 0) + .expect("request repo without recording"); + assert!(runtime.transport_prefetch_snapshot().requests.is_empty()); + } + + #[test] + fn phase1_runtime_prefetch_dispatches_and_later_request_waits_on_same_task() { + let count = Arc::new(AtomicUsize::new(0)); + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + CountingSuccessTransportExecutor { + count: Arc::clone(&count), + }, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new_with_failure_scope_and_prefetch_recording( + coordinator, + pool, + Arc::new(|base: &str| base.to_string()), + Arc::new(|_base: &str| Some("rsync://example.test/".to_string())), + SyncPreference::RrdpThenRsync, + true, + ); + let ca = sample_ca("rsync://example.test/repo/root.mft"); + let mut recorder = + crate::parallel::transport_prefetch::TransportPrefetchRecorder::default(); + recorder.record_registered_request( + &super::Phase1RepoSyncRuntime::::build_identity(&ca), + &super::Phase1RepoSyncRuntime::::build_requester(&ca), + 0, + "rsync://example.test/repo/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RrdpThenRsync, + ); + let snapshot = recorder.snapshot(SyncPreference::RrdpThenRsync); + + let stats = runtime + .prefetch_transport_requests(&snapshot, time::OffsetDateTime::UNIX_EPOCH) + .expect("prefetch transport requests"); + assert_eq!(stats.loaded_requests, 1); + assert_eq!(stats.enqueued_tasks, 1); + + let status = runtime + .request_publication_point_repo(&ca, 0) + .expect("request after prefetch"); + assert!(matches!( + status, + super::RepoSyncRequestStatus::Pending { + state: RepoRuntimeState::WaitingRrdp, + .. + } + )); + let _ = runtime + .recv_repo_result_timeout(Duration::from_secs(1)) + .expect("repo event") + .expect("event"); + assert_eq!(count.load(Ordering::SeqCst), 1); + } + + #[test] + fn phase1_runtime_does_not_persist_prefetch_only_requests() { + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + SuccessTransportExecutor, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new_with_failure_scope_and_prefetch_recording( + coordinator, + pool, + Arc::new(|base: &str| base.to_string()), + Arc::new(|_base: &str| Some("rsync://example.test/".to_string())), + SyncPreference::RrdpThenRsync, + true, + ); + let snapshot = TransportPrefetchSnapshot::new( + SyncPreference::RrdpThenRsync, + vec![crate::parallel::transport_prefetch::TransportPrefetchRequest::from_registered_request( + &crate::parallel::types::RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/", + ), + &crate::parallel::types::RepoRequester::with_tal_rir( + "arin", + "arin", + "rsync://example.test/repo/root.mft", + "rsync://example.test/repo/", + "arin:root", + ), + 0, + "rsync://example.test/repo/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RrdpThenRsync, + )], + ); + + let stats = runtime + .prefetch_transport_requests(&snapshot, time::OffsetDateTime::UNIX_EPOCH) + .expect("prefetch transport requests"); + assert_eq!(stats.enqueued_tasks, 1); + assert!( + runtime.transport_prefetch_snapshot().requests.is_empty(), + "prefetch-only requests should not be carried forward forever" + ); + } + + #[test] + fn phase1_runtime_prefetch_skips_requests_when_scope_resolver_differs() { + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + SuccessTransportExecutor, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new_with_failure_scope_and_prefetch_recording( + coordinator, + pool, + Arc::new(|_base: &str| "rsync://example.test/different/".to_string()), + Arc::new(|_base: &str| Some("rsync://example.test/".to_string())), + SyncPreference::RrdpThenRsync, + true, + ); + let snapshot = TransportPrefetchSnapshot::new( + SyncPreference::RrdpThenRsync, + vec![crate::parallel::transport_prefetch::TransportPrefetchRequest::from_registered_request( + &crate::parallel::types::RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/", + ), + &crate::parallel::types::RepoRequester::with_tal_rir( + "arin", + "arin", + "rsync://example.test/repo/root.mft", + "rsync://example.test/repo/", + "arin:root", + ), + 0, + "rsync://example.test/repo/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RrdpThenRsync, + )], + ); + + let stats = runtime + .prefetch_transport_requests(&snapshot, time::OffsetDateTime::UNIX_EPOCH) + .expect("prefetch transport requests"); + assert_eq!(stats.loaded_requests, 1); + assert_eq!(stats.enqueued_tasks, 0); + assert_eq!(stats.skipped_incompatible, 1); + } + + #[test] + fn phase1_runtime_transitions_rrdp_failure_to_rsync_success() { + let rrdp_count = Arc::new(AtomicUsize::new(0)); + let rsync_count = Arc::new(AtomicUsize::new(0)); + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + FailRrdpThenSucceedRsyncExecutor { + rrdp_count: Arc::clone(&rrdp_count), + rsync_count: Arc::clone(&rsync_count), + }, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new( + coordinator, + pool, + Arc::new(|_base: &str| "rsync://example.test/module/".to_string()), + SyncPreference::RrdpThenRsync, + ); + + let outcome = runtime + .sync_publication_point_repo(&sample_ca("rsync://example.test/repo/root.mft")) + .expect("sync repo"); + assert!(outcome.repo_sync_ok); + assert_eq!(outcome.repo_sync_source.as_deref(), Some("rsync")); + assert_eq!( + outcome.repo_sync_phase.as_deref(), + Some("rrdp_failed_rsync_ok") + ); + assert_eq!(rrdp_count.load(Ordering::SeqCst), 1); + assert_eq!(rsync_count.load(Ordering::SeqCst), 1); + } + + #[test] + fn phase1_runtime_terminal_failure_keeps_rsync_failure_duration() { + let rrdp_count = Arc::new(AtomicUsize::new(0)); + let rsync_count = Arc::new(AtomicUsize::new(0)); + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + FailRrdpThenFailRsyncExecutor { + rrdp_count: Arc::clone(&rrdp_count), + rsync_count: Arc::clone(&rsync_count), + }, + ) + .expect("pool"); + let runtime = Phase1RepoSyncRuntime::new( + coordinator, + pool, + Arc::new(|_base: &str| "rsync://example.test/module/".to_string()), + SyncPreference::RrdpThenRsync, + ); + + let outcome = runtime + .sync_publication_point_repo(&sample_ca("rsync://example.test/repo/root.mft")) + .expect("sync repo"); + assert!(!outcome.repo_sync_ok); + assert_eq!( + outcome.repo_sync_phase.as_deref(), + Some("rrdp_failed_rsync_failed") + ); + assert_eq!(outcome.repo_sync_duration_ms, 12); + assert_eq!(rrdp_count.load(Ordering::SeqCst), 1); + assert_eq!(rsync_count.load(Ordering::SeqCst), 1); + } + + #[test] + fn phase1_runtime_prefetch_submits_transport_task_before_consumption() { + let rrdp_count = Arc::new(AtomicUsize::new(0)); + let rsync_count = Arc::new(AtomicUsize::new(0)); + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 1 }, + FailRrdpThenSucceedRsyncExecutor { + rrdp_count: Arc::clone(&rrdp_count), + rsync_count: Arc::clone(&rsync_count), + }, + ) + .expect("pool"); + let runtime = Arc::new(Phase1RepoSyncRuntime::new( + coordinator, + pool, + Arc::new(|_base: &str| "rsync://example.test/module/".to_string()), + SyncPreference::RrdpThenRsync, + )); + + let child = DiscoveredChildCaInstance { + handle: sample_ca("rsync://example.test/repo/child.mft"), + discovered_from: crate::audit::DiscoveredFrom { + parent_manifest_rsync_uri: "rsync://example.test/repo/root.mft".to_string(), + child_ca_certificate_rsync_uri: "rsync://example.test/repo/child.cer".to_string(), + child_ca_certificate_sha256_hex: "00".repeat(32), + }, + child_entry_projection: None, + }; + + runtime + .prefetch_discovered_children(std::slice::from_ref(&child)) + .expect("prefetch"); + + let started = Instant::now(); + while rrdp_count.load(Ordering::SeqCst) == 0 && started.elapsed() < Duration::from_secs(1) { + std::thread::sleep(Duration::from_millis(10)); + } + assert_eq!(rrdp_count.load(Ordering::SeqCst), 1); + + let outcome = runtime + .sync_publication_point_repo(&child.handle) + .expect("sync child repo"); + assert!(outcome.repo_sync_ok); + assert_eq!(rsync_count.load(Ordering::SeqCst), 1); + } +} diff --git a/crates/panda-rpki-validator/src/parallel/repo_scheduler.rs b/crates/panda-rpki-validator/src/parallel/repo_scheduler.rs new file mode 100644 index 0000000..08a6e29 --- /dev/null +++ b/crates/panda-rpki-validator/src/parallel/repo_scheduler.rs @@ -0,0 +1,2244 @@ +use std::collections::{HashMap, HashSet}; + +use crate::parallel::dead_repo_blacklist::DeadRepoBlacklist; +use crate::parallel::types::{ + InFlightRepoEntry, RepoDedupKey, RepoIdentity, RepoKey, RepoRequester, RepoRuntimeState, + RepoSyncResultEnvelope, RepoSyncResultKind, RepoSyncResultRef, RepoSyncTask, RepoTaskState, + RepoTransportErrorClass, RepoTransportMode, RepoTransportResultEnvelope, + RepoTransportResultKind, RepoTransportTask, +}; +use crate::policy::SyncPreference; +use crate::report::Warning; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RepoRequestAction { + Enqueued(RepoSyncTask), + Waiting, + Reused(RepoSyncResultRef), + FailedReuse { detail: String }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RepoCompletion { + pub repo_key: RepoKey, + pub released_requesters: Vec, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TransportTaskState { + Pending, + Running, + Finished, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TransportInFlightEntry { + pub state: TransportTaskState, + pub task: RepoTransportTask, + pub waiting_requesters: Vec, + pub last_result: Option, + pub started_at: Option, + pub finished_at: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RepoRuntimeRecord { + pub identity: RepoIdentity, + pub state: RepoRuntimeState, + pub rrdp_notification_key: Option, + pub rsync_scope_key: String, + pub rsync_failure_scope_key: Option, + pub requesters: Vec, + pub validation_time: time::OffsetDateTime, + pub priority: u8, + pub last_success: Option, + pub terminal_failure: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum TransportRequestAction { + Enqueue(RepoTransportTask), + Waiting { state: RepoRuntimeState }, + ReusedSuccess(RepoTransportResultEnvelope), + ReusedTerminalFailure(RepoTransportResultEnvelope), +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TransportCompletion { + pub released_requesters: Vec, + pub follow_up_tasks: Vec, +} + +#[derive(Default)] +pub struct TransportStateTables { + rrdp_inflight: HashMap, + rsync_inflight: HashMap, + rsync_failure_by_scope: HashMap, + rsync_failure_probe_inflight: HashMap, + rsync_failure_scope_reachable: HashSet, + runtime_records: HashMap, + /// Frozen run-start snapshot of the dead-repo blacklist (#141). Entries + /// admitted during this run only take effect from the next run. + dead_repo_blacklist: Option, +} + +impl TransportStateTables { + pub fn new() -> Self { + Self::default() + } + + pub fn set_dead_repo_blacklist(&mut self, blacklist: DeadRepoBlacklist) { + self.dead_repo_blacklist = Some(blacklist); + } + + fn dead_repo_terminal_envelope( + identity: &RepoIdentity, + requesters: &[RepoRequester], + rsync_scope_uri: &str, + rsync_failure_scope_uri: Option, + ) -> RepoTransportResultEnvelope { + let first_requester = requesters + .first() + .expect("blacklist terminal record must keep at least one requester"); + RepoTransportResultEnvelope { + dedup_key: RepoDedupKey::RsyncScope { + rsync_scope_uri: rsync_scope_uri.to_string(), + }, + rsync_failure_scope_uri, + repo_identity: identity.clone(), + mode: RepoTransportMode::Rsync, + tal_id: first_requester.tal_id.clone(), + rir_id: first_requester.rir_id.clone(), + timing_ms: 0, + result: RepoTransportResultKind::Failed { + detail: format!( + "dead repo blacklist: rsync transport persistently unreachable: {}", + identity.rsync_base_uri + ), + warnings: vec![Warning::new(format!( + "dead_repo_blacklist_skip_all: repository {} skipped after repeated transport failures", + identity.rsync_base_uri + )) + .with_context(identity.rsync_base_uri.clone())], + error_class: RepoTransportErrorClass::Unknown, + }, + } + } + + pub fn runtime_record(&self, identity: &RepoIdentity) -> Option<&RepoRuntimeRecord> { + self.runtime_records.get(identity) + } + + pub fn finalized_runtime_records_for_transport( + &self, + dedup_key: &RepoDedupKey, + ) -> Vec { + self.runtime_records + .values() + .filter(|record| match dedup_key { + RepoDedupKey::RrdpNotify { notification_uri } => { + record.rrdp_notification_key.as_deref() == Some(notification_uri.as_str()) + } + RepoDedupKey::RsyncScope { rsync_scope_uri } => { + record.rsync_scope_key == *rsync_scope_uri + } + }) + .filter(|record| { + matches!( + record.state, + RepoRuntimeState::RrdpOk + | RepoRuntimeState::RsyncOk + | RepoRuntimeState::FailedTerminal + ) + }) + .cloned() + .collect() + } + + pub fn finalized_runtime_records_for_transport_result( + &self, + result: &RepoTransportResultEnvelope, + ) -> Vec { + let failure_scope = reusable_rsync_failure_scope(result); + self.runtime_records + .values() + .filter(|record| { + let exact_match = match &result.dedup_key { + RepoDedupKey::RrdpNotify { notification_uri } => { + record.rrdp_notification_key.as_deref() == Some(notification_uri.as_str()) + } + RepoDedupKey::RsyncScope { rsync_scope_uri } => { + record.rsync_scope_key == *rsync_scope_uri + } + }; + let failure_scope_match = failure_scope + .map(|scope| record.rsync_failure_scope_key.as_deref() == Some(scope)) + .unwrap_or(false); + exact_match || failure_scope_match + }) + .filter(|record| { + matches!( + record.state, + RepoRuntimeState::RrdpOk + | RepoRuntimeState::RsyncOk + | RepoRuntimeState::FailedTerminal + ) + }) + .cloned() + .collect() + } + + pub fn reset_run_state(&mut self) { + self.rrdp_inflight.clear(); + self.rsync_inflight.clear(); + self.rsync_failure_by_scope.clear(); + self.rsync_failure_probe_inflight.clear(); + self.rsync_failure_scope_reachable.clear(); + self.runtime_records.clear(); + } + + pub fn register_transport_request( + &mut self, + identity: RepoIdentity, + requester: RepoRequester, + validation_time: time::OffsetDateTime, + priority: u8, + rsync_scope_uri: String, + rsync_failure_scope_uri: Option, + sync_preference: SyncPreference, + ) -> TransportRequestAction { + if let Some(record) = self.runtime_records.get_mut(&identity) { + record.requesters.push(requester.clone()); + return match record.state { + RepoRuntimeState::WaitingRrdp => { + if let Some(key) = record.rrdp_notification_key.as_ref() { + if let Some(entry) = self.rrdp_inflight.get_mut(key) { + entry.waiting_requesters.push(requester); + } + } + TransportRequestAction::Waiting { + state: RepoRuntimeState::WaitingRrdp, + } + } + RepoRuntimeState::RrdpOk | RepoRuntimeState::RsyncOk => { + TransportRequestAction::ReusedSuccess( + record + .last_success + .clone() + .expect("success state must keep last_success"), + ) + } + RepoRuntimeState::RrdpFailedPendingRsync | RepoRuntimeState::WaitingRsync => { + if let Some(entry) = self.rsync_inflight.get_mut(&record.rsync_scope_key) { + entry.waiting_requesters.push(requester); + } + record.state = RepoRuntimeState::WaitingRsync; + TransportRequestAction::Waiting { + state: RepoRuntimeState::WaitingRsync, + } + } + RepoRuntimeState::FailedTerminal => TransportRequestAction::ReusedTerminalFailure( + record + .terminal_failure + .clone() + .expect("terminal failure must keep last result"), + ), + RepoRuntimeState::Init => TransportRequestAction::Waiting { + state: RepoRuntimeState::Init, + }, + }; + } + + // Dead-repo blacklist fast paths (#141), consulted only for fresh + // identities; existing records already encode their terminal state. + if let Some(blacklist) = self.dead_repo_blacklist.as_ref() { + let rrdp_wanted = sync_preference == SyncPreference::RrdpThenRsync + && identity.notification_uri.is_some(); + let rrdp_dead = rrdp_wanted + && identity + .notification_uri + .as_deref() + .map(|uri| blacklist.is_blacklisted(RepoTransportMode::Rrdp, uri)) + .unwrap_or(false); + let rsync_dead = blacklist + .is_blacklisted(RepoTransportMode::Rsync, identity.rsync_base_uri.as_str()); + if rsync_dead && (!rrdp_wanted || rrdp_dead) { + crate::progress_log::emit( + "dead_repo_blacklist_skip_all", + serde_json::json!({ + "repo_key_notification_uri": identity.notification_uri, + "repo_key_rsync_base_uri": identity.rsync_base_uri, + }), + ); + let envelope = Self::dead_repo_terminal_envelope( + &identity, + std::slice::from_ref(&requester), + &rsync_scope_uri, + rsync_failure_scope_uri.clone(), + ); + self.runtime_records.insert( + identity.clone(), + RepoRuntimeRecord { + identity, + state: RepoRuntimeState::FailedTerminal, + rrdp_notification_key: None, + rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri, + requesters: vec![requester], + validation_time, + priority, + last_success: None, + terminal_failure: Some(envelope.clone()), + }, + ); + return TransportRequestAction::ReusedTerminalFailure(envelope); + } + if rrdp_dead { + crate::progress_log::emit( + "dead_repo_blacklist_skip_rrdp", + serde_json::json!({ + "repo_key_notification_uri": identity.notification_uri, + "repo_key_rsync_base_uri": identity.rsync_base_uri, + }), + ); + let mut action = self.register_rsync_request( + identity, + requester, + validation_time, + priority, + rsync_scope_uri, + rsync_failure_scope_uri, + ); + // A blacklisted rrdp transport is a known-persistent failure, + // so the rsync fallback must keep the short retry profile + // (#141): the prefetch layer records no real rrdp failure in + // skipped runs, which would otherwise silently drop the task + // back to the default 15s window from the second skipped run + // on. The coordinator wrapper only ever sets the flag to true, + // so setting it here survives registration. + if let TransportRequestAction::Enqueue(task) = &mut action { + task.retry_short_timeout = true; + } + return action; + } + } + + if sync_preference == SyncPreference::RrdpThenRsync { + if let Some(notification_uri) = identity.notification_uri.clone() { + if let Some(entry) = self.rrdp_inflight.get_mut(¬ification_uri) { + if let Some(result) = entry.last_result.clone() { + return match result.result { + RepoTransportResultKind::Success { .. } => { + self.runtime_records.insert( + identity.clone(), + RepoRuntimeRecord { + identity, + state: RepoRuntimeState::RrdpOk, + rrdp_notification_key: Some(notification_uri), + rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), + requesters: vec![requester], + validation_time, + priority, + last_success: Some(result.clone()), + terminal_failure: None, + }, + ); + TransportRequestAction::ReusedSuccess(result) + } + RepoTransportResultKind::Failed { .. } => { + self.runtime_records.insert( + identity.clone(), + RepoRuntimeRecord { + identity: identity.clone(), + state: RepoRuntimeState::RrdpFailedPendingRsync, + rrdp_notification_key: Some(notification_uri), + rsync_scope_key: rsync_scope_uri.clone(), + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), + requesters: vec![requester.clone()], + validation_time, + priority, + last_success: None, + terminal_failure: None, + }, + ); + self.register_rsync_request( + identity, + requester, + validation_time, + priority, + rsync_scope_uri, + rsync_failure_scope_uri, + ) + } + }; + } + + entry.waiting_requesters.push(requester.clone()); + self.runtime_records.insert( + identity.clone(), + RepoRuntimeRecord { + identity, + state: RepoRuntimeState::WaitingRrdp, + rrdp_notification_key: Some(notification_uri), + rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), + requesters: vec![requester], + validation_time, + priority, + last_success: None, + terminal_failure: None, + }, + ); + return TransportRequestAction::Waiting { + state: RepoRuntimeState::WaitingRrdp, + }; + } + + let task = RepoTransportTask { + dedup_key: RepoDedupKey::RrdpNotify { + notification_uri: notification_uri.clone(), + }, + rsync_failure_scope_uri: None, + repo_identity: identity.clone(), + mode: RepoTransportMode::Rrdp, + retry_short_timeout: false, + tal_id: requester.tal_id.clone(), + rir_id: requester.rir_id.clone(), + validation_time, + priority, + requesters: vec![requester.clone()], + }; + self.rrdp_inflight.insert( + notification_uri.clone(), + TransportInFlightEntry { + state: TransportTaskState::Pending, + task: task.clone(), + waiting_requesters: Vec::new(), + last_result: None, + started_at: None, + finished_at: None, + }, + ); + self.runtime_records.insert( + identity.clone(), + RepoRuntimeRecord { + identity, + state: RepoRuntimeState::WaitingRrdp, + rrdp_notification_key: Some(notification_uri), + rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), + requesters: vec![requester], + validation_time, + priority, + last_success: None, + terminal_failure: None, + }, + ); + return TransportRequestAction::Enqueue(task); + } + } + + self.register_rsync_request( + identity, + requester, + validation_time, + priority, + rsync_scope_uri, + rsync_failure_scope_uri, + ) + } + + fn register_rsync_request( + &mut self, + identity: RepoIdentity, + requester: RepoRequester, + validation_time: time::OffsetDateTime, + priority: u8, + rsync_scope_uri: String, + rsync_failure_scope_uri: Option, + ) -> TransportRequestAction { + if let Some(entry) = self.rsync_inflight.get_mut(&rsync_scope_uri) { + if let Some(result) = entry.last_result.clone() { + return match result.result { + RepoTransportResultKind::Success { .. } => { + self.runtime_records.insert( + identity.clone(), + RepoRuntimeRecord { + identity, + state: RepoRuntimeState::RsyncOk, + rrdp_notification_key: None, + rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), + requesters: vec![requester], + validation_time, + priority, + last_success: Some(result.clone()), + terminal_failure: None, + }, + ); + TransportRequestAction::ReusedSuccess(result) + } + RepoTransportResultKind::Failed { .. } => { + self.runtime_records.insert( + identity.clone(), + RepoRuntimeRecord { + identity, + state: RepoRuntimeState::FailedTerminal, + rrdp_notification_key: None, + rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), + requesters: vec![requester], + validation_time, + priority, + last_success: None, + terminal_failure: Some(result.clone()), + }, + ); + TransportRequestAction::ReusedTerminalFailure(result) + } + }; + } + + entry.waiting_requesters.push(requester.clone()); + self.runtime_records.insert( + identity.clone(), + RepoRuntimeRecord { + identity, + state: RepoRuntimeState::WaitingRsync, + rrdp_notification_key: None, + rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), + requesters: vec![requester], + validation_time, + priority, + last_success: None, + terminal_failure: None, + }, + ); + return TransportRequestAction::Waiting { + state: RepoRuntimeState::WaitingRsync, + }; + } + + if let Some(failure_scope_uri) = rsync_failure_scope_uri.as_ref() { + if let Some(result) = self.rsync_failure_by_scope.get(failure_scope_uri).cloned() { + self.runtime_records.insert( + identity.clone(), + RepoRuntimeRecord { + identity, + state: RepoRuntimeState::FailedTerminal, + rrdp_notification_key: None, + rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), + requesters: vec![requester], + validation_time, + priority, + last_success: None, + terminal_failure: Some(result.clone()), + }, + ); + return TransportRequestAction::ReusedTerminalFailure(result); + } + + if !self + .rsync_failure_scope_reachable + .contains(failure_scope_uri) + { + if self + .rsync_failure_probe_inflight + .get(failure_scope_uri) + .is_some() + { + self.runtime_records.insert( + identity.clone(), + RepoRuntimeRecord { + identity, + state: RepoRuntimeState::WaitingRsync, + rrdp_notification_key: None, + rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), + requesters: vec![requester], + validation_time, + priority, + last_success: None, + terminal_failure: None, + }, + ); + return TransportRequestAction::Waiting { + state: RepoRuntimeState::WaitingRsync, + }; + } + } + } + + let task = RepoTransportTask { + dedup_key: RepoDedupKey::RsyncScope { + rsync_scope_uri: rsync_scope_uri.clone(), + }, + rsync_failure_scope_uri: rsync_failure_scope_uri.clone(), + repo_identity: identity.clone(), + mode: RepoTransportMode::Rsync, + retry_short_timeout: false, + tal_id: requester.tal_id.clone(), + rir_id: requester.rir_id.clone(), + validation_time, + priority, + requesters: vec![requester.clone()], + }; + self.rsync_inflight.insert( + rsync_scope_uri.clone(), + TransportInFlightEntry { + state: TransportTaskState::Pending, + task: task.clone(), + waiting_requesters: Vec::new(), + last_result: None, + started_at: None, + finished_at: None, + }, + ); + if let Some(failure_scope_uri) = rsync_failure_scope_uri.as_ref() { + if !self + .rsync_failure_scope_reachable + .contains(failure_scope_uri) + { + self.rsync_failure_probe_inflight + .insert(failure_scope_uri.clone(), rsync_scope_uri.clone()); + } + } + self.runtime_records.insert( + identity.clone(), + RepoRuntimeRecord { + identity, + state: RepoRuntimeState::WaitingRsync, + rrdp_notification_key: None, + rsync_scope_key: rsync_scope_uri, + rsync_failure_scope_key: rsync_failure_scope_uri.clone(), + requesters: vec![requester], + validation_time, + priority, + last_success: None, + terminal_failure: None, + }, + ); + TransportRequestAction::Enqueue(task) + } + + fn schedule_rsync_for_record( + record: &mut RepoRuntimeRecord, + rsync_inflight: &mut HashMap, + rsync_failure_by_scope: &HashMap, + rsync_failure_probe_inflight: &mut HashMap, + rsync_failure_scope_reachable: &HashSet, + dead_repo_blacklist: Option<&DeadRepoBlacklist>, + follow_up_tasks: &mut Vec, + ) { + let rsync_scope_uri = record.rsync_scope_key.clone(); + if let Some(entry) = rsync_inflight.get_mut(&rsync_scope_uri) { + if let Some(result) = entry.last_result.clone() { + match result.result { + RepoTransportResultKind::Success { .. } => { + record.state = RepoRuntimeState::RsyncOk; + record.last_success = Some(result); + } + RepoTransportResultKind::Failed { .. } => { + record.state = RepoRuntimeState::FailedTerminal; + record.terminal_failure = Some(result); + } + } + return; + } + entry.waiting_requesters.extend(record.requesters.clone()); + record.state = RepoRuntimeState::WaitingRsync; + return; + } + + if let Some(failure_scope_uri) = record.rsync_failure_scope_key.as_ref() { + if let Some(result) = rsync_failure_by_scope.get(failure_scope_uri).cloned() { + record.state = RepoRuntimeState::FailedTerminal; + record.terminal_failure = Some(result); + return; + } + if !rsync_failure_scope_reachable.contains(failure_scope_uri) + && rsync_failure_probe_inflight.contains_key(failure_scope_uri) + { + record.state = RepoRuntimeState::WaitingRsync; + return; + } + } + + // Dead-repo blacklist (#141): the rsync transport of this repo is + // persistently unreachable; terminate instantly instead of enqueueing + // another doomed fetch (rrdp was already attempted or skipped). + if let Some(blacklist) = dead_repo_blacklist { + if blacklist.is_blacklisted( + RepoTransportMode::Rsync, + record.identity.rsync_base_uri.as_str(), + ) { + crate::progress_log::emit( + "dead_repo_blacklist_skip_all", + serde_json::json!({ + "repo_key_notification_uri": record.identity.notification_uri, + "repo_key_rsync_base_uri": record.identity.rsync_base_uri, + "after_rrdp_failure": true, + }), + ); + let envelope = Self::dead_repo_terminal_envelope( + &record.identity, + &record.requesters, + &rsync_scope_uri, + record.rsync_failure_scope_key.clone(), + ); + record.state = RepoRuntimeState::FailedTerminal; + record.terminal_failure = Some(envelope); + return; + } + } + + let first_requester = record + .requesters + .first() + .expect("rsync record must keep at least one requester"); + let task = RepoTransportTask { + dedup_key: RepoDedupKey::RsyncScope { + rsync_scope_uri: rsync_scope_uri.clone(), + }, + rsync_failure_scope_uri: record.rsync_failure_scope_key.clone(), + repo_identity: record.identity.clone(), + mode: RepoTransportMode::Rsync, + retry_short_timeout: false, + tal_id: first_requester.tal_id.clone(), + rir_id: first_requester.rir_id.clone(), + validation_time: record.validation_time, + priority: record.priority, + requesters: record.requesters.clone(), + }; + rsync_inflight.insert( + rsync_scope_uri.clone(), + TransportInFlightEntry { + state: TransportTaskState::Pending, + task: task.clone(), + waiting_requesters: Vec::new(), + last_result: None, + started_at: None, + finished_at: None, + }, + ); + if let Some(failure_scope_uri) = record.rsync_failure_scope_key.as_ref() { + if !rsync_failure_scope_reachable.contains(failure_scope_uri) { + rsync_failure_probe_inflight.insert(failure_scope_uri.clone(), rsync_scope_uri); + } + } + record.state = RepoRuntimeState::WaitingRsync; + follow_up_tasks.push(task); + } + + pub fn mark_transport_running( + &mut self, + dedup_key: &RepoDedupKey, + started_at: time::OffsetDateTime, + ) -> Result<(), String> { + match dedup_key { + RepoDedupKey::RrdpNotify { notification_uri } => { + let entry = self + .rrdp_inflight + .get_mut(notification_uri) + .ok_or_else(|| format!("rrdp transport not found: {notification_uri}"))?; + entry.state = TransportTaskState::Running; + entry.started_at = Some(started_at); + } + RepoDedupKey::RsyncScope { rsync_scope_uri } => { + let entry = self + .rsync_inflight + .get_mut(rsync_scope_uri) + .ok_or_else(|| format!("rsync transport not found: {rsync_scope_uri}"))?; + entry.state = TransportTaskState::Running; + entry.started_at = Some(started_at); + } + } + Ok(()) + } + + pub fn complete_transport_result( + &mut self, + result: RepoTransportResultEnvelope, + finished_at: time::OffsetDateTime, + ) -> Result { + match (&result.dedup_key, &result.result) { + ( + RepoDedupKey::RrdpNotify { notification_uri }, + RepoTransportResultKind::Success { .. }, + ) => { + let entry = self + .rrdp_inflight + .get_mut(notification_uri) + .ok_or_else(|| format!("rrdp transport not found: {notification_uri}"))?; + entry.state = TransportTaskState::Finished; + entry.finished_at = Some(finished_at); + entry.last_result = Some(result.clone()); + let released_requesters = std::mem::take(&mut entry.waiting_requesters); + for record in self.runtime_records.values_mut() { + if record.rrdp_notification_key.as_deref() == Some(notification_uri) + && record.state == RepoRuntimeState::WaitingRrdp + { + record.state = RepoRuntimeState::RrdpOk; + record.last_success = Some(result.clone()); + } + } + Ok(TransportCompletion { + released_requesters, + follow_up_tasks: Vec::new(), + }) + } + ( + RepoDedupKey::RrdpNotify { notification_uri }, + RepoTransportResultKind::Failed { .. }, + ) => { + let entry = self + .rrdp_inflight + .get_mut(notification_uri) + .ok_or_else(|| format!("rrdp transport not found: {notification_uri}"))?; + entry.state = TransportTaskState::Finished; + entry.finished_at = Some(finished_at); + entry.last_result = Some(result.clone()); + let mut follow_up_tasks = Vec::new(); + for record in self.runtime_records.values_mut() { + if record.rrdp_notification_key.as_deref() == Some(notification_uri) + && record.state == RepoRuntimeState::WaitingRrdp + { + record.state = RepoRuntimeState::RrdpFailedPendingRsync; + Self::schedule_rsync_for_record( + record, + &mut self.rsync_inflight, + &self.rsync_failure_by_scope, + &mut self.rsync_failure_probe_inflight, + &self.rsync_failure_scope_reachable, + self.dead_repo_blacklist.as_ref(), + &mut follow_up_tasks, + ); + } + } + Ok(TransportCompletion { + released_requesters: Vec::new(), + follow_up_tasks, + }) + } + ( + RepoDedupKey::RsyncScope { rsync_scope_uri }, + RepoTransportResultKind::Success { .. }, + ) => { + let mut follow_up_tasks = Vec::new(); + let entry = self + .rsync_inflight + .get_mut(rsync_scope_uri) + .ok_or_else(|| format!("rsync transport not found: {rsync_scope_uri}"))?; + entry.state = TransportTaskState::Finished; + entry.finished_at = Some(finished_at); + entry.last_result = Some(result.clone()); + if let Some(failure_scope_uri) = result.rsync_failure_scope_uri.as_ref() { + self.rsync_failure_probe_inflight.remove(failure_scope_uri); + self.rsync_failure_scope_reachable + .insert(failure_scope_uri.clone()); + } + let released_requesters = std::mem::take(&mut entry.waiting_requesters); + for record in self.runtime_records.values_mut() { + if record.rsync_scope_key == *rsync_scope_uri + && matches!( + record.state, + RepoRuntimeState::WaitingRsync + | RepoRuntimeState::RrdpFailedPendingRsync + ) + { + record.state = RepoRuntimeState::RsyncOk; + record.last_success = Some(result.clone()); + } + } + if let Some(failure_scope_uri) = result.rsync_failure_scope_uri.as_ref() { + for record in self.runtime_records.values_mut() { + if record.rsync_scope_key != *rsync_scope_uri + && record.rsync_failure_scope_key.as_deref() + == Some(failure_scope_uri.as_str()) + && matches!(record.state, RepoRuntimeState::WaitingRsync) + { + Self::schedule_rsync_for_record( + record, + &mut self.rsync_inflight, + &self.rsync_failure_by_scope, + &mut self.rsync_failure_probe_inflight, + &self.rsync_failure_scope_reachable, + self.dead_repo_blacklist.as_ref(), + &mut follow_up_tasks, + ); + } + } + } + Ok(TransportCompletion { + released_requesters, + follow_up_tasks, + }) + } + ( + RepoDedupKey::RsyncScope { rsync_scope_uri }, + RepoTransportResultKind::Failed { .. }, + ) => { + let entry = self + .rsync_inflight + .get_mut(rsync_scope_uri) + .ok_or_else(|| format!("rsync transport not found: {rsync_scope_uri}"))?; + entry.state = TransportTaskState::Finished; + entry.finished_at = Some(finished_at); + entry.last_result = Some(result.clone()); + let reusable_failure_scope = + reusable_rsync_failure_scope(&result).map(str::to_string); + if let Some(failure_scope_uri) = result.rsync_failure_scope_uri.as_ref() { + self.rsync_failure_probe_inflight.remove(failure_scope_uri); + if reusable_failure_scope.as_deref() == Some(failure_scope_uri.as_str()) { + self.rsync_failure_by_scope + .insert(failure_scope_uri.clone(), result.clone()); + } else { + self.rsync_failure_scope_reachable + .insert(failure_scope_uri.clone()); + } + } + let released_requesters = std::mem::take(&mut entry.waiting_requesters); + for record in self.runtime_records.values_mut() { + if (record.rsync_scope_key == *rsync_scope_uri + || reusable_failure_scope + .as_deref() + .map(|scope| record.rsync_failure_scope_key.as_deref() == Some(scope)) + .unwrap_or(false)) + && matches!( + record.state, + RepoRuntimeState::WaitingRsync + | RepoRuntimeState::RrdpFailedPendingRsync + ) + { + record.state = RepoRuntimeState::FailedTerminal; + record.terminal_failure = Some(result.clone()); + } + } + if let Some(failure_scope_uri) = result.rsync_failure_scope_uri.as_ref() { + if reusable_failure_scope.as_deref() != Some(failure_scope_uri.as_str()) { + let mut follow_up_tasks = Vec::new(); + for record in self.runtime_records.values_mut() { + if record.rsync_scope_key != *rsync_scope_uri + && record.rsync_failure_scope_key.as_deref() + == Some(failure_scope_uri.as_str()) + && matches!(record.state, RepoRuntimeState::WaitingRsync) + { + Self::schedule_rsync_for_record( + record, + &mut self.rsync_inflight, + &self.rsync_failure_by_scope, + &mut self.rsync_failure_probe_inflight, + &self.rsync_failure_scope_reachable, + self.dead_repo_blacklist.as_ref(), + &mut follow_up_tasks, + ); + } + } + return Ok(TransportCompletion { + released_requesters, + follow_up_tasks, + }); + } + } + Ok(TransportCompletion { + released_requesters, + follow_up_tasks: Vec::new(), + }) + } + } + } +} + +fn reusable_rsync_failure_scope(result: &RepoTransportResultEnvelope) -> Option<&str> { + let scope = result.rsync_failure_scope_uri.as_deref()?; + match (&result.mode, &result.result) { + (RepoTransportMode::Rsync, RepoTransportResultKind::Failed { detail, .. }) + if is_host_level_rsync_failure(detail) => + { + Some(scope) + } + _ => None, + } +} + +pub(crate) fn is_host_level_rsync_failure(detail: &str) -> bool { + let lower = detail.to_ascii_lowercase(); + lower.contains("timeout waiting for daemon connection") + || lower.contains("failed to connect") + || lower.contains("no route to host") + || lower.contains("network is unreachable") + || lower.contains("connection refused") + || lower.contains("name or service not known") + || lower.contains("temporary failure in name resolution") +} + +#[derive(Default)] +pub struct InFlightRepoTable { + entries: HashMap, +} + +impl InFlightRepoTable { + pub fn new() -> Self { + Self::default() + } + + pub fn len(&self) -> usize { + self.entries.len() + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + pub fn reset_run_state(&mut self) { + self.entries.clear(); + } + + pub fn get(&self, key: &RepoKey) -> Option<&InFlightRepoEntry> { + self.entries.get(key) + } + + pub fn last_result(&self, key: &RepoKey) -> Option<&RepoSyncResultEnvelope> { + self.entries + .get(key) + .and_then(|entry| entry.last_result.as_ref()) + } + + pub fn register_request( + &mut self, + repo_key: RepoKey, + requester: RepoRequester, + validation_time: time::OffsetDateTime, + sync_preference: SyncPreference, + priority: u8, + ) -> RepoRequestAction { + match self.entries.get_mut(&repo_key) { + Some(entry) => match entry.state { + RepoTaskState::Pending | RepoTaskState::Running => { + entry.waiting_requesters.push(requester); + RepoRequestAction::Waiting + } + RepoTaskState::Succeeded | RepoTaskState::Reused => { + RepoRequestAction::Reused(entry.result_ref.clone().expect("result_ref exists")) + } + RepoTaskState::Failed => RepoRequestAction::FailedReuse { + detail: entry + .last_error + .clone() + .unwrap_or_else(|| "repo sync failed".to_string()), + }, + }, + None => { + let task = RepoSyncTask { + repo_key: repo_key.clone(), + validation_time, + sync_preference, + tal_id: requester.tal_id.clone(), + rir_id: requester.rir_id.clone(), + priority, + requesters: vec![requester], + }; + self.entries.insert( + repo_key, + InFlightRepoEntry { + state: RepoTaskState::Pending, + task_ref: Some(task.clone()), + waiting_requesters: Vec::new(), + result_ref: None, + last_result: None, + last_error: None, + started_at: None, + finished_at: None, + }, + ); + RepoRequestAction::Enqueued(task) + } + } + } + + pub fn mark_running( + &mut self, + repo_key: &RepoKey, + started_at: time::OffsetDateTime, + ) -> Result<(), String> { + let entry = self + .entries + .get_mut(repo_key) + .ok_or_else(|| format!("repo not found: {}", repo_key.rsync_base_uri))?; + match entry.state { + RepoTaskState::Pending => { + entry.state = RepoTaskState::Running; + entry.started_at = Some(started_at); + Ok(()) + } + other => Err(format!("repo cannot transition to running from {other:?}")), + } + } + + pub fn complete_success( + &mut self, + repo_key: &RepoKey, + result: RepoSyncResultEnvelope, + finished_at: time::OffsetDateTime, + ) -> Result { + let entry = self + .entries + .get_mut(repo_key) + .ok_or_else(|| format!("repo not found: {}", repo_key.rsync_base_uri))?; + match entry.state { + RepoTaskState::Pending | RepoTaskState::Running => { + let result_ref = match &result.result { + RepoSyncResultKind::Success(result_ref) + | RepoSyncResultKind::Reused(result_ref) => result_ref.clone(), + RepoSyncResultKind::Failed { detail } => { + return Err(format!( + "success completion called with failure result: {detail}" + )); + } + }; + entry.state = RepoTaskState::Succeeded; + entry.result_ref = Some(result_ref); + entry.last_result = Some(result); + entry.last_error = None; + entry.finished_at = Some(finished_at); + entry.task_ref = None; + let released_requesters = std::mem::take(&mut entry.waiting_requesters); + Ok(RepoCompletion { + repo_key: repo_key.clone(), + released_requesters, + }) + } + other => Err(format!("repo cannot transition to success from {other:?}")), + } + } + + pub fn complete_failure( + &mut self, + repo_key: &RepoKey, + result: RepoSyncResultEnvelope, + finished_at: time::OffsetDateTime, + ) -> Result { + let entry = self + .entries + .get_mut(repo_key) + .ok_or_else(|| format!("repo not found: {}", repo_key.rsync_base_uri))?; + match entry.state { + RepoTaskState::Pending | RepoTaskState::Running => { + let detail = match &result.result { + RepoSyncResultKind::Failed { detail } => detail.clone(), + RepoSyncResultKind::Success(_) | RepoSyncResultKind::Reused(_) => { + return Err("failure completion called with success result".to_string()); + } + }; + entry.state = RepoTaskState::Failed; + entry.result_ref = None; + entry.last_result = Some(result); + entry.last_error = Some(detail); + entry.finished_at = Some(finished_at); + entry.task_ref = None; + let released_requesters = std::mem::take(&mut entry.waiting_requesters); + Ok(RepoCompletion { + repo_key: repo_key.clone(), + released_requesters, + }) + } + other => Err(format!("repo cannot transition to failure from {other:?}")), + } + } +} + +#[cfg(test)] +mod tests { + use crate::parallel::repo_scheduler::{InFlightRepoTable, RepoRequestAction}; + use crate::parallel::types::{ + RepoKey, RepoRequester, RepoSyncResultEnvelope, RepoSyncResultKind, RepoSyncResultRef, + RepoTaskState, + }; + use crate::policy::SyncPreference; + + fn requester(tal_id: &str, rir_id: &str, manifest: &str) -> RepoRequester { + RepoRequester { + tal_id: tal_id.to_string(), + rir_id: rir_id.to_string(), + parent_node_id: None, + ca_instance_handle_id: format!("{tal_id}:{manifest}"), + publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), + manifest_rsync_uri: manifest.to_string(), + } + } + + #[test] + fn inflight_repo_table_stores_entries_by_repo_key() { + let key = RepoKey::new("rsync://example.test/repo/", None); + let mut table = InFlightRepoTable::new(); + let action = table.register_request( + key.clone(), + requester("arin", "arin", "rsync://example.test/repo/root.mft"), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + assert!(matches!(action, RepoRequestAction::Enqueued(_))); + assert_eq!(table.len(), 1); + assert_eq!( + table.get(&key).map(|entry| entry.state), + Some(RepoTaskState::Pending) + ); + } + + #[test] + fn running_repo_request_is_merged_into_waiting_list() { + let key = RepoKey::new("rsync://example.test/repo/", None); + let mut table = InFlightRepoTable::new(); + let _ = table.register_request( + key.clone(), + requester("arin", "arin", "rsync://example.test/repo/root.mft"), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + table + .mark_running(&key, time::OffsetDateTime::UNIX_EPOCH) + .expect("mark running"); + + let action = table.register_request( + key.clone(), + requester("apnic", "apnic", "rsync://example.test/repo/other.mft"), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + assert_eq!(action, RepoRequestAction::Waiting); + assert_eq!(table.get(&key).unwrap().waiting_requesters.len(), 1); + } + + #[test] + fn complete_success_releases_waiting_requesters_and_reuses_result() { + let key = RepoKey::new("rsync://example.test/repo/", None); + let mut table = InFlightRepoTable::new(); + let _ = table.register_request( + key.clone(), + requester("arin", "arin", "rsync://example.test/repo/root.mft"), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + table + .mark_running(&key, time::OffsetDateTime::UNIX_EPOCH) + .expect("mark running"); + let waiting = requester("ripe", "ripe", "rsync://example.test/repo/child.mft"); + let _ = table.register_request( + key.clone(), + waiting.clone(), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + + let result_ref = RepoSyncResultRef { + repo_key: key.clone(), + source: "rrdp".to_string(), + }; + let completion = table + .complete_success( + &key, + RepoSyncResultEnvelope { + repo_key: key.clone(), + tal_id: "arin".to_string(), + rir_id: "arin".to_string(), + result: RepoSyncResultKind::Success(result_ref.clone()), + phase: Some("rrdp_ok".to_string()), + timing_ms: 10, + warnings: Vec::new(), + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete success"); + assert_eq!(completion.released_requesters, vec![waiting]); + assert_eq!(table.get(&key).unwrap().state, RepoTaskState::Succeeded); + + let action = table.register_request( + key, + requester("afrinic", "afrinic", "rsync://example.test/repo/again.mft"), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + assert_eq!(action, RepoRequestAction::Reused(result_ref)); + } + + #[test] + fn complete_failure_releases_waiting_requesters_and_reuses_failure() { + let key = RepoKey::new("rsync://example.test/repo/", None); + let mut table = InFlightRepoTable::new(); + let _ = table.register_request( + key.clone(), + requester("arin", "arin", "rsync://example.test/repo/root.mft"), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + table + .mark_running(&key, time::OffsetDateTime::UNIX_EPOCH) + .expect("mark running"); + let waiting = requester("ripe", "ripe", "rsync://example.test/repo/child.mft"); + let _ = table.register_request( + key.clone(), + waiting.clone(), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + + let completion = table + .complete_failure( + &key, + RepoSyncResultEnvelope { + repo_key: key.clone(), + tal_id: "arin".to_string(), + rir_id: "arin".to_string(), + result: RepoSyncResultKind::Failed { + detail: "network timeout".to_string(), + }, + phase: Some("rrdp_failed_rsync_failed".to_string()), + timing_ms: 10, + warnings: Vec::new(), + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete failure"); + assert_eq!(completion.released_requesters, vec![waiting]); + assert_eq!(table.get(&key).unwrap().state, RepoTaskState::Failed); + + let action = table.register_request( + key, + requester("afrinic", "afrinic", "rsync://example.test/repo/again.mft"), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + assert_eq!( + action, + RepoRequestAction::FailedReuse { + detail: "network timeout".to_string() + } + ); + } + + #[test] + fn requesters_from_different_tals_do_not_lose_identity() { + let key = RepoKey::new("rsync://shared.example/repo/", None); + let mut table = InFlightRepoTable::new(); + let _ = table.register_request( + key.clone(), + requester("arin", "arin", "rsync://shared.example/repo/a.mft"), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + table + .mark_running(&key, time::OffsetDateTime::UNIX_EPOCH) + .expect("mark running"); + let wait_apnic = requester("apnic", "apnic", "rsync://shared.example/repo/b.mft"); + let wait_ripe = requester("ripe", "ripe", "rsync://shared.example/repo/c.mft"); + let _ = table.register_request( + key.clone(), + wait_apnic.clone(), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + let _ = table.register_request( + key.clone(), + wait_ripe.clone(), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + let completion = table + .complete_success( + &key, + RepoSyncResultEnvelope { + repo_key: key.clone(), + tal_id: "arin".to_string(), + rir_id: "arin".to_string(), + result: RepoSyncResultKind::Success(RepoSyncResultRef { + repo_key: key.clone(), + source: "rrdp".to_string(), + }), + phase: Some("rrdp_ok".to_string()), + timing_ms: 10, + warnings: Vec::new(), + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete success"); + assert_eq!(completion.released_requesters, vec![wait_apnic, wait_ripe]); + } +} + +#[cfg(test)] +mod transport_tests { + use crate::parallel::repo_scheduler::{ + TransportRequestAction, TransportStateTables, TransportTaskState, + }; + use crate::parallel::types::{ + RepoDedupKey, RepoIdentity, RepoRequester, RepoRuntimeState, RepoTransportMode, + RepoTransportResultEnvelope, RepoTransportResultKind, + }; + use crate::policy::SyncPreference; + + fn requester(id: &str) -> RepoRequester { + RepoRequester::with_tal_rir( + "apnic", + "apnic", + format!("rsync://example.test/repo/{id}.mft"), + "rsync://example.test/repo/".to_string(), + format!("node:{id}"), + ) + } + + #[test] + fn register_transport_request_enqueues_initial_rrdp_task() { + let mut tables = TransportStateTables::new(); + let identity = RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/", + ); + let action = tables.register_transport_request( + identity.clone(), + requester("root"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RrdpThenRsync, + ); + let task = match action { + TransportRequestAction::Enqueue(task) => task, + other => panic!("expected enqueue, got {other:?}"), + }; + assert_eq!(task.mode, RepoTransportMode::Rrdp); + assert_eq!( + task.dedup_key, + RepoDedupKey::RrdpNotify { + notification_uri: "https://example.test/notify.xml".to_string() + } + ); + let key = "https://example.test/notify.xml".to_string(); + assert_eq!( + tables.rrdp_inflight.get(&key).unwrap().state, + TransportTaskState::Pending + ); + assert_eq!( + tables.runtime_records.get(&identity).unwrap().state, + RepoRuntimeState::WaitingRrdp + ); + } + + #[test] + fn register_transport_request_waits_on_existing_rrdp_task() { + let mut tables = TransportStateTables::new(); + let identity = RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/a/", + ); + let _ = tables.register_transport_request( + identity.clone(), + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RrdpThenRsync, + ); + let other_identity = RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/b/", + ); + let action = tables.register_transport_request( + other_identity.clone(), + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RrdpThenRsync, + ); + assert_eq!( + action, + TransportRequestAction::Waiting { + state: RepoRuntimeState::WaitingRrdp + } + ); + assert_eq!( + tables.runtime_records.get(&other_identity).unwrap().state, + RepoRuntimeState::WaitingRrdp + ); + assert_eq!( + tables + .rrdp_inflight + .get("https://example.test/notify.xml") + .unwrap() + .waiting_requesters + .len(), + 1 + ); + } + + #[test] + fn complete_rrdp_success_reuses_for_later_identity_requests() { + let mut tables = TransportStateTables::new(); + let identity = RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/a/", + ); + let _ = tables.register_transport_request( + identity.clone(), + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RrdpThenRsync, + ); + tables + .mark_transport_running( + &RepoDedupKey::RrdpNotify { + notification_uri: "https://example.test/notify.xml".to_string(), + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("mark running"); + let completion = tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: RepoDedupKey::RrdpNotify { + notification_uri: "https://example.test/notify.xml".to_string(), + }, + rsync_failure_scope_uri: None, + repo_identity: identity.clone(), + mode: RepoTransportMode::Rrdp, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 10, + result: RepoTransportResultKind::Success { + source: "rrdp".to_string(), + warnings: Vec::new(), + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete success"); + assert!(completion.follow_up_tasks.is_empty()); + let later_identity = RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/b/", + ); + let action = tables.register_transport_request( + later_identity.clone(), + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RrdpThenRsync, + ); + assert!(matches!(action, TransportRequestAction::ReusedSuccess(_))); + assert_eq!( + tables.runtime_records.get(&later_identity).unwrap().state, + RepoRuntimeState::RrdpOk + ); + + let same_identity_action = tables.register_transport_request( + later_identity.clone(), + requester("b-again"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RrdpThenRsync, + ); + assert!(matches!( + same_identity_action, + TransportRequestAction::ReusedSuccess(_) + )); + + let finalized = tables.finalized_runtime_records_for_transport(&RepoDedupKey::RrdpNotify { + notification_uri: "https://example.test/notify.xml".to_string(), + }); + assert_eq!(finalized.len(), 2); + } + + #[test] + fn complete_rrdp_failure_enqueues_rsync_follow_up() { + let mut tables = TransportStateTables::new(); + let identity = RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/a/", + ); + let _ = tables.register_transport_request( + identity.clone(), + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RrdpThenRsync, + ); + tables + .mark_transport_running( + &RepoDedupKey::RrdpNotify { + notification_uri: "https://example.test/notify.xml".to_string(), + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("mark running"); + let completion = tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: RepoDedupKey::RrdpNotify { + notification_uri: "https://example.test/notify.xml".to_string(), + }, + rsync_failure_scope_uri: None, + repo_identity: identity.clone(), + mode: RepoTransportMode::Rrdp, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 10, + result: RepoTransportResultKind::Failed { + detail: "rrdp timeout".to_string(), + warnings: Vec::new(), + error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete failure"); + assert_eq!(completion.follow_up_tasks.len(), 1); + assert_eq!(completion.follow_up_tasks[0].mode, RepoTransportMode::Rsync); + assert_eq!( + tables.runtime_records.get(&identity).unwrap().state, + RepoRuntimeState::WaitingRsync + ); + + let same_identity_action = tables.register_transport_request( + identity.clone(), + requester("a-again"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RrdpThenRsync, + ); + assert_eq!( + same_identity_action, + TransportRequestAction::Waiting { + state: RepoRuntimeState::WaitingRsync + } + ); + + let later_identity = RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/b/", + ); + let later_action = tables.register_transport_request( + later_identity.clone(), + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RrdpThenRsync, + ); + assert_eq!( + later_action, + TransportRequestAction::Waiting { + state: RepoRuntimeState::WaitingRsync + } + ); + assert_eq!( + tables.runtime_records.get(&later_identity).unwrap().state, + RepoRuntimeState::WaitingRsync + ); + } + + #[test] + fn host_failure_scope_probes_once_then_reuses_terminal_failure() { + let mut tables = TransportStateTables::new(); + let identity_a = RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/a/", + ); + let identity_b = RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/b/", + ); + let _ = tables.register_transport_request( + identity_a.clone(), + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/a/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RrdpThenRsync, + ); + let _ = tables.register_transport_request( + identity_b.clone(), + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/b/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RrdpThenRsync, + ); + tables + .mark_transport_running( + &RepoDedupKey::RrdpNotify { + notification_uri: "https://example.test/notify.xml".to_string(), + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("mark running"); + let completion = tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: RepoDedupKey::RrdpNotify { + notification_uri: "https://example.test/notify.xml".to_string(), + }, + rsync_failure_scope_uri: None, + repo_identity: identity_a.clone(), + mode: RepoTransportMode::Rrdp, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 10, + result: RepoTransportResultKind::Failed { + detail: "rrdp timeout".to_string(), + warnings: Vec::new(), + error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete rrdp failure"); + assert_eq!(completion.follow_up_tasks.len(), 1); + assert_eq!( + completion.follow_up_tasks[0] + .rsync_failure_scope_uri + .as_deref(), + Some("rsync://example.test/") + ); + let probe_task = completion.follow_up_tasks[0].clone(); + + tables + .mark_transport_running(&probe_task.dedup_key, time::OffsetDateTime::UNIX_EPOCH) + .expect("mark rsync running"); + let completion = tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: probe_task.dedup_key, + rsync_failure_scope_uri: probe_task.rsync_failure_scope_uri, + repo_identity: probe_task.repo_identity, + mode: RepoTransportMode::Rsync, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 15_000, + result: RepoTransportResultKind::Failed { + detail: "rsync error: timeout waiting for daemon connection".to_string(), + warnings: Vec::new(), + error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete rsync failure"); + assert!(completion.follow_up_tasks.is_empty()); + assert_eq!( + tables.runtime_records.get(&identity_a).unwrap().state, + RepoRuntimeState::FailedTerminal + ); + assert_eq!( + tables.runtime_records.get(&identity_b).unwrap().state, + RepoRuntimeState::FailedTerminal + ); + let finalized = tables.finalized_runtime_records_for_transport_result( + tables + .runtime_records + .get(&identity_a) + .unwrap() + .terminal_failure + .as_ref() + .unwrap(), + ); + assert_eq!(finalized.len(), 2); + } + + #[test] + fn complete_rsync_failure_reuses_terminal_failure() { + let mut tables = TransportStateTables::new(); + let identity = RepoIdentity::new(None, "rsync://example.test/repo/a/"); + let _ = tables.register_transport_request( + identity.clone(), + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RsyncOnly, + ); + tables + .mark_transport_running( + &RepoDedupKey::RsyncScope { + rsync_scope_uri: "rsync://example.test/module/".to_string(), + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("mark running"); + let _ = tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: RepoDedupKey::RsyncScope { + rsync_scope_uri: "rsync://example.test/module/".to_string(), + }, + rsync_failure_scope_uri: None, + repo_identity: identity.clone(), + mode: RepoTransportMode::Rsync, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 20, + result: RepoTransportResultKind::Failed { + detail: "rsync timeout".to_string(), + warnings: Vec::new(), + error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete rsync failure"); + let later_identity = RepoIdentity::new(None, "rsync://example.test/repo/b/"); + let action = tables.register_transport_request( + later_identity, + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RsyncOnly, + ); + assert!(matches!( + action, + TransportRequestAction::ReusedTerminalFailure(_) + )); + + let finalized = tables.finalized_runtime_records_for_transport(&RepoDedupKey::RsyncScope { + rsync_scope_uri: "rsync://example.test/module/".to_string(), + }); + assert_eq!(finalized.len(), 2); + + let same_identity_action = tables.register_transport_request( + identity, + requester("a-again"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RsyncOnly, + ); + assert!(matches!( + same_identity_action, + TransportRequestAction::ReusedTerminalFailure(_) + )); + } + + #[test] + fn complete_rsync_success_reuses_for_later_identity_requests() { + let mut tables = TransportStateTables::new(); + let identity = RepoIdentity::new(None, "rsync://example.test/repo/a/"); + let action = tables.register_transport_request( + identity.clone(), + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RsyncOnly, + ); + assert!(matches!(action, TransportRequestAction::Enqueue(_))); + tables + .mark_transport_running( + &RepoDedupKey::RsyncScope { + rsync_scope_uri: "rsync://example.test/module/".to_string(), + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("mark running"); + tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: RepoDedupKey::RsyncScope { + rsync_scope_uri: "rsync://example.test/module/".to_string(), + }, + rsync_failure_scope_uri: None, + repo_identity: identity.clone(), + mode: RepoTransportMode::Rsync, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 20, + result: RepoTransportResultKind::Success { + source: "rsync".to_string(), + warnings: Vec::new(), + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete rsync success"); + + let later_identity = RepoIdentity::new(None, "rsync://example.test/repo/b/"); + let action = tables.register_transport_request( + later_identity.clone(), + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RsyncOnly, + ); + assert!(matches!(action, TransportRequestAction::ReusedSuccess(_))); + assert_eq!( + tables.runtime_records.get(&later_identity).unwrap().state, + RepoRuntimeState::RsyncOk + ); + } + + #[test] + fn register_rsync_request_waits_on_existing_rsync_task() { + let mut tables = TransportStateTables::new(); + let identity_a = RepoIdentity::new(None, "rsync://example.test/repo/a/"); + let identity_b = RepoIdentity::new(None, "rsync://example.test/repo/b/"); + let _ = tables.register_transport_request( + identity_a, + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RsyncOnly, + ); + let action = tables.register_transport_request( + identity_b.clone(), + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RsyncOnly, + ); + assert_eq!( + action, + TransportRequestAction::Waiting { + state: RepoRuntimeState::WaitingRsync + } + ); + assert_eq!( + tables.runtime_records.get(&identity_b).unwrap().state, + RepoRuntimeState::WaitingRsync + ); + assert_eq!( + tables + .rsync_inflight + .get("rsync://example.test/module/") + .unwrap() + .waiting_requesters + .len(), + 1 + ); + } + + #[test] + fn host_failure_scope_success_marks_host_reachable_and_schedules_waiters() { + let mut tables = TransportStateTables::new(); + let identity_a = RepoIdentity::new(None, "rsync://example.test/repo/a/"); + let identity_b = RepoIdentity::new(None, "rsync://example.test/repo/b/"); + let action_a = tables.register_transport_request( + identity_a.clone(), + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/a/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RsyncOnly, + ); + let probe_task = match action_a { + TransportRequestAction::Enqueue(task) => task, + other => panic!("expected first host probe enqueue, got {other:?}"), + }; + let action_b = tables.register_transport_request( + identity_b.clone(), + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/b/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RsyncOnly, + ); + assert_eq!( + action_b, + TransportRequestAction::Waiting { + state: RepoRuntimeState::WaitingRsync + } + ); + + tables + .mark_transport_running(&probe_task.dedup_key, time::OffsetDateTime::UNIX_EPOCH) + .expect("mark rsync running"); + let completion = tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: probe_task.dedup_key, + rsync_failure_scope_uri: probe_task.rsync_failure_scope_uri, + repo_identity: probe_task.repo_identity, + mode: RepoTransportMode::Rsync, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 5, + result: RepoTransportResultKind::Success { + source: "rsync".to_string(), + warnings: Vec::new(), + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete rsync success"); + + assert_eq!( + tables.runtime_records.get(&identity_a).unwrap().state, + RepoRuntimeState::RsyncOk + ); + assert_eq!(completion.follow_up_tasks.len(), 1); + assert_eq!( + completion.follow_up_tasks[0].dedup_key, + RepoDedupKey::RsyncScope { + rsync_scope_uri: "rsync://example.test/repo/b/".to_string() + } + ); + assert_eq!( + tables.runtime_records.get(&identity_b).unwrap().state, + RepoRuntimeState::WaitingRsync + ); + } + + #[test] + fn non_host_level_rsync_failure_does_not_poison_host_scope() { + let mut tables = TransportStateTables::new(); + let identity_a = RepoIdentity::new(None, "rsync://example.test/repo/a/"); + let identity_b = RepoIdentity::new(None, "rsync://example.test/repo/b/"); + let action_a = tables.register_transport_request( + identity_a.clone(), + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/a/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RsyncOnly, + ); + let probe_task = match action_a { + TransportRequestAction::Enqueue(task) => task, + other => panic!("expected first host probe enqueue, got {other:?}"), + }; + let _ = tables.register_transport_request( + identity_b.clone(), + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/b/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RsyncOnly, + ); + + tables + .mark_transport_running(&probe_task.dedup_key, time::OffsetDateTime::UNIX_EPOCH) + .expect("mark rsync running"); + let completion = tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: probe_task.dedup_key, + rsync_failure_scope_uri: probe_task.rsync_failure_scope_uri, + repo_identity: probe_task.repo_identity, + mode: RepoTransportMode::Rsync, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 5, + result: RepoTransportResultKind::Failed { + detail: "rsync file digest mismatch after download".to_string(), + warnings: Vec::new(), + error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete rsync failure"); + + assert_eq!( + tables.runtime_records.get(&identity_a).unwrap().state, + RepoRuntimeState::FailedTerminal + ); + assert_eq!(completion.follow_up_tasks.len(), 1); + assert_eq!( + completion.follow_up_tasks[0].dedup_key, + RepoDedupKey::RsyncScope { + rsync_scope_uri: "rsync://example.test/repo/b/".to_string() + } + ); + } + + #[test] + fn cached_host_level_failure_reuses_for_later_rsync_only_requests() { + let mut tables = TransportStateTables::new(); + let identity_a = RepoIdentity::new(None, "rsync://example.test/repo/a/"); + let action_a = tables.register_transport_request( + identity_a, + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/a/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RsyncOnly, + ); + let probe_task = match action_a { + TransportRequestAction::Enqueue(task) => task, + other => panic!("expected first host probe enqueue, got {other:?}"), + }; + tables + .mark_transport_running(&probe_task.dedup_key, time::OffsetDateTime::UNIX_EPOCH) + .expect("mark rsync running"); + let _ = tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: probe_task.dedup_key, + rsync_failure_scope_uri: probe_task.rsync_failure_scope_uri, + repo_identity: probe_task.repo_identity, + mode: RepoTransportMode::Rsync, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 15_000, + result: RepoTransportResultKind::Failed { + detail: "rsync error: failed to connect to daemon".to_string(), + warnings: Vec::new(), + error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete host failure"); + + let later_identity = RepoIdentity::new(None, "rsync://example.test/repo/c/"); + let action = tables.register_transport_request( + later_identity.clone(), + requester("c"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/c/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RsyncOnly, + ); + assert!(matches!( + action, + TransportRequestAction::ReusedTerminalFailure(_) + )); + assert_eq!( + tables.runtime_records.get(&later_identity).unwrap().state, + RepoRuntimeState::FailedTerminal + ); + } + + #[test] + fn reset_run_state_clears_host_failure_scope_cache() { + let mut tables = TransportStateTables::new(); + let identity = RepoIdentity::new(None, "rsync://example.test/repo/a/"); + let action = tables.register_transport_request( + identity, + requester("a"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/a/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RsyncOnly, + ); + let probe_task = match action { + TransportRequestAction::Enqueue(task) => task, + other => panic!("expected host probe enqueue, got {other:?}"), + }; + tables + .mark_transport_running(&probe_task.dedup_key, time::OffsetDateTime::UNIX_EPOCH) + .expect("mark rsync running"); + let _ = tables + .complete_transport_result( + RepoTransportResultEnvelope { + dedup_key: probe_task.dedup_key, + rsync_failure_scope_uri: probe_task.rsync_failure_scope_uri, + repo_identity: probe_task.repo_identity, + mode: RepoTransportMode::Rsync, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 15_000, + result: RepoTransportResultKind::Failed { + detail: "temporary failure in name resolution".to_string(), + warnings: Vec::new(), + error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, + }, + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete host failure"); + + tables.reset_run_state(); + let later_identity = RepoIdentity::new(None, "rsync://example.test/repo/b/"); + let action = tables.register_transport_request( + later_identity, + requester("b"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/b/".to_string(), + Some("rsync://example.test/".to_string()), + SyncPreference::RsyncOnly, + ); + assert!(matches!(action, TransportRequestAction::Enqueue(_))); + } + + #[test] + fn register_transport_request_skips_rrdp_when_sync_preference_is_rsync_only() { + let mut tables = TransportStateTables::new(); + let identity = RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/", + ); + let action = tables.register_transport_request( + identity.clone(), + requester("root"), + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/module/".to_string(), + None, + SyncPreference::RsyncOnly, + ); + let task = match action { + TransportRequestAction::Enqueue(task) => task, + other => panic!("expected enqueue, got {other:?}"), + }; + assert_eq!(task.mode, RepoTransportMode::Rsync); + assert_eq!( + task.dedup_key, + RepoDedupKey::RsyncScope { + rsync_scope_uri: "rsync://example.test/module/".to_string() + } + ); + assert!(tables.rrdp_inflight.is_empty()); + assert_eq!( + tables.runtime_records.get(&identity).unwrap().state, + RepoRuntimeState::WaitingRsync + ); + } +} diff --git a/crates/panda-rpki-validator/src/parallel/repo_worker.rs b/crates/panda-rpki-validator/src/parallel/repo_worker.rs new file mode 100644 index 0000000..ec226ae --- /dev/null +++ b/crates/panda-rpki-validator/src/parallel/repo_worker.rs @@ -0,0 +1,1270 @@ +use std::sync::mpsc::{self, Receiver, RecvTimeoutError, Sender}; +use std::sync::{Arc, Mutex}; +use std::thread::{self, JoinHandle}; +use std::time::Duration; + +use super::config::ParallelPhase1Config; +use super::types::{ + RepoSyncResultEnvelope, RepoSyncTask, RepoTransportMode, RepoTransportResultEnvelope, + RepoTransportResultKind, RepoTransportTask, +}; +use crate::analysis::timing::TimingHandle; +use crate::audit_downloads::DownloadLogHandle; +use crate::current_repo_index::CurrentRepoIndexHandle; +use crate::fetch::rsync::RsyncFetcher; +use crate::fetch::rsync_system::{ + RsyncFailFastProfile, with_scoped_rsync_fail_fast_profile, with_scoped_rsync_timeout_override, +}; +use crate::policy::Policy; +use crate::storage::RocksStore; +use crate::sync::repo::{ + RepoSyncSource, run_rrdp_transport, run_rsync_transport, sync_publication_point, +}; +use crate::sync::rrdp::Fetcher; + +const RETRY_SHORT_TIMEOUT: Duration = Duration::from_secs(1); + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RepoWorkerPoolConfig { + pub max_workers: usize, +} + +impl From<&ParallelPhase1Config> for RepoWorkerPoolConfig { + fn from(value: &ParallelPhase1Config) -> Self { + Self { + max_workers: value.max_repo_sync_workers_global, + } + } +} + +pub trait RepoSyncExecutor: Send + Sync + 'static { + fn execute(&self, task: RepoSyncTask) -> RepoSyncResultEnvelope; +} + +pub trait RepoTransportExecutor: Send + Sync + 'static { + fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope; +} + +pub struct LiveRrdpTransportExecutor { + store: Arc, + current_repo_index: CurrentRepoIndexHandle, + http_fetcher: Arc, + timing: Option, + download_log: Option, +} + +impl LiveRrdpTransportExecutor { + pub fn new( + store: Arc, + current_repo_index: CurrentRepoIndexHandle, + http_fetcher: Arc, + timing: Option, + download_log: Option, + ) -> Self { + Self { + store, + current_repo_index, + http_fetcher, + timing, + download_log, + } + } +} + +impl RepoTransportExecutor for LiveRrdpTransportExecutor { + fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { + let started = std::time::Instant::now(); + debug_assert_eq!(task.mode, RepoTransportMode::Rrdp); + let notification_uri = task + .repo_identity + .notification_uri + .as_deref() + .expect("rrdp transport requires notification uri"); + let sync_result = if task.retry_short_timeout { + crate::fetch::http::with_scoped_http_timeout_override(RETRY_SHORT_TIMEOUT, || { + run_rrdp_transport( + self.store.as_ref(), + notification_uri, + Some(&self.current_repo_index), + self.http_fetcher.as_ref(), + self.timing.as_ref(), + self.download_log.as_ref(), + ) + }) + } else { + run_rrdp_transport( + self.store.as_ref(), + notification_uri, + Some(&self.current_repo_index), + self.http_fetcher.as_ref(), + self.timing.as_ref(), + self.download_log.as_ref(), + ) + }; + match sync_result { + Ok(_) => RepoTransportResultEnvelope { + dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri, + repo_identity: task.repo_identity, + mode: RepoTransportMode::Rrdp, + tal_id: task.tal_id, + rir_id: task.rir_id, + timing_ms: started.elapsed().as_millis() as u64, + result: RepoTransportResultKind::Success { + source: "rrdp".to_string(), + warnings: Vec::new(), + }, + }, + Err(err) => { + let error_class = + crate::parallel::dead_repo_blacklist::classify_rrdp_sync_error(&err); + RepoTransportResultEnvelope { + dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri, + repo_identity: task.repo_identity, + mode: RepoTransportMode::Rrdp, + tal_id: task.tal_id, + rir_id: task.rir_id, + timing_ms: started.elapsed().as_millis() as u64, + result: RepoTransportResultKind::Failed { + detail: err.to_string(), + warnings: Vec::new(), + error_class, + }, + } + } + } + } +} + +pub struct LiveRsyncTransportExecutor { + store: Arc, + current_repo_index: CurrentRepoIndexHandle, + rsync_fetcher: Arc, + timing: Option, + download_log: Option, +} + +impl LiveRsyncTransportExecutor { + pub fn new( + store: Arc, + current_repo_index: CurrentRepoIndexHandle, + rsync_fetcher: Arc, + timing: Option, + download_log: Option, + ) -> Self { + Self { + store, + current_repo_index, + rsync_fetcher, + timing, + download_log, + } + } +} + +impl RepoTransportExecutor for LiveRsyncTransportExecutor { + fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { + let started = std::time::Instant::now(); + debug_assert_eq!(task.mode, RepoTransportMode::Rsync); + let sync_result = if task.retry_short_timeout { + with_scoped_rsync_timeout_override(RETRY_SHORT_TIMEOUT, || { + with_scoped_rsync_fail_fast_profile( + RsyncFailFastProfile { + initial_wall_clock_timeout: RETRY_SHORT_TIMEOUT, + max_wall_clock_timeout: RETRY_SHORT_TIMEOUT, + max_attempts: 1, + }, + || { + run_rsync_transport( + self.store.as_ref(), + &task.repo_identity.rsync_base_uri, + Some(&self.current_repo_index), + self.rsync_fetcher.as_ref(), + self.timing.as_ref(), + self.download_log.as_ref(), + ) + }, + ) + }) + } else { + run_rsync_transport( + self.store.as_ref(), + &task.repo_identity.rsync_base_uri, + Some(&self.current_repo_index), + self.rsync_fetcher.as_ref(), + self.timing.as_ref(), + self.download_log.as_ref(), + ) + }; + match sync_result { + Ok(_) => RepoTransportResultEnvelope { + dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri, + repo_identity: task.repo_identity, + mode: RepoTransportMode::Rsync, + tal_id: task.tal_id, + rir_id: task.rir_id, + timing_ms: started.elapsed().as_millis() as u64, + result: RepoTransportResultKind::Success { + source: "rsync".to_string(), + warnings: Vec::new(), + }, + }, + Err(err) => { + let error_class = + crate::parallel::dead_repo_blacklist::classify_rsync_repo_sync_error(&err); + RepoTransportResultEnvelope { + dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri, + repo_identity: task.repo_identity, + mode: RepoTransportMode::Rsync, + tal_id: task.tal_id, + rir_id: task.rir_id, + timing_ms: started.elapsed().as_millis() as u64, + result: RepoTransportResultKind::Failed { + detail: err.to_string(), + warnings: Vec::new(), + error_class, + }, + } + } + } + } +} + +pub struct LiveRepoTransportExecutor { + rrdp: LiveRrdpTransportExecutor, + rsync: LiveRsyncTransportExecutor, +} + +impl LiveRepoTransportExecutor { + pub fn new( + store: Arc, + current_repo_index: CurrentRepoIndexHandle, + http_fetcher: Arc, + rsync_fetcher: Arc, + timing: Option, + download_log: Option, + ) -> Self { + Self { + rrdp: LiveRrdpTransportExecutor::new( + Arc::clone(&store), + current_repo_index.clone(), + http_fetcher, + timing.clone(), + download_log.clone(), + ), + rsync: LiveRsyncTransportExecutor::new( + store, + current_repo_index, + rsync_fetcher, + timing, + download_log, + ), + } + } +} + +impl RepoTransportExecutor + for LiveRepoTransportExecutor +{ + fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { + match task.mode { + RepoTransportMode::Rrdp => self.rrdp.execute_transport(task), + RepoTransportMode::Rsync => self.rsync.execute_transport(task), + } + } +} + +pub struct LiveRepoSyncExecutor { + store: Arc, + policy: Policy, + http_fetcher: Arc, + rsync_fetcher: Arc, + timing: Option, + download_log: Option, +} + +impl LiveRepoSyncExecutor { + pub fn new( + store: Arc, + policy: Policy, + http_fetcher: Arc, + rsync_fetcher: Arc, + timing: Option, + download_log: Option, + ) -> Self { + Self { + store, + policy, + http_fetcher, + rsync_fetcher, + timing, + download_log, + } + } +} + +impl RepoSyncExecutor + for LiveRepoSyncExecutor +{ + fn execute(&self, task: RepoSyncTask) -> RepoSyncResultEnvelope { + let started = std::time::Instant::now(); + crate::progress_log::emit( + "phase1_repo_worker_execute_start", + serde_json::json!({ + "repo_key_rsync_base_uri": task.repo_key.rsync_base_uri, + "repo_key_notification_uri": task.repo_key.notification_uri, + "tal_id": task.tal_id, + "rir_id": task.rir_id, + }), + ); + match sync_publication_point( + self.store.as_ref(), + &self.policy, + task.repo_key.notification_uri.as_deref(), + &task.repo_key.rsync_base_uri, + self.http_fetcher.as_ref(), + self.rsync_fetcher.as_ref(), + self.timing.as_ref(), + self.download_log.as_ref(), + ) { + Ok(res) => { + let timing_ms = started.elapsed().as_millis() as u64; + crate::progress_log::emit( + "phase1_repo_worker_execute_finish", + serde_json::json!({ + "repo_key_rsync_base_uri": task.repo_key.rsync_base_uri, + "repo_key_notification_uri": task.repo_key.notification_uri, + "result": "success", + "phase": repo_sync_phase_label(res.phase), + "timing_ms": timing_ms, + }), + ); + RepoSyncResultEnvelope { + repo_key: task.repo_key.clone(), + tal_id: task.tal_id, + rir_id: task.rir_id, + result: super::types::RepoSyncResultKind::Success( + super::types::RepoSyncResultRef { + repo_key: task.repo_key, + source: repo_sync_source_label(res.source).to_string(), + }, + ), + phase: Some(repo_sync_phase_label(res.phase).to_string()), + timing_ms, + warnings: res.warnings, + } + } + Err(err) => { + let timing_ms = started.elapsed().as_millis() as u64; + crate::progress_log::emit( + "phase1_repo_worker_execute_finish", + serde_json::json!({ + "repo_key_rsync_base_uri": task.repo_key.rsync_base_uri, + "repo_key_notification_uri": task.repo_key.notification_uri, + "result": "failed", + "phase": "repo_sync_failed", + "timing_ms": timing_ms, + "error": err.to_string(), + }), + ); + RepoSyncResultEnvelope { + repo_key: task.repo_key, + tal_id: task.tal_id, + rir_id: task.rir_id, + result: super::types::RepoSyncResultKind::Failed { + detail: err.to_string(), + }, + phase: Some("repo_sync_failed".to_string()), + timing_ms, + warnings: Vec::new(), + } + } + } + } +} + +fn repo_sync_source_label(source: RepoSyncSource) -> &'static str { + match source { + RepoSyncSource::Rrdp => "rrdp", + RepoSyncSource::Rsync => "rsync", + } +} + +fn repo_sync_phase_label(phase: crate::sync::repo::RepoSyncPhase) -> &'static str { + match phase { + crate::sync::repo::RepoSyncPhase::RrdpOk => "rrdp_ok", + crate::sync::repo::RepoSyncPhase::RrdpFailedRsyncOk => "rrdp_failed_rsync_ok", + crate::sync::repo::RepoSyncPhase::RsyncOnlyOk => "rsync_only_ok", + crate::sync::repo::RepoSyncPhase::ReplayRrdpOk => "replay_rrdp_ok", + crate::sync::repo::RepoSyncPhase::ReplayRsyncOk => "replay_rsync_ok", + crate::sync::repo::RepoSyncPhase::ReplayNoopRrdp => "replay_noop_rrdp", + crate::sync::repo::RepoSyncPhase::ReplayNoopRsync => "replay_noop_rsync", + } +} + +enum RepoWorkerMessage { + Task(RepoSyncTask), + Shutdown, +} + +enum TransportWorkerMessage { + Task(RepoTransportTask), + Shutdown, +} + +pub struct RepoWorkerPool { + config: RepoWorkerPoolConfig, + task_tx: Sender, + result_rx: Receiver, + workers: Vec>, + _executor: Arc, +} + +pub struct RepoTransportWorkerPool { + config: RepoWorkerPoolConfig, + task_tx: Sender, + result_rx: Receiver, + workers: Vec>, + _executor: Arc, +} + +impl RepoWorkerPool { + pub fn new(config: RepoWorkerPoolConfig, executor: E) -> Result { + if config.max_workers == 0 { + return Err("RepoWorkerPool requires at least one worker".to_string()); + } + + let executor = Arc::new(executor); + let (task_tx, task_rx) = mpsc::channel::(); + let (result_tx, result_rx) = mpsc::channel::(); + let shared_task_rx = Arc::new(Mutex::new(task_rx)); + + let mut workers = Vec::with_capacity(config.max_workers); + for idx in 0..config.max_workers { + let task_rx = Arc::clone(&shared_task_rx); + let result_tx = result_tx.clone(); + let executor = Arc::clone(&executor); + workers.push( + thread::Builder::new() + .name(format!("repo-sync-worker-{idx}")) + .spawn(move || worker_loop(task_rx, result_tx, executor)) + .map_err(|e| format!("spawn repo worker failed: {e}"))?, + ); + } + + Ok(Self { + config, + task_tx, + result_rx, + workers, + _executor: executor, + }) + } + + pub fn worker_count(&self) -> usize { + self.config.max_workers + } + + pub fn submit(&self, task: RepoSyncTask) -> Result<(), String> { + self.task_tx + .send(RepoWorkerMessage::Task(task)) + .map_err(|e| format!("submit repo task failed: {e}")) + } + + pub fn recv_result_timeout( + &self, + timeout: Duration, + ) -> Result, String> { + match self.result_rx.recv_timeout(timeout) { + Ok(msg) => Ok(Some(msg)), + Err(RecvTimeoutError::Timeout) => Ok(None), + Err(RecvTimeoutError::Disconnected) => { + Err("repo worker result channel disconnected".to_string()) + } + } + } + + pub fn shutdown(mut self) -> Result<(), String> { + self.shutdown_inner() + } + + fn shutdown_inner(&mut self) -> Result<(), String> { + if self.workers.is_empty() { + return Ok(()); + } + + for _ in 0..self.workers.len() { + self.task_tx + .send(RepoWorkerMessage::Shutdown) + .map_err(|e| format!("send shutdown to repo worker failed: {e}"))?; + } + + let mut first_err: Option = None; + for handle in self.workers.drain(..) { + if let Err(e) = handle.join() { + if first_err.is_none() { + first_err = Some(format!("join repo worker failed: {e:?}")); + } + } + } + + if let Some(err) = first_err { + return Err(err); + } + Ok(()) + } +} + +impl Drop for RepoWorkerPool { + fn drop(&mut self) { + let _ = self.shutdown_inner(); + } +} + +impl RepoTransportWorkerPool { + pub fn new(config: RepoWorkerPoolConfig, executor: E) -> Result { + if config.max_workers == 0 { + return Err("RepoTransportWorkerPool requires at least one worker".to_string()); + } + let executor = Arc::new(executor); + let (task_tx, task_rx) = mpsc::channel::(); + let (result_tx, result_rx) = mpsc::channel::(); + let shared_task_rx = Arc::new(Mutex::new(task_rx)); + let mut workers = Vec::with_capacity(config.max_workers); + for idx in 0..config.max_workers { + let task_rx = Arc::clone(&shared_task_rx); + let result_tx = result_tx.clone(); + let executor = Arc::clone(&executor); + workers.push( + thread::Builder::new() + .name(format!("repo-transport-worker-{idx}")) + .spawn(move || transport_worker_loop(task_rx, result_tx, executor)) + .map_err(|e| format!("spawn repo transport worker failed: {e}"))?, + ); + } + Ok(Self { + config, + task_tx, + result_rx, + workers, + _executor: executor, + }) + } + + pub fn submit(&self, task: RepoTransportTask) -> Result<(), String> { + self.task_tx + .send(TransportWorkerMessage::Task(task)) + .map_err(|e| format!("submit repo transport task failed: {e}")) + } + + pub fn recv_result_timeout( + &self, + timeout: Duration, + ) -> Result, String> { + match self.result_rx.recv_timeout(timeout) { + Ok(msg) => Ok(Some(msg)), + Err(RecvTimeoutError::Timeout) => Ok(None), + Err(RecvTimeoutError::Disconnected) => { + Err("repo transport worker result channel disconnected".to_string()) + } + } + } + + pub fn worker_count(&self) -> usize { + self.config.max_workers + } + + pub fn shutdown(mut self) -> Result<(), String> { + self.shutdown_inner() + } + + fn shutdown_inner(&mut self) -> Result<(), String> { + if self.workers.is_empty() { + return Ok(()); + } + for _ in 0..self.workers.len() { + self.task_tx + .send(TransportWorkerMessage::Shutdown) + .map_err(|e| format!("send shutdown to repo transport worker failed: {e}"))?; + } + let mut first_err: Option = None; + for handle in self.workers.drain(..) { + if let Err(e) = handle.join() { + if first_err.is_none() { + first_err = Some(format!("join repo transport worker failed: {e:?}")); + } + } + } + if let Some(err) = first_err { + return Err(err); + } + Ok(()) + } +} + +impl Drop for RepoTransportWorkerPool { + fn drop(&mut self) { + let _ = self.shutdown_inner(); + } +} + +fn worker_loop( + task_rx: Arc>>, + result_tx: Sender, + executor: Arc, +) { + loop { + let message = { + let rx = task_rx.lock().expect("repo worker receiver lock poisoned"); + rx.recv() + }; + + match message { + Ok(RepoWorkerMessage::Task(task)) => { + let result = executor.execute(task); + if result_tx.send(result).is_err() { + break; + } + } + Ok(RepoWorkerMessage::Shutdown) | Err(_) => break, + } + } +} + +fn transport_worker_loop( + task_rx: Arc>>, + result_tx: Sender, + executor: Arc, +) { + loop { + let message = { + let rx = task_rx + .lock() + .expect("repo transport worker receiver lock poisoned"); + rx.recv() + }; + + match message { + Ok(TransportWorkerMessage::Task(task)) => { + let result = executor.execute_transport(task); + if result_tx.send(result).is_err() { + break; + } + } + Ok(TransportWorkerMessage::Shutdown) | Err(_) => break, + } + } +} + +#[cfg(test)] +mod tests { + use base64::Engine; + use sha2::Digest; + use std::collections::HashMap; + use std::fs; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, Barrier, Mutex}; + use std::time::Duration; + + use super::{ + LiveRepoSyncExecutor, LiveRrdpTransportExecutor, LiveRsyncTransportExecutor, + RepoSyncExecutor, RepoTransportExecutor, RepoTransportWorkerPool, RepoWorkerPool, + RepoWorkerPoolConfig, + }; + use crate::current_repo_index::CurrentRepoIndex; + use crate::fetch::rsync::{ + LocalDirRsyncFetcher, RsyncFetchError, RsyncFetchResult, RsyncFetcher, + }; + use crate::parallel::config::ParallelPhase1Config; + use crate::parallel::types::{ + RepoDedupKey, RepoIdentity, RepoKey, RepoRequester, RepoSyncResultEnvelope, + RepoSyncResultKind, RepoSyncResultRef, RepoSyncTask, RepoTransportMode, + RepoTransportResultEnvelope, RepoTransportResultKind, RepoTransportTask, + }; + use crate::policy::SyncPreference; + use crate::report::Warning; + use crate::storage::RocksStore; + use crate::sync::rrdp::Fetcher; + + fn sample_task(name: &str) -> RepoSyncTask { + RepoSyncTask { + repo_key: RepoKey::new(format!("rsync://example.test/{name}/"), None), + validation_time: time::OffsetDateTime::UNIX_EPOCH, + sync_preference: SyncPreference::RrdpThenRsync, + tal_id: "arin".to_string(), + rir_id: "arin".to_string(), + priority: 0, + requesters: vec![RepoRequester { + tal_id: "arin".to_string(), + rir_id: "arin".to_string(), + parent_node_id: None, + ca_instance_handle_id: format!("node:{name}"), + publication_point_rsync_uri: format!("rsync://example.test/{name}/"), + manifest_rsync_uri: format!("rsync://example.test/{name}/root.mft"), + }], + } + } + + fn sample_rrdp_transport_task( + notification_uri: &str, + rsync_base_uri: &str, + ) -> RepoTransportTask { + RepoTransportTask { + dedup_key: RepoDedupKey::RrdpNotify { + notification_uri: notification_uri.to_string(), + }, + rsync_failure_scope_uri: None, + repo_identity: RepoIdentity::new(Some(notification_uri.to_string()), rsync_base_uri), + mode: RepoTransportMode::Rrdp, + retry_short_timeout: false, + tal_id: "arin".to_string(), + rir_id: "arin".to_string(), + validation_time: time::OffsetDateTime::UNIX_EPOCH, + priority: 0, + requesters: vec![RepoRequester::with_tal_rir( + "arin", + "arin", + format!("{rsync_base_uri}root.mft"), + rsync_base_uri.to_string(), + "node:rrdp", + )], + } + } + + fn sample_rsync_transport_task( + rsync_scope_uri: &str, + rsync_base_uri: &str, + ) -> RepoTransportTask { + RepoTransportTask { + dedup_key: RepoDedupKey::RsyncScope { + rsync_scope_uri: rsync_scope_uri.to_string(), + }, + rsync_failure_scope_uri: None, + repo_identity: RepoIdentity::new(None, rsync_base_uri), + mode: RepoTransportMode::Rsync, + retry_short_timeout: false, + tal_id: "arin".to_string(), + rir_id: "arin".to_string(), + validation_time: time::OffsetDateTime::UNIX_EPOCH, + priority: 0, + requesters: vec![RepoRequester::with_tal_rir( + "arin", + "arin", + format!("{rsync_base_uri}root.mft"), + rsync_base_uri.to_string(), + "node:rsync", + )], + } + } + + struct SuccessExecutor; + + impl RepoSyncExecutor for SuccessExecutor { + fn execute(&self, task: RepoSyncTask) -> RepoSyncResultEnvelope { + RepoSyncResultEnvelope { + repo_key: task.repo_key.clone(), + tal_id: task.tal_id, + rir_id: task.rir_id, + result: RepoSyncResultKind::Success(RepoSyncResultRef { + repo_key: task.repo_key, + source: "rrdp".to_string(), + }), + phase: Some("rrdp_ok".to_string()), + timing_ms: 12, + warnings: Vec::new(), + } + } + } + + struct FailureExecutor; + + impl RepoSyncExecutor for FailureExecutor { + fn execute(&self, task: RepoSyncTask) -> RepoSyncResultEnvelope { + RepoSyncResultEnvelope { + repo_key: task.repo_key, + tal_id: task.tal_id, + rir_id: task.rir_id, + result: RepoSyncResultKind::Failed { + detail: "timeout".to_string(), + }, + phase: Some("repo_sync_failed".to_string()), + timing_ms: 33, + warnings: vec![Warning::new("timeout")], + } + } + } + + struct BarrierExecutor { + barrier: Arc, + active: Arc, + peak: Arc, + } + + #[derive(Clone)] + struct PanicHttpFetcher; + + impl Fetcher for PanicHttpFetcher { + fn fetch(&self, _uri: &str) -> Result, String> { + panic!("http fetch should not be used in this test") + } + } + + #[derive(Clone)] + struct ErrorHttpFetcher; + + impl Fetcher for ErrorHttpFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + Err(format!("fetch blocked for {uri}")) + } + } + + #[derive(Clone)] + struct FailingRsyncFetcher; + + #[derive(Clone, Default)] + struct MockHttpFetcher { + map: Arc>>>, + } + + impl MockHttpFetcher { + fn new() -> Self { + Self::default() + } + + fn insert(&self, uri: &str, bytes: Vec) { + self.map + .lock() + .expect("http fixture lock") + .insert(uri.to_string(), bytes); + } + } + + impl RsyncFetcher for FailingRsyncFetcher { + fn fetch_objects(&self, _rsync_base_uri: &str) -> RsyncFetchResult)>> { + Err(RsyncFetchError::Fetch("boom".to_string())) + } + } + + impl Fetcher for MockHttpFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + self.map + .lock() + .expect("http fixture lock") + .get(uri) + .cloned() + .ok_or_else(|| format!("missing fixture for {uri}")) + } + } + + fn sha256_hex(bytes: &[u8]) -> String { + hex::encode(sha2::Sha256::digest(bytes)) + } + + fn rrdp_notification_xml(session_id: &str, serial: u64, snapshot_uri: &str) -> String { + let snapshot_body = rrdp_snapshot_xml( + session_id, + &[("rsync://example.test/repo/a.roa", b"a".as_ref())], + ); + let snapshot_hash = sha256_hex(snapshot_body.as_bytes()); + format!( + r#" + + +"# + ) + } + + fn rrdp_snapshot_xml(session_id: &str, objects: &[(&str, &[u8])]) -> String { + let mut body = String::from(&format!( + r#" + +"# + )); + for (uri, bytes) in objects { + body.push_str(&format!( + " {}\n", + base64::engine::general_purpose::STANDARD.encode(bytes) + )); + } + body.push_str(""); + body + } + + impl RepoSyncExecutor for BarrierExecutor { + fn execute(&self, task: RepoSyncTask) -> RepoSyncResultEnvelope { + let now = self.active.fetch_add(1, Ordering::SeqCst) + 1; + loop { + let peak = self.peak.load(Ordering::SeqCst); + if now > peak { + if self + .peak + .compare_exchange(peak, now, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + break; + } + } else { + break; + } + } + self.barrier.wait(); + self.active.fetch_sub(1, Ordering::SeqCst); + RepoSyncResultEnvelope { + repo_key: task.repo_key.clone(), + tal_id: task.tal_id, + rir_id: task.rir_id, + result: RepoSyncResultKind::Success(RepoSyncResultRef { + repo_key: task.repo_key, + source: "rsync".to_string(), + }), + phase: Some("rsync_only_ok".to_string()), + timing_ms: 1, + warnings: Vec::new(), + } + } + } + + struct SuccessTransportExecutor; + + impl RepoTransportExecutor for SuccessTransportExecutor { + fn execute_transport(&self, task: RepoTransportTask) -> RepoTransportResultEnvelope { + RepoTransportResultEnvelope { + dedup_key: task.dedup_key, + rsync_failure_scope_uri: task.rsync_failure_scope_uri, + repo_identity: task.repo_identity, + mode: task.mode, + tal_id: task.tal_id, + rir_id: task.rir_id, + timing_ms: 5, + result: RepoTransportResultKind::Success { + source: match task.mode { + RepoTransportMode::Rrdp => "rrdp".to_string(), + RepoTransportMode::Rsync => "rsync".to_string(), + }, + warnings: Vec::new(), + }, + } + } + } + + #[test] + fn repo_worker_pool_config_uses_parallel_phase1_budget() { + let cfg = ParallelPhase1Config { + max_repo_sync_workers_global: 9, + ..ParallelPhase1Config::default() + }; + let pool_cfg = RepoWorkerPoolConfig::from(&cfg); + assert_eq!(pool_cfg.max_workers, 9); + } + + #[test] + fn repo_worker_pool_rejects_zero_workers() { + let err = RepoWorkerPool::new(RepoWorkerPoolConfig { max_workers: 0 }, SuccessExecutor) + .err() + .expect("zero workers should fail"); + assert!(err.contains("at least one worker")); + } + + #[test] + fn repo_worker_pool_processes_tasks_and_returns_results() { + let pool = RepoWorkerPool::new(RepoWorkerPoolConfig { max_workers: 2 }, SuccessExecutor) + .expect("pool"); + pool.submit(sample_task("a")).expect("submit task a"); + pool.submit(sample_task("b")).expect("submit task b"); + + let mut results = Vec::new(); + results.push( + pool.recv_result_timeout(Duration::from_secs(1)) + .expect("recv result 1") + .expect("result 1"), + ); + results.push( + pool.recv_result_timeout(Duration::from_secs(1)) + .expect("recv result 2") + .expect("result 2"), + ); + + assert_eq!(results.len(), 2); + assert!( + results + .iter() + .all(|res| matches!(res.result, RepoSyncResultKind::Success(_))) + ); + } + + #[test] + fn repo_worker_pool_returns_failure_envelopes() { + let pool = RepoWorkerPool::new(RepoWorkerPoolConfig { max_workers: 1 }, FailureExecutor) + .expect("pool"); + pool.submit(sample_task("fail")).expect("submit"); + let result = pool + .recv_result_timeout(Duration::from_secs(1)) + .expect("recv") + .expect("result"); + assert!(matches!( + result.result, + RepoSyncResultKind::Failed { ref detail } if detail == "timeout" + )); + assert_eq!(result.warnings, vec![Warning::new("timeout")]); + } + + #[test] + fn repo_worker_pool_processes_work_in_parallel() { + let barrier = Arc::new(Barrier::new(2)); + let active = Arc::new(AtomicUsize::new(0)); + let peak = Arc::new(AtomicUsize::new(0)); + let executor = BarrierExecutor { + barrier: Arc::clone(&barrier), + active: Arc::clone(&active), + peak: Arc::clone(&peak), + }; + let pool = + RepoWorkerPool::new(RepoWorkerPoolConfig { max_workers: 2 }, executor).expect("pool"); + pool.submit(sample_task("a")).expect("submit a"); + pool.submit(sample_task("b")).expect("submit b"); + + let _ = pool + .recv_result_timeout(Duration::from_secs(1)) + .expect("recv 1") + .expect("result 1"); + let _ = pool + .recv_result_timeout(Duration::from_secs(1)) + .expect("recv 2") + .expect("result 2"); + + assert!(peak.load(Ordering::SeqCst) >= 2); + } + + #[test] + fn repo_worker_pool_exposes_configured_worker_count() { + let pool = RepoWorkerPool::new(RepoWorkerPoolConfig { max_workers: 3 }, SuccessExecutor) + .expect("pool"); + assert_eq!(pool.worker_count(), 3); + } + + #[test] + fn repo_transport_worker_pool_processes_transport_tasks() { + let pool = RepoTransportWorkerPool::new( + RepoWorkerPoolConfig { max_workers: 2 }, + SuccessTransportExecutor, + ) + .expect("pool"); + pool.submit(sample_rrdp_transport_task( + "https://example.test/notify.xml", + "rsync://example.test/repo/", + )) + .expect("submit rrdp"); + pool.submit(sample_rsync_transport_task( + "rsync://example.test/module/", + "rsync://example.test/repo/", + )) + .expect("submit rsync"); + + let first = pool + .recv_result_timeout(Duration::from_secs(1)) + .expect("recv first") + .expect("first result"); + let second = pool + .recv_result_timeout(Duration::from_secs(1)) + .expect("recv second") + .expect("second result"); + assert!(matches!( + first.result, + RepoTransportResultKind::Success { .. } + )); + assert!(matches!( + second.result, + RepoTransportResultKind::Success { .. } + )); + } + + #[test] + fn live_repo_sync_executor_runs_rsync_sync_and_updates_store() { + let td = tempfile::tempdir().expect("tempdir"); + fs::create_dir_all(td.path().join("nested")).expect("mkdir"); + fs::write(td.path().join("a.roa"), b"a").expect("write a"); + fs::write(td.path().join("nested").join("b.cer"), b"b").expect("write b"); + + let store_dir = tempfile::tempdir().expect("store tempdir"); + let store = Arc::new(RocksStore::open(store_dir.path()).expect("open store")); + let executor = LiveRepoSyncExecutor::new( + Arc::clone(&store), + crate::policy::Policy::default(), + Arc::new(PanicHttpFetcher), + Arc::new(LocalDirRsyncFetcher::new(td.path())), + None, + None, + ); + + let result = executor.execute(sample_task("repo")); + assert!(matches!(result.result, RepoSyncResultKind::Success(_))); + assert_eq!(result.tal_id, "arin"); + assert_eq!(result.rir_id, "arin"); + assert_eq!(result.warnings.len(), 0); + + let view = store + .get_repository_view_entry("rsync://example.test/repo/a.roa") + .expect("read view") + .expect("entry exists"); + assert_eq!(view.rsync_uri, "rsync://example.test/repo/a.roa"); + assert!(view.current_hash.is_some()); + } + + #[test] + fn live_repo_sync_executor_returns_failure_when_rsync_fails() { + let store_dir = tempfile::tempdir().expect("store tempdir"); + let store = Arc::new(RocksStore::open(store_dir.path()).expect("open store")); + let executor = LiveRepoSyncExecutor::new( + Arc::clone(&store), + crate::policy::Policy::default(), + Arc::new(PanicHttpFetcher), + Arc::new(FailingRsyncFetcher), + None, + None, + ); + + let result = executor.execute(sample_task("repo")); + assert!(matches!( + result.result, + RepoSyncResultKind::Failed { ref detail } if detail.contains("rsync fallback failed") + )); + assert!( + store + .get_repository_view_entry("rsync://example.test/repo/a.roa") + .expect("read view") + .is_none() + ); + } + + #[test] + fn live_rrdp_transport_executor_succeeds_on_valid_notification() { + let td = tempfile::tempdir().expect("tempdir"); + let store = Arc::new(RocksStore::open(td.path()).expect("open store")); + let http = MockHttpFetcher::new(); + let notify = "https://example.test/notification.xml"; + let snapshot = "https://example.test/snapshot.xml"; + let snapshot_bytes = rrdp_snapshot_xml( + "123e4567-e89b-12d3-a456-426614174000", + &[("rsync://example.test/repo/a.roa", b"a".as_ref())], + ); + let notification_bytes = + rrdp_notification_xml("123e4567-e89b-12d3-a456-426614174000", 1, snapshot); + http.insert(notify, notification_bytes.into_bytes()); + http.insert(snapshot, snapshot_bytes.into_bytes()); + let current_repo_index = CurrentRepoIndex::shared(); + let executor = LiveRrdpTransportExecutor::new( + Arc::clone(&store), + current_repo_index.clone(), + Arc::new(http), + None, + None, + ); + let result = executor.execute_transport(sample_rrdp_transport_task( + notify, + "rsync://example.test/repo/", + )); + assert!( + matches!(result.result, RepoTransportResultKind::Success { .. }), + "{result:?}" + ); + let index = current_repo_index.read().expect("index read lock"); + assert!( + index + .get_by_uri("rsync://example.test/repo/a.roa") + .is_some() + ); + assert_eq!( + index.list_scope_uris(notify), + vec!["rsync://example.test/repo/a.roa".to_string()] + ); + } + + #[test] + fn live_rrdp_transport_executor_reports_failure_without_rsync_fallback() { + let td = tempfile::tempdir().expect("tempdir"); + let store = Arc::new(RocksStore::open(td.path()).expect("open store")); + let executor = LiveRrdpTransportExecutor::new( + Arc::clone(&store), + CurrentRepoIndex::shared(), + Arc::new(ErrorHttpFetcher), + None, + None, + ); + let result = executor.execute_transport(sample_rrdp_transport_task( + "https://example.test/notification.xml", + "rsync://example.test/repo/", + )); + assert!(matches!( + result.result, + RepoTransportResultKind::Failed { .. } + )); + } + + #[test] + fn live_rsync_transport_executor_succeeds_on_local_repo() { + let td = tempfile::tempdir().expect("tempdir"); + fs::create_dir_all(td.path().join("nested")).expect("mkdir"); + fs::write(td.path().join("a.roa"), b"a").expect("write"); + fs::write(td.path().join("nested").join("b.cer"), b"b").expect("write"); + let store_dir = tempfile::tempdir().expect("store tempdir"); + let store = Arc::new(RocksStore::open(store_dir.path()).expect("open store")); + let current_repo_index = CurrentRepoIndex::shared(); + let executor = LiveRsyncTransportExecutor::new( + Arc::clone(&store), + current_repo_index.clone(), + Arc::new(LocalDirRsyncFetcher::new(td.path())), + None, + None, + ); + let result = executor.execute_transport(sample_rsync_transport_task( + "rsync://example.test/repo/", + "rsync://example.test/repo/", + )); + assert!(matches!( + result.result, + RepoTransportResultKind::Success { .. } + )); + let index = current_repo_index.read().expect("index read lock"); + assert!( + index + .get_by_uri("rsync://example.test/repo/a.roa") + .is_some() + ); + assert!( + index + .get_by_uri("rsync://example.test/repo/nested/b.cer") + .is_some() + ); + assert_eq!( + index.list_scope_uris("rsync://example.test/repo/"), + vec![ + "rsync://example.test/repo/a.roa".to_string(), + "rsync://example.test/repo/nested/b.cer".to_string() + ] + ); + } + + #[test] + fn live_rsync_transport_executor_reports_failure() { + let store_dir = tempfile::tempdir().expect("store tempdir"); + let store = Arc::new(RocksStore::open(store_dir.path()).expect("open store")); + let executor = LiveRsyncTransportExecutor::new( + Arc::clone(&store), + CurrentRepoIndex::shared(), + Arc::new(FailingRsyncFetcher), + None, + None, + ); + let result = executor.execute_transport(sample_rsync_transport_task( + "rsync://example.test/module/", + "rsync://example.test/repo/", + )); + assert!(matches!( + result.result, + RepoTransportResultKind::Failed { .. } + )); + } +} diff --git a/crates/panda-rpki-validator/src/parallel/run_coordinator.rs b/crates/panda-rpki-validator/src/parallel/run_coordinator.rs new file mode 100644 index 0000000..7d62627 --- /dev/null +++ b/crates/panda-rpki-validator/src/parallel/run_coordinator.rs @@ -0,0 +1,879 @@ +use std::collections::VecDeque; + +use crate::current_repo_index::{CurrentRepoIndex, CurrentRepoIndexHandle}; +use crate::parallel::config::ParallelPhase1Config; +use crate::parallel::dead_repo_blacklist::{ + DeadRepoBlacklist, DeadRepoFailureOutcome, DeadRepoSuccessOutcome, +}; +use crate::parallel::repo_scheduler::{ + InFlightRepoTable, RepoCompletion, RepoRequestAction, TransportCompletion, + TransportRequestAction, TransportStateTables, +}; +use crate::parallel::stats::ParallelRunStats; +use crate::parallel::types::{ + RepoDedupKey, RepoIdentity, RepoKey, RepoRequester, RepoSyncResultEnvelope, RepoSyncTask, + RepoTransportErrorClass, RepoTransportMode, RepoTransportResultEnvelope, + RepoTransportResultKind, RepoTransportTask, TalInputSpec, +}; +use crate::policy::SyncPreference; + +pub struct GlobalRunCoordinator { + pub config: ParallelPhase1Config, + pub tal_inputs: Vec, + pub current_repo_index: CurrentRepoIndexHandle, + pub in_flight_repos: InFlightRepoTable, + pub transport_tables: TransportStateTables, + pub pending_repo_tasks: VecDeque, + pub pending_transport_tasks: VecDeque, + pub stats: ParallelRunStats, + /// Mutable working copy of the dead-repo blacklist (#141): failure/success + /// counters update during the run and persist at run end. Skip decisions + /// inside `transport_tables` use the frozen run-start snapshot. + pub dead_repo_blacklist: Option, +} + +impl GlobalRunCoordinator { + pub fn new(config: ParallelPhase1Config, tal_inputs: Vec) -> Self { + let dead_repo_blacklist = config.dead_repo_blacklist.as_ref().map(|blacklist_config| { + let (blacklist, warning) = DeadRepoBlacklist::load(&blacklist_config.path); + if let Some(warning) = warning { + crate::progress_log::emit( + "dead_repo_blacklist_load_warning", + serde_json::json!({ "warning": warning }), + ); + } + blacklist + }); + let mut transport_tables = TransportStateTables::new(); + if let Some(blacklist) = dead_repo_blacklist.as_ref() { + // Frozen run-start snapshot for skip decisions. + transport_tables.set_dead_repo_blacklist(blacklist.clone()); + } + Self { + config, + tal_inputs, + current_repo_index: CurrentRepoIndex::shared(), + in_flight_repos: InFlightRepoTable::new(), + transport_tables, + pending_repo_tasks: VecDeque::new(), + pending_transport_tasks: VecDeque::new(), + stats: ParallelRunStats::default(), + dead_repo_blacklist, + } + } + + pub fn current_repo_index_handle(&self) -> CurrentRepoIndexHandle { + self.current_repo_index.clone() + } + + pub fn register_repo_request( + &mut self, + repo_key: RepoKey, + requester: RepoRequester, + validation_time: time::OffsetDateTime, + sync_preference: SyncPreference, + priority: u8, + ) -> RepoRequestAction { + let action = self.in_flight_repos.register_request( + repo_key, + requester, + validation_time, + sync_preference, + priority, + ); + + match &action { + RepoRequestAction::Enqueued(task) => { + self.stats.repo_tasks_total += 1; + self.pending_repo_tasks.push_back(task.clone()); + self.stats.repo_queue_depth = self.pending_repo_tasks.len(); + } + RepoRequestAction::Reused(_) | RepoRequestAction::FailedReuse { .. } => { + self.stats.repo_tasks_reused += 1; + } + RepoRequestAction::Waiting => {} + } + + action + } + + pub fn pop_next_repo_task(&mut self) -> Option { + let next = self.pending_repo_tasks.pop_front(); + self.stats.repo_queue_depth = self.pending_repo_tasks.len(); + next + } + + pub fn mark_repo_running( + &mut self, + repo_key: &RepoKey, + started_at: time::OffsetDateTime, + ) -> Result<(), String> { + self.in_flight_repos.mark_running(repo_key, started_at)?; + self.stats.repo_tasks_running += 1; + Ok(()) + } + + pub fn complete_repo_success( + &mut self, + result: RepoSyncResultEnvelope, + finished_at: time::OffsetDateTime, + ) -> Result { + let repo_key = result.repo_key.clone(); + let completion = self + .in_flight_repos + .complete_success(&repo_key, result, finished_at)?; + self.stats.repo_tasks_running = self.stats.repo_tasks_running.saturating_sub(1); + Ok(completion) + } + + pub fn complete_repo_failure( + &mut self, + result: RepoSyncResultEnvelope, + finished_at: time::OffsetDateTime, + ) -> Result { + let repo_key = result.repo_key.clone(); + let completion = self + .in_flight_repos + .complete_failure(&repo_key, result, finished_at)?; + self.stats.repo_tasks_running = self.stats.repo_tasks_running.saturating_sub(1); + self.stats.repo_tasks_failed += 1; + Ok(completion) + } + + pub fn register_transport_request( + &mut self, + identity: RepoIdentity, + requester: RepoRequester, + validation_time: time::OffsetDateTime, + priority: u8, + rsync_scope_uri: String, + rsync_failure_scope_uri: Option, + sync_preference: SyncPreference, + retry_short_timeout: bool, + ) -> TransportRequestAction { + let mut action = self.transport_tables.register_transport_request( + identity, + requester, + validation_time, + priority, + rsync_scope_uri, + rsync_failure_scope_uri, + sync_preference, + ); + match &mut action { + TransportRequestAction::Enqueue(task) => { + if retry_short_timeout { + task.retry_short_timeout = true; + } + self.stats.repo_tasks_total += 1; + self.pending_transport_tasks.push_back(task.clone()); + self.stats.repo_queue_depth = self.pending_transport_tasks.len(); + } + TransportRequestAction::ReusedSuccess(_) + | TransportRequestAction::ReusedTerminalFailure(_) => { + self.stats.repo_tasks_reused += 1; + } + TransportRequestAction::Waiting { .. } => {} + } + action + } + + pub fn push_transport_task(&mut self, task: RepoTransportTask) { + self.stats.repo_tasks_total += 1; + self.pending_transport_tasks.push_back(task); + self.stats.repo_queue_depth = self.pending_transport_tasks.len(); + } + + pub fn pop_next_transport_task(&mut self) -> Option { + let next = self.pending_transport_tasks.pop_front(); + self.stats.repo_queue_depth = self.pending_transport_tasks.len(); + next + } + + pub fn mark_transport_running( + &mut self, + dedup_key: &crate::parallel::types::RepoDedupKey, + started_at: time::OffsetDateTime, + ) -> Result<(), String> { + self.transport_tables + .mark_transport_running(dedup_key, started_at)?; + self.stats.repo_tasks_running += 1; + Ok(()) + } + + pub fn complete_transport_result( + &mut self, + result: RepoTransportResultEnvelope, + finished_at: time::OffsetDateTime, + ) -> Result { + let completion = self + .transport_tables + .complete_transport_result(result.clone(), finished_at)?; + self.stats.repo_tasks_running = self.stats.repo_tasks_running.saturating_sub(1); + if matches!( + result.result, + crate::parallel::types::RepoTransportResultKind::Failed { .. } + ) && result.mode == crate::parallel::types::RepoTransportMode::Rsync + { + self.stats.repo_tasks_failed += 1; + } + self.update_dead_repo_blacklist(&result, finished_at); + Ok(completion) + } + + /// Update dead-repo blacklist counters (#141) from a real worker result. + /// Runs on the single pump thread; synthesized terminal envelopes never + /// pass through here, so no double counting is possible. + fn update_dead_repo_blacklist( + &mut self, + result: &RepoTransportResultEnvelope, + finished_at: time::OffsetDateTime, + ) { + let Some(blacklist_config) = self.config.dead_repo_blacklist.clone() else { + return; + }; + let Some(blacklist) = self.dead_repo_blacklist.as_mut() else { + return; + }; + let now_unix = finished_at.unix_timestamp().max(0) as u64; + let key: Option<(RepoTransportMode, String)> = match (&result.dedup_key, &result.result) { + ( + RepoDedupKey::RrdpNotify { notification_uri }, + RepoTransportResultKind::Success { .. }, + ) => Some((RepoTransportMode::Rrdp, notification_uri.clone())), + (RepoDedupKey::RsyncScope { .. }, RepoTransportResultKind::Success { .. }) => Some(( + RepoTransportMode::Rsync, + result.repo_identity.rsync_base_uri.clone(), + )), + ( + RepoDedupKey::RrdpNotify { notification_uri }, + RepoTransportResultKind::Failed { + error_class: RepoTransportErrorClass::TransportFetch, + .. + }, + ) => Some((RepoTransportMode::Rrdp, notification_uri.clone())), + ( + RepoDedupKey::RsyncScope { .. }, + RepoTransportResultKind::Failed { + error_class: RepoTransportErrorClass::TransportFetch, + .. + }, + ) => Some(( + RepoTransportMode::Rsync, + result.repo_identity.rsync_base_uri.clone(), + )), + _ => None, + }; + let Some((transport, uri)) = key else { + return; + }; + match &result.result { + RepoTransportResultKind::Success { .. } => { + if let Some(outcome) = blacklist.record_transport_success(transport, &uri) { + crate::progress_log::emit( + "dead_repo_blacklist_remove", + serde_json::json!({ + "transport": transport.as_str(), + "uri": uri, + "reason": match outcome { + DeadRepoSuccessOutcome::CounterReset => "counter_reset", + DeadRepoSuccessOutcome::BlacklistRemoved => "fetch_success", + }, + }), + ); + } + } + RepoTransportResultKind::Failed { .. } => { + if let DeadRepoFailureOutcome::Admitted = blacklist.record_transport_failure( + transport, + &uri, + now_unix, + blacklist_config.fail_threshold, + blacklist_config.capacity, + ) { + crate::progress_log::emit( + "dead_repo_blacklist_admit", + serde_json::json!({ + "transport": transport.as_str(), + "uri": uri, + "fail_threshold": blacklist_config.fail_threshold, + }), + ); + } + } + } + } + + pub fn dead_repo_blacklist(&self) -> Option<&DeadRepoBlacklist> { + self.dead_repo_blacklist.as_ref() + } + + pub fn runtime_record( + &self, + identity: &RepoIdentity, + ) -> Option<&crate::parallel::repo_scheduler::RepoRuntimeRecord> { + self.transport_tables.runtime_record(identity) + } + + pub fn finalized_runtime_records_for_transport( + &self, + dedup_key: &RepoDedupKey, + ) -> Vec { + self.transport_tables + .finalized_runtime_records_for_transport(dedup_key) + } + + pub fn finalized_runtime_records_for_transport_result( + &self, + result: &RepoTransportResultEnvelope, + ) -> Vec { + self.transport_tables + .finalized_runtime_records_for_transport_result(result) + } + + pub fn reset_run_state(&mut self) { + self.in_flight_repos.reset_run_state(); + self.transport_tables.reset_run_state(); + self.pending_repo_tasks.clear(); + self.pending_transport_tasks.clear(); + self.stats = ParallelRunStats::default(); + if let Ok(mut index) = self.current_repo_index.write() { + index.clear(); + } + } +} + +#[cfg(test)] +mod tests { + use crate::parallel::config::ParallelPhase1Config; + use crate::parallel::repo_scheduler::RepoRequestAction; + use crate::parallel::run_coordinator::GlobalRunCoordinator; + use crate::parallel::types::{ + RepoIdentity, RepoKey, RepoRequester, RepoSyncResultEnvelope, RepoSyncResultKind, + RepoSyncResultRef, TalInputSpec, + }; + use crate::policy::SyncPreference; + use crate::storage::{RepositoryViewEntry, RepositoryViewState}; + + fn requester(tal_id: &str, rir_id: &str, manifest: &str) -> RepoRequester { + RepoRequester { + tal_id: tal_id.to_string(), + rir_id: rir_id.to_string(), + parent_node_id: None, + ca_instance_handle_id: format!("{tal_id}:{manifest}"), + publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), + manifest_rsync_uri: manifest.to_string(), + } + } + + #[test] + fn coordinator_holds_tal_inputs_and_default_stats() { + let coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + assert_eq!(coordinator.tal_inputs.len(), 1); + assert_eq!(coordinator.tal_inputs[0].tal_id, "arin"); + assert_eq!(coordinator.stats.repo_tasks_total, 0); + assert!(coordinator.in_flight_repos.is_empty()); + } + + #[test] + fn coordinator_enqueues_new_repo_task_once() { + let mut coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let key = RepoKey::new("rsync://example.test/repo/", None); + let action = coordinator.register_repo_request( + key.clone(), + requester("arin", "arin", "rsync://example.test/repo/root.mft"), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 7, + ); + let task = match action { + RepoRequestAction::Enqueued(task) => task, + other => panic!("expected enqueue, got {other:?}"), + }; + assert_eq!(task.priority, 7); + assert_eq!(coordinator.stats.repo_tasks_total, 1); + assert_eq!(coordinator.stats.repo_queue_depth, 1); + + let popped = coordinator.pop_next_repo_task().expect("task queued"); + assert_eq!(popped.repo_key, key); + assert_eq!(coordinator.stats.repo_queue_depth, 0); + } + + #[test] + fn coordinator_reset_run_state_clears_runtime_only_state() { + let mut coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let identity = RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/", + ); + let requester = requester("arin", "arin", "rsync://example.test/repo/root.mft"); + let action = coordinator.register_transport_request( + identity.clone(), + requester, + time::OffsetDateTime::UNIX_EPOCH, + 0, + "rsync://example.test/repo/".to_string(), + None, + SyncPreference::RrdpThenRsync, + false, + ); + assert!(matches!( + action, + crate::parallel::repo_scheduler::TransportRequestAction::Enqueue(_) + )); + assert!(coordinator.runtime_record(&identity).is_some()); + assert_eq!(coordinator.pending_transport_tasks.len(), 1); + + { + let mut index = coordinator + .current_repo_index + .write() + .expect("index write lock"); + index + .apply_repository_view_entries(&[RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/a.roa".to_string(), + current_hash: Some("aa".repeat(32)), + repository_source: Some("rsync://example.test/repo/".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }]) + .expect("apply current object"); + assert_eq!(index.active_uri_count(), 1); + } + + coordinator.reset_run_state(); + + assert!(coordinator.runtime_record(&identity).is_none()); + assert!(coordinator.pending_transport_tasks.is_empty()); + assert!(coordinator.pending_repo_tasks.is_empty()); + assert_eq!( + coordinator + .current_repo_index + .read() + .expect("index read lock") + .active_uri_count(), + 0 + ); + assert_eq!(coordinator.stats.repo_tasks_total, 0); + } + + #[test] + fn coordinator_merges_waiting_requesters_and_reuses_success() { + let mut coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![ + TalInputSpec::from_url("https://example.test/arin.tal"), + TalInputSpec::from_url("https://example.test/apnic.tal"), + ], + ); + let key = RepoKey::new("rsync://shared.example/repo/", None); + let _ = coordinator.register_repo_request( + key.clone(), + requester("arin", "arin", "rsync://shared.example/repo/root.mft"), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + let _ = coordinator.pop_next_repo_task(); + coordinator + .mark_repo_running(&key, time::OffsetDateTime::UNIX_EPOCH) + .expect("running"); + + let wait_action = coordinator.register_repo_request( + key.clone(), + requester("apnic", "apnic", "rsync://shared.example/repo/child.mft"), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + assert_eq!(wait_action, RepoRequestAction::Waiting); + + let completion = coordinator + .complete_repo_success( + RepoSyncResultEnvelope { + repo_key: key.clone(), + tal_id: "arin".to_string(), + rir_id: "arin".to_string(), + result: RepoSyncResultKind::Success(RepoSyncResultRef { + repo_key: key.clone(), + source: "rrdp".to_string(), + }), + phase: Some("rrdp_ok".to_string()), + timing_ms: 12, + warnings: Vec::new(), + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("success"); + assert_eq!(completion.released_requesters.len(), 1); + assert_eq!(coordinator.stats.repo_tasks_running, 0); + + let reuse_action = coordinator.register_repo_request( + key, + requester("ripe", "ripe", "rsync://shared.example/repo/again.mft"), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + assert!(matches!(reuse_action, RepoRequestAction::Reused(_))); + assert_eq!(coordinator.stats.repo_tasks_reused, 1); + } + + #[test] + fn coordinator_reuses_failures_without_enqueuing_again() { + let mut coordinator = GlobalRunCoordinator::new( + ParallelPhase1Config::default(), + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + let key = RepoKey::new("rsync://shared.example/repo/", None); + let _ = coordinator.register_repo_request( + key.clone(), + requester("arin", "arin", "rsync://shared.example/repo/root.mft"), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + let _ = coordinator.pop_next_repo_task(); + coordinator + .mark_repo_running(&key, time::OffsetDateTime::UNIX_EPOCH) + .expect("running"); + coordinator + .complete_repo_failure( + RepoSyncResultEnvelope { + repo_key: key.clone(), + tal_id: "arin".to_string(), + rir_id: "arin".to_string(), + result: RepoSyncResultKind::Failed { + detail: "timeout".to_string(), + }, + phase: Some("rrdp_failed_rsync_failed".to_string()), + timing_ms: 12, + warnings: Vec::new(), + }, + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("failure"); + assert_eq!(coordinator.stats.repo_tasks_failed, 1); + + let action = coordinator.register_repo_request( + key, + requester("ripe", "ripe", "rsync://shared.example/repo/child.mft"), + time::OffsetDateTime::UNIX_EPOCH, + SyncPreference::RrdpThenRsync, + 0, + ); + assert_eq!( + action, + RepoRequestAction::FailedReuse { + detail: "timeout".to_string() + } + ); + assert_eq!(coordinator.stats.repo_tasks_total, 1); + assert_eq!(coordinator.stats.repo_tasks_reused, 1); + } + + // ---- Dead-repo blacklist (#141) integration tests ---- + + use crate::parallel::dead_repo_blacklist::{DeadRepoBlacklist, DeadRepoBlacklistConfig}; + use crate::parallel::types::{ + RepoDedupKey, RepoTransportErrorClass, RepoTransportMode, RepoTransportResultEnvelope, + RepoTransportResultKind, + }; + + const DEAD_NOTIFY: &str = "https://dead.example/notification.xml"; + const DEAD_BASE: &str = "rsync://dead.example/repo/"; + + fn blacklist_config(fail_threshold: u32) -> DeadRepoBlacklistConfig { + DeadRepoBlacklistConfig { + path: std::env::temp_dir().join(format!( + "dead_repo_blacklist_coordinator_test_{}.json", + std::process::id() + )), + fail_threshold, + capacity: 256, + } + } + + fn coordinator_with_blacklist( + fail_threshold: u32, + seed: DeadRepoBlacklist, + ) -> GlobalRunCoordinator { + let mut config = ParallelPhase1Config::default(); + config.dead_repo_blacklist = Some(blacklist_config(fail_threshold)); + let mut coordinator = GlobalRunCoordinator::new( + config, + vec![TalInputSpec::from_url("https://example.test/arin.tal")], + ); + coordinator.dead_repo_blacklist = Some(seed.clone()); + coordinator.transport_tables.set_dead_repo_blacklist(seed); + coordinator + } + + fn dead_identity() -> RepoIdentity { + RepoIdentity::new(Some(DEAD_NOTIFY.to_string()), DEAD_BASE) + } + + fn dead_requester() -> RepoRequester { + requester("arin", "arin", "rsync://dead.example/repo/root.mft") + } + + fn transport_envelope( + mode: RepoTransportMode, + error_class: RepoTransportErrorClass, + ) -> RepoTransportResultEnvelope { + let (dedup_key, detail) = match mode { + RepoTransportMode::Rrdp => ( + RepoDedupKey::RrdpNotify { + notification_uri: DEAD_NOTIFY.to_string(), + }, + "http request failed: connect timeout".to_string(), + ), + RepoTransportMode::Rsync => ( + RepoDedupKey::RsyncScope { + rsync_scope_uri: DEAD_BASE.to_string(), + }, + "rsync error: timeout waiting for daemon connection".to_string(), + ), + }; + RepoTransportResultEnvelope { + dedup_key, + rsync_failure_scope_uri: None, + repo_identity: dead_identity(), + mode, + tal_id: "arin".to_string(), + rir_id: "arin".to_string(), + timing_ms: 1, + result: RepoTransportResultKind::Failed { + detail, + warnings: Vec::new(), + error_class, + }, + } + } + + fn register_dead( + coordinator: &mut GlobalRunCoordinator, + ) -> crate::parallel::repo_scheduler::TransportRequestAction { + coordinator.register_transport_request( + dead_identity(), + dead_requester(), + time::OffsetDateTime::UNIX_EPOCH, + 0, + DEAD_BASE.to_string(), + None, + SyncPreference::RrdpThenRsync, + false, + ) + } + + #[test] + fn blacklist_admission_after_threshold_transport_failures() { + let mut coordinator = coordinator_with_blacklist(2, DeadRepoBlacklist::new()); + for _ in 0..2 { + let action = register_dead(&mut coordinator); + assert!(matches!( + action, + crate::parallel::repo_scheduler::TransportRequestAction::Enqueue(_) + )); + coordinator + .complete_transport_result( + transport_envelope( + RepoTransportMode::Rrdp, + RepoTransportErrorClass::TransportFetch, + ), + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete"); + // Simulate the next run: run-local scheduler state resets, the + // persistent blacklist working copy carries over. + coordinator.reset_run_state(); + } + let blacklist = coordinator.dead_repo_blacklist().expect("blacklist"); + assert!(blacklist.is_blacklisted(RepoTransportMode::Rrdp, DEAD_NOTIFY)); + // Admission mid-run does not affect this run's frozen snapshot: the + // scheduler still enqueues rrdp for a fresh identity registration. + coordinator.reset_run_state(); + let action = register_dead(&mut coordinator); + assert!(matches!( + action, + crate::parallel::repo_scheduler::TransportRequestAction::Enqueue(_) + )); + } + + #[test] + fn blacklist_ignores_protocol_and_unknown_failures() { + let mut coordinator = coordinator_with_blacklist(1, DeadRepoBlacklist::new()); + for error_class in [ + RepoTransportErrorClass::Protocol, + RepoTransportErrorClass::Unknown, + RepoTransportErrorClass::Storage, + ] { + let action = register_dead(&mut coordinator); + assert!(matches!( + action, + crate::parallel::repo_scheduler::TransportRequestAction::Enqueue(_) + )); + coordinator + .complete_transport_result( + transport_envelope(RepoTransportMode::Rrdp, error_class), + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete"); + coordinator.reset_run_state(); + } + assert!( + coordinator + .dead_repo_blacklist() + .expect("blacklist") + .is_empty() + ); + } + + #[test] + fn blacklist_skip_rrdp_routes_straight_to_rsync() { + let mut seed = DeadRepoBlacklist::new(); + seed.record_transport_failure(RepoTransportMode::Rrdp, DEAD_NOTIFY, 100, 1, 256); + let mut coordinator = coordinator_with_blacklist(3, seed); + let action = register_dead(&mut coordinator); + let task = match action { + crate::parallel::repo_scheduler::TransportRequestAction::Enqueue(task) => task, + other => panic!("expected rsync enqueue, got {other:?}"), + }; + assert_eq!(task.mode, RepoTransportMode::Rsync); + // register_dead passes retry_short_timeout=false (mirroring a prefetch + // snapshot that recorded no real rrdp failure for skipped runs); the + // skip path must still keep the short retry profile (#141). + assert!(task.retry_short_timeout); + } + + #[test] + fn blacklist_dual_dead_terminates_immediately() { + let mut seed = DeadRepoBlacklist::new(); + seed.record_transport_failure(RepoTransportMode::Rrdp, DEAD_NOTIFY, 100, 1, 256); + seed.record_transport_failure(RepoTransportMode::Rsync, DEAD_BASE, 100, 1, 256); + let mut coordinator = coordinator_with_blacklist(3, seed); + let action = register_dead(&mut coordinator); + let envelope = match action { + crate::parallel::repo_scheduler::TransportRequestAction::ReusedTerminalFailure( + envelope, + ) => envelope, + other => panic!("expected terminal failure, got {other:?}"), + }; + match &envelope.result { + RepoTransportResultKind::Failed { + detail, warnings, .. + } => { + assert!(detail.contains("dead repo blacklist")); + assert!( + warnings + .iter() + .any(|warning| warning.message.contains("dead_repo_blacklist_skip_all")) + ); + } + other => panic!("expected failed result, got {other:?}"), + } + let record = coordinator + .runtime_record(&dead_identity()) + .expect("record"); + assert_eq!( + record.state, + crate::parallel::types::RepoRuntimeState::FailedTerminal + ); + assert_eq!(coordinator.stats.repo_tasks_total, 0); + assert_eq!(coordinator.stats.repo_tasks_reused, 1); + } + + #[test] + fn blacklist_rsync_dead_terminates_after_rrdp_failure() { + // Only rsync blacklisted: rrdp is attempted; when it fails, the rsync + // fallback is short-circuited into an instant terminal failure. + let mut seed = DeadRepoBlacklist::new(); + seed.record_transport_failure(RepoTransportMode::Rsync, DEAD_BASE, 100, 1, 256); + let mut coordinator = coordinator_with_blacklist(3, seed); + let action = register_dead(&mut coordinator); + assert!(matches!( + action, + crate::parallel::repo_scheduler::TransportRequestAction::Enqueue(_) + )); + let completion = coordinator + .complete_transport_result( + transport_envelope( + RepoTransportMode::Rrdp, + RepoTransportErrorClass::TransportFetch, + ), + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete"); + assert!(completion.follow_up_tasks.is_empty()); + let record = coordinator + .runtime_record(&dead_identity()) + .expect("record"); + assert_eq!( + record.state, + crate::parallel::types::RepoRuntimeState::FailedTerminal + ); + let terminal = record.terminal_failure.as_ref().expect("terminal failure"); + match &terminal.result { + RepoTransportResultKind::Failed { detail, .. } => { + assert!(detail.contains("dead repo blacklist")); + } + other => panic!("expected failed result, got {other:?}"), + } + } + + #[test] + fn blacklist_success_resets_failure_counter() { + let mut coordinator = coordinator_with_blacklist(3, DeadRepoBlacklist::new()); + // One transport failure starts counting. + let action = register_dead(&mut coordinator); + assert!(matches!( + action, + crate::parallel::repo_scheduler::TransportRequestAction::Enqueue(_) + )); + coordinator + .complete_transport_result( + transport_envelope( + RepoTransportMode::Rrdp, + RepoTransportErrorClass::TransportFetch, + ), + time::OffsetDateTime::UNIX_EPOCH, + ) + .expect("complete"); + assert_eq!( + coordinator.dead_repo_blacklist().expect("blacklist").len(), + 1 + ); + + // A later success clears the entry. + coordinator.reset_run_state(); + let action = register_dead(&mut coordinator); + assert!(matches!( + action, + crate::parallel::repo_scheduler::TransportRequestAction::Enqueue(_) + )); + let mut success = + transport_envelope(RepoTransportMode::Rrdp, RepoTransportErrorClass::Unknown); + success.result = RepoTransportResultKind::Success { + source: "rrdp".to_string(), + warnings: Vec::new(), + }; + coordinator + .complete_transport_result(success, time::OffsetDateTime::UNIX_EPOCH) + .expect("complete"); + assert!( + coordinator + .dead_repo_blacklist() + .expect("blacklist") + .is_empty() + ); + } +} diff --git a/crates/panda-rpki-validator/src/parallel/stats.rs b/crates/panda-rpki-validator/src/parallel/stats.rs new file mode 100644 index 0000000..157539a --- /dev/null +++ b/crates/panda-rpki-validator/src/parallel/stats.rs @@ -0,0 +1,25 @@ +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ParallelRunStats { + pub repo_tasks_total: usize, + pub repo_tasks_reused: usize, + pub repo_tasks_running: usize, + pub repo_tasks_failed: usize, + pub inflight_snapshot_bytes: usize, + pub repo_queue_depth: usize, +} + +#[cfg(test)] +mod tests { + use super::ParallelRunStats; + + #[test] + fn parallel_run_stats_default_to_zero() { + let stats = ParallelRunStats::default(); + assert_eq!(stats.repo_tasks_total, 0); + assert_eq!(stats.repo_tasks_reused, 0); + assert_eq!(stats.repo_tasks_running, 0); + assert_eq!(stats.repo_tasks_failed, 0); + assert_eq!(stats.inflight_snapshot_bytes, 0); + assert_eq!(stats.repo_queue_depth, 0); + } +} diff --git a/crates/panda-rpki-validator/src/parallel/transport_prefetch.rs b/crates/panda-rpki-validator/src/parallel/transport_prefetch.rs new file mode 100644 index 0000000..fac75e5 --- /dev/null +++ b/crates/panda-rpki-validator/src/parallel/transport_prefetch.rs @@ -0,0 +1,512 @@ +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +use crate::parallel::types::{ + RepoDedupKey, RepoIdentity, RepoRequester, RepoTransportMode, RepoTransportTask, +}; +use crate::policy::SyncPreference; + +pub const TRANSPORT_PREFETCH_SCHEMA_VERSION: u32 = 1; + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct TransportPrefetchSnapshot { + pub schema_version: u32, + pub generated_at_unix: i64, + pub sync_preference: SyncPreference, + pub requests: Vec, +} + +impl TransportPrefetchSnapshot { + pub fn new(sync_preference: SyncPreference, requests: Vec) -> Self { + Self { + schema_version: TRANSPORT_PREFETCH_SCHEMA_VERSION, + generated_at_unix: time::OffsetDateTime::now_utc().unix_timestamp(), + sync_preference, + requests, + } + } + + pub fn is_compatible_with(&self, sync_preference: SyncPreference) -> bool { + self.schema_version == TRANSPORT_PREFETCH_SCHEMA_VERSION + && self.sync_preference == sync_preference + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TransportPrefetchRequest { + pub dedup_key: TransportPrefetchDedupKey, + pub rsync_scope_uri: String, + pub rsync_failure_scope_uri: Option, + pub repo_identity: TransportPrefetchRepoIdentity, + pub mode: TransportPrefetchMode, + #[serde(default)] + pub last_result: Option, + #[serde(default)] + pub last_rsync_result: Option, + pub tal_id: String, + pub rir_id: String, + pub priority: u8, + pub requesters: Vec, +} + +impl TransportPrefetchRequest { + pub fn from_registered_request( + identity: &RepoIdentity, + requester: &RepoRequester, + priority: u8, + rsync_scope_uri: String, + rsync_failure_scope_uri: Option, + sync_preference: SyncPreference, + ) -> Self { + let (dedup_key, mode) = if sync_preference == SyncPreference::RrdpThenRsync { + if let Some(notification_uri) = identity.notification_uri.clone() { + ( + TransportPrefetchDedupKey::RrdpNotify { notification_uri }, + TransportPrefetchMode::Rrdp, + ) + } else { + ( + TransportPrefetchDedupKey::RsyncScope { + rsync_scope_uri: rsync_scope_uri.clone(), + }, + TransportPrefetchMode::Rsync, + ) + } + } else { + ( + TransportPrefetchDedupKey::RsyncScope { + rsync_scope_uri: rsync_scope_uri.clone(), + }, + TransportPrefetchMode::Rsync, + ) + }; + + Self { + dedup_key, + rsync_scope_uri, + rsync_failure_scope_uri, + repo_identity: TransportPrefetchRepoIdentity::from_identity(identity), + mode, + last_result: None, + last_rsync_result: None, + tal_id: requester.tal_id.clone(), + rir_id: requester.rir_id.clone(), + priority, + requesters: vec![TransportPrefetchRequester::from_requester(requester)], + } + } + + pub fn from_task(task: &RepoTransportTask, rsync_scope_uri: String) -> Self { + Self { + dedup_key: TransportPrefetchDedupKey::from_repo_key(&task.dedup_key), + rsync_scope_uri, + rsync_failure_scope_uri: task.rsync_failure_scope_uri.clone(), + repo_identity: TransportPrefetchRepoIdentity::from_identity(&task.repo_identity), + mode: TransportPrefetchMode::from_mode(task.mode), + last_result: None, + last_rsync_result: None, + tal_id: task.tal_id.clone(), + rir_id: task.rir_id.clone(), + priority: task.priority, + requesters: task + .requesters + .iter() + .map(TransportPrefetchRequester::from_requester) + .collect(), + } + } + + pub fn to_identity(&self) -> RepoIdentity { + self.repo_identity.to_identity() + } + + pub fn to_requester(&self) -> RepoRequester { + self.requesters + .first() + .map(TransportPrefetchRequester::to_requester) + .unwrap_or_else(|| RepoRequester { + tal_id: self.tal_id.clone(), + rir_id: self.rir_id.clone(), + parent_node_id: None, + ca_instance_handle_id: format!( + "{}:{}", + self.tal_id, self.repo_identity.rsync_base_uri + ), + publication_point_rsync_uri: self.repo_identity.rsync_base_uri.clone(), + manifest_rsync_uri: format!("{}prefetch.mft", self.repo_identity.rsync_base_uri), + }) + } + + fn recorder_key(&self) -> String { + self.dedup_key.stable_key() + } + + pub fn retry_short_timeout(&self) -> bool { + matches!( + self.last_result, + Some(TransportPrefetchLastResult { ok: false }) + ) + } + + pub fn retry_short_rsync_timeout(&self) -> bool { + matches!( + self.last_rsync_result, + Some(TransportPrefetchLastResult { ok: false }) + ) || (self.mode == TransportPrefetchMode::Rsync && self.retry_short_timeout()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TransportPrefetchDedupKey { + RrdpNotify { notification_uri: String }, + RsyncScope { rsync_scope_uri: String }, +} + +impl TransportPrefetchDedupKey { + fn from_repo_key(key: &RepoDedupKey) -> Self { + match key { + RepoDedupKey::RrdpNotify { notification_uri } => Self::RrdpNotify { + notification_uri: notification_uri.clone(), + }, + RepoDedupKey::RsyncScope { rsync_scope_uri } => Self::RsyncScope { + rsync_scope_uri: rsync_scope_uri.clone(), + }, + } + } + + fn stable_key(&self) -> String { + match self { + Self::RrdpNotify { notification_uri } => format!("rrdp:{notification_uri}"), + Self::RsyncScope { rsync_scope_uri } => format!("rsync:{rsync_scope_uri}"), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TransportPrefetchRepoIdentity { + pub notification_uri: Option, + pub rsync_base_uri: String, +} + +impl TransportPrefetchRepoIdentity { + fn from_identity(identity: &RepoIdentity) -> Self { + Self { + notification_uri: identity.notification_uri.clone(), + rsync_base_uri: identity.rsync_base_uri.clone(), + } + } + + fn to_identity(&self) -> RepoIdentity { + RepoIdentity::new(self.notification_uri.clone(), self.rsync_base_uri.clone()) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TransportPrefetchMode { + Rrdp, + Rsync, +} + +impl TransportPrefetchMode { + fn from_mode(mode: RepoTransportMode) -> Self { + match mode { + RepoTransportMode::Rrdp => Self::Rrdp, + RepoTransportMode::Rsync => Self::Rsync, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TransportPrefetchLastResult { + pub ok: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct TransportPrefetchRequester { + pub tal_id: String, + pub rir_id: String, + pub parent_node_id: Option, + pub ca_instance_handle_id: String, + pub publication_point_rsync_uri: String, + pub manifest_rsync_uri: String, +} + +impl TransportPrefetchRequester { + fn from_requester(requester: &RepoRequester) -> Self { + Self { + tal_id: requester.tal_id.clone(), + rir_id: requester.rir_id.clone(), + parent_node_id: requester.parent_node_id, + ca_instance_handle_id: requester.ca_instance_handle_id.clone(), + publication_point_rsync_uri: requester.publication_point_rsync_uri.clone(), + manifest_rsync_uri: requester.manifest_rsync_uri.clone(), + } + } + + fn to_requester(&self) -> RepoRequester { + RepoRequester { + tal_id: self.tal_id.clone(), + rir_id: self.rir_id.clone(), + parent_node_id: self.parent_node_id, + ca_instance_handle_id: self.ca_instance_handle_id.clone(), + publication_point_rsync_uri: self.publication_point_rsync_uri.clone(), + manifest_rsync_uri: self.manifest_rsync_uri.clone(), + } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct TransportPrefetchRecorder { + requests_by_key: BTreeMap, + results_by_key: BTreeMap, + rsync_results_by_scope: BTreeMap, + request_order: Vec, +} + +impl TransportPrefetchRecorder { + pub fn record_registered_request( + &mut self, + identity: &RepoIdentity, + requester: &RepoRequester, + priority: u8, + rsync_scope_uri: String, + rsync_failure_scope_uri: Option, + sync_preference: SyncPreference, + ) { + let request = TransportPrefetchRequest::from_registered_request( + identity, + requester, + priority, + rsync_scope_uri, + rsync_failure_scope_uri, + sync_preference, + ); + self.record_request(request); + } + + pub fn record_task(&mut self, task: &RepoTransportTask, rsync_scope_uri: String) { + let request = TransportPrefetchRequest::from_task(task, rsync_scope_uri); + self.record_request(request); + } + + pub fn record_result(&mut self, result: &crate::parallel::types::RepoTransportResultEnvelope) { + let key = TransportPrefetchDedupKey::from_repo_key(&result.dedup_key).stable_key(); + let last_result = TransportPrefetchLastResult { + ok: matches!( + result.result, + crate::parallel::types::RepoTransportResultKind::Success { .. } + ), + }; + self.results_by_key.insert(key.clone(), last_result); + if let Some(request) = self.requests_by_key.get_mut(&key) { + request.last_result = Some(last_result); + } + if let crate::parallel::types::RepoDedupKey::RsyncScope { rsync_scope_uri } = + &result.dedup_key + { + self.rsync_results_by_scope + .insert(rsync_scope_uri.clone(), last_result); + for request in self.requests_by_key.values_mut() { + if request.rsync_scope_uri == *rsync_scope_uri { + request.last_rsync_result = Some(last_result); + } + } + } + } + + pub fn snapshot(&self, sync_preference: SyncPreference) -> TransportPrefetchSnapshot { + TransportPrefetchSnapshot::new( + sync_preference, + self.request_order + .iter() + .filter_map(|key| self.requests_by_key.get(key)) + .cloned() + .collect(), + ) + } + + pub fn len(&self) -> usize { + self.requests_by_key.len() + } + + fn record_request(&mut self, request: TransportPrefetchRequest) { + let key = request.recorder_key(); + if !self.requests_by_key.contains_key(&key) { + self.request_order.push(key.clone()); + let mut request = request; + if request.last_result.is_none() { + request.last_result = self.results_by_key.get(&key).copied(); + } + if request.last_rsync_result.is_none() { + request.last_rsync_result = self + .rsync_results_by_scope + .get(&request.rsync_scope_uri) + .copied(); + } + self.requests_by_key.insert(key, request); + } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct TransportPrefetchDispatchStats { + pub loaded_requests: u64, + pub enqueued_tasks: u64, + pub waiting_requests: u64, + pub reused_results: u64, + pub skipped_incompatible: u64, +} + +#[cfg(test)] +mod tests { + use super::*; + + fn requester(uri: &str) -> RepoRequester { + RepoRequester { + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + parent_node_id: None, + ca_instance_handle_id: format!("apnic:{uri}"), + publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), + manifest_rsync_uri: uri.to_string(), + } + } + + fn task(notification_uri: &str, manifest_uri: &str) -> RepoTransportTask { + RepoTransportTask { + dedup_key: RepoDedupKey::RrdpNotify { + notification_uri: notification_uri.to_string(), + }, + rsync_failure_scope_uri: Some("rsync://example.test/".to_string()), + repo_identity: RepoIdentity::new( + Some(notification_uri.to_string()), + "rsync://example.test/repo/", + ), + mode: RepoTransportMode::Rrdp, + retry_short_timeout: false, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + validation_time: time::OffsetDateTime::UNIX_EPOCH, + priority: 0, + requesters: vec![requester(manifest_uri)], + } + } + + #[test] + fn recorder_deduplicates_by_transport_key() { + let mut recorder = TransportPrefetchRecorder::default(); + recorder.record_task( + &task( + "https://example.test/notification.xml", + "rsync://example.test/repo/a.mft", + ), + "rsync://example.test/repo/".to_string(), + ); + recorder.record_task( + &task( + "https://example.test/notification.xml", + "rsync://example.test/repo/b.mft", + ), + "rsync://example.test/repo/".to_string(), + ); + + let snapshot = recorder.snapshot(SyncPreference::RrdpThenRsync); + assert_eq!(snapshot.requests.len(), 1); + assert_eq!( + snapshot.requests[0].requesters[0].manifest_rsync_uri, + "rsync://example.test/repo/a.mft" + ); + } + + #[test] + fn recorder_preserves_first_discovery_order() { + let mut recorder = TransportPrefetchRecorder::default(); + recorder.record_task( + &task( + "https://z.example.test/notification.xml", + "rsync://z.example.test/repo/root.mft", + ), + "rsync://z.example.test/repo/".to_string(), + ); + recorder.record_task( + &task( + "https://a.example.test/notification.xml", + "rsync://a.example.test/repo/root.mft", + ), + "rsync://a.example.test/repo/".to_string(), + ); + + let snapshot = recorder.snapshot(SyncPreference::RrdpThenRsync); + assert_eq!( + snapshot.requests[0] + .repo_identity + .notification_uri + .as_deref(), + Some("https://z.example.test/notification.xml") + ); + assert_eq!( + snapshot.requests[1] + .repo_identity + .notification_uri + .as_deref(), + Some("https://a.example.test/notification.xml") + ); + } + + #[test] + fn snapshot_checks_schema_and_sync_preference() { + let snapshot = TransportPrefetchSnapshot::new(SyncPreference::RrdpThenRsync, Vec::new()); + assert!(snapshot.is_compatible_with(SyncPreference::RrdpThenRsync)); + assert!(!snapshot.is_compatible_with(SyncPreference::RsyncOnly)); + } + + #[test] + fn recorder_marks_failed_rrdp_and_rsync_for_next_run_short_timeouts() { + let mut recorder = TransportPrefetchRecorder::default(); + let rrdp_task = task( + "https://example.test/notification.xml", + "rsync://example.test/repo/a.mft", + ); + recorder.record_task(&rrdp_task, "rsync://example.test/repo/".to_string()); + recorder.record_result(&crate::parallel::types::RepoTransportResultEnvelope { + dedup_key: RepoDedupKey::RrdpNotify { + notification_uri: "https://example.test/notification.xml".to_string(), + }, + rsync_failure_scope_uri: None, + repo_identity: rrdp_task.repo_identity.clone(), + mode: RepoTransportMode::Rrdp, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 1, + result: crate::parallel::types::RepoTransportResultKind::Failed { + detail: "timeout".to_string(), + warnings: Vec::new(), + error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, + }, + }); + recorder.record_result(&crate::parallel::types::RepoTransportResultEnvelope { + dedup_key: RepoDedupKey::RsyncScope { + rsync_scope_uri: "rsync://example.test/repo/".to_string(), + }, + rsync_failure_scope_uri: None, + repo_identity: rrdp_task.repo_identity.clone(), + mode: RepoTransportMode::Rsync, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + timing_ms: 1, + result: crate::parallel::types::RepoTransportResultKind::Failed { + detail: "timeout".to_string(), + warnings: Vec::new(), + error_class: crate::parallel::types::RepoTransportErrorClass::Unknown, + }, + }); + + let snapshot = recorder.snapshot(SyncPreference::RrdpThenRsync); + assert_eq!(snapshot.requests.len(), 1); + assert!(snapshot.requests[0].retry_short_timeout()); + assert!(snapshot.requests[0].retry_short_rsync_timeout()); + } +} diff --git a/crates/panda-rpki-validator/src/parallel/types.rs b/crates/panda-rpki-validator/src/parallel/types.rs new file mode 100644 index 0000000..bf0195e --- /dev/null +++ b/crates/panda-rpki-validator/src/parallel/types.rs @@ -0,0 +1,537 @@ +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; + +use crate::policy::SyncPreference; +use crate::report::Warning; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum TalSource { + Url(String), + DerBytes { + tal_url: String, + tal_bytes: Vec, + ta_der: Vec, + }, + FilePath(PathBuf), + FilePathWithTa { + tal_path: PathBuf, + ta_path: PathBuf, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TalInputSpec { + pub tal_id: String, + pub rir_id: String, + pub source: TalSource, +} + +impl TalInputSpec { + pub fn from_url(url: impl Into) -> Self { + let url = url.into(); + let tal_id = derive_tal_id_from_url_like(&url); + Self { + rir_id: tal_id.clone(), + tal_id, + source: TalSource::Url(url), + } + } + + pub fn from_file_path(path: impl Into) -> Self { + let path = path.into(); + let tal_id = derive_tal_id_from_path(&path); + Self { + rir_id: tal_id.clone(), + tal_id, + source: TalSource::FilePath(path), + } + } + + pub fn from_file_path_with_ta( + tal_path: impl Into, + ta_path: impl Into, + ) -> Self { + let tal_path = tal_path.into(); + let ta_path = ta_path.into(); + let tal_id = derive_tal_id_from_path(&tal_path); + Self { + rir_id: tal_id.clone(), + tal_id, + source: TalSource::FilePathWithTa { tal_path, ta_path }, + } + } + + pub fn from_ta_der(tal_url: impl Into, tal_bytes: Vec, ta_der: Vec) -> Self { + let tal_url = tal_url.into(); + let tal_id = derive_tal_id_from_url_like(&tal_url); + Self { + rir_id: tal_id.clone(), + tal_id, + source: TalSource::DerBytes { + tal_url, + tal_bytes, + ta_der, + }, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct RepoIdentity { + pub notification_uri: Option, + pub rsync_base_uri: String, +} + +impl RepoIdentity { + pub fn new(notification_uri: Option, rsync_base_uri: impl Into) -> Self { + Self { + notification_uri, + rsync_base_uri: rsync_base_uri.into(), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum RepoDedupKey { + RrdpNotify { notification_uri: String }, + RsyncScope { rsync_scope_uri: String }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RepoTransportMode { + Rrdp, + Rsync, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RepoTransportTask { + pub dedup_key: RepoDedupKey, + pub rsync_failure_scope_uri: Option, + pub repo_identity: RepoIdentity, + pub mode: RepoTransportMode, + pub retry_short_timeout: bool, + pub tal_id: String, + pub rir_id: String, + pub validation_time: time::OffsetDateTime, + pub priority: u8, + pub requesters: Vec, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RepoTransportErrorClass { + /// Transport-layer fetch failure (connect timeout/refused/TLS/DNS/broken + /// connection). Only this class counts toward the dead-repo blacklist. + TransportFetch, + /// Protocol-level failure (HTTP status, XML/parse, content mismatch). + Protocol, + /// Local storage failure. + Storage, + /// Unclassified (synthesized or legacy results); never counted. + Unknown, +} + +impl Default for RepoTransportErrorClass { + fn default() -> Self { + RepoTransportErrorClass::Unknown + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RepoTransportResultKind { + Success { + source: String, + warnings: Vec, + }, + Failed { + detail: String, + warnings: Vec, + error_class: RepoTransportErrorClass, + }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RepoTransportResultEnvelope { + pub dedup_key: RepoDedupKey, + pub rsync_failure_scope_uri: Option, + pub repo_identity: RepoIdentity, + pub mode: RepoTransportMode, + pub tal_id: String, + pub rir_id: String, + pub timing_ms: u64, + pub result: RepoTransportResultKind, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RepoRuntimeState { + Init, + WaitingRrdp, + RrdpOk, + RrdpFailedPendingRsync, + WaitingRsync, + RsyncOk, + FailedTerminal, +} + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub struct RepoKey { + pub rsync_base_uri: String, + pub notification_uri: Option, +} + +impl RepoKey { + pub fn new(rsync_base_uri: impl Into, notification_uri: Option) -> Self { + Self { + rsync_base_uri: rsync_base_uri.into(), + notification_uri, + } + } + + pub fn as_identity(&self) -> RepoIdentity { + RepoIdentity { + notification_uri: self.notification_uri.clone(), + rsync_base_uri: self.rsync_base_uri.clone(), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RepoRequester { + pub tal_id: String, + pub rir_id: String, + pub parent_node_id: Option, + pub ca_instance_handle_id: String, + pub publication_point_rsync_uri: String, + pub manifest_rsync_uri: String, +} + +impl RepoRequester { + pub fn with_tal_rir( + tal_id: impl Into, + rir_id: impl Into, + manifest_rsync_uri: impl Into, + publication_point_rsync_uri: impl Into, + ca_instance_handle_id: impl Into, + ) -> Self { + Self { + tal_id: tal_id.into(), + rir_id: rir_id.into(), + parent_node_id: None, + ca_instance_handle_id: ca_instance_handle_id.into(), + publication_point_rsync_uri: publication_point_rsync_uri.into(), + manifest_rsync_uri: manifest_rsync_uri.into(), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RepoSyncTask { + pub repo_key: RepoKey, + pub validation_time: time::OffsetDateTime, + pub sync_preference: SyncPreference, + pub tal_id: String, + pub rir_id: String, + pub priority: u8, + pub requesters: Vec, +} + +impl RepoSyncTask { + pub fn as_transport_task( + &self, + dedup_key: RepoDedupKey, + mode: RepoTransportMode, + ) -> RepoTransportTask { + RepoTransportTask { + dedup_key, + rsync_failure_scope_uri: None, + repo_identity: self.repo_key.as_identity(), + mode, + retry_short_timeout: false, + tal_id: self.tal_id.clone(), + rir_id: self.rir_id.clone(), + validation_time: self.validation_time, + priority: self.priority, + requesters: self.requesters.clone(), + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RepoTaskState { + Pending, + Running, + Succeeded, + Failed, + Reused, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RepoSyncResultRef { + pub repo_key: RepoKey, + pub source: String, +} + +impl RepoSyncResultRef { + pub fn as_identity(&self) -> RepoIdentity { + self.repo_key.as_identity() + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct InFlightRepoEntry { + pub state: RepoTaskState, + pub task_ref: Option, + pub waiting_requesters: Vec, + pub result_ref: Option, + pub last_result: Option, + pub last_error: Option, + pub started_at: Option, + pub finished_at: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RepoSyncResultEnvelope { + pub repo_key: RepoKey, + pub tal_id: String, + pub rir_id: String, + pub result: RepoSyncResultKind, + pub phase: Option, + pub timing_ms: u64, + pub warnings: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RepoSyncResultKind { + Success(RepoSyncResultRef), + Failed { detail: String }, + Reused(RepoSyncResultRef), +} + +fn derive_tal_id_from_url_like(s: &str) -> String { + if let Ok(url) = url::Url::parse(s) { + if let Some(last) = url + .path_segments() + .and_then(|segments| segments.filter(|seg| !seg.is_empty()).next_back()) + { + let stem = last.rsplit_once('.').map(|(stem, _)| stem).unwrap_or(last); + let trimmed = stem.trim(); + if !trimmed.is_empty() { + return trimmed.to_string(); + } + } + if let Some(host) = url.host_str() { + return host.to_string(); + } + } + "unknown-tal".to_string() +} + +fn derive_tal_id_from_path(path: &Path) -> String { + path.file_stem() + .and_then(|stem| stem.to_str()) + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .unwrap_or("unknown-tal") + .to_string() +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use crate::policy::SyncPreference; + use crate::report::Warning; + + use super::{ + RepoDedupKey, RepoIdentity, RepoKey, RepoRequester, RepoRuntimeState, RepoSyncTask, + RepoTaskState, RepoTransportMode, RepoTransportResultEnvelope, RepoTransportResultKind, + TalInputSpec, TalSource, derive_tal_id_from_path, derive_tal_id_from_url_like, + }; + + #[test] + fn tal_input_spec_from_url_derives_tal_and_rir_ids() { + let spec = TalInputSpec::from_url("https://example.test/tals/apnic.tal"); + assert_eq!(spec.tal_id, "apnic"); + assert_eq!(spec.rir_id, "apnic"); + assert_eq!( + spec.source, + TalSource::Url("https://example.test/tals/apnic.tal".to_string()) + ); + } + + #[test] + fn tal_input_spec_from_file_path_derives_file_stem() { + let spec = TalInputSpec::from_file_path("local/arin.tal"); + assert_eq!(spec.tal_id, "arin"); + assert_eq!(spec.rir_id, "arin"); + } + + #[test] + fn tal_input_spec_from_ta_der_preserves_payload() { + let spec = TalInputSpec::from_ta_der( + "https://example.test/ripe.tal", + vec![4, 5, 6], + vec![1, 2, 3], + ); + assert_eq!(spec.tal_id, "ripe"); + assert_eq!(spec.rir_id, "ripe"); + assert_eq!( + spec.source, + TalSource::DerBytes { + tal_url: "https://example.test/ripe.tal".to_string(), + tal_bytes: vec![4, 5, 6], + ta_der: vec![1, 2, 3], + } + ); + } + + #[test] + fn repo_key_equality_uses_rsync_base_and_notification() { + let a = RepoKey::new( + "rsync://example.test/repo/", + Some("https://example.test/notify.xml".to_string()), + ); + let b = RepoKey::new( + "rsync://example.test/repo/", + Some("https://example.test/notify.xml".to_string()), + ); + let c = RepoKey::new("rsync://example.test/repo/", None); + assert_eq!(a, b); + assert_ne!(a, c); + } + + #[test] + fn repo_task_state_variants_are_distinct() { + assert_ne!(RepoTaskState::Pending, RepoTaskState::Running); + assert_ne!(RepoTaskState::Succeeded, RepoTaskState::Failed); + assert_ne!(RepoTaskState::Failed, RepoTaskState::Reused); + } + + #[test] + fn repo_identity_preserves_raw_inputs() { + let ident = RepoIdentity::new( + Some("https://example.test/notify.xml".to_string()), + "rsync://example.test/repo/", + ); + assert_eq!( + ident.notification_uri.as_deref(), + Some("https://example.test/notify.xml") + ); + assert_eq!(ident.rsync_base_uri, "rsync://example.test/repo/"); + } + + #[test] + fn repo_key_can_be_viewed_as_repo_identity() { + let key = RepoKey::new( + "rsync://example.test/repo/", + Some("https://example.test/notify.xml".to_string()), + ); + let ident = key.as_identity(); + assert_eq!(ident.rsync_base_uri, "rsync://example.test/repo/"); + assert_eq!( + ident.notification_uri.as_deref(), + Some("https://example.test/notify.xml") + ); + } + + #[test] + fn repo_sync_task_maps_to_rrdp_transport_task() { + let task = RepoSyncTask { + repo_key: RepoKey::new( + "rsync://example.test/repo/", + Some("https://example.test/notify.xml".to_string()), + ), + validation_time: time::OffsetDateTime::UNIX_EPOCH, + sync_preference: SyncPreference::RrdpThenRsync, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + priority: 1, + requesters: vec![RepoRequester::with_tal_rir( + "apnic", + "apnic", + "rsync://example.test/repo/root.mft", + "rsync://example.test/repo/", + "node:1", + )], + }; + let transport = task.as_transport_task( + RepoDedupKey::RrdpNotify { + notification_uri: "https://example.test/notify.xml".to_string(), + }, + RepoTransportMode::Rrdp, + ); + assert_eq!(transport.mode, RepoTransportMode::Rrdp); + assert_eq!(transport.tal_id, "apnic"); + assert_eq!(transport.rir_id, "apnic"); + assert_eq!(transport.requesters.len(), 1); + assert_eq!( + transport.repo_identity.notification_uri.as_deref(), + Some("https://example.test/notify.xml") + ); + } + + #[test] + fn repo_transport_result_envelope_supports_success_and_failure_shapes() { + let identity = RepoIdentity::new(None, "rsync://example.test/repo/"); + let ok = RepoTransportResultEnvelope { + dedup_key: RepoDedupKey::RsyncScope { + rsync_scope_uri: "rsync://example.test/module/".to_string(), + }, + rsync_failure_scope_uri: None, + repo_identity: identity.clone(), + mode: RepoTransportMode::Rsync, + tal_id: "arin".to_string(), + rir_id: "arin".to_string(), + timing_ms: 12, + result: RepoTransportResultKind::Success { + source: "rsync".to_string(), + warnings: vec![Warning::new("ok")], + }, + }; + let fail = RepoTransportResultEnvelope { + dedup_key: RepoDedupKey::RsyncScope { + rsync_scope_uri: "rsync://example.test/module/".to_string(), + }, + rsync_failure_scope_uri: None, + repo_identity: identity, + mode: RepoTransportMode::Rsync, + tal_id: "arin".to_string(), + rir_id: "arin".to_string(), + timing_ms: 30, + result: RepoTransportResultKind::Failed { + detail: "timeout".to_string(), + warnings: vec![Warning::new("timeout")], + error_class: super::RepoTransportErrorClass::Unknown, + }, + }; + assert!(matches!(ok.result, RepoTransportResultKind::Success { .. })); + assert!(matches!( + fail.result, + RepoTransportResultKind::Failed { .. } + )); + } + + #[test] + fn repo_runtime_state_variants_are_distinct() { + assert_ne!(RepoRuntimeState::Init, RepoRuntimeState::WaitingRrdp); + assert_ne!(RepoRuntimeState::RrdpOk, RepoRuntimeState::RsyncOk); + assert_ne!( + RepoRuntimeState::RrdpFailedPendingRsync, + RepoRuntimeState::FailedTerminal + ); + } + + #[test] + fn derive_tal_id_helpers_fall_back_safely() { + assert_eq!( + derive_tal_id_from_url_like("https://example.test/path/afrinic.tal"), + "afrinic" + ); + assert_eq!( + derive_tal_id_from_path(Path::new("foo/lacnic.tal")), + "lacnic" + ); + } +} diff --git a/crates/panda-rpki-validator/src/policy.rs b/crates/panda-rpki-validator/src/policy.rs new file mode 100644 index 0000000..5ba6a93 --- /dev/null +++ b/crates/panda-rpki-validator/src/policy.rs @@ -0,0 +1,235 @@ +use serde::{Deserialize, Serialize}; + +use crate::ta_constraints::TaConstraintsByTal; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SyncPreference { + RrdpThenRsync, + RsyncOnly, +} + +impl Default for SyncPreference { + fn default() -> Self { + Self::RrdpThenRsync + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CaFailedFetchPolicy { + ReuseCurrentInstanceVcir, + StopAllOutput, +} + +impl Default for CaFailedFetchPolicy { + fn default() -> Self { + Self::ReuseCurrentInstanceVcir + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SignedObjectFailurePolicy { + DropObject, + DropPublicationPoint, +} + +impl Default for SignedObjectFailurePolicy { + fn default() -> Self { + Self::DropObject + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum ResourceValidationMode { + #[serde(rename = "validation-update-03")] + ValidationUpdate03, + #[serde(rename = "rfc6487")] + Rfc6487, +} + +impl Default for ResourceValidationMode { + fn default() -> Self { + Self::ValidationUpdate03 + } +} + +impl ResourceValidationMode { + pub fn parse_cli_value(raw: &str) -> Result { + match raw.trim() { + "validation-update-03" => Ok(Self::ValidationUpdate03), + "rfc6487" => Ok(Self::Rfc6487), + value => Err(format!( + "unknown resource validation mode: {value}; supported: validation-update-03,rfc6487" + )), + } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct StrictPolicy { + pub name: bool, + pub cms_der: bool, + pub signed_attrs: bool, +} + +impl StrictPolicy { + pub fn none() -> Self { + Self::default() + } + + pub fn all() -> Self { + Self { + name: true, + cms_der: true, + signed_attrs: true, + } + } + + pub fn parse_cli_spec(spec: Option<&str>) -> Result { + let Some(spec) = spec else { + return Ok(Self::all()); + }; + let spec = spec.trim(); + if spec.is_empty() || spec == "all" { + return Ok(Self::all()); + } + if spec == "none" { + return Ok(Self::none()); + } + + let mut out = Self::none(); + for raw in spec.split(',') { + let item = raw.trim(); + match item { + "name" => out.name = true, + "cms-der" | "cms_der" => out.cms_der = true, + "signed-attrs" | "signed_attrs" => out.signed_attrs = true, + "all" => out = Self::all(), + "none" => out = Self::none(), + "" => return Err("empty strict policy name".to_string()), + _ => { + return Err(format!( + "unknown strict policy: {item}; supported: name,cms-der,signed-attrs,all,none" + )); + } + } + } + Ok(out) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct Policy { + pub sync_preference: SyncPreference, + pub ca_failed_fetch_policy: CaFailedFetchPolicy, + pub signed_object_failure_policy: SignedObjectFailurePolicy, + pub resource_validation_mode: ResourceValidationMode, + pub strict: StrictPolicy, + /// Locally configured, per-TAL EE certificate resource constraints. They + /// are intentionally CLI/runtime-only rather than policy-file input. + #[serde(skip)] + pub ta_constraints: TaConstraintsByTal, +} + +impl Default for Policy { + fn default() -> Self { + Self { + sync_preference: SyncPreference::default(), + ca_failed_fetch_policy: CaFailedFetchPolicy::default(), + signed_object_failure_policy: SignedObjectFailurePolicy::default(), + resource_validation_mode: ResourceValidationMode::default(), + strict: StrictPolicy::default(), + ta_constraints: TaConstraintsByTal::default(), + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum PolicyParseError { + #[error("policy TOML parse error: {0}")] + Toml(String), +} + +impl Policy { + pub fn from_toml_str(s: &str) -> Result { + toml::from_str(s).map_err(|e| PolicyParseError::Toml(e.to_string())) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn strict_policy_parses_cli_specs() { + assert_eq!( + StrictPolicy::parse_cli_spec(None).unwrap(), + StrictPolicy::all() + ); + assert_eq!( + StrictPolicy::parse_cli_spec(Some("name,signed-attrs")).unwrap(), + StrictPolicy { + name: true, + cms_der: false, + signed_attrs: true, + } + ); + assert_eq!( + StrictPolicy::parse_cli_spec(Some("none")).unwrap(), + StrictPolicy::none() + ); + assert!(StrictPolicy::parse_cli_spec(Some("bogus")).is_err()); + } + + #[test] + fn resource_validation_mode_parses_cli_values() { + assert_eq!( + ResourceValidationMode::parse_cli_value("validation-update-03").unwrap(), + ResourceValidationMode::ValidationUpdate03 + ); + assert_eq!( + ResourceValidationMode::parse_cli_value("rfc6487").unwrap(), + ResourceValidationMode::Rfc6487 + ); + assert!(ResourceValidationMode::parse_cli_value("bogus").is_err()); + } + + #[test] + fn policy_toml_accepts_strict_table() { + let policy = Policy::from_toml_str( + r#" + [strict] + name = true + cms_der = true + signed_attrs = false + "#, + ) + .expect("parse policy"); + assert_eq!( + policy.strict, + StrictPolicy { + name: true, + cms_der: true, + signed_attrs: false, + } + ); + } + + #[test] + fn policy_toml_accepts_resource_validation_mode() { + let policy = Policy::from_toml_str( + r#" + resource_validation_mode = "rfc6487" + "#, + ) + .expect("parse policy"); + assert_eq!( + policy.resource_validation_mode, + ResourceValidationMode::Rfc6487 + ); + } +} diff --git a/crates/panda-rpki-validator/src/progress_log.rs b/crates/panda-rpki-validator/src/progress_log.rs new file mode 100644 index 0000000..3612082 --- /dev/null +++ b/crates/panda-rpki-validator/src/progress_log.rs @@ -0,0 +1,68 @@ +use serde_json::Value; +use time::format_description::well_known::Rfc3339; + +fn progress_enabled() -> bool { + std::env::var("RPKI_PROGRESS_LOG") + .ok() + .map(|v| matches!(v.as_str(), "1" | "true" | "TRUE" | "yes" | "YES")) + .unwrap_or(false) +} + +pub fn slow_threshold_secs() -> f64 { + std::env::var("RPKI_PROGRESS_SLOW_SECS") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|v| *v >= 0.0) + .unwrap_or(30.0) +} + +pub fn stage_fresh_slow_threshold_ms() -> u64 { + std::env::var("RPKI_PROGRESS_STAGE_FRESH_SLOW_MS") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(1_000) +} + +pub fn pp_control_slow_threshold_ms() -> u64 { + std::env::var("RPKI_PROGRESS_PP_CONTROL_SLOW_MS") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(100) +} + +pub fn pp_cache_slow_threshold_ms() -> u64 { + std::env::var("RPKI_PROGRESS_PP_CACHE_SLOW_MS") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(50) +} + +pub fn control_loop_slow_threshold_ms() -> u64 { + std::env::var("RPKI_PROGRESS_CONTROL_LOOP_SLOW_MS") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(1_000) +} + +pub fn emit(kind: &str, payload: Value) { + if !progress_enabled() { + return; + } + let ts = time::OffsetDateTime::now_utc() + .format(&Rfc3339) + .unwrap_or_else(|_| "1970-01-01T00:00:00Z".to_string()); + let mut map = serde_json::Map::new(); + map.insert("ts".to_string(), Value::String(ts)); + map.insert("kind".to_string(), Value::String(kind.to_string())); + match payload { + Value::Object(obj) => { + for (k, v) in obj { + map.insert(k, v); + } + } + other => { + map.insert("value".to_string(), other); + } + } + eprintln!("[progress] {}", Value::Object(map)); +} diff --git a/crates/panda-rpki-validator/src/replay/archive.rs b/crates/panda-rpki-validator/src/replay/archive.rs new file mode 100644 index 0000000..f33f258 --- /dev/null +++ b/crates/panda-rpki-validator/src/replay/archive.rs @@ -0,0 +1,986 @@ +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; + +use serde::Deserialize; +use sha2::Digest; + +#[derive(Debug, thiserror::Error)] +pub enum ReplayArchiveError { + #[error("read {entity} failed: {path}: {detail}")] + ReadFile { + entity: &'static str, + path: String, + detail: String, + }, + + #[error("parse {entity} JSON failed: {path}: {detail}")] + ParseJson { + entity: &'static str, + path: String, + detail: String, + }, + + #[error("unsupported {entity} version: expected 1, got {version}")] + UnsupportedVersion { entity: &'static str, version: u32 }, + + #[error("capture directory not found: {0}")] + MissingCaptureDirectory(String), + + #[error("capture.json captureId mismatch: locks={locks_capture}, capture={capture_json}")] + CaptureIdMismatch { + locks_capture: String, + capture_json: String, + }, + + #[error("RRDP lock entry invalid for {notify_uri}: {detail}")] + InvalidRrdpLock { notify_uri: String, detail: String }, + + #[error("RRDP repo bucket not found for {notify_uri}: {path}")] + MissingRrdpRepoBucket { notify_uri: String, path: String }, + + #[error("RRDP repo meta rpkiNotify mismatch: expected {expected}, actual {actual}")] + RrdpMetaMismatch { expected: String, actual: String }, + + #[error("RRDP session directory not found for {notify_uri}: {path}")] + MissingRrdpSessionDir { notify_uri: String, path: String }, + + #[error("locked notification file not found for {notify_uri}: {path}")] + MissingLockedNotification { notify_uri: String, path: String }, + + #[error("locked snapshot file not found for {notify_uri} at serial {serial} in {session_dir}")] + MissingLockedSnapshot { + notify_uri: String, + serial: u64, + session_dir: String, + }, + + #[error( + "multiple locked snapshot files found for {notify_uri} at serial {serial} in {session_dir}" + )] + AmbiguousLockedSnapshot { + notify_uri: String, + serial: u64, + session_dir: String, + }, + + #[error("rsync URI is not a valid module/base URI: {uri}: {detail}")] + InvalidRsyncUri { uri: String, detail: String }, + + #[error("rsync module bucket not found for {module_uri}: {path}")] + MissingRsyncModuleBucket { module_uri: String, path: String }, + + #[error("rsync module meta mismatch: expected {expected}, actual {actual}")] + RsyncMetaMismatch { expected: String, actual: String }, + + #[error("rsync module tree not found for {module_uri}: {path}")] + MissingRsyncTree { module_uri: String, path: String }, + + #[error("no replay lock found for RRDP notification URI: {0}")] + MissingRrdpLock(String), + + #[error("no replay lock found for rsync module: {0}")] + MissingRsyncLock(String), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ReplayTransport { + Rrdp, + Rsync, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub struct ReplayRrdpLock { + pub transport: ReplayTransport, + pub session: Option, + pub serial: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub struct ReplayRsyncLock { + pub transport: ReplayTransport, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub struct ReplayLocks { + pub version: u32, + pub capture: String, + pub rrdp: BTreeMap, + pub rsync: BTreeMap, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub struct ReplayCaptureMeta { + pub version: u32, + #[serde(rename = "captureId")] + pub capture_id: String, + #[serde(rename = "createdAt")] + pub created_at: String, + pub notes: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub struct ReplayRrdpRepoMeta { + pub version: u32, + #[serde(rename = "rpkiNotify")] + pub rpki_notify: String, + #[serde(rename = "createdAt")] + pub created_at: String, + #[serde(rename = "lastSeenAt")] + pub last_seen_at: String, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub struct ReplayRsyncModuleMeta { + pub version: u32, + pub module: String, + #[serde(rename = "createdAt")] + pub created_at: String, + #[serde(rename = "lastSeenAt")] + pub last_seen_at: String, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ReplayRrdpRepo { + pub notify_uri: String, + pub bucket_hash: String, + pub bucket_dir: PathBuf, + pub meta: ReplayRrdpRepoMeta, + pub locked_session: String, + pub locked_serial: u64, + pub session_dir: PathBuf, + pub locked_notification_path: PathBuf, + pub locked_snapshot_path: PathBuf, + pub available_delta_paths: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ReplayRsyncModule { + pub module_uri: String, + pub bucket_hash: String, + pub bucket_dir: PathBuf, + pub meta: ReplayRsyncModuleMeta, + pub tree_dir: PathBuf, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ReplayArchiveIndex { + pub archive_root: PathBuf, + pub capture_root: PathBuf, + pub locks_path: PathBuf, + pub locks: ReplayLocks, + pub capture_meta: ReplayCaptureMeta, + pub rrdp_repos: BTreeMap, + pub rsync_modules: BTreeMap, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ReplayArchiveLoadMode { + Strict, + AllowMissingRsyncModules, +} + +impl ReplayArchiveIndex { + pub fn load( + archive_root: impl AsRef, + locks_path: impl AsRef, + ) -> Result { + Self::load_with_mode(archive_root, locks_path, ReplayArchiveLoadMode::Strict) + } + + pub fn load_allow_missing_rsync_modules( + archive_root: impl AsRef, + locks_path: impl AsRef, + ) -> Result { + Self::load_with_mode( + archive_root, + locks_path, + ReplayArchiveLoadMode::AllowMissingRsyncModules, + ) + } + + fn load_with_mode( + archive_root: impl AsRef, + locks_path: impl AsRef, + load_mode: ReplayArchiveLoadMode, + ) -> Result { + let archive_root = archive_root.as_ref().to_path_buf(); + let locks_path = locks_path.as_ref().to_path_buf(); + + let locks: ReplayLocks = read_json_file(&locks_path, "payload replay locks")?; + ensure_version("payload replay locks", locks.version)?; + + let capture_root = archive_root + .join("v1") + .join("captures") + .join(&locks.capture); + if !capture_root.is_dir() { + return Err(ReplayArchiveError::MissingCaptureDirectory( + capture_root.display().to_string(), + )); + } + + let capture_meta_path = capture_root.join("capture.json"); + let capture_meta: ReplayCaptureMeta = read_json_file(&capture_meta_path, "capture meta")?; + ensure_version("capture meta", capture_meta.version)?; + if capture_meta.capture_id != locks.capture { + return Err(ReplayArchiveError::CaptureIdMismatch { + locks_capture: locks.capture.clone(), + capture_json: capture_meta.capture_id.clone(), + }); + } + + let mut rrdp_repos = BTreeMap::new(); + for (notify_uri, lock) in &locks.rrdp { + match lock.transport { + ReplayTransport::Rrdp => { + let repo = load_rrdp_repo(&capture_root, notify_uri, lock)?; + rrdp_repos.insert(notify_uri.clone(), repo); + } + ReplayTransport::Rsync => { + validate_rsync_transport_lock(notify_uri, lock)?; + } + } + } + + let mut rsync_modules = BTreeMap::new(); + for (module_uri, lock) in &locks.rsync { + if lock.transport != ReplayTransport::Rsync { + return Err(ReplayArchiveError::InvalidRrdpLock { + notify_uri: module_uri.clone(), + detail: "rsync lock transport must be rsync".to_string(), + }); + } + match load_rsync_module(&capture_root, module_uri) { + Ok(module) => { + rsync_modules.insert(module.module_uri.clone(), module); + } + Err(ReplayArchiveError::MissingRsyncModuleBucket { .. }) + if load_mode == ReplayArchiveLoadMode::AllowMissingRsyncModules => {} + Err(err) => return Err(err), + } + } + + Ok(Self { + archive_root, + capture_root, + locks_path, + locks, + capture_meta, + rrdp_repos, + rsync_modules, + }) + } + + pub fn rrdp_lock(&self, notify_uri: &str) -> Option<&ReplayRrdpLock> { + self.locks.rrdp.get(notify_uri) + } + + pub fn rrdp_repo(&self, notify_uri: &str) -> Option<&ReplayRrdpRepo> { + self.rrdp_repos.get(notify_uri) + } + + pub fn require_rrdp_repo( + &self, + notify_uri: &str, + ) -> Result<&ReplayRrdpRepo, ReplayArchiveError> { + self.rrdp_repos + .get(notify_uri) + .ok_or_else(|| ReplayArchiveError::MissingRrdpLock(notify_uri.to_string())) + } + + pub fn rsync_module(&self, module_uri: &str) -> Option<&ReplayRsyncModule> { + self.rsync_modules.get(module_uri) + } + + pub fn resolve_rsync_module_for_base_uri( + &self, + rsync_base_uri: &str, + ) -> Result<&ReplayRsyncModule, ReplayArchiveError> { + let module_uri = canonical_rsync_module(rsync_base_uri)?; + self.rsync_modules + .get(&module_uri) + .ok_or(ReplayArchiveError::MissingRsyncLock(module_uri)) + } +} + +fn load_rrdp_repo( + capture_root: &Path, + notify_uri: &str, + lock: &ReplayRrdpLock, +) -> Result { + let session = lock + .session + .as_deref() + .ok_or_else(|| ReplayArchiveError::InvalidRrdpLock { + notify_uri: notify_uri.to_string(), + detail: "transport=rrdp requires non-null session".to_string(), + })?; + let serial = lock + .serial + .ok_or_else(|| ReplayArchiveError::InvalidRrdpLock { + notify_uri: notify_uri.to_string(), + detail: "transport=rrdp requires non-null serial".to_string(), + })?; + + let bucket_hash = sha256_hex(notify_uri.as_bytes()); + let bucket_dir = capture_root.join("rrdp").join("repos").join(&bucket_hash); + if !bucket_dir.is_dir() { + return Err(ReplayArchiveError::MissingRrdpRepoBucket { + notify_uri: notify_uri.to_string(), + path: bucket_dir.display().to_string(), + }); + } + + let meta_path = bucket_dir.join("meta.json"); + let meta: ReplayRrdpRepoMeta = read_json_file(&meta_path, "RRDP repo meta")?; + ensure_version("RRDP repo meta", meta.version)?; + if meta.rpki_notify != notify_uri { + return Err(ReplayArchiveError::RrdpMetaMismatch { + expected: notify_uri.to_string(), + actual: meta.rpki_notify.clone(), + }); + } + + let session_dir = bucket_dir.join(session); + if !session_dir.is_dir() { + return Err(ReplayArchiveError::MissingRrdpSessionDir { + notify_uri: notify_uri.to_string(), + path: session_dir.display().to_string(), + }); + } + + let locked_notification_path = session_dir.join(format!("notification-{serial}.xml")); + if !locked_notification_path.is_file() { + return Err(ReplayArchiveError::MissingLockedNotification { + notify_uri: notify_uri.to_string(), + path: locked_notification_path.display().to_string(), + }); + } + + let mut snapshot_candidates: Vec = fs::read_dir(&session_dir) + .map_err(|e| ReplayArchiveError::ReadFile { + entity: "RRDP session directory", + path: session_dir.display().to_string(), + detail: e.to_string(), + })? + .filter_map(|entry| entry.ok().map(|e| e.path())) + .filter(|path| path.is_file()) + .filter(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| { + name.starts_with(&format!("snapshot-{serial}-")) && name.ends_with(".xml") + }) + }) + .collect(); + snapshot_candidates.sort(); + let locked_snapshot_path = match snapshot_candidates.len() { + 0 => { + return Err(ReplayArchiveError::MissingLockedSnapshot { + notify_uri: notify_uri.to_string(), + serial, + session_dir: session_dir.display().to_string(), + }); + } + 1 => snapshot_candidates.remove(0), + _ => { + return Err(ReplayArchiveError::AmbiguousLockedSnapshot { + notify_uri: notify_uri.to_string(), + serial, + session_dir: session_dir.display().to_string(), + }); + } + }; + + let mut available_delta_paths: Vec = fs::read_dir(&session_dir) + .map_err(|e| ReplayArchiveError::ReadFile { + entity: "RRDP session directory", + path: session_dir.display().to_string(), + detail: e.to_string(), + })? + .filter_map(|entry| entry.ok().map(|e| e.path())) + .filter(|path| path.is_file()) + .filter(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with("delta-") && name.ends_with(".xml")) + }) + .collect(); + available_delta_paths.sort(); + + Ok(ReplayRrdpRepo { + notify_uri: notify_uri.to_string(), + bucket_hash, + bucket_dir, + meta, + locked_session: session.to_string(), + locked_serial: serial, + session_dir, + locked_notification_path, + locked_snapshot_path, + available_delta_paths, + }) +} + +fn validate_rsync_transport_lock( + notify_uri: &str, + lock: &ReplayRrdpLock, +) -> Result<(), ReplayArchiveError> { + if lock.session.is_some() || lock.serial.is_some() { + return Err(ReplayArchiveError::InvalidRrdpLock { + notify_uri: notify_uri.to_string(), + detail: "transport=rsync requires null session and serial".to_string(), + }); + } + Ok(()) +} + +fn load_rsync_module( + capture_root: &Path, + module_uri: &str, +) -> Result { + let canonical_module = canonical_rsync_module(module_uri)?; + let bucket_hash = sha256_hex(canonical_module.as_bytes()); + let bucket_dir = capture_root + .join("rsync") + .join("modules") + .join(&bucket_hash); + if !bucket_dir.is_dir() { + return Err(ReplayArchiveError::MissingRsyncModuleBucket { + module_uri: canonical_module.clone(), + path: bucket_dir.display().to_string(), + }); + } + + let meta_path = bucket_dir.join("meta.json"); + let meta: ReplayRsyncModuleMeta = if meta_path.is_file() { + let meta: ReplayRsyncModuleMeta = read_json_file(&meta_path, "rsync module meta")?; + ensure_version("rsync module meta", meta.version)?; + if meta.module != canonical_module { + return Err(ReplayArchiveError::RsyncMetaMismatch { + expected: canonical_module.clone(), + actual: meta.module.clone(), + }); + } + meta + } else { + ReplayRsyncModuleMeta { + version: 1, + module: canonical_module.clone(), + created_at: String::new(), + last_seen_at: String::new(), + } + }; + + let tree_dir = bucket_dir.join("tree"); + if !tree_dir.is_dir() { + return Err(ReplayArchiveError::MissingRsyncTree { + module_uri: canonical_module.clone(), + path: tree_dir.display().to_string(), + }); + } + + Ok(ReplayRsyncModule { + module_uri: canonical_module, + bucket_hash, + bucket_dir, + meta, + tree_dir, + }) +} + +pub fn canonical_rsync_module(rsync_uri: &str) -> Result { + let raw = rsync_uri.trim(); + let prefix = "rsync://"; + let rest = raw + .strip_prefix(prefix) + .ok_or_else(|| ReplayArchiveError::InvalidRsyncUri { + uri: rsync_uri.to_string(), + detail: "URI must start with rsync://".to_string(), + })?; + + let mut parts = rest.split('/'); + let authority = parts.next().unwrap_or_default(); + let module = parts.next().unwrap_or_default(); + if authority.is_empty() || module.is_empty() { + return Err(ReplayArchiveError::InvalidRsyncUri { + uri: rsync_uri.to_string(), + detail: "URI must contain authority and module".to_string(), + }); + } + + Ok(format!("rsync://{authority}/{module}/")) +} + +pub fn sha256_hex(bytes: &[u8]) -> String { + let digest = sha2::Sha256::digest(bytes); + hex::encode(digest) +} + +fn ensure_version(entity: &'static str, version: u32) -> Result<(), ReplayArchiveError> { + if version == 1 { + Ok(()) + } else { + Err(ReplayArchiveError::UnsupportedVersion { entity, version }) + } +} + +fn read_json_file Deserialize<'de>>( + path: &Path, + entity: &'static str, +) -> Result { + let bytes = fs::read(path).map_err(|e| ReplayArchiveError::ReadFile { + entity, + path: path.display().to_string(), + detail: e.to_string(), + })?; + serde_json::from_slice(&bytes).map_err(|e| ReplayArchiveError::ParseJson { + entity, + path: path.display().to_string(), + detail: e.to_string(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn build_minimal_archive() -> (tempfile::TempDir, PathBuf, PathBuf, String, String) { + let temp = tempfile::tempdir().expect("tempdir"); + let archive_root = temp.path().join("payload-archive"); + let capture = "capture-001"; + let capture_root = archive_root.join("v1").join("captures").join(capture); + std::fs::create_dir_all(&capture_root).expect("mkdir capture root"); + std::fs::write( + capture_root.join("capture.json"), + format!( + r#"{{"version":1,"captureId":"{capture}","createdAt":"2026-03-13T00:00:00Z","notes":""}}"# + ), + ) + .expect("write capture json"); + + let notify_uri = "https://rrdp.example.test/notification.xml".to_string(); + let session = "11111111-1111-1111-1111-111111111111".to_string(); + let serial = 42u64; + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let repo_dir = capture_root.join("rrdp").join("repos").join(&repo_hash); + let session_dir = repo_dir.join(&session); + std::fs::create_dir_all(&session_dir).expect("mkdir session dir"); + std::fs::write( + repo_dir.join("meta.json"), + format!( + r#"{{"version":1,"rpkiNotify":"{notify_uri}","createdAt":"2026-03-13T00:00:00Z","lastSeenAt":"2026-03-13T00:00:01Z"}}"# + ), + ) + .expect("write repo meta"); + std::fs::write(session_dir.join("notification-42.xml"), b"") + .expect("write notification"); + std::fs::write(session_dir.join("snapshot-42-deadbeef.xml"), b"") + .expect("write snapshot"); + + let module_uri = "rsync://rsync.example.test/repo/".to_string(); + let mod_hash = sha256_hex(module_uri.as_bytes()); + let mod_dir = capture_root.join("rsync").join("modules").join(&mod_hash); + let tree_dir = mod_dir.join("tree").join("rsync.example.test").join("repo"); + std::fs::create_dir_all(&tree_dir).expect("mkdir tree dir"); + std::fs::write( + mod_dir.join("meta.json"), + format!( + r#"{{"version":1,"module":"{module_uri}","createdAt":"2026-03-13T00:00:00Z","lastSeenAt":"2026-03-13T00:00:01Z"}}"# + ), + ) + .expect("write rsync meta"); + std::fs::write(tree_dir.join("a.roa"), b"roa").expect("write rsync object"); + + let locks_path = temp.path().join("locks.json"); + std::fs::write( + &locks_path, + format!( + r#"{{ + "version":1, + "capture":"{capture}", + "rrdp":{{ + "{notify_uri}":{{"transport":"rrdp","session":"{session}","serial":{serial}}}, + "https://rrdp-fallback.example.test/notification.xml":{{"transport":"rsync","session":null,"serial":null}} + }}, + "rsync":{{ + "{module_uri}":{{"transport":"rsync"}} + }} +}}"# + ), + ) + .expect("write locks"); + + (temp, archive_root, locks_path, notify_uri, module_uri) + } + + #[test] + fn canonical_rsync_module_normalizes_subpaths_and_trailing_slash() { + let normalized = canonical_rsync_module("rsync://example.test/repo/sub/dir/file.roa") + .expect("normalize module"); + assert_eq!(normalized, "rsync://example.test/repo/"); + + let normalized = canonical_rsync_module("rsync://example.test/repo") + .expect("normalize module without slash"); + assert_eq!(normalized, "rsync://example.test/repo/"); + } + + #[test] + fn replay_archive_index_loads_rrdp_and_rsync_entries() { + let (_temp, archive_root, locks_path, notify_uri, module_uri) = build_minimal_archive(); + let index = + ReplayArchiveIndex::load(&archive_root, &locks_path).expect("load replay index"); + + assert_eq!(index.capture_meta.capture_id, "capture-001"); + assert_eq!(index.rrdp_repos.len(), 1); + assert_eq!(index.locks.rrdp.len(), 2); + assert_eq!(index.rsync_modules.len(), 1); + + let repo = index.require_rrdp_repo(¬ify_uri).expect("rrdp repo"); + assert_eq!(repo.locked_serial, 42); + assert!( + repo.locked_notification_path + .ends_with("notification-42.xml") + ); + assert!( + repo.locked_snapshot_path + .ends_with("snapshot-42-deadbeef.xml") + ); + assert!(repo.available_delta_paths.is_empty()); + + let module = index + .resolve_rsync_module_for_base_uri("rsync://rsync.example.test/repo/sub/path") + .expect("resolve module from base"); + assert_eq!(module.module_uri, module_uri); + assert!( + module + .tree_dir + .join("rsync.example.test") + .join("repo") + .join("a.roa") + .is_file() + ); + } + + #[test] + fn replay_archive_index_rejects_rsync_transport_with_session_or_serial() { + let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_minimal_archive(); + let bad = locks_path.with_file_name("bad-locks.json"); + std::fs::write( + &bad, + r#"{ + "version":1, + "capture":"capture-001", + "rrdp":{ + "https://rrdp-fallback.example.test/notification.xml":{"transport":"rsync","session":"oops","serial":1} + }, + "rsync":{} +}"#, + ) + .expect("write bad locks"); + + let err = ReplayArchiveIndex::load(&archive_root, &bad).unwrap_err(); + assert!( + matches!(err, ReplayArchiveError::InvalidRrdpLock { .. }), + "{err}" + ); + } + + #[test] + fn replay_archive_index_rejects_mismatched_rrdp_meta() { + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_minimal_archive(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let meta_path = archive_root + .join("v1/captures/capture-001/rrdp/repos") + .join(repo_hash) + .join("meta.json"); + std::fs::write( + &meta_path, + r#"{"version":1,"rpkiNotify":"https://other.example/notification.xml","createdAt":"2026-03-13T00:00:00Z","lastSeenAt":"2026-03-13T00:00:01Z"}"#, + ) + .expect("rewrite meta"); + + let err = ReplayArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayArchiveError::RrdpMetaMismatch { .. }), + "{err}" + ); + } + + #[test] + fn replay_archive_index_rejects_missing_snapshot() { + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_minimal_archive(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let session_dir = archive_root + .join("v1/captures/capture-001/rrdp/repos") + .join(repo_hash) + .join("11111111-1111-1111-1111-111111111111"); + std::fs::remove_file(session_dir.join("snapshot-42-deadbeef.xml")) + .expect("remove snapshot"); + + let err = ReplayArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayArchiveError::MissingLockedSnapshot { .. }), + "{err}" + ); + } + + #[test] + fn replay_archive_index_rejects_ambiguous_snapshot_candidates() { + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_minimal_archive(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let session_dir = archive_root + .join("v1/captures/capture-001/rrdp/repos") + .join(repo_hash) + .join("11111111-1111-1111-1111-111111111111"); + std::fs::write(session_dir.join("snapshot-42-another.xml"), b"") + .expect("write second snapshot"); + + let err = ReplayArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayArchiveError::AmbiguousLockedSnapshot { .. }), + "{err}" + ); + } + + #[test] + fn canonical_rsync_module_rejects_invalid_uris() { + let err = canonical_rsync_module("https://example.test/repo/").unwrap_err(); + assert!( + matches!(err, ReplayArchiveError::InvalidRsyncUri { .. }), + "{err}" + ); + + let err = canonical_rsync_module("rsync://example.test").unwrap_err(); + assert!( + matches!(err, ReplayArchiveError::InvalidRsyncUri { .. }), + "{err}" + ); + } + + #[test] + fn replay_archive_index_rejects_missing_capture_directory() { + let temp = tempfile::tempdir().expect("tempdir"); + let archive_root = temp.path().join("payload-archive"); + std::fs::create_dir_all(&archive_root).expect("mkdir archive root"); + let locks_path = temp.path().join("locks.json"); + std::fs::write( + &locks_path, + r#"{"version":1,"capture":"missing","rrdp":{},"rsync":{}}"#, + ) + .expect("write locks"); + + let err = ReplayArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayArchiveError::MissingCaptureDirectory(_)), + "{err}" + ); + } + + #[test] + fn replay_archive_index_rejects_missing_rrdp_bucket_and_session_and_notification() { + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_minimal_archive(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let repo_dir = archive_root + .join("v1/captures/capture-001/rrdp/repos") + .join(&repo_hash); + + std::fs::remove_dir_all(&repo_dir).expect("remove repo dir"); + let err = ReplayArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayArchiveError::MissingRrdpRepoBucket { .. }), + "{err}" + ); + + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_minimal_archive(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let session_dir = archive_root + .join("v1/captures/capture-001/rrdp/repos") + .join(&repo_hash) + .join("11111111-1111-1111-1111-111111111111"); + std::fs::remove_dir_all(&session_dir).expect("remove session dir"); + let err = ReplayArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayArchiveError::MissingRrdpSessionDir { .. }), + "{err}" + ); + + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_minimal_archive(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let session_dir = archive_root + .join("v1/captures/capture-001/rrdp/repos") + .join(&repo_hash) + .join("11111111-1111-1111-1111-111111111111"); + std::fs::remove_file(session_dir.join("notification-42.xml")).expect("remove notification"); + let err = ReplayArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayArchiveError::MissingLockedNotification { .. }), + "{err}" + ); + } + + #[test] + fn replay_archive_index_rejects_invalid_json_and_missing_rsync_bucket() { + let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_minimal_archive(); + std::fs::write(&locks_path, b"not json").expect("corrupt locks"); + let err = ReplayArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!( + err, + ReplayArchiveError::ParseJson { + entity: "payload replay locks", + .. + } + ), + "{err}" + ); + + let (_temp, archive_root, locks_path, _notify_uri, module_uri) = build_minimal_archive(); + let mod_hash = sha256_hex(module_uri.as_bytes()); + let mod_dir = archive_root + .join("v1/captures/capture-001/rsync/modules") + .join(&mod_hash); + std::fs::remove_dir_all(&mod_dir).expect("remove module dir"); + let err = ReplayArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayArchiveError::MissingRsyncModuleBucket { .. }), + "{err}" + ); + } + + #[test] + fn replay_archive_index_can_skip_missing_rsync_modules_in_lenient_mode() { + let (_temp, archive_root, locks_path, _notify_uri, module_uri) = build_minimal_archive(); + let mod_hash = sha256_hex(module_uri.as_bytes()); + let mod_dir = archive_root + .join("v1/captures/capture-001/rsync/modules") + .join(mod_hash); + std::fs::remove_dir_all(&mod_dir).expect("remove module dir"); + + let index = + ReplayArchiveIndex::load_allow_missing_rsync_modules(&archive_root, &locks_path) + .expect("load lenient replay index"); + assert!(index.rsync_modules.is_empty()); + } + #[test] + fn replay_archive_index_accepts_missing_rsync_module_meta_when_tree_exists() { + let (_temp, archive_root, locks_path, _notify_uri, module_uri) = build_minimal_archive(); + let mod_hash = sha256_hex(module_uri.as_bytes()); + let meta_path = archive_root + .join("v1/captures/capture-001/rsync/modules") + .join(mod_hash) + .join("meta.json"); + std::fs::remove_file(&meta_path).expect("remove rsync module meta"); + + let index = + ReplayArchiveIndex::load_allow_missing_rsync_modules(&archive_root, &locks_path) + .expect("load replay index without rsync meta"); + let module = index + .rsync_modules + .get(&module_uri) + .expect("module present"); + assert_eq!(module.meta.module, module_uri); + assert_eq!(module.meta.version, 1); + } + + #[test] + fn replay_archive_index_rejects_capture_id_mismatch() { + let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_minimal_archive(); + let capture_json = archive_root.join("v1/captures/capture-001/capture.json"); + std::fs::write( + &capture_json, + r#"{"version":1,"captureId":"other-capture","createdAt":"2026-03-13T00:00:00Z","notes":""}"#, + ) + .expect("rewrite capture json"); + + let err = ReplayArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayArchiveError::CaptureIdMismatch { .. }), + "{err}" + ); + } + + #[test] + fn replay_archive_index_rejects_unsupported_locks_version() { + let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_minimal_archive(); + std::fs::write( + &locks_path, + r#"{"version":2,"capture":"capture-001","rrdp":{},"rsync":{}}"#, + ) + .expect("rewrite locks version"); + + let err = ReplayArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!( + err, + ReplayArchiveError::UnsupportedVersion { + entity: "payload replay locks", + version: 2 + } + ), + "{err}" + ); + } + + #[test] + fn replay_archive_index_reports_missing_rrdp_lock_and_missing_rsync_lock() { + let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_minimal_archive(); + let index = + ReplayArchiveIndex::load(&archive_root, &locks_path).expect("load replay index"); + + let err = index + .require_rrdp_repo("https://missing.example/notification.xml") + .unwrap_err(); + assert!( + matches!(err, ReplayArchiveError::MissingRrdpLock(_)), + "{err}" + ); + + let err = index + .resolve_rsync_module_for_base_uri("rsync://missing.example/repo/path") + .unwrap_err(); + assert!( + matches!(err, ReplayArchiveError::MissingRsyncLock(_)), + "{err}" + ); + } + + #[test] + fn replay_archive_index_rejects_rsync_meta_mismatch() { + let (_temp, archive_root, locks_path, _notify_uri, module_uri) = build_minimal_archive(); + let mod_hash = sha256_hex(module_uri.as_bytes()); + let meta_path = archive_root + .join("v1/captures/capture-001/rsync/modules") + .join(mod_hash) + .join("meta.json"); + std::fs::write( + &meta_path, + r#"{"version":1,"module":"rsync://other.example/repo/","createdAt":"2026-03-13T00:00:00Z","lastSeenAt":"2026-03-13T00:00:01Z"}"#, + ) + .expect("rewrite rsync meta"); + + let err = ReplayArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayArchiveError::RsyncMetaMismatch { .. }), + "{err}" + ); + } + #[test] + fn replay_archive_index_rejects_missing_rsync_module_tree() { + let (_temp, archive_root, locks_path, _notify_uri, module_uri) = build_minimal_archive(); + let mod_hash = sha256_hex(module_uri.as_bytes()); + let tree_dir = archive_root + .join("v1/captures/capture-001/rsync/modules") + .join(mod_hash) + .join("tree"); + std::fs::remove_dir_all(&tree_dir).expect("remove tree dir"); + + let err = ReplayArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayArchiveError::MissingRsyncTree { .. }), + "{err}" + ); + } +} diff --git a/crates/panda-rpki-validator/src/replay/delta_archive.rs b/crates/panda-rpki-validator/src/replay/delta_archive.rs new file mode 100644 index 0000000..cac5d4f --- /dev/null +++ b/crates/panda-rpki-validator/src/replay/delta_archive.rs @@ -0,0 +1,1161 @@ +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; + +use serde::Deserialize; + +use crate::replay::archive::{ + ReplayArchiveError, ReplayRrdpRepoMeta, ReplayRsyncModuleMeta, ReplayTransport, + canonical_rsync_module, sha256_hex, +}; + +#[derive(Debug, thiserror::Error)] +pub enum ReplayDeltaArchiveError { + #[error(transparent)] + Base(#[from] ReplayArchiveError), + + #[error("delta capture directory not found: {0}")] + MissingDeltaCaptureDirectory(String), + + #[error("delta capture.json captureId mismatch: locks={locks_capture}, capture={capture_json}")] + CaptureIdMismatch { + locks_capture: String, + capture_json: String, + }, + + #[error( + "delta base.json baseCapture mismatch: locks={locks_base_capture}, base_json={base_json_base_capture}" + )] + BaseCaptureMismatch { + locks_base_capture: String, + base_json_base_capture: String, + }, + + #[error( + "delta base.json baseLocksSha256 mismatch: locks={locks_sha256}, base_json={base_json_sha256}" + )] + BaseLocksShaMismatch { + locks_sha256: String, + base_json_sha256: String, + }, + + #[error("base locks sha256 mismatch: expected {expected}, actual {actual}")] + BaseLocksBytesShaMismatch { expected: String, actual: String }, + + #[error("delta repo bucket not found for {notify_uri}: {path}")] + MissingDeltaRepoBucket { notify_uri: String, path: String }, + + #[error("delta repo meta mismatch: expected {expected}, actual {actual}")] + RrdpMetaMismatch { expected: String, actual: String }, + + #[error( + "delta transition kind mismatch for {notify_uri}: locks={locks_kind}, transition={transition_kind}" + )] + TransitionKindMismatch { + notify_uri: String, + locks_kind: String, + transition_kind: String, + }, + + #[error("delta transition base mismatch for {notify_uri}")] + TransitionBaseMismatch { notify_uri: String }, + + #[error("delta transition target mismatch for {notify_uri}")] + TransitionTargetMismatch { notify_uri: String }, + + #[error("delta serial list mismatch for {notify_uri}")] + DeltaSerialListMismatch { notify_uri: String }, + + #[error("delta notification session directory not found for {notify_uri}: {path}")] + MissingDeltaSessionDir { notify_uri: String, path: String }, + + #[error("target notification file not found for {notify_uri}: {path}")] + MissingTargetNotification { notify_uri: String, path: String }, + + #[error("delta file not found for {notify_uri} serial={serial}: {path}")] + MissingDeltaFile { + notify_uri: String, + serial: u64, + path: String, + }, + + #[error("delta target archive missing for {notify_uri}: {path}")] + MissingTargetArchive { notify_uri: String, path: String }, + + #[error("delta rsync module bucket not found for {module_uri}: {path}")] + MissingRsyncModuleBucket { module_uri: String, path: String }, + + #[error("delta rsync module meta mismatch: expected {expected}, actual {actual}")] + RsyncMetaMismatch { expected: String, actual: String }, + + #[error("delta rsync files.json module mismatch: expected {expected}, actual {actual}")] + RsyncFilesModuleMismatch { expected: String, actual: String }, + + #[error( + "delta rsync file count mismatch for {module_uri}: declared={declared}, actual={actual}" + )] + RsyncFileCountMismatch { + module_uri: String, + declared: usize, + actual: usize, + }, + + #[error("delta rsync overlay file not found for {module_uri}: {path}")] + MissingRsyncOverlayFile { module_uri: String, path: String }, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub struct ReplayDeltaLocks { + pub version: u32, + pub capture: String, + #[serde(rename = "baseCapture")] + pub base_capture: String, + #[serde(rename = "baseLocksSha256")] + pub base_locks_sha256: String, + pub rrdp: BTreeMap, + pub rsync: BTreeMap, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub struct ReplayDeltaBaseMeta { + pub version: u32, + #[serde(rename = "baseCapture")] + pub base_capture: String, + #[serde(rename = "baseLocksSha256")] + pub base_locks_sha256: String, + #[serde(rename = "createdAt")] + pub created_at: String, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ReplayDeltaRrdpKind { + Unchanged, + Delta, + FallbackRsync, + SessionReset, + Gap, +} + +impl ReplayDeltaRrdpKind { + pub fn as_str(self) -> &'static str { + match self { + Self::Unchanged => "unchanged", + Self::Delta => "delta", + Self::FallbackRsync => "fallback-rsync", + Self::SessionReset => "session-reset", + Self::Gap => "gap", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub struct ReplayDeltaRrdpState { + pub transport: ReplayTransport, + pub session: Option, + pub serial: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub struct ReplayDeltaRrdpEntry { + pub kind: ReplayDeltaRrdpKind, + pub base: ReplayDeltaRrdpState, + pub target: ReplayDeltaRrdpState, + #[serde(rename = "delta_count")] + pub delta_count: usize, + pub deltas: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub struct ReplayDeltaRsyncEntry { + #[serde(rename = "file_count")] + pub file_count: usize, + #[serde(rename = "overlay_only")] + pub overlay_only: bool, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub struct ReplayDeltaTransition { + pub kind: ReplayDeltaRrdpKind, + pub base: ReplayDeltaRrdpState, + pub target: ReplayDeltaRrdpState, + #[serde(rename = "delta_count")] + pub delta_count: usize, + pub deltas: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Deserialize)] +pub struct ReplayDeltaRsyncFiles { + pub version: u32, + pub module: String, + #[serde(rename = "fileCount")] + pub file_count: usize, + pub files: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ReplayDeltaRrdpRepo { + pub notify_uri: String, + pub bucket_hash: String, + pub bucket_dir: PathBuf, + pub meta: ReplayRrdpRepoMeta, + pub transition: ReplayDeltaTransition, + pub target_notification_path: Option, + pub delta_paths: Vec<(u64, PathBuf)>, + pub target_archive_path: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ReplayDeltaRsyncModule { + pub module_uri: String, + pub bucket_hash: String, + pub bucket_dir: PathBuf, + pub meta: ReplayRsyncModuleMeta, + pub overlay_only: bool, + pub files: ReplayDeltaRsyncFiles, + pub tree_dir: PathBuf, + pub overlay_files: Vec<(String, PathBuf)>, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ReplayDeltaArchiveIndex { + pub archive_root: PathBuf, + pub capture_root: PathBuf, + pub delta_locks_path: PathBuf, + pub delta_locks: ReplayDeltaLocks, + pub capture_meta: crate::replay::archive::ReplayCaptureMeta, + pub base_meta: ReplayDeltaBaseMeta, + pub rrdp_repos: BTreeMap, + pub rsync_modules: BTreeMap, +} + +impl ReplayDeltaArchiveIndex { + pub fn load( + delta_archive_root: impl AsRef, + delta_locks_path: impl AsRef, + ) -> Result { + let archive_root = delta_archive_root.as_ref().to_path_buf(); + let delta_locks_path = delta_locks_path.as_ref().to_path_buf(); + + let delta_locks: ReplayDeltaLocks = + read_delta_json_file(&delta_locks_path, "payload delta locks")?; + ensure_delta_version("payload delta locks", delta_locks.version)?; + + let capture_root = archive_root + .join("v1") + .join("captures") + .join(&delta_locks.capture); + if !capture_root.is_dir() { + return Err(ReplayDeltaArchiveError::MissingDeltaCaptureDirectory( + capture_root.display().to_string(), + )); + } + + let capture_meta: crate::replay::archive::ReplayCaptureMeta = + read_delta_json_file(&capture_root.join("capture.json"), "delta capture meta")?; + ensure_delta_version("delta capture meta", capture_meta.version)?; + if capture_meta.capture_id != delta_locks.capture { + return Err(ReplayDeltaArchiveError::CaptureIdMismatch { + locks_capture: delta_locks.capture.clone(), + capture_json: capture_meta.capture_id.clone(), + }); + } + + let base_meta: ReplayDeltaBaseMeta = + read_delta_json_file(&capture_root.join("base.json"), "delta base meta")?; + ensure_delta_version("delta base meta", base_meta.version)?; + if base_meta.base_capture != delta_locks.base_capture { + return Err(ReplayDeltaArchiveError::BaseCaptureMismatch { + locks_base_capture: delta_locks.base_capture.clone(), + base_json_base_capture: base_meta.base_capture.clone(), + }); + } + if base_meta.base_locks_sha256.to_ascii_lowercase() + != delta_locks.base_locks_sha256.to_ascii_lowercase() + { + return Err(ReplayDeltaArchiveError::BaseLocksShaMismatch { + locks_sha256: delta_locks.base_locks_sha256.clone(), + base_json_sha256: base_meta.base_locks_sha256.clone(), + }); + } + + let mut rrdp_repos = BTreeMap::new(); + for (notify_uri, entry) in &delta_locks.rrdp { + let repo = load_delta_rrdp_repo(&capture_root, notify_uri, entry)?; + rrdp_repos.insert(notify_uri.clone(), repo); + } + + let mut rsync_modules = BTreeMap::new(); + for (module_uri, entry) in &delta_locks.rsync { + let module = load_delta_rsync_module(&capture_root, module_uri, entry)?; + rsync_modules.insert(module.module_uri.clone(), module); + } + + Ok(Self { + archive_root, + capture_root, + delta_locks_path, + delta_locks, + capture_meta, + base_meta, + rrdp_repos, + rsync_modules, + }) + } + + pub fn rrdp_repo(&self, notify_uri: &str) -> Option<&ReplayDeltaRrdpRepo> { + self.rrdp_repos.get(notify_uri) + } + + pub fn rsync_module(&self, module_uri: &str) -> Option<&ReplayDeltaRsyncModule> { + self.rsync_modules.get(module_uri) + } + + pub fn resolve_rsync_module_for_base_uri( + &self, + rsync_base_uri: &str, + ) -> Result<&ReplayDeltaRsyncModule, ReplayDeltaArchiveError> { + let module_uri = + canonical_rsync_module(rsync_base_uri).map_err(ReplayDeltaArchiveError::Base)?; + self.rsync_modules.get(&module_uri).ok_or_else(|| { + ReplayDeltaArchiveError::MissingRsyncModuleBucket { + module_uri, + path: "".to_string(), + } + }) + } + + pub fn validate_base_locks_sha256_bytes( + &self, + base_locks_bytes: &[u8], + ) -> Result<(), ReplayDeltaArchiveError> { + let actual = sha256_hex(base_locks_bytes); + let expected = self.delta_locks.base_locks_sha256.to_ascii_lowercase(); + if actual != expected { + return Err(ReplayDeltaArchiveError::BaseLocksBytesShaMismatch { expected, actual }); + } + Ok(()) + } + + pub fn validate_base_locks_sha256_file( + &self, + path: &Path, + ) -> Result<(), ReplayDeltaArchiveError> { + let bytes = fs::read(path).map_err(|e| { + ReplayDeltaArchiveError::Base(ReplayArchiveError::ReadFile { + entity: "base locks file", + path: path.display().to_string(), + detail: e.to_string(), + }) + })?; + self.validate_base_locks_sha256_bytes(&bytes) + } +} + +fn load_delta_rrdp_repo( + capture_root: &Path, + notify_uri: &str, + entry: &ReplayDeltaRrdpEntry, +) -> Result { + let bucket_hash = sha256_hex(notify_uri.as_bytes()); + let bucket_dir = capture_root.join("rrdp").join("repos").join(&bucket_hash); + if !bucket_dir.is_dir() { + return Err(ReplayDeltaArchiveError::MissingDeltaRepoBucket { + notify_uri: notify_uri.to_string(), + path: bucket_dir.display().to_string(), + }); + } + + let meta: ReplayRrdpRepoMeta = + read_delta_json_file(&bucket_dir.join("meta.json"), "delta RRDP repo meta")?; + ensure_delta_version("delta RRDP repo meta", meta.version)?; + if meta.rpki_notify != notify_uri { + return Err(ReplayDeltaArchiveError::RrdpMetaMismatch { + expected: notify_uri.to_string(), + actual: meta.rpki_notify.clone(), + }); + } + + let transition: ReplayDeltaTransition = + read_delta_json_file(&bucket_dir.join("transition.json"), "delta transition")?; + if transition.kind != entry.kind { + return Err(ReplayDeltaArchiveError::TransitionKindMismatch { + notify_uri: notify_uri.to_string(), + locks_kind: entry.kind.as_str().to_string(), + transition_kind: transition.kind.as_str().to_string(), + }); + } + if transition.base != entry.base { + return Err(ReplayDeltaArchiveError::TransitionBaseMismatch { + notify_uri: notify_uri.to_string(), + }); + } + if transition.target != entry.target { + return Err(ReplayDeltaArchiveError::TransitionTargetMismatch { + notify_uri: notify_uri.to_string(), + }); + } + if transition.delta_count != entry.delta_count || transition.deltas != entry.deltas { + return Err(ReplayDeltaArchiveError::DeltaSerialListMismatch { + notify_uri: notify_uri.to_string(), + }); + } + + let (target_notification_path, delta_paths, target_archive_path) = match entry.kind { + ReplayDeltaRrdpKind::Delta => { + let session = entry.target.session.as_ref().ok_or_else(|| { + ReplayDeltaArchiveError::TransitionTargetMismatch { + notify_uri: notify_uri.to_string(), + } + })?; + let serial = entry.target.serial.ok_or_else(|| { + ReplayDeltaArchiveError::TransitionTargetMismatch { + notify_uri: notify_uri.to_string(), + } + })?; + let session_dir = bucket_dir.join(session); + if !session_dir.is_dir() { + return Err(ReplayDeltaArchiveError::MissingDeltaSessionDir { + notify_uri: notify_uri.to_string(), + path: session_dir.display().to_string(), + }); + } + let notification = session_dir.join(format!("notification-target-{serial}.xml")); + if !notification.is_file() { + return Err(ReplayDeltaArchiveError::MissingTargetNotification { + notify_uri: notify_uri.to_string(), + path: notification.display().to_string(), + }); + } + let mut delta_paths = Vec::new(); + for delta_serial in &entry.deltas { + let pattern = format!("delta-{delta_serial}-"); + let deltas_dir = session_dir.join("deltas"); + let mut matches = if deltas_dir.is_dir() { + fs::read_dir(&deltas_dir) + .map_err(|e| { + ReplayDeltaArchiveError::Base(ReplayArchiveError::ReadFile { + entity: "delta deltas dir", + path: deltas_dir.display().to_string(), + detail: e.to_string(), + }) + })? + .filter_map(|entry| entry.ok().map(|e| e.path())) + .filter(|path| path.is_file()) + .filter(|path| { + path.file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n.starts_with(&pattern) && n.ends_with(".xml")) + }) + .collect::>() + } else { + Vec::new() + }; + matches.sort(); + let path = matches.into_iter().next().ok_or_else(|| { + ReplayDeltaArchiveError::MissingDeltaFile { + notify_uri: notify_uri.to_string(), + serial: *delta_serial, + path: deltas_dir + .join(format!("delta-{delta_serial}-.xml")) + .display() + .to_string(), + } + })?; + delta_paths.push((*delta_serial, path)); + } + let target_archive = bucket_dir.join(format!("target-archive-{serial}.bin")); + let target_archive_path = if target_archive.is_file() { + Some(target_archive) + } else { + None + }; + (Some(notification), delta_paths, target_archive_path) + } + ReplayDeltaRrdpKind::Unchanged | ReplayDeltaRrdpKind::FallbackRsync => { + (None, Vec::new(), None) + } + ReplayDeltaRrdpKind::SessionReset | ReplayDeltaRrdpKind::Gap => (None, Vec::new(), None), + }; + + Ok(ReplayDeltaRrdpRepo { + notify_uri: notify_uri.to_string(), + bucket_hash, + bucket_dir, + meta, + transition, + target_notification_path, + delta_paths, + target_archive_path, + }) +} + +fn load_delta_rsync_module( + capture_root: &Path, + module_uri: &str, + entry: &ReplayDeltaRsyncEntry, +) -> Result { + let canonical = canonical_rsync_module(module_uri).map_err(ReplayDeltaArchiveError::Base)?; + let bucket_hash = sha256_hex(canonical.as_bytes()); + let bucket_dir = capture_root + .join("rsync") + .join("modules") + .join(&bucket_hash); + if !bucket_dir.is_dir() { + return Err(ReplayDeltaArchiveError::MissingRsyncModuleBucket { + module_uri: canonical.clone(), + path: bucket_dir.display().to_string(), + }); + } + + let meta_path = bucket_dir.join("meta.json"); + let meta: ReplayRsyncModuleMeta = if meta_path.is_file() { + let meta: ReplayRsyncModuleMeta = + read_delta_json_file(&meta_path, "delta rsync module meta")?; + ensure_delta_version("delta rsync module meta", meta.version)?; + if meta.module != canonical { + return Err(ReplayDeltaArchiveError::RsyncMetaMismatch { + expected: canonical.clone(), + actual: meta.module.clone(), + }); + } + meta + } else { + ReplayRsyncModuleMeta { + version: 1, + module: canonical.clone(), + created_at: String::new(), + last_seen_at: String::new(), + } + }; + + let files: ReplayDeltaRsyncFiles = + read_delta_json_file(&bucket_dir.join("files.json"), "delta rsync files")?; + ensure_delta_version("delta rsync files", files.version)?; + if files.module != canonical { + return Err(ReplayDeltaArchiveError::RsyncFilesModuleMismatch { + expected: canonical.clone(), + actual: files.module.clone(), + }); + } + if files.file_count != entry.file_count || files.file_count != files.files.len() { + return Err(ReplayDeltaArchiveError::RsyncFileCountMismatch { + module_uri: canonical.clone(), + declared: entry.file_count, + actual: files.files.len(), + }); + } + + let tree_dir = bucket_dir.join("tree"); + let mut overlay_files = Vec::new(); + for uri in &files.files { + let rel = uri.strip_prefix(&canonical).ok_or_else(|| { + ReplayDeltaArchiveError::RsyncFilesModuleMismatch { + expected: canonical.clone(), + actual: uri.clone(), + } + })?; + let tree_root = module_tree_root(&canonical, &tree_dir)?; + let path = tree_root.join(rel); + if !path.is_file() { + return Err(ReplayDeltaArchiveError::MissingRsyncOverlayFile { + module_uri: canonical.clone(), + path: path.display().to_string(), + }); + } + overlay_files.push((uri.clone(), path)); + } + + Ok(ReplayDeltaRsyncModule { + module_uri: canonical, + bucket_hash, + bucket_dir, + meta, + overlay_only: entry.overlay_only, + files, + tree_dir, + overlay_files, + }) +} + +fn module_tree_root(module_uri: &str, tree_dir: &Path) -> Result { + let rest = module_uri.strip_prefix("rsync://").ok_or_else(|| { + ReplayDeltaArchiveError::Base(ReplayArchiveError::InvalidRsyncUri { + uri: module_uri.to_string(), + detail: "URI must start with rsync://".to_string(), + }) + })?; + let mut parts = rest.trim_end_matches('/').split('/'); + let authority = parts.next().unwrap_or_default(); + let module = parts.next().unwrap_or_default(); + Ok(tree_dir.join(authority).join(module)) +} + +fn ensure_delta_version(entity: &'static str, version: u32) -> Result<(), ReplayDeltaArchiveError> { + if version == 1 { + Ok(()) + } else { + Err(ReplayDeltaArchiveError::Base( + ReplayArchiveError::UnsupportedVersion { entity, version }, + )) + } +} + +fn read_delta_json_file Deserialize<'de>>( + path: &Path, + entity: &'static str, +) -> Result { + let bytes = fs::read(path).map_err(|e| { + ReplayDeltaArchiveError::Base(ReplayArchiveError::ReadFile { + entity, + path: path.display().to_string(), + detail: e.to_string(), + }) + })?; + serde_json::from_slice(&bytes).map_err(|e| { + ReplayDeltaArchiveError::Base(ReplayArchiveError::ParseJson { + entity, + path: path.display().to_string(), + detail: e.to_string(), + }) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn build_delta_fixture() -> (tempfile::TempDir, PathBuf, PathBuf, String, String) { + let temp = tempfile::tempdir().expect("tempdir"); + let archive_root = temp.path().join("payload-delta-archive"); + let capture = "delta-cap"; + let base_capture = "base-cap"; + let base_sha = "deadbeef"; + let capture_root = archive_root.join("v1").join("captures").join(capture); + std::fs::create_dir_all(&capture_root).expect("mkdir capture root"); + std::fs::write( + capture_root.join("capture.json"), + format!( + r#"{{"version":1,"captureId":"{capture}","createdAt":"2026-03-15T00:00:00Z","notes":""}}"# + ), + ) + .expect("write capture json"); + std::fs::write( + capture_root.join("base.json"), + format!( + r#"{{"version":1,"baseCapture":"{base_capture}","baseLocksSha256":"{base_sha}","createdAt":"2026-03-15T00:00:00Z"}}"# + ), + ) + .expect("write base json"); + + let notify_uri = "https://rrdp.example.test/notification.xml".to_string(); + let session = "11111111-1111-1111-1111-111111111111".to_string(); + let _target_serial = 12u64; + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let session_dir = capture_root + .join("rrdp/repos") + .join(&repo_hash) + .join(&session); + let deltas_dir = session_dir.join("deltas"); + std::fs::create_dir_all(&deltas_dir).expect("mkdir deltas"); + std::fs::write( + session_dir.parent().unwrap().join("meta.json"), + format!( + r#"{{"version":1,"rpkiNotify":"{notify_uri}","createdAt":"2026-03-15T00:00:00Z","lastSeenAt":"2026-03-15T00:00:01Z"}}"# + ), + ) + .expect("write meta"); + std::fs::write( + session_dir.parent().unwrap().join("transition.json"), + format!( + r#"{{"kind":"delta","base":{{"transport":"rrdp","session":"{session}","serial":10}},"target":{{"transport":"rrdp","session":"{session}","serial":12}},"delta_count":2,"deltas":[11,12]}}"# + ), + ) + .expect("write transition"); + std::fs::write( + session_dir.join("notification-target-12.xml"), + b"", + ) + .expect("write notification"); + std::fs::write( + deltas_dir.join("delta-11-aaaa.xml"), + b"", + ) + .expect("write delta 11"); + std::fs::write( + deltas_dir.join("delta-12-bbbb.xml"), + b"", + ) + .expect("write delta 12"); + std::fs::write( + session_dir.parent().unwrap().join("target-archive-12.bin"), + b"bin", + ) + .expect("write target archive"); + + let module_uri = "rsync://rsync.example.test/repo/".to_string(); + let module_hash = sha256_hex(module_uri.as_bytes()); + let module_bucket = capture_root.join("rsync/modules").join(&module_hash); + let tree_root = module_bucket + .join("tree") + .join("rsync.example.test") + .join("repo"); + std::fs::create_dir_all(tree_root.join("sub")).expect("mkdir tree root"); + std::fs::write( + module_bucket.join("meta.json"), + format!( + r#"{{"version":1,"module":"{module_uri}","createdAt":"2026-03-15T00:00:00Z","lastSeenAt":"2026-03-15T00:00:01Z"}}"# + ), + ) + .expect("write rsync meta"); + std::fs::write( + module_bucket.join("files.json"), + format!( + r#"{{"version":1,"module":"{module_uri}","fileCount":2,"files":["{module_uri}a.roa","{module_uri}sub/b.cer"]}}"# + ), + ) + .expect("write files json"); + std::fs::write(tree_root.join("a.roa"), b"roa").expect("write a.roa"); + std::fs::write(tree_root.join("sub").join("b.cer"), b"cer").expect("write b.cer"); + + let locks_path = temp.path().join("locks-delta.json"); + std::fs::write( + &locks_path, + format!( + r#"{{"version":1,"capture":"{capture}","baseCapture":"{base_capture}","baseLocksSha256":"{base_sha}","rrdp":{{"{notify_uri}":{{"kind":"delta","base":{{"transport":"rrdp","session":"{session}","serial":10}},"target":{{"transport":"rrdp","session":"{session}","serial":12}},"delta_count":2,"deltas":[11,12]}}}},"rsync":{{"{module_uri}":{{"file_count":2,"overlay_only":true}}}}}}"# + ), + ) + .expect("write locks-delta"); + (temp, archive_root, locks_path, notify_uri, module_uri) + } + + #[test] + fn delta_archive_index_loads_unchanged_and_fallback_rsync_rrdp_entries() { + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); + let capture_root = archive_root.join("v1/captures/delta-cap"); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let repo_dir = capture_root.join("rrdp/repos").join(&repo_hash); + std::fs::write( + repo_dir.join("transition.json"), + r#"{"kind":"unchanged","base":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10},"target":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10},"delta_count":0,"deltas":[]}"#, + ) + .expect("rewrite transition unchanged"); + std::fs::write( + &locks_path, + r#"{"version":1,"capture":"delta-cap","baseCapture":"base-cap","baseLocksSha256":"deadbeef","rrdp":{"https://rrdp.example.test/notification.xml":{"kind":"unchanged","base":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10},"target":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10},"delta_count":0,"deltas":[]}},"rsync":{"rsync://rsync.example.test/repo/":{"file_count":2,"overlay_only":true}}}"#, + ).expect("rewrite locks unchanged"); + let index = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path) + .expect("load unchanged index"); + let repo = index.rrdp_repo(¬ify_uri).expect("rrdp repo"); + assert_eq!(repo.transition.kind, ReplayDeltaRrdpKind::Unchanged); + assert!(repo.target_notification_path.is_none()); + assert!(repo.delta_paths.is_empty()); + assert!(repo.target_archive_path.is_none()); + + std::fs::write( + repo_dir.join("transition.json"), + r#"{"kind":"fallback-rsync","base":{"transport":"rsync","session":null,"serial":null},"target":{"transport":"rsync","session":null,"serial":null},"delta_count":0,"deltas":[]}"#, + ).expect("rewrite transition fallback-rsync"); + std::fs::write( + &locks_path, + r#"{"version":1,"capture":"delta-cap","baseCapture":"base-cap","baseLocksSha256":"deadbeef","rrdp":{"https://rrdp.example.test/notification.xml":{"kind":"fallback-rsync","base":{"transport":"rsync","session":null,"serial":null},"target":{"transport":"rsync","session":null,"serial":null},"delta_count":0,"deltas":[]}},"rsync":{"rsync://rsync.example.test/repo/":{"file_count":2,"overlay_only":true}}}"#, + ).expect("rewrite locks fallback-rsync"); + let index = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path) + .expect("load fallback-rsync index"); + let repo = index.rrdp_repo(¬ify_uri).expect("rrdp repo"); + assert_eq!(repo.transition.kind, ReplayDeltaRrdpKind::FallbackRsync); + assert!(repo.target_notification_path.is_none()); + assert!(repo.delta_paths.is_empty()); + } + + #[test] + fn delta_archive_index_resolves_rsync_module_from_base_uri() { + let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_delta_fixture(); + let index = + ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).expect("load delta index"); + let module = index + .resolve_rsync_module_for_base_uri("rsync://rsync.example.test/repo/sub/path") + .expect("resolve module"); + assert_eq!(module.module_uri, "rsync://rsync.example.test/repo/"); + } + + #[test] + fn delta_archive_index_rejects_unsupported_versions_and_meta_mismatches() { + let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_delta_fixture(); + std::fs::write( + &locks_path, + r#"{"version":2,"capture":"delta-cap","baseCapture":"base-cap","baseLocksSha256":"deadbeef","rrdp":{},"rsync":{}}"#, + ).expect("rewrite locks version"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!( + err, + ReplayDeltaArchiveError::Base(ReplayArchiveError::UnsupportedVersion { + entity: "payload delta locks", + version: 2 + }) + ), + "{err}" + ); + + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + std::fs::write( + archive_root.join("v1/captures/delta-cap/rrdp/repos").join(&repo_hash).join("meta.json"), + r#"{"version":1,"rpkiNotify":"https://other.example/notification.xml","createdAt":"2026-03-15T00:00:00Z","lastSeenAt":"2026-03-15T00:00:01Z"}"#, + ).expect("rewrite rrdp meta"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::RrdpMetaMismatch { .. }), + "{err}" + ); + + let (_temp, archive_root, locks_path, _notify_uri, module_uri) = build_delta_fixture(); + let module_hash = sha256_hex(module_uri.as_bytes()); + std::fs::write( + archive_root.join("v1/captures/delta-cap/rsync/modules").join(&module_hash).join("meta.json"), + r#"{"version":1,"module":"rsync://other.example/repo/","createdAt":"2026-03-15T00:00:00Z","lastSeenAt":"2026-03-15T00:00:01Z"}"#, + ).expect("rewrite rsync meta"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::RsyncMetaMismatch { .. }), + "{err}" + ); + } + + #[test] + fn delta_archive_index_rejects_transition_base_target_and_serial_mismatches() { + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let repo_dir = archive_root + .join("v1/captures/delta-cap/rrdp/repos") + .join(&repo_hash); + + std::fs::write( + repo_dir.join("transition.json"), + r#"{"kind":"delta","base":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":9},"target":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":12},"delta_count":2,"deltas":[11,12]}"#, + ).expect("rewrite transition base"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::TransitionBaseMismatch { .. }), + "{err}" + ); + + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let repo_dir = archive_root + .join("v1/captures/delta-cap/rrdp/repos") + .join(&repo_hash); + std::fs::write( + repo_dir.join("transition.json"), + r#"{"kind":"delta","base":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10},"target":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":13},"delta_count":2,"deltas":[11,12]}"#, + ).expect("rewrite transition target"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!( + err, + ReplayDeltaArchiveError::TransitionTargetMismatch { .. } + ), + "{err}" + ); + + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let repo_dir = archive_root + .join("v1/captures/delta-cap/rrdp/repos") + .join(&repo_hash); + std::fs::write( + repo_dir.join("transition.json"), + r#"{"kind":"delta","base":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10},"target":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":12},"delta_count":1,"deltas":[12]}"#, + ).expect("rewrite transition deltas"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::DeltaSerialListMismatch { .. }), + "{err}" + ); + } + + #[test] + fn delta_archive_index_rejects_missing_session_dir_and_overlay_files() { + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let session_dir = archive_root + .join("v1/captures/delta-cap/rrdp/repos") + .join(&repo_hash) + .join("11111111-1111-1111-1111-111111111111"); + std::fs::remove_dir_all(&session_dir).expect("remove session dir"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::MissingDeltaSessionDir { .. }), + "{err}" + ); + + let (_temp, archive_root, locks_path, _notify_uri, module_uri) = build_delta_fixture(); + let module_hash = sha256_hex(module_uri.as_bytes()); + let overlay_path = archive_root + .join("v1/captures/delta-cap/rsync/modules") + .join(&module_hash) + .join("tree/rsync.example.test/repo/sub/b.cer"); + std::fs::remove_file(overlay_path).expect("remove overlay file"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::MissingRsyncOverlayFile { .. }), + "{err}" + ); + } + + #[test] + fn delta_archive_index_accepts_missing_rsync_module_meta_when_files_and_tree_exist() { + let (_temp, archive_root, locks_path, _notify_uri, module_uri) = build_delta_fixture(); + let module_hash = sha256_hex(module_uri.as_bytes()); + let meta_path = archive_root + .join("v1/captures/delta-cap/rsync/modules") + .join(module_hash) + .join("meta.json"); + std::fs::remove_file(&meta_path).expect("remove delta rsync module meta"); + + let index = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path) + .expect("load delta replay index without rsync meta"); + let module = index + .rsync_modules + .get(&module_uri) + .expect("module present"); + assert_eq!(module.meta.module, module_uri); + assert_eq!(module.meta.version, 1); + } + + #[test] + fn delta_archive_index_accepts_correct_base_locks_sha_and_rejects_missing_module_resolution() { + let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_delta_fixture(); + let index = + ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).expect("load delta index"); + let locks_bytes = std::fs::read(&locks_path).expect("read locks bytes"); + assert!( + index + .validate_base_locks_sha256_bytes(&locks_bytes) + .is_err() + ); + let err = index + .resolve_rsync_module_for_base_uri("rsync://missing.example/repo/path") + .unwrap_err(); + assert!( + matches!( + err, + ReplayDeltaArchiveError::MissingRsyncModuleBucket { .. } + ), + "{err}" + ); + } + + #[test] + fn delta_archive_index_loads_session_reset_and_gap_entries_without_target_files() { + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let repo_dir = archive_root + .join("v1/captures/delta-cap/rrdp/repos") + .join(&repo_hash); + + for kind in ["session-reset", "gap"] { + std::fs::write( + repo_dir.join("transition.json"), + format!( + r#"{{"kind":"{kind}","base":{{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10}},"target":{{"transport":"rrdp","session":"22222222-2222-2222-2222-222222222222","serial":12}},"delta_count":0,"deltas":[]}}"#, + ), + ) + .expect("rewrite transition kind"); + std::fs::write( + &locks_path, + format!( + r#"{{"version":1,"capture":"delta-cap","baseCapture":"base-cap","baseLocksSha256":"deadbeef","rrdp":{{"{notify_uri}":{{"kind":"{kind}","base":{{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10}},"target":{{"transport":"rrdp","session":"22222222-2222-2222-2222-222222222222","serial":12}},"delta_count":0,"deltas":[]}}}},"rsync":{{"rsync://rsync.example.test/repo/":{{"file_count":2,"overlay_only":true}}}}}}"#, + ), + ) + .expect("rewrite locks kind"); + let index = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path) + .expect("load delta index"); + let repo = index.rrdp_repo(¬ify_uri).expect("rrdp repo"); + assert!(repo.target_notification_path.is_none()); + assert!(repo.delta_paths.is_empty()); + } + } + #[test] + fn delta_archive_index_loads_rrdp_and_rsync_entries() { + let (_temp, archive_root, locks_path, notify_uri, module_uri) = build_delta_fixture(); + let index = + ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).expect("load delta index"); + assert_eq!(index.capture_meta.capture_id, "delta-cap"); + assert_eq!(index.base_meta.base_capture, "base-cap"); + assert_eq!(index.rrdp_repos.len(), 1); + assert_eq!(index.rsync_modules.len(), 1); + + let repo = index.rrdp_repo(¬ify_uri).expect("rrdp repo"); + assert_eq!(repo.transition.kind, ReplayDeltaRrdpKind::Delta); + assert_eq!(repo.transition.delta_count, 2); + assert_eq!(repo.delta_paths.len(), 2); + assert!(repo.target_notification_path.as_ref().unwrap().is_file()); + assert!(repo.target_archive_path.as_ref().unwrap().is_file()); + + let module = index.rsync_module(&module_uri).expect("rsync module"); + assert_eq!(module.files.file_count, 2); + assert_eq!(module.overlay_files.len(), 2); + assert!(module.overlay_files.iter().all(|(_, path)| path.is_file())); + } + + #[test] + fn delta_archive_index_rejects_capture_and_sha_mismatches() { + let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_delta_fixture(); + let capture_root = archive_root.join("v1/captures/delta-cap"); + std::fs::write( + capture_root.join("capture.json"), + r#"{"version":1,"captureId":"other-cap","createdAt":"2026-03-15T00:00:00Z","notes":""}"#, + ) + .expect("rewrite capture json"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::CaptureIdMismatch { .. }), + "{err}" + ); + + let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_delta_fixture(); + let capture_root = archive_root.join("v1/captures/delta-cap"); + std::fs::write( + capture_root.join("base.json"), + r#"{"version":1,"baseCapture":"base-cap","baseLocksSha256":"beefdead","createdAt":"2026-03-15T00:00:00Z"}"#, + ) + .expect("rewrite base json sha"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::BaseLocksShaMismatch { .. }), + "{err}" + ); + } + + #[test] + fn delta_archive_index_rejects_missing_target_notification_and_repo_bucket() { + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let session_dir = archive_root + .join("v1/captures/delta-cap/rrdp/repos") + .join(&repo_hash) + .join("11111111-1111-1111-1111-111111111111"); + std::fs::remove_file(session_dir.join("notification-target-12.xml")) + .expect("remove target notification"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!( + err, + ReplayDeltaArchiveError::MissingTargetNotification { .. } + ), + "{err}" + ); + + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let repo_dir = archive_root + .join("v1/captures/delta-cap/rrdp/repos") + .join(&repo_hash); + std::fs::remove_dir_all(repo_dir).expect("remove repo dir"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::MissingDeltaRepoBucket { .. }), + "{err}" + ); + } + #[test] + fn delta_archive_index_rejects_base_meta_mismatch() { + let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_delta_fixture(); + let capture_root = archive_root.join("v1/captures/delta-cap"); + std::fs::write( + capture_root.join("base.json"), + r#"{"version":1,"baseCapture":"other","baseLocksSha256":"deadbeef","createdAt":"2026-03-15T00:00:00Z"}"#, + ) + .expect("rewrite base json"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::BaseCaptureMismatch { .. }), + "{err}" + ); + } + + #[test] + fn delta_archive_index_rejects_transition_mismatch_and_missing_delta_file() { + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let repo_dir = archive_root + .join("v1/captures/delta-cap/rrdp/repos") + .join(&repo_hash); + std::fs::write( + repo_dir.join("transition.json"), + r#"{"kind":"unchanged","base":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":10},"target":{"transport":"rrdp","session":"11111111-1111-1111-1111-111111111111","serial":12},"delta_count":2,"deltas":[11,12]}"#, + ) + .expect("rewrite transition"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::TransitionKindMismatch { .. }), + "{err}" + ); + + let (_temp, archive_root, locks_path, notify_uri, _module_uri) = build_delta_fixture(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let delta_path = archive_root + .join("v1/captures/delta-cap/rrdp/repos") + .join(&repo_hash) + .join("11111111-1111-1111-1111-111111111111/deltas/delta-12-bbbb.xml"); + std::fs::remove_file(delta_path).expect("remove delta"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::MissingDeltaFile { .. }), + "{err}" + ); + } + + #[test] + fn delta_archive_index_validates_base_locks_sha256_bytes_and_file() { + let (_temp, archive_root, locks_path, _notify_uri, _module_uri) = build_delta_fixture(); + let index = + ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).expect("load delta index"); + let err = index + .validate_base_locks_sha256_bytes(b"not-the-right-base-locks") + .unwrap_err(); + assert!( + matches!( + err, + ReplayDeltaArchiveError::BaseLocksBytesShaMismatch { .. } + ), + "{err}" + ); + + let temp_file = tempfile::NamedTempFile::new().expect("tempfile"); + std::fs::write(temp_file.path(), b"still-wrong").expect("write base locks file"); + let err = index + .validate_base_locks_sha256_file(temp_file.path()) + .unwrap_err(); + assert!( + matches!( + err, + ReplayDeltaArchiveError::BaseLocksBytesShaMismatch { .. } + ), + "{err}" + ); + } + + #[test] + fn delta_archive_index_rejects_rsync_files_mismatch() { + let (_temp, archive_root, locks_path, _notify_uri, module_uri) = build_delta_fixture(); + let module_hash = sha256_hex(module_uri.as_bytes()); + let module_dir = archive_root + .join("v1/captures/delta-cap/rsync/modules") + .join(&module_hash); + std::fs::write( + module_dir.join("files.json"), + format!( + r#"{{"version":1,"module":"{module_uri}","fileCount":3,"files":["{module_uri}a.roa"]}}"# + ), + ) + .expect("rewrite files json"); + let err = ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayDeltaArchiveError::RsyncFileCountMismatch { .. }), + "{err}" + ); + } +} diff --git a/crates/panda-rpki-validator/src/replay/delta_fetch_http.rs b/crates/panda-rpki-validator/src/replay/delta_fetch_http.rs new file mode 100644 index 0000000..89dfbfe --- /dev/null +++ b/crates/panda-rpki-validator/src/replay/delta_fetch_http.rs @@ -0,0 +1,368 @@ +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use crate::replay::delta_archive::{ + ReplayDeltaArchiveError, ReplayDeltaArchiveIndex, ReplayDeltaRrdpKind, +}; +use crate::sync::rrdp::{Fetcher, parse_notification}; + +#[derive(Debug, thiserror::Error)] +pub enum PayloadDeltaReplayHttpFetcherError { + #[error(transparent)] + DeltaIndex(#[from] ReplayDeltaArchiveError), + + #[error("read delta replay RRDP file failed: {path}: {detail}")] + ReadFile { path: String, detail: String }, + + #[error("parse target notification failed for {notify_uri}: {detail}")] + ParseNotification { notify_uri: String, detail: String }, + + #[error( + "target notification session/serial mismatch for {notify_uri}: expected session={expected_session} serial={expected_serial}, got session={actual_session} serial={actual_serial}" + )] + NotificationTargetMismatch { + notify_uri: String, + expected_session: String, + expected_serial: u64, + actual_session: String, + actual_serial: u64, + }, + + #[error( + "delta serial list mismatch between target notification and transition for {notify_uri}" + )] + DeltaSerialMismatch { notify_uri: String }, + + #[error("duplicate delta replay HTTP URI mapping for {uri}: {first_path} vs {second_path}")] + DuplicateUriMapping { + uri: String, + first_path: String, + second_path: String, + }, + + #[error( + "delta replay notification URI is {kind} and should not be fetched as RRDP: {notify_uri}" + )] + NotificationKindNotFetchable { notify_uri: String, kind: String }, + + #[error("delta replay HTTP URI not found in archive: {0}")] + MissingUri(String), +} + +#[derive(Clone, Debug)] +pub struct PayloadDeltaReplayHttpFetcher { + index: Arc, + routes: BTreeMap, + repo_kinds: BTreeMap, +} + +impl PayloadDeltaReplayHttpFetcher { + pub fn new( + index: Arc, + ) -> Result { + let mut routes = BTreeMap::new(); + let mut repo_kinds = BTreeMap::new(); + for (notify_uri, repo) in &index.rrdp_repos { + repo_kinds.insert(notify_uri.clone(), repo.transition.kind); + if repo.transition.kind != ReplayDeltaRrdpKind::Delta { + continue; + } + let notification_path = repo + .target_notification_path + .as_ref() + .expect("delta repo target notification indexed"); + insert_unique_route(&mut routes, notify_uri, notification_path)?; + + let notification_xml = fs::read(notification_path).map_err(|e| { + PayloadDeltaReplayHttpFetcherError::ReadFile { + path: notification_path.display().to_string(), + detail: e.to_string(), + } + })?; + let notification = parse_notification(¬ification_xml).map_err(|e| { + PayloadDeltaReplayHttpFetcherError::ParseNotification { + notify_uri: notify_uri.clone(), + detail: e.to_string(), + } + })?; + let expected_session = repo.transition.target.session.as_deref().unwrap_or(""); + let expected_serial = repo.transition.target.serial.unwrap_or_default(); + let actual_session = notification.session_id.to_string(); + if actual_session != expected_session || notification.serial != expected_serial { + return Err( + PayloadDeltaReplayHttpFetcherError::NotificationTargetMismatch { + notify_uri: notify_uri.clone(), + expected_session: expected_session.to_string(), + expected_serial, + actual_session, + actual_serial: notification.serial, + }, + ); + } + let transition_serials = repo + .delta_paths + .iter() + .map(|(serial, _)| *serial) + .collect::>(); + let mut notification_delta_map = BTreeMap::new(); + for dref in notification.deltas { + notification_delta_map.insert(dref.serial, dref.uri); + } + for serial in &transition_serials { + if !notification_delta_map.contains_key(serial) { + return Err(PayloadDeltaReplayHttpFetcherError::DeltaSerialMismatch { + notify_uri: notify_uri.clone(), + }); + } + } + for (serial, path) in &repo.delta_paths { + let uri = notification_delta_map + .get(serial) + .expect("delta uri present for transition serial"); + insert_unique_route(&mut routes, uri, path)?; + } + } + Ok(Self { + index, + routes, + repo_kinds, + }) + } + + pub fn from_index( + index: Arc, + ) -> Result { + Self::new(index) + } + + pub fn archive_index(&self) -> &ReplayDeltaArchiveIndex { + self.index.as_ref() + } +} + +impl Fetcher for PayloadDeltaReplayHttpFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + if let Some(path) = self.routes.get(uri) { + return fs::read(path).map_err(|e| { + PayloadDeltaReplayHttpFetcherError::ReadFile { + path: path.display().to_string(), + detail: e.to_string(), + } + .to_string() + }); + } + if let Some(kind) = self.repo_kinds.get(uri) { + return Err( + PayloadDeltaReplayHttpFetcherError::NotificationKindNotFetchable { + notify_uri: uri.to_string(), + kind: kind.as_str().to_string(), + } + .to_string(), + ); + } + Err(PayloadDeltaReplayHttpFetcherError::MissingUri(uri.to_string()).to_string()) + } +} + +fn insert_unique_route( + routes: &mut BTreeMap, + uri: &str, + path: &Path, +) -> Result<(), PayloadDeltaReplayHttpFetcherError> { + if let Some(existing) = routes.get(uri) { + if existing != path { + return Err(PayloadDeltaReplayHttpFetcherError::DuplicateUriMapping { + uri: uri.to_string(), + first_path: existing.display().to_string(), + second_path: path.display().to_string(), + }); + } + return Ok(()); + } + routes.insert(uri.to_string(), path.to_path_buf()); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::replay::archive::sha256_hex; + use crate::replay::delta_archive::ReplayDeltaArchiveIndex; + + fn build_delta_http_fixture( + kind: ReplayDeltaRrdpKind, + ) -> (tempfile::TempDir, PathBuf, PathBuf, String, String, String) { + let temp = tempfile::tempdir().expect("tempdir"); + let archive_root = temp.path().join("payload-delta-archive"); + let capture = "delta-http"; + let capture_root = archive_root.join("v1").join("captures").join(capture); + std::fs::create_dir_all(&capture_root).expect("mkdir capture root"); + std::fs::write( + capture_root.join("capture.json"), + format!(r#"{{"version":1,"captureId":"{capture}","createdAt":"2026-03-16T00:00:00Z","notes":""}}"#), + ) + .expect("write capture meta"); + std::fs::write( + capture_root.join("base.json"), + r#"{"version":1,"baseCapture":"base-cap","baseLocksSha256":"deadbeef","createdAt":"2026-03-16T00:00:00Z"}"#, + ) + .expect("write base meta"); + + let notify_uri = "https://rrdp.example.test/notification.xml".to_string(); + let snapshot_uri = "https://rrdp.example.test/snapshot.xml".to_string(); + let delta1_uri = "https://rrdp.example.test/d1.xml".to_string(); + let delta2_uri = "https://rrdp.example.test/d2.xml".to_string(); + let session = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".to_string(); + let target_serial = 12u64; + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let session_dir = capture_root + .join("rrdp/repos") + .join(&repo_hash) + .join(&session); + let deltas_dir = session_dir.join("deltas"); + std::fs::create_dir_all(&deltas_dir).expect("mkdir deltas dir"); + std::fs::write( + session_dir.parent().unwrap().join("meta.json"), + format!(r#"{{"version":1,"rpkiNotify":"{notify_uri}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), + ) + .expect("write repo meta"); + + let notification_xml = format!( + r#" + + + + +"# + ); + std::fs::write( + session_dir.join("notification-target-12.xml"), + notification_xml, + ) + .expect("write target notification"); + std::fs::write( + deltas_dir.join("delta-11-aaaa.xml"), + b"", + ) + .expect("write delta1"); + std::fs::write( + deltas_dir.join("delta-12-bbbb.xml"), + b"", + ) + .expect("write delta2"); + std::fs::write( + session_dir.parent().unwrap().join("transition.json"), + format!( + r#"{{"kind":"{}","base":{{"transport":"rrdp","session":"{session}","serial":10}},"target":{{"transport":"rrdp","session":"{session}","serial":12}},"delta_count":2,"deltas":[11,12]}}"#, + kind.as_str() + ), + ) + .expect("write transition"); + let locks_path = temp.path().join("locks-delta.json"); + std::fs::write( + &locks_path, + format!( + r#"{{"version":1,"capture":"{capture}","baseCapture":"base-cap","baseLocksSha256":"deadbeef","rrdp":{{"{notify_uri}":{{"kind":"{}","base":{{"transport":"rrdp","session":"{session}","serial":10}},"target":{{"transport":"rrdp","session":"{session}","serial":12}},"delta_count":2,"deltas":[11,12]}}}},"rsync":{{}}}}"#, + kind.as_str() + ), + ) + .expect("write locks"); + ( + temp, + archive_root, + locks_path, + notify_uri, + delta1_uri, + delta2_uri, + ) + } + + #[test] + fn delta_http_fetcher_rejects_session_reset_and_gap_notification_kind() { + for kind in [ReplayDeltaRrdpKind::SessionReset, ReplayDeltaRrdpKind::Gap] { + let (_temp, archive_root, locks_path, notify_uri, _delta1_uri, _delta2_uri) = + build_delta_http_fixture(kind); + let index = Arc::new( + ReplayDeltaArchiveIndex::load(&archive_root, &locks_path) + .expect("load delta index"), + ); + let fetcher = PayloadDeltaReplayHttpFetcher::from_index(index).expect("build fetcher"); + let err = fetcher.fetch(¬ify_uri).unwrap_err(); + assert!(err.contains(kind.as_str()), "{err}"); + } + } + #[test] + fn delta_http_fetcher_reads_target_notification_and_delta_files() { + let (_temp, archive_root, locks_path, notify_uri, delta1_uri, delta2_uri) = + build_delta_http_fixture(ReplayDeltaRrdpKind::Delta); + let index = Arc::new( + ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).expect("load delta index"), + ); + let fetcher = PayloadDeltaReplayHttpFetcher::from_index(index).expect("build fetcher"); + let notification = fetcher.fetch(¬ify_uri).expect("fetch notification"); + assert!( + std::str::from_utf8(¬ification) + .unwrap() + .contains("notification") + ); + assert_eq!( + fetcher.fetch(&delta1_uri).expect("fetch delta1"), + b"".to_vec() + ); + assert_eq!( + fetcher.fetch(&delta2_uri).expect("fetch delta2"), + b"".to_vec() + ); + } + + #[test] + fn delta_http_fetcher_rejects_non_delta_notification_kinds_and_missing_uri() { + let (_temp, archive_root, locks_path, notify_uri, _delta1_uri, _delta2_uri) = + build_delta_http_fixture(ReplayDeltaRrdpKind::Unchanged); + let index = Arc::new( + ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).expect("load delta index"), + ); + let fetcher = PayloadDeltaReplayHttpFetcher::from_index(index).expect("build fetcher"); + let err = fetcher.fetch(¬ify_uri).unwrap_err(); + assert!(err.contains("unchanged"), "{err}"); + let err = fetcher + .fetch("https://missing.example/test.xml") + .unwrap_err(); + assert!(err.contains("not found in archive"), "{err}"); + } + + #[test] + fn delta_http_fetcher_rejects_target_notification_mismatch() { + let (_temp, archive_root, locks_path, notify_uri, _delta1_uri, _delta2_uri) = + build_delta_http_fixture(ReplayDeltaRrdpKind::Delta); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let notification = archive_root + .join("v1/captures/delta-http/rrdp/repos") + .join(repo_hash) + .join("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa") + .join("notification-target-12.xml"); + std::fs::write( + ¬ification, + r#" + + + + +"#, + ) + .expect("rewrite notification"); + let index = Arc::new( + ReplayDeltaArchiveIndex::load(&archive_root, &locks_path).expect("load delta index"), + ); + let err = PayloadDeltaReplayHttpFetcher::from_index(index).unwrap_err(); + assert!( + matches!( + err, + PayloadDeltaReplayHttpFetcherError::NotificationTargetMismatch { .. } + ), + "{err}" + ); + } +} diff --git a/crates/panda-rpki-validator/src/replay/delta_fetch_rsync.rs b/crates/panda-rpki-validator/src/replay/delta_fetch_rsync.rs new file mode 100644 index 0000000..51face7 --- /dev/null +++ b/crates/panda-rpki-validator/src/replay/delta_fetch_rsync.rs @@ -0,0 +1,471 @@ +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use crate::fetch::rsync::{RsyncFetchError, RsyncFetchResult, RsyncFetcher}; +use crate::replay::archive::{ReplayArchiveIndex, canonical_rsync_module}; +use crate::replay::delta_archive::ReplayDeltaArchiveIndex; +use crate::storage::{RepositoryViewState, RocksStore}; + +#[derive(Clone, Debug)] +pub struct PayloadDeltaReplayRsyncFetcher { + base_index: Arc, + delta_index: Arc, +} + +impl PayloadDeltaReplayRsyncFetcher { + pub fn new( + base_index: Arc, + delta_index: Arc, + ) -> Self { + Self { + base_index, + delta_index, + } + } + + pub fn base_index(&self) -> &ReplayArchiveIndex { + self.base_index.as_ref() + } + + pub fn delta_index(&self) -> &ReplayDeltaArchiveIndex { + self.delta_index.as_ref() + } +} + +pub struct PayloadDeltaReplayCurrentStoreRsyncFetcher<'a> { + store: &'a RocksStore, + delta_index: Arc, +} + +impl<'a> PayloadDeltaReplayCurrentStoreRsyncFetcher<'a> { + pub fn new(store: &'a RocksStore, delta_index: Arc) -> Self { + Self { store, delta_index } + } + + pub fn delta_index(&self) -> &ReplayDeltaArchiveIndex { + self.delta_index.as_ref() + } +} + +impl RsyncFetcher for PayloadDeltaReplayRsyncFetcher { + fn fetch_objects(&self, rsync_base_uri: &str) -> RsyncFetchResult)>> { + let module_uri = canonical_rsync_module(rsync_base_uri) + .map_err(|e| RsyncFetchError::Fetch(e.to_string()))?; + let normalized_base = if rsync_base_uri.ends_with('/') { + rsync_base_uri.to_string() + } else { + format!("{rsync_base_uri}/") + }; + + let mut merged: BTreeMap> = BTreeMap::new(); + let mut saw_base = false; + let overlay_only = self + .delta_index + .rsync_module(&module_uri) + .map(|module| module.overlay_only) + .unwrap_or(false); + if !overlay_only { + if let Ok(base_module) = self + .base_index + .resolve_rsync_module_for_base_uri(rsync_base_uri) + { + let base_tree_root = module_tree_root(&module_uri, &base_module.tree_dir) + .map_err(RsyncFetchError::Fetch)?; + if base_tree_root.is_dir() { + let mut base_objects = Vec::new(); + walk_dir_collect( + &base_tree_root, + &base_tree_root, + &module_uri, + &mut base_objects, + ) + .map_err(RsyncFetchError::Fetch)?; + for (uri, bytes) in base_objects { + merged.insert(uri, bytes); + } + saw_base = true; + } + } + } + + let mut saw_overlay = false; + if let Some(delta_module) = self.delta_index.rsync_module(&module_uri) { + for (uri, path) in &delta_module.overlay_files { + let bytes = fs::read(path).map_err(|e| { + RsyncFetchError::Fetch(format!( + "read delta rsync overlay failed: {}: {e}", + path.display() + )) + })?; + merged.insert(uri.clone(), bytes); + saw_overlay = true; + } + } + + if !saw_base && !saw_overlay { + return Err(RsyncFetchError::Fetch(format!( + "delta replay base rsync module not found and no delta overlay exists for module: {module_uri}" + ))); + } + + let filtered: Vec<(String, Vec)> = merged + .into_iter() + .filter(|(uri, _)| uri.starts_with(&normalized_base)) + .collect(); + if filtered.is_empty() { + return Err(RsyncFetchError::Fetch(format!( + "delta replay rsync subtree not found: {normalized_base}" + ))); + } + Ok(filtered) + } +} + +impl RsyncFetcher for PayloadDeltaReplayCurrentStoreRsyncFetcher<'_> { + fn fetch_objects(&self, rsync_base_uri: &str) -> RsyncFetchResult)>> { + let module_uri = canonical_rsync_module(rsync_base_uri) + .map_err(|e| RsyncFetchError::Fetch(e.to_string()))?; + let normalized_base = if rsync_base_uri.ends_with('/') { + rsync_base_uri.to_string() + } else { + format!("{rsync_base_uri}/") + }; + + let mut merged: BTreeMap> = BTreeMap::new(); + let mut saw_base = false; + let overlay_only = self + .delta_index + .rsync_module(&module_uri) + .map(|module| module.overlay_only) + .unwrap_or(false); + + if !overlay_only { + let entries = self + .store + .list_repository_view_entries_with_prefix(&module_uri) + .map_err(|e| RsyncFetchError::Fetch(format!("list repository view failed: {e}")))?; + for entry in entries { + if entry.state != RepositoryViewState::Present { + continue; + } + let bytes = self + .store + .load_current_object_bytes_by_uri(&entry.rsync_uri) + .map_err(|e| { + RsyncFetchError::Fetch(format!( + "load current object failed for {}: {e}", + entry.rsync_uri + )) + })? + .ok_or_else(|| { + RsyncFetchError::Fetch(format!( + "current object missing for {}", + entry.rsync_uri + )) + })?; + merged.insert(entry.rsync_uri, bytes); + saw_base = true; + } + } + + let mut saw_overlay = false; + if let Some(delta_module) = self.delta_index.rsync_module(&module_uri) { + for (uri, path) in &delta_module.overlay_files { + let bytes = fs::read(path).map_err(|e| { + RsyncFetchError::Fetch(format!( + "read delta rsync overlay failed: {}: {e}", + path.display() + )) + })?; + merged.insert(uri.clone(), bytes); + saw_overlay = true; + } + } + + if !saw_base && !saw_overlay { + return Err(RsyncFetchError::Fetch(format!( + "delta replay current-store base not found and no delta overlay exists for module: {module_uri}" + ))); + } + + let filtered: Vec<(String, Vec)> = merged + .into_iter() + .filter(|(uri, _)| uri.starts_with(&normalized_base)) + .collect(); + if filtered.is_empty() { + return Err(RsyncFetchError::Fetch(format!( + "delta replay rsync subtree not found: {normalized_base}" + ))); + } + Ok(filtered) + } +} + +fn module_tree_root(module_uri: &str, tree_dir: &Path) -> Result { + let rest = module_uri + .strip_prefix("rsync://") + .ok_or_else(|| format!("invalid rsync module URI: {module_uri}"))?; + let mut parts = rest.trim_end_matches('/').split('/'); + let authority = parts + .next() + .ok_or_else(|| format!("invalid rsync module URI: {module_uri}"))?; + let module = parts + .next() + .ok_or_else(|| format!("invalid rsync module URI: {module_uri}"))?; + Ok(tree_dir.join(authority).join(module)) +} + +fn walk_dir_collect( + root: &Path, + current: &Path, + rsync_base_uri: &str, + out: &mut Vec<(String, Vec)>, +) -> Result<(), String> { + let rd = fs::read_dir(current).map_err(|e| e.to_string())?; + for entry in rd { + let entry = entry.map_err(|e| e.to_string())?; + let path = entry.path(); + let meta = entry.metadata().map_err(|e| e.to_string())?; + if meta.is_dir() { + walk_dir_collect(root, &path, rsync_base_uri, out)?; + continue; + } + if !meta.is_file() { + continue; + } + let rel = path + .strip_prefix(root) + .map_err(|e| e.to_string())? + .to_string_lossy() + .replace('\\', "/"); + let base = if rsync_base_uri.ends_with('/') { + rsync_base_uri.to_string() + } else { + format!("{rsync_base_uri}/") + }; + let uri = format!("{base}{rel}"); + let bytes = fs::read(&path).map_err(|e| e.to_string())?; + out.push((uri, bytes)); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::replay::archive::ReplayArchiveIndex; + use crate::replay::delta_archive::ReplayDeltaArchiveIndex; + use crate::storage::{RawByHashEntry, RepositoryViewEntry, RepositoryViewState, RocksStore}; + + fn build_base_and_delta_rsync_fixture() + -> (tempfile::TempDir, PathBuf, PathBuf, PathBuf, PathBuf) { + let temp = tempfile::tempdir().expect("tempdir"); + let base_archive = temp.path().join("payload-archive"); + let base_capture_root = base_archive.join("v1/captures/base-cap"); + std::fs::create_dir_all(&base_capture_root).expect("mkdir base capture"); + std::fs::write( + base_capture_root.join("capture.json"), + r#"{"version":1,"captureId":"base-cap","createdAt":"2026-03-16T00:00:00Z","notes":""}"#, + ) + .expect("write base capture meta"); + let module_uri = "rsync://rsync.example.test/repo/"; + let module_hash = crate::replay::archive::sha256_hex(module_uri.as_bytes()); + let base_bucket = base_capture_root.join("rsync/modules").join(&module_hash); + let base_tree = base_bucket.join("tree/rsync.example.test/repo"); + std::fs::create_dir_all(base_tree.join("sub")).expect("mkdir base tree"); + std::fs::write(base_bucket.join("meta.json"), format!(r#"{{"version":1,"module":"{module_uri}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#)).expect("write base meta"); + std::fs::write(base_tree.join("a.roa"), b"base-a").expect("write base a"); + std::fs::write(base_tree.join("sub").join("b.cer"), b"base-b").expect("write base b"); + let base_locks = temp.path().join("base-locks.json"); + std::fs::write(&base_locks, format!(r#"{{"version":1,"capture":"base-cap","rrdp":{{}},"rsync":{{"{module_uri}":{{"transport":"rsync"}}}}}}"#)).expect("write base locks"); + + let delta_archive = temp.path().join("payload-delta-archive"); + let delta_capture_root = delta_archive.join("v1/captures/delta-cap"); + std::fs::create_dir_all(&delta_capture_root).expect("mkdir delta capture"); + std::fs::write(delta_capture_root.join("capture.json"), r#"{"version":1,"captureId":"delta-cap","createdAt":"2026-03-16T00:00:00Z","notes":""}"#).expect("write delta capture meta"); + std::fs::write(delta_capture_root.join("base.json"), r#"{"version":1,"baseCapture":"base-cap","baseLocksSha256":"deadbeef","createdAt":"2026-03-16T00:00:00Z"}"#).expect("write delta base meta"); + let delta_bucket = delta_capture_root.join("rsync/modules").join(&module_hash); + let delta_tree = delta_bucket.join("tree/rsync.example.test/repo"); + std::fs::create_dir_all(delta_tree.join("sub")).expect("mkdir delta tree"); + std::fs::write(delta_bucket.join("meta.json"), format!(r#"{{"version":1,"module":"{module_uri}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#)).expect("write delta meta"); + std::fs::write(delta_bucket.join("files.json"), format!(r#"{{"version":1,"module":"{module_uri}","fileCount":1,"files":["{module_uri}sub/b.cer"]}}"#)).expect("write files json"); + std::fs::write(delta_tree.join("sub").join("b.cer"), b"delta-b") + .expect("write delta overlay"); + let delta_locks = temp.path().join("locks-delta.json"); + std::fs::write(&delta_locks, format!(r#"{{"version":1,"capture":"delta-cap","baseCapture":"base-cap","baseLocksSha256":"deadbeef","rrdp":{{}},"rsync":{{"{module_uri}":{{"file_count":1,"overlay_only":false}}}}}}"#)).expect("write delta locks"); + (temp, base_archive, base_locks, delta_archive, delta_locks) + } + + #[test] + fn delta_rsync_current_store_fetcher_merges_current_store_and_overlay() { + let (_temp, _base_archive, _base_locks, delta_archive, delta_locks) = + build_base_and_delta_rsync_fixture(); + let temp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(temp.path()).expect("open rocksdb"); + store + .put_raw_by_hash_entry(&RawByHashEntry { + sha256_hex: crate::replay::archive::sha256_hex(b"base-a"), + bytes: b"base-a".to_vec(), + origin_uris: vec!["rsync://rsync.example.test/repo/a.roa".to_string()], + object_type: Some("roa".to_string()), + encoding: None, + }) + .expect("put raw by hash"); + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: "rsync://rsync.example.test/repo/a.roa".to_string(), + current_hash: Some(crate::replay::archive::sha256_hex(b"base-a")), + repository_source: Some("rsync".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }) + .expect("put repository view"); + + let delta = Arc::new( + ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"), + ); + let fetcher = PayloadDeltaReplayCurrentStoreRsyncFetcher::new(&store, delta); + let mut objects = fetcher + .fetch_objects("rsync://rsync.example.test/repo/") + .expect("fetch current-store objects"); + objects.sort_by(|a, b| a.0.cmp(&b.0)); + assert_eq!(objects.len(), 2); + assert_eq!(objects[0].0, "rsync://rsync.example.test/repo/a.roa"); + assert_eq!(objects[0].1, b"base-a"); + assert_eq!(objects[1].0, "rsync://rsync.example.test/repo/sub/b.cer"); + assert_eq!(objects[1].1, b"delta-b"); + } + + #[test] + fn delta_rsync_fetcher_uses_base_only_when_delta_has_no_module_entry() { + let (_temp, base_archive, base_locks, delta_archive, delta_locks) = + build_base_and_delta_rsync_fixture(); + std::fs::write( + &delta_locks, + r#"{"version":1,"capture":"delta-cap","baseCapture":"base-cap","baseLocksSha256":"deadbeef","rrdp":{},"rsync":{}}"#, + ) + .expect("rewrite delta locks no rsync"); + let base = Arc::new( + ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index"), + ); + let delta = Arc::new( + ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"), + ); + let fetcher = PayloadDeltaReplayRsyncFetcher::new(base, delta); + let mut objects = fetcher + .fetch_objects("rsync://rsync.example.test/repo/") + .expect("fetch base only objects"); + objects.sort_by(|a, b| a.0.cmp(&b.0)); + assert_eq!(objects.len(), 2); + assert_eq!(objects[0].1, b"base-a"); + assert_eq!(objects[1].1, b"base-b"); + } + + #[test] + fn delta_rsync_fetcher_reports_missing_base_subtree_and_exposes_indexes() { + let (_temp, base_archive, base_locks, delta_archive, delta_locks) = + build_base_and_delta_rsync_fixture(); + let base = Arc::new( + ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index"), + ); + let delta = Arc::new( + ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"), + ); + let fetcher = PayloadDeltaReplayRsyncFetcher::new(base.clone(), delta.clone()); + assert_eq!(fetcher.base_index().rsync_modules.len(), 1); + assert_eq!(fetcher.delta_index().rsync_modules.len(), 1); + let err = fetcher + .fetch_objects("rsync://rsync.example.test/repo/missing/") + .unwrap_err(); + match err { + RsyncFetchError::Fetch(msg) => { + assert!( + msg.contains("delta replay rsync subtree not found"), + "{msg}" + ) + } + } + } + + #[test] + fn delta_rsync_fetcher_can_use_overlay_without_base_module() { + let (_temp, base_archive, base_locks, delta_archive, delta_locks) = + build_base_and_delta_rsync_fixture(); + let module_hash = + crate::replay::archive::sha256_hex("rsync://rsync.example.test/repo/".as_bytes()); + std::fs::remove_dir_all( + base_archive + .join("v1/captures/base-cap/rsync/modules") + .join(&module_hash), + ) + .expect("remove base module dir"); + let base = Arc::new( + ReplayArchiveIndex::load_allow_missing_rsync_modules(&base_archive, &base_locks) + .expect("load lenient base index"), + ); + let delta = Arc::new( + ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"), + ); + let fetcher = PayloadDeltaReplayRsyncFetcher::new(base, delta); + let objects = fetcher + .fetch_objects("rsync://rsync.example.test/repo/sub/") + .expect("fetch subtree from overlay only"); + assert_eq!(objects.len(), 1); + assert_eq!(objects[0].0, "rsync://rsync.example.test/repo/sub/b.cer"); + assert_eq!(objects[0].1, b"delta-b"); + } + + #[test] + fn delta_rsync_fetcher_merges_base_and_overlay() { + let (_temp, base_archive, base_locks, delta_archive, delta_locks) = + build_base_and_delta_rsync_fixture(); + let base = Arc::new( + ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index"), + ); + let delta = Arc::new( + ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"), + ); + let fetcher = PayloadDeltaReplayRsyncFetcher::new(base, delta); + let mut objects = fetcher + .fetch_objects("rsync://rsync.example.test/repo/") + .expect("fetch merged objects"); + objects.sort_by(|a, b| a.0.cmp(&b.0)); + assert_eq!(objects.len(), 2); + assert_eq!(objects[0].0, "rsync://rsync.example.test/repo/a.roa"); + assert_eq!(objects[0].1, b"base-a"); + assert_eq!(objects[1].0, "rsync://rsync.example.test/repo/sub/b.cer"); + assert_eq!(objects[1].1, b"delta-b"); + } + + #[test] + fn delta_rsync_fetcher_reads_subtree_and_rejects_missing_base_module() { + let (_temp, base_archive, base_locks, delta_archive, delta_locks) = + build_base_and_delta_rsync_fixture(); + let base = Arc::new( + ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index"), + ); + let delta = Arc::new( + ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"), + ); + let fetcher = PayloadDeltaReplayRsyncFetcher::new(base, delta); + let objects = fetcher + .fetch_objects("rsync://rsync.example.test/repo/sub/") + .expect("fetch subtree"); + assert_eq!(objects.len(), 1); + assert_eq!(objects[0].0, "rsync://rsync.example.test/repo/sub/b.cer"); + assert_eq!(objects[0].1, b"delta-b"); + + let err = fetcher + .fetch_objects("rsync://missing.example/repo/") + .unwrap_err(); + match err { + RsyncFetchError::Fetch(msg) => assert!( + msg.contains("delta replay base rsync module not found") + || msg.contains("no replay lock found for rsync module"), + "{msg}" + ), + } + } +} diff --git a/crates/panda-rpki-validator/src/replay/fetch_http.rs b/crates/panda-rpki-validator/src/replay/fetch_http.rs new file mode 100644 index 0000000..2ed6061 --- /dev/null +++ b/crates/panda-rpki-validator/src/replay/fetch_http.rs @@ -0,0 +1,399 @@ +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use crate::replay::archive::{ReplayArchiveError, ReplayArchiveIndex, ReplayTransport}; +use crate::sync::rrdp::{Fetcher, parse_notification}; + +#[derive(Debug, thiserror::Error)] +pub enum ReplayHttpFetcherError { + #[error(transparent)] + Archive(#[from] ReplayArchiveError), + + #[error("read replay RRDP file failed: {path}: {detail}")] + ReadFile { path: String, detail: String }, + + #[error("parse locked notification failed for {notify_uri}: {detail}")] + ParseNotification { notify_uri: String, detail: String }, + + #[error( + "locked notification session/serial mismatch for {notify_uri}: expected session={expected_session} serial={expected_serial}, got session={actual_session} serial={actual_serial}" + )] + NotificationLockMismatch { + notify_uri: String, + expected_session: String, + expected_serial: u64, + actual_session: String, + actual_serial: u64, + }, + + #[error("locked delta file not found for {notify_uri} at serial {serial}: {path}")] + MissingLockedDelta { + notify_uri: String, + serial: u64, + path: String, + }, + + #[error("duplicate replay HTTP URI mapping for {uri}: {first_path} vs {second_path}")] + DuplicateUriMapping { + uri: String, + first_path: String, + second_path: String, + }, + + #[error("RRDP notification URI is locked to rsync transport in replay: {0}")] + LockedToRsync(String), + + #[error("replay HTTP URI not found in archive: {0}")] + MissingUri(String), +} + +#[derive(Clone, Debug)] +pub struct PayloadReplayHttpFetcher { + index: Arc, + routes: BTreeMap, +} + +impl PayloadReplayHttpFetcher { + pub fn new(index: Arc) -> Result { + let mut routes = BTreeMap::new(); + for (notify_uri, lock) in &index.locks.rrdp { + if lock.transport != ReplayTransport::Rrdp { + continue; + } + let repo = index.require_rrdp_repo(notify_uri)?; + insert_unique_route(&mut routes, notify_uri, &repo.locked_notification_path)?; + + let notification_xml = fs::read(&repo.locked_notification_path).map_err(|e| { + ReplayHttpFetcherError::ReadFile { + path: repo.locked_notification_path.display().to_string(), + detail: e.to_string(), + } + })?; + let notification = parse_notification(¬ification_xml).map_err(|e| { + ReplayHttpFetcherError::ParseNotification { + notify_uri: notify_uri.clone(), + detail: e.to_string(), + } + })?; + let actual_session = notification.session_id.to_string(); + if actual_session != repo.locked_session || notification.serial != repo.locked_serial { + return Err(ReplayHttpFetcherError::NotificationLockMismatch { + notify_uri: notify_uri.clone(), + expected_session: repo.locked_session.clone(), + expected_serial: repo.locked_serial, + actual_session, + actual_serial: notification.serial, + }); + } + + insert_unique_route( + &mut routes, + ¬ification.snapshot_uri, + &repo.locked_snapshot_path, + )?; + + for delta in notification.deltas { + let expected = repo.session_dir.join(format!( + "delta-{}-{}.xml", + delta.serial, + hex::encode(delta.hash_sha256) + )); + if expected.is_file() { + insert_unique_route(&mut routes, &delta.uri, &expected)?; + } + } + } + Ok(Self { index, routes }) + } + + pub fn from_paths( + archive_root: impl AsRef, + locks_path: impl AsRef, + ) -> Result { + let index = Arc::new(ReplayArchiveIndex::load(archive_root, locks_path)?); + Self::new(index) + } + + pub fn archive_index(&self) -> &ReplayArchiveIndex { + self.index.as_ref() + } + + pub fn mapped_route(&self, uri: &str) -> Option<&Path> { + self.routes.get(uri).map(PathBuf::as_path) + } +} + +impl Fetcher for PayloadReplayHttpFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + if let Some(path) = self.routes.get(uri) { + return fs::read(path).map_err(|e| { + ReplayHttpFetcherError::ReadFile { + path: path.display().to_string(), + detail: e.to_string(), + } + .to_string() + }); + } + + if self + .index + .rrdp_lock(uri) + .is_some_and(|lock| lock.transport == ReplayTransport::Rsync) + { + return Err(ReplayHttpFetcherError::LockedToRsync(uri.to_string()).to_string()); + } + + Err(ReplayHttpFetcherError::MissingUri(uri.to_string()).to_string()) + } +} + +fn insert_unique_route( + routes: &mut BTreeMap, + uri: &str, + path: &Path, +) -> Result<(), ReplayHttpFetcherError> { + if let Some(existing) = routes.get(uri) { + if existing != path { + return Err(ReplayHttpFetcherError::DuplicateUriMapping { + uri: uri.to_string(), + first_path: existing.display().to_string(), + second_path: path.display().to_string(), + }); + } + return Ok(()); + } + routes.insert(uri.to_string(), path.to_path_buf()); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::replay::archive::sha256_hex; + + fn build_archive_with_rrdp_and_rsync( + with_delta: bool, + ) -> ( + tempfile::TempDir, + PathBuf, + PathBuf, + String, + String, + String, + Option, + ) { + let temp = tempfile::tempdir().expect("tempdir"); + let archive_root = temp.path().join("payload-archive"); + let capture = "capture-http"; + let capture_root = archive_root.join("v1").join("captures").join(capture); + std::fs::create_dir_all(&capture_root).expect("mkdir capture root"); + std::fs::write( + capture_root.join("capture.json"), + format!( + r#"{{"version":1,"captureId":"{capture}","createdAt":"2026-03-13T00:00:00Z","notes":""}}"# + ), + ) + .expect("write capture meta"); + + let notify_uri = "https://rrdp.example.test/notification.xml".to_string(); + let snapshot_uri = "https://rrdp.example.test/snapshot.xml".to_string(); + let delta_uri = "https://rrdp.example.test/delta.xml".to_string(); + let rsync_locked_notify = "https://rrdp-fallback.example.test/notification.xml".to_string(); + let session = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".to_string(); + let serial = 99u64; + let bucket_hash = sha256_hex(notify_uri.as_bytes()); + let session_dir = capture_root + .join("rrdp/repos") + .join(bucket_hash) + .join(&session); + std::fs::create_dir_all(&session_dir).expect("mkdir session dir"); + std::fs::write( + session_dir.parent().unwrap().join("meta.json"), + format!( + r#"{{"version":1,"rpkiNotify":"{notify_uri}","createdAt":"2026-03-13T00:00:00Z","lastSeenAt":"2026-03-13T00:00:01Z"}}"# + ), + ) + .expect("write repo meta"); + + let snapshot_hash = "00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff"; + let delta_hash = "ffeeddccbbaa99887766554433221100ffeeddccbbaa99887766554433221100"; + let notification_xml = if with_delta { + format!( + r#" + + + +"# + ) + } else { + format!( + r#" + + +"# + ) + }; + std::fs::write(session_dir.join("notification-99.xml"), notification_xml) + .expect("write notification"); + std::fs::write( + session_dir.join(format!("snapshot-99-{snapshot_hash}.xml")), + b"", + ) + .expect("write snapshot"); + if with_delta { + std::fs::write( + session_dir.join(format!("delta-99-{delta_hash}.xml")), + b"", + ) + .expect("write delta"); + } + + let module_uri = "rsync://rsync.example.test/repo/".to_string(); + let module_hash = sha256_hex(module_uri.as_bytes()); + let tree_root = capture_root + .join("rsync/modules") + .join(&module_hash) + .join("tree") + .join("rsync.example.test") + .join("repo"); + std::fs::create_dir_all(&tree_root).expect("mkdir tree root"); + let module_bucket_dir = capture_root.join("rsync/modules").join(&module_hash); + std::fs::write( + module_bucket_dir.join("meta.json"), + format!( + r#"{{"version":1,"module":"{module_uri}","createdAt":"2026-03-13T00:00:00Z","lastSeenAt":"2026-03-13T00:00:01Z"}}"# + ), + ) + .expect("write rsync meta"); + std::fs::write(tree_root.join("x.cer"), b"cer").expect("write tree file"); + + let locks_path = temp.path().join("locks.json"); + std::fs::write( + &locks_path, + format!( + r#"{{ + "version":1, + "capture":"{capture}", + "rrdp":{{ + "{notify_uri}":{{"transport":"rrdp","session":"{session}","serial":{serial}}}, + "{rsync_locked_notify}":{{"transport":"rsync","session":null,"serial":null}} + }}, + "rsync":{{ + "{module_uri}":{{"transport":"rsync"}} + }} +}}"# + ), + ) + .expect("write locks"); + + ( + temp, + archive_root, + locks_path, + notify_uri, + snapshot_uri, + rsync_locked_notify, + with_delta.then_some(delta_uri), + ) + } + + #[test] + fn payload_replay_http_fetcher_reads_locked_notification_and_snapshot_and_delta() { + let (_temp, archive_root, locks_path, notify_uri, snapshot_uri, _rsync_locked, delta_uri) = + build_archive_with_rrdp_and_rsync(true); + let fetcher = PayloadReplayHttpFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay http fetcher"); + + let notification = fetcher.fetch(¬ify_uri).expect("fetch notification"); + assert!( + std::str::from_utf8(¬ification) + .expect("utf8") + .contains(""); + + let delta = fetcher + .fetch(delta_uri.as_deref().expect("delta uri")) + .expect("fetch delta"); + assert_eq!(delta, b""); + } + + #[test] + fn payload_replay_http_fetcher_rejects_notification_locked_to_rsync() { + let (_temp, archive_root, locks_path, _notify_uri, _snapshot_uri, rsync_locked, _delta_uri) = + build_archive_with_rrdp_and_rsync(false); + let fetcher = PayloadReplayHttpFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay http fetcher"); + let err = fetcher.fetch(&rsync_locked).unwrap_err(); + assert!(err.contains("locked to rsync transport"), "{err}"); + } + + #[test] + fn payload_replay_http_fetcher_rejects_unknown_uri() { + let ( + _temp, + archive_root, + locks_path, + _notify_uri, + _snapshot_uri, + _rsync_locked, + _delta_uri, + ) = build_archive_with_rrdp_and_rsync(false); + let fetcher = PayloadReplayHttpFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay http fetcher"); + let err = fetcher + .fetch("https://unknown.example/test.xml") + .unwrap_err(); + assert!(err.contains("not found in archive"), "{err}"); + } + + #[test] + fn payload_replay_http_fetcher_rejects_notification_lock_mismatch() { + let (_temp, archive_root, locks_path, notify_uri, _snapshot_uri, _rsync_locked, _delta_uri) = + build_archive_with_rrdp_and_rsync(false); + let bucket_hash = sha256_hex(notify_uri.as_bytes()); + let notification = archive_root + .join("v1/captures/capture-http/rrdp/repos") + .join(bucket_hash) + .join("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa") + .join("notification-99.xml"); + std::fs::write( + ¬ification, + r#" + + +"#, + ) + .expect("rewrite notification"); + + let err = PayloadReplayHttpFetcher::from_paths(&archive_root, &locks_path).unwrap_err(); + assert!( + matches!(err, ReplayHttpFetcherError::NotificationLockMismatch { .. }), + "{err}" + ); + } + + #[test] + fn payload_replay_http_fetcher_skips_missing_locked_delta_route() { + let (_temp, archive_root, locks_path, notify_uri, _snapshot_uri, _rsync_locked, _delta_uri) = + build_archive_with_rrdp_and_rsync(true); + let bucket_hash = sha256_hex(notify_uri.as_bytes()); + let delta = archive_root + .join("v1/captures/capture-http/rrdp/repos") + .join(bucket_hash) + .join("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa") + .join("delta-99-ffeeddccbbaa99887766554433221100ffeeddccbbaa99887766554433221100.xml"); + std::fs::remove_file(delta).expect("remove delta"); + + let fetcher = PayloadReplayHttpFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay http fetcher without delta route"); + let err = fetcher + .fetch("https://rrdp.example.test/delta.xml") + .unwrap_err(); + assert!(err.contains("not found in archive"), "{err}"); + } +} diff --git a/crates/panda-rpki-validator/src/replay/fetch_rsync.rs b/crates/panda-rpki-validator/src/replay/fetch_rsync.rs new file mode 100644 index 0000000..df8fb91 --- /dev/null +++ b/crates/panda-rpki-validator/src/replay/fetch_rsync.rs @@ -0,0 +1,253 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use crate::fetch::rsync::{RsyncFetchError, RsyncFetchResult, RsyncFetcher}; +use crate::replay::archive::{ReplayArchiveIndex, canonical_rsync_module}; + +#[derive(Clone, Debug)] +pub struct PayloadReplayRsyncFetcher { + index: Arc, +} + +impl PayloadReplayRsyncFetcher { + pub fn new(index: Arc) -> Self { + Self { index } + } + + pub fn from_paths( + archive_root: impl AsRef, + locks_path: impl AsRef, + ) -> Result { + let index = Arc::new( + ReplayArchiveIndex::load(archive_root, locks_path).map_err(|e| e.to_string())?, + ); + Ok(Self::new(index)) + } + + pub fn archive_index(&self) -> &ReplayArchiveIndex { + self.index.as_ref() + } +} + +impl RsyncFetcher for PayloadReplayRsyncFetcher { + fn fetch_objects(&self, rsync_base_uri: &str) -> RsyncFetchResult)>> { + let (module_uri, relative_path) = split_rsync_base_uri(rsync_base_uri) + .map_err(|e| RsyncFetchError::Fetch(e.to_string()))?; + let module = self + .index + .resolve_rsync_module_for_base_uri(rsync_base_uri) + .map_err(|e| RsyncFetchError::Fetch(e.to_string()))?; + let module_root = module_tree_root(&module_uri, &module.tree_dir) + .map_err(|e| RsyncFetchError::Fetch(e.to_string()))?; + let start = if relative_path.as_os_str().is_empty() { + module_root + } else { + module_root.join(&relative_path) + }; + if !start.is_dir() { + return Err(RsyncFetchError::Fetch(format!( + "replay rsync subtree not found: {}", + start.display() + ))); + } + let mut out = Vec::new(); + walk_dir_collect(&start, &start, rsync_base_uri, &mut out) + .map_err(RsyncFetchError::Fetch)?; + Ok(out) + } +} + +fn split_rsync_base_uri(rsync_base_uri: &str) -> Result<(String, PathBuf), String> { + let module_uri = canonical_rsync_module(rsync_base_uri).map_err(|e| e.to_string())?; + let rest = rsync_base_uri + .strip_prefix(&module_uri) + .unwrap_or_default() + .trim_matches('/'); + let relative_path = if rest.is_empty() { + PathBuf::new() + } else { + let mut path = PathBuf::new(); + for segment in rest.split('/') { + if !segment.is_empty() { + path.push(segment); + } + } + path + }; + Ok((module_uri, relative_path)) +} + +fn module_tree_root(module_uri: &str, tree_dir: &Path) -> Result { + let rest = module_uri + .strip_prefix("rsync://") + .ok_or_else(|| format!("invalid rsync module URI: {module_uri}"))?; + let mut parts = rest.trim_end_matches('/').split('/'); + let authority = parts + .next() + .ok_or_else(|| format!("invalid rsync module URI: {module_uri}"))?; + let module = parts + .next() + .ok_or_else(|| format!("invalid rsync module URI: {module_uri}"))?; + Ok(tree_dir.join(authority).join(module)) +} + +fn walk_dir_collect( + root: &Path, + current: &Path, + rsync_base_uri: &str, + out: &mut Vec<(String, Vec)>, +) -> Result<(), String> { + let rd = fs::read_dir(current).map_err(|e| e.to_string())?; + for entry in rd { + let entry = entry.map_err(|e| e.to_string())?; + let path = entry.path(); + let meta = entry.metadata().map_err(|e| e.to_string())?; + if meta.is_dir() { + walk_dir_collect(root, &path, rsync_base_uri, out)?; + continue; + } + if !meta.is_file() { + continue; + } + let rel = path + .strip_prefix(root) + .map_err(|e| e.to_string())? + .to_string_lossy() + .replace('\\', "/"); + let base = if rsync_base_uri.ends_with('/') { + rsync_base_uri.to_string() + } else { + format!("{rsync_base_uri}/") + }; + let uri = format!("{base}{rel}"); + let bytes = fs::read(&path).map_err(|e| e.to_string())?; + out.push((uri, bytes)); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn build_rsync_archive() -> (tempfile::TempDir, PathBuf, PathBuf) { + let temp = tempfile::tempdir().expect("tempdir"); + let archive_root = temp.path().join("payload-archive"); + let capture = "capture-rsync"; + let capture_root = archive_root.join("v1").join("captures").join(capture); + std::fs::create_dir_all(&capture_root).expect("mkdir capture root"); + std::fs::write( + capture_root.join("capture.json"), + format!( + r#"{{"version":1,"captureId":"{capture}","createdAt":"2026-03-13T00:00:00Z","notes":""}}"# + ), + ) + .expect("write capture meta"); + + let module_uri = "rsync://rsync.example.test/repo/"; + let mod_hash = crate::replay::archive::sha256_hex(module_uri.as_bytes()); + let module_root = capture_root + .join("rsync/modules") + .join(&mod_hash) + .join("tree") + .join("rsync.example.test") + .join("repo"); + std::fs::create_dir_all(module_root.join("child")).expect("mkdir module tree"); + let module_bucket_dir = capture_root.join("rsync/modules").join(&mod_hash); + std::fs::write( + module_bucket_dir.join("meta.json"), + format!( + r#"{{"version":1,"module":"{module_uri}","createdAt":"2026-03-13T00:00:00Z","lastSeenAt":"2026-03-13T00:00:01Z"}}"# + ), + ) + .expect("write module meta"); + std::fs::write(module_root.join("a.roa"), b"a").expect("write a.roa"); + std::fs::write(module_root.join("child").join("b.cer"), b"b").expect("write b.cer"); + + let locks_path = temp.path().join("locks.json"); + std::fs::write( + &locks_path, + format!( + r#"{{"version":1,"capture":"{capture}","rrdp":{{}},"rsync":{{"{module_uri}":{{"transport":"rsync"}}}}}}"# + ), + ) + .expect("write locks"); + + (temp, archive_root, locks_path) + } + + #[test] + fn payload_replay_rsync_fetcher_reads_module_root() { + let (_temp, archive_root, locks_path) = build_rsync_archive(); + let fetcher = PayloadReplayRsyncFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay rsync fetcher"); + let mut objects = fetcher + .fetch_objects("rsync://rsync.example.test/repo/") + .expect("fetch root objects"); + objects.sort_by(|a, b| a.0.cmp(&b.0)); + + assert_eq!(objects.len(), 2); + assert_eq!(objects[0].0, "rsync://rsync.example.test/repo/a.roa"); + assert_eq!(objects[0].1, b"a"); + assert_eq!(objects[1].0, "rsync://rsync.example.test/repo/child/b.cer"); + assert_eq!(objects[1].1, b"b"); + } + + #[test] + fn payload_replay_rsync_fetcher_reads_subtree_only() { + let (_temp, archive_root, locks_path) = build_rsync_archive(); + let fetcher = PayloadReplayRsyncFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay rsync fetcher"); + let objects = fetcher + .fetch_objects("rsync://rsync.example.test/repo/child/") + .expect("fetch child subtree"); + + assert_eq!(objects.len(), 1); + assert_eq!(objects[0].0, "rsync://rsync.example.test/repo/child/b.cer"); + assert_eq!(objects[0].1, b"b"); + } + + #[test] + fn payload_replay_rsync_fetcher_rejects_missing_module_and_subtree() { + let (_temp, archive_root, locks_path) = build_rsync_archive(); + let fetcher = PayloadReplayRsyncFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay rsync fetcher"); + + let err = fetcher + .fetch_objects("rsync://missing.example/repo/") + .unwrap_err(); + match err { + RsyncFetchError::Fetch(msg) => assert!( + msg.contains("no replay lock found for rsync module"), + "{msg}" + ), + } + + let err = fetcher + .fetch_objects("rsync://rsync.example.test/repo/missing/") + .unwrap_err(); + match err { + RsyncFetchError::Fetch(msg) => { + assert!(msg.contains("replay rsync subtree not found"), "{msg}") + } + } + } + + #[test] + fn payload_replay_rsync_fetcher_rejects_invalid_uri_and_exposes_index() { + let (_temp, archive_root, locks_path) = build_rsync_archive(); + let fetcher = PayloadReplayRsyncFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay rsync fetcher"); + assert_eq!(fetcher.archive_index().rsync_modules.len(), 1); + + let err = fetcher + .fetch_objects("https://not-rsync.example/repo/") + .unwrap_err(); + match err { + RsyncFetchError::Fetch(msg) => { + assert!(msg.contains("URI must start with rsync://"), "{msg}") + } + } + } +} diff --git a/crates/panda-rpki-validator/src/replay/mod.rs b/crates/panda-rpki-validator/src/replay/mod.rs new file mode 100644 index 0000000..56f5ac9 --- /dev/null +++ b/crates/panda-rpki-validator/src/replay/mod.rs @@ -0,0 +1,6 @@ +pub mod archive; +pub mod delta_archive; +pub mod delta_fetch_http; +pub mod delta_fetch_rsync; +pub mod fetch_http; +pub mod fetch_rsync; diff --git a/crates/panda-rpki-validator/src/report.rs b/crates/panda-rpki-validator/src/report.rs new file mode 100644 index 0000000..942511d --- /dev/null +++ b/crates/panda-rpki-validator/src/report.rs @@ -0,0 +1,57 @@ +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RfcRef(pub &'static str); + +/// Stable categories for warnings written to audit reports and exported metrics. +/// +/// New warning sites should use a specific category when one is available. The +/// default preserves the behaviour and schema of existing callers while making +/// their metric label explicit. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum WarningCategory { + #[default] + Unclassified, + BerCompatibleCmsEncoding, +} + +impl WarningCategory { + pub const fn as_str(self) -> &'static str { + match self { + Self::Unclassified => "unclassified", + Self::BerCompatibleCmsEncoding => "ber_compatible_cms_encoding", + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Warning { + pub message: String, + pub category: WarningCategory, + pub rfc_refs: Vec, + pub context: Option, +} + +impl Warning { + pub fn new(message: impl Into) -> Self { + Self { + message: message.into(), + category: WarningCategory::Unclassified, + rfc_refs: Vec::new(), + context: None, + } + } + + pub fn with_category(mut self, category: WarningCategory) -> Self { + self.category = category; + self + } + + pub fn with_rfc_refs(mut self, refs: &[RfcRef]) -> Self { + self.rfc_refs.extend_from_slice(refs); + self + } + + pub fn with_context(mut self, context: impl Into) -> Self { + self.context = Some(context.into()); + self + } +} diff --git a/crates/panda-rpki-validator/src/storage.rs b/crates/panda-rpki-validator/src/storage.rs new file mode 100644 index 0000000..a0561ee --- /dev/null +++ b/crates/panda-rpki-validator/src/storage.rs @@ -0,0 +1,4862 @@ +mod config; +mod keys; +mod pack; +mod pp_cache_index; + +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +use base64::Engine; +use rocksdb::checkpoint::Checkpoint; +use rocksdb::{ColumnFamily, DB, Direction, IteratorMode, Options, WriteBatch}; +use serde::{Deserialize, Serialize}; + +use crate::blob_store::{ExternalRawStoreDb, ExternalRepoBytesDb, RawObjectStore}; +use crate::data_model::rc::{AsResourceSet, IpResourceSet}; + +use config::*; +pub use config::{ + ALL_COLUMN_FAMILY_NAMES, CF_CHILD_CERTIFICATE_CACHE_PROJECTION, CF_MANIFEST_REPLAY_META, + CF_PUBLICATION_POINT_CACHE_PROJECTION, CF_RAW_BY_HASH, CF_REPOSITORY_VIEW, + CF_ROA_CACHE_PROJECTION, CF_RRDP_SOURCE, CF_RRDP_SOURCE_MEMBER, CF_RRDP_URI_OWNER, + CF_TRANSPORT_PREFETCH, CF_VCIR, CF_VCIR_FAILED_FETCH_REUSE_IDENTITY, column_family_descriptors, +}; +use keys::*; +use pack::compute_sha256_32; +pub use pack::{PackBytes, PackFile, PackTime}; +pub use pp_cache_index::{PpCacheIndexLoadStats, PpCacheIndexRefreshStats}; +use pp_cache_index::{ + PpCacheIndexLookup, PpCacheMmapIndexSet, compact_pp_cache_index, default_pp_cache_index_dir, + load_pp_cache_mmap_index, load_pp_cache_mmap_index_set, pp_cache_index_directory_stats, + write_pp_cache_index_atomic, write_pp_cache_index_segment, +}; +#[derive(Debug, thiserror::Error)] +pub enum StorageError { + #[error("rocksdb error: {0}")] + RocksDb(String), + + #[error("missing column family: {0}")] + MissingColumnFamily(&'static str), + + #[error("cbor codec error for {entity}: {detail}")] + Codec { + entity: &'static str, + detail: String, + }, + + #[error("invalid {entity}: {detail}")] + InvalidData { + entity: &'static str, + detail: String, + }, +} + +pub type StorageResult = Result; + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct RepositoryBlobVerificationSummary { + pub current_objects: u64, + pub bytes_verified: u64, + pub batches: u64, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct ChildCertificateCacheMmapLookup { + pub projections: Vec>, + pub hits: usize, + pub misses: usize, + pub file_bytes: u64, +} + +pub struct RocksStore { + db: DB, + external_raw_store: Option, + external_repo_bytes: Option, + publication_point_cache_index_dir: PathBuf, + child_certificate_cache_index_dir: PathBuf, + publication_point_cache_projection_index: Mutex, +} + +enum PublicationPointCacheProjectionIndexState { + Uninitialized, + Disabled, + BuildingFromEmpty { + index: HashMap>, + bytes: usize, + limit: usize, + }, + Loaded { + index: HashMap>, + bytes: usize, + }, + LoadedMmap { + mmap: PpCacheMmapIndexSet, + dirty: HashMap>, + dirty_bytes: usize, + load_stats: PpCacheIndexLoadStats, + }, +} + +#[derive(Clone, Copy)] +pub(crate) enum PublicationPointCacheProjectionWriteAction<'a> { + Keep, + Write(&'a PublicationPointCacheProjection), + Delete { manifest_rsync_uri: &'a str }, +} + +fn process_vm_rss_kb() -> Option { + let status = std::fs::read_to_string("/proc/self/status").ok()?; + status.lines().find_map(|line| { + let rest = line.strip_prefix("VmRSS:")?; + rest.split_whitespace().next()?.parse::().ok() + }) +} + +fn default_child_certificate_cache_index_dir(db_path: &Path) -> PathBuf { + let file_name = db_path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("work-db"); + db_path.with_file_name(format!("{file_name}.child-cert-cache-index")) +} + +fn child_certificate_cache_segment_file_name(manifest_rsync_uri: &str) -> String { + format!( + "{}.idx", + hex::encode(compute_sha256_32(manifest_rsync_uri.as_bytes())) + ) +} + +const ROCKSDB_MEMORY_PROPERTY_NAMES: &[(&str, &str)] = &[ + ("cur_size_all_mem_tables", "rocksdb.cur-size-all-mem-tables"), + ("size_all_mem_tables", "rocksdb.size-all-mem-tables"), + ( + "estimate_table_readers_mem", + "rocksdb.estimate-table-readers-mem", + ), + ("block_cache_capacity", "rocksdb.block-cache-capacity"), + ("block_cache_usage", "rocksdb.block-cache-usage"), + ( + "block_cache_pinned_usage", + "rocksdb.block-cache-pinned-usage", + ), + ("num_snapshots", "rocksdb.num-snapshots"), + ("background_errors", "rocksdb.background-errors"), +]; + +const PP_CACHE_RAW_INDEX_ENV: &str = "RPKI_PP_CACHE_RAW_INDEX"; +const PP_CACHE_RAW_INDEX_EMPTY_BUILD_LIMIT_ENV: &str = + "RPKI_PP_CACHE_RAW_INDEX_EMPTY_BUILD_LIMIT_BYTES"; +const DEFAULT_PP_CACHE_RAW_INDEX_EMPTY_BUILD_LIMIT_BYTES: usize = 32 * 1024 * 1024; +const PP_CACHE_INDEX_COMPACTION_SEGMENT_THRESHOLD: usize = 16; +const PP_CACHE_INDEX_COMPACTION_BYTES_THRESHOLD: u64 = 1_610_612_736; + +fn pp_cache_raw_index_enabled() -> bool { + match std::env::var(PP_CACHE_RAW_INDEX_ENV) { + Ok(value) => !matches!( + value.trim().to_ascii_lowercase().as_str(), + "0" | "false" | "off" | "no" + ), + Err(_) => true, + } +} + +fn pp_cache_raw_index_empty_build_limit_bytes() -> usize { + std::env::var(PP_CACHE_RAW_INDEX_EMPTY_BUILD_LIMIT_ENV) + .ok() + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or(DEFAULT_PP_CACHE_RAW_INDEX_EMPTY_BUILD_LIMIT_BYTES) +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct RocksDbMemoryProperties { + pub cur_size_all_mem_tables: Option, + pub size_all_mem_tables: Option, + pub estimate_table_readers_mem: Option, + pub block_cache_capacity: Option, + pub block_cache_usage: Option, + pub block_cache_pinned_usage: Option, + pub num_snapshots: Option, + pub background_errors: Option, + pub errors: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct RocksDbColumnFamilyMemoryProperties { + pub name: String, + pub properties: RocksDbMemoryProperties, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct RocksDbMemoryDbSnapshot { + pub label: String, + pub properties: RocksDbMemoryProperties, + pub column_families: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct RocksDbMemoryTotals { + pub cur_size_all_mem_tables: u64, + pub size_all_mem_tables: u64, + pub estimate_table_readers_mem: u64, + pub block_cache_capacity: u64, + pub block_cache_usage: u64, + pub block_cache_pinned_usage: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct RocksDbMemorySnapshot { + pub databases: Vec, + pub totals: RocksDbMemoryTotals, +} + +impl RocksDbMemoryTotals { + fn add_properties(&mut self, properties: &RocksDbMemoryProperties) { + self.cur_size_all_mem_tables += properties.cur_size_all_mem_tables.unwrap_or(0); + self.size_all_mem_tables += properties.size_all_mem_tables.unwrap_or(0); + self.estimate_table_readers_mem += properties.estimate_table_readers_mem.unwrap_or(0); + self.block_cache_capacity += properties.block_cache_capacity.unwrap_or(0); + self.block_cache_usage += properties.block_cache_usage.unwrap_or(0); + self.block_cache_pinned_usage += properties.block_cache_pinned_usage.unwrap_or(0); + } +} + +fn set_memory_property(properties: &mut RocksDbMemoryProperties, name: &str, value: u64) { + match name { + "cur_size_all_mem_tables" => properties.cur_size_all_mem_tables = Some(value), + "size_all_mem_tables" => properties.size_all_mem_tables = Some(value), + "estimate_table_readers_mem" => properties.estimate_table_readers_mem = Some(value), + "block_cache_capacity" => properties.block_cache_capacity = Some(value), + "block_cache_usage" => properties.block_cache_usage = Some(value), + "block_cache_pinned_usage" => properties.block_cache_pinned_usage = Some(value), + "num_snapshots" => properties.num_snapshots = Some(value), + "background_errors" => properties.background_errors = Some(value), + _ => {} + } +} + +fn parse_rocksdb_property_int(raw: Option) -> Option { + raw.and_then(|value| value.trim().parse::().ok()) +} + +fn memory_properties_for_db(db: &DB) -> RocksDbMemoryProperties { + let mut properties = RocksDbMemoryProperties::default(); + for (field_name, property_name) in ROCKSDB_MEMORY_PROPERTY_NAMES { + match db.property_value(*property_name) { + Ok(value) => { + if let Some(parsed) = parse_rocksdb_property_int(value) { + set_memory_property(&mut properties, field_name, parsed); + } + } + Err(err) => properties.errors.push(format!("{property_name}: {}", err)), + } + } + properties +} + +fn memory_properties_for_cf(db: &DB, cf_name: &'static str) -> RocksDbColumnFamilyMemoryProperties { + let mut properties = RocksDbMemoryProperties::default(); + let Some(cf) = db.cf_handle(cf_name) else { + properties + .errors + .push(format!("missing column family: {cf_name}")); + return RocksDbColumnFamilyMemoryProperties { + name: cf_name.to_string(), + properties, + }; + }; + for (field_name, property_name) in ROCKSDB_MEMORY_PROPERTY_NAMES { + match db.property_value_cf(cf, *property_name) { + Ok(value) => { + if let Some(parsed) = parse_rocksdb_property_int(value) { + set_memory_property(&mut properties, field_name, parsed); + } + } + Err(err) => properties.errors.push(format!("{property_name}: {}", err)), + } + } + RocksDbColumnFamilyMemoryProperties { + name: cf_name.to_string(), + properties, + } +} + +pub(crate) fn memory_db_snapshot_for_column_families( + label: impl Into, + db: &DB, + column_families: Option<&[&'static str]>, +) -> RocksDbMemoryDbSnapshot { + RocksDbMemoryDbSnapshot { + label: label.into(), + properties: memory_properties_for_db(db), + column_families: column_families + .map(|names| { + names + .iter() + .map(|name| memory_properties_for_cf(db, name)) + .collect() + }) + .unwrap_or_default(), + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RrdpDeltaOp { + Upsert { rsync_uri: String, bytes: Vec }, + Delete { rsync_uri: String }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RepositoryViewState { + Present, + Withdrawn, + Replaced, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RepositoryViewEntry { + pub rsync_uri: String, + pub current_hash: Option, + pub repository_source: Option, + pub object_type: Option, + pub state: RepositoryViewState, +} + +impl RepositoryViewEntry { + pub fn validate_internal(&self) -> StorageResult<()> { + validate_non_empty("repository_view.rsync_uri", &self.rsync_uri)?; + if let Some(source) = &self.repository_source { + validate_non_empty("repository_view.repository_source", source)?; + } + match self.state { + RepositoryViewState::Present | RepositoryViewState::Replaced => { + let hash = self + .current_hash + .as_deref() + .ok_or(StorageError::InvalidData { + entity: "repository_view", + detail: "current_hash is required when state is present or replaced" + .to_string(), + })?; + validate_sha256_hex("repository_view.current_hash", hash)?; + } + RepositoryViewState::Withdrawn => { + if let Some(hash) = &self.current_hash { + validate_sha256_hex("repository_view.current_hash", hash)?; + } + } + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RawByHashEntry { + pub sha256_hex: String, + pub bytes: Vec, + pub origin_uris: Vec, + pub object_type: Option, + pub encoding: Option, +} + +impl RawByHashEntry { + pub fn from_bytes(sha256_hex: impl Into, bytes: Vec) -> Self { + Self { + sha256_hex: sha256_hex.into(), + bytes, + origin_uris: Vec::new(), + object_type: None, + encoding: None, + } + } + + pub fn validate_internal(&self) -> StorageResult<()> { + validate_sha256_hex("raw_by_hash.sha256_hex", &self.sha256_hex)?; + if self.bytes.is_empty() { + return Err(StorageError::InvalidData { + entity: "raw_by_hash", + detail: "bytes must not be empty".to_string(), + }); + } + let computed = hex::encode(compute_sha256_32(&self.bytes)); + if computed != self.sha256_hex.to_ascii_lowercase() { + return Err(StorageError::InvalidData { + entity: "raw_by_hash", + detail: "sha256_hex does not match bytes".to_string(), + }); + } + let mut seen = HashSet::with_capacity(self.origin_uris.len()); + for uri in &self.origin_uris { + validate_non_empty("raw_by_hash.origin_uris[]", uri)?; + if !seen.insert(uri.as_str()) { + return Err(StorageError::InvalidData { + entity: "raw_by_hash", + detail: format!("duplicate origin URI: {uri}"), + }); + } + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CurrentObjectWithHash { + pub current_hash_hex: String, + pub current_hash: [u8; 32], + pub bytes: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ValidatedManifestMeta { + pub validated_manifest_number: Vec, + pub validated_manifest_this_update: PackTime, + pub validated_manifest_next_update: PackTime, +} + +impl ValidatedManifestMeta { + pub fn validate_internal(&self) -> StorageResult<()> { + validate_manifest_number_be( + "validated_manifest_meta.validated_manifest_number", + &self.validated_manifest_number, + )?; + let this_update = parse_time( + "validated_manifest_meta.validated_manifest_this_update", + &self.validated_manifest_this_update, + )?; + let next_update = parse_time( + "validated_manifest_meta.validated_manifest_next_update", + &self.validated_manifest_next_update, + )?; + if next_update < this_update { + return Err(StorageError::InvalidData { + entity: "validated_manifest_meta", + detail: "validated_manifest_next_update must be >= validated_manifest_this_update" + .to_string(), + }); + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ManifestReplayMeta { + pub manifest_rsync_uri: String, + pub manifest_number_be: Vec, + pub manifest_this_update: PackTime, + pub manifest_sha256: Vec, + pub updated_at_validation_time: PackTime, +} + +impl ManifestReplayMeta { + pub fn from_vcir(vcir: &ValidatedCaInstanceResult) -> Self { + Self { + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + manifest_number_be: vcir + .validated_manifest_meta + .validated_manifest_number + .clone(), + manifest_this_update: vcir + .validated_manifest_meta + .validated_manifest_this_update + .clone(), + manifest_sha256: vcir.ccr_manifest_projection.manifest_sha256.clone(), + updated_at_validation_time: vcir.last_successful_validation_time.clone(), + } + } + + pub fn validate_internal(&self) -> StorageResult<()> { + validate_non_empty( + "manifest_replay_meta.manifest_rsync_uri", + &self.manifest_rsync_uri, + )?; + validate_manifest_number_be( + "manifest_replay_meta.manifest_number_be", + &self.manifest_number_be, + )?; + parse_time( + "manifest_replay_meta.manifest_this_update", + &self.manifest_this_update, + )?; + validate_sha256_digest_bytes( + "manifest_replay_meta.manifest_sha256", + &self.manifest_sha256, + )?; + parse_time( + "manifest_replay_meta.updated_at_validation_time", + &self.updated_at_validation_time, + )?; + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct VcirCcrManifestProjection { + pub manifest_rsync_uri: String, + pub manifest_sha256: Vec, + pub manifest_size: u64, + pub manifest_ee_aki: Vec, + pub manifest_number_be: Vec, + pub manifest_this_update: PackTime, + pub manifest_sia_locations_der: Vec>, + pub subordinate_skis: Vec>, +} + +impl VcirCcrManifestProjection { + pub fn validate_internal(&self) -> StorageResult<()> { + validate_non_empty( + "vcir.ccr_manifest_projection.manifest_rsync_uri", + &self.manifest_rsync_uri, + )?; + validate_sha256_digest_bytes( + "vcir.ccr_manifest_projection.manifest_sha256", + &self.manifest_sha256, + )?; + if self.manifest_size < 1000 { + return Err(StorageError::InvalidData { + entity: "vcir.ccr_manifest_projection.manifest_size", + detail: format!("must be >= 1000, got {}", self.manifest_size), + }); + } + validate_fixed_len_bytes( + "vcir.ccr_manifest_projection.manifest_ee_aki", + &self.manifest_ee_aki, + 20, + )?; + validate_manifest_number_be( + "vcir.ccr_manifest_projection.manifest_number_be", + &self.manifest_number_be, + )?; + parse_time( + "vcir.ccr_manifest_projection.manifest_this_update", + &self.manifest_this_update, + )?; + if self.manifest_sia_locations_der.is_empty() { + return Err(StorageError::InvalidData { + entity: "vcir.ccr_manifest_projection.manifest_sia_locations_der", + detail: "must contain at least one AccessDescription".to_string(), + }); + } + for location in &self.manifest_sia_locations_der { + validate_full_der_with_tag( + "vcir.ccr_manifest_projection.manifest_sia_locations_der[]", + location, + Some(0x30), + )?; + } + validate_sorted_unique_fixed_len_bytes( + "vcir.ccr_manifest_projection.subordinate_skis", + &self.subordinate_skis, + 20, + )?; + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct VcirInstanceGate { + pub manifest_next_update: PackTime, + pub current_crl_next_update: PackTime, + pub self_ca_not_after: PackTime, + pub instance_effective_until: PackTime, +} + +impl VcirInstanceGate { + pub fn validate_internal(&self) -> StorageResult<()> { + let manifest_next_update = parse_time( + "vcir.instance_gate.manifest_next_update", + &self.manifest_next_update, + )?; + let current_crl_next_update = parse_time( + "vcir.instance_gate.current_crl_next_update", + &self.current_crl_next_update, + )?; + let self_ca_not_after = parse_time( + "vcir.instance_gate.self_ca_not_after", + &self.self_ca_not_after, + )?; + let instance_effective_until = parse_time( + "vcir.instance_gate.instance_effective_until", + &self.instance_effective_until, + )?; + let expected = manifest_next_update + .min(current_crl_next_update) + .min(self_ca_not_after); + if instance_effective_until != expected { + return Err(StorageError::InvalidData { + entity: "vcir.instance_gate", + detail: "instance_effective_until must equal min(manifest_next_update, current_crl_next_update, self_ca_not_after)".to_string(), + }); + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct VcirFailedFetchReuseIdentity { + #[serde(rename = "c")] + #[serde(with = "serde_bytes_32")] + pub current_ca_sha256: [u8; 32], + #[serde(rename = "t")] + #[serde(with = "serde_bytes_32")] + pub ta_context_digest: [u8; 32], + #[serde(rename = "p")] + #[serde(with = "serde_bytes_32")] + pub ca_validation_context_digest: [u8; 32], + #[serde(rename = "f")] + #[serde(with = "serde_bytes_32")] + pub policy_fingerprint: [u8; 32], + #[serde(rename = "nb")] + pub effective_not_before: PackTime, + #[serde(rename = "nu")] + pub effective_until: PackTime, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct VcirReuseRecordsCleared { + pub vcir_records: u64, + pub failed_fetch_identity_records: u64, +} + +impl VcirFailedFetchReuseIdentity { + pub fn validate_internal(&self) -> StorageResult<()> { + let effective_not_before = parse_time( + "vcir_failed_fetch_reuse_identity.effective_not_before", + &self.effective_not_before, + )?; + let effective_until = parse_time( + "vcir_failed_fetch_reuse_identity.effective_until", + &self.effective_until, + )?; + if effective_not_before >= effective_until { + return Err(StorageError::InvalidData { + entity: "vcir_failed_fetch_reuse_identity.effective_window", + detail: "effective_not_before must be before effective_until".to_string(), + }); + } + Ok(()) + } + + pub fn contains_validation_time(&self, validation_time: time::OffsetDateTime) -> bool { + let Ok(effective_not_before) = self.effective_not_before.parse() else { + return false; + }; + let Ok(effective_until) = self.effective_until.parse() else { + return false; + }; + validation_time >= effective_not_before && validation_time < effective_until + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct VcirChildEntry { + pub child_manifest_rsync_uri: String, + pub child_cert_rsync_uri: String, + pub child_cert_hash: String, + pub child_ski: String, + pub child_rsync_base_uri: String, + pub child_publication_point_rsync_uri: String, + pub child_rrdp_notification_uri: Option, + pub child_effective_ip_resources: Option, + pub child_effective_as_resources: Option, + pub accepted_at_validation_time: PackTime, +} + +impl VcirChildEntry { + pub fn validate_internal(&self) -> StorageResult<()> { + validate_non_empty( + "vcir.child_entries[].child_manifest_rsync_uri", + &self.child_manifest_rsync_uri, + )?; + validate_non_empty( + "vcir.child_entries[].child_cert_rsync_uri", + &self.child_cert_rsync_uri, + )?; + validate_sha256_hex( + "vcir.child_entries[].child_cert_hash", + &self.child_cert_hash, + )?; + validate_non_empty("vcir.child_entries[].child_ski", &self.child_ski)?; + validate_non_empty( + "vcir.child_entries[].child_rsync_base_uri", + &self.child_rsync_base_uri, + )?; + validate_non_empty( + "vcir.child_entries[].child_publication_point_rsync_uri", + &self.child_publication_point_rsync_uri, + )?; + if let Some(uri) = &self.child_rrdp_notification_uri { + validate_non_empty("vcir.child_entries[].child_rrdp_notification_uri", uri)?; + } + parse_time( + "vcir.child_entries[].accepted_at_validation_time", + &self.accepted_at_validation_time, + )?; + Ok(()) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum VcirOutputType { + Vrp, + Aspa, + RouterKey, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum VcirSourceObjectType { + Roa, + Aspa, + RouterKey, + Other, +} + +impl VcirSourceObjectType { + pub fn as_str(self) -> &'static str { + match self { + Self::Roa => "roa", + Self::Aspa => "aspa", + Self::RouterKey => "router_key", + Self::Other => "other", + } + } +} + +struct FixedBytesVisitor; + +impl<'de, const N: usize> serde::de::Visitor<'de> for FixedBytesVisitor { + type Value = [u8; N]; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(formatter, "{N} bytes") + } + + fn visit_bytes(self, value: &[u8]) -> Result + where + E: serde::de::Error, + { + if value.len() != N { + return Err(E::invalid_length(value.len(), &self)); + } + let mut out = [0u8; N]; + out.copy_from_slice(value); + Ok(out) + } + + fn visit_byte_buf(self, value: Vec) -> Result + where + E: serde::de::Error, + { + self.visit_bytes(&value) + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let mut out = [0u8; N]; + for (idx, slot) in out.iter_mut().enumerate() { + *slot = seq + .next_element()? + .ok_or_else(|| serde::de::Error::invalid_length(idx, &self))?; + } + Ok(out) + } +} + +fn deserialize_fixed_bytes<'de, D, const N: usize>(deserializer: D) -> Result<[u8; N], D::Error> +where + D: serde::Deserializer<'de>, +{ + deserializer.deserialize_bytes(FixedBytesVisitor::) +} + +mod serde_bytes_16 { + pub(super) fn serialize(value: &[u8; 16], serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_bytes(value) + } + + pub(super) fn deserialize<'de, D>(deserializer: D) -> Result<[u8; 16], D::Error> + where + D: serde::Deserializer<'de>, + { + super::deserialize_fixed_bytes::(deserializer) + } +} + +mod serde_bytes_32 { + pub(super) fn serialize(value: &[u8; 32], serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_bytes(value) + } + + pub(super) fn deserialize<'de, D>(deserializer: D) -> Result<[u8; 32], D::Error> + where + D: serde::Deserializer<'de>, + { + super::deserialize_fixed_bytes::(deserializer) + } +} + +struct ByteVecVisitor; + +impl<'de> serde::de::Visitor<'de> for ByteVecVisitor { + type Value = Vec; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("byte vector") + } + + fn visit_bytes(self, value: &[u8]) -> Result + where + E: serde::de::Error, + { + Ok(value.to_vec()) + } + + fn visit_byte_buf(self, value: Vec) -> Result + where + E: serde::de::Error, + { + Ok(value) + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let mut out = Vec::with_capacity(seq.size_hint().unwrap_or(0)); + while let Some(byte) = seq.next_element()? { + out.push(byte); + } + Ok(out) + } +} + +mod serde_byte_vec { + pub(super) fn serialize(value: &[u8], serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.serialize_bytes(value) + } + + pub(super) fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_bytes(super::ByteVecVisitor) + } +} + +mod serde_optional_byte_vec { + pub(super) fn serialize(value: &Option>, serializer: S) -> Result + where + S: serde::Serializer, + { + match value { + Some(bytes) => serializer.serialize_some(bytes), + None => serializer.serialize_none(), + } + } + + pub(super) fn deserialize<'de, D>(deserializer: D) -> Result>, D::Error> + where + D: serde::Deserializer<'de>, + { + struct OptionalByteVecVisitor; + + impl<'de> serde::de::Visitor<'de> for OptionalByteVecVisitor { + type Value = Option>; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("optional byte vector") + } + + fn visit_none(self) -> Result + where + E: serde::de::Error, + { + Ok(None) + } + + fn visit_some(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer + .deserialize_bytes(super::ByteVecVisitor) + .map(Some) + } + } + + deserializer.deserialize_option(OptionalByteVecVisitor) + } +} + +mod serde_optional_bytes_32 { + pub(super) fn serialize(value: &Option<[u8; 32]>, serializer: S) -> Result + where + S: serde::Serializer, + { + match value { + Some(bytes) => serializer.serialize_some(bytes.as_slice()), + None => serializer.serialize_none(), + } + } + + pub(super) fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> + where + D: serde::Deserializer<'de>, + { + struct OptionalBytes32Visitor; + + impl<'de> serde::de::Visitor<'de> for OptionalBytes32Visitor { + type Value = Option<[u8; 32]>; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("optional 32-byte array") + } + + fn visit_none(self) -> Result + where + E: serde::de::Error, + { + Ok(None) + } + + fn visit_some(self, deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + super::deserialize_fixed_bytes::(deserializer).map(Some) + } + } + + deserializer.deserialize_option(OptionalBytes32Visitor) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum VcirLocalOutputPayload { + Vrp { + asn: u32, + afi: crate::data_model::roa::RoaAfi, + prefix_len: u16, + #[serde(with = "serde_bytes_16")] + addr: [u8; 16], + max_length: u16, + }, + Aspa { + customer_as_id: u32, + provider_as_ids: Vec, + }, + RouterKey { + as_id: u32, + #[serde(with = "serde_byte_vec")] + ski: Vec, + #[serde(with = "serde_byte_vec")] + spki_der: Vec, + }, +} + +impl VcirLocalOutputPayload { + pub fn typed_body_bytes(&self) -> u64 { + match self { + Self::Vrp { .. } => 4 + 1 + 2 + 16 + 2, + Self::Aspa { + provider_as_ids, .. + } => 4 + (provider_as_ids.len() as u64 * 4), + Self::RouterKey { ski, spki_der, .. } => 4 + ski.len() as u64 + spki_der.len() as u64, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct VcirLocalOutput { + #[serde(rename = "t")] + pub output_type: VcirOutputType, + #[serde(rename = "e")] + pub item_effective_until: PackTime, + #[serde(rename = "u")] + pub source_object_uri: String, + #[serde(rename = "k")] + pub source_object_type: VcirSourceObjectType, + #[serde(rename = "h")] + #[serde(with = "serde_bytes_32")] + pub source_object_hash: [u8; 32], + #[serde(rename = "c")] + #[serde(with = "serde_bytes_32")] + pub source_ee_cert_hash: [u8; 32], + #[serde(rename = "p")] + pub payload: VcirLocalOutputPayload, + #[serde(rename = "r")] + #[serde(with = "serde_bytes_32")] + pub rule_hash: [u8; 32], +} + +impl VcirLocalOutput { + pub fn output_id(&self) -> String { + self.rule_hash_hex() + } + + pub fn source_object_hash_hex(&self) -> String { + hex::encode(self.source_object_hash) + } + + pub fn source_ee_cert_hash_hex(&self) -> String { + hex::encode(self.source_ee_cert_hash) + } + + pub fn rule_hash_hex(&self) -> String { + hex::encode(self.rule_hash) + } + + pub fn source_object_type_name(&self) -> &'static str { + self.source_object_type.as_str() + } + + pub fn payload_json(&self) -> String { + match &self.payload { + VcirLocalOutputPayload::Vrp { + asn, + afi, + prefix_len, + addr, + max_length, + } => { + let prefix = match afi { + crate::data_model::roa::RoaAfi::Ipv4 => { + let ip = std::net::Ipv4Addr::new(addr[0], addr[1], addr[2], addr[3]); + format!("{ip}/{prefix_len}") + } + crate::data_model::roa::RoaAfi::Ipv6 => { + let ip = std::net::Ipv6Addr::from(*addr); + format!("{ip}/{prefix_len}") + } + }; + format!(r#"{{"asn":{asn},"max_length":{max_length},"prefix":"{prefix}"}}"#) + } + VcirLocalOutputPayload::Aspa { + customer_as_id, + provider_as_ids, + } => { + let providers = provider_as_ids + .iter() + .map(u32::to_string) + .collect::>() + .join(","); + format!(r#"{{"customer_as_id":{customer_as_id},"provider_as_ids":[{providers}]}}"#) + } + VcirLocalOutputPayload::RouterKey { + as_id, + ski, + spki_der, + } => { + let ski_hex = hex::encode(ski); + let spki_der_base64 = base64::engine::general_purpose::STANDARD.encode(spki_der); + format!( + r#"{{"as_id":{as_id},"ski_hex":"{ski_hex}","spki_der_base64":"{spki_der_base64}"}}"# + ) + } + } + } + + pub fn validate_internal(&self) -> StorageResult<()> { + parse_time( + "vcir.local_outputs[].item_effective_until", + &self.item_effective_until, + )?; + validate_non_empty( + "vcir.local_outputs[].source_object_uri", + &self.source_object_uri, + )?; + validate_local_output_type_matches_payload(self)?; + Ok(()) + } +} + +fn validate_local_output_type_matches_payload(output: &VcirLocalOutput) -> StorageResult<()> { + let matches_payload = matches!( + (&output.output_type, &output.payload), + (VcirOutputType::Vrp, VcirLocalOutputPayload::Vrp { .. }) + | (VcirOutputType::Aspa, VcirLocalOutputPayload::Aspa { .. }) + | ( + VcirOutputType::RouterKey, + VcirLocalOutputPayload::RouterKey { .. } + ) + ); + if !matches_payload { + return Err(StorageError::InvalidData { + entity: "vcir.local_outputs[]", + detail: "output_type must match payload variant".to_string(), + }); + } + Ok(()) +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RoaCacheCrlProjection { + #[serde(rename = "u")] + pub uri: String, + #[serde(rename = "h")] + pub sha256: String, +} + +impl RoaCacheCrlProjection { + pub fn validate_internal(&self) -> StorageResult<()> { + validate_non_empty("roa_cache_projection.crls[].uri", &self.uri)?; + validate_sha256_hex("roa_cache_projection.crls[].sha256", &self.sha256)?; + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RoaCacheObjectMeta { + pub source_object_uri: String, + pub source_object_hash: [u8; 32], + pub ee_serial: Vec, + pub crl_uri: String, + pub earliest_safe_reuse_time: PackTime, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RoaCacheProjectionContext { + pub ca_validation_context_digest: [u8; 32], + pub policy_fingerprint: [u8; 32], + pub object_meta: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RoaCacheLocalOutputProjection { + #[serde(rename = "e")] + pub item_effective_until: PackTime, + #[serde(rename = "c")] + #[serde(with = "serde_bytes_32")] + pub source_ee_cert_hash: [u8; 32], + #[serde(rename = "p")] + pub payload: VcirLocalOutputPayload, + #[serde(rename = "r")] + #[serde(with = "serde_bytes_32")] + pub rule_hash: [u8; 32], +} + +impl RoaCacheLocalOutputProjection { + fn from_local_output(output: &VcirLocalOutput) -> Option { + if output.output_type != VcirOutputType::Vrp + || output.source_object_type != VcirSourceObjectType::Roa + { + return None; + } + Some(Self { + item_effective_until: output.item_effective_until.clone(), + source_ee_cert_hash: output.source_ee_cert_hash, + payload: output.payload.clone(), + rule_hash: output.rule_hash, + }) + } + + pub fn validate_internal(&self) -> StorageResult<()> { + parse_time( + "roa_cache_projection.entries[].outputs[].item_effective_until", + &self.item_effective_until, + )?; + if !matches!(self.payload, VcirLocalOutputPayload::Vrp { .. }) { + return Err(StorageError::InvalidData { + entity: "roa_cache_projection.entries[].outputs[]", + detail: "payload must be VRP".to_string(), + }); + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RoaCacheObjectProjection { + #[serde(rename = "u")] + pub source_object_uri: String, + #[serde(rename = "h")] + #[serde(with = "serde_bytes_32")] + pub source_object_hash: [u8; 32], + #[serde(rename = "s", default, skip_serializing_if = "Option::is_none")] + #[serde(with = "serde_optional_byte_vec")] + pub ee_serial: Option>, + #[serde(rename = "c", default, skip_serializing_if = "Option::is_none")] + pub crl_uri: Option, + #[serde(rename = "n", default, skip_serializing_if = "Option::is_none")] + pub earliest_safe_reuse_time_unix: Option, + #[serde(rename = "x")] + pub outputs_effective_until_unix: i64, + #[serde(rename = "o")] + pub outputs: Vec, +} + +impl RoaCacheObjectProjection { + pub fn validate_internal(&self) -> StorageResult<()> { + validate_non_empty( + "roa_cache_projection.entries[].source_object_uri", + &self.source_object_uri, + )?; + if let Some(serial) = &self.ee_serial { + if serial.is_empty() { + return Err(StorageError::InvalidData { + entity: "roa_cache_projection.entries[].ee_serial", + detail: "must not be empty when present".to_string(), + }); + } + } + if let Some(crl_uri) = &self.crl_uri { + validate_non_empty("roa_cache_projection.entries[].crl_uri", crl_uri)?; + } + if let Some(earliest_safe_reuse_time_unix) = self.earliest_safe_reuse_time_unix + && earliest_safe_reuse_time_unix >= self.outputs_effective_until_unix + { + return Err(StorageError::InvalidData { + entity: "roa_cache_projection.entries[].effective_window", + detail: "earliest_safe_reuse_time_unix must be before outputs_effective_until_unix" + .to_string(), + }); + } + if self.outputs.is_empty() { + return Err(StorageError::InvalidData { + entity: "roa_cache_projection.entries[]", + detail: "outputs must not be empty".to_string(), + }); + } + for output in &self.outputs { + output.validate_internal()?; + } + let expected_effective_until = self + .outputs + .iter() + .map(|output| { + output + .item_effective_until + .parse() + .map(|time| time.unix_timestamp()) + .map_err(|detail| StorageError::InvalidData { + entity: "roa_cache_projection.entries[].outputs_effective_until_unix", + detail, + }) + }) + .collect::>>()? + .into_iter() + .min() + .expect("outputs must not be empty"); + if self.outputs_effective_until_unix != expected_effective_until { + return Err(StorageError::InvalidData { + entity: "roa_cache_projection.entries[].outputs_effective_until_unix", + detail: "must equal the earliest output item_effective_until".to_string(), + }); + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RoaCacheProjection { + #[serde(rename = "m")] + pub manifest_rsync_uri: String, + #[serde(rename = "e")] + pub instance_effective_until: PackTime, + #[serde(rename = "i")] + pub issuer_ca_sha256_hex: Option, + #[serde(rename = "p", default, skip_serializing_if = "Option::is_none")] + #[serde(with = "serde_optional_bytes_32")] + pub ca_validation_context_digest: Option<[u8; 32]>, + #[serde(rename = "f", default, skip_serializing_if = "Option::is_none")] + #[serde(with = "serde_optional_bytes_32")] + pub policy_fingerprint: Option<[u8; 32]>, + #[serde(rename = "c")] + pub crl_sha256_by_uri: Vec, + #[serde(rename = "r")] + pub entries: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ChildCertificateCacheRouterKeyProjection { + #[serde(rename = "a")] + pub as_id: u32, + #[serde(rename = "s")] + #[serde(with = "serde_byte_vec")] + pub ski: Vec, + #[serde(rename = "p")] + #[serde(with = "serde_byte_vec")] + pub spki_der: Vec, + #[serde(rename = "e")] + pub item_effective_until: PackTime, +} + +impl ChildCertificateCacheRouterKeyProjection { + pub fn validate_internal(&self) -> StorageResult<()> { + if self.ski.is_empty() { + return Err(StorageError::InvalidData { + entity: "child_certificate_cache_projection.router_keys[].ski", + detail: "must not be empty".to_string(), + }); + } + if self.spki_der.is_empty() { + return Err(StorageError::InvalidData { + entity: "child_certificate_cache_projection.router_keys[].spki_der", + detail: "must not be empty".to_string(), + }); + } + parse_time( + "child_certificate_cache_projection.router_keys[].item_effective_until", + &self.item_effective_until, + )?; + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ChildCertificateCachePayload { + ChildCa { + #[serde(rename = "cm")] + child_manifest_rsync_uri: String, + #[serde(rename = "ski")] + child_ski: String, + #[serde(rename = "rb")] + child_rsync_base_uri: String, + #[serde(rename = "pp")] + child_publication_point_rsync_uri: String, + #[serde(rename = "rn")] + child_rrdp_notification_uri: Option, + #[serde(rename = "ip")] + child_effective_ip_resources: Option, + #[serde(rename = "as")] + child_effective_as_resources: Option, + }, + Router { + #[serde(rename = "r")] + router_keys: Vec, + }, +} + +impl ChildCertificateCachePayload { + pub fn validate_internal(&self) -> StorageResult<()> { + match self { + Self::ChildCa { + child_manifest_rsync_uri, + child_ski, + child_rsync_base_uri, + child_publication_point_rsync_uri, + child_rrdp_notification_uri, + .. + } => { + validate_non_empty( + "child_certificate_cache_projection.child_manifest_rsync_uri", + child_manifest_rsync_uri, + )?; + validate_non_empty("child_certificate_cache_projection.child_ski", child_ski)?; + validate_non_empty( + "child_certificate_cache_projection.child_rsync_base_uri", + child_rsync_base_uri, + )?; + validate_non_empty( + "child_certificate_cache_projection.child_publication_point_rsync_uri", + child_publication_point_rsync_uri, + )?; + if let Some(uri) = child_rrdp_notification_uri { + validate_non_empty( + "child_certificate_cache_projection.child_rrdp_notification_uri", + uri, + )?; + } + Ok(()) + } + Self::Router { router_keys } => { + if router_keys.is_empty() { + return Err(StorageError::InvalidData { + entity: "child_certificate_cache_projection.router_keys", + detail: "must not be empty".to_string(), + }); + } + for router_key in router_keys { + router_key.validate_internal()?; + } + Ok(()) + } + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ChildCertificateCacheProjection { + #[serde(rename = "sv")] + pub schema_version: u32, + #[serde(rename = "av")] + pub algorithm_version: u32, + #[serde(rename = "k")] + pub cache_key_sha256_hex: String, + #[serde(rename = "cu")] + pub child_cert_uri: String, + #[serde(rename = "ch")] + pub child_cert_sha256_hex: String, + #[serde(rename = "cs")] + #[serde(with = "serde_byte_vec")] + pub child_cert_serial: Vec, + #[serde(rename = "ih")] + pub issuer_ca_sha256_hex: String, + #[serde(rename = "cru")] + pub issuer_crl_uri: String, + #[serde(rename = "crh")] + pub issuer_crl_sha256_hex: String, + #[serde(rename = "pc")] + #[serde(with = "serde_bytes_32")] + pub ca_validation_context_digest: [u8; 32], + #[serde(rename = "pf")] + #[serde(with = "serde_bytes_32")] + pub validation_policy_fingerprint: [u8; 32], + #[serde(rename = "nb")] + pub effective_not_before: PackTime, + #[serde(rename = "nu")] + pub effective_until: PackTime, + #[serde(rename = "p")] + pub payload: ChildCertificateCachePayload, +} + +pub const CHILD_CERTIFICATE_CACHE_SCHEMA_VERSION: u32 = 2; +pub const CHILD_CERTIFICATE_CACHE_ALGORITHM_VERSION: u32 = 3; + +impl ChildCertificateCacheProjection { + pub fn validate_internal(&self) -> StorageResult<()> { + if self.schema_version != CHILD_CERTIFICATE_CACHE_SCHEMA_VERSION { + return Err(StorageError::InvalidData { + entity: "child_certificate_cache_projection.schema_version", + detail: format!("unsupported schema_version {}", self.schema_version), + }); + } + if self.algorithm_version != CHILD_CERTIFICATE_CACHE_ALGORITHM_VERSION { + return Err(StorageError::InvalidData { + entity: "child_certificate_cache_projection.algorithm_version", + detail: format!("unsupported algorithm_version {}", self.algorithm_version), + }); + } + validate_sha256_hex( + "child_certificate_cache_projection.cache_key_sha256_hex", + &self.cache_key_sha256_hex, + )?; + validate_non_empty( + "child_certificate_cache_projection.child_cert_uri", + &self.child_cert_uri, + )?; + validate_sha256_hex( + "child_certificate_cache_projection.child_cert_sha256_hex", + &self.child_cert_sha256_hex, + )?; + if self.child_cert_serial.is_empty() { + return Err(StorageError::InvalidData { + entity: "child_certificate_cache_projection.child_cert_serial", + detail: "must not be empty".to_string(), + }); + } + validate_sha256_hex( + "child_certificate_cache_projection.issuer_ca_sha256_hex", + &self.issuer_ca_sha256_hex, + )?; + validate_non_empty( + "child_certificate_cache_projection.issuer_crl_uri", + &self.issuer_crl_uri, + )?; + validate_sha256_hex( + "child_certificate_cache_projection.issuer_crl_sha256_hex", + &self.issuer_crl_sha256_hex, + )?; + let effective_not_before = parse_time( + "child_certificate_cache_projection.effective_not_before", + &self.effective_not_before, + )?; + let effective_until = parse_time( + "child_certificate_cache_projection.effective_until", + &self.effective_until, + )?; + if effective_not_before >= effective_until { + return Err(StorageError::InvalidData { + entity: "child_certificate_cache_projection.effective_window", + detail: "effective_not_before must be before effective_until".to_string(), + }); + } + self.payload.validate_internal()?; + Ok(()) + } +} + +pub const PUBLICATION_POINT_CACHE_SCHEMA_VERSION: u32 = 1; +pub const PUBLICATION_POINT_CACHE_ALGORITHM_VERSION: u32 = 1; + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PublicationPointCacheOutput { + #[serde(rename = "t")] + pub output_type: VcirOutputType, + #[serde(rename = "nb")] + pub item_effective_not_before: PackTime, + #[serde(rename = "nu")] + pub item_effective_until: PackTime, + #[serde(rename = "u")] + pub source_object_uri: String, + #[serde(rename = "k")] + pub source_object_type: VcirSourceObjectType, + #[serde(rename = "h")] + #[serde(with = "serde_bytes_32")] + pub source_object_hash: [u8; 32], + #[serde(rename = "c")] + #[serde(with = "serde_bytes_32")] + pub source_ee_cert_hash: [u8; 32], + #[serde(rename = "p")] + pub payload: VcirLocalOutputPayload, + #[serde(rename = "r")] + #[serde(with = "serde_bytes_32")] + pub rule_hash: [u8; 32], +} + +impl PublicationPointCacheOutput { + fn from_local_output(output: &VcirLocalOutput, default_not_before: &PackTime) -> Self { + Self { + output_type: output.output_type, + item_effective_not_before: default_not_before.clone(), + item_effective_until: output.item_effective_until.clone(), + source_object_uri: output.source_object_uri.clone(), + source_object_type: output.source_object_type, + source_object_hash: output.source_object_hash, + source_ee_cert_hash: output.source_ee_cert_hash, + payload: output.payload.clone(), + rule_hash: output.rule_hash, + } + } + + pub fn to_local_output(&self) -> VcirLocalOutput { + VcirLocalOutput { + output_type: self.output_type, + item_effective_until: self.item_effective_until.clone(), + source_object_uri: self.source_object_uri.clone(), + source_object_type: self.source_object_type, + source_object_hash: self.source_object_hash, + source_ee_cert_hash: self.source_ee_cert_hash, + payload: self.payload.clone(), + rule_hash: self.rule_hash, + } + } + + pub fn validate_internal(&self) -> StorageResult<()> { + parse_time( + "publication_point_cache_projection.outputs[].item_effective_not_before", + &self.item_effective_not_before, + )?; + parse_time( + "publication_point_cache_projection.outputs[].item_effective_until", + &self.item_effective_until, + )?; + validate_non_empty( + "publication_point_cache_projection.outputs[].source_object_uri", + &self.source_object_uri, + )?; + VcirLocalOutput { + output_type: self.output_type, + item_effective_until: self.item_effective_until.clone(), + source_object_uri: self.source_object_uri.clone(), + source_object_type: self.source_object_type, + source_object_hash: self.source_object_hash, + source_ee_cert_hash: self.source_ee_cert_hash, + payload: self.payload.clone(), + rule_hash: self.rule_hash, + } + .validate_internal() + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PublicationPointCacheChild { + #[serde(rename = "cm")] + pub child_manifest_rsync_uri: String, + #[serde(rename = "cu")] + pub child_cert_rsync_uri: String, + #[serde(rename = "ch")] + pub child_cert_hash: String, + #[serde(rename = "ski")] + pub child_ski: String, + #[serde(rename = "rb")] + pub child_rsync_base_uri: String, + #[serde(rename = "pp")] + pub child_publication_point_rsync_uri: String, + #[serde(rename = "rn")] + pub child_rrdp_notification_uri: Option, + #[serde(rename = "ip")] + pub child_effective_ip_resources: Option, + #[serde(rename = "as")] + pub child_effective_as_resources: Option, + #[serde(rename = "nb")] + pub child_effective_not_before: PackTime, + #[serde(rename = "nu")] + pub child_effective_until: PackTime, +} + +impl PublicationPointCacheChild { + fn from_child_entry( + entry: &VcirChildEntry, + default_not_before: &PackTime, + default_until: &PackTime, + ) -> Self { + Self { + child_manifest_rsync_uri: entry.child_manifest_rsync_uri.clone(), + child_cert_rsync_uri: entry.child_cert_rsync_uri.clone(), + child_cert_hash: entry.child_cert_hash.clone(), + child_ski: entry.child_ski.clone(), + child_rsync_base_uri: entry.child_rsync_base_uri.clone(), + child_publication_point_rsync_uri: entry.child_publication_point_rsync_uri.clone(), + child_rrdp_notification_uri: entry.child_rrdp_notification_uri.clone(), + child_effective_ip_resources: entry.child_effective_ip_resources.clone(), + child_effective_as_resources: entry.child_effective_as_resources.clone(), + child_effective_not_before: default_not_before.clone(), + child_effective_until: default_until.clone(), + } + } + + pub fn to_child_entry(&self, accepted_at_validation_time: PackTime) -> VcirChildEntry { + VcirChildEntry { + child_manifest_rsync_uri: self.child_manifest_rsync_uri.clone(), + child_cert_rsync_uri: self.child_cert_rsync_uri.clone(), + child_cert_hash: self.child_cert_hash.clone(), + child_ski: self.child_ski.clone(), + child_rsync_base_uri: self.child_rsync_base_uri.clone(), + child_publication_point_rsync_uri: self.child_publication_point_rsync_uri.clone(), + child_rrdp_notification_uri: self.child_rrdp_notification_uri.clone(), + child_effective_ip_resources: self.child_effective_ip_resources.clone(), + child_effective_as_resources: self.child_effective_as_resources.clone(), + accepted_at_validation_time, + } + } + + pub fn validate_internal(&self) -> StorageResult<()> { + self.to_child_entry(self.child_effective_not_before.clone()) + .validate_internal()?; + parse_time( + "publication_point_cache_projection.children[].child_effective_not_before", + &self.child_effective_not_before, + )?; + parse_time( + "publication_point_cache_projection.children[].child_effective_until", + &self.child_effective_until, + )?; + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PublicationPointCacheObject { + #[serde(rename = "r")] + pub artifact_role: VcirArtifactRole, + #[serde(rename = "k")] + pub artifact_kind: VcirArtifactKind, + #[serde(rename = "u")] + pub uri: Option, + #[serde(rename = "h")] + pub sha256: String, + #[serde(rename = "t")] + pub object_type: Option, + #[serde(rename = "s")] + pub validation_status: VcirArtifactValidationStatus, + /// See `VcirRelatedArtifact::reject_reason`. + #[serde(rename = "e", default, skip_serializing_if = "Option::is_none")] + pub reject_reason: Option, +} + +impl PublicationPointCacheObject { + fn from_related_artifact(artifact: &VcirRelatedArtifact) -> Self { + Self { + artifact_role: artifact.artifact_role, + artifact_kind: artifact.artifact_kind, + uri: artifact.uri.clone(), + sha256: artifact.sha256.clone(), + object_type: artifact.object_type.clone(), + validation_status: artifact.validation_status, + reject_reason: artifact.reject_reason.clone(), + } + } + + pub fn to_related_artifact(&self) -> VcirRelatedArtifact { + VcirRelatedArtifact { + artifact_role: self.artifact_role, + artifact_kind: self.artifact_kind, + uri: self.uri.clone(), + sha256: self.sha256.clone(), + object_type: self.object_type.clone(), + validation_status: self.validation_status, + reject_reason: self.reject_reason.clone(), + } + } + + pub fn validate_internal(&self) -> StorageResult<()> { + self.to_related_artifact().validate_internal() + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PublicationPointCacheProjection { + #[serde(rename = "sv")] + pub schema_version: u32, + #[serde(rename = "av")] + pub algorithm_version: u32, + #[serde(rename = "m")] + pub manifest_rsync_uri: String, + #[serde(rename = "pp")] + pub publication_point_rsync_uri: String, + #[serde(rename = "cu")] + pub ca_cert_uri: Option, + #[serde(rename = "ch")] + #[serde(with = "serde_bytes_32")] + pub ca_cert_sha256: [u8; 32], + #[serde(rename = "mh")] + #[serde(with = "serde_bytes_32")] + pub manifest_sha256: [u8; 32], + #[serde(rename = "tal")] + pub tal_id: String, + #[serde(rename = "ta")] + #[serde(with = "serde_bytes_32")] + pub ta_context_digest: [u8; 32], + #[serde(rename = "pc")] + #[serde(with = "serde_bytes_32")] + pub ca_validation_context_digest: [u8; 32], + #[serde(rename = "pf")] + #[serde(with = "serde_bytes_32")] + pub validation_policy_fingerprint: [u8; 32], + #[serde(rename = "nb")] + pub instance_effective_not_before: PackTime, + #[serde(rename = "nu")] + pub instance_effective_until: PackTime, + #[serde(rename = "mt")] + pub manifest_this_update: PackTime, + #[serde(rename = "mn")] + pub manifest_next_update: PackTime, + #[serde(rename = "cn")] + pub current_crl_next_update: PackTime, + #[serde(rename = "ca")] + pub self_ca_not_after: PackTime, + #[serde(rename = "ccr")] + pub ccr_manifest_projection: VcirCcrManifestProjection, + #[serde(rename = "o")] + pub outputs: Vec, + #[serde(rename = "c")] + pub children: Vec, + #[serde(rename = "ra")] + pub related_objects: Vec, + #[serde(rename = "s")] + pub summary: VcirSummary, +} + +impl PublicationPointCacheProjection { + pub fn from_vcir_with_context( + vcir: &ValidatedCaInstanceResult, + publication_point_rsync_uri: String, + ca_cert_uri: Option, + ca_cert_sha256: [u8; 32], + manifest_sha256: [u8; 32], + ta_context_digest: [u8; 32], + ca_validation_context_digest: [u8; 32], + validation_policy_fingerprint: [u8; 32], + ) -> StorageResult { + let projection = Self { + schema_version: PUBLICATION_POINT_CACHE_SCHEMA_VERSION, + algorithm_version: PUBLICATION_POINT_CACHE_ALGORITHM_VERSION, + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri, + ca_cert_uri, + ca_cert_sha256, + manifest_sha256, + tal_id: vcir.tal_id.clone(), + ta_context_digest, + ca_validation_context_digest, + validation_policy_fingerprint, + instance_effective_not_before: vcir.last_successful_validation_time.clone(), + instance_effective_until: vcir.instance_gate.instance_effective_until.clone(), + manifest_this_update: vcir + .validated_manifest_meta + .validated_manifest_this_update + .clone(), + manifest_next_update: vcir.instance_gate.manifest_next_update.clone(), + current_crl_next_update: vcir.instance_gate.current_crl_next_update.clone(), + self_ca_not_after: vcir.instance_gate.self_ca_not_after.clone(), + ccr_manifest_projection: vcir.ccr_manifest_projection.clone(), + outputs: vcir + .local_outputs + .iter() + .map(|output| { + PublicationPointCacheOutput::from_local_output( + output, + &vcir.last_successful_validation_time, + ) + }) + .collect(), + children: vcir + .child_entries + .iter() + .map(|child| { + PublicationPointCacheChild::from_child_entry( + child, + &vcir.last_successful_validation_time, + &vcir.instance_gate.instance_effective_until, + ) + }) + .collect(), + related_objects: vcir + .related_artifacts + .iter() + .map(PublicationPointCacheObject::from_related_artifact) + .collect(), + summary: vcir.summary.clone(), + }; + projection.validate_internal()?; + Ok(projection) + } + + pub fn to_vcir_for_reuse( + &self, + validation_time: time::OffsetDateTime, + ) -> ValidatedCaInstanceResult { + let validation_time = PackTime::from_utc_offset_datetime(validation_time); + ValidatedCaInstanceResult { + manifest_rsync_uri: self.manifest_rsync_uri.clone(), + parent_manifest_rsync_uri: None, + tal_id: self.tal_id.clone(), + ca_subject_name: String::from("publication-point-cache"), + ca_ski: hex::encode(self.ca_cert_sha256), + issuer_ski: hex::encode(self.ca_validation_context_digest), + last_successful_validation_time: validation_time.clone(), + current_manifest_rsync_uri: self.manifest_rsync_uri.clone(), + current_crl_rsync_uri: self + .related_objects + .iter() + .find(|object| object.artifact_role == VcirArtifactRole::CurrentCrl) + .and_then(|object| object.uri.clone()) + .unwrap_or_default(), + validated_manifest_meta: ValidatedManifestMeta { + validated_manifest_number: self.ccr_manifest_projection.manifest_number_be.clone(), + validated_manifest_this_update: self.manifest_this_update.clone(), + validated_manifest_next_update: self.manifest_next_update.clone(), + }, + ccr_manifest_projection: self.ccr_manifest_projection.clone(), + instance_gate: VcirInstanceGate { + manifest_next_update: self.manifest_next_update.clone(), + current_crl_next_update: self.current_crl_next_update.clone(), + self_ca_not_after: self.self_ca_not_after.clone(), + instance_effective_until: self.instance_effective_until.clone(), + }, + child_entries: self + .children + .iter() + .map(|child| child.to_child_entry(validation_time.clone())) + .collect(), + local_outputs: self + .outputs + .iter() + .map(PublicationPointCacheOutput::to_local_output) + .collect(), + related_artifacts: self + .related_objects + .iter() + .map(PublicationPointCacheObject::to_related_artifact) + .collect(), + summary: self.summary.clone(), + audit_summary: VcirAuditSummary { + failed_fetch_eligible: false, + last_failed_fetch_reason: None, + warning_count: 0, + audit_flags: vec!["publication_point_cache_projection".to_string()], + }, + } + } + + pub fn validate_internal(&self) -> StorageResult<()> { + if self.schema_version != PUBLICATION_POINT_CACHE_SCHEMA_VERSION { + return Err(StorageError::InvalidData { + entity: "publication_point_cache_projection.schema_version", + detail: format!("unsupported schema_version {}", self.schema_version), + }); + } + if self.algorithm_version != PUBLICATION_POINT_CACHE_ALGORITHM_VERSION { + return Err(StorageError::InvalidData { + entity: "publication_point_cache_projection.algorithm_version", + detail: format!("unsupported algorithm_version {}", self.algorithm_version), + }); + } + validate_non_empty( + "publication_point_cache_projection.manifest_rsync_uri", + &self.manifest_rsync_uri, + )?; + validate_non_empty( + "publication_point_cache_projection.publication_point_rsync_uri", + &self.publication_point_rsync_uri, + )?; + if let Some(uri) = &self.ca_cert_uri { + validate_non_empty("publication_point_cache_projection.ca_cert_uri", uri)?; + } + validate_non_empty("publication_point_cache_projection.tal_id", &self.tal_id)?; + parse_time( + "publication_point_cache_projection.instance_effective_not_before", + &self.instance_effective_not_before, + )?; + parse_time( + "publication_point_cache_projection.instance_effective_until", + &self.instance_effective_until, + )?; + parse_time( + "publication_point_cache_projection.manifest_this_update", + &self.manifest_this_update, + )?; + parse_time( + "publication_point_cache_projection.manifest_next_update", + &self.manifest_next_update, + )?; + parse_time( + "publication_point_cache_projection.current_crl_next_update", + &self.current_crl_next_update, + )?; + parse_time( + "publication_point_cache_projection.self_ca_not_after", + &self.self_ca_not_after, + )?; + self.ccr_manifest_projection.validate_internal()?; + for output in &self.outputs { + output.validate_internal()?; + } + for child in &self.children { + child.validate_internal()?; + } + for object in &self.related_objects { + object.validate_internal()?; + } + Ok(()) + } +} + +impl RoaCacheProjection { + pub fn from_vcir(vcir: &ValidatedCaInstanceResult) -> StorageResult> { + Self::from_vcir_with_context(vcir, None) + } + + pub fn from_vcir_with_context( + vcir: &ValidatedCaInstanceResult, + context: Option<&RoaCacheProjectionContext>, + ) -> StorageResult> { + let Some(context) = context else { + return Ok(None); + }; + let mut issuer_ca_sha256_hex = None; + let mut crl_sha256_by_uri = Vec::new(); + for artifact in &vcir.related_artifacts { + if artifact.validation_status != VcirArtifactValidationStatus::Accepted { + continue; + } + match (artifact.artifact_role, artifact.artifact_kind) { + ( + VcirArtifactRole::IssuerCert | VcirArtifactRole::TrustAnchorCert, + VcirArtifactKind::Cer, + ) => { + issuer_ca_sha256_hex = Some(artifact.sha256.clone()); + } + (_, VcirArtifactKind::Crl) => { + if let Some(uri) = artifact.uri.as_ref() { + crl_sha256_by_uri.push(RoaCacheCrlProjection { + uri: uri.clone(), + sha256: artifact.sha256.clone(), + }); + } + } + _ => {} + } + } + crl_sha256_by_uri.sort_by(|left, right| left.uri.cmp(&right.uri)); + + let meta_by_uri = context + .object_meta + .iter() + .map(|meta| (meta.source_object_uri.as_str(), meta)) + .collect::>(); + let vcir_validation_time = + vcir.last_successful_validation_time + .parse() + .map_err(|detail| StorageError::InvalidData { + entity: "roa_cache_projection.entries[].earliest_safe_reuse_time_unix", + detail, + })?; + let mut entries: Vec = Vec::new(); + let mut entry_index_by_uri: HashMap = HashMap::new(); + for output in &vcir.local_outputs { + let Some(projected_output) = RoaCacheLocalOutputProjection::from_local_output(output) + else { + continue; + }; + let projected_output_effective_until = projected_output + .item_effective_until + .parse() + .map(|time| time.unix_timestamp()) + .map_err(|detail| StorageError::InvalidData { + entity: "roa_cache_projection.entries[].outputs_effective_until_unix", + detail, + })?; + let Some(meta) = meta_by_uri.get(output.source_object_uri.as_str()).copied() else { + continue; + }; + if meta.source_object_hash != output.source_object_hash { + return Err(StorageError::InvalidData { + entity: "roa_cache_projection.entries[]", + detail: format!( + "metadata source object hash mismatch for {}", + output.source_object_uri + ), + }); + } + let earliest_safe_reuse_time_unix = meta + .earliest_safe_reuse_time + .parse() + .map(|time| time.max(vcir_validation_time).unix_timestamp()) + .map_err(|detail| StorageError::InvalidData { + entity: "roa_cache_projection.entries[].earliest_safe_reuse_time_unix", + detail, + })?; + if let Some(entry_index) = entry_index_by_uri.get(output.source_object_uri.as_str()) { + let entry = &mut entries[*entry_index]; + if entry.source_object_hash != output.source_object_hash { + return Err(StorageError::InvalidData { + entity: "roa_cache_projection.entries[]", + detail: format!( + "source object hash mismatch for {}", + output.source_object_uri + ), + }); + } + entry.outputs_effective_until_unix = entry + .outputs_effective_until_unix + .min(projected_output_effective_until); + entry.outputs.push(projected_output); + } else { + entry_index_by_uri.insert(output.source_object_uri.clone(), entries.len()); + entries.push(RoaCacheObjectProjection { + source_object_uri: output.source_object_uri.clone(), + source_object_hash: output.source_object_hash, + ee_serial: Some(meta.ee_serial.clone()), + crl_uri: Some(meta.crl_uri.clone()), + earliest_safe_reuse_time_unix: Some(earliest_safe_reuse_time_unix), + outputs_effective_until_unix: projected_output_effective_until, + outputs: vec![projected_output], + }); + } + } + if entries.is_empty() { + return Ok(None); + } + entries.sort_by(|left, right| left.source_object_uri.cmp(&right.source_object_uri)); + + let projection = Self { + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + instance_effective_until: vcir.instance_gate.instance_effective_until.clone(), + issuer_ca_sha256_hex, + ca_validation_context_digest: Some(context.ca_validation_context_digest), + policy_fingerprint: Some(context.policy_fingerprint), + crl_sha256_by_uri, + entries, + }; + projection.validate_internal()?; + Ok(Some(projection)) + } + + pub fn validate_internal(&self) -> StorageResult<()> { + validate_non_empty( + "roa_cache_projection.manifest_rsync_uri", + &self.manifest_rsync_uri, + )?; + parse_time( + "roa_cache_projection.instance_effective_until", + &self.instance_effective_until, + )?; + if let Some(hash) = &self.issuer_ca_sha256_hex { + validate_sha256_hex("roa_cache_projection.issuer_ca_sha256_hex", hash)?; + } + if self.ca_validation_context_digest.is_some() != self.policy_fingerprint.is_some() { + return Err(StorageError::InvalidData { + entity: "roa_cache_projection.context", + detail: "ca_validation_context_digest and policy_fingerprint must be both present or both absent" + .to_string(), + }); + } + let mut seen_crls = HashSet::with_capacity(self.crl_sha256_by_uri.len()); + for crl in &self.crl_sha256_by_uri { + crl.validate_internal()?; + if !seen_crls.insert(crl.uri.as_str()) { + return Err(StorageError::InvalidData { + entity: "roa_cache_projection.crls[]", + detail: format!("duplicate CRL URI: {}", crl.uri), + }); + } + } + if self.entries.is_empty() { + return Err(StorageError::InvalidData { + entity: "roa_cache_projection.entries", + detail: "must not be empty".to_string(), + }); + } + let mut seen_entries = HashSet::with_capacity(self.entries.len()); + for entry in &self.entries { + entry.validate_internal()?; + if !seen_entries.insert(entry.source_object_uri.as_str()) { + return Err(StorageError::InvalidData { + entity: "roa_cache_projection.entries[]", + detail: format!("duplicate ROA URI: {}", entry.source_object_uri), + }); + } + } + Ok(()) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum VcirArtifactRole { + Manifest, + CurrentCrl, + ChildCaCert, + SignedObject, + EeCert, + IssuerCert, + Tal, + TrustAnchorCert, + Other, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum VcirArtifactKind { + Cer, + Crl, + Mft, + Roa, + Aspa, + Gbr, + Tal, + Other, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum VcirArtifactValidationStatus { + Accepted, + Rejected, + WarningOnly, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct VcirRelatedArtifact { + #[serde(rename = "r")] + pub artifact_role: VcirArtifactRole, + #[serde(rename = "k")] + pub artifact_kind: VcirArtifactKind, + #[serde(rename = "u")] + pub uri: Option, + #[serde(rename = "h")] + pub sha256: String, + #[serde(rename = "t")] + pub object_type: Option, + #[serde(rename = "s")] + pub validation_status: VcirArtifactValidationStatus, + /// Reject reason captured at fresh validation time for `Rejected` artifacts. + /// `None` for accepted/warning artifacts and for cache entries written + /// before this field existed. + #[serde(rename = "e", default, skip_serializing_if = "Option::is_none")] + pub reject_reason: Option, +} + +impl VcirRelatedArtifact { + pub fn validate_internal(&self) -> StorageResult<()> { + if let Some(uri) = &self.uri { + validate_non_empty("vcir.related_artifacts[].uri", uri)?; + } + validate_sha256_hex("vcir.related_artifacts[].sha256", &self.sha256)?; + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct VcirSummary { + #[serde(rename = "v")] + pub local_vrp_count: u32, + #[serde(rename = "a")] + pub local_aspa_count: u32, + #[serde(rename = "r")] + pub local_router_key_count: u32, + #[serde(rename = "c")] + pub child_count: u32, + #[serde(rename = "o")] + pub accepted_object_count: u32, + #[serde(rename = "x")] + pub rejected_object_count: u32, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct VcirAuditSummary { + #[serde(rename = "f")] + pub failed_fetch_eligible: bool, + #[serde(rename = "r")] + pub last_failed_fetch_reason: Option, + #[serde(rename = "w")] + pub warning_count: u32, + #[serde(rename = "a")] + pub audit_flags: Vec, +} + +impl VcirAuditSummary { + pub fn validate_internal(&self) -> StorageResult<()> { + if let Some(reason) = &self.last_failed_fetch_reason { + validate_non_empty("vcir.audit_summary.last_failed_fetch_reason", reason)?; + } + for flag in &self.audit_flags { + validate_non_empty("vcir.audit_summary.audit_flags[]", flag)?; + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct ValidatedCaInstanceResult { + #[serde(rename = "m")] + pub manifest_rsync_uri: String, + #[serde(rename = "pm")] + pub parent_manifest_rsync_uri: Option, + #[serde(rename = "tal")] + pub tal_id: String, + #[serde(rename = "subj")] + pub ca_subject_name: String, + #[serde(rename = "ski")] + pub ca_ski: String, + #[serde(rename = "aki")] + pub issuer_ski: String, + #[serde(rename = "vt")] + pub last_successful_validation_time: PackTime, + #[serde(rename = "cm")] + pub current_manifest_rsync_uri: String, + #[serde(rename = "crl")] + pub current_crl_rsync_uri: String, + #[serde(rename = "mm")] + pub validated_manifest_meta: ValidatedManifestMeta, + #[serde(rename = "ccr")] + pub ccr_manifest_projection: VcirCcrManifestProjection, + #[serde(rename = "g")] + pub instance_gate: VcirInstanceGate, + #[serde(rename = "ch")] + pub child_entries: Vec, + #[serde(rename = "lo")] + pub local_outputs: Vec, + #[serde(rename = "ra")] + pub related_artifacts: Vec, + #[serde(rename = "s")] + pub summary: VcirSummary, + #[serde(rename = "as")] + pub audit_summary: VcirAuditSummary, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct VcirFieldSizeBreakdown { + pub local_output_count: u64, + pub local_output_source_uri_bytes: u64, + pub local_output_source_type_bytes: u64, + pub local_output_source_hash_hex_bytes: u64, + pub local_output_source_ee_hash_hex_bytes: u64, + pub local_output_payload_json_bytes: u64, + pub local_output_rule_hash_hex_bytes: u64, + pub local_output_source_hash_binary_bytes: u64, + pub local_output_source_ee_hash_binary_bytes: u64, + pub local_output_payload_typed_body_bytes: u64, + pub local_output_rule_hash_binary_bytes: u64, + pub related_artifact_count: u64, + pub related_artifact_uri_bytes: u64, + pub related_artifact_hash_hex_bytes: u64, + pub related_artifact_type_bytes: u64, + pub child_entry_count: u64, + pub child_entry_uri_bytes: u64, + pub child_entry_hash_hex_bytes: u64, +} + +fn serialized_cbor_len(value: &T) -> u64 { + serde_cbor::to_vec(value) + .map(|bytes| bytes.len() as u64) + .unwrap_or(0) +} + +impl VcirFieldSizeBreakdown { + pub fn from_vcir(vcir: &ValidatedCaInstanceResult) -> Self { + let mut out = Self::default(); + out.local_output_count = vcir.local_outputs.len() as u64; + for local in &vcir.local_outputs { + out.local_output_source_uri_bytes += local.source_object_uri.len() as u64; + out.local_output_source_type_bytes += local.source_object_type_name().len() as u64; + out.local_output_source_hash_hex_bytes += 64; + out.local_output_source_ee_hash_hex_bytes += 64; + out.local_output_payload_json_bytes += local.payload_json().len() as u64; + out.local_output_rule_hash_hex_bytes += 64; + out.local_output_source_hash_binary_bytes += 32; + out.local_output_source_ee_hash_binary_bytes += 32; + out.local_output_payload_typed_body_bytes += local.payload.typed_body_bytes(); + out.local_output_rule_hash_binary_bytes += 32; + } + out.related_artifact_count = vcir.related_artifacts.len() as u64; + for artifact in &vcir.related_artifacts { + if let Some(uri) = &artifact.uri { + out.related_artifact_uri_bytes += uri.len() as u64; + } + out.related_artifact_hash_hex_bytes += artifact.sha256.len() as u64; + if let Some(object_type) = &artifact.object_type { + out.related_artifact_type_bytes += object_type.len() as u64; + } + } + out.child_entry_count = vcir.child_entries.len() as u64; + for child in &vcir.child_entries { + out.child_entry_uri_bytes += child.child_manifest_rsync_uri.len() as u64 + + child.child_cert_rsync_uri.len() as u64 + + child.child_rsync_base_uri.len() as u64 + + child.child_publication_point_rsync_uri.len() as u64 + + child + .child_rrdp_notification_uri + .as_ref() + .map(|uri| uri.len() as u64) + .unwrap_or(0); + out.child_entry_hash_hex_bytes += + child.child_cert_hash.len() as u64 + child.child_ski.len() as u64; + } + out + } + + pub fn add_assign(&mut self, other: &Self) { + self.local_output_count += other.local_output_count; + self.local_output_source_uri_bytes += other.local_output_source_uri_bytes; + self.local_output_source_type_bytes += other.local_output_source_type_bytes; + self.local_output_source_hash_hex_bytes += other.local_output_source_hash_hex_bytes; + self.local_output_source_ee_hash_hex_bytes += other.local_output_source_ee_hash_hex_bytes; + self.local_output_payload_json_bytes += other.local_output_payload_json_bytes; + self.local_output_rule_hash_hex_bytes += other.local_output_rule_hash_hex_bytes; + self.local_output_source_hash_binary_bytes += other.local_output_source_hash_binary_bytes; + self.local_output_source_ee_hash_binary_bytes += + other.local_output_source_ee_hash_binary_bytes; + self.local_output_payload_typed_body_bytes += other.local_output_payload_typed_body_bytes; + self.local_output_rule_hash_binary_bytes += other.local_output_rule_hash_binary_bytes; + self.related_artifact_count += other.related_artifact_count; + self.related_artifact_uri_bytes += other.related_artifact_uri_bytes; + self.related_artifact_hash_hex_bytes += other.related_artifact_hash_hex_bytes; + self.related_artifact_type_bytes += other.related_artifact_type_bytes; + self.child_entry_count += other.child_entry_count; + self.child_entry_uri_bytes += other.child_entry_uri_bytes; + self.child_entry_hash_hex_bytes += other.child_entry_hash_hex_bytes; + } + + pub fn local_output_old_projection_bytes(&self) -> u64 { + self.local_output_source_type_bytes + + self.local_output_source_hash_hex_bytes + + self.local_output_source_ee_hash_hex_bytes + + self.local_output_payload_json_bytes + + self.local_output_rule_hash_hex_bytes + } + + pub fn local_output_typed_projection_bytes(&self) -> u64 { + self.local_output_count + + self.local_output_source_hash_binary_bytes + + self.local_output_source_ee_hash_binary_bytes + + self.local_output_payload_typed_body_bytes + + self.local_output_rule_hash_binary_bytes + } + + pub fn local_output_projection_saved_bytes(&self) -> u64 { + self.local_output_old_projection_bytes() + .saturating_sub(self.local_output_typed_projection_bytes()) + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct VcirReplaceTimingBreakdown { + pub validate_ms: u64, + pub vcir_encode_ms: u64, + pub vcir_value_bytes: u64, + pub replay_meta_encode_ms: u64, + pub replay_meta_value_bytes: u64, + pub roa_cache_projection_encode_ms: u64, + pub roa_cache_projection_value_bytes: u64, + pub publication_point_cache_projection_encode_ms: u64, + pub publication_point_cache_projection_value_bytes: u64, + pub batch_build_ms: u64, + pub write_batch_ms: u64, + pub total_encoded_bytes: u64, + pub field_sizes: VcirFieldSizeBreakdown, + pub rss_before_kb: Option, + pub rss_after_validate_kb: Option, + pub rss_after_vcir_encode_kb: Option, + pub rss_after_replay_meta_encode_kb: Option, + pub rss_after_roa_cache_projection_encode_kb: Option, + pub rss_after_publication_point_cache_projection_encode_kb: Option, + pub rss_after_write_batch_kb: Option, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct VcirStorageSummary { + pub entry_count: u64, + pub vcir_value_bytes: u64, + pub vcir_value_bytes_max: u64, + pub vcir_value_bytes_max_manifest_rsync_uri: Option, + pub core_fields: VcirCoreFieldSizeBreakdown, + pub ccr_projection: VcirCcrProjectionSizeBreakdown, + pub child_resources: VcirChildResourceSizeBreakdown, + pub field_sizes: VcirFieldSizeBreakdown, + pub local_output_old_projection_bytes: u64, + pub local_output_typed_projection_bytes: u64, + pub local_output_projection_saved_bytes: u64, + pub top_entries_by_vcir_value_bytes: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct VcirStorageEntrySummary { + pub manifest_rsync_uri: String, + pub vcir_value_bytes: u64, + pub local_vrp_count: u32, + pub local_aspa_count: u32, + pub local_router_key_count: u32, + pub accepted_object_count: u32, + pub rejected_object_count: u32, + pub child_count: u32, + pub core_fields: VcirCoreFieldSizeBreakdown, + pub ccr_projection: VcirCcrProjectionSizeBreakdown, + pub child_resources: VcirChildResourceSizeBreakdown, + pub field_sizes: VcirFieldSizeBreakdown, + pub local_output_old_projection_bytes: u64, + pub local_output_typed_projection_bytes: u64, + pub local_output_projection_saved_bytes: u64, +} + +impl VcirStorageEntrySummary { + fn from_vcir(vcir: &ValidatedCaInstanceResult, vcir_value_bytes: u64) -> Self { + let field_sizes = VcirFieldSizeBreakdown::from_vcir(vcir); + Self { + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + vcir_value_bytes, + local_vrp_count: vcir.summary.local_vrp_count, + local_aspa_count: vcir.summary.local_aspa_count, + local_router_key_count: vcir.summary.local_router_key_count, + accepted_object_count: vcir.summary.accepted_object_count, + rejected_object_count: vcir.summary.rejected_object_count, + child_count: vcir.summary.child_count, + core_fields: VcirCoreFieldSizeBreakdown::from_vcir(vcir), + ccr_projection: VcirCcrProjectionSizeBreakdown::from_projection( + &vcir.ccr_manifest_projection, + ), + child_resources: VcirChildResourceSizeBreakdown::from_vcir(vcir), + local_output_old_projection_bytes: field_sizes.local_output_old_projection_bytes(), + local_output_typed_projection_bytes: field_sizes.local_output_typed_projection_bytes(), + local_output_projection_saved_bytes: field_sizes.local_output_projection_saved_bytes(), + field_sizes, + } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct VcirCoreFieldSizeBreakdown { + pub manifest_rsync_uri_bytes: u64, + pub parent_manifest_rsync_uri_bytes: u64, + pub tal_id_bytes: u64, + pub ca_subject_name_bytes: u64, + pub ca_ski_bytes: u64, + pub issuer_ski_bytes: u64, + pub current_manifest_rsync_uri_bytes: u64, + pub current_crl_rsync_uri_bytes: u64, + pub validated_manifest_number_bytes: u64, +} + +impl VcirCoreFieldSizeBreakdown { + fn from_vcir(vcir: &ValidatedCaInstanceResult) -> Self { + Self { + manifest_rsync_uri_bytes: vcir.manifest_rsync_uri.len() as u64, + parent_manifest_rsync_uri_bytes: vcir + .parent_manifest_rsync_uri + .as_ref() + .map(|uri| uri.len() as u64) + .unwrap_or(0), + tal_id_bytes: vcir.tal_id.len() as u64, + ca_subject_name_bytes: vcir.ca_subject_name.len() as u64, + ca_ski_bytes: vcir.ca_ski.len() as u64, + issuer_ski_bytes: vcir.issuer_ski.len() as u64, + current_manifest_rsync_uri_bytes: vcir.current_manifest_rsync_uri.len() as u64, + current_crl_rsync_uri_bytes: vcir.current_crl_rsync_uri.len() as u64, + validated_manifest_number_bytes: vcir + .validated_manifest_meta + .validated_manifest_number + .len() as u64, + } + } + + fn add_assign(&mut self, other: &Self) { + self.manifest_rsync_uri_bytes += other.manifest_rsync_uri_bytes; + self.parent_manifest_rsync_uri_bytes += other.parent_manifest_rsync_uri_bytes; + self.tal_id_bytes += other.tal_id_bytes; + self.ca_subject_name_bytes += other.ca_subject_name_bytes; + self.ca_ski_bytes += other.ca_ski_bytes; + self.issuer_ski_bytes += other.issuer_ski_bytes; + self.current_manifest_rsync_uri_bytes += other.current_manifest_rsync_uri_bytes; + self.current_crl_rsync_uri_bytes += other.current_crl_rsync_uri_bytes; + self.validated_manifest_number_bytes += other.validated_manifest_number_bytes; + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct VcirChildResourceSizeBreakdown { + pub effective_ip_resource_cbor_bytes: u64, + pub effective_as_resource_cbor_bytes: u64, +} + +impl VcirChildResourceSizeBreakdown { + fn from_vcir(vcir: &ValidatedCaInstanceResult) -> Self { + let mut out = Self::default(); + for child in &vcir.child_entries { + out.effective_ip_resource_cbor_bytes += + serialized_cbor_len(&child.child_effective_ip_resources); + out.effective_as_resource_cbor_bytes += + serialized_cbor_len(&child.child_effective_as_resources); + } + out + } + + fn add_assign(&mut self, other: &Self) { + self.effective_ip_resource_cbor_bytes += other.effective_ip_resource_cbor_bytes; + self.effective_as_resource_cbor_bytes += other.effective_as_resource_cbor_bytes; + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct VcirCcrProjectionSizeBreakdown { + pub manifest_rsync_uri_bytes: u64, + pub manifest_sha256_bytes: u64, + pub manifest_ee_aki_bytes: u64, + pub manifest_number_bytes: u64, + pub manifest_sia_locations_count: u64, + pub manifest_sia_locations_der_bytes: u64, + pub subordinate_ski_count: u64, + pub subordinate_ski_bytes: u64, +} + +impl VcirCcrProjectionSizeBreakdown { + fn from_projection(projection: &VcirCcrManifestProjection) -> Self { + Self { + manifest_rsync_uri_bytes: projection.manifest_rsync_uri.len() as u64, + manifest_sha256_bytes: projection.manifest_sha256.len() as u64, + manifest_ee_aki_bytes: projection.manifest_ee_aki.len() as u64, + manifest_number_bytes: projection.manifest_number_be.len() as u64, + manifest_sia_locations_count: projection.manifest_sia_locations_der.len() as u64, + manifest_sia_locations_der_bytes: projection + .manifest_sia_locations_der + .iter() + .map(|location| location.len() as u64) + .sum(), + subordinate_ski_count: projection.subordinate_skis.len() as u64, + subordinate_ski_bytes: projection + .subordinate_skis + .iter() + .map(|ski| ski.len() as u64) + .sum(), + } + } + + fn add_assign(&mut self, other: &Self) { + self.manifest_rsync_uri_bytes += other.manifest_rsync_uri_bytes; + self.manifest_sha256_bytes += other.manifest_sha256_bytes; + self.manifest_ee_aki_bytes += other.manifest_ee_aki_bytes; + self.manifest_number_bytes += other.manifest_number_bytes; + self.manifest_sia_locations_count += other.manifest_sia_locations_count; + self.manifest_sia_locations_der_bytes += other.manifest_sia_locations_der_bytes; + self.subordinate_ski_count += other.subordinate_ski_count; + self.subordinate_ski_bytes += other.subordinate_ski_bytes; + } +} + +fn push_top_vcir_storage_entry( + entries: &mut Vec, + entry: VcirStorageEntrySummary, +) { + const TOP_N: usize = 20; + entries.push(entry); + entries.sort_by(|left, right| { + right + .vcir_value_bytes + .cmp(&left.vcir_value_bytes) + .then_with(|| left.manifest_rsync_uri.cmp(&right.manifest_rsync_uri)) + }); + entries.truncate(TOP_N); +} + +impl ValidatedCaInstanceResult { + pub fn validate_internal(&self) -> StorageResult<()> { + validate_non_empty("vcir.manifest_rsync_uri", &self.manifest_rsync_uri)?; + if let Some(parent_manifest_rsync_uri) = &self.parent_manifest_rsync_uri { + validate_non_empty("vcir.parent_manifest_rsync_uri", parent_manifest_rsync_uri)?; + } + validate_non_empty("vcir.tal_id", &self.tal_id)?; + validate_non_empty("vcir.ca_subject_name", &self.ca_subject_name)?; + validate_non_empty("vcir.ca_ski", &self.ca_ski)?; + validate_non_empty("vcir.issuer_ski", &self.issuer_ski)?; + parse_time( + "vcir.last_successful_validation_time", + &self.last_successful_validation_time, + )?; + validate_non_empty( + "vcir.current_manifest_rsync_uri", + &self.current_manifest_rsync_uri, + )?; + validate_non_empty("vcir.current_crl_rsync_uri", &self.current_crl_rsync_uri)?; + self.validated_manifest_meta.validate_internal()?; + self.ccr_manifest_projection.validate_internal()?; + self.instance_gate.validate_internal()?; + + let expected_manifest_next = self + .validated_manifest_meta + .validated_manifest_next_update + .parse() + .map_err(|detail| StorageError::InvalidData { + entity: "vcir", + detail: format!( + "validated_manifest_meta.validated_manifest_next_update invalid: {detail}" + ), + })?; + let instance_manifest_next = + self.instance_gate + .manifest_next_update + .parse() + .map_err(|detail| StorageError::InvalidData { + entity: "vcir", + detail: format!("instance_gate.manifest_next_update invalid: {detail}"), + })?; + if expected_manifest_next != instance_manifest_next { + return Err(StorageError::InvalidData { + entity: "vcir", + detail: "instance_gate.manifest_next_update must equal validated_manifest_meta.validated_manifest_next_update".to_string(), + }); + } + + let mut child_manifests = HashSet::with_capacity(self.child_entries.len()); + for child in &self.child_entries { + child.validate_internal()?; + if !child_manifests.insert(child.child_manifest_rsync_uri.as_str()) { + return Err(StorageError::InvalidData { + entity: "vcir", + detail: format!( + "duplicate child_manifest_rsync_uri: {}", + child.child_manifest_rsync_uri + ), + }); + } + } + + let mut vrp_count = 0u32; + let mut aspa_count = 0u32; + let mut router_key_count = 0u32; + for output in &self.local_outputs { + output.validate_internal()?; + match output.output_type { + VcirOutputType::Vrp => vrp_count += 1, + VcirOutputType::Aspa => aspa_count += 1, + VcirOutputType::RouterKey => router_key_count += 1, + } + } + let mut output_ids = self + .local_outputs + .iter() + .map(VcirLocalOutput::output_id) + .collect::>(); + output_ids.sort_unstable(); + if let Some(duplicate) = output_ids + .windows(2) + .find_map(|pair| (pair[0] == pair[1]).then(|| pair[0].clone())) + { + return Err(StorageError::InvalidData { + entity: "vcir", + detail: format!("duplicate output_id: {duplicate}"), + }); + } + if self.summary.local_vrp_count != vrp_count { + return Err(StorageError::InvalidData { + entity: "vcir.summary", + detail: format!( + "local_vrp_count={} does not match local_outputs count {}", + self.summary.local_vrp_count, vrp_count + ), + }); + } + if self.summary.local_aspa_count != aspa_count { + return Err(StorageError::InvalidData { + entity: "vcir.summary", + detail: format!( + "local_aspa_count={} does not match local_outputs count {}", + self.summary.local_aspa_count, aspa_count + ), + }); + } + if self.summary.local_router_key_count != router_key_count { + return Err(StorageError::InvalidData { + entity: "vcir.summary", + detail: format!( + "local_router_key_count={} does not match local_outputs count {}", + self.summary.local_router_key_count, router_key_count + ), + }); + } + if self.summary.child_count != self.child_entries.len() as u32 { + return Err(StorageError::InvalidData { + entity: "vcir.summary", + detail: format!( + "child_count={} does not match child_entries length {}", + self.summary.child_count, + self.child_entries.len() + ), + }); + } + + for artifact in &self.related_artifacts { + artifact.validate_internal()?; + } + self.audit_summary.validate_internal()?; + Ok(()) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RrdpSourceSyncState { + Empty, + SnapshotOnly, + DeltaReady, + Error, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RrdpSourceRecord { + pub notify_uri: String, + pub last_session_id: Option, + pub last_serial: Option, + pub first_seen_at: PackTime, + pub last_seen_at: PackTime, + pub last_sync_at: Option, + pub sync_state: RrdpSourceSyncState, + pub last_snapshot_uri: Option, + pub last_snapshot_hash: Option, + pub last_error: Option, +} + +impl RrdpSourceRecord { + pub fn validate_internal(&self) -> StorageResult<()> { + validate_non_empty("rrdp_source.notify_uri", &self.notify_uri)?; + if let Some(session_id) = &self.last_session_id { + validate_non_empty("rrdp_source.last_session_id", session_id)?; + } + parse_time("rrdp_source.first_seen_at", &self.first_seen_at)?; + parse_time("rrdp_source.last_seen_at", &self.last_seen_at)?; + if let Some(last_sync_at) = &self.last_sync_at { + parse_time("rrdp_source.last_sync_at", last_sync_at)?; + } + if let Some(last_snapshot_uri) = &self.last_snapshot_uri { + validate_non_empty("rrdp_source.last_snapshot_uri", last_snapshot_uri)?; + } + if let Some(last_snapshot_hash) = &self.last_snapshot_hash { + validate_sha256_hex("rrdp_source.last_snapshot_hash", last_snapshot_hash)?; + } + if let Some(last_error) = &self.last_error { + validate_non_empty("rrdp_source.last_error", last_error)?; + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RrdpSourceMemberRecord { + pub notify_uri: String, + pub rsync_uri: String, + pub current_hash: Option, + pub object_type: Option, + pub present: bool, + pub last_confirmed_session_id: String, + pub last_confirmed_serial: u64, + pub last_changed_at: PackTime, +} + +impl RrdpSourceMemberRecord { + pub fn validate_internal(&self) -> StorageResult<()> { + validate_non_empty("rrdp_source_member.notify_uri", &self.notify_uri)?; + validate_non_empty("rrdp_source_member.rsync_uri", &self.rsync_uri)?; + validate_non_empty( + "rrdp_source_member.last_confirmed_session_id", + &self.last_confirmed_session_id, + )?; + if self.present { + let hash = self + .current_hash + .as_deref() + .ok_or(StorageError::InvalidData { + entity: "rrdp_source_member", + detail: "current_hash is required when present=true".to_string(), + })?; + validate_sha256_hex("rrdp_source_member.current_hash", hash)?; + } else if let Some(hash) = &self.current_hash { + validate_sha256_hex("rrdp_source_member.current_hash", hash)?; + } + parse_time("rrdp_source_member.last_changed_at", &self.last_changed_at)?; + Ok(()) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RrdpUriOwnerState { + Active, + Conflict, + Withdrawn, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RrdpUriOwnerRecord { + pub rsync_uri: String, + pub notify_uri: String, + pub current_hash: Option, + pub last_confirmed_session_id: String, + pub last_confirmed_serial: u64, + pub last_changed_at: PackTime, + pub owner_state: RrdpUriOwnerState, +} + +impl RrdpUriOwnerRecord { + pub fn validate_internal(&self) -> StorageResult<()> { + validate_non_empty("rrdp_uri_owner.rsync_uri", &self.rsync_uri)?; + validate_non_empty("rrdp_uri_owner.notify_uri", &self.notify_uri)?; + validate_non_empty( + "rrdp_uri_owner.last_confirmed_session_id", + &self.last_confirmed_session_id, + )?; + if let Some(hash) = &self.current_hash { + validate_sha256_hex("rrdp_uri_owner.current_hash", hash)?; + } + parse_time("rrdp_uri_owner.last_changed_at", &self.last_changed_at)?; + Ok(()) + } +} + +fn write_roa_cache_projection_to_batch( + projection_cf: &ColumnFamily, + batch: &mut WriteBatch, + vcir: &ValidatedCaInstanceResult, + context: Option<&RoaCacheProjectionContext>, + timing: Option<&mut VcirReplaceTimingBreakdown>, +) -> StorageResult<()> { + let projection_key = roa_cache_projection_key(&vcir.manifest_rsync_uri); + let projection = RoaCacheProjection::from_vcir_with_context(vcir, context)?; + match projection { + Some(projection) => { + let projection_value = encode_cbor(&projection, "roa_cache_projection")?; + if let Some(timing) = timing { + timing.roa_cache_projection_value_bytes = projection_value.len() as u64; + } + batch.put_cf(projection_cf, projection_key.as_bytes(), projection_value); + } + None => { + batch.delete_cf(projection_cf, projection_key.as_bytes()); + } + } + Ok(()) +} + +fn write_vcir_failed_fetch_reuse_identity_to_batch( + failed_fetch_reuse_identity_cf: &ColumnFamily, + batch: &mut WriteBatch, + manifest_rsync_uri: &str, + failed_fetch_reuse_identity: Option<&VcirFailedFetchReuseIdentity>, +) -> StorageResult<()> { + let key = vcir_failed_fetch_reuse_identity_key(manifest_rsync_uri); + match failed_fetch_reuse_identity { + Some(identity) => { + identity.validate_internal()?; + let value = encode_cbor(identity, "vcir_failed_fetch_reuse_identity")?; + batch.put_cf(failed_fetch_reuse_identity_cf, key.as_bytes(), value); + } + None => batch.delete_cf(failed_fetch_reuse_identity_cf, key.as_bytes()), + } + Ok(()) +} + +fn write_publication_point_cache_projection_to_batch( + projection_cf: &ColumnFamily, + batch: &mut WriteBatch, + action: PublicationPointCacheProjectionWriteAction<'_>, + timing: Option<&mut VcirReplaceTimingBreakdown>, +) -> StorageResult<()> { + match action { + PublicationPointCacheProjectionWriteAction::Keep => Ok(()), + PublicationPointCacheProjectionWriteAction::Write(projection) => { + projection.validate_internal()?; + let key = publication_point_cache_projection_key(&projection.manifest_rsync_uri); + let value = encode_cbor(projection, "publication_point_cache_projection")?; + if let Some(timing) = timing { + timing.publication_point_cache_projection_value_bytes = value.len() as u64; + } + batch.put_cf(projection_cf, key.as_bytes(), value); + Ok(()) + } + PublicationPointCacheProjectionWriteAction::Delete { manifest_rsync_uri } => { + let key = publication_point_cache_projection_key(manifest_rsync_uri); + batch.delete_cf(projection_cf, key.as_bytes()); + Ok(()) + } + } +} + +impl RocksStore { + pub fn create_read_only_checkpoint(source: &Path, destination: &Path) -> StorageResult<()> { + if destination.exists() { + return Err(StorageError::InvalidData { + entity: "work_db_checkpoint", + detail: format!( + "checkpoint destination already exists: {}", + destination.display() + ), + }); + } + if let Some(parent) = destination.parent() { + std::fs::create_dir_all(parent) + .map_err(|error| StorageError::RocksDb(error.to_string()))?; + } + let mut options = Options::default(); + let blob_mode = work_db_blob_mode_from_env(); + let memory_profile = work_db_memory_profile_from_env(); + configure_work_db_options(&mut options, blob_mode, memory_profile); + let db = DB::open_cf_descriptors_read_only( + &options, + source, + column_family_descriptors_for_blob_mode(blob_mode), + false, + ) + .map_err(|error| StorageError::RocksDb(error.to_string()))?; + let checkpoint = + Checkpoint::new(&db).map_err(|error| StorageError::RocksDb(error.to_string()))?; + checkpoint + .create_checkpoint(destination) + .map_err(|error| StorageError::RocksDb(error.to_string())) + } + + pub fn open(path: &Path) -> StorageResult { + let mut base_opts = Options::default(); + base_opts.create_if_missing(true); + base_opts.create_missing_column_families(true); + let blob_mode = work_db_blob_mode_from_env(); + let memory_profile = work_db_memory_profile_from_env(); + configure_work_db_options(&mut base_opts, blob_mode, memory_profile); + + let db = DB::open_cf_descriptors( + &base_opts, + path, + column_family_descriptors_for_blob_mode(blob_mode), + ) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + + Ok(Self { + db, + external_raw_store: None, + external_repo_bytes: None, + publication_point_cache_index_dir: default_pp_cache_index_dir(path), + child_certificate_cache_index_dir: default_child_certificate_cache_index_dir(path), + publication_point_cache_projection_index: Mutex::new( + PublicationPointCacheProjectionIndexState::Uninitialized, + ), + }) + } + + pub fn open_with_external_raw_store(path: &Path, raw_store_path: &Path) -> StorageResult { + Self::open_with_external_stores(path, Some(raw_store_path), None) + } + + pub fn open_with_external_repo_bytes( + path: &Path, + repo_bytes_path: &Path, + ) -> StorageResult { + Self::open_with_external_stores(path, None, Some(repo_bytes_path)) + } + + pub fn open_with_external_repo_bytes_read_only( + path: &Path, + repo_bytes_path: &Path, + ) -> StorageResult { + let mut store = Self::open(path)?; + store.external_repo_bytes = Some(ExternalRepoBytesDb::open_read_only(repo_bytes_path)?); + Ok(store) + } + + pub fn open_with_external_stores( + path: &Path, + raw_store_path: Option<&Path>, + repo_bytes_path: Option<&Path>, + ) -> StorageResult { + let mut store = Self::open(path)?; + if let Some(raw_store_path) = raw_store_path { + store.external_raw_store = Some(ExternalRawStoreDb::open(raw_store_path)?); + } + if let Some(repo_bytes_path) = repo_bytes_path { + store.external_repo_bytes = Some(ExternalRepoBytesDb::open(repo_bytes_path)?); + } + Ok(store) + } + + pub(crate) fn external_raw_store_ref(&self) -> Option<&ExternalRawStoreDb> { + self.external_raw_store.as_ref() + } + + pub(crate) fn external_repo_bytes_ref(&self) -> Option<&ExternalRepoBytesDb> { + self.external_repo_bytes.as_ref() + } + + pub fn memory_snapshot(&self) -> RocksDbMemorySnapshot { + let mut databases = Vec::new(); + databases.push(memory_db_snapshot_for_column_families( + "work-db", + &self.db, + Some(ALL_COLUMN_FAMILY_NAMES), + )); + if let Some(raw_store) = self.external_raw_store.as_ref() { + databases.push(raw_store.memory_snapshot("raw-store.db")); + } + if let Some(repo_bytes) = self.external_repo_bytes.as_ref() { + databases.push(repo_bytes.memory_snapshot("repo-bytes.db")); + } + + let mut totals = RocksDbMemoryTotals::default(); + for db in &databases { + totals.add_properties(&db.properties); + } + RocksDbMemorySnapshot { databases, totals } + } + + pub fn publication_point_cache_mmap_index_load_stats(&self) -> Option { + let guard = self.publication_point_cache_projection_index.lock().ok()?; + match &*guard { + PublicationPointCacheProjectionIndexState::LoadedMmap { load_stats, .. } => { + Some(load_stats.clone()) + } + PublicationPointCacheProjectionIndexState::Uninitialized + | PublicationPointCacheProjectionIndexState::Disabled + | PublicationPointCacheProjectionIndexState::BuildingFromEmpty { .. } + | PublicationPointCacheProjectionIndexState::Loaded { .. } => None, + } + } + + fn try_load_publication_point_cache_mmap_index_for_update( + &self, + reason: &'static str, + ) -> StorageResult<()> { + if !pp_cache_raw_index_enabled() { + return Ok(()); + } + let mut guard = self + .publication_point_cache_projection_index + .lock() + .map_err(|e| { + StorageError::RocksDb(format!("publication point cache index lock poisoned: {e}")) + })?; + if !matches!( + *guard, + PublicationPointCacheProjectionIndexState::Uninitialized + ) { + return Ok(()); + } + match load_pp_cache_mmap_index_set(&self.publication_point_cache_index_dir) { + Ok((mmap, stats)) => { + crate::progress_log::emit( + "publication_point_cache_mmap_index_load", + serde_json::json!({ + "state": "loaded", + "reason": reason, + "entries": stats.entries, + "bytes": stats.bytes, + "file_bytes": stats.file_bytes, + "load_ms": stats.load_ms, + }), + ); + *guard = PublicationPointCacheProjectionIndexState::LoadedMmap { + mmap, + dirty: HashMap::new(), + dirty_bytes: 0, + load_stats: stats, + }; + } + Err(e) => { + crate::progress_log::emit( + "publication_point_cache_mmap_index_load", + serde_json::json!({ + "state": "deferred_fallback_scan", + "reason": reason, + "error": e.to_string(), + }), + ); + } + } + Ok(()) + } + + fn cf(&self, name: &'static str) -> StorageResult<&ColumnFamily> { + self.db + .cf_handle(name) + .ok_or(StorageError::MissingColumnFamily(name)) + } + + pub fn put_repository_view_entry(&self, entry: &RepositoryViewEntry) -> StorageResult<()> { + entry.validate_internal()?; + let cf = self.cf(CF_REPOSITORY_VIEW)?; + let key = repository_view_key(&entry.rsync_uri); + let value = encode_cbor(entry, "repository_view")?; + self.db + .put_cf(cf, key.as_bytes(), value) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(()) + } + + pub fn get_repository_view_entry( + &self, + rsync_uri: &str, + ) -> StorageResult> { + let cf = self.cf(CF_REPOSITORY_VIEW)?; + let key = repository_view_key(rsync_uri); + let Some(bytes) = self + .db + .get_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + else { + return Ok(None); + }; + let entry = decode_cbor::(&bytes, "repository_view")?; + entry.validate_internal()?; + Ok(Some(entry)) + } + + pub fn delete_repository_view_entry(&self, rsync_uri: &str) -> StorageResult<()> { + let cf = self.cf(CF_REPOSITORY_VIEW)?; + let key = repository_view_key(rsync_uri); + self.db + .delete_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(()) + } + + pub fn put_projection_batch( + &self, + repository_view_entries: &[RepositoryViewEntry], + member_records: &[RrdpSourceMemberRecord], + owner_records: &[RrdpUriOwnerRecord], + ) -> StorageResult<()> { + if repository_view_entries.is_empty() + && member_records.is_empty() + && owner_records.is_empty() + { + return Ok(()); + } + + let repo_cf = self.cf(CF_REPOSITORY_VIEW)?; + let member_cf = self.cf(CF_RRDP_SOURCE_MEMBER)?; + let owner_cf = self.cf(CF_RRDP_URI_OWNER)?; + let mut batch = WriteBatch::default(); + + for entry in repository_view_entries { + entry.validate_internal()?; + let key = repository_view_key(&entry.rsync_uri); + let value = encode_cbor(entry, "repository_view")?; + batch.put_cf(repo_cf, key.as_bytes(), value); + } + for record in member_records { + record.validate_internal()?; + let key = rrdp_source_member_key(&record.notify_uri, &record.rsync_uri); + let value = encode_cbor(record, "rrdp_source_member")?; + batch.put_cf(member_cf, key.as_bytes(), value); + } + for record in owner_records { + record.validate_internal()?; + let key = rrdp_uri_owner_key(&record.rsync_uri); + let value = encode_cbor(record, "rrdp_uri_owner")?; + batch.put_cf(owner_cf, key.as_bytes(), value); + } + + self.write_batch(batch) + } + + pub fn list_repository_view_entries_with_prefix( + &self, + rsync_uri_prefix: &str, + ) -> StorageResult> { + let cf = self.cf(CF_REPOSITORY_VIEW)?; + let prefix = repository_view_prefix(rsync_uri_prefix); + let mode = IteratorMode::From(prefix.as_bytes(), Direction::Forward); + self.db + .iterator_cf(cf, mode) + .take_while(|res| match res { + Ok((key, _)) => key.starts_with(prefix.as_bytes()), + Err(_) => false, + }) + .map(|res| { + let (_key, value) = res.map_err(|e| StorageError::RocksDb(e.to_string()))?; + let entry = decode_cbor::(&value, "repository_view")?; + entry.validate_internal()?; + Ok(entry) + }) + .collect() + } + + pub fn verify_current_repository_blobs( + &self, + batch_size: usize, + ) -> StorageResult { + if batch_size == 0 { + return Err(StorageError::InvalidData { + entity: "repository_blob_verification", + detail: "batch_size must be greater than zero".to_string(), + }); + } + let cf = self.cf(CF_REPOSITORY_VIEW)?; + let prefix = repository_view_prefix(""); + let mode = IteratorMode::From(prefix.as_bytes(), Direction::Forward); + let mut summary = RepositoryBlobVerificationSummary::default(); + let mut batch = Vec::with_capacity(batch_size); + for item in self.db.iterator_cf(cf, mode) { + let (key, value) = item.map_err(|error| StorageError::RocksDb(error.to_string()))?; + if !key.starts_with(prefix.as_bytes()) { + break; + } + let entry = decode_cbor::(&value, "repository_view")?; + entry.validate_internal()?; + if !matches!( + entry.state, + RepositoryViewState::Present | RepositoryViewState::Replaced + ) { + continue; + } + let hash = entry + .current_hash + .clone() + .ok_or(StorageError::InvalidData { + entity: "repository_blob_verification", + detail: format!("current hash missing for {}", entry.rsync_uri), + })?; + batch.push((entry.rsync_uri, hash)); + if batch.len() >= batch_size { + verify_repository_blob_batch(self, &batch, &mut summary)?; + batch.clear(); + } + } + if !batch.is_empty() { + verify_repository_blob_batch(self, &batch, &mut summary)?; + } + Ok(summary) + } + + pub fn put_raw_by_hash_entry(&self, entry: &RawByHashEntry) -> StorageResult<()> { + entry.validate_internal()?; + if let Some(raw_store) = self.external_raw_store.as_ref() { + return raw_store.put_raw_entry(entry); + } + let cf = self.cf(CF_RAW_BY_HASH)?; + let key = raw_by_hash_key(&entry.sha256_hex); + let value = encode_cbor(entry, "raw_by_hash")?; + self.db + .put_cf(cf, key.as_bytes(), value) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(()) + } + + pub fn put_raw_by_hash_entries_batch(&self, entries: &[RawByHashEntry]) -> StorageResult<()> { + if entries.is_empty() { + return Ok(()); + } + if let Some(raw_store) = self.external_raw_store.as_ref() { + return raw_store.put_raw_entries_batch(entries); + } + + let cf = self.cf(CF_RAW_BY_HASH)?; + let mut batch = WriteBatch::default(); + for entry in entries { + entry.validate_internal()?; + let key = raw_by_hash_key(&entry.sha256_hex); + let value = encode_cbor(entry, "raw_by_hash")?; + batch.put_cf(cf, key.as_bytes(), value); + } + self.write_batch(batch) + } + + pub fn put_raw_by_hash_entries_batch_unchecked( + &self, + entries: &[RawByHashEntry], + ) -> StorageResult<()> { + if entries.is_empty() { + return Ok(()); + } + if let Some(raw_store) = self.external_raw_store.as_ref() { + return raw_store.put_raw_entries_batch(entries); + } + + let cf = self.cf(CF_RAW_BY_HASH)?; + let mut batch = WriteBatch::default(); + for entry in entries { + let key = raw_by_hash_key(&entry.sha256_hex); + let value = encode_cbor(entry, "raw_by_hash")?; + batch.put_cf(cf, key.as_bytes(), value); + } + self.write_batch(batch) + } + + pub fn put_blob_bytes_batch(&self, blobs: &[(String, Vec)]) -> StorageResult<()> { + if blobs.is_empty() { + return Ok(()); + } + if let Some(repo_bytes) = self.external_repo_bytes.as_ref() { + if repo_bytes.is_read_only() { + return repo_bytes.require_existing_blob_bytes_batch(blobs); + } + return repo_bytes.put_blob_bytes_batch(blobs); + } + if let Some(raw_store) = self.external_raw_store.as_ref() { + return raw_store.put_blob_bytes_batch(blobs); + } + let cf = self.cf(CF_RAW_BLOB)?; + let mut batch = WriteBatch::default(); + for (sha256_hex, bytes) in blobs { + validate_sha256_hex("raw_blob.sha256_hex", sha256_hex)?; + if bytes.is_empty() { + return Err(StorageError::InvalidData { + entity: "raw_blob", + detail: "bytes must not be empty".to_string(), + }); + } + let key = raw_blob_key(sha256_hex); + batch.put_cf(cf, key.as_bytes(), bytes.as_slice()); + } + self.write_batch(batch) + } + + pub fn delete_raw_by_hash_entry(&self, sha256_hex: &str) -> StorageResult<()> { + validate_sha256_hex("raw_by_hash.sha256_hex", sha256_hex)?; + if let Some(raw_store) = self.external_raw_store.as_ref() { + return raw_store.delete_raw_entry(sha256_hex); + } + let cf = self.cf(CF_RAW_BY_HASH)?; + let key = raw_by_hash_key(sha256_hex); + self.db + .delete_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(()) + } + + pub fn get_raw_by_hash_entry(&self, sha256_hex: &str) -> StorageResult> { + if let Some(raw_store) = self.external_raw_store.as_ref() { + return raw_store.get_raw_entry(sha256_hex); + } + let cf = self.cf(CF_RAW_BY_HASH)?; + let key = raw_by_hash_key(sha256_hex); + let Some(bytes) = self + .db + .get_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + else { + return Ok(None); + }; + let entry = decode_cbor::(&bytes, "raw_by_hash")?; + entry.validate_internal()?; + Ok(Some(entry)) + } + + pub fn get_raw_by_hash_entries_batch( + &self, + sha256_hexes: &[String], + ) -> StorageResult>> { + if sha256_hexes.is_empty() { + return Ok(Vec::new()); + } + if let Some(raw_store) = self.external_raw_store.as_ref() { + return raw_store.get_raw_entries_batch(sha256_hexes); + } + + let cf = self.cf(CF_RAW_BY_HASH)?; + let keys: Vec = sha256_hexes + .iter() + .map(|hash| raw_by_hash_key(hash)) + .collect(); + self.db + .multi_get_cf(keys.iter().map(|key| (cf, key.as_bytes()))) + .into_iter() + .map(|res| { + let maybe = res.map_err(|e| StorageError::RocksDb(e.to_string()))?; + match maybe { + Some(bytes) => { + let entry = decode_cbor::(&bytes, "raw_by_hash")?; + entry.validate_internal()?; + Ok(Some(entry)) + } + None => Ok(None), + } + }) + .collect() + } + + pub fn get_blob_bytes(&self, sha256_hex: &str) -> StorageResult>> { + if let Some(repo_bytes) = self.external_repo_bytes.as_ref() { + return repo_bytes.get_blob_bytes(sha256_hex); + } + if let Some(raw_store) = self.external_raw_store.as_ref() { + return raw_store.get_blob_bytes(sha256_hex); + } + validate_sha256_hex("raw_blob.sha256_hex", sha256_hex)?; + let cf = self.cf(CF_RAW_BLOB)?; + let key = raw_blob_key(sha256_hex); + if let Some(bytes) = self + .db + .get_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + { + return Ok(Some(bytes)); + } + self.get_raw_by_hash_entry(sha256_hex) + .map(|entry| entry.map(|entry| entry.bytes)) + } + + pub fn get_blob_bytes_batch( + &self, + sha256_hexes: &[String], + ) -> StorageResult>>> { + if sha256_hexes.is_empty() { + return Ok(Vec::new()); + } + if let Some(repo_bytes) = self.external_repo_bytes.as_ref() { + return repo_bytes.get_blob_bytes_batch(sha256_hexes); + } + if let Some(raw_store) = self.external_raw_store.as_ref() { + return raw_store.get_blob_bytes_batch(sha256_hexes); + } + + let cf = self.cf(CF_RAW_BLOB)?; + let keys: Vec = sha256_hexes + .iter() + .map(|hash| { + validate_sha256_hex("raw_blob.sha256_hex", hash)?; + Ok::(raw_blob_key(hash)) + }) + .collect::>()?; + let blob_results: Vec>> = self + .db + .multi_get_cf(keys.iter().map(|key| (cf, key.as_bytes()))) + .into_iter() + .map(|res| res.map_err(|e| StorageError::RocksDb(e.to_string()))) + .collect::>()?; + + let mut out = Vec::with_capacity(sha256_hexes.len()); + for (sha256_hex, maybe_blob) in sha256_hexes.iter().zip(blob_results.into_iter()) { + if maybe_blob.is_some() { + out.push(maybe_blob); + } else { + out.push( + self.get_raw_by_hash_entry(sha256_hex)? + .map(|entry| entry.bytes), + ); + } + } + Ok(out) + } + + pub fn put_vcir(&self, vcir: &ValidatedCaInstanceResult) -> StorageResult<()> { + self.put_vcir_with_publication_point_cache_projection(vcir, None) + } + + pub fn put_vcir_with_failed_fetch_reuse_identity( + &self, + vcir: &ValidatedCaInstanceResult, + failed_fetch_reuse_identity: &VcirFailedFetchReuseIdentity, + ) -> StorageResult<()> { + self.put_vcir_with_projection_action( + vcir, + None, + PublicationPointCacheProjectionWriteAction::Keep, + Some(failed_fetch_reuse_identity), + ) + } + + pub fn put_vcir_with_publication_point_cache_projection( + &self, + vcir: &ValidatedCaInstanceResult, + publication_point_projection: Option<&PublicationPointCacheProjection>, + ) -> StorageResult<()> { + self.put_vcir_with_projections(vcir, None, publication_point_projection) + } + + pub fn put_vcir_with_projections( + &self, + vcir: &ValidatedCaInstanceResult, + roa_cache_context: Option<&RoaCacheProjectionContext>, + publication_point_projection: Option<&PublicationPointCacheProjection>, + ) -> StorageResult<()> { + let publication_point_projection_action = publication_point_projection + .map(PublicationPointCacheProjectionWriteAction::Write) + .unwrap_or(PublicationPointCacheProjectionWriteAction::Keep); + self.put_vcir_with_projection_action( + vcir, + roa_cache_context, + publication_point_projection_action, + None, + ) + } + + fn put_vcir_with_projection_action( + &self, + vcir: &ValidatedCaInstanceResult, + roa_cache_context: Option<&RoaCacheProjectionContext>, + publication_point_projection_action: PublicationPointCacheProjectionWriteAction<'_>, + failed_fetch_reuse_identity: Option<&VcirFailedFetchReuseIdentity>, + ) -> StorageResult<()> { + vcir.validate_internal()?; + let vcir_cf = self.cf(CF_VCIR)?; + let failed_fetch_reuse_identity_cf = self.cf(CF_VCIR_FAILED_FETCH_REUSE_IDENTITY)?; + let replay_cf = self.cf(CF_MANIFEST_REPLAY_META)?; + let projection_cf = self.cf(CF_ROA_CACHE_PROJECTION)?; + let pp_projection_cf = self.cf(CF_PUBLICATION_POINT_CACHE_PROJECTION)?; + let replay_meta = ManifestReplayMeta::from_vcir(vcir); + replay_meta.validate_internal()?; + let mut batch = WriteBatch::default(); + let key = vcir_key(&vcir.manifest_rsync_uri); + let value = encode_cbor(vcir, "vcir")?; + batch.put_cf(vcir_cf, key.as_bytes(), value); + write_vcir_failed_fetch_reuse_identity_to_batch( + failed_fetch_reuse_identity_cf, + &mut batch, + &vcir.manifest_rsync_uri, + failed_fetch_reuse_identity, + )?; + let replay_key = manifest_replay_meta_key(&replay_meta.manifest_rsync_uri); + let replay_value = encode_cbor(&replay_meta, "manifest_replay_meta")?; + batch.put_cf(replay_cf, replay_key.as_bytes(), replay_value); + write_roa_cache_projection_to_batch( + projection_cf, + &mut batch, + vcir, + roa_cache_context, + None, + )?; + write_publication_point_cache_projection_to_batch( + pp_projection_cf, + &mut batch, + publication_point_projection_action, + None, + )?; + self.write_batch(batch)?; + self.apply_publication_point_cache_projection_index_action( + publication_point_projection_action, + )?; + Ok(()) + } + + pub fn replace_vcir_and_manifest_replay_meta( + &self, + vcir: &ValidatedCaInstanceResult, + ) -> StorageResult { + self.replace_vcir_manifest_replay_meta_and_publication_point_cache_projection(vcir, None) + } + + pub fn replace_vcir_manifest_replay_meta_and_publication_point_cache_projection( + &self, + vcir: &ValidatedCaInstanceResult, + publication_point_projection: Option<&PublicationPointCacheProjection>, + ) -> StorageResult { + self.replace_vcir_manifest_replay_meta_and_projections( + vcir, + None, + publication_point_projection, + ) + } + + pub fn replace_vcir_manifest_replay_meta_and_projections( + &self, + vcir: &ValidatedCaInstanceResult, + roa_cache_context: Option<&RoaCacheProjectionContext>, + publication_point_projection: Option<&PublicationPointCacheProjection>, + ) -> StorageResult { + let publication_point_projection_action = publication_point_projection + .map(PublicationPointCacheProjectionWriteAction::Write) + .unwrap_or(PublicationPointCacheProjectionWriteAction::Keep); + self.replace_vcir_manifest_replay_meta_and_projection_action( + vcir, + roa_cache_context, + publication_point_projection_action, + ) + } + + pub(crate) fn replace_vcir_manifest_replay_meta_and_projection_action( + &self, + vcir: &ValidatedCaInstanceResult, + roa_cache_context: Option<&RoaCacheProjectionContext>, + publication_point_projection_action: PublicationPointCacheProjectionWriteAction<'_>, + ) -> StorageResult { + self.replace_vcir_manifest_replay_meta_and_projection_action_with_failed_fetch_reuse_identity( + vcir, + roa_cache_context, + publication_point_projection_action, + None, + ) + } + + pub(crate) fn replace_vcir_manifest_replay_meta_and_projection_action_with_failed_fetch_reuse_identity( + &self, + vcir: &ValidatedCaInstanceResult, + roa_cache_context: Option<&RoaCacheProjectionContext>, + publication_point_projection_action: PublicationPointCacheProjectionWriteAction<'_>, + failed_fetch_reuse_identity: Option<&VcirFailedFetchReuseIdentity>, + ) -> StorageResult { + let mut timing = VcirReplaceTimingBreakdown { + rss_before_kb: process_vm_rss_kb(), + ..VcirReplaceTimingBreakdown::default() + }; + + let validate_started = std::time::Instant::now(); + vcir.validate_internal()?; + timing.validate_ms = validate_started.elapsed().as_millis() as u64; + timing.rss_after_validate_kb = process_vm_rss_kb(); + timing.field_sizes = VcirFieldSizeBreakdown::from_vcir(vcir); + + let batch_build_started = std::time::Instant::now(); + let vcir_cf = self.cf(CF_VCIR)?; + let failed_fetch_reuse_identity_cf = self.cf(CF_VCIR_FAILED_FETCH_REUSE_IDENTITY)?; + let replay_cf = self.cf(CF_MANIFEST_REPLAY_META)?; + let projection_cf = self.cf(CF_ROA_CACHE_PROJECTION)?; + let pp_projection_cf = self.cf(CF_PUBLICATION_POINT_CACHE_PROJECTION)?; + let mut batch = WriteBatch::default(); + + let vcir_key = vcir_key(&vcir.manifest_rsync_uri); + let vcir_encode_started = std::time::Instant::now(); + let vcir_value = encode_cbor(vcir, "vcir")?; + timing.vcir_encode_ms = vcir_encode_started.elapsed().as_millis() as u64; + timing.vcir_value_bytes = vcir_value.len() as u64; + batch.put_cf(vcir_cf, vcir_key.as_bytes(), vcir_value); + timing.rss_after_vcir_encode_kb = process_vm_rss_kb(); + write_vcir_failed_fetch_reuse_identity_to_batch( + failed_fetch_reuse_identity_cf, + &mut batch, + &vcir.manifest_rsync_uri, + failed_fetch_reuse_identity, + )?; + + let replay_meta_encode_started = std::time::Instant::now(); + let replay_meta = ManifestReplayMeta::from_vcir(vcir); + replay_meta.validate_internal()?; + let replay_key = manifest_replay_meta_key(&replay_meta.manifest_rsync_uri); + let replay_value = encode_cbor(&replay_meta, "manifest_replay_meta")?; + timing.replay_meta_encode_ms = replay_meta_encode_started.elapsed().as_millis() as u64; + timing.replay_meta_value_bytes = replay_value.len() as u64; + batch.put_cf(replay_cf, replay_key.as_bytes(), replay_value); + timing.rss_after_replay_meta_encode_kb = process_vm_rss_kb(); + + let projection_encode_started = std::time::Instant::now(); + write_roa_cache_projection_to_batch( + projection_cf, + &mut batch, + vcir, + roa_cache_context, + Some(&mut timing), + )?; + timing.roa_cache_projection_encode_ms = + projection_encode_started.elapsed().as_millis() as u64; + timing.rss_after_roa_cache_projection_encode_kb = process_vm_rss_kb(); + + let pp_projection_encode_started = std::time::Instant::now(); + write_publication_point_cache_projection_to_batch( + pp_projection_cf, + &mut batch, + publication_point_projection_action, + Some(&mut timing), + )?; + timing.publication_point_cache_projection_encode_ms = + pp_projection_encode_started.elapsed().as_millis() as u64; + timing.rss_after_publication_point_cache_projection_encode_kb = process_vm_rss_kb(); + + timing.total_encoded_bytes = timing.vcir_value_bytes + + timing.replay_meta_value_bytes + + timing.roa_cache_projection_value_bytes + + timing.publication_point_cache_projection_value_bytes; + timing.batch_build_ms = batch_build_started.elapsed().as_millis() as u64; + + let write_batch_started = std::time::Instant::now(); + self.write_batch(batch)?; + self.apply_publication_point_cache_projection_index_action( + publication_point_projection_action, + )?; + timing.write_batch_ms = write_batch_started.elapsed().as_millis() as u64; + timing.rss_after_write_batch_kb = process_vm_rss_kb(); + Ok(timing) + } + + pub fn get_vcir( + &self, + manifest_rsync_uri: &str, + ) -> StorageResult> { + let cf = self.cf(CF_VCIR)?; + let key = vcir_key(manifest_rsync_uri); + let Some(bytes) = self + .db + .get_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + else { + return Ok(None); + }; + let vcir = decode_cbor::(&bytes, "vcir")?; + vcir.validate_internal()?; + Ok(Some(vcir)) + } + + pub fn get_vcir_failed_fetch_reuse_identity( + &self, + manifest_rsync_uri: &str, + ) -> StorageResult> { + let cf = self.cf(CF_VCIR_FAILED_FETCH_REUSE_IDENTITY)?; + let key = vcir_failed_fetch_reuse_identity_key(manifest_rsync_uri); + let Some(bytes) = self + .db + .get_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + else { + return Ok(None); + }; + let identity = decode_cbor::( + &bytes, + "vcir_failed_fetch_reuse_identity", + )?; + identity.validate_internal()?; + Ok(Some(identity)) + } + + pub fn get_manifest_replay_meta( + &self, + manifest_rsync_uri: &str, + ) -> StorageResult> { + let cf = self.cf(CF_MANIFEST_REPLAY_META)?; + let key = manifest_replay_meta_key(manifest_rsync_uri); + let Some(bytes) = self + .db + .get_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + else { + return Ok(None); + }; + let meta = decode_cbor::(&bytes, "manifest_replay_meta")?; + meta.validate_internal()?; + Ok(Some(meta)) + } + + pub fn get_roa_cache_projection( + &self, + manifest_rsync_uri: &str, + ) -> StorageResult> { + let cf = self.cf(CF_ROA_CACHE_PROJECTION)?; + let key = roa_cache_projection_key(manifest_rsync_uri); + let Some(bytes) = self + .db + .get_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + else { + return Ok(None); + }; + let projection = decode_cbor::(&bytes, "roa_cache_projection")?; + projection.validate_internal()?; + Ok(Some(projection)) + } + + pub fn put_child_certificate_cache_projection( + &self, + projection: &ChildCertificateCacheProjection, + ) -> StorageResult<()> { + projection.validate_internal()?; + let cf = self.cf(CF_CHILD_CERTIFICATE_CACHE_PROJECTION)?; + let key = child_certificate_cache_projection_key(&projection.cache_key_sha256_hex); + let value = encode_cbor(projection, "child_certificate_cache_projection")?; + self.db + .put_cf(cf, key.as_bytes(), value) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(()) + } + + pub fn get_child_certificate_cache_projection( + &self, + cache_key_sha256_hex: &str, + ) -> StorageResult> { + validate_sha256_hex( + "child_certificate_cache_projection.cache_key_sha256_hex", + cache_key_sha256_hex, + )?; + let cf = self.cf(CF_CHILD_CERTIFICATE_CACHE_PROJECTION)?; + let key = child_certificate_cache_projection_key(cache_key_sha256_hex); + let Some(bytes) = self + .db + .get_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + else { + return Ok(None); + }; + let projection = decode_cbor::( + &bytes, + "child_certificate_cache_projection", + )?; + projection.validate_internal()?; + Ok(Some(projection)) + } + + pub fn get_child_certificate_cache_projections_batch( + &self, + cache_key_sha256_hexes: &[String], + ) -> StorageResult>> { + if cache_key_sha256_hexes.is_empty() { + return Ok(Vec::new()); + } + + for cache_key_sha256_hex in cache_key_sha256_hexes { + validate_sha256_hex( + "child_certificate_cache_projection.cache_key_sha256_hex", + cache_key_sha256_hex, + )?; + } + + let cf = self.cf(CF_CHILD_CERTIFICATE_CACHE_PROJECTION)?; + let keys: Vec = cache_key_sha256_hexes + .iter() + .map(|key| child_certificate_cache_projection_key(key)) + .collect(); + self.db + .multi_get_cf(keys.iter().map(|key| (cf, key.as_bytes()))) + .into_iter() + .map(|res| { + let Some(bytes) = res.map_err(|e| StorageError::RocksDb(e.to_string()))? else { + return Ok(None); + }; + let projection = decode_cbor::( + &bytes, + "child_certificate_cache_projection", + )?; + projection.validate_internal()?; + Ok(Some(projection)) + }) + .collect() + } + + fn child_certificate_cache_segment_path(&self, manifest_rsync_uri: &str) -> PathBuf { + self.child_certificate_cache_index_dir + .join(child_certificate_cache_segment_file_name( + manifest_rsync_uri, + )) + } + + pub fn get_child_certificate_cache_projections_mmap_segment( + &self, + manifest_rsync_uri: &str, + cache_key_sha256_hexes: &[String], + ) -> StorageResult> { + if cache_key_sha256_hexes.is_empty() { + return Ok(Some(ChildCertificateCacheMmapLookup::default())); + } + validate_non_empty( + "child_certificate_cache_mmap_segment.manifest_rsync_uri", + manifest_rsync_uri, + )?; + for cache_key_sha256_hex in cache_key_sha256_hexes { + validate_sha256_hex( + "child_certificate_cache_projection.cache_key_sha256_hex", + cache_key_sha256_hex, + )?; + } + + let path = self.child_certificate_cache_segment_path(manifest_rsync_uri); + if !path.exists() { + return Ok(None); + } + + let (index, _) = load_pp_cache_mmap_index(&path)?; + let file_bytes = index.file_bytes(); + let mut hits = 0usize; + let mut misses = 0usize; + let mut projections = Vec::with_capacity(cache_key_sha256_hexes.len()); + for cache_key_sha256_hex in cache_key_sha256_hexes { + match index.lookup(cache_key_sha256_hex) { + Some(PpCacheIndexLookup::Hit(bytes)) => { + let projection = decode_cbor::( + bytes, + "child_certificate_cache_projection_mmap_segment", + )?; + projection.validate_internal()?; + hits = hits.saturating_add(1); + projections.push(Some(projection)); + } + Some(PpCacheIndexLookup::Deleted) | None => { + misses = misses.saturating_add(1); + projections.push(None); + } + } + } + + Ok(Some(ChildCertificateCacheMmapLookup { + projections, + hits, + misses, + file_bytes, + })) + } + + pub fn write_child_certificate_cache_mmap_segment( + &self, + manifest_rsync_uri: &str, + projections: &[ChildCertificateCacheProjection], + ) -> StorageResult { + validate_non_empty( + "child_certificate_cache_mmap_segment.manifest_rsync_uri", + manifest_rsync_uri, + )?; + let mut entries = Vec::with_capacity(projections.len()); + for projection in projections { + projection.validate_internal()?; + entries.push(( + projection.cache_key_sha256_hex.clone(), + encode_cbor( + projection, + "child_certificate_cache_projection_mmap_segment", + )?, + )); + } + let path = self.child_certificate_cache_segment_path(manifest_rsync_uri); + write_pp_cache_index_atomic(&path, entries) + } + + pub fn write_child_certificate_cache_mmap_segment_overlay( + &self, + manifest_rsync_uri: &str, + cache_key_sha256_hexes: &[String], + projections: &[ChildCertificateCacheProjection], + ) -> StorageResult { + validate_non_empty( + "child_certificate_cache_mmap_segment.manifest_rsync_uri", + manifest_rsync_uri, + )?; + let mut dirty = HashMap::>::with_capacity(projections.len()); + for projection in projections { + projection.validate_internal()?; + dirty.insert( + projection.cache_key_sha256_hex.clone(), + encode_cbor( + projection, + "child_certificate_cache_projection_mmap_segment", + )?, + ); + } + + let path = self.child_certificate_cache_segment_path(manifest_rsync_uri); + let existing = if path.exists() { + Some(load_pp_cache_mmap_index(&path)?.0) + } else { + None + }; + let mut entries = Vec::with_capacity(cache_key_sha256_hexes.len()); + let mut emitted = HashSet::::new(); + for cache_key_sha256_hex in cache_key_sha256_hexes { + validate_sha256_hex( + "child_certificate_cache_projection.cache_key_sha256_hex", + cache_key_sha256_hex, + )?; + if !emitted.insert(cache_key_sha256_hex.clone()) { + continue; + } + if let Some(value) = dirty.remove(cache_key_sha256_hex) { + entries.push((cache_key_sha256_hex.clone(), value)); + continue; + } + if let Some(existing) = existing.as_ref() { + if let Some(PpCacheIndexLookup::Hit(bytes)) = existing.lookup(cache_key_sha256_hex) + { + entries.push((cache_key_sha256_hex.clone(), bytes.to_vec())); + } + } + } + for (key, value) in dirty { + if emitted.insert(key.clone()) { + entries.push((key, value)); + } + } + + write_pp_cache_index_atomic(&path, entries) + } + + pub fn get_publication_point_cache_projection( + &self, + manifest_rsync_uri: &str, + ) -> StorageResult> { + self.get_publication_point_cache_projection_from_db(manifest_rsync_uri) + } + + pub fn get_publication_point_cache_projection_cached( + &self, + manifest_rsync_uri: &str, + ) -> StorageResult> { + let mut owned_bytes: Option> = None; + { + let mut guard = self + .publication_point_cache_projection_index + .lock() + .map_err(|e| { + StorageError::RocksDb(format!( + "publication point cache index lock poisoned: {e}" + )) + })?; + if matches!( + *guard, + PublicationPointCacheProjectionIndexState::Uninitialized + ) { + let load_started = std::time::Instant::now(); + let raw_index_enabled = pp_cache_raw_index_enabled(); + *guard = if !raw_index_enabled { + crate::progress_log::emit( + "publication_point_cache_raw_index", + serde_json::json!({ + "state": "disabled_by_env", + "entries": 0, + "bytes": 0, + "load_ms": load_started.elapsed().as_millis() as u64, + }), + ); + PublicationPointCacheProjectionIndexState::Disabled + } else { + match load_pp_cache_mmap_index_set(&self.publication_point_cache_index_dir) { + Ok((mmap, stats)) => { + crate::progress_log::emit( + "publication_point_cache_mmap_index_load", + serde_json::json!({ + "state": "loaded", + "entries": stats.entries, + "bytes": stats.bytes, + "file_bytes": stats.file_bytes, + "load_ms": stats.load_ms, + }), + ); + PublicationPointCacheProjectionIndexState::LoadedMmap { + mmap, + dirty: HashMap::new(), + dirty_bytes: 0, + load_stats: stats, + } + } + Err(e) => { + crate::progress_log::emit( + "publication_point_cache_mmap_index_load", + serde_json::json!({ + "state": "fallback_scan", + "error": e.to_string(), + "load_ms": load_started.elapsed().as_millis() as u64, + }), + ); + let scan_started = std::time::Instant::now(); + let (index, bytes) = + self.load_publication_point_cache_projection_index()?; + if index.is_empty() { + let limit = pp_cache_raw_index_empty_build_limit_bytes(); + crate::progress_log::emit( + "publication_point_cache_raw_index", + serde_json::json!({ + "state": "empty_building_bounded", + "entries": 0, + "bytes": 0, + "empty_build_limit_bytes": limit, + "load_ms": scan_started.elapsed().as_millis() as u64, + }), + ); + PublicationPointCacheProjectionIndexState::BuildingFromEmpty { + index, + bytes: 0, + limit, + } + } else { + crate::progress_log::emit( + "publication_point_cache_raw_index", + serde_json::json!({ + "state": "loaded", + "entries": index.len(), + "bytes": bytes, + "load_ms": scan_started.elapsed().as_millis() as u64, + }), + ); + PublicationPointCacheProjectionIndexState::Loaded { index, bytes } + } + } + } + }; + } + match &*guard { + PublicationPointCacheProjectionIndexState::Loaded { index, .. } => { + owned_bytes = index.get(manifest_rsync_uri).cloned(); + } + PublicationPointCacheProjectionIndexState::BuildingFromEmpty { index, .. } => { + owned_bytes = index.get(manifest_rsync_uri).cloned(); + } + PublicationPointCacheProjectionIndexState::LoadedMmap { mmap, dirty, .. } => { + if let Some(bytes) = dirty.get(manifest_rsync_uri).cloned() { + if bytes.is_empty() { + return Ok(None); + } + owned_bytes = Some(bytes); + } else if let Some(lookup) = mmap.lookup(manifest_rsync_uri) { + match lookup { + PpCacheIndexLookup::Hit(bytes) => { + let projection = decode_cbor::( + bytes, + "publication_point_cache_projection", + )?; + projection.validate_internal()?; + return Ok(Some(projection)); + } + PpCacheIndexLookup::Deleted => return Ok(None), + } + } + } + PublicationPointCacheProjectionIndexState::Disabled + | PublicationPointCacheProjectionIndexState::Uninitialized => {} + } + } + let bytes = owned_bytes; + let Some(bytes) = bytes else { + return Ok(None); + }; + let projection = decode_cbor::( + bytes.as_ref(), + "publication_point_cache_projection", + )?; + projection.validate_internal()?; + Ok(Some(projection)) + } + + fn get_publication_point_cache_projection_from_db( + &self, + manifest_rsync_uri: &str, + ) -> StorageResult> { + let cf = self.cf(CF_PUBLICATION_POINT_CACHE_PROJECTION)?; + let key = publication_point_cache_projection_key(manifest_rsync_uri); + let Some(bytes) = self + .db + .get_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + else { + return Ok(None); + }; + let projection = decode_cbor::( + &bytes, + "publication_point_cache_projection", + )?; + projection.validate_internal()?; + Ok(Some(projection)) + } + + fn load_publication_point_cache_projection_index( + &self, + ) -> StorageResult<(HashMap>, usize)> { + let cf = self.cf(CF_PUBLICATION_POINT_CACHE_PROJECTION)?; + let mode = IteratorMode::Start; + let mut index = HashMap::new(); + let mut bytes_total = 0usize; + for res in self.db.iterator_cf(cf, mode) { + let (key, value) = res.map_err(|e| StorageError::RocksDb(e.to_string()))?; + let Some(manifest_rsync_uri) = + publication_point_cache_projection_key_manifest_uri(&key) + else { + continue; + }; + bytes_total = bytes_total.saturating_add(value.len()); + index.insert(manifest_rsync_uri, Arc::<[u8]>::from(value.to_vec())); + } + Ok((index, bytes_total)) + } + + fn load_publication_point_cache_projection_entries( + &self, + ) -> StorageResult)>> { + let cf = self.cf(CF_PUBLICATION_POINT_CACHE_PROJECTION)?; + let mode = IteratorMode::Start; + let mut entries = Vec::new(); + for res in self.db.iterator_cf(cf, mode) { + let (key, value) = res.map_err(|e| StorageError::RocksDb(e.to_string()))?; + let Some(manifest_rsync_uri) = + publication_point_cache_projection_key_manifest_uri(&key) + else { + continue; + }; + entries.push((manifest_rsync_uri, value.to_vec())); + } + Ok(entries) + } + + fn apply_publication_point_cache_projection_index_action( + &self, + action: PublicationPointCacheProjectionWriteAction<'_>, + ) -> StorageResult<()> { + match action { + PublicationPointCacheProjectionWriteAction::Keep => Ok(()), + PublicationPointCacheProjectionWriteAction::Write(projection) => { + self.update_publication_point_cache_projection_index(projection) + } + PublicationPointCacheProjectionWriteAction::Delete { manifest_rsync_uri } => { + self.delete_publication_point_cache_projection_index_entry(manifest_rsync_uri) + } + } + } + + fn update_publication_point_cache_projection_index( + &self, + projection: &PublicationPointCacheProjection, + ) -> StorageResult<()> { + self.try_load_publication_point_cache_mmap_index_for_update("write")?; + let mut guard = self + .publication_point_cache_projection_index + .lock() + .map_err(|e| { + StorageError::RocksDb(format!("publication point cache index lock poisoned: {e}")) + })?; + let bytes = encode_cbor(projection, "publication_point_cache_projection")?; + match &mut *guard { + PublicationPointCacheProjectionIndexState::Loaded { + index, + bytes: total_bytes, + } => { + if let Some(previous) = index.insert( + projection.manifest_rsync_uri.clone(), + Arc::<[u8]>::from(bytes.clone()), + ) { + *total_bytes = total_bytes.saturating_sub(previous.len()); + } + *total_bytes = total_bytes.saturating_add(bytes.len()); + } + PublicationPointCacheProjectionIndexState::BuildingFromEmpty { + index, + bytes: total_bytes, + limit, + } => { + if let Some(previous) = index.remove(&projection.manifest_rsync_uri) { + *total_bytes = total_bytes.saturating_sub(previous.len()); + } + if total_bytes.saturating_add(bytes.len()) <= *limit { + *total_bytes += bytes.len(); + index.insert( + projection.manifest_rsync_uri.clone(), + Arc::<[u8]>::from(bytes), + ); + } else { + *guard = PublicationPointCacheProjectionIndexState::Disabled; + } + } + PublicationPointCacheProjectionIndexState::LoadedMmap { + dirty, dirty_bytes, .. + } => { + if let Some(previous) = dirty.insert( + projection.manifest_rsync_uri.clone(), + Arc::<[u8]>::from(bytes.clone()), + ) { + *dirty_bytes = dirty_bytes.saturating_sub(previous.len()); + } + *dirty_bytes = dirty_bytes.saturating_add(bytes.len()); + } + PublicationPointCacheProjectionIndexState::Uninitialized + | PublicationPointCacheProjectionIndexState::Disabled => {} + } + Ok(()) + } + + fn delete_publication_point_cache_projection_index_entry( + &self, + manifest_rsync_uri: &str, + ) -> StorageResult<()> { + self.try_load_publication_point_cache_mmap_index_for_update("delete")?; + let mut guard = self + .publication_point_cache_projection_index + .lock() + .map_err(|e| { + StorageError::RocksDb(format!("publication point cache index lock poisoned: {e}")) + })?; + match &mut *guard { + PublicationPointCacheProjectionIndexState::Loaded { + index, + bytes: total_bytes, + } => { + if let Some(previous) = index.remove(manifest_rsync_uri) { + *total_bytes = total_bytes.saturating_sub(previous.len()); + } + } + PublicationPointCacheProjectionIndexState::BuildingFromEmpty { + index, + bytes: total_bytes, + .. + } => { + if let Some(previous) = index.remove(manifest_rsync_uri) { + *total_bytes = total_bytes.saturating_sub(previous.len()); + } + } + PublicationPointCacheProjectionIndexState::LoadedMmap { + dirty, dirty_bytes, .. + } => { + if let Some(previous) = + dirty.insert(manifest_rsync_uri.to_string(), Arc::<[u8]>::from([])) + { + *dirty_bytes = dirty_bytes.saturating_sub(previous.len()); + } + } + PublicationPointCacheProjectionIndexState::Uninitialized + | PublicationPointCacheProjectionIndexState::Disabled => {} + } + Ok(()) + } + + pub fn refresh_publication_point_cache_mmap_index( + &self, + ) -> StorageResult> { + if !pp_cache_raw_index_enabled() { + return Ok(None); + } + self.try_load_publication_point_cache_mmap_index_for_update("refresh")?; + enum RefreshAction { + Entries { + entries: Vec<(String, Vec)>, + write_segment: bool, + old_entries: usize, + dirty_entries: usize, + }, + ScanDb, + } + let action = { + let guard = self + .publication_point_cache_projection_index + .lock() + .map_err(|e| { + StorageError::RocksDb(format!( + "publication point cache index lock poisoned: {e}" + )) + })?; + match &*guard { + PublicationPointCacheProjectionIndexState::LoadedMmap { mmap, dirty, .. } => { + RefreshAction::Entries { + entries: dirty + .iter() + .map(|(key, value)| (key.clone(), value.as_ref().to_vec())) + .collect::>(), + write_segment: true, + old_entries: mmap.entries(), + dirty_entries: dirty.len(), + } + } + PublicationPointCacheProjectionIndexState::Loaded { index, .. } + | PublicationPointCacheProjectionIndexState::BuildingFromEmpty { index, .. } => { + RefreshAction::Entries { + entries: index + .iter() + .map(|(key, value)| (key.clone(), value.as_ref().to_vec())) + .collect::>(), + write_segment: false, + old_entries: 0, + dirty_entries: 0, + } + } + PublicationPointCacheProjectionIndexState::Disabled + | PublicationPointCacheProjectionIndexState::Uninitialized => RefreshAction::ScanDb, + } + }; + let (entries, write_segment, old_entries, dirty_entries) = match action { + RefreshAction::Entries { + entries, + write_segment, + old_entries, + dirty_entries, + } => (entries, write_segment, old_entries, dirty_entries), + RefreshAction::ScanDb => ( + self.load_publication_point_cache_projection_entries()?, + false, + 0, + 0, + ), + }; + if entries.is_empty() { + return Ok(None); + } + let current_path = self.publication_point_cache_index_dir.join("current.idx"); + let mut stats = if write_segment && current_path.exists() { + write_pp_cache_index_segment(&self.publication_point_cache_index_dir, entries)? + } else { + write_pp_cache_index_atomic(¤t_path, entries)? + }; + stats.old_entries = old_entries; + stats.dirty_entries = dirty_entries; + crate::progress_log::emit( + "publication_point_cache_mmap_index_refresh", + serde_json::json!({ + "state": stats.state, + "old_entries": stats.old_entries, + "dirty_entries": stats.dirty_entries, + "new_entries": stats.new_entries, + "file_bytes": stats.file_bytes, + "write_ms": stats.write_ms, + }), + ); + let directory_stats = + pp_cache_index_directory_stats(&self.publication_point_cache_index_dir)?; + let compaction_reason = if directory_stats.segment_count + >= PP_CACHE_INDEX_COMPACTION_SEGMENT_THRESHOLD + { + Some(format!( + "segment_count>={PP_CACHE_INDEX_COMPACTION_SEGMENT_THRESHOLD}" + )) + } else if directory_stats.total_file_bytes >= PP_CACHE_INDEX_COMPACTION_BYTES_THRESHOLD { + Some(format!( + "total_file_bytes>={PP_CACHE_INDEX_COMPACTION_BYTES_THRESHOLD}" + )) + } else { + None + }; + if let Some(reason) = compaction_reason { + crate::progress_log::emit( + "publication_point_cache_mmap_index_compaction", + serde_json::json!({ + "state": "started", + "reason": reason, + "segment_count": directory_stats.segment_count, + "total_file_bytes": directory_stats.total_file_bytes, + }), + ); + match compact_pp_cache_index(&self.publication_point_cache_index_dir) { + Ok(mut compact_stats) => { + compact_stats.compaction_reason = Some(reason); + compact_stats.old_entries = stats.old_entries; + compact_stats.dirty_entries = stats.dirty_entries; + crate::progress_log::emit( + "publication_point_cache_mmap_index_compaction", + serde_json::json!({ + "state": "completed", + "reason": compact_stats.compaction_reason, + "segments_before": compact_stats.compaction_segments_before, + "total_file_bytes_before": compact_stats.compaction_total_file_bytes_before, + "live_entries": compact_stats.compaction_live_entries, + "file_bytes": compact_stats.compaction_file_bytes, + "reclaimed_bytes": compact_stats.compaction_reclaimed_bytes, + "deleted_segments": compact_stats.compaction_deleted_segments, + "compaction_ms": compact_stats.compaction_ms, + }), + ); + stats.compaction_triggered = compact_stats.compaction_triggered; + stats.compaction_reason = compact_stats.compaction_reason; + stats.compaction_segments_before = compact_stats.compaction_segments_before; + stats.compaction_total_file_bytes_before = + compact_stats.compaction_total_file_bytes_before; + stats.compaction_live_entries = compact_stats.compaction_live_entries; + stats.compaction_file_bytes = compact_stats.compaction_file_bytes; + stats.compaction_reclaimed_bytes = compact_stats.compaction_reclaimed_bytes; + stats.compaction_ms = compact_stats.compaction_ms; + stats.compaction_deleted_segments = compact_stats.compaction_deleted_segments; + } + Err(e) => { + let error = e.to_string(); + crate::progress_log::emit( + "publication_point_cache_mmap_index_compaction", + serde_json::json!({ + "state": "failed", + "reason": reason, + "segment_count": directory_stats.segment_count, + "total_file_bytes": directory_stats.total_file_bytes, + "error": error, + }), + ); + stats.compaction_triggered = true; + stats.compaction_reason = Some(reason); + stats.compaction_segments_before = directory_stats.segment_count; + stats.compaction_total_file_bytes_before = directory_stats.total_file_bytes; + stats.compaction_error = Some(error); + } + } + } + let mut guard = self + .publication_point_cache_projection_index + .lock() + .map_err(|e| { + StorageError::RocksDb(format!("publication point cache index lock poisoned: {e}")) + })?; + if let PublicationPointCacheProjectionIndexState::LoadedMmap { + dirty, dirty_bytes, .. + } = &mut *guard + { + dirty.clear(); + *dirty_bytes = 0; + } + Ok(Some(stats)) + } + + pub fn put_transport_prefetch_snapshot( + &self, + snapshot: &crate::parallel::transport_prefetch::TransportPrefetchSnapshot, + ) -> StorageResult<()> { + let cf = self.cf(CF_TRANSPORT_PREFETCH)?; + let value = encode_cbor(snapshot, "transport_prefetch_snapshot")?; + self.db + .put_cf(cf, TRANSPORT_PREFETCH_LAST_SNAPSHOT_KEY.as_bytes(), value) + .map_err(|e| StorageError::RocksDb(e.to_string())) + } + + pub fn get_transport_prefetch_snapshot( + &self, + ) -> StorageResult> { + let cf = self.cf(CF_TRANSPORT_PREFETCH)?; + let Some(bytes) = self + .db + .get_cf(cf, TRANSPORT_PREFETCH_LAST_SNAPSHOT_KEY.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + else { + return Ok(None); + }; + decode_cbor::( + &bytes, + "transport_prefetch_snapshot", + ) + .map(Some) + } + + pub fn list_vcirs(&self) -> StorageResult> { + let cf = self.cf(CF_VCIR)?; + let mode = IteratorMode::Start; + let mut out = Vec::new(); + for res in self.db.iterator_cf(cf, mode) { + let (_key, bytes) = res.map_err(|e| StorageError::RocksDb(e.to_string()))?; + let vcir = decode_cbor::(&bytes, "vcir")?; + vcir.validate_internal()?; + out.push(vcir); + } + Ok(out) + } + + /// Remove every reusable VCIR and its failed-fetch identity while retaining + /// manifest replay metadata. The replay metadata is a fresh-validation + /// guard, not a reusable validation result; removing it changes the normal + /// traversal semantics for manifests that are not selected for replay. + /// + /// This is intentionally narrower than [`Self::delete_vcir`]. It preserves + /// manifest replay metadata while clearing reusable validation records. + pub fn clear_vcir_reuse_records(&self) -> StorageResult { + let vcir_cf = self.cf(CF_VCIR)?; + let identity_cf = self.cf(CF_VCIR_FAILED_FETCH_REUSE_IDENTITY)?; + let mut batch = WriteBatch::default(); + let mut summary = VcirReuseRecordsCleared::default(); + + for entry in self.db.iterator_cf(vcir_cf, IteratorMode::Start) { + let (key, _value) = entry.map_err(|error| StorageError::RocksDb(error.to_string()))?; + batch.delete_cf(vcir_cf, key); + summary.vcir_records += 1; + } + for entry in self.db.iterator_cf(identity_cf, IteratorMode::Start) { + let (key, _value) = entry.map_err(|error| StorageError::RocksDb(error.to_string()))?; + batch.delete_cf(identity_cf, key); + summary.failed_fetch_identity_records += 1; + } + if summary.vcir_records > 0 || summary.failed_fetch_identity_records > 0 { + self.write_batch(batch)?; + } + Ok(summary) + } + + pub fn summarize_vcir_storage(&self) -> StorageResult { + let cf = self.cf(CF_VCIR)?; + let mode = IteratorMode::Start; + let mut summary = VcirStorageSummary::default(); + for res in self.db.iterator_cf(cf, mode) { + let (_key, bytes) = res.map_err(|e| StorageError::RocksDb(e.to_string()))?; + let vcir = decode_cbor::(&bytes, "vcir")?; + vcir.validate_internal()?; + summary.entry_count += 1; + let value_bytes = bytes.len() as u64; + summary.vcir_value_bytes += value_bytes; + if value_bytes > summary.vcir_value_bytes_max { + summary.vcir_value_bytes_max = value_bytes; + summary.vcir_value_bytes_max_manifest_rsync_uri = + Some(vcir.manifest_rsync_uri.clone()); + } + let entry_summary = VcirStorageEntrySummary::from_vcir(&vcir, value_bytes); + summary.core_fields.add_assign(&entry_summary.core_fields); + summary + .ccr_projection + .add_assign(&entry_summary.ccr_projection); + summary + .child_resources + .add_assign(&entry_summary.child_resources); + summary.field_sizes.add_assign(&entry_summary.field_sizes); + push_top_vcir_storage_entry( + &mut summary.top_entries_by_vcir_value_bytes, + entry_summary, + ); + } + summary.local_output_old_projection_bytes = + summary.field_sizes.local_output_old_projection_bytes(); + summary.local_output_typed_projection_bytes = + summary.field_sizes.local_output_typed_projection_bytes(); + summary.local_output_projection_saved_bytes = + summary.field_sizes.local_output_projection_saved_bytes(); + Ok(summary) + } + + pub fn delete_vcir(&self, manifest_rsync_uri: &str) -> StorageResult<()> { + let vcir_cf = self.cf(CF_VCIR)?; + let failed_fetch_reuse_identity_cf = self.cf(CF_VCIR_FAILED_FETCH_REUSE_IDENTITY)?; + let replay_cf = self.cf(CF_MANIFEST_REPLAY_META)?; + let projection_cf = self.cf(CF_ROA_CACHE_PROJECTION)?; + let mut batch = WriteBatch::default(); + let key = vcir_key(manifest_rsync_uri); + batch.delete_cf(vcir_cf, key.as_bytes()); + let failed_fetch_reuse_identity_key = + vcir_failed_fetch_reuse_identity_key(manifest_rsync_uri); + batch.delete_cf( + failed_fetch_reuse_identity_cf, + failed_fetch_reuse_identity_key.as_bytes(), + ); + let replay_key = manifest_replay_meta_key(manifest_rsync_uri); + batch.delete_cf(replay_cf, replay_key.as_bytes()); + let projection_key = roa_cache_projection_key(manifest_rsync_uri); + batch.delete_cf(projection_cf, projection_key.as_bytes()); + self.write_batch(batch) + } + + pub fn put_rrdp_source_record(&self, record: &RrdpSourceRecord) -> StorageResult<()> { + record.validate_internal()?; + let cf = self.cf(CF_RRDP_SOURCE)?; + let key = rrdp_source_key(&record.notify_uri); + let value = encode_cbor(record, "rrdp_source")?; + self.db + .put_cf(cf, key.as_bytes(), value) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(()) + } + + pub fn get_rrdp_source_record( + &self, + notify_uri: &str, + ) -> StorageResult> { + let cf = self.cf(CF_RRDP_SOURCE)?; + let key = rrdp_source_key(notify_uri); + let Some(bytes) = self + .db + .get_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + else { + return Ok(None); + }; + let record = decode_cbor::(&bytes, "rrdp_source")?; + record.validate_internal()?; + Ok(Some(record)) + } + + pub fn put_rrdp_source_member_record( + &self, + record: &RrdpSourceMemberRecord, + ) -> StorageResult<()> { + record.validate_internal()?; + let cf = self.cf(CF_RRDP_SOURCE_MEMBER)?; + let key = rrdp_source_member_key(&record.notify_uri, &record.rsync_uri); + let value = encode_cbor(record, "rrdp_source_member")?; + self.db + .put_cf(cf, key.as_bytes(), value) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(()) + } + + pub fn get_rrdp_source_member_record( + &self, + notify_uri: &str, + rsync_uri: &str, + ) -> StorageResult> { + let cf = self.cf(CF_RRDP_SOURCE_MEMBER)?; + let key = rrdp_source_member_key(notify_uri, rsync_uri); + let Some(bytes) = self + .db + .get_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + else { + return Ok(None); + }; + let record = decode_cbor::(&bytes, "rrdp_source_member")?; + record.validate_internal()?; + Ok(Some(record)) + } + + pub fn list_rrdp_source_member_records( + &self, + notify_uri: &str, + ) -> StorageResult> { + let cf = self.cf(CF_RRDP_SOURCE_MEMBER)?; + let prefix = rrdp_source_member_prefix(notify_uri); + let mode = IteratorMode::From(prefix.as_bytes(), Direction::Forward); + self.db + .iterator_cf(cf, mode) + .take_while(|res| match res { + Ok((key, _)) => key.starts_with(prefix.as_bytes()), + Err(_) => false, + }) + .map(|res| { + let (_key, value) = res.map_err(|e| StorageError::RocksDb(e.to_string()))?; + let record = decode_cbor::(&value, "rrdp_source_member")?; + record.validate_internal()?; + Ok(record) + }) + .collect() + } + + pub fn list_current_rrdp_source_members( + &self, + notify_uri: &str, + ) -> StorageResult> { + let mut records = self.list_rrdp_source_member_records(notify_uri)?; + records.retain(|record| record.present); + records.sort_by(|a, b| a.rsync_uri.cmp(&b.rsync_uri)); + Ok(records) + } + + pub fn is_current_rrdp_source_member( + &self, + notify_uri: &str, + rsync_uri: &str, + ) -> StorageResult { + Ok(matches!( + self.get_rrdp_source_member_record(notify_uri, rsync_uri)?, + Some(record) if record.present + )) + } + + pub fn load_current_object_bytes_by_uri( + &self, + rsync_uri: &str, + ) -> StorageResult>> { + Ok(self + .load_current_object_with_hash_by_uri(rsync_uri)? + .map(|obj| obj.bytes)) + } + + pub fn load_current_object_with_hash_by_uri( + &self, + rsync_uri: &str, + ) -> StorageResult> { + let Some(view) = self.get_repository_view_entry(rsync_uri)? else { + return Ok(None); + }; + + match view.state { + RepositoryViewState::Withdrawn => Ok(None), + RepositoryViewState::Present | RepositoryViewState::Replaced => { + let hash = view + .current_hash + .as_deref() + .ok_or(StorageError::InvalidData { + entity: "repository_view", + detail: format!("current_hash missing for current object URI: {rsync_uri}"), + })?; + let bytes = self + .get_blob_bytes(hash)? + .ok_or(StorageError::InvalidData { + entity: "repository_view", + detail: format!( + "blob bytes missing for current object URI: {rsync_uri} (hash={hash})" + ), + })?; + let current_hash = decode_sha256_hex_32("repository_view.current_hash", hash)?; + Ok(Some(CurrentObjectWithHash { + current_hash_hex: hash.to_ascii_lowercase(), + current_hash, + bytes, + })) + } + } + } + + pub fn put_rrdp_uri_owner_record(&self, record: &RrdpUriOwnerRecord) -> StorageResult<()> { + record.validate_internal()?; + let cf = self.cf(CF_RRDP_URI_OWNER)?; + let key = rrdp_uri_owner_key(&record.rsync_uri); + let value = encode_cbor(record, "rrdp_uri_owner")?; + self.db + .put_cf(cf, key.as_bytes(), value) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(()) + } + + pub fn get_rrdp_uri_owner_record( + &self, + rsync_uri: &str, + ) -> StorageResult> { + let cf = self.cf(CF_RRDP_URI_OWNER)?; + let key = rrdp_uri_owner_key(rsync_uri); + let Some(bytes) = self + .db + .get_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + else { + return Ok(None); + }; + let record = decode_cbor::(&bytes, "rrdp_uri_owner")?; + record.validate_internal()?; + Ok(Some(record)) + } + + pub fn delete_rrdp_uri_owner_record(&self, rsync_uri: &str) -> StorageResult<()> { + let cf = self.cf(CF_RRDP_URI_OWNER)?; + let key = rrdp_uri_owner_key(rsync_uri); + self.db + .delete_cf(cf, key.as_bytes()) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(()) + } + + #[allow(dead_code)] + + pub fn write_batch(&self, batch: WriteBatch) -> StorageResult<()> { + self.db + .write(batch) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(()) + } +} + +fn verify_repository_blob_batch( + store: &RocksStore, + batch: &[(String, String)], + summary: &mut RepositoryBlobVerificationSummary, +) -> StorageResult<()> { + let hashes = batch + .iter() + .map(|(_, hash)| hash.clone()) + .collect::>(); + let blobs = store.get_blob_bytes_batch(&hashes)?; + for ((uri, expected_hash), blob) in batch.iter().zip(blobs.into_iter()) { + let bytes = blob.as_ref().ok_or(StorageError::InvalidData { + entity: "repository_blob_verification", + detail: format!("blob missing for URI {uri} (hash={expected_hash})"), + })?; + let actual_hash = hex::encode(compute_sha256_32(bytes)); + if !actual_hash.eq_ignore_ascii_case(expected_hash) { + return Err(StorageError::InvalidData { + entity: "repository_blob_verification", + detail: format!( + "blob hash mismatch for URI {uri}: expected={expected_hash}, actual={actual_hash}" + ), + }); + } + summary.current_objects += 1; + summary.bytes_verified += bytes.len() as u64; + } + summary.batches += 1; + Ok(()) +} + +#[cfg(test)] +#[path = "storage/tests.rs"] +mod tests; diff --git a/crates/panda-rpki-validator/src/storage/config.rs b/crates/panda-rpki-validator/src/storage/config.rs new file mode 100644 index 0000000..bd4d282 --- /dev/null +++ b/crates/panda-rpki-validator/src/storage/config.rs @@ -0,0 +1,178 @@ +use rocksdb::{ColumnFamilyDescriptor, DBCompressionType, Options}; + +pub const CF_REPOSITORY_VIEW: &str = "repository_view"; +pub const CF_RAW_BY_HASH: &str = "raw_by_hash"; +pub const CF_RAW_BLOB: &str = "raw_blob"; +pub const CF_VCIR: &str = "vcir"; +pub const CF_VCIR_FAILED_FETCH_REUSE_IDENTITY: &str = "vcir_failed_fetch_reuse_identity"; +pub const CF_MANIFEST_REPLAY_META: &str = "manifest_replay_meta"; +pub const CF_ROA_CACHE_PROJECTION: &str = "roa_cache_projection"; +pub const CF_PUBLICATION_POINT_CACHE_PROJECTION: &str = "publication_point_cache_projection"; +pub const CF_CHILD_CERTIFICATE_CACHE_PROJECTION: &str = "child_certificate_cache_projection"; +pub const CF_RRDP_SOURCE: &str = "rrdp_source"; +pub const CF_RRDP_SOURCE_MEMBER: &str = "rrdp_source_member"; +pub const CF_RRDP_URI_OWNER: &str = "rrdp_uri_owner"; +pub const CF_TRANSPORT_PREFETCH: &str = "transport_prefetch"; + +pub const ALL_COLUMN_FAMILY_NAMES: &[&str] = &[ + CF_REPOSITORY_VIEW, + CF_RAW_BY_HASH, + CF_RAW_BLOB, + CF_VCIR, + CF_VCIR_FAILED_FETCH_REUSE_IDENTITY, + CF_MANIFEST_REPLAY_META, + CF_ROA_CACHE_PROJECTION, + CF_PUBLICATION_POINT_CACHE_PROJECTION, + CF_CHILD_CERTIFICATE_CACHE_PROJECTION, + CF_RRDP_SOURCE, + CF_RRDP_SOURCE_MEMBER, + CF_RRDP_URI_OWNER, + CF_TRANSPORT_PREFETCH, +]; + +pub(super) const REPOSITORY_VIEW_KEY_PREFIX: &str = "repo_view:"; +pub(super) const RAW_BY_HASH_KEY_PREFIX: &str = "rawbyhash:"; +pub(super) const RAW_BLOB_KEY_PREFIX: &str = "rawblob:"; +pub(super) const VCIR_KEY_PREFIX: &str = "vcir:"; +pub(super) const VCIR_FAILED_FETCH_REUSE_IDENTITY_KEY_PREFIX: &str = + "vcir_failed_fetch_reuse_identity:"; +pub(super) const MANIFEST_REPLAY_META_KEY_PREFIX: &str = "manifest_replay_meta:"; +pub(super) const ROA_CACHE_PROJECTION_KEY_PREFIX: &str = "roa_cache_projection:"; +pub(super) const PUBLICATION_POINT_CACHE_PROJECTION_KEY_PREFIX: &str = + "publication_point_cache_projection:"; +pub(super) const CHILD_CERTIFICATE_CACHE_PROJECTION_KEY_PREFIX: &str = + "child_certificate_cache_projection:"; +pub(super) const RRDP_SOURCE_KEY_PREFIX: &str = "rrdp_source:"; +pub(super) const RRDP_SOURCE_MEMBER_KEY_PREFIX: &str = "rrdp_source_member:"; +pub(super) const RRDP_URI_OWNER_KEY_PREFIX: &str = "rrdp_uri_owner:"; +pub(super) const TRANSPORT_PREFETCH_LAST_SNAPSHOT_KEY: &str = "transport_prefetch:last_snapshot"; + +const WORK_DB_BLOB_MODE_ENV: &str = "RPKI_WORK_DB_BLOB_MODE"; +const WORK_DB_MEMORY_PROFILE_ENV: &str = "RPKI_WORK_DB_MEMORY_PROFILE"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum WorkDbBlobMode { + Current, + Disabled, + Lz4, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum WorkDbMemoryProfile { + Default, + Compact, +} + +pub(super) fn parse_work_db_blob_mode(raw: &str) -> Option { + match raw.trim().to_ascii_lowercase().as_str() { + "" | "default" => Some(default_work_db_blob_mode()), + "current" | "legacy" => Some(WorkDbBlobMode::Current), + "disabled" | "disable" | "off" | "none" | "no_blob" | "no-blob" => { + Some(WorkDbBlobMode::Disabled) + } + "lz4" | "blob_lz4" | "blob-lz4" => Some(WorkDbBlobMode::Lz4), + _ => None, + } +} + +pub(super) fn default_work_db_blob_mode() -> WorkDbBlobMode { + WorkDbBlobMode::Disabled +} + +pub(super) fn work_db_blob_mode_from_env() -> WorkDbBlobMode { + let Ok(raw) = std::env::var(WORK_DB_BLOB_MODE_ENV) else { + return default_work_db_blob_mode(); + }; + match parse_work_db_blob_mode(&raw) { + Some(mode) => mode, + None => { + eprintln!( + "warning: unsupported {WORK_DB_BLOB_MODE_ENV}={raw:?}; using default work-db blobdb mode" + ); + default_work_db_blob_mode() + } + } +} + +pub(super) fn parse_work_db_memory_profile(raw: &str) -> Option { + match raw.trim().to_ascii_lowercase().as_str() { + "" | "default" | "off" | "none" => Some(WorkDbMemoryProfile::Default), + "compact" | "low" | "low_memory" | "low-memory" => Some(WorkDbMemoryProfile::Compact), + _ => None, + } +} + +pub(super) fn work_db_memory_profile_from_env() -> WorkDbMemoryProfile { + let Ok(raw) = std::env::var(WORK_DB_MEMORY_PROFILE_ENV) else { + return WorkDbMemoryProfile::Default; + }; + match parse_work_db_memory_profile(&raw) { + Some(profile) => profile, + None => { + eprintln!( + "warning: unsupported {WORK_DB_MEMORY_PROFILE_ENV}={raw:?}; using default work-db memory profile" + ); + WorkDbMemoryProfile::Default + } + } +} + +pub(super) fn configure_work_db_options( + opts: &mut Options, + blob_mode: WorkDbBlobMode, + memory_profile: WorkDbMemoryProfile, +) { + opts.set_compression_type(DBCompressionType::Lz4); + apply_work_db_memory_profile(opts, memory_profile); + match blob_mode { + WorkDbBlobMode::Current => enable_blobdb_current(opts), + WorkDbBlobMode::Disabled => {} + WorkDbBlobMode::Lz4 => { + enable_blobdb_current(opts); + opts.set_blob_compression_type(DBCompressionType::Lz4); + } + } +} + +pub(super) fn cf_opts(blob_mode: WorkDbBlobMode, memory_profile: WorkDbMemoryProfile) -> Options { + let mut opts = Options::default(); + configure_work_db_options(&mut opts, blob_mode, memory_profile); + opts +} + +pub fn column_family_descriptors() -> Vec { + column_family_descriptors_for_blob_mode(work_db_blob_mode_from_env()) +} + +pub(super) fn column_family_descriptors_for_blob_mode( + blob_mode: WorkDbBlobMode, +) -> Vec { + let memory_profile = work_db_memory_profile_from_env(); + ALL_COLUMN_FAMILY_NAMES + .iter() + .map(|name| ColumnFamilyDescriptor::new(*name, cf_opts(blob_mode, memory_profile))) + .collect() +} + +fn apply_work_db_memory_profile(opts: &mut Options, memory_profile: WorkDbMemoryProfile) { + match memory_profile { + WorkDbMemoryProfile::Default => {} + WorkDbMemoryProfile::Compact => { + opts.set_write_buffer_size(16 * 1024 * 1024); + opts.set_max_write_buffer_number(1); + } + } +} + +pub(super) fn enable_blobdb_current(opts: &mut Options) { + #[allow(unused_mut)] + let mut _enabled = false; + + #[allow(dead_code)] + fn _set(opts: &mut Options) { + opts.set_enable_blob_files(true); + opts.set_min_blob_size(1024); + } + + _set(opts); +} diff --git a/crates/panda-rpki-validator/src/storage/keys.rs b/crates/panda-rpki-validator/src/storage/keys.rs new file mode 100644 index 0000000..4aa6cfb --- /dev/null +++ b/crates/panda-rpki-validator/src/storage/keys.rs @@ -0,0 +1,217 @@ +use serde::{Serialize, de::DeserializeOwned}; + +use crate::data_model::common::der_take_tlv; + +use super::config::*; +use super::pack::PackTime; +use super::{StorageError, StorageResult}; + +pub(super) fn repository_view_key(rsync_uri: &str) -> String { + format!("{REPOSITORY_VIEW_KEY_PREFIX}{rsync_uri}") +} + +pub(super) fn repository_view_prefix(rsync_uri_prefix: &str) -> String { + format!("{REPOSITORY_VIEW_KEY_PREFIX}{rsync_uri_prefix}") +} + +pub(super) fn raw_by_hash_key(sha256_hex: &str) -> String { + format!("{RAW_BY_HASH_KEY_PREFIX}{sha256_hex}") +} + +pub(super) fn raw_blob_key(sha256_hex: &str) -> String { + format!("{RAW_BLOB_KEY_PREFIX}{sha256_hex}") +} + +pub(super) fn vcir_key(manifest_rsync_uri: &str) -> String { + format!("{VCIR_KEY_PREFIX}{manifest_rsync_uri}") +} + +pub(super) fn vcir_failed_fetch_reuse_identity_key(manifest_rsync_uri: &str) -> String { + format!("{VCIR_FAILED_FETCH_REUSE_IDENTITY_KEY_PREFIX}{manifest_rsync_uri}") +} + +pub(super) fn manifest_replay_meta_key(manifest_rsync_uri: &str) -> String { + format!("{MANIFEST_REPLAY_META_KEY_PREFIX}{manifest_rsync_uri}") +} + +pub(super) fn roa_cache_projection_key(manifest_rsync_uri: &str) -> String { + format!("{ROA_CACHE_PROJECTION_KEY_PREFIX}{manifest_rsync_uri}") +} + +pub(super) fn publication_point_cache_projection_key(manifest_rsync_uri: &str) -> String { + format!("{PUBLICATION_POINT_CACHE_PROJECTION_KEY_PREFIX}{manifest_rsync_uri}") +} + +pub(super) fn child_certificate_cache_projection_key(cache_key_sha256_hex: &str) -> String { + format!("{CHILD_CERTIFICATE_CACHE_PROJECTION_KEY_PREFIX}{cache_key_sha256_hex}") +} + +pub(super) fn publication_point_cache_projection_key_manifest_uri(key: &[u8]) -> Option { + let key = std::str::from_utf8(key).ok()?; + key.strip_prefix(PUBLICATION_POINT_CACHE_PROJECTION_KEY_PREFIX) + .map(str::to_string) +} + +pub(super) fn rrdp_source_key(notify_uri: &str) -> String { + format!("{RRDP_SOURCE_KEY_PREFIX}{notify_uri}") +} + +pub(super) fn rrdp_source_member_key(notify_uri: &str, rsync_uri: &str) -> String { + format!("{RRDP_SOURCE_MEMBER_KEY_PREFIX}{notify_uri}:{rsync_uri}") +} + +pub(super) fn rrdp_source_member_prefix(notify_uri: &str) -> String { + format!("{RRDP_SOURCE_MEMBER_KEY_PREFIX}{notify_uri}:") +} + +pub(super) fn rrdp_uri_owner_key(rsync_uri: &str) -> String { + format!("{RRDP_URI_OWNER_KEY_PREFIX}{rsync_uri}") +} + +pub(super) fn encode_cbor(value: &T, entity: &'static str) -> StorageResult> { + serde_cbor::to_vec(value).map_err(|e| StorageError::Codec { + entity, + detail: e.to_string(), + }) +} + +pub(super) fn decode_cbor( + bytes: &[u8], + entity: &'static str, +) -> StorageResult { + serde_cbor::from_slice(bytes).map_err(|e| StorageError::Codec { + entity, + detail: e.to_string(), + }) +} + +pub(super) fn validate_non_empty(field: &'static str, value: &str) -> StorageResult<()> { + if value.is_empty() { + return Err(StorageError::InvalidData { + entity: field, + detail: "must not be empty".to_string(), + }); + } + Ok(()) +} + +pub(super) fn validate_sha256_hex(field: &'static str, value: &str) -> StorageResult<()> { + if value.len() != 64 || !value.as_bytes().iter().all(u8::is_ascii_hexdigit) { + return Err(StorageError::InvalidData { + entity: field, + detail: "must be a 64-character lowercase or uppercase SHA-256 hex string".to_string(), + }); + } + Ok(()) +} + +pub(super) fn decode_sha256_hex_32(field: &'static str, value: &str) -> StorageResult<[u8; 32]> { + validate_sha256_hex(field, value)?; + let mut out = [0u8; 32]; + hex::decode_to_slice(value, &mut out).map_err(|e| StorageError::InvalidData { + entity: field, + detail: format!("hex decode failed: {e}"), + })?; + Ok(out) +} + +pub(super) fn validate_manifest_number_be(field: &'static str, value: &[u8]) -> StorageResult<()> { + if value.is_empty() { + return Err(StorageError::InvalidData { + entity: field, + detail: "must not be empty".to_string(), + }); + } + if value.len() > 20 { + return Err(StorageError::InvalidData { + entity: field, + detail: "must be at most 20 octets".to_string(), + }); + } + if value.len() > 1 && value[0] == 0 { + return Err(StorageError::InvalidData { + entity: field, + detail: "must be minimal big-endian without leading zeros".to_string(), + }); + } + Ok(()) +} + +pub(super) fn validate_sha256_digest_bytes(field: &'static str, value: &[u8]) -> StorageResult<()> { + if value.len() != 32 { + return Err(StorageError::InvalidData { + entity: field, + detail: format!("must be 32 bytes, got {}", value.len()), + }); + } + Ok(()) +} + +pub(super) fn validate_fixed_len_bytes( + field: &'static str, + value: &[u8], + expected_len: usize, +) -> StorageResult<()> { + if value.len() != expected_len { + return Err(StorageError::InvalidData { + entity: field, + detail: format!("must be {expected_len} bytes, got {}", value.len()), + }); + } + Ok(()) +} + +pub(super) fn validate_sorted_unique_fixed_len_bytes( + field: &'static str, + values: &[Vec], + expected_len: usize, +) -> StorageResult<()> { + for value in values { + validate_fixed_len_bytes(field, value, expected_len)?; + } + for window in values.windows(2) { + if window[0] >= window[1] { + return Err(StorageError::InvalidData { + entity: field, + detail: "must be strictly sorted and unique".to_string(), + }); + } + } + Ok(()) +} + +pub(super) fn validate_full_der_with_tag( + field: &'static str, + der: &[u8], + expected_tag: Option, +) -> StorageResult<()> { + let (tag, _value, rem) = der_take_tlv(der).map_err(|detail| StorageError::InvalidData { + entity: field, + detail, + })?; + if !rem.is_empty() { + return Err(StorageError::InvalidData { + entity: field, + detail: "trailing bytes after DER object".to_string(), + }); + } + if let Some(expected_tag) = expected_tag { + if tag != expected_tag { + return Err(StorageError::InvalidData { + entity: field, + detail: format!("unexpected tag 0x{tag:02X}, expected 0x{expected_tag:02X}"), + }); + } + } + Ok(()) +} + +pub(super) fn parse_time( + field: &'static str, + value: &PackTime, +) -> StorageResult { + value.parse().map_err(|detail| StorageError::InvalidData { + entity: field, + detail, + }) +} diff --git a/crates/panda-rpki-validator/src/storage/pack.rs b/crates/panda-rpki-validator/src/storage/pack.rs new file mode 100644 index 0000000..deda515 --- /dev/null +++ b/crates/panda-rpki-validator/src/storage/pack.rs @@ -0,0 +1,200 @@ +use serde::{Deserialize, Serialize}; +use sha2::Digest; + +use crate::blob_store::{ExternalRawStoreDb, ExternalRepoBytesDb, RawObjectStore}; + +#[derive(Clone, Debug)] +pub enum PackBytes { + Eager(std::sync::Arc<[u8]>), + LazyExternal { + sha256_hex: String, + store: std::sync::Arc, + cache: std::sync::Arc>>, + }, + LazyRepoBytes { + sha256_hex: String, + store: std::sync::Arc, + cache: std::sync::Arc>>, + }, +} + +impl PackBytes { + pub fn eager(bytes: Vec) -> Self { + Self::Eager(std::sync::Arc::from(bytes)) + } + + pub fn lazy_external(sha256_hex: String, store: std::sync::Arc) -> Self { + Self::LazyExternal { + sha256_hex, + store, + cache: std::sync::Arc::new(std::sync::OnceLock::new()), + } + } + + pub fn lazy_repo_bytes(sha256_hex: String, store: std::sync::Arc) -> Self { + Self::LazyRepoBytes { + sha256_hex, + store, + cache: std::sync::Arc::new(std::sync::OnceLock::new()), + } + } + + pub fn as_slice(&self) -> Result<&[u8], String> { + match self { + Self::Eager(bytes) => Ok(bytes.as_ref()), + Self::LazyExternal { + sha256_hex, + store, + cache, + } => { + if cache.get().is_none() { + let bytes = store + .get_blob_bytes(sha256_hex) + .map_err(|e| e.to_string())? + .ok_or_else(|| format!("missing raw blob for sha256={sha256_hex}"))?; + let _ = cache.set(std::sync::Arc::from(bytes)); + } + let bytes = cache + .get() + .ok_or_else(|| format!("missing raw blob cache for sha256={sha256_hex}"))?; + Ok(bytes.as_ref()) + } + Self::LazyRepoBytes { + sha256_hex, + store, + cache, + } => { + if cache.get().is_none() { + let bytes = store + .get_blob_bytes(sha256_hex) + .map_err(|e| e.to_string())? + .ok_or_else(|| format!("missing repo bytes for sha256={sha256_hex}"))?; + let _ = cache.set(std::sync::Arc::from(bytes)); + } + let bytes = cache + .get() + .ok_or_else(|| format!("missing repo bytes cache for sha256={sha256_hex}"))?; + Ok(bytes.as_ref()) + } + } + } + + pub fn to_vec(&self) -> Result, String> { + Ok(self.as_slice()?.to_vec()) + } +} + +impl PartialEq for PackBytes { + fn eq(&self, other: &Self) -> bool { + match (self.as_slice(), other.as_slice()) { + (Ok(a), Ok(b)) => a == b, + _ => false, + } + } +} + +impl Eq for PackBytes {} + +#[derive(Clone, Debug)] +pub struct PackFile { + pub rsync_uri: String, + pub bytes: PackBytes, + pub sha256: [u8; 32], +} + +impl PackFile { + pub fn new(rsync_uri: impl Into, bytes: PackBytes, sha256: [u8; 32]) -> Self { + Self { + rsync_uri: rsync_uri.into(), + bytes, + sha256, + } + } + + pub fn from_bytes_with_sha256( + rsync_uri: impl Into, + bytes: Vec, + sha256: [u8; 32], + ) -> Self { + Self::new(rsync_uri, PackBytes::eager(bytes), sha256) + } + + pub fn from_lazy_external_raw_store( + rsync_uri: impl Into, + sha256_hex: String, + sha256: [u8; 32], + store: std::sync::Arc, + ) -> Self { + Self::new( + rsync_uri, + PackBytes::lazy_external(sha256_hex, store), + sha256, + ) + } + + pub fn from_lazy_repo_bytes( + rsync_uri: impl Into, + sha256_hex: String, + sha256: [u8; 32], + store: std::sync::Arc, + ) -> Self { + Self::new( + rsync_uri, + PackBytes::lazy_repo_bytes(sha256_hex, store), + sha256, + ) + } + + pub fn from_bytes_compute_sha256(rsync_uri: impl Into, bytes: Vec) -> Self { + let sha256 = compute_sha256_32(&bytes); + Self::new(rsync_uri, PackBytes::eager(bytes), sha256) + } + + pub fn bytes(&self) -> Result<&[u8], String> { + self.bytes.as_slice() + } + + pub fn bytes_cloned(&self) -> Result, String> { + self.bytes.to_vec() + } + + pub fn compute_sha256(&self) -> Result<[u8; 32], String> { + Ok(compute_sha256_32(self.bytes()?)) + } +} + +impl PartialEq for PackFile { + fn eq(&self, other: &Self) -> bool { + self.rsync_uri == other.rsync_uri + && self.sha256 == other.sha256 + && self.bytes == other.bytes + } +} + +impl Eq for PackFile {} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct PackTime { + pub rfc3339_utc: String, +} + +impl PackTime { + pub fn from_utc_offset_datetime(t: time::OffsetDateTime) -> Self { + use time::format_description::well_known::Rfc3339; + let utc = t.to_offset(time::UtcOffset::UTC); + let s = utc.format(&Rfc3339).expect("format RFC 3339 UTC time"); + Self { rfc3339_utc: s } + } + + pub fn parse(&self) -> Result { + use time::format_description::well_known::Rfc3339; + time::OffsetDateTime::parse(&self.rfc3339_utc, &Rfc3339).map_err(|e| e.to_string()) + } +} + +pub(super) fn compute_sha256_32(bytes: &[u8]) -> [u8; 32] { + let digest = sha2::Sha256::digest(bytes); + let mut out = [0u8; 32]; + out.copy_from_slice(&digest); + out +} diff --git a/crates/panda-rpki-validator/src/storage/pp_cache_index.rs b/crates/panda-rpki-validator/src/storage/pp_cache_index.rs new file mode 100644 index 0000000..68065be --- /dev/null +++ b/crates/panda-rpki-validator/src/storage/pp_cache_index.rs @@ -0,0 +1,671 @@ +use std::collections::{BTreeMap, HashMap}; +use std::fs::{self, File}; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +use memmap2::Mmap; +use serde::Serialize; + +use super::{StorageError, StorageResult}; + +const MAGIC: &[u8; 12] = b"RPKIPPIDX\0\0\0"; +const VERSION: u32 = 1; +const HEADER_LEN: usize = 64; +const ENTRY_LEN: usize = 24; + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct PpCacheIndexLoadStats { + pub source: String, + pub entries: usize, + pub bytes: usize, + pub file_bytes: u64, + pub load_ms: u64, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct PpCacheIndexRefreshStats { + pub state: String, + pub old_entries: usize, + pub dirty_entries: usize, + pub new_entries: usize, + pub file_bytes: u64, + pub write_ms: u64, + pub compaction_triggered: bool, + pub compaction_reason: Option, + pub compaction_segments_before: usize, + pub compaction_total_file_bytes_before: u64, + pub compaction_live_entries: usize, + pub compaction_file_bytes: u64, + pub compaction_reclaimed_bytes: u64, + pub compaction_ms: u64, + pub compaction_deleted_segments: usize, + pub compaction_error: Option, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)] +pub struct PpCacheIndexDirectoryStats { + pub segment_count: usize, + pub total_file_bytes: u64, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct PpCacheIndexEntry { + value_offset: usize, + value_len: usize, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PpCacheIndexLookup<'a> { + Hit(&'a [u8]), + Deleted, +} + +#[derive(Debug)] +pub struct PpCacheMmapIndex { + mmap: Arc, + entries: HashMap, + bytes: usize, + file_bytes: u64, +} + +impl PpCacheMmapIndex { + pub fn open(path: &Path) -> StorageResult { + let file = File::open(path).map_err(|e| StorageError::RocksDb(e.to_string()))?; + let file_len = file + .metadata() + .map_err(|e| StorageError::RocksDb(e.to_string()))? + .len() as usize; + if file_len < HEADER_LEN { + return Err(StorageError::InvalidData { + entity: "publication_point_cache_mmap_index", + detail: "file too small".to_string(), + }); + } + // SAFETY: The mmap is read-only, held by Arc for all returned slices, and the file + // format is bounds-checked before any slice is exposed. + let mmap = unsafe { Mmap::map(&file) }.map_err(|e| StorageError::RocksDb(e.to_string()))?; + if &mmap[0..MAGIC.len()] != MAGIC { + return Err(StorageError::InvalidData { + entity: "publication_point_cache_mmap_index", + detail: "invalid magic".to_string(), + }); + } + let version = read_u32(&mmap, 12)?; + if version != VERSION { + return Err(StorageError::InvalidData { + entity: "publication_point_cache_mmap_index", + detail: format!("unsupported version {version}"), + }); + } + let entry_count = read_u64(&mmap, 16)? as usize; + let entry_table_offset = read_u64(&mmap, 24)? as usize; + let entry_table_len = read_u64(&mmap, 32)? as usize; + let blob_offset = read_u64(&mmap, 40)? as usize; + let blob_len = read_u64(&mmap, 48)? as usize; + checked_range(file_len, entry_table_offset, entry_table_len)?; + checked_range(file_len, blob_offset, blob_len)?; + if entry_table_len != entry_count.saturating_mul(ENTRY_LEN) { + return Err(StorageError::InvalidData { + entity: "publication_point_cache_mmap_index", + detail: "entry table length mismatch".to_string(), + }); + } + + let mut entries = HashMap::with_capacity(entry_count); + let mut bytes = 0usize; + for offset in (entry_table_offset..entry_table_offset + entry_table_len).step_by(ENTRY_LEN) + { + let key_offset = read_u64(&mmap, offset)? as usize; + let key_len = read_u32(&mmap, offset + 8)? as usize; + let value_offset = read_u64(&mmap, offset + 12)? as usize; + let value_len = read_u32(&mmap, offset + 20)? as usize; + checked_range(blob_len, key_offset, key_len)?; + checked_range(blob_len, value_offset, value_len)?; + let key_start = blob_offset + key_offset; + let key_end = key_start + key_len; + let key = std::str::from_utf8(&mmap[key_start..key_end]) + .map_err(|e| StorageError::InvalidData { + entity: "publication_point_cache_mmap_index.key", + detail: e.to_string(), + })? + .to_string(); + bytes = bytes.saturating_add(value_len); + entries.insert( + key, + PpCacheIndexEntry { + value_offset, + value_len, + }, + ); + } + Ok(Self { + mmap: Arc::new(mmap), + entries, + bytes, + file_bytes: file_len as u64, + }) + } + + pub fn lookup(&self, manifest_rsync_uri: &str) -> Option> { + let entry = self.entries.get(manifest_rsync_uri)?; + if entry.value_len == 0 { + return Some(PpCacheIndexLookup::Deleted); + } + let blob_offset = read_u64(self.mmap.as_ref(), 40).ok()? as usize; + let start = blob_offset + entry.value_offset; + let end = start + entry.value_len; + Some(PpCacheIndexLookup::Hit(&self.mmap[start..end])) + } + + #[cfg(test)] + pub fn get(&self, manifest_rsync_uri: &str) -> Option<&[u8]> { + match self.lookup(manifest_rsync_uri)? { + PpCacheIndexLookup::Hit(bytes) => Some(bytes), + PpCacheIndexLookup::Deleted => None, + } + } + + pub fn entries(&self) -> usize { + self.entries.len() + } + + pub fn bytes(&self) -> usize { + self.bytes + } + + pub fn file_bytes(&self) -> u64 { + self.file_bytes + } +} + +#[derive(Debug)] +pub struct PpCacheMmapIndexSet { + indexes: Vec, + entries: usize, + bytes: usize, + file_bytes: u64, +} + +impl PpCacheMmapIndexSet { + pub fn lookup(&self, manifest_rsync_uri: &str) -> Option> { + self.indexes + .iter() + .find_map(|index| index.lookup(manifest_rsync_uri)) + } + + #[cfg(test)] + pub fn get(&self, manifest_rsync_uri: &str) -> Option<&[u8]> { + match self.lookup(manifest_rsync_uri)? { + PpCacheIndexLookup::Hit(bytes) => Some(bytes), + PpCacheIndexLookup::Deleted => None, + } + } + + pub fn entries(&self) -> usize { + self.entries + } + + pub fn bytes(&self) -> usize { + self.bytes + } + + pub fn file_bytes(&self) -> u64 { + self.file_bytes + } +} + +pub fn default_pp_cache_index_dir(db_path: &Path) -> PathBuf { + let file_name = db_path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("work-db"); + db_path.with_file_name(format!("{file_name}.pp-cache-index")) +} + +pub fn load_pp_cache_mmap_index( + path: &Path, +) -> StorageResult<(PpCacheMmapIndex, PpCacheIndexLoadStats)> { + let started = Instant::now(); + let index = PpCacheMmapIndex::open(path)?; + let file_bytes = fs::metadata(path) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + .len(); + let stats = PpCacheIndexLoadStats { + source: "mmap".to_string(), + entries: index.entries(), + bytes: index.bytes(), + file_bytes, + load_ms: started.elapsed().as_millis() as u64, + }; + Ok((index, stats)) +} + +pub fn load_pp_cache_mmap_index_set( + dir: &Path, +) -> StorageResult<(PpCacheMmapIndexSet, PpCacheIndexLoadStats)> { + let started = Instant::now(); + let mut paths = Vec::new(); + if dir.exists() { + let mut segments = fs::read_dir(dir) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + .filter_map(Result::ok) + .map(|entry| entry.path()) + .filter(|path| { + path.file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| name.starts_with("segment-") && name.ends_with(".idx")) + }) + .collect::>(); + segments.sort(); + segments.reverse(); + paths.extend(segments); + let current = dir.join("current.idx"); + if current.exists() { + paths.push(current); + } + } + if paths.is_empty() { + return Err(StorageError::InvalidData { + entity: "publication_point_cache_mmap_index", + detail: "index file missing".to_string(), + }); + } + + let mut indexes = Vec::new(); + let mut entries = 0usize; + let mut bytes = 0usize; + let mut file_bytes = 0u64; + for path in paths { + let (index, _) = load_pp_cache_mmap_index(&path)?; + entries = entries.saturating_add(index.entries()); + bytes = bytes.saturating_add(index.bytes()); + file_bytes = file_bytes.saturating_add(index.file_bytes()); + indexes.push(index); + } + let set = PpCacheMmapIndexSet { + indexes, + entries, + bytes, + file_bytes, + }; + let stats = PpCacheIndexLoadStats { + source: "mmap".to_string(), + entries: set.entries(), + bytes: set.bytes(), + file_bytes: set.file_bytes(), + load_ms: started.elapsed().as_millis() as u64, + }; + Ok((set, stats)) +} + +pub fn pp_cache_index_directory_stats(dir: &Path) -> StorageResult { + let mut stats = PpCacheIndexDirectoryStats::default(); + if !dir.exists() { + return Ok(stats); + } + for entry in fs::read_dir(dir).map_err(|e| StorageError::RocksDb(e.to_string()))? { + let path = entry + .map_err(|e| StorageError::RocksDb(e.to_string()))? + .path(); + let Some(name) = path.file_name().and_then(|name| name.to_str()) else { + continue; + }; + let is_index = + name == "current.idx" || (name.starts_with("segment-") && name.ends_with(".idx")); + if !is_index { + continue; + } + let len = fs::metadata(&path) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + .len(); + stats.total_file_bytes = stats.total_file_bytes.saturating_add(len); + if name.starts_with("segment-") && name.ends_with(".idx") { + stats.segment_count += 1; + } + } + Ok(stats) +} + +pub fn compact_pp_cache_index(dir: &Path) -> StorageResult { + let started = Instant::now(); + fs::create_dir_all(dir).map_err(|e| StorageError::RocksDb(e.to_string()))?; + let before = pp_cache_index_directory_stats(dir)?; + let (index_set, _) = load_pp_cache_mmap_index_set(dir)?; + let mut compact_entries = Vec::with_capacity(index_set.entries()); + let mut seen = HashMap::::with_capacity(index_set.entries()); + for index in &index_set.indexes { + for (key, entry) in &index.entries { + if seen.insert(key.clone(), ()).is_some() { + continue; + } + if entry.value_len == 0 { + continue; + } + let blob_offset = read_u64(index.mmap.as_ref(), 40)? as usize; + let start = blob_offset + entry.value_offset; + let end = start + entry.value_len; + compact_entries.push((key.clone(), index.mmap[start..end].to_vec())); + } + } + + let current = dir.join("current.idx"); + let mut stats = write_pp_cache_index_atomic(¤t, compact_entries)?; + stats.state = "compacted".to_string(); + stats.compaction_triggered = true; + stats.compaction_segments_before = before.segment_count; + stats.compaction_total_file_bytes_before = before.total_file_bytes; + stats.compaction_live_entries = stats.new_entries; + stats.compaction_file_bytes = stats.file_bytes; + stats.compaction_reclaimed_bytes = before.total_file_bytes.saturating_sub(stats.file_bytes); + + let mut deleted_segments = 0usize; + for entry in fs::read_dir(dir).map_err(|e| StorageError::RocksDb(e.to_string()))? { + let path = entry + .map_err(|e| StorageError::RocksDb(e.to_string()))? + .path(); + let Some(name) = path.file_name().and_then(|name| name.to_str()) else { + continue; + }; + if name.starts_with("segment-") && name.ends_with(".idx") { + fs::remove_file(&path).map_err(|e| StorageError::RocksDb(e.to_string()))?; + deleted_segments += 1; + } + } + stats.compaction_deleted_segments = deleted_segments; + stats.compaction_ms = started.elapsed().as_millis() as u64; + Ok(stats) +} + +pub fn write_pp_cache_index_segment( + dir: &Path, + entries: I, +) -> StorageResult +where + I: IntoIterator)>, +{ + fs::create_dir_all(dir).map_err(|e| StorageError::RocksDb(e.to_string()))?; + let current = dir.join("current.idx"); + if !current.exists() { + return write_pp_cache_index_atomic(¤t, entries); + } + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + .as_nanos(); + let segment = dir.join(format!("segment-{now:020}-{}.idx", std::process::id())); + let mut stats = write_pp_cache_index_atomic(&segment, entries)?; + stats.state = "segment_written".to_string(); + Ok(stats) +} + +pub fn write_pp_cache_index_atomic( + path: &Path, + entries: I, +) -> StorageResult +where + I: IntoIterator)>, +{ + let started = Instant::now(); + let parent = path.parent().ok_or_else(|| StorageError::InvalidData { + entity: "publication_point_cache_mmap_index.path", + detail: "missing parent".to_string(), + })?; + fs::create_dir_all(parent).map_err(|e| StorageError::RocksDb(e.to_string()))?; + + let mut ordered = BTreeMap::>::new(); + for (key, value) in entries { + ordered.insert(key, value); + } + + let tmp_path = parent.join("next.tmp"); + write_pp_cache_index_file( + &tmp_path, + ordered.iter().map(|(k, v)| (k.as_str(), v.as_slice())), + )?; + let file_bytes = fs::metadata(&tmp_path) + .map_err(|e| StorageError::RocksDb(e.to_string()))? + .len(); + fs::rename(&tmp_path, path).map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(PpCacheIndexRefreshStats { + state: "written".to_string(), + old_entries: 0, + dirty_entries: 0, + new_entries: ordered.len(), + file_bytes, + write_ms: started.elapsed().as_millis() as u64, + ..PpCacheIndexRefreshStats::default() + }) +} + +fn write_pp_cache_index_file<'a, I>(path: &Path, entries: I) -> StorageResult<()> +where + I: IntoIterator, +{ + let entries = entries.into_iter().collect::>(); + let entry_table_offset = HEADER_LEN; + let entry_table_len = entries.len() * ENTRY_LEN; + let blob_offset = entry_table_offset + entry_table_len; + let mut table = Vec::with_capacity(entry_table_len); + let mut blob = Vec::new(); + for (key, value) in entries.iter() { + let key_offset = blob.len(); + blob.extend_from_slice(key.as_bytes()); + let value_offset = blob.len(); + blob.extend_from_slice(value); + write_u64(&mut table, key_offset as u64); + write_u32(&mut table, key.len() as u32); + write_u64(&mut table, value_offset as u64); + write_u32(&mut table, value.len() as u32); + } + + let mut header = Vec::with_capacity(HEADER_LEN); + header.extend_from_slice(MAGIC); + write_u32(&mut header, VERSION); + write_u64(&mut header, entries.len() as u64); + write_u64(&mut header, entry_table_offset as u64); + write_u64(&mut header, entry_table_len as u64); + write_u64(&mut header, blob_offset as u64); + write_u64(&mut header, blob.len() as u64); + header.resize(HEADER_LEN, 0); + + let mut file = File::create(path).map_err(|e| StorageError::RocksDb(e.to_string()))?; + file.write_all(&header) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + file.write_all(&table) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + file.write_all(&blob) + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + file.sync_all() + .map_err(|e| StorageError::RocksDb(e.to_string()))?; + Ok(()) +} + +fn checked_range(total: usize, offset: usize, len: usize) -> StorageResult<()> { + if offset.checked_add(len).is_none_or(|end| end > total) { + return Err(StorageError::InvalidData { + entity: "publication_point_cache_mmap_index.range", + detail: "out of bounds".to_string(), + }); + } + Ok(()) +} + +fn read_u32(bytes: &[u8], offset: usize) -> StorageResult { + checked_range(bytes.len(), offset, 4)?; + Ok(u32::from_le_bytes( + bytes[offset..offset + 4].try_into().unwrap(), + )) +} + +fn read_u64(bytes: &[u8], offset: usize) -> StorageResult { + checked_range(bytes.len(), offset, 8)?; + Ok(u64::from_le_bytes( + bytes[offset..offset + 8].try_into().unwrap(), + )) +} + +fn write_u32(out: &mut Vec, value: u32) { + out.extend_from_slice(&value.to_le_bytes()); +} + +fn write_u64(out: &mut Vec, value: u64) { + out.extend_from_slice(&value.to_le_bytes()); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pp_cache_index_roundtrips_multiple_entries() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("current.idx"); + let stats = write_pp_cache_index_atomic( + &path, + vec![ + ("rsync://example.test/a.mft".to_string(), b"aaa".to_vec()), + ("rsync://example.test/b.mft".to_string(), b"bbbb".to_vec()), + ], + ) + .expect("write index"); + assert_eq!(stats.new_entries, 2); + + let (index, load) = load_pp_cache_mmap_index(&path).expect("load index"); + assert_eq!(load.entries, 2); + assert_eq!(index.get("rsync://example.test/a.mft"), Some(&b"aaa"[..])); + assert_eq!(index.get("rsync://example.test/b.mft"), Some(&b"bbbb"[..])); + assert_eq!(index.get("rsync://example.test/missing.mft"), None); + } + + #[test] + fn pp_cache_index_rejects_bad_magic() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("current.idx"); + fs::write(&path, b"bad").expect("write bad"); + let err = load_pp_cache_mmap_index(&path).expect_err("bad index rejected"); + assert!(err.to_string().contains("file too small")); + } + + #[test] + fn pp_cache_index_last_duplicate_wins() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("current.idx"); + write_pp_cache_index_atomic( + &path, + vec![ + ("rsync://example.test/a.mft".to_string(), b"old".to_vec()), + ("rsync://example.test/a.mft".to_string(), b"new".to_vec()), + ], + ) + .expect("write index"); + + let (index, _) = load_pp_cache_mmap_index(&path).expect("load index"); + assert_eq!(index.entries(), 1); + assert_eq!(index.get("rsync://example.test/a.mft"), Some(&b"new"[..])); + } + + #[test] + fn pp_cache_index_set_prefers_newest_segment() { + let dir = tempfile::tempdir().expect("tempdir"); + let current = dir.path().join("current.idx"); + write_pp_cache_index_atomic( + ¤t, + vec![("rsync://example.test/a.mft".to_string(), b"old".to_vec())], + ) + .expect("write current index"); + write_pp_cache_index_segment( + dir.path(), + vec![("rsync://example.test/a.mft".to_string(), b"new".to_vec())], + ) + .expect("write segment index"); + + let (set, stats) = load_pp_cache_mmap_index_set(dir.path()).expect("load index set"); + assert_eq!(stats.entries, 2); + assert_eq!(set.get("rsync://example.test/a.mft"), Some(&b"new"[..])); + } + + #[test] + fn pp_cache_index_set_tombstone_shadows_older_entry() { + let dir = tempfile::tempdir().expect("tempdir"); + let current = dir.path().join("current.idx"); + write_pp_cache_index_atomic( + ¤t, + vec![("rsync://example.test/a.mft".to_string(), b"old".to_vec())], + ) + .expect("write current index"); + write_pp_cache_index_segment( + dir.path(), + vec![("rsync://example.test/a.mft".to_string(), Vec::new())], + ) + .expect("write tombstone segment index"); + + let (set, stats) = load_pp_cache_mmap_index_set(dir.path()).expect("load index set"); + assert_eq!(stats.entries, 2); + assert_eq!( + set.lookup("rsync://example.test/a.mft"), + Some(PpCacheIndexLookup::Deleted) + ); + assert_eq!(set.get("rsync://example.test/a.mft"), None); + } + + #[test] + fn pp_cache_index_compaction_keeps_latest_values_and_removes_segments() { + let dir = tempfile::tempdir().expect("tempdir"); + let current = dir.path().join("current.idx"); + write_pp_cache_index_atomic( + ¤t, + vec![ + ("rsync://example.test/a.mft".to_string(), b"old-a".to_vec()), + ("rsync://example.test/b.mft".to_string(), b"old-b".to_vec()), + ("rsync://example.test/c.mft".to_string(), b"old-c".to_vec()), + ], + ) + .expect("write current index"); + write_pp_cache_index_segment( + dir.path(), + vec![ + ("rsync://example.test/a.mft".to_string(), b"new-a".to_vec()), + ("rsync://example.test/b.mft".to_string(), Vec::new()), + ], + ) + .expect("write first segment"); + write_pp_cache_index_segment( + dir.path(), + vec![("rsync://example.test/d.mft".to_string(), b"new-d".to_vec())], + ) + .expect("write second segment"); + + let before = pp_cache_index_directory_stats(dir.path()).expect("stats before"); + assert_eq!(before.segment_count, 2); + + let stats = compact_pp_cache_index(dir.path()).expect("compact"); + assert!(stats.compaction_triggered); + assert_eq!(stats.compaction_segments_before, 2); + assert_eq!(stats.compaction_live_entries, 3); + assert_eq!(stats.compaction_deleted_segments, 2); + assert!(stats.compaction_reclaimed_bytes > 0); + + let after = pp_cache_index_directory_stats(dir.path()).expect("stats after"); + assert_eq!(after.segment_count, 0); + let segments = fs::read_dir(dir.path()) + .expect("read dir") + .filter_map(Result::ok) + .filter(|entry| { + entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with("segment-")) + }) + .count(); + assert_eq!(segments, 0); + + let (index, _) = load_pp_cache_mmap_index(¤t).expect("load compacted current"); + assert_eq!(index.entries(), 3); + assert_eq!(index.get("rsync://example.test/a.mft"), Some(&b"new-a"[..])); + assert_eq!(index.get("rsync://example.test/b.mft"), None); + assert_eq!(index.get("rsync://example.test/c.mft"), Some(&b"old-c"[..])); + assert_eq!(index.get("rsync://example.test/d.mft"), Some(&b"new-d"[..])); + } +} diff --git a/crates/panda-rpki-validator/src/storage/tests.rs b/crates/panda-rpki-validator/src/storage/tests.rs new file mode 100644 index 0000000..34d8f3b --- /dev/null +++ b/crates/panda-rpki-validator/src/storage/tests.rs @@ -0,0 +1,2460 @@ +use super::*; + +fn pack_time(hour: i64) -> PackTime { + PackTime::from_utc_offset_datetime( + time::OffsetDateTime::UNIX_EPOCH + time::Duration::hours(hour), + ) +} + +fn sha256_hex(input: &[u8]) -> String { + hex::encode(compute_sha256_32(input)) +} + +fn sha256_32(input: &[u8]) -> [u8; 32] { + compute_sha256_32(input) +} + +fn sample_child_certificate_cache_projection( + cache_key_sha256_hex: String, + child_cert_uri: &str, + child_cert_sha256_hex: &str, +) -> ChildCertificateCacheProjection { + ChildCertificateCacheProjection { + schema_version: CHILD_CERTIFICATE_CACHE_SCHEMA_VERSION, + algorithm_version: CHILD_CERTIFICATE_CACHE_ALGORITHM_VERSION, + cache_key_sha256_hex, + child_cert_uri: child_cert_uri.to_string(), + child_cert_sha256_hex: child_cert_sha256_hex.to_string(), + child_cert_serial: vec![1], + issuer_ca_sha256_hex: sha256_hex(b"issuer-ca"), + issuer_crl_uri: "rsync://example.test/repo/issuer.crl".to_string(), + issuer_crl_sha256_hex: sha256_hex(b"issuer-crl"), + ca_validation_context_digest: sha256_32(b"parent-context"), + validation_policy_fingerprint: sha256_32(b"policy"), + effective_not_before: pack_time(0), + effective_until: pack_time(24), + payload: ChildCertificateCachePayload::ChildCa { + child_manifest_rsync_uri: format!("{child_cert_uri}.mft"), + child_ski: "11".repeat(20), + child_rsync_base_uri: "rsync://example.test/repo/child/".to_string(), + child_publication_point_rsync_uri: "rsync://example.test/repo/child/".to_string(), + child_rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), + child_effective_ip_resources: None, + child_effective_as_resources: None, + }, + } +} + +#[test] +fn parse_work_db_blob_mode_accepts_supported_values() { + assert_eq!(default_work_db_blob_mode(), WorkDbBlobMode::Disabled); + assert_eq!( + parse_work_db_blob_mode("default"), + Some(WorkDbBlobMode::Disabled) + ); + assert_eq!( + parse_work_db_blob_mode("current"), + Some(WorkDbBlobMode::Current) + ); + assert_eq!( + parse_work_db_blob_mode("legacy"), + Some(WorkDbBlobMode::Current) + ); + assert_eq!( + parse_work_db_blob_mode("disabled"), + Some(WorkDbBlobMode::Disabled) + ); + assert_eq!( + parse_work_db_blob_mode("no-blob"), + Some(WorkDbBlobMode::Disabled) + ); + assert_eq!(parse_work_db_blob_mode("lz4"), Some(WorkDbBlobMode::Lz4)); + assert_eq!( + parse_work_db_blob_mode("blob-lz4"), + Some(WorkDbBlobMode::Lz4) + ); + assert_eq!(parse_work_db_blob_mode("unexpected"), None); +} + +#[test] +fn parse_work_db_memory_profile_accepts_supported_values() { + assert_eq!( + parse_work_db_memory_profile("default"), + Some(WorkDbMemoryProfile::Default) + ); + assert_eq!( + parse_work_db_memory_profile("none"), + Some(WorkDbMemoryProfile::Default) + ); + assert_eq!( + parse_work_db_memory_profile("compact"), + Some(WorkDbMemoryProfile::Compact) + ); + assert_eq!( + parse_work_db_memory_profile("low-memory"), + Some(WorkDbMemoryProfile::Compact) + ); + assert_eq!(parse_work_db_memory_profile("unexpected"), None); +} + +#[test] +fn vcir_field_size_breakdown_counts_local_outputs_and_artifacts() { + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let breakdown = VcirFieldSizeBreakdown::from_vcir(&vcir); + assert_eq!(breakdown.local_output_count, 2); + assert_eq!( + breakdown.local_output_payload_json_bytes, + vcir.local_outputs + .iter() + .map(|output| output.payload_json().len() as u64) + .sum::() + ); + assert_eq!( + breakdown.local_output_rule_hash_hex_bytes, + vcir.local_outputs.len() as u64 * 64 + ); + assert_eq!(breakdown.related_artifact_count, 2); + assert!(breakdown.related_artifact_uri_bytes > 0); + assert_eq!(breakdown.child_entry_count, 1); + assert!(breakdown.child_entry_uri_bytes > 0); + assert_eq!( + breakdown.local_output_old_projection_bytes(), + breakdown.local_output_source_type_bytes + + breakdown.local_output_source_hash_hex_bytes + + breakdown.local_output_source_ee_hash_hex_bytes + + breakdown.local_output_payload_json_bytes + + breakdown.local_output_rule_hash_hex_bytes + ); + assert!( + breakdown.local_output_old_projection_bytes() + > breakdown.local_output_typed_projection_bytes() + ); +} + +fn sample_repository_view_entry(rsync_uri: &str, bytes: &[u8]) -> RepositoryViewEntry { + RepositoryViewEntry { + rsync_uri: rsync_uri.to_string(), + current_hash: Some(sha256_hex(bytes)), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("cer".to_string()), + state: RepositoryViewState::Present, + } +} + +fn sample_raw_by_hash_entry(bytes: Vec) -> RawByHashEntry { + RawByHashEntry { + sha256_hex: sha256_hex(&bytes), + bytes, + origin_uris: vec!["rsync://example.test/repo/object.cer".to_string()], + object_type: Some("cer".to_string()), + encoding: Some("der".to_string()), + } +} + +fn sample_ccr_manifest_projection( + manifest_rsync_uri: &str, + manifest_this_update: PackTime, + subordinate_skis: Vec>, +) -> VcirCcrManifestProjection { + VcirCcrManifestProjection { + manifest_rsync_uri: manifest_rsync_uri.to_string(), + manifest_sha256: vec![0x11; 32], + manifest_size: 4096, + manifest_ee_aki: vec![0x22; 20], + manifest_number_be: vec![3], + manifest_this_update, + manifest_sia_locations_der: vec![vec![ + 0x30, 0x11, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x05, 0x86, 0x05, + b'r', b's', b'y', b'n', b'c', + ]], + subordinate_skis, + } +} + +fn sample_vcir(manifest_rsync_uri: &str) -> ValidatedCaInstanceResult { + let roa_bytes = b"roa-object".to_vec(); + let ee_bytes = b"ee-cert".to_vec(); + let child_bytes = b"child-cert".to_vec(); + let child_ski = "1234567890abcdef1234567890abcdef12345678".to_string(); + ValidatedCaInstanceResult { + manifest_rsync_uri: manifest_rsync_uri.to_string(), + parent_manifest_rsync_uri: Some("rsync://example.test/repo/parent/parent.mft".to_string()), + tal_id: "apnic".to_string(), + ca_subject_name: "CN=Example CA".to_string(), + ca_ski: "00112233445566778899aabbccddeeff00112233".to_string(), + issuer_ski: "ffeeddccbbaa99887766554433221100ffeeddcc".to_string(), + last_successful_validation_time: pack_time(0), + current_manifest_rsync_uri: manifest_rsync_uri.to_string(), + current_crl_rsync_uri: "rsync://example.test/repo/current.crl".to_string(), + validated_manifest_meta: ValidatedManifestMeta { + validated_manifest_number: vec![3], + validated_manifest_this_update: pack_time(0), + validated_manifest_next_update: pack_time(24), + }, + ccr_manifest_projection: sample_ccr_manifest_projection( + manifest_rsync_uri, + pack_time(0), + vec![hex::decode(&child_ski).expect("decode child ski")], + ), + instance_gate: VcirInstanceGate { + manifest_next_update: pack_time(24), + current_crl_next_update: pack_time(12), + self_ca_not_after: pack_time(48), + instance_effective_until: pack_time(12), + }, + child_entries: vec![VcirChildEntry { + child_manifest_rsync_uri: "rsync://example.test/repo/child/child.mft".to_string(), + child_cert_rsync_uri: "rsync://example.test/repo/child/child.cer".to_string(), + child_cert_hash: sha256_hex(&child_bytes), + child_ski, + child_rsync_base_uri: "rsync://example.test/repo/child/".to_string(), + child_publication_point_rsync_uri: "rsync://example.test/repo/child/".to_string(), + child_rrdp_notification_uri: Some("https://example.test/child-notify.xml".to_string()), + child_effective_ip_resources: None, + child_effective_as_resources: None, + accepted_at_validation_time: pack_time(0), + }], + local_outputs: vec![ + VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: pack_time(12), + source_object_uri: "rsync://example.test/repo/object.roa".to_string(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: sha256_32(&roa_bytes), + source_ee_cert_hash: sha256_32(&ee_bytes), + payload: VcirLocalOutputPayload::Vrp { + asn: 64496, + afi: crate::data_model::roa::RoaAfi::Ipv4, + prefix_len: 24, + addr: { + let mut addr = [0u8; 16]; + addr[..4].copy_from_slice(&[203, 0, 113, 0]); + addr + }, + max_length: 24, + }, + rule_hash: sha256_32(b"vrp-rule-1"), + }, + VcirLocalOutput { + output_type: VcirOutputType::Aspa, + item_effective_until: pack_time(10), + source_object_uri: "rsync://example.test/repo/object.asa".to_string(), + source_object_type: VcirSourceObjectType::Aspa, + source_object_hash: sha256_32(b"aspa-object"), + source_ee_cert_hash: sha256_32(b"aspa-ee-cert"), + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: 64496, + provider_as_ids: vec![64497], + }, + rule_hash: sha256_32(b"aspa-rule-1"), + }, + ], + related_artifacts: vec![ + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::Manifest, + artifact_kind: VcirArtifactKind::Mft, + uri: Some(manifest_rsync_uri.to_string()), + sha256: sha256_hex(b"manifest-object"), + object_type: Some("mft".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::CurrentCrl, + artifact_kind: VcirArtifactKind::Crl, + uri: Some("rsync://example.test/repo/current.crl".to_string()), + sha256: sha256_hex(b"current-crl"), + object_type: Some("crl".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }, + ], + summary: VcirSummary { + local_vrp_count: 1, + local_aspa_count: 1, + local_router_key_count: 0, + child_count: 1, + accepted_object_count: 4, + rejected_object_count: 0, + }, + audit_summary: VcirAuditSummary { + failed_fetch_eligible: true, + last_failed_fetch_reason: None, + warning_count: 0, + audit_flags: vec!["validated-fresh".to_string()], + }, + } +} + +fn roa_cache_projection_context(vcir: &ValidatedCaInstanceResult) -> RoaCacheProjectionContext { + let roa_output = vcir + .local_outputs + .iter() + .find(|output| { + output.output_type == VcirOutputType::Vrp + && output.source_object_type == VcirSourceObjectType::Roa + }) + .expect("sample VCIR has a ROA output"); + RoaCacheProjectionContext { + ca_validation_context_digest: [0x31; 32], + policy_fingerprint: [0x32; 32], + object_meta: vec![RoaCacheObjectMeta { + source_object_uri: roa_output.source_object_uri.clone(), + source_object_hash: roa_output.source_object_hash, + ee_serial: vec![0x01], + crl_uri: vcir.current_crl_rsync_uri.clone(), + earliest_safe_reuse_time: vcir.last_successful_validation_time.clone(), + }], + } +} + +#[test] +fn vcir_ccr_manifest_projection_validate_accepts_valid_projection() { + let projection = sample_ccr_manifest_projection( + "rsync://example.test/repo/current.mft", + pack_time(0), + vec![vec![0x33; 20], vec![0x44; 20]], + ); + projection.validate_internal().expect("valid projection"); +} + +#[test] +fn vcir_ccr_manifest_projection_validate_rejects_invalid_fields() { + let mut bad_hash = sample_ccr_manifest_projection( + "rsync://example.test/repo/current.mft", + pack_time(0), + vec![vec![0x33; 20]], + ); + bad_hash.manifest_sha256 = vec![0x11; 31]; + assert!(matches!( + bad_hash.validate_internal(), + Err(StorageError::InvalidData { .. }) + )); + + let mut bad_locations = sample_ccr_manifest_projection( + "rsync://example.test/repo/current.mft", + pack_time(0), + vec![vec![0x33; 20]], + ); + bad_locations.manifest_sia_locations_der = vec![vec![0x04, 0x00]]; + assert!(matches!( + bad_locations.validate_internal(), + Err(StorageError::InvalidData { .. }) + )); + + let bad_subordinates = sample_ccr_manifest_projection( + "rsync://example.test/repo/current.mft", + pack_time(0), + vec![vec![0x44; 20], vec![0x33; 20]], + ); + assert!(matches!( + bad_subordinates.validate_internal(), + Err(StorageError::InvalidData { .. }) + )); +} + +#[test] +fn roa_cache_projection_from_vcir_keeps_only_roa_vrp_outputs() { + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let context = roa_cache_projection_context(&vcir); + let projection = RoaCacheProjection::from_vcir_with_context(&vcir, Some(&context)) + .expect("projection build") + .expect("projection exists"); + + assert_eq!(projection.manifest_rsync_uri, vcir.manifest_rsync_uri); + assert_eq!(projection.instance_effective_until, pack_time(12)); + assert_eq!(projection.crl_sha256_by_uri.len(), 1); + assert_eq!(projection.entries.len(), 1); + assert_eq!( + projection.entries[0].source_object_uri, + "rsync://example.test/repo/object.roa" + ); + assert_eq!( + projection.entries[0].outputs_effective_until_unix, + 12 * 3600 + ); + assert_eq!(projection.entries[0].outputs.len(), 1); + assert!(matches!( + projection.entries[0].outputs[0].payload, + VcirLocalOutputPayload::Vrp { .. } + )); +} + +#[test] +fn roa_cache_projection_groups_multiple_outputs_by_roa_uri() { + let mut vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let mut second = vcir.local_outputs[0].clone(); + second.rule_hash = sha256_32(b"vrp-rule-2"); + if let VcirLocalOutputPayload::Vrp { max_length, .. } = &mut second.payload { + *max_length = 25; + } + vcir.local_outputs.push(second); + vcir.summary.local_vrp_count = 2; + + let context = roa_cache_projection_context(&vcir); + let projection = RoaCacheProjection::from_vcir_with_context(&vcir, Some(&context)) + .expect("projection build") + .expect("projection exists"); + + assert_eq!(projection.entries.len(), 1); + assert_eq!(projection.entries[0].outputs.len(), 2); + assert_eq!( + projection.entries[0].outputs_effective_until_unix, + 12 * 3600 + ); +} + +#[test] +fn publication_point_cache_projection_roundtrips_with_vcir() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let projection = PublicationPointCacheProjection::from_vcir_with_context( + &vcir, + "rsync://example.test/repo/".to_string(), + Some("rsync://example.test/repo/ca.cer".to_string()), + sha256_32(b"ca-cert"), + sha256_32(b"manifest"), + sha256_32(b"ta-context"), + sha256_32(b"parent-context"), + sha256_32(b"policy"), + ) + .expect("build publication point projection"); + + store + .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) + .expect("put vcir with publication point projection"); + + let got_vcir = store + .get_vcir(&vcir.manifest_rsync_uri) + .expect("get vcir") + .expect("vcir exists"); + assert_eq!(got_vcir, vcir); + let got_projection = store + .get_publication_point_cache_projection(&vcir.manifest_rsync_uri) + .expect("get publication point projection") + .expect("projection exists"); + assert_eq!(got_projection, projection); + assert_eq!(got_projection.outputs.len(), 2); + assert_eq!(got_projection.children.len(), 1); + assert_eq!(got_projection.related_objects.len(), 2); +} + +#[test] +fn publication_point_cache_projection_index_updates_after_first_read() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let mut projection = PublicationPointCacheProjection::from_vcir_with_context( + &vcir, + "rsync://example.test/repo/".to_string(), + Some("rsync://example.test/repo/ca.cer".to_string()), + sha256_32(b"ca-cert"), + sha256_32(b"manifest-old"), + sha256_32(b"ta-context"), + sha256_32(b"parent-context"), + sha256_32(b"policy"), + ) + .expect("build old publication point projection"); + + store + .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) + .expect("put old projection"); + let got_old = store + .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) + .expect("get old projection") + .expect("old projection exists"); + assert_eq!(got_old.manifest_sha256, sha256_32(b"manifest-old")); + + projection.manifest_sha256 = sha256_32(b"manifest-new"); + store + .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) + .expect("put new projection"); + let got_new = store + .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) + .expect("get new projection") + .expect("new projection exists"); + assert_eq!(got_new.manifest_sha256, sha256_32(b"manifest-new")); +} + +#[test] +fn publication_point_cache_projection_cached_empty_db_accepts_bounded_new_entries() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let projection = PublicationPointCacheProjection::from_vcir_with_context( + &vcir, + "rsync://example.test/repo/".to_string(), + Some("rsync://example.test/repo/ca.cer".to_string()), + sha256_32(b"ca-cert"), + sha256_32(b"manifest"), + sha256_32(b"ta-context"), + sha256_32(b"parent-context"), + sha256_32(b"policy"), + ) + .expect("build publication point projection"); + + assert!( + store + .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) + .expect("cached empty lookup") + .is_none() + ); + store + .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) + .expect("put projection after cached empty lookup"); + assert_eq!( + store + .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) + .expect("cached lookup sees bounded new entry") + .expect("cached projection exists"), + projection + ); + assert_eq!( + store + .get_publication_point_cache_projection(&vcir.manifest_rsync_uri) + .expect("direct db lookup") + .expect("projection exists"), + projection + ); +} + +#[test] +fn publication_point_cache_mmap_index_refresh_roundtrips_after_reopen() { + let td = tempfile::tempdir().expect("tempdir"); + let db_path = td.path().join("work-db"); + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let projection = PublicationPointCacheProjection::from_vcir_with_context( + &vcir, + "rsync://example.test/repo/".to_string(), + Some("rsync://example.test/repo/ca.cer".to_string()), + sha256_32(b"ca-cert"), + sha256_32(b"manifest"), + sha256_32(b"ta-context"), + sha256_32(b"parent-context"), + sha256_32(b"policy"), + ) + .expect("build publication point projection"); + + { + let store = RocksStore::open(&db_path).expect("open rocksdb"); + store + .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) + .expect("put projection"); + let stats = store + .refresh_publication_point_cache_mmap_index() + .expect("refresh mmap index") + .expect("refresh stats"); + assert_eq!(stats.new_entries, 1); + assert_eq!(stats.state, "written"); + } + + let store = RocksStore::open(&db_path).expect("reopen rocksdb"); + let got = store + .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) + .expect("get cached projection from mmap") + .expect("projection exists"); + assert_eq!(got, projection); +} + +#[test] +fn publication_point_cache_mmap_index_dirty_overlay_wins_and_refreshes_segment() { + let td = tempfile::tempdir().expect("tempdir"); + let db_path = td.path().join("work-db"); + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let mut projection = PublicationPointCacheProjection::from_vcir_with_context( + &vcir, + "rsync://example.test/repo/".to_string(), + Some("rsync://example.test/repo/ca.cer".to_string()), + sha256_32(b"ca-cert"), + sha256_32(b"manifest-old"), + sha256_32(b"ta-context"), + sha256_32(b"parent-context"), + sha256_32(b"policy"), + ) + .expect("build publication point projection"); + + { + let store = RocksStore::open(&db_path).expect("open rocksdb"); + store + .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) + .expect("put old projection"); + store + .refresh_publication_point_cache_mmap_index() + .expect("refresh base mmap index"); + } + + { + let store = RocksStore::open(&db_path).expect("reopen rocksdb"); + assert_eq!( + store + .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) + .expect("get old projection") + .expect("old projection exists") + .manifest_sha256, + sha256_32(b"manifest-old") + ); + projection.manifest_sha256 = sha256_32(b"manifest-new"); + store + .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) + .expect("put new projection"); + assert_eq!( + store + .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) + .expect("get dirty projection") + .expect("dirty projection exists") + .manifest_sha256, + sha256_32(b"manifest-new") + ); + let stats = store + .refresh_publication_point_cache_mmap_index() + .expect("refresh dirty mmap segment") + .expect("refresh stats"); + assert_eq!(stats.state, "segment_written"); + assert_eq!(stats.dirty_entries, 1); + } + + let store = RocksStore::open(&db_path).expect("reopen rocksdb after segment"); + let got = store + .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) + .expect("get refreshed projection") + .expect("projection exists"); + assert_eq!(got.manifest_sha256, sha256_32(b"manifest-new")); +} + +#[test] +fn publication_point_cache_mmap_index_write_before_read_refreshes_segment() { + let td = tempfile::tempdir().expect("tempdir"); + let db_path = td.path().join("work-db"); + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let mut projection = PublicationPointCacheProjection::from_vcir_with_context( + &vcir, + "rsync://example.test/repo/".to_string(), + Some("rsync://example.test/repo/ca.cer".to_string()), + sha256_32(b"ca-cert"), + sha256_32(b"manifest-old"), + sha256_32(b"ta-context"), + sha256_32(b"parent-context"), + sha256_32(b"policy"), + ) + .expect("build publication point projection"); + + { + let store = RocksStore::open(&db_path).expect("open rocksdb"); + store + .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) + .expect("put old projection"); + let stats = store + .refresh_publication_point_cache_mmap_index() + .expect("refresh base mmap index") + .expect("refresh stats"); + assert_eq!(stats.state, "written"); + } + + { + let store = RocksStore::open(&db_path).expect("reopen rocksdb"); + projection.manifest_sha256 = sha256_32(b"manifest-new"); + store + .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) + .expect("put new projection before any cached read"); + let stats = store + .refresh_publication_point_cache_mmap_index() + .expect("refresh dirty mmap segment") + .expect("refresh stats"); + assert_eq!(stats.state, "segment_written"); + assert_eq!(stats.dirty_entries, 1); + assert_eq!(stats.new_entries, 1); + } + + let store = RocksStore::open(&db_path).expect("reopen rocksdb after segment"); + let got = store + .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) + .expect("get refreshed projection") + .expect("projection exists"); + assert_eq!(got.manifest_sha256, sha256_32(b"manifest-new")); +} + +#[test] +fn publication_point_cache_mmap_index_delete_before_read_refreshes_tombstone_segment() { + let td = tempfile::tempdir().expect("tempdir"); + let db_path = td.path().join("work-db"); + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let projection = PublicationPointCacheProjection::from_vcir_with_context( + &vcir, + "rsync://example.test/repo/".to_string(), + Some("rsync://example.test/repo/ca.cer".to_string()), + sha256_32(b"ca-cert"), + sha256_32(b"manifest"), + sha256_32(b"ta-context"), + sha256_32(b"parent-context"), + sha256_32(b"policy"), + ) + .expect("build publication point projection"); + + { + let store = RocksStore::open(&db_path).expect("open rocksdb"); + store + .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) + .expect("put projection"); + let stats = store + .refresh_publication_point_cache_mmap_index() + .expect("refresh base mmap index") + .expect("refresh stats"); + assert_eq!(stats.state, "written"); + } + + { + let store = RocksStore::open(&db_path).expect("reopen rocksdb"); + store + .replace_vcir_manifest_replay_meta_and_projection_action( + &vcir, + None, + PublicationPointCacheProjectionWriteAction::Delete { + manifest_rsync_uri: &vcir.manifest_rsync_uri, + }, + ) + .expect("delete projection before any cached read"); + assert!( + store + .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) + .expect("dirty tombstone lookup") + .is_none() + ); + let stats = store + .refresh_publication_point_cache_mmap_index() + .expect("refresh tombstone mmap segment") + .expect("refresh stats"); + assert_eq!(stats.state, "segment_written"); + assert_eq!(stats.dirty_entries, 1); + assert_eq!(stats.new_entries, 1); + } + + let store = RocksStore::open(&db_path).expect("reopen rocksdb after tombstone segment"); + assert!( + store + .get_publication_point_cache_projection_cached(&vcir.manifest_rsync_uri) + .expect("tombstone shadows old current index") + .is_none() + ); + assert!( + store + .get_publication_point_cache_projection(&vcir.manifest_rsync_uri) + .expect("direct db lookup") + .is_none() + ); +} + +#[test] +fn publication_point_cache_projection_rejects_version_mismatch() { + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let mut projection = PublicationPointCacheProjection::from_vcir_with_context( + &vcir, + "rsync://example.test/repo/".to_string(), + Some("rsync://example.test/repo/ca.cer".to_string()), + sha256_32(b"ca-cert"), + sha256_32(b"manifest"), + sha256_32(b"ta-context"), + sha256_32(b"parent-context"), + sha256_32(b"policy"), + ) + .expect("build publication point projection"); + projection.schema_version = PUBLICATION_POINT_CACHE_SCHEMA_VERSION + 1; + assert!(matches!( + projection.validate_internal(), + Err(StorageError::InvalidData { .. }) + )); +} + +#[test] +fn roa_cache_projection_rejects_duplicate_uri_with_different_hash() { + let mut vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let mut duplicate = vcir.local_outputs[0].clone(); + duplicate.source_object_hash = sha256_32(b"different-roa"); + duplicate.rule_hash = sha256_32(b"vrp-rule-2"); + vcir.local_outputs.push(duplicate); + vcir.summary.local_vrp_count = 2; + + let context = roa_cache_projection_context(&vcir); + let err = RoaCacheProjection::from_vcir_with_context(&vcir, Some(&context)) + .expect_err("same ROA URI with different hash must fail"); + assert!(err.to_string().contains("source object hash mismatch")); +} + +fn sample_rrdp_source_record(notify_uri: &str) -> RrdpSourceRecord { + RrdpSourceRecord { + notify_uri: notify_uri.to_string(), + last_session_id: Some("session-1".to_string()), + last_serial: Some(42), + first_seen_at: pack_time(0), + last_seen_at: pack_time(1), + last_sync_at: Some(pack_time(1)), + sync_state: RrdpSourceSyncState::DeltaReady, + last_snapshot_uri: Some("https://rrdp.example.test/snapshot.xml".to_string()), + last_snapshot_hash: Some(sha256_hex(b"snapshot-bytes")), + last_error: None, + } +} + +fn sample_rrdp_source_member_record( + notify_uri: &str, + rsync_uri: &str, + serial: u64, +) -> RrdpSourceMemberRecord { + RrdpSourceMemberRecord { + notify_uri: notify_uri.to_string(), + rsync_uri: rsync_uri.to_string(), + current_hash: Some(sha256_hex(rsync_uri.as_bytes())), + object_type: Some("cer".to_string()), + present: true, + last_confirmed_session_id: "session-1".to_string(), + last_confirmed_serial: serial, + last_changed_at: pack_time(serial as i64), + } +} + +fn sample_rrdp_uri_owner_record(notify_uri: &str, rsync_uri: &str) -> RrdpUriOwnerRecord { + RrdpUriOwnerRecord { + rsync_uri: rsync_uri.to_string(), + notify_uri: notify_uri.to_string(), + current_hash: Some(sha256_hex(rsync_uri.as_bytes())), + last_confirmed_session_id: "session-1".to_string(), + last_confirmed_serial: 7, + last_changed_at: pack_time(7), + owner_state: RrdpUriOwnerState::Active, + } +} + +#[test] +fn repository_view_and_raw_by_hash_roundtrip() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let entry1 = sample_repository_view_entry("rsync://example.test/repo/a.cer", b"object-a"); + let entry2 = sample_repository_view_entry("rsync://example.test/repo/sub/b.roa", b"object-b"); + store + .put_repository_view_entry(&entry1) + .expect("put repository view entry1"); + store + .put_repository_view_entry(&entry2) + .expect("put repository view entry2"); + + let got1 = store + .get_repository_view_entry(&entry1.rsync_uri) + .expect("get repository view entry1") + .expect("entry1 exists"); + assert_eq!(got1, entry1); + + let got_prefix = store + .list_repository_view_entries_with_prefix("rsync://example.test/repo/sub/") + .expect("list repository view prefix"); + assert_eq!(got_prefix, vec![entry2.clone()]); + + store + .delete_repository_view_entry(&entry1.rsync_uri) + .expect("delete repository view entry1"); + assert!( + store + .get_repository_view_entry(&entry1.rsync_uri) + .expect("get deleted repository view entry1") + .is_none() + ); + + let raw = sample_raw_by_hash_entry(b"raw-der-object".to_vec()); + store + .put_raw_by_hash_entry(&raw) + .expect("put raw_by_hash entry"); + let got_raw = store + .get_raw_by_hash_entry(&raw.sha256_hex) + .expect("get raw_by_hash entry") + .expect("raw entry exists"); + assert_eq!(got_raw, raw); +} + +#[test] +fn raw_by_hash_routes_to_external_raw_store_when_configured() { + let td = tempfile::tempdir().expect("tempdir"); + let main_db = td.path().join("main-db"); + let raw_db = td.path().join("raw-store.db"); + + let raw = sample_raw_by_hash_entry(b"external-raw".to_vec()); + { + let store = + RocksStore::open_with_external_raw_store(&main_db, &raw_db).expect("open store"); + store.put_raw_by_hash_entry(&raw).expect("put external raw"); + + let got = store + .get_raw_by_hash_entry(&raw.sha256_hex) + .expect("get external raw") + .expect("raw exists"); + assert_eq!(got, raw); + } + + let main_store = RocksStore::open(&main_db).expect("open main only"); + assert!( + main_store + .get_raw_by_hash_entry(&raw.sha256_hex) + .expect("read main store") + .is_none(), + "main db should not contain raw entry when external raw store is configured" + ); +} + +#[test] +fn put_blob_bytes_batch_uses_internal_blob_cf_without_raw_entry() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let bytes = b"internal-blob-only".to_vec(); + let hash = sha256_hex(&bytes); + + store + .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) + .expect("put blob bytes"); + + assert_eq!( + store.get_blob_bytes(&hash).expect("get blob bytes"), + Some(bytes.clone()) + ); + assert!( + store + .get_raw_by_hash_entry(&hash) + .expect("get raw entry") + .is_none() + ); +} + +#[test] +fn put_blob_bytes_batch_routes_to_external_raw_store_without_raw_entry() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open_with_external_raw_store( + &td.path().join("main-db"), + &td.path().join("raw-store.db"), + ) + .expect("open store"); + let bytes = b"external-blob-only".to_vec(); + let hash = sha256_hex(&bytes); + + store + .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) + .expect("put external blob bytes"); + + assert_eq!(store.get_blob_bytes(&hash).unwrap(), Some(bytes)); + assert!(store.get_raw_by_hash_entry(&hash).unwrap().is_none()); +} + +#[test] +fn repo_bytes_db_is_physically_separate_from_external_raw_store() { + let td = tempfile::tempdir().expect("tempdir"); + let main_db = td.path().join("main-db"); + let raw_db = td.path().join("raw-store.db"); + let repo_bytes_db = td.path().join("repo-bytes.db"); + let store = + RocksStore::open_with_external_stores(&main_db, Some(&raw_db), Some(&repo_bytes_db)) + .expect("open store"); + let repo_bytes = b"repo-object".to_vec(); + let repo_hash = sha256_hex(&repo_bytes); + let raw = sample_raw_by_hash_entry(b"raw-evidence".to_vec()); + + store + .put_blob_bytes_batch(&[(repo_hash.clone(), repo_bytes.clone())]) + .expect("put repo bytes"); + store.put_raw_by_hash_entry(&raw).expect("put raw evidence"); + + assert_eq!(store.get_blob_bytes(&repo_hash).unwrap(), Some(repo_bytes)); + assert_eq!( + store.get_raw_by_hash_entry(&raw.sha256_hex).unwrap(), + Some(raw.clone()) + ); + drop(store); + + let raw_only = RocksStore::open_with_external_raw_store(&td.path().join("raw-reader"), &raw_db) + .expect("open raw only"); + assert!( + raw_only.get_blob_bytes(&repo_hash).unwrap().is_none(), + "repo object bytes must not be written into raw-store.db" + ); + + let repo_only = + RocksStore::open_with_external_repo_bytes(&td.path().join("repo-reader"), &repo_bytes_db) + .expect("open repo bytes only"); + assert_eq!( + repo_only.get_blob_bytes(&repo_hash).unwrap(), + Some(b"repo-object".to_vec()) + ); + assert!( + repo_only.get_blob_bytes(&raw.sha256_hex).unwrap().is_none(), + "raw evidence bytes must not be written into repo-bytes.db" + ); +} + +#[test] +fn memory_snapshot_includes_work_db_and_external_stores() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open_with_external_stores( + &td.path().join("main-db"), + Some(&td.path().join("raw-store.db")), + Some(&td.path().join("repo-bytes.db")), + ) + .expect("open store"); + + let snapshot = store.memory_snapshot(); + let labels: Vec<&str> = snapshot + .databases + .iter() + .map(|db| db.label.as_str()) + .collect(); + assert_eq!(labels, vec!["work-db", "raw-store.db", "repo-bytes.db"]); + assert!( + snapshot.databases[0] + .column_families + .iter() + .any(|cf| cf.name == CF_REPOSITORY_VIEW) + ); + serde_json::to_value(&snapshot).expect("serialize memory snapshot"); +} + +#[test] +fn put_blob_bytes_batch_accepts_empty_batch_with_external_raw_store() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open_with_external_raw_store( + &td.path().join("main-db"), + &td.path().join("raw-store.db"), + ) + .expect("open store"); + + store + .put_blob_bytes_batch(&[]) + .expect("empty external blob batch should be a no-op"); +} + +#[test] +fn get_blob_bytes_internal_falls_back_to_raw_entry_when_blob_missing() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let raw = sample_raw_by_hash_entry(b"raw-fallback".to_vec()); + + store.put_raw_by_hash_entry(&raw).expect("put raw entry"); + + assert_eq!( + store + .get_blob_bytes(&raw.sha256_hex) + .expect("get blob bytes via raw fallback"), + Some(raw.bytes.clone()) + ); +} + +#[test] +fn get_blob_bytes_batch_internal_prefers_blob_cf_and_falls_back_to_raw_entry() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let blob_bytes = b"blob-cf-object".to_vec(); + let blob_hash = sha256_hex(&blob_bytes); + store + .put_blob_bytes_batch(&[(blob_hash.clone(), blob_bytes.clone())]) + .expect("put blob bytes"); + + let raw = sample_raw_by_hash_entry(b"raw-fallback-batch".to_vec()); + store.put_raw_by_hash_entry(&raw).expect("put raw fallback"); + + let batch = store + .get_blob_bytes_batch(&[blob_hash.clone(), raw.sha256_hex.clone(), "00".repeat(32)]) + .expect("get blob bytes batch"); + assert_eq!(batch, vec![Some(blob_bytes), Some(raw.bytes.clone()), None]); +} + +#[test] +fn get_blob_bytes_batch_routes_to_external_raw_store_without_raw_entry() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open_with_external_raw_store( + &td.path().join("main-db"), + &td.path().join("raw-store.db"), + ) + .expect("open store"); + let bytes = b"external-batch-blob".to_vec(); + let hash = sha256_hex(&bytes); + + store + .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) + .expect("put external blob bytes"); + + assert_eq!( + store + .get_blob_bytes_batch(&[hash, "00".repeat(32)]) + .expect("get external blob batch"), + vec![Some(bytes), None] + ); +} + +#[test] +fn get_blob_bytes_rejects_invalid_hash_for_internal_store() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let err = store + .get_blob_bytes("not-a-valid-hash") + .expect_err("invalid hash must fail"); + assert!(matches!(err, StorageError::InvalidData { .. })); +} + +#[test] +fn get_blob_bytes_batch_rejects_invalid_hash_for_internal_store() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let err = store + .get_blob_bytes_batch(&["not-a-valid-hash".to_string()]) + .expect_err("invalid hash must fail"); + assert!(matches!(err, StorageError::InvalidData { .. })); +} + +#[test] +fn get_blob_bytes_batch_returns_empty_for_empty_request_internal() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + assert!( + store + .get_blob_bytes_batch(&[]) + .expect("empty blob batch request") + .is_empty() + ); +} + +#[test] +fn put_blob_bytes_batch_accepts_empty_batch() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + store + .put_blob_bytes_batch(&[]) + .expect("empty blob batch should be a no-op"); +} + +#[test] +fn put_blob_bytes_batch_rejects_empty_bytes() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let err = store + .put_blob_bytes_batch(&[(sha256_hex(b"valid"), Vec::new())]) + .expect_err("empty bytes must fail"); + assert!(matches!(err, StorageError::InvalidData { .. })); +} + +#[test] +fn delete_raw_by_hash_entry_internal_preserves_blob_bytes() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let bytes = b"blob-persists-after-raw-delete".to_vec(); + let hash = sha256_hex(&bytes); + let raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone()); + + store + .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) + .expect("put blob bytes"); + store.put_raw_by_hash_entry(&raw).expect("put raw entry"); + + store + .delete_raw_by_hash_entry(&hash) + .expect("delete raw entry only"); + + assert!(store.get_raw_by_hash_entry(&hash).unwrap().is_none()); + assert_eq!(store.get_blob_bytes(&hash).unwrap(), Some(bytes)); +} + +#[test] +fn delete_raw_by_hash_entry_rejects_invalid_hash() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let err = store + .delete_raw_by_hash_entry("not-a-valid-hash") + .expect_err("invalid hash must fail"); + assert!(matches!(err, StorageError::InvalidData { .. })); +} + +#[test] +fn delete_raw_by_hash_entry_routes_to_external_raw_store() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open_with_external_raw_store( + &td.path().join("main-db"), + &td.path().join("raw-store.db"), + ) + .expect("open store"); + let raw = sample_raw_by_hash_entry(b"external-delete".to_vec()); + + store.put_raw_by_hash_entry(&raw).expect("put raw entry"); + store + .delete_raw_by_hash_entry(&raw.sha256_hex) + .expect("delete external raw entry"); + + assert!( + store + .get_raw_by_hash_entry(&raw.sha256_hex) + .unwrap() + .is_none() + ); + assert!(store.get_blob_bytes(&raw.sha256_hex).unwrap().is_none()); +} + +#[test] +fn repository_view_and_raw_by_hash_validation_errors_are_reported() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let invalid_view = RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/withdrawn.cer".to_string(), + current_hash: None, + repository_source: None, + object_type: None, + state: RepositoryViewState::Present, + }; + let err = store + .put_repository_view_entry(&invalid_view) + .expect_err("missing current_hash must fail"); + assert!(err.to_string().contains("current_hash is required")); + + let invalid_raw = RawByHashEntry { + sha256_hex: sha256_hex(b"expected"), + bytes: b"actual".to_vec(), + origin_uris: vec!["rsync://example.test/repo/object.cer".to_string()], + object_type: None, + encoding: None, + }; + let err = store + .put_raw_by_hash_entry(&invalid_raw) + .expect_err("mismatched raw_by_hash entry must fail"); + assert!(err.to_string().contains("does not match bytes")); +} + +#[test] +fn vcir_roundtrip_and_validation_failures_are_reported() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + let roa_context = roa_cache_projection_context(&vcir); + store + .put_vcir_with_projections(&vcir, Some(&roa_context), None) + .expect("put vcir"); + let got = store + .get_vcir(&vcir.manifest_rsync_uri) + .expect("get vcir") + .expect("vcir exists"); + assert_eq!(got, vcir); + let replay_meta = store + .get_manifest_replay_meta(&vcir.manifest_rsync_uri) + .expect("get manifest replay meta") + .expect("manifest replay meta exists"); + assert_eq!( + replay_meta, + ManifestReplayMeta { + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + manifest_number_be: vcir + .validated_manifest_meta + .validated_manifest_number + .clone(), + manifest_this_update: vcir + .validated_manifest_meta + .validated_manifest_this_update + .clone(), + manifest_sha256: vcir.ccr_manifest_projection.manifest_sha256.clone(), + updated_at_validation_time: vcir.last_successful_validation_time.clone(), + } + ); + let projection = store + .get_roa_cache_projection(&vcir.manifest_rsync_uri) + .expect("get roa cache projection") + .expect("roa cache projection exists"); + assert_eq!(projection.manifest_rsync_uri, vcir.manifest_rsync_uri); + assert_eq!(projection.entries.len(), 1); + assert_eq!( + projection.entries[0].source_object_uri, + "rsync://example.test/repo/object.roa" + ); + + let mut invalid = sample_vcir("rsync://example.test/repo/invalid.mft"); + invalid.summary.local_vrp_count = 9; + let err = store + .put_vcir(&invalid) + .expect_err("invalid vcir must fail"); + assert!(err.to_string().contains("local_vrp_count=9")); + + let mut invalid = sample_vcir("rsync://example.test/repo/invalid-2.mft"); + invalid.instance_gate.instance_effective_until = pack_time(11); + let err = store + .put_vcir(&invalid) + .expect_err("invalid instance gate must fail"); + assert!(err.to_string().contains("instance_effective_until")); + + store + .delete_vcir(&vcir.manifest_rsync_uri) + .expect("delete vcir"); + assert!( + store + .get_vcir(&vcir.manifest_rsync_uri) + .expect("get deleted vcir") + .is_none() + ); + assert!( + store + .get_manifest_replay_meta(&vcir.manifest_rsync_uri) + .expect("get deleted manifest replay meta") + .is_none() + ); + assert!( + store + .get_roa_cache_projection(&vcir.manifest_rsync_uri) + .expect("get deleted roa cache projection") + .is_none() + ); +} + +#[test] +fn clear_vcir_reuse_records_preserves_manifest_replay_meta() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let vcir = sample_vcir("rsync://example.test/repo/current.mft"); + store.put_vcir(&vcir).expect("put vcir"); + let replay_meta = store + .get_manifest_replay_meta(&vcir.manifest_rsync_uri) + .expect("get replay meta") + .expect("replay meta exists"); + + let cleared = store + .clear_vcir_reuse_records() + .expect("clear reusable VCIR records"); + assert_eq!(cleared.vcir_records, 1); + assert_eq!(cleared.failed_fetch_identity_records, 0); + assert!( + store + .get_vcir(&vcir.manifest_rsync_uri) + .expect("get cleared vcir") + .is_none() + ); + assert_eq!( + store + .get_manifest_replay_meta(&vcir.manifest_rsync_uri) + .expect("get preserved replay meta"), + Some(replay_meta) + ); +} + +#[test] +fn transport_prefetch_snapshot_roundtrips() { + use crate::parallel::transport_prefetch::{ + TransportPrefetchDedupKey, TransportPrefetchMode, TransportPrefetchRepoIdentity, + TransportPrefetchRequest, TransportPrefetchRequester, TransportPrefetchSnapshot, + }; + use crate::policy::SyncPreference; + + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + assert!( + store + .get_transport_prefetch_snapshot() + .expect("get empty prefetch snapshot") + .is_none() + ); + + let snapshot = TransportPrefetchSnapshot::new( + SyncPreference::RrdpThenRsync, + vec![TransportPrefetchRequest { + dedup_key: TransportPrefetchDedupKey::RrdpNotify { + notification_uri: "https://example.test/notification.xml".to_string(), + }, + rsync_scope_uri: "rsync://example.test/repo/".to_string(), + rsync_failure_scope_uri: Some("rsync://example.test/".to_string()), + repo_identity: TransportPrefetchRepoIdentity { + notification_uri: Some("https://example.test/notification.xml".to_string()), + rsync_base_uri: "rsync://example.test/repo/".to_string(), + }, + mode: TransportPrefetchMode::Rrdp, + last_result: None, + last_rsync_result: None, + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + priority: 0, + requesters: vec![TransportPrefetchRequester { + tal_id: "apnic".to_string(), + rir_id: "apnic".to_string(), + parent_node_id: None, + ca_instance_handle_id: "apnic:rsync://example.test/repo/root.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/root.mft".to_string(), + }], + }], + ); + store + .put_transport_prefetch_snapshot(&snapshot) + .expect("put prefetch snapshot"); + let got = store + .get_transport_prefetch_snapshot() + .expect("get prefetch snapshot") + .expect("snapshot exists"); + assert_eq!(got, snapshot); +} + +#[test] +fn manifest_replay_meta_validation_reports_invalid_fields() { + let mut meta = ManifestReplayMeta { + manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(), + manifest_number_be: vec![3], + manifest_this_update: pack_time(0), + manifest_sha256: vec![0x11; 32], + updated_at_validation_time: pack_time(1), + }; + meta.validate_internal().expect("valid replay meta"); + + meta.manifest_sha256 = vec![0x11; 31]; + let err = meta + .validate_internal() + .expect_err("short manifest sha must fail"); + assert!(err.to_string().contains("must be 32 bytes")); + + meta.manifest_sha256 = vec![0x11; 32]; + meta.manifest_number_be = vec![0, 3]; + let err = meta + .validate_internal() + .expect_err("non-minimal manifest number must fail"); + assert!(err.to_string().contains("minimal big-endian")); +} + +#[test] +fn list_vcirs_returns_all_entries() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let vcir1 = sample_vcir("rsync://example.test/repo/a.mft"); + let vcir2 = sample_vcir("rsync://example.test/repo/b.mft"); + store.put_vcir(&vcir1).expect("put vcir1"); + store.put_vcir(&vcir2).expect("put vcir2"); + + let mut got = store.list_vcirs().expect("list vcirs"); + got.sort_by(|a, b| a.manifest_rsync_uri.cmp(&b.manifest_rsync_uri)); + assert_eq!(got, vec![vcir1, vcir2]); +} + +#[test] +fn summarize_vcir_storage_aggregates_values_and_field_sizes() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let vcir1 = sample_vcir("rsync://example.test/repo/a.mft"); + let vcir2 = sample_vcir("rsync://example.test/repo/b.mft"); + store.put_vcir(&vcir1).expect("put vcir1"); + store.put_vcir(&vcir2).expect("put vcir2"); + + let summary = store.summarize_vcir_storage().expect("summarize vcirs"); + let mut expected_fields = VcirFieldSizeBreakdown::default(); + expected_fields.add_assign(&VcirFieldSizeBreakdown::from_vcir(&vcir1)); + expected_fields.add_assign(&VcirFieldSizeBreakdown::from_vcir(&vcir2)); + + assert_eq!(summary.entry_count, 2); + assert!(summary.vcir_value_bytes > 0); + assert!(summary.vcir_value_bytes_max > 0); + assert!(summary.vcir_value_bytes_max_manifest_rsync_uri.is_some()); + assert_eq!(summary.top_entries_by_vcir_value_bytes.len(), 2); + assert!( + summary.top_entries_by_vcir_value_bytes[0].vcir_value_bytes + >= summary.top_entries_by_vcir_value_bytes[1].vcir_value_bytes + ); + assert_eq!(summary.field_sizes, expected_fields); + assert!(summary.core_fields.manifest_rsync_uri_bytes > 0); + assert!(summary.ccr_projection.manifest_sha256_bytes > 0); + assert!(summary.child_resources.effective_ip_resource_cbor_bytes > 0); + assert_eq!( + summary.local_output_old_projection_bytes, + expected_fields.local_output_old_projection_bytes() + ); + assert_eq!( + summary.local_output_typed_projection_bytes, + expected_fields.local_output_typed_projection_bytes() + ); + assert_eq!( + summary.local_output_projection_saved_bytes, + expected_fields.local_output_projection_saved_bytes() + ); +} + +#[test] +fn replace_vcir_and_manifest_replay_meta_replaces_current_entry() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let mut previous = sample_vcir("rsync://example.test/repo/current.mft"); + previous.local_outputs = vec![VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: pack_time(10), + source_object_uri: "rsync://example.test/repo/old.roa".to_string(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: sha256_32(b"old-roa"), + source_ee_cert_hash: sha256_32(b"old-ee"), + payload: VcirLocalOutputPayload::Vrp { + asn: 64496, + afi: crate::data_model::roa::RoaAfi::Ipv4, + prefix_len: 24, + addr: { + let mut addr = [0u8; 16]; + addr[..4].copy_from_slice(&[203, 0, 113, 0]); + addr + }, + max_length: 24, + }, + rule_hash: sha256_32(b"old-rule"), + }]; + previous.summary.local_vrp_count = 1; + previous.summary.local_aspa_count = 0; + previous.summary.local_router_key_count = 0; + let previous_roa_context = roa_cache_projection_context(&previous); + let previous_timing = store + .replace_vcir_manifest_replay_meta_and_projections( + &previous, + Some(&previous_roa_context), + None, + ) + .expect("store previous vcir"); + assert!(previous_timing.vcir_value_bytes > 0); + assert!(previous_timing.replay_meta_value_bytes > 0); + assert!(previous_timing.roa_cache_projection_value_bytes > 0); + assert_eq!( + previous_timing.total_encoded_bytes, + previous_timing.vcir_value_bytes + + previous_timing.replay_meta_value_bytes + + previous_timing.roa_cache_projection_value_bytes + ); + assert!( + store + .get_roa_cache_projection(&previous.manifest_rsync_uri) + .expect("get previous projection") + .is_some() + ); + + let mut current = sample_vcir("rsync://example.test/repo/current.mft"); + current.local_outputs = vec![VcirLocalOutput { + output_type: VcirOutputType::Aspa, + item_effective_until: pack_time(11), + source_object_uri: "rsync://example.test/repo/new.asa".to_string(), + source_object_type: VcirSourceObjectType::Aspa, + source_object_hash: sha256_32(b"new-aspa"), + source_ee_cert_hash: sha256_32(b"new-ee"), + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: 64496, + provider_as_ids: vec![64497], + }, + rule_hash: sha256_32(b"new-rule"), + }]; + current.summary.local_vrp_count = 0; + current.summary.local_aspa_count = 1; + let current_timing = store + .replace_vcir_and_manifest_replay_meta(¤t) + .expect("replace vcir and replay meta"); + assert!(current_timing.vcir_value_bytes > 0); + assert!(current_timing.replay_meta_value_bytes > 0); + assert_eq!(current_timing.roa_cache_projection_value_bytes, 0); + assert_eq!( + current_timing.total_encoded_bytes, + current_timing.vcir_value_bytes + current_timing.replay_meta_value_bytes + ); + + let got = store + .get_vcir(¤t.manifest_rsync_uri) + .expect("get replaced vcir") + .expect("vcir exists"); + assert_eq!(got, current); + let replay_meta = store + .get_manifest_replay_meta(¤t.manifest_rsync_uri) + .expect("get replaced replay meta") + .expect("replay meta exists"); + assert_eq!( + replay_meta.manifest_number_be, + current.validated_manifest_meta.validated_manifest_number + ); + assert_eq!( + replay_meta.manifest_sha256, + current.ccr_manifest_projection.manifest_sha256 + ); + assert!( + store + .get_roa_cache_projection(¤t.manifest_rsync_uri) + .expect("get current projection") + .is_none() + ); +} + +#[test] +fn get_child_certificate_cache_projections_batch_preserves_order_and_misses() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let first_key = sha256_hex(b"child-cache-key-first"); + let missing_key = sha256_hex(b"child-cache-key-missing"); + let second_key = sha256_hex(b"child-cache-key-second"); + let first_hash = sha256_hex(b"first-child-cert"); + let second_hash = sha256_hex(b"second-child-cert"); + let first = sample_child_certificate_cache_projection( + first_key.clone(), + "rsync://example.test/repo/first.cer", + &first_hash, + ); + let second = sample_child_certificate_cache_projection( + second_key.clone(), + "rsync://example.test/repo/second.cer", + &second_hash, + ); + + store + .put_child_certificate_cache_projection(&first) + .expect("put first projection"); + store + .put_child_certificate_cache_projection(&second) + .expect("put second projection"); + + let got = store + .get_child_certificate_cache_projections_batch(&[ + second_key.clone(), + missing_key, + first_key.clone(), + ]) + .expect("batch get child projections"); + + assert_eq!(got.len(), 3); + assert_eq!( + got[0] + .as_ref() + .expect("second projection") + .child_cert_sha256_hex, + second_hash + ); + assert!(got[1].is_none()); + assert_eq!( + got[2] + .as_ref() + .expect("first projection") + .child_cert_sha256_hex, + first_hash + ); +} + +#[test] +fn child_certificate_cache_mmap_segment_preserves_order_and_misses() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let manifest_uri = "rsync://example.test/repo/parent.mft"; + let first_key = sha256_hex(b"child-cache-mmap-key-first"); + let missing_key = sha256_hex(b"child-cache-mmap-key-missing"); + let second_key = sha256_hex(b"child-cache-mmap-key-second"); + let first_hash = sha256_hex(b"first-child-cert-mmap"); + let second_hash = sha256_hex(b"second-child-cert-mmap"); + let first = sample_child_certificate_cache_projection( + first_key.clone(), + "rsync://example.test/repo/first.cer", + &first_hash, + ); + let second = sample_child_certificate_cache_projection( + second_key.clone(), + "rsync://example.test/repo/second.cer", + &second_hash, + ); + + assert!( + store + .get_child_certificate_cache_projections_mmap_segment( + manifest_uri, + &[first_key.clone()] + ) + .expect("missing segment lookup") + .is_none() + ); + + let write_stats = store + .write_child_certificate_cache_mmap_segment(manifest_uri, &[first.clone(), second.clone()]) + .expect("write child projection segment"); + assert_eq!(write_stats.new_entries, 2); + assert!(write_stats.file_bytes > 0); + + let got = store + .get_child_certificate_cache_projections_mmap_segment( + manifest_uri, + &[second_key.clone(), missing_key, first_key.clone()], + ) + .expect("lookup child projection segment") + .expect("segment exists"); + + assert_eq!(got.hits, 2); + assert_eq!(got.misses, 1); + assert!(got.file_bytes > 0); + assert_eq!(got.projections.len(), 3); + assert_eq!( + got.projections[0] + .as_ref() + .expect("second projection") + .child_cert_sha256_hex, + second_hash + ); + assert!(got.projections[1].is_none()); + assert_eq!( + got.projections[2] + .as_ref() + .expect("first projection") + .child_cert_sha256_hex, + first_hash + ); +} + +#[test] +fn child_certificate_cache_mmap_segment_overlay_preserves_existing_values() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let manifest_uri = "rsync://example.test/repo/parent-overlay.mft"; + let first_key = sha256_hex(b"child-cache-overlay-key-first"); + let second_key = sha256_hex(b"child-cache-overlay-key-second"); + let missing_key = sha256_hex(b"child-cache-overlay-key-missing"); + let first_hash = sha256_hex(b"first-child-cert-overlay"); + let old_second_hash = sha256_hex(b"old-second-child-cert-overlay"); + let new_second_hash = sha256_hex(b"new-second-child-cert-overlay"); + let first = sample_child_certificate_cache_projection( + first_key.clone(), + "rsync://example.test/repo/first-overlay.cer", + &first_hash, + ); + let old_second = sample_child_certificate_cache_projection( + second_key.clone(), + "rsync://example.test/repo/second-overlay.cer", + &old_second_hash, + ); + let new_second = sample_child_certificate_cache_projection( + second_key.clone(), + "rsync://example.test/repo/second-overlay.cer", + &new_second_hash, + ); + + store + .write_child_certificate_cache_mmap_segment( + manifest_uri, + &[first.clone(), old_second.clone()], + ) + .expect("write initial segment"); + let stats = store + .write_child_certificate_cache_mmap_segment_overlay( + manifest_uri, + &[first_key.clone(), second_key.clone(), missing_key.clone()], + std::slice::from_ref(&new_second), + ) + .expect("write segment overlay"); + assert_eq!(stats.new_entries, 2); + + let got = store + .get_child_certificate_cache_projections_mmap_segment( + manifest_uri, + &[first_key.clone(), second_key.clone(), missing_key], + ) + .expect("lookup segment") + .expect("segment exists"); + assert_eq!(got.hits, 2); + assert_eq!(got.misses, 1); + assert_eq!( + got.projections[0] + .as_ref() + .expect("first projection") + .child_cert_sha256_hex, + first_hash + ); + assert_eq!( + got.projections[1] + .as_ref() + .expect("updated second projection") + .child_cert_sha256_hex, + new_second_hash + ); + assert!(got.projections[2].is_none()); +} + +#[test] +fn storage_helpers_cover_optional_validation_paths() { + let withdrawn = RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/withdrawn.cer".to_string(), + current_hash: Some(sha256_hex(b"withdrawn")), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("cer".to_string()), + state: RepositoryViewState::Withdrawn, + }; + withdrawn + .validate_internal() + .expect("withdrawn repository view validates"); + + let raw = RawByHashEntry::from_bytes(sha256_hex(b"helper-bytes"), b"helper-bytes".to_vec()); + raw.validate_internal() + .expect("raw_by_hash helper validates"); + + let empty_raw = RawByHashEntry { + sha256_hex: sha256_hex(b"x"), + bytes: Vec::new(), + origin_uris: Vec::new(), + object_type: None, + encoding: None, + }; + let err = empty_raw + .validate_internal() + .expect_err("empty raw bytes must fail"); + assert!(err.to_string().contains("bytes must not be empty")); + + let duplicate_origin_raw = RawByHashEntry { + sha256_hex: sha256_hex(b"dup-origin"), + bytes: b"dup-origin".to_vec(), + origin_uris: vec![ + "rsync://example.test/repo/object.cer".to_string(), + "rsync://example.test/repo/object.cer".to_string(), + ], + object_type: Some("cer".to_string()), + encoding: Some("der".to_string()), + }; + let err = duplicate_origin_raw + .validate_internal() + .expect_err("duplicate origin URI must fail"); + assert!(err.to_string().contains("duplicate origin URI")); +} + +#[test] +fn rrdp_source_optional_fields_and_owner_without_hash_validate() { + let source = RrdpSourceRecord { + notify_uri: "https://rrdp.example.test/notification.xml".to_string(), + last_session_id: None, + last_serial: None, + first_seen_at: pack_time(0), + last_seen_at: pack_time(1), + last_sync_at: None, + sync_state: RrdpSourceSyncState::Empty, + last_snapshot_uri: None, + last_snapshot_hash: None, + last_error: Some("network timeout".to_string()), + }; + source + .validate_internal() + .expect("source with optional fields validates"); + + let owner = RrdpUriOwnerRecord { + rsync_uri: "rsync://example.test/repo/object.cer".to_string(), + notify_uri: "https://rrdp.example.test/notification.xml".to_string(), + current_hash: None, + last_confirmed_session_id: "session-1".to_string(), + last_confirmed_serial: 5, + last_changed_at: pack_time(5), + owner_state: RrdpUriOwnerState::Withdrawn, + }; + owner + .validate_internal() + .expect("owner without hash validates when withdrawn"); +} + +#[test] +fn rrdp_source_binding_records_roundtrip_and_prefix_iteration() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let notify_uri = "https://rrdp.example.test/notification.xml"; + let source = sample_rrdp_source_record(notify_uri); + store + .put_rrdp_source_record(&source) + .expect("put rrdp source record"); + let got_source = store + .get_rrdp_source_record(notify_uri) + .expect("get rrdp source record") + .expect("rrdp source exists"); + assert_eq!(got_source, source); + + let member1 = + sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/a.cer", 1); + let member2 = + sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/b.roa", 2); + let other_member = sample_rrdp_source_member_record( + "https://other.example.test/notification.xml", + "rsync://other.example.test/repo/c.cer", + 3, + ); + store + .put_rrdp_source_member_record(&member1) + .expect("put member1"); + store + .put_rrdp_source_member_record(&member2) + .expect("put member2"); + store + .put_rrdp_source_member_record(&other_member) + .expect("put other member"); + + let mut members = store + .list_rrdp_source_member_records(notify_uri) + .expect("list rrdp source members"); + members.sort_by(|a, b| a.rsync_uri.cmp(&b.rsync_uri)); + assert_eq!(members, vec![member1.clone(), member2.clone()]); + + let got_member = store + .get_rrdp_source_member_record(notify_uri, &member1.rsync_uri) + .expect("get member1") + .expect("member1 exists"); + assert_eq!(got_member, member1); + + let owner = sample_rrdp_uri_owner_record(notify_uri, &member1.rsync_uri); + store + .put_rrdp_uri_owner_record(&owner) + .expect("put uri owner record"); + let got_owner = store + .get_rrdp_uri_owner_record(&member1.rsync_uri) + .expect("get uri owner record") + .expect("uri owner exists"); + assert_eq!(got_owner, owner); + store + .delete_rrdp_uri_owner_record(&member1.rsync_uri) + .expect("delete uri owner record"); + assert!( + store + .get_rrdp_uri_owner_record(&member1.rsync_uri) + .expect("get deleted uri owner") + .is_none() + ); + + let mut invalid_source = sample_rrdp_source_record("https://invalid.example/notification.xml"); + invalid_source.last_snapshot_hash = Some("bad".to_string()); + let err = store + .put_rrdp_source_record(&invalid_source) + .expect_err("invalid source hash must fail"); + assert!(err.to_string().contains("last_snapshot_hash")); + + let invalid_member = RrdpSourceMemberRecord { + notify_uri: notify_uri.to_string(), + rsync_uri: "rsync://example.test/repo/deleted.cer".to_string(), + current_hash: None, + object_type: None, + present: true, + last_confirmed_session_id: "session-1".to_string(), + last_confirmed_serial: 10, + last_changed_at: pack_time(10), + }; + let err = store + .put_rrdp_source_member_record(&invalid_member) + .expect_err("present member without hash must fail"); + assert!(err.to_string().contains("current_hash is required")); +} +#[test] +fn projection_batch_roundtrip_writes_repository_view_member_and_owner_records() { + let dir = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(dir.path()).expect("open store"); + + let view = RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/a.roa".to_string(), + current_hash: Some(hex::encode([1u8; 32])), + repository_source: Some("https://example.test/notify.xml".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }; + let member = RrdpSourceMemberRecord { + notify_uri: "https://example.test/notify.xml".to_string(), + rsync_uri: "rsync://example.test/repo/a.roa".to_string(), + current_hash: Some(hex::encode([1u8; 32])), + object_type: Some("roa".to_string()), + present: true, + last_confirmed_session_id: "session-1".to_string(), + last_confirmed_serial: 7, + last_changed_at: pack_time(1), + }; + let owner = RrdpUriOwnerRecord { + rsync_uri: "rsync://example.test/repo/a.roa".to_string(), + notify_uri: "https://example.test/notify.xml".to_string(), + current_hash: Some(hex::encode([1u8; 32])), + last_confirmed_session_id: "session-1".to_string(), + last_confirmed_serial: 7, + last_changed_at: pack_time(1), + owner_state: RrdpUriOwnerState::Active, + }; + + store + .put_projection_batch(&[view.clone()], &[member.clone()], &[owner.clone()]) + .expect("write projection batch"); + + assert_eq!( + store + .get_repository_view_entry(&view.rsync_uri) + .expect("get view") + .expect("present view"), + view + ); + assert_eq!( + store + .get_rrdp_source_member_record(&member.notify_uri, &member.rsync_uri) + .expect("get member") + .expect("present member"), + member + ); + assert_eq!( + store + .get_rrdp_uri_owner_record(&owner.rsync_uri) + .expect("get owner") + .expect("present owner"), + owner + ); +} + +#[test] +fn current_rrdp_source_member_helpers_filter_present_records() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let notify_uri = "https://rrdp.example.test/notification.xml"; + let mut present_a = + sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/a.cer", 1); + let mut withdrawn_b = + sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/b.roa", 2); + withdrawn_b.present = false; + let present_c = + sample_rrdp_source_member_record(notify_uri, "rsync://example.test/repo/c.crl", 3); + let other_source = sample_rrdp_source_member_record( + "https://other.example.test/notification.xml", + "rsync://other.example.test/repo/x.cer", + 4, + ); + present_a.last_confirmed_serial = 10; + + store + .put_rrdp_source_member_record(&present_a) + .expect("put present a"); + store + .put_rrdp_source_member_record(&withdrawn_b) + .expect("put withdrawn b"); + store + .put_rrdp_source_member_record(&present_c) + .expect("put present c"); + store + .put_rrdp_source_member_record(&other_source) + .expect("put other source"); + + let members = store + .list_current_rrdp_source_members(notify_uri) + .expect("list current members"); + assert_eq!( + members + .iter() + .map(|record| record.rsync_uri.as_str()) + .collect::>(), + vec![ + "rsync://example.test/repo/a.cer", + "rsync://example.test/repo/c.crl", + ] + ); + + assert!( + store + .is_current_rrdp_source_member(notify_uri, &present_a.rsync_uri) + .expect("current a") + ); + assert!( + !store + .is_current_rrdp_source_member(notify_uri, &withdrawn_b.rsync_uri) + .expect("withdrawn b") + ); + assert!( + !store + .is_current_rrdp_source_member(notify_uri, &other_source.rsync_uri) + .expect("other source") + ); +} + +#[test] +fn load_current_object_bytes_by_uri_uses_repository_view_and_raw_by_hash() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + + let present_bytes = b"present-object".to_vec(); + let present_hash = sha256_hex(&present_bytes); + let mut present_raw = RawByHashEntry::from_bytes(present_hash.clone(), present_bytes.clone()); + present_raw + .origin_uris + .push("rsync://example.test/repo/present.roa".to_string()); + present_raw.object_type = Some("roa".to_string()); + store + .put_raw_by_hash_entry(&present_raw) + .expect("put present raw"); + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/present.roa".to_string(), + current_hash: Some(present_hash), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }) + .expect("put present view"); + + let replaced_bytes = b"replaced-object".to_vec(); + let replaced_hash = sha256_hex(&replaced_bytes); + let mut replaced_raw = + RawByHashEntry::from_bytes(replaced_hash.clone(), replaced_bytes.clone()); + replaced_raw + .origin_uris + .push("rsync://example.test/repo/replaced.cer".to_string()); + replaced_raw.object_type = Some("cer".to_string()); + store + .put_raw_by_hash_entry(&replaced_raw) + .expect("put replaced raw"); + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/replaced.cer".to_string(), + current_hash: Some(replaced_hash), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("cer".to_string()), + state: RepositoryViewState::Replaced, + }) + .expect("put replaced view"); + + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/withdrawn.crl".to_string(), + current_hash: Some(sha256_hex(b"withdrawn")), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("crl".to_string()), + state: RepositoryViewState::Withdrawn, + }) + .expect("put withdrawn view"); + + assert_eq!( + store + .load_current_object_bytes_by_uri("rsync://example.test/repo/present.roa") + .expect("load present"), + Some(present_bytes) + ); + assert_eq!( + store + .load_current_object_bytes_by_uri("rsync://example.test/repo/replaced.cer") + .expect("load replaced"), + Some(replaced_bytes) + ); + assert_eq!( + store + .load_current_object_bytes_by_uri("rsync://example.test/repo/withdrawn.crl") + .expect("load withdrawn"), + None + ); + assert_eq!( + store + .load_current_object_bytes_by_uri("rsync://example.test/repo/missing.roa") + .expect("load missing"), + None + ); +} + +#[test] +fn load_current_object_bytes_by_uri_errors_when_raw_by_hash_is_missing() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let rsync_uri = "rsync://example.test/repo/missing.cer"; + + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: rsync_uri.to_string(), + current_hash: Some(hex::encode([0x11; 32])), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("cer".to_string()), + state: RepositoryViewState::Present, + }) + .expect("put view"); + let err = store + .load_current_object_bytes_by_uri(rsync_uri) + .expect_err("missing raw_by_hash should error"); + assert!(matches!(err, StorageError::InvalidData { .. })); +} + +#[test] +fn load_current_object_with_hash_by_uri_returns_hash_and_bytes() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let rsync_uri = "rsync://example.test/repo/present.roa"; + let bytes = b"present-object".to_vec(); + let hash = sha256_hex(&bytes); + + let mut raw = RawByHashEntry::from_bytes(hash.clone(), bytes.clone()); + raw.origin_uris.push(rsync_uri.to_string()); + raw.object_type = Some("roa".to_string()); + store.put_raw_by_hash_entry(&raw).expect("put raw"); + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: rsync_uri.to_string(), + current_hash: Some(hash.clone()), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }) + .expect("put view"); + + let got = store + .load_current_object_with_hash_by_uri(rsync_uri) + .expect("load current object") + .expect("current object exists"); + assert_eq!(got.current_hash_hex, hash); + assert_eq!(got.current_hash, compute_sha256_32(&bytes)); + assert_eq!(got.bytes, bytes); +} + +#[test] +fn load_current_object_with_hash_by_uri_uses_internal_blob_cf_without_raw_entry() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let rsync_uri = "rsync://example.test/repo/blob-only.roa"; + let bytes = b"blob-only-current-object".to_vec(); + let hash = sha256_hex(&bytes); + + store + .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) + .expect("put blob bytes"); + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: rsync_uri.to_string(), + current_hash: Some(hash.clone()), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }) + .expect("put view"); + + let got = store + .load_current_object_with_hash_by_uri(rsync_uri) + .expect("load current object") + .expect("current object exists"); + assert_eq!(got.current_hash_hex, hash); + assert_eq!(got.current_hash, compute_sha256_32(&bytes)); + assert_eq!(got.bytes, bytes); + assert!( + store + .get_raw_by_hash_entry(&got.current_hash_hex) + .expect("get raw entry") + .is_none() + ); +} + +#[test] +fn load_current_object_bytes_by_uri_uses_internal_blob_cf_without_raw_entry() { + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let rsync_uri = "rsync://example.test/repo/blob-only-bytes.roa"; + let bytes = b"blob-only-current-object-bytes".to_vec(); + let hash = sha256_hex(&bytes); + + store + .put_blob_bytes_batch(&[(hash, bytes.clone())]) + .expect("put blob bytes"); + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: rsync_uri.to_string(), + current_hash: Some(sha256_hex(&bytes)), + repository_source: Some("https://rrdp.example.test/notification.xml".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }) + .expect("put view"); + + assert_eq!( + store + .load_current_object_bytes_by_uri(rsync_uri) + .expect("load current object bytes"), + Some(bytes) + ); +} + +#[test] +fn pack_file_can_lazy_load_bytes_from_external_raw_store() { + let td = tempfile::tempdir().expect("tempdir"); + let raw_store = std::sync::Arc::new( + ExternalRawStoreDb::open(td.path().join("raw-store.db")).expect("open raw store"), + ); + let bytes = b"lazy-pack-file".to_vec(); + let sha256_hex = sha256_hex(&bytes); + raw_store + .put_raw_entry(&RawByHashEntry::from_bytes( + sha256_hex.clone(), + bytes.clone(), + )) + .expect("put raw entry"); + + let file = PackFile::from_lazy_external_raw_store( + "rsync://example.test/repo/a.roa", + sha256_hex, + compute_sha256_32(&bytes), + raw_store, + ); + + assert_eq!(file.bytes().expect("lazy bytes"), bytes.as_slice()); + assert_eq!(file.bytes_cloned().expect("cloned bytes"), bytes); +} + +#[test] +fn pack_file_can_lazy_load_bytes_from_external_repo_bytes_store() { + let td = tempfile::tempdir().expect("tempdir"); + let repo_bytes_store = std::sync::Arc::new( + ExternalRepoBytesDb::open(td.path().join("repo-bytes.db")).expect("open repo bytes"), + ); + let bytes = b"repo-object-pack-file".to_vec(); + let sha256_hex = sha256_hex(&bytes); + repo_bytes_store + .put_blob_bytes_batch(&[(sha256_hex.clone(), bytes.clone())]) + .expect("put repo bytes"); + + let file = PackFile::from_lazy_repo_bytes( + "rsync://example.test/repo/a.roa", + sha256_hex, + compute_sha256_32(&bytes), + repo_bytes_store, + ); + + assert_eq!(file.bytes().expect("lazy repo bytes"), bytes.as_slice()); + assert_eq!(file.bytes_cloned().expect("cloned repo bytes"), bytes); + assert_eq!(file.compute_sha256().expect("compute sha256"), file.sha256); +} + +#[test] +fn read_only_checkpoint_isolated_from_source_work_db() { + let td = tempfile::tempdir().expect("tempdir"); + let source_path = td.path().join("source-work-db"); + let checkpoint_path = td.path().join("checkpoint-work-db"); + let uri = "rsync://example.test/repo/a.roa"; + let bytes = b"checkpoint-object"; + let hash = sha256_hex(bytes); + { + let source = RocksStore::open(&source_path).expect("open source"); + source + .put_blob_bytes_batch(&[(hash.clone(), bytes.to_vec())]) + .expect("put blob"); + source + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: uri.to_string(), + current_hash: Some(hash), + repository_source: Some("fixture".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }) + .expect("put view"); + } + + RocksStore::create_read_only_checkpoint(&source_path, &checkpoint_path) + .expect("create checkpoint"); + let checkpoint = RocksStore::open(&checkpoint_path).expect("open checkpoint"); + checkpoint + .delete_repository_view_entry(uri) + .expect("delete checkpoint entry"); + drop(checkpoint); + + let source = RocksStore::open(&source_path).expect("reopen source"); + assert!( + source + .get_repository_view_entry(uri) + .expect("get source entry") + .is_some() + ); +} + +#[test] +fn read_only_external_repo_bytes_rejects_writes() { + let td = tempfile::tempdir().expect("tempdir"); + let path = td.path().join("repo-bytes.db"); + let hash = sha256_hex(b"repo-object"); + { + let writable = ExternalRepoBytesDb::open(&path).expect("open writable repo bytes"); + writable + .put_blob_bytes_batch(&[(hash.clone(), b"repo-object".to_vec())]) + .expect("seed repo bytes"); + } + let read_only = ExternalRepoBytesDb::open_read_only(&path).expect("open read-only repo bytes"); + assert_eq!( + read_only + .get_blob_bytes(&hash) + .expect("read repo bytes") + .expect("blob"), + b"repo-object" + ); + assert!( + read_only + .put_blob_bytes_batch(&[(hash, b"repo-object".to_vec())]) + .is_err() + ); +} + +#[test] +fn read_only_store_accepts_only_idempotent_existing_blob_apply() { + let td = tempfile::tempdir().expect("tempdir"); + let work_db_path = td.path().join("work-db"); + let repo_bytes_path = td.path().join("repo-bytes.db"); + let bytes = b"repo-object".to_vec(); + let hash = sha256_hex(&bytes); + { + let repo_bytes = ExternalRepoBytesDb::open(&repo_bytes_path).expect("open repo bytes"); + repo_bytes + .put_blob_bytes_batch(&[(hash.clone(), bytes.clone())]) + .expect("seed repo bytes"); + } + let store = + RocksStore::open_with_external_repo_bytes_read_only(&work_db_path, &repo_bytes_path) + .expect("open read-only store"); + store + .put_blob_bytes_batch(&[(hash.clone(), bytes)]) + .expect("idempotent apply"); + assert!( + store + .put_blob_bytes_batch(&[(hash, b"different".to_vec())]) + .is_err() + ); +} + +#[test] +fn repository_blob_verification_detects_missing_external_blob() { + let td = tempfile::tempdir().expect("tempdir"); + let repo_bytes_path = td.path().join("repo-bytes.db"); + ExternalRepoBytesDb::open(&repo_bytes_path).expect("create repo bytes"); + let store = RocksStore::open_with_external_repo_bytes_read_only( + &td.path().join("work-db"), + &repo_bytes_path, + ) + .expect("open work db"); + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/missing.roa".to_string(), + current_hash: Some(sha256_hex(b"missing")), + repository_source: Some("fixture".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }) + .expect("put view"); + let error = store + .verify_current_repository_blobs(16) + .expect_err("missing blob must fail"); + assert!(error.to_string().contains("missing.roa"), "{error}"); +} + +#[test] +fn vcir_related_artifact_reject_reason_serde_backward_compatible() { + // Legacy cache JSON written before the reject_reason ("e") field existed. + let legacy = r#"{"r":"manifest","k":"mft","u":"rsync://example.test/a.mft","h":"0000000000000000000000000000000000000000000000000000000000000000","t":"mft","s":"accepted"}"#; + let artifact: VcirRelatedArtifact = serde_json::from_str(legacy).expect("legacy artifact"); + assert_eq!(artifact.reject_reason, None); + + // None reason is skipped on serialization, keeping cache entries compact. + let json_none = serde_json::to_string(&artifact).expect("serialize none"); + assert!(!json_none.contains("\"e\"")); + + // A recorded reason round-trips through the "e" field. + let mut with_reason = artifact.clone(); + with_reason.validation_status = VcirArtifactValidationStatus::Rejected; + with_reason.reject_reason = Some("bad object".to_string()); + let json = serde_json::to_string(&with_reason).expect("serialize"); + assert!(json.contains("\"e\":\"bad object\"")); + let decoded: VcirRelatedArtifact = serde_json::from_str(&json).expect("deserialize"); + assert_eq!(decoded.reject_reason.as_deref(), Some("bad object")); + + // PublicationPointCacheObject conversion carries the reason both ways. + let cache_object = PublicationPointCacheObject::from_related_artifact(&with_reason); + assert_eq!(cache_object.reject_reason.as_deref(), Some("bad object")); + let back = cache_object.to_related_artifact(); + assert_eq!(back.reject_reason.as_deref(), Some("bad object")); + + // Legacy PublicationPointCacheObject JSON without "e" still deserializes. + let legacy_object: PublicationPointCacheObject = + serde_json::from_str(legacy).expect("legacy cache object"); + assert_eq!(legacy_object.reject_reason, None); +} diff --git a/crates/panda-rpki-validator/src/sync/mod.rs b/crates/panda-rpki-validator/src/sync/mod.rs new file mode 100644 index 0000000..7d88f66 --- /dev/null +++ b/crates/panda-rpki-validator/src/sync/mod.rs @@ -0,0 +1,3 @@ +pub mod repo; +pub mod rrdp; +pub(crate) mod store_projection; diff --git a/crates/panda-rpki-validator/src/sync/repo.rs b/crates/panda-rpki-validator/src/sync/repo.rs new file mode 100644 index 0000000..827c03b --- /dev/null +++ b/crates/panda-rpki-validator/src/sync/repo.rs @@ -0,0 +1,728 @@ +use crate::analysis::timing::TimingHandle; +use crate::audit::AuditDownloadKind; +use crate::audit_downloads::DownloadLogHandle; +use crate::current_repo_index::CurrentRepoIndexHandle; +use crate::fetch::rsync::{RsyncFetchError, RsyncFetcher}; +use crate::policy::{Policy, SyncPreference}; +use crate::replay::archive::{ReplayArchiveIndex, ReplayTransport}; +use crate::replay::delta_archive::{ReplayDeltaArchiveIndex, ReplayDeltaRrdpKind}; +use crate::report::{RfcRef, Warning}; +use crate::storage::RocksStore; +use crate::sync::rrdp::sync_from_notification_with_timing_and_download_log; +use crate::sync::rrdp::{Fetcher as HttpFetcher, RrdpSyncError, load_rrdp_local_state}; +use crate::sync::store_projection::{ + build_repository_view_present_entry, build_repository_view_withdrawn_entry, + prepare_repo_bytes_batch_owned, +}; +use std::collections::HashSet; + +#[cfg(test)] +use crate::storage::RrdpSourceSyncState; +#[cfg(test)] +use crate::sync::rrdp::persist_rrdp_local_state; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RepoSyncSource { + Rrdp, + Rsync, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RepoSyncPhase { + RrdpOk, + RrdpFailedRsyncOk, + RsyncOnlyOk, + ReplayRrdpOk, + ReplayRsyncOk, + ReplayNoopRrdp, + ReplayNoopRsync, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RepoSyncResult { + pub source: RepoSyncSource, + pub phase: RepoSyncPhase, + pub objects_written: usize, + pub warnings: Vec, +} + +#[derive(Debug, thiserror::Error)] +pub enum RepoSyncError { + #[error("RRDP sync failed: {0}")] + Rrdp(#[from] RrdpSyncError), + + #[error("rsync fallback failed: {0}")] + Rsync(#[from] RsyncFetchError), + + #[error("replay sync error: {0}")] + Replay(String), + + #[error("storage error: {0}")] + Storage(String), +} + +/// Sync a publication point into the current repository view. +/// +/// v1 behavior: +/// - If `rrdp_notification_uri` is present and `policy.sync_preference` is `rrdp_then_rsync`, +/// try RRDP snapshot sync first (RFC 8182 §3.4.1-§3.4.3). +/// - On RRDP failure, fall back to rsync (RFC 8182 §3.4.5). +/// - If `sync_preference` is `rsync_only` or there is no RRDP URI, use rsync. +pub fn sync_publication_point( + store: &RocksStore, + policy: &Policy, + rrdp_notification_uri: Option<&str>, + rsync_base_uri: &str, + http_fetcher: &dyn HttpFetcher, + rsync_fetcher: &dyn RsyncFetcher, + timing: Option<&TimingHandle>, + download_log: Option<&DownloadLogHandle>, +) -> Result { + match (policy.sync_preference, rrdp_notification_uri) { + (SyncPreference::RrdpThenRsync, Some(notification_uri)) => { + match try_rrdp_sync_with_retry( + store, + notification_uri, + None, + http_fetcher, + timing, + download_log, + ) { + Ok(written) => { + if let Some(t) = timing.as_ref() { + t.record_count("repo_sync_rrdp_ok_total", 1); + t.record_count("repo_sync_rrdp_objects_written_total", written as u64); + } + crate::progress_log::emit( + "repo_sync_rrdp_ok", + serde_json::json!({ + "notify_uri": notification_uri, + "objects_written": written, + }), + ); + Ok(RepoSyncResult { + source: RepoSyncSource::Rrdp, + phase: RepoSyncPhase::RrdpOk, + objects_written: written, + warnings: Vec::new(), + }) + } + Err(err) => { + if let Some(t) = timing.as_ref() { + t.record_count("repo_sync_rrdp_failed_total", 1); + } + crate::progress_log::emit( + "rrdp_fallback_rsync", + serde_json::json!({ + "notify_uri": notification_uri, + "rsync_base_uri": rsync_base_uri, + "rrdp_error": err.to_string(), + }), + ); + crate::progress_log::emit( + "rrdp_failed_fallback_rsync", + serde_json::json!({ + "notify_uri": notification_uri, + "rsync_base_uri": rsync_base_uri, + "rrdp_error": err.to_string(), + }), + ); + let warnings = vec![ + Warning::new(format!("RRDP failed; falling back to rsync: {err}")) + .with_rfc_refs(&[RfcRef("RFC 8182 §3.4.5")]) + .with_context(notification_uri), + ]; + let written = rsync_sync_into_current_store( + store, + rsync_base_uri, + None, + rsync_fetcher, + timing, + download_log, + )?; + if let Some(t) = timing.as_ref() { + t.record_count("repo_sync_rsync_peer_aligned_profile_total", 1); + } + if let Some(t) = timing.as_ref() { + t.record_count("repo_sync_rsync_fallback_ok_total", 1); + t.record_count("repo_sync_rsync_objects_written_total", written as u64); + } + Ok(RepoSyncResult { + source: RepoSyncSource::Rsync, + phase: RepoSyncPhase::RrdpFailedRsyncOk, + objects_written: written, + warnings, + }) + } + } + } + _ => { + let written = rsync_sync_into_current_store( + store, + rsync_base_uri, + None, + rsync_fetcher, + timing, + download_log, + )?; + if let Some(t) = timing.as_ref() { + t.record_count("repo_sync_rsync_peer_aligned_profile_total", 1); + } + crate::progress_log::emit( + "repo_sync_rsync_direct", + serde_json::json!({ + "rsync_base_uri": rsync_base_uri, + "objects_written": written, + }), + ); + if let Some(t) = timing.as_ref() { + t.record_count("repo_sync_rsync_direct_total", 1); + t.record_count("repo_sync_rsync_objects_written_total", written as u64); + } + Ok(RepoSyncResult { + source: RepoSyncSource::Rsync, + phase: RepoSyncPhase::RsyncOnlyOk, + objects_written: written, + warnings: Vec::new(), + }) + } + } +} + +pub fn sync_publication_point_replay( + store: &RocksStore, + replay_index: &ReplayArchiveIndex, + rrdp_notification_uri: Option<&str>, + rsync_base_uri: &str, + http_fetcher: &dyn HttpFetcher, + rsync_fetcher: &dyn RsyncFetcher, + timing: Option<&TimingHandle>, + download_log: Option<&DownloadLogHandle>, +) -> Result { + match resolve_replay_transport(replay_index, rrdp_notification_uri, rsync_base_uri)? { + ReplayResolvedTransport::Rrdp(notification_uri) => { + let written = try_rrdp_sync_with_retry( + store, + notification_uri, + None, + http_fetcher, + timing, + download_log, + )?; + if let Some(t) = timing.as_ref() { + t.record_count("repo_sync_rrdp_ok_total", 1); + t.record_count("repo_sync_rrdp_objects_written_total", written as u64); + } + Ok(RepoSyncResult { + source: RepoSyncSource::Rrdp, + phase: RepoSyncPhase::ReplayRrdpOk, + objects_written: written, + warnings: Vec::new(), + }) + } + ReplayResolvedTransport::Rsync => { + let written = rsync_sync_into_current_store( + store, + rsync_base_uri, + None, + rsync_fetcher, + timing, + download_log, + )?; + if let Some(t) = timing.as_ref() { + t.record_count("repo_sync_rsync_direct_total", 1); + t.record_count("repo_sync_rsync_objects_written_total", written as u64); + } + Ok(RepoSyncResult { + source: RepoSyncSource::Rsync, + phase: RepoSyncPhase::ReplayRsyncOk, + objects_written: written, + warnings: Vec::new(), + }) + } + } +} + +pub fn sync_publication_point_replay_delta( + store: &RocksStore, + delta_index: &ReplayDeltaArchiveIndex, + rrdp_notification_uri: Option<&str>, + rsync_base_uri: &str, + http_fetcher: &dyn HttpFetcher, + rsync_fetcher: &dyn RsyncFetcher, + timing: Option<&TimingHandle>, + download_log: Option<&DownloadLogHandle>, +) -> Result { + match resolve_replay_delta_transport(store, delta_index, rrdp_notification_uri, rsync_base_uri)? + { + ReplayDeltaResolvedTransport::Rrdp(notification_uri) => { + let written = try_rrdp_sync_with_retry( + store, + notification_uri, + None, + http_fetcher, + timing, + download_log, + )?; + if let Some(t) = timing.as_ref() { + t.record_count("repo_sync_rrdp_ok_total", 1); + t.record_count("repo_sync_rrdp_objects_written_total", written as u64); + } + Ok(RepoSyncResult { + source: RepoSyncSource::Rrdp, + phase: RepoSyncPhase::ReplayRrdpOk, + objects_written: written, + warnings: Vec::new(), + }) + } + ReplayDeltaResolvedTransport::Rsync => { + let written = rsync_sync_into_current_store( + store, + rsync_base_uri, + None, + rsync_fetcher, + timing, + download_log, + )?; + if let Some(t) = timing.as_ref() { + t.record_count("repo_sync_rsync_direct_total", 1); + t.record_count("repo_sync_rsync_objects_written_total", written as u64); + } + Ok(RepoSyncResult { + source: RepoSyncSource::Rsync, + phase: RepoSyncPhase::ReplayRsyncOk, + objects_written: written, + warnings: Vec::new(), + }) + } + ReplayDeltaResolvedTransport::Noop(source) => Ok(RepoSyncResult { + source, + phase: match source { + RepoSyncSource::Rrdp => RepoSyncPhase::ReplayNoopRrdp, + RepoSyncSource::Rsync => RepoSyncPhase::ReplayNoopRsync, + }, + objects_written: 0, + warnings: Vec::new(), + }), + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ReplayResolvedTransport<'a> { + Rrdp(&'a str), + Rsync, +} + +enum ReplayDeltaResolvedTransport<'a> { + Rrdp(&'a str), + Rsync, + Noop(RepoSyncSource), +} + +fn resolve_replay_transport<'a>( + replay_index: &'a ReplayArchiveIndex, + rrdp_notification_uri: Option<&'a str>, + rsync_base_uri: &str, +) -> Result, RepoSyncError> { + if let Some(notification_uri) = rrdp_notification_uri { + let lock = replay_index.rrdp_lock(notification_uri).ok_or_else(|| { + RepoSyncError::Replay(format!( + "replay RRDP lock missing for notification URI: {notification_uri}" + )) + })?; + return Ok(match lock.transport { + ReplayTransport::Rrdp => ReplayResolvedTransport::Rrdp(notification_uri), + ReplayTransport::Rsync => ReplayResolvedTransport::Rsync, + }); + } + + replay_index + .resolve_rsync_module_for_base_uri(rsync_base_uri) + .map_err(|e| RepoSyncError::Replay(e.to_string()))?; + Ok(ReplayResolvedTransport::Rsync) +} + +fn resolve_replay_delta_transport<'a>( + store: &RocksStore, + delta_index: &'a ReplayDeltaArchiveIndex, + rrdp_notification_uri: Option<&'a str>, + rsync_base_uri: &str, +) -> Result, RepoSyncError> { + if let Some(notification_uri) = rrdp_notification_uri { + let repo = delta_index.rrdp_repo(notification_uri).ok_or_else(|| { + RepoSyncError::Replay(format!( + "delta replay RRDP entry missing for notification URI: {notification_uri}" + )) + })?; + validate_delta_replay_base_state_for_repo(store, notification_uri, &repo.transition.base)?; + return match repo.transition.kind { + ReplayDeltaRrdpKind::Delta => Ok(ReplayDeltaResolvedTransport::Rrdp(notification_uri)), + ReplayDeltaRrdpKind::Unchanged => Ok(ReplayDeltaResolvedTransport::Noop( + match repo.transition.target.transport { + ReplayTransport::Rrdp => RepoSyncSource::Rrdp, + ReplayTransport::Rsync => RepoSyncSource::Rsync, + }, + )), + ReplayDeltaRrdpKind::FallbackRsync => Ok(ReplayDeltaResolvedTransport::Rsync), + ReplayDeltaRrdpKind::SessionReset => Err(RepoSyncError::Replay(format!( + "delta replay kind session-reset requires fresh full replay for {notification_uri}" + ))), + ReplayDeltaRrdpKind::Gap => Err(RepoSyncError::Replay(format!( + "delta replay kind gap requires fresh full replay for {notification_uri}" + ))), + }; + } + + delta_index + .resolve_rsync_module_for_base_uri(rsync_base_uri) + .map_err(|e| RepoSyncError::Replay(e.to_string()))?; + Ok(ReplayDeltaResolvedTransport::Rsync) +} + +fn validate_delta_replay_base_state_for_repo( + store: &RocksStore, + notification_uri: &str, + base: &crate::replay::delta_archive::ReplayDeltaRrdpState, +) -> Result<(), RepoSyncError> { + match base.transport { + ReplayTransport::Rrdp => { + let state = load_rrdp_local_state(store, notification_uri) + .map_err(RepoSyncError::Storage)? + .ok_or_else(|| { + RepoSyncError::Replay(format!( + "delta replay base state missing for {notification_uri}: expected RRDP session={} serial={}", + base.session.as_deref().unwrap_or(""), + base.serial + .map(|v| v.to_string()) + .unwrap_or_else(|| "".to_string()) + )) + })?; + let expected_session = base.session.as_deref().unwrap_or(""); + let expected_serial = base.serial.unwrap_or_default(); + if state.session_id != expected_session || state.serial != expected_serial { + return Err(RepoSyncError::Replay(format!( + "delta replay base state mismatch for {notification_uri}: expected session={} serial={}, actual session={} serial={}", + expected_session, expected_serial, state.session_id, state.serial + ))); + } + } + ReplayTransport::Rsync => {} + } + Ok(()) +} + +fn try_rrdp_sync( + store: &RocksStore, + notification_uri: &str, + current_repo_index: Option<&CurrentRepoIndexHandle>, + http_fetcher: &dyn HttpFetcher, + timing: Option<&TimingHandle>, + download_log: Option<&DownloadLogHandle>, +) -> Result { + let notification_xml = { + let _step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "fetch_notification")); + let _total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_fetch_notification_total")); + let mut dl_span = download_log + .map(|dl| dl.span_download(AuditDownloadKind::RrdpNotification, notification_uri)); + match http_fetcher.fetch(notification_uri) { + Ok(v) => { + if let Some(t) = timing.as_ref() { + t.record_count("rrdp_notification_fetch_ok_total", 1); + } + if let Some(s) = dl_span.as_mut() { + s.set_bytes(v.len() as u64); + s.set_ok(); + } + v + } + Err(e) => { + if let Some(t) = timing.as_ref() { + t.record_count("rrdp_notification_fetch_fail_total", 1); + } + if let Some(s) = dl_span.as_mut() { + s.set_err(e.clone()); + } + return Err(RrdpSyncError::Fetch(e)); + } + } + }; + if let Some(t) = timing.as_ref() { + t.record_count( + "rrdp_notification_bytes_total", + notification_xml.len() as u64, + ); + } + + sync_from_notification_with_timing_and_download_log( + store, + notification_uri, + current_repo_index, + ¬ification_xml, + http_fetcher, + timing, + download_log, + ) +} + +fn is_retryable_http_fetch_error(msg: &str) -> bool { + if msg.contains("http request failed:") || msg.contains("http read body failed:") { + return true; + } + let Some(rest) = msg.strip_prefix("http status ") else { + return false; + }; + let code = rest + .trim() + .split_whitespace() + .next() + .and_then(|s| s.parse::().ok()) + .unwrap_or(0); + code == 408 || code == 429 || (500..600).contains(&code) +} + +fn try_rrdp_sync_with_retry( + store: &RocksStore, + notification_uri: &str, + current_repo_index: Option<&CurrentRepoIndexHandle>, + http_fetcher: &dyn HttpFetcher, + timing: Option<&TimingHandle>, + download_log: Option<&DownloadLogHandle>, +) -> Result { + let attempt = 1usize; + crate::progress_log::emit( + "rrdp_sync_attempt", + serde_json::json!({ + "notify_uri": notification_uri, + "attempt": attempt, + }), + ); + if let Some(t) = timing.as_ref() { + t.record_count("rrdp_retry_attempt_total", 1); + } + + match try_rrdp_sync( + store, + notification_uri, + current_repo_index, + http_fetcher, + timing, + download_log, + ) { + Ok(written) => { + crate::progress_log::emit( + "rrdp_sync_success", + serde_json::json!({ + "notify_uri": notification_uri, + "attempt": attempt, + "objects_written": written, + }), + ); + Ok(written) + } + Err(err) => { + let retryable = match &err { + RrdpSyncError::Fetch(msg) => is_retryable_http_fetch_error(msg), + _ => false, + }; + crate::progress_log::emit( + "rrdp_sync_failed", + serde_json::json!({ + "notify_uri": notification_uri, + "attempt": attempt, + "retryable": retryable, + "error": err.to_string(), + }), + ); + if let Some(t) = timing.as_ref() { + match &err { + RrdpSyncError::Fetch(_) => t.record_count("rrdp_failed_fetch_total", 1), + RrdpSyncError::Rrdp(_) => t.record_count("rrdp_failed_protocol_total", 1), + RrdpSyncError::Storage(_) => t.record_count("rrdp_failed_storage_total", 1), + } + } + Err(err) + } + } +} + +pub(crate) fn run_rrdp_transport( + store: &RocksStore, + notification_uri: &str, + current_repo_index: Option<&CurrentRepoIndexHandle>, + http_fetcher: &dyn HttpFetcher, + timing: Option<&TimingHandle>, + download_log: Option<&DownloadLogHandle>, +) -> Result { + try_rrdp_sync_with_retry( + store, + notification_uri, + current_repo_index, + http_fetcher, + timing, + download_log, + ) +} + +fn rsync_sync_into_current_store( + store: &RocksStore, + rsync_base_uri: &str, + current_repo_index: Option<&CurrentRepoIndexHandle>, + rsync_fetcher: &dyn RsyncFetcher, + timing: Option<&TimingHandle>, + download_log: Option<&DownloadLogHandle>, +) -> Result { + let started = std::time::Instant::now(); + let sync_scope_uri = rsync_fetcher.dedup_key(rsync_base_uri); + crate::progress_log::emit( + "rsync_sync_start", + serde_json::json!({ + "rsync_base_uri": rsync_base_uri, + "sync_scope_uri": &sync_scope_uri, + }), + ); + let _s = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(rsync_base_uri, "rsync_fetch_objects")); + let _p = timing.as_ref().map(|t| t.span_phase("rsync_fetch_total")); + let mut dl_span = + download_log.map(|dl| dl.span_download(AuditDownloadKind::Rsync, rsync_base_uri)); + let mut new_set: HashSet = HashSet::new(); + let mut fetched_objects: Vec<(String, Vec)> = Vec::new(); + let (object_count, bytes_total) = + match rsync_fetcher.visit_objects(rsync_base_uri, &mut |uri, bytes| { + new_set.insert(uri.clone()); + fetched_objects.push((uri, bytes)); + Ok(()) + }) { + Ok(v) => { + if let Some(s) = dl_span.as_mut() { + s.set_objects(v.0 as u64, v.1); + s.set_bytes(v.1); + s.set_ok(); + } + v + } + Err(e) => { + if let Some(s) = dl_span.as_mut() { + s.set_err(e.to_string()); + } + return Err(e.into()); + } + }; + crate::progress_log::emit( + "rsync_sync_fetch_done", + serde_json::json!({ + "rsync_base_uri": rsync_base_uri, + "sync_scope_uri": &sync_scope_uri, + "object_count": object_count, + "bytes_total": bytes_total, + "duration_ms": started.elapsed().as_millis() as u64, + }), + ); + if let Some(t) = timing.as_ref() { + t.record_count("rsync_objects_fetched_total", object_count as u64); + t.record_count("rsync_objects_bytes_total", bytes_total); + } + drop(_p); + + let existing_view = store + .list_repository_view_entries_with_prefix(&sync_scope_uri) + .map_err(|e| RepoSyncError::Storage(e.to_string()))?; + + let _proj = timing + .as_ref() + .map(|t| t.span_phase("rsync_write_current_store_total")); + let prepared_bytes = + prepare_repo_bytes_batch_owned(fetched_objects).map_err(RepoSyncError::Storage)?; + let mut repository_view_entries = Vec::new(); + for entry in existing_view { + if !new_set.contains(&entry.rsync_uri) { + repository_view_entries.push(build_repository_view_withdrawn_entry( + &sync_scope_uri, + &entry.rsync_uri, + entry.current_hash, + )); + } + } + + for uri in &new_set { + let current_hash = prepared_bytes + .uri_to_hash + .get(uri) + .cloned() + .ok_or_else(|| { + RepoSyncError::Storage(format!("missing raw_by_hash mapping for {uri}")) + })?; + repository_view_entries.push(build_repository_view_present_entry( + &sync_scope_uri, + uri, + ¤t_hash, + )); + } + + store + .put_blob_bytes_batch(&prepared_bytes.blobs_to_write) + .map_err(|e| RepoSyncError::Storage(e.to_string()))?; + store + .put_projection_batch(&repository_view_entries, &[], &[]) + .map_err(|e| RepoSyncError::Storage(e.to_string()))?; + if let Some(index) = current_repo_index { + index + .write() + .map_err(|_| RepoSyncError::Storage("current repo index lock poisoned".to_string()))? + .apply_repository_view_entries(&repository_view_entries) + .map_err(RepoSyncError::Storage)?; + } + + let total_duration_ms = started.elapsed().as_millis() as u64; + crate::progress_log::emit( + "rsync_sync_done", + serde_json::json!({ + "rsync_base_uri": rsync_base_uri, + "sync_scope_uri": &sync_scope_uri, + "object_count": object_count, + "bytes_total": bytes_total, + "duration_ms": total_duration_ms, + }), + ); + if (total_duration_ms as f64) / 1000.0 >= crate::progress_log::slow_threshold_secs() { + crate::progress_log::emit( + "rsync_sync_slow", + serde_json::json!({ + "rsync_base_uri": rsync_base_uri, + "sync_scope_uri": &sync_scope_uri, + "object_count": object_count, + "bytes_total": bytes_total, + "duration_ms": total_duration_ms, + }), + ); + } + + Ok(object_count) +} + +pub(crate) fn run_rsync_transport( + store: &RocksStore, + rsync_base_uri: &str, + current_repo_index: Option<&CurrentRepoIndexHandle>, + rsync_fetcher: &dyn RsyncFetcher, + timing: Option<&TimingHandle>, + download_log: Option<&DownloadLogHandle>, +) -> Result { + rsync_sync_into_current_store( + store, + rsync_base_uri, + current_repo_index, + rsync_fetcher, + timing, + download_log, + ) +} + +#[cfg(test)] +#[path = "repo/tests.rs"] +mod tests; diff --git a/crates/panda-rpki-validator/src/sync/repo/tests.rs b/crates/panda-rpki-validator/src/sync/repo/tests.rs new file mode 100644 index 0000000..d35cc6f --- /dev/null +++ b/crates/panda-rpki-validator/src/sync/repo/tests.rs @@ -0,0 +1,1332 @@ +use super::*; +use crate::analysis::timing::{TimingHandle, TimingMeta}; +use crate::fetch::rsync::LocalDirRsyncFetcher; +use crate::replay::archive::{ReplayArchiveIndex, sha256_hex}; +use crate::replay::delta_archive::ReplayDeltaArchiveIndex; +use crate::replay::delta_fetch_http::PayloadDeltaReplayHttpFetcher; +use crate::replay::delta_fetch_rsync::PayloadDeltaReplayRsyncFetcher; +use crate::replay::fetch_http::PayloadReplayHttpFetcher; +use crate::replay::fetch_rsync::PayloadReplayRsyncFetcher; +use crate::storage::RepositoryViewState; +use crate::sync::rrdp::Fetcher as HttpFetcher; +use crate::sync::rrdp::RrdpState; +use crate::sync::store_projection::{build_repository_view_present_entry, compute_sha256_hex}; +use base64::Engine; +use sha2::Digest; +use std::collections::HashMap; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +struct DummyHttpFetcher; + +impl HttpFetcher for DummyHttpFetcher { + fn fetch(&self, _url: &str) -> Result, String> { + panic!("http fetcher must not be used in rsync-only mode") + } +} + +struct PanicRsyncFetcher; +impl RsyncFetcher for PanicRsyncFetcher { + fn fetch_objects( + &self, + _rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + panic!("rsync must not be used in this test") + } +} + +struct MapFetcher { + map: HashMap>, +} + +impl HttpFetcher for MapFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + self.map + .get(uri) + .cloned() + .ok_or_else(|| format!("not found: {uri}")) + } +} + +fn assert_current_object(store: &RocksStore, uri: &str, expected: &[u8]) { + assert_eq!( + store + .load_current_object_bytes_by_uri(uri) + .expect("load current object"), + Some(expected.to_vec()) + ); +} + +#[test] +fn rsync_sync_uses_fetcher_dedup_scope_for_repository_view_projection() { + struct ScopeFetcher; + impl RsyncFetcher for ScopeFetcher { + fn fetch_objects( + &self, + _rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + Ok(vec![( + "rsync://example.net/repo/child/a.mft".to_string(), + b"manifest".to_vec(), + )]) + } + + fn dedup_key(&self, _rsync_base_uri: &str) -> String { + "rsync://example.net/repo/".to_string() + } + } + + let td = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(td.path()).expect("open rocksdb"); + let seeded = build_repository_view_present_entry( + "rsync://example.net/repo/", + "rsync://example.net/repo/sibling/old.roa", + &compute_sha256_hex(b"old"), + ); + store + .put_projection_batch(&[seeded], &[], &[]) + .expect("seed repository view"); + + let fetcher = ScopeFetcher; + let written = rsync_sync_into_current_store( + &store, + "rsync://example.net/repo/child/", + None, + &fetcher, + None, + None, + ) + .expect("sync ok"); + assert_eq!(written, 1); + + let entries = store + .list_repository_view_entries_with_prefix("rsync://example.net/repo/") + .expect("list repository view"); + let sibling = entries + .iter() + .find(|entry| entry.rsync_uri == "rsync://example.net/repo/sibling/old.roa") + .expect("sibling entry exists"); + assert_eq!(sibling.state, RepositoryViewState::Withdrawn); + let child = entries + .iter() + .find(|entry| entry.rsync_uri == "rsync://example.net/repo/child/a.mft") + .expect("child entry exists"); + assert_eq!(child.state, RepositoryViewState::Present); +} + +fn notification_xml( + session_id: &str, + serial: u64, + snapshot_uri: &str, + snapshot_hash: &str, +) -> Vec { + format!( + r#""# + ) + .into_bytes() +} + +fn snapshot_xml(session_id: &str, serial: u64, published: &[(&str, &[u8])]) -> Vec { + let mut out = format!( + r#""# + ); + for (uri, bytes) in published { + let b64 = base64::engine::general_purpose::STANDARD.encode(bytes); + out.push_str(&format!(r#"{b64}"#)); + } + out.push_str(""); + out.into_bytes() +} + +fn build_replay_archive_fixture() -> ( + tempfile::TempDir, + std::path::PathBuf, + std::path::PathBuf, + String, + String, + String, + String, +) { + let temp = tempfile::tempdir().expect("tempdir"); + let archive_root = temp.path().join("payload-archive"); + let capture = "repo-replay"; + let capture_root = archive_root.join("v1").join("captures").join(capture); + std::fs::create_dir_all(&capture_root).expect("mkdir capture root"); + std::fs::write( + capture_root.join("capture.json"), + format!( + r#"{{"version":1,"captureId":"{capture}","createdAt":"2026-03-13T00:00:00Z","notes":""}}"# + ), + ) + .expect("write capture json"); + + let notify_uri = "https://rrdp.example.test/notification.xml".to_string(); + let snapshot_uri = "https://rrdp.example.test/snapshot.xml".to_string(); + let session = "00000000-0000-0000-0000-000000000001".to_string(); + let serial = 7u64; + let published_uri = "rsync://example.test/repo/a.mft".to_string(); + let published_bytes = b"mft"; + let snapshot = snapshot_xml(&session, serial, &[(&published_uri, published_bytes)]); + let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); + let notification = notification_xml(&session, serial, &snapshot_uri, &snapshot_hash); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let session_dir = capture_root + .join("rrdp/repos") + .join(&repo_hash) + .join(&session); + std::fs::create_dir_all(&session_dir).expect("mkdir session dir"); + std::fs::write( + session_dir.parent().unwrap().join("meta.json"), + format!( + r#"{{"version":1,"rpkiNotify":"{notify_uri}","createdAt":"2026-03-13T00:00:00Z","lastSeenAt":"2026-03-13T00:00:01Z"}}"# + ), + ) + .expect("write repo meta"); + std::fs::write(session_dir.join("notification-7.xml"), notification) + .expect("write notification"); + std::fs::write( + session_dir.join(format!("snapshot-7-{snapshot_hash}.xml")), + &snapshot, + ) + .expect("write snapshot"); + + let rsync_base_uri = "rsync://rsync.example.test/repo/".to_string(); + let rsync_locked_notify = "https://rrdp-fallback.example.test/notification.xml".to_string(); + let mod_hash = sha256_hex(rsync_base_uri.as_bytes()); + let module_bucket_dir = capture_root.join("rsync/modules").join(&mod_hash); + let module_root = module_bucket_dir + .join("tree") + .join("rsync.example.test") + .join("repo"); + std::fs::create_dir_all(module_root.join("sub")).expect("mkdir module tree"); + std::fs::write( + module_bucket_dir.join("meta.json"), + format!( + r#"{{"version":1,"module":"{rsync_base_uri}","createdAt":"2026-03-13T00:00:00Z","lastSeenAt":"2026-03-13T00:00:01Z"}}"# + ), + ) + .expect("write rsync meta"); + std::fs::write(module_root.join("sub").join("fallback.cer"), b"cer") + .expect("write rsync object"); + + let locks_path = temp.path().join("locks.json"); + std::fs::write( + &locks_path, + format!( + r#"{{ + "version":1, + "capture":"{capture}", + "rrdp":{{ + "{notify_uri}":{{"transport":"rrdp","session":"{session}","serial":{serial}}}, + "{rsync_locked_notify}":{{"transport":"rsync","session":null,"serial":null}} + }}, + "rsync":{{ + "{rsync_base_uri}":{{"transport":"rsync"}} + }} +}}"# + ), + ) + .expect("write locks"); + + ( + temp, + archive_root, + locks_path, + notify_uri, + rsync_locked_notify, + rsync_base_uri, + published_uri, + ) +} + +fn build_delta_replay_fixture() -> ( + tempfile::TempDir, + std::path::PathBuf, + std::path::PathBuf, + std::path::PathBuf, + std::path::PathBuf, + String, + String, + String, +) { + let temp = tempfile::tempdir().expect("tempdir"); + + let base_archive = temp.path().join("payload-archive"); + let base_capture_root = base_archive.join("v1/captures/base-cap"); + std::fs::create_dir_all(&base_capture_root).expect("mkdir base capture"); + std::fs::write( + base_capture_root.join("capture.json"), + r#"{"version":1,"captureId":"base-cap","createdAt":"2026-03-16T00:00:00Z","notes":""}"#, + ) + .expect("write base capture meta"); + + let notify_uri = "https://rrdp.example.test/notification.xml".to_string(); + let snapshot_uri = "https://rrdp.example.test/snapshot.xml".to_string(); + let session = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".to_string(); + let base_serial = 10u64; + let delta1_uri = "https://rrdp.example.test/d1.xml".to_string(); + let delta2_uri = "https://rrdp.example.test/d2.xml".to_string(); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let base_session_dir = base_capture_root + .join("rrdp/repos") + .join(&repo_hash) + .join(&session); + std::fs::create_dir_all(&base_session_dir).expect("mkdir base session dir"); + std::fs::write( + base_session_dir.parent().unwrap().join("meta.json"), + format!(r#"{{"version":1,"rpkiNotify":"{notify_uri}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), + ) + .expect("write base rrdp meta"); + let base_snapshot = snapshot_xml( + &session, + base_serial, + &[("rsync://example.test/repo/a.mft", b"base")], + ); + let base_snapshot_hash = hex::encode(sha2::Sha256::digest(&base_snapshot)); + let base_notification = + notification_xml(&session, base_serial, &snapshot_uri, &base_snapshot_hash); + std::fs::write( + base_session_dir.join("notification-10.xml"), + base_notification, + ) + .expect("write base notif"); + std::fs::write( + base_session_dir.join(format!("snapshot-10-{base_snapshot_hash}.xml")), + base_snapshot, + ) + .expect("write base snapshot"); + + let module_uri = "rsync://rsync.example.test/repo/".to_string(); + let module_hash = sha256_hex(module_uri.as_bytes()); + let base_module_bucket = base_capture_root.join("rsync/modules").join(&module_hash); + let base_module_tree = base_module_bucket.join("tree/rsync.example.test/repo"); + std::fs::create_dir_all(base_module_tree.join("sub")).expect("mkdir base rsync tree"); + std::fs::write( + base_module_bucket.join("meta.json"), + format!(r#"{{"version":1,"module":"{module_uri}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), + ) + .expect("write base module meta"); + std::fs::write(base_module_tree.join("a.mft"), b"base").expect("write base a.mft"); + std::fs::write(base_module_tree.join("sub").join("x.cer"), b"base-cer") + .expect("write base x.cer"); + + let base_locks = temp.path().join("base-locks.json"); + let fallback_notify = "https://rrdp-fallback.example.test/notification.xml".to_string(); + let base_locks_body = format!( + r#"{{"version":1,"capture":"base-cap","rrdp":{{"{notify_uri}":{{"transport":"rrdp","session":"{session}","serial":10}},"{fallback_notify}":{{"transport":"rsync","session":null,"serial":null}}}},"rsync":{{"{module_uri}":{{"transport":"rsync"}}}}}}"# + ); + std::fs::write(&base_locks, &base_locks_body).expect("write base locks"); + let base_locks_sha = sha256_hex(base_locks_body.as_bytes()); + + let delta_archive = temp.path().join("payload-delta-archive"); + let delta_capture_root = delta_archive.join("v1/captures/delta-cap"); + std::fs::create_dir_all(&delta_capture_root).expect("mkdir delta capture"); + std::fs::write( + delta_capture_root.join("capture.json"), + r#"{"version":1,"captureId":"delta-cap","createdAt":"2026-03-16T00:00:00Z","notes":""}"#, + ) + .expect("write delta capture meta"); + std::fs::write( + delta_capture_root.join("base.json"), + format!(r#"{{"version":1,"baseCapture":"base-cap","baseLocksSha256":"{base_locks_sha}","createdAt":"2026-03-16T00:00:00Z"}}"#), + ) + .expect("write delta base meta"); + + let delta_session_dir = delta_capture_root + .join("rrdp/repos") + .join(&repo_hash) + .join(&session); + let delta_deltas_dir = delta_session_dir.join("deltas"); + std::fs::create_dir_all(&delta_deltas_dir).expect("mkdir delta deltas"); + std::fs::write( + delta_session_dir.parent().unwrap().join("meta.json"), + format!(r#"{{"version":1,"rpkiNotify":"{notify_uri}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), + ) + .expect("write delta meta"); + std::fs::write( + delta_session_dir.parent().unwrap().join("transition.json"), + format!(r#"{{"kind":"delta","base":{{"transport":"rrdp","session":"{session}","serial":10}},"target":{{"transport":"rrdp","session":"{session}","serial":12}},"delta_count":2,"deltas":[11,12]}}"#), + ) + .expect("write delta transition"); + let delta1 = format!( + r#"{}"#, + base64::engine::general_purpose::STANDARD.encode(b"delta-a") + ); + let delta2 = format!( + r#"{}"#, + base64::engine::general_purpose::STANDARD.encode(b"delta-b") + ); + let delta1_hash = hex::encode(sha2::Sha256::digest(delta1.as_bytes())); + let delta2_hash = hex::encode(sha2::Sha256::digest(delta2.as_bytes())); + let target_notification = format!( + r#" + + + + +"# + ); + std::fs::write( + delta_session_dir.join("notification-target-12.xml"), + target_notification, + ) + .expect("write target notification"); + std::fs::write(delta_deltas_dir.join("delta-11-aaaa.xml"), delta1).expect("write delta11"); + std::fs::write(delta_deltas_dir.join("delta-12-bbbb.xml"), delta2).expect("write delta12"); + + let delta_module_bucket = delta_capture_root.join("rsync/modules").join(&module_hash); + let delta_module_tree = delta_module_bucket.join("tree/rsync.example.test/repo"); + std::fs::create_dir_all(delta_module_tree.join("sub")).expect("mkdir delta rsync tree"); + std::fs::write( + delta_module_bucket.join("meta.json"), + format!(r#"{{"version":1,"module":"{module_uri}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), + ) + .expect("write delta rsync meta"); + std::fs::write( + delta_module_bucket.join("files.json"), + format!(r#"{{"version":1,"module":"{module_uri}","fileCount":1,"files":["{module_uri}sub/x.cer"]}}"#), + ) + .expect("write delta files"); + std::fs::write(delta_module_tree.join("sub").join("x.cer"), b"overlay-cer") + .expect("write overlay file"); + + let fallback_hash = sha256_hex(fallback_notify.as_bytes()); + let fallback_repo_dir = delta_capture_root.join("rrdp/repos").join(&fallback_hash); + std::fs::create_dir_all(&fallback_repo_dir).expect("mkdir fallback repo dir"); + std::fs::write( + fallback_repo_dir.join("meta.json"), + format!(r#"{{"version":1,"rpkiNotify":"{fallback_notify}","createdAt":"2026-03-16T00:00:00Z","lastSeenAt":"2026-03-16T00:00:01Z"}}"#), + ) + .expect("write fallback meta"); + std::fs::write( + fallback_repo_dir.join("transition.json"), + r#"{"kind":"fallback-rsync","base":{"transport":"rsync","session":null,"serial":null},"target":{"transport":"rsync","session":null,"serial":null},"delta_count":0,"deltas":[]}"#, + ) + .expect("write fallback transition"); + + let delta_locks = temp.path().join("locks-delta.json"); + std::fs::write( + &delta_locks, + format!(r#"{{"version":1,"capture":"delta-cap","baseCapture":"base-cap","baseLocksSha256":"{base_locks_sha}","rrdp":{{"{notify_uri}":{{"kind":"delta","base":{{"transport":"rrdp","session":"{session}","serial":10}},"target":{{"transport":"rrdp","session":"{session}","serial":12}},"delta_count":2,"deltas":[11,12]}},"{fallback_notify}":{{"kind":"fallback-rsync","base":{{"transport":"rsync","session":null,"serial":null}},"target":{{"transport":"rsync","session":null,"serial":null}},"delta_count":0,"deltas":[]}}}},"rsync":{{"{module_uri}":{{"file_count":1,"overlay_only":false}}}}}}"#), + ) + .expect("write delta locks"); + + ( + temp, + base_archive, + base_locks, + delta_archive, + delta_locks, + notify_uri, + fallback_notify, + module_uri, + ) +} + +fn timing_to_json(temp_dir: &std::path::Path, timing: &TimingHandle) -> serde_json::Value { + let timing_path = temp_dir.join("timing_retry.json"); + timing.write_json(&timing_path, 50).expect("write json"); + serde_json::from_slice(&std::fs::read(&timing_path).expect("read json")).expect("parse json") +} + +#[test] +fn rsync_sync_writes_current_store_and_records_counts() { + let temp = tempfile::tempdir().expect("tempdir"); + + let repo_dir = temp.path().join("repo"); + std::fs::create_dir_all(repo_dir.join("sub")).expect("mkdir"); + std::fs::write(repo_dir.join("a.mft"), b"mft").expect("write"); + std::fs::write(repo_dir.join("sub").join("b.roa"), b"roa").expect("write"); + std::fs::write(repo_dir.join("sub").join("c.cer"), b"cer").expect("write"); + + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + + let timing = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + tal_url: None, + db_path: Some(store_dir.to_string_lossy().into_owned()), + }); + + let policy = Policy { + sync_preference: SyncPreference::RsyncOnly, + ..Policy::default() + }; + let http = DummyHttpFetcher; + let rsync = LocalDirRsyncFetcher::new(&repo_dir); + + let download_log = DownloadLogHandle::new(); + let out = sync_publication_point( + &store, + &policy, + None, + "rsync://example.test/repo/", + &http, + &rsync, + Some(&timing), + Some(&download_log), + ) + .expect("sync ok"); + + assert_eq!(out.source, RepoSyncSource::Rsync); + assert_eq!(out.objects_written, 3); + + let events = download_log.snapshot_events(); + assert_eq!(events.len(), 1); + assert_eq!(events[0].kind, AuditDownloadKind::Rsync); + assert!(events[0].success); + assert_eq!(events[0].bytes, Some(9)); + let objects = events[0].objects.as_ref().expect("objects stat"); + assert_eq!(objects.objects_count, 3); + assert_eq!(objects.objects_bytes_total, 9); + + assert_current_object(&store, "rsync://example.test/repo/a.mft", b"mft"); + assert_current_object(&store, "rsync://example.test/repo/sub/b.roa", b"roa"); + assert_current_object(&store, "rsync://example.test/repo/sub/c.cer", b"cer"); + + let view = store + .get_repository_view_entry("rsync://example.test/repo/a.mft") + .expect("get repository view") + .expect("repository view entry present"); + assert_eq!( + view.current_hash.as_deref(), + Some(hex::encode(sha2::Sha256::digest(b"mft")).as_str()) + ); + assert_eq!( + view.repository_source.as_deref(), + Some("rsync://example.test/repo/") + ); + + let current_bytes = store + .load_current_object_bytes_by_uri("rsync://example.test/repo/sub/b.roa") + .expect("load current bytes") + .expect("current object bytes exist"); + assert_eq!(current_bytes, b"roa".to_vec()); + assert!( + store + .get_raw_by_hash_entry(hex::encode(sha2::Sha256::digest(b"roa")).as_str()) + .expect("get raw_by_hash") + .is_none() + ); + + let timing_path = temp.path().join("timing.json"); + timing.write_json(&timing_path, 5).expect("write json"); + let v: serde_json::Value = + serde_json::from_slice(&std::fs::read(&timing_path).expect("read json")) + .expect("parse json"); + let counts = v.get("counts").expect("counts"); + assert_eq!( + counts + .get("rsync_objects_fetched_total") + .and_then(|v| v.as_u64()), + Some(3) + ); + assert_eq!( + counts + .get("rsync_objects_bytes_total") + .and_then(|v| v.as_u64()), + Some(3 * 3) + ); +} + +#[test] +fn rsync_second_sync_marks_missing_repository_view_entries_withdrawn() { + let temp = tempfile::tempdir().expect("tempdir"); + + let repo_dir = temp.path().join("repo"); + std::fs::create_dir_all(repo_dir.join("sub")).expect("mkdir"); + std::fs::write(repo_dir.join("a.mft"), b"mft-v1").expect("write a"); + std::fs::write(repo_dir.join("sub").join("b.roa"), b"roa-v1").expect("write b"); + + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + let policy = Policy { + sync_preference: SyncPreference::RsyncOnly, + ..Policy::default() + }; + let http = DummyHttpFetcher; + let rsync = LocalDirRsyncFetcher::new(&repo_dir); + + sync_publication_point( + &store, + &policy, + None, + "rsync://example.test/repo/", + &http, + &rsync, + None, + None, + ) + .expect("first sync ok"); + + std::fs::remove_file(repo_dir.join("sub").join("b.roa")).expect("remove b"); + std::fs::write(repo_dir.join("c.crl"), b"crl-v2").expect("write c"); + + sync_publication_point( + &store, + &policy, + None, + "rsync://example.test/repo/", + &http, + &rsync, + None, + None, + ) + .expect("second sync ok"); + + let withdrawn = store + .get_repository_view_entry("rsync://example.test/repo/sub/b.roa") + .expect("get withdrawn repo view") + .expect("withdrawn entry exists"); + assert_eq!( + withdrawn.state, + crate::storage::RepositoryViewState::Withdrawn + ); + assert_eq!( + withdrawn.repository_source.as_deref(), + Some("rsync://example.test/repo/") + ); + + let added = store + .get_repository_view_entry("rsync://example.test/repo/c.crl") + .expect("get added repo view") + .expect("added entry exists"); + assert_eq!(added.state, crate::storage::RepositoryViewState::Present); +} + +#[test] +fn rrdp_fetch_error_falls_back_to_rsync_without_retry() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + + let timing = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + tal_url: None, + db_path: Some(store_dir.to_string_lossy().into_owned()), + }); + + let notification_uri = "https://example.test/notification.xml"; + let published_uri = "rsync://example.test/repo/a.mft"; + let published_bytes = b"x"; + struct AlwaysFailHttp { + notification_calls: AtomicUsize, + } + + impl HttpFetcher for AlwaysFailHttp { + fn fetch(&self, _uri: &str) -> Result, String> { + self.notification_calls.fetch_add(1, Ordering::SeqCst); + Err("http request failed: simulated transient".to_string()) + } + } + + struct SingleObjectRsync { + uri: String, + bytes: Vec, + } + impl RsyncFetcher for SingleObjectRsync { + fn fetch_objects( + &self, + _rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + Ok(vec![(self.uri.clone(), self.bytes.clone())]) + } + } + + let http = AlwaysFailHttp { + notification_calls: AtomicUsize::new(0), + }; + + let policy = Policy { + sync_preference: SyncPreference::RrdpThenRsync, + ..Policy::default() + }; + + let download_log = DownloadLogHandle::new(); + let out = sync_publication_point( + &store, + &policy, + Some(notification_uri), + "rsync://example.test/repo/", + &http, + &SingleObjectRsync { + uri: published_uri.to_string(), + bytes: published_bytes.to_vec(), + }, + Some(&timing), + Some(&download_log), + ) + .expect("sync ok"); + + assert_eq!(out.source, RepoSyncSource::Rsync); + assert_current_object(&store, published_uri, published_bytes); + assert_eq!(http.notification_calls.load(Ordering::SeqCst), 1); + + let events = download_log.snapshot_events(); + assert_eq!(events.len(), 2, "expected 1x notification + 1x rsync"); + assert_eq!( + events + .iter() + .filter(|e| e.kind == AuditDownloadKind::RrdpNotification) + .count(), + 1 + ); + assert_eq!( + events + .iter() + .filter(|e| e.kind == AuditDownloadKind::RrdpNotification && !e.success) + .count(), + 1 + ); + assert_eq!( + events + .iter() + .filter(|e| e.kind == AuditDownloadKind::Rsync) + .count(), + 1 + ); + + let v = timing_to_json(temp.path(), &timing); + let counts = v.get("counts").expect("counts"); + assert_eq!( + counts + .get("rrdp_retry_attempt_total") + .and_then(|v| v.as_u64()), + Some(1) + ); + assert_eq!( + counts + .get("repo_sync_rrdp_failed_total") + .and_then(|v| v.as_u64()), + Some(1) + ); + assert_eq!( + counts + .get("repo_sync_rsync_fallback_ok_total") + .and_then(|v| v.as_u64()), + Some(1) + ); +} + +#[test] +fn rrdp_protocol_error_does_not_retry_and_falls_back_to_rsync() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + + let timing = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + tal_url: None, + db_path: Some(store_dir.to_string_lossy().into_owned()), + }); + + let notification_uri = "https://example.test/notification.xml"; + let snapshot_uri = "https://example.test/snapshot.xml"; + let published_uri = "rsync://example.test/repo/a.mft"; + let published_bytes = b"x"; + + let snapshot = snapshot_xml( + "9df4b597-af9e-4dca-bdda-719cce2c4e28", + 1, + &[(published_uri, published_bytes)], + ); + // Intentionally wrong hash to trigger protocol error (SnapshotHashMismatch). + let wrong_hash = "00".repeat(32); + let notif = notification_xml( + "9df4b597-af9e-4dca-bdda-719cce2c4e28", + 1, + snapshot_uri, + &wrong_hash, + ); + + let mut map = HashMap::new(); + map.insert(notification_uri.to_string(), notif); + map.insert(snapshot_uri.to_string(), snapshot); + let http = MapFetcher { map }; + + struct EmptyRsyncFetcher; + impl RsyncFetcher for EmptyRsyncFetcher { + fn fetch_objects( + &self, + _rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + Ok(Vec::new()) + } + } + + let policy = Policy { + sync_preference: SyncPreference::RrdpThenRsync, + ..Policy::default() + }; + + let download_log = DownloadLogHandle::new(); + let out = sync_publication_point( + &store, + &policy, + Some(notification_uri), + "rsync://example.test/repo/", + &http, + &EmptyRsyncFetcher, + Some(&timing), + Some(&download_log), + ) + .expect("sync ok"); + + assert_eq!(out.source, RepoSyncSource::Rsync); + assert!( + out.warnings + .iter() + .any(|w| w.message.contains("RRDP failed; falling back to rsync")), + "expected RRDP fallback warning" + ); + + let events = download_log.snapshot_events(); + assert_eq!( + events.len(), + 3, + "expected notification + snapshot + rsync fallback" + ); + assert_eq!(events[0].kind, AuditDownloadKind::RrdpNotification); + assert!(events[0].success); + assert_eq!(events[1].kind, AuditDownloadKind::RrdpSnapshot); + assert!(!events[1].success); + assert_eq!(events[2].kind, AuditDownloadKind::Rsync); + assert!(events[2].success); + + let v = timing_to_json(temp.path(), &timing); + let counts = v.get("counts").expect("counts"); + assert_eq!( + counts + .get("rrdp_retry_attempt_total") + .and_then(|v| v.as_u64()), + Some(1) + ); + assert_eq!( + counts + .get("rrdp_failed_protocol_total") + .and_then(|v| v.as_u64()), + Some(1) + ); + assert_eq!( + counts + .get("repo_sync_rrdp_failed_total") + .and_then(|v| v.as_u64()), + Some(1) + ); + assert_eq!( + counts + .get("repo_sync_rsync_fallback_ok_total") + .and_then(|v| v.as_u64()), + Some(1) + ); +} + +#[test] +fn rrdp_delta_fetches_are_logged_even_if_snapshot_fallback_is_used() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + + let timing = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + tal_url: None, + db_path: Some(store_dir.to_string_lossy().into_owned()), + }); + + let notification_uri = "https://example.test/notification.xml"; + let snapshot_uri = "https://example.test/snapshot.xml"; + let delta_2_uri = "https://example.test/delta_2.xml"; + let delta_3_uri = "https://example.test/delta_3.xml"; + let published_uri = "rsync://example.test/repo/a.mft"; + let published_bytes = b"x"; + + let sid = "9df4b597-af9e-4dca-bdda-719cce2c4e28"; + + // Seed old RRDP state so sync_from_notification tries deltas (RFC 8182 §3.4.1). + let state = RrdpState { + session_id: sid.to_string(), + serial: 1, + }; + persist_rrdp_local_state( + &store, + notification_uri, + &state, + RrdpSourceSyncState::DeltaReady, + Some(snapshot_uri), + None, + ) + .expect("seed state"); + + let delta_2 = format!( + r#""# + ) + .into_bytes(); + let delta_3 = format!( + r#""# + ) + .into_bytes(); + let delta_2_hash = hex::encode(sha2::Sha256::digest(&delta_2)); + let delta_3_hash = hex::encode(sha2::Sha256::digest(&delta_3)); + + let snapshot = snapshot_xml(sid, 3, &[(published_uri, published_bytes)]); + let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); + let notif = format!( + r#""# + ) + .into_bytes(); + + let mut map = HashMap::new(); + map.insert(notification_uri.to_string(), notif); + map.insert(snapshot_uri.to_string(), snapshot); + map.insert(delta_2_uri.to_string(), delta_2); + map.insert(delta_3_uri.to_string(), delta_3); + let http = MapFetcher { map }; + + let policy = Policy { + sync_preference: SyncPreference::RrdpThenRsync, + ..Policy::default() + }; + + let download_log = DownloadLogHandle::new(); + let out = sync_publication_point( + &store, + &policy, + Some(notification_uri), + "rsync://example.test/repo/", + &http, + &PanicRsyncFetcher, + Some(&timing), + Some(&download_log), + ) + .expect("sync ok"); + + assert_eq!(out.source, RepoSyncSource::Rrdp); + assert_eq!(out.objects_written, 1); + assert_current_object(&store, published_uri, published_bytes); + + let events = download_log.snapshot_events(); + assert_eq!(events.len(), 4); + assert_eq!( + events + .iter() + .filter(|e| e.kind == AuditDownloadKind::RrdpNotification) + .count(), + 1 + ); + assert_eq!( + events + .iter() + .filter(|e| e.kind == AuditDownloadKind::RrdpDelta) + .count(), + 2 + ); + assert_eq!( + events + .iter() + .filter(|e| e.kind == AuditDownloadKind::RrdpSnapshot) + .count(), + 1 + ); + assert!(events.iter().all(|e| e.success)); +} + +#[test] +fn replay_sync_uses_rrdp_when_locked_to_rrdp() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + + let ( + _archive_temp, + archive_root, + locks_path, + notify_uri, + _rsync_locked_notify, + _rsync_base_uri, + published_uri, + ) = build_replay_archive_fixture(); + let replay_index = + ReplayArchiveIndex::load(&archive_root, &locks_path).expect("load replay index"); + let http = PayloadReplayHttpFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay http fetcher"); + let rsync = PayloadReplayRsyncFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay rsync fetcher"); + + let out = sync_publication_point_replay( + &store, + &replay_index, + Some(¬ify_uri), + "rsync://example.test/repo/", + &http, + &rsync, + None, + None, + ) + .expect("replay sync ok"); + + assert_eq!(out.source, RepoSyncSource::Rrdp); + assert_eq!(out.objects_written, 1); + assert_current_object(&store, &published_uri, b"mft"); +} + +#[test] +fn replay_sync_uses_rsync_when_notification_is_locked_to_rsync() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + + let ( + _archive_temp, + archive_root, + locks_path, + _notify_uri, + rsync_locked_notify, + rsync_base_uri, + _published_uri, + ) = build_replay_archive_fixture(); + let replay_index = + ReplayArchiveIndex::load(&archive_root, &locks_path).expect("load replay index"); + let http = PayloadReplayHttpFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay http fetcher"); + let rsync = PayloadReplayRsyncFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay rsync fetcher"); + + let out = sync_publication_point_replay( + &store, + &replay_index, + Some(&rsync_locked_notify), + &rsync_base_uri, + &http, + &rsync, + None, + None, + ) + .expect("replay rsync sync ok"); + + assert_eq!(out.source, RepoSyncSource::Rsync); + assert_eq!(out.objects_written, 1); + assert_eq!(out.warnings.len(), 0); + assert_current_object( + &store, + "rsync://rsync.example.test/repo/sub/fallback.cer", + b"cer", + ); +} + +#[test] +fn replay_sync_errors_when_lock_is_missing() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + + let ( + _archive_temp, + archive_root, + locks_path, + _notify_uri, + _rsync_locked_notify, + _rsync_base_uri, + _published_uri, + ) = build_replay_archive_fixture(); + let replay_index = + ReplayArchiveIndex::load(&archive_root, &locks_path).expect("load replay index"); + let http = PayloadReplayHttpFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay http fetcher"); + let rsync = PayloadReplayRsyncFetcher::from_paths(&archive_root, &locks_path) + .expect("build replay rsync fetcher"); + + let err = sync_publication_point_replay( + &store, + &replay_index, + Some("https://missing.example/notification.xml"), + "rsync://missing.example/repo/", + &http, + &rsync, + None, + None, + ) + .unwrap_err(); + assert!(matches!(err, RepoSyncError::Replay(_)), "{err}"); +} + +#[test] +fn delta_replay_sync_applies_rrdp_deltas_when_base_state_matches() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + let ( + _fixture, + base_archive, + base_locks, + delta_archive, + delta_locks, + notify_uri, + _fallback_notify, + module_uri, + ) = build_delta_replay_fixture(); + let base_index = + Arc::new(ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index")); + let delta_index = Arc::new( + ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"), + ); + let http = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) + .expect("build delta http fetcher"); + let rsync = PayloadDeltaReplayRsyncFetcher::new(base_index, delta_index.clone()); + + let state = RrdpState { + session_id: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".to_string(), + serial: 10, + }; + persist_rrdp_local_state( + &store, + ¬ify_uri, + &state, + RrdpSourceSyncState::DeltaReady, + None, + None, + ) + .expect("seed base state"); + + let out = sync_publication_point_replay_delta( + &store, + &delta_index, + Some(¬ify_uri), + &module_uri, + &http, + &rsync, + None, + None, + ) + .expect("delta sync ok"); + + assert_eq!(out.source, RepoSyncSource::Rrdp); + assert_eq!(out.objects_written, 2); + assert_current_object(&store, "rsync://example.test/repo/a.mft", b"delta-a"); + assert_current_object(&store, "rsync://example.test/repo/sub/b.roa", b"delta-b"); + let new_state = load_rrdp_local_state(&store, ¬ify_uri) + .expect("load current state") + .expect("rrdp state present"); + assert_eq!(new_state.serial, 12); +} + +#[test] +fn delta_replay_sync_rejects_base_state_mismatch() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + let ( + _fixture, + base_archive, + base_locks, + delta_archive, + delta_locks, + notify_uri, + _fallback_notify, + module_uri, + ) = build_delta_replay_fixture(); + let base_index = + Arc::new(ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index")); + let delta_index = Arc::new( + ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"), + ); + let http = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) + .expect("build delta http fetcher"); + let rsync = PayloadDeltaReplayRsyncFetcher::new(base_index, delta_index.clone()); + + let err = sync_publication_point_replay_delta( + &store, + &delta_index, + Some(¬ify_uri), + &module_uri, + &http, + &rsync, + None, + None, + ) + .unwrap_err(); + assert!(matches!(err, RepoSyncError::Replay(_)), "{err}"); +} + +#[test] +fn delta_replay_sync_noops_unchanged_rrdp_repo() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + let ( + _fixture, + base_archive, + base_locks, + delta_archive, + delta_locks, + notify_uri, + _fallback_notify, + module_uri, + ) = build_delta_replay_fixture(); + let state = RrdpState { + session_id: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".to_string(), + serial: 10, + }; + persist_rrdp_local_state( + &store, + ¬ify_uri, + &state, + RrdpSourceSyncState::DeltaReady, + None, + None, + ) + .expect("seed base state"); + + let base_locks_body = std::fs::read_to_string(&base_locks).expect("read base locks"); + let base_locks_sha = sha256_hex(base_locks_body.as_bytes()); + std::fs::write( + &delta_locks, + format!(r#"{{"version":1,"capture":"delta-cap","baseCapture":"base-cap","baseLocksSha256":"{base_locks_sha}","rrdp":{{"{notify_uri}":{{"kind":"unchanged","base":{{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":10}},"target":{{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":10}},"delta_count":0,"deltas":[]}},"https://rrdp-fallback.example.test/notification.xml":{{"kind":"fallback-rsync","base":{{"transport":"rsync","session":null,"serial":null}},"target":{{"transport":"rsync","session":null,"serial":null}},"delta_count":0,"deltas":[]}}}},"rsync":{{"rsync://rsync.example.test/repo/":{{"file_count":1,"overlay_only":true}}}}}}"#), + ) + .expect("rewrite delta locks"); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let repo_dir = delta_archive + .join("v1/captures/delta-cap/rrdp/repos") + .join(&repo_hash); + std::fs::write( + repo_dir.join("transition.json"), + r#"{"kind":"unchanged","base":{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":10},"target":{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":10},"delta_count":0,"deltas":[]}"#, + ).expect("rewrite transition"); + let delta_index = + ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"); + let http = PayloadDeltaReplayHttpFetcher::from_index(Arc::new(delta_index.clone())) + .expect("build delta http fetcher"); + let base_index = + Arc::new(ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index")); + let rsync = PayloadDeltaReplayRsyncFetcher::new(base_index, Arc::new(delta_index.clone())); + + let out = sync_publication_point_replay_delta( + &store, + &delta_index, + Some(¬ify_uri), + &module_uri, + &http, + &rsync, + None, + None, + ) + .expect("unchanged delta sync ok"); + assert_eq!(out.source, RepoSyncSource::Rrdp); + assert_eq!(out.objects_written, 0); +} + +#[test] +fn delta_replay_sync_uses_rsync_overlay_for_fallback_rsync_kind() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + let ( + _fixture, + base_archive, + base_locks, + delta_archive, + delta_locks, + _notify_uri, + fallback_notify, + module_uri, + ) = build_delta_replay_fixture(); + let base_index = + Arc::new(ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index")); + let delta_index = Arc::new( + ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"), + ); + let http = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) + .expect("build delta http fetcher"); + let rsync = PayloadDeltaReplayRsyncFetcher::new(base_index, delta_index.clone()); + + let out = sync_publication_point_replay_delta( + &store, + &delta_index, + Some(&fallback_notify), + &module_uri, + &http, + &rsync, + None, + None, + ) + .expect("fallback-rsync delta sync ok"); + assert_eq!(out.source, RepoSyncSource::Rsync); + assert_eq!(out.objects_written, 2); + assert_current_object(&store, "rsync://rsync.example.test/repo/a.mft", b"base"); + assert_current_object( + &store, + "rsync://rsync.example.test/repo/sub/x.cer", + b"overlay-cer", + ); +} + +#[test] +fn delta_replay_sync_rejects_session_reset_and_gap() { + for kind in ["session-reset", "gap"] { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + let ( + _fixture, + base_archive, + base_locks, + delta_archive, + delta_locks, + notify_uri, + _fallback_notify, + module_uri, + ) = build_delta_replay_fixture(); + let base_index = Arc::new( + ReplayArchiveIndex::load(&base_archive, &base_locks).expect("load base index"), + ); + let state = RrdpState { + session_id: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa".to_string(), + serial: 10, + }; + persist_rrdp_local_state( + &store, + ¬ify_uri, + &state, + RrdpSourceSyncState::DeltaReady, + None, + None, + ) + .expect("seed base state"); + + let locks_body = std::fs::read_to_string(&delta_locks).expect("read delta locks"); + let rewritten = locks_body.replace("\"kind\":\"delta\"", &format!("\"kind\":\"{}\"", kind)); + std::fs::write(&delta_locks, rewritten).expect("rewrite locks kind"); + let repo_hash = sha256_hex(notify_uri.as_bytes()); + let repo_dir = delta_archive + .join("v1/captures/delta-cap/rrdp/repos") + .join(&repo_hash); + std::fs::write( + repo_dir.join("transition.json"), + format!( + r#"{{"kind":"{kind}","base":{{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":10}},"target":{{"transport":"rrdp","session":"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa","serial":12}},"delta_count":2,"deltas":[11,12]}}"#, + ), + ) + .expect("rewrite transition kind"); + let delta_index = Arc::new( + ReplayDeltaArchiveIndex::load(&delta_archive, &delta_locks).expect("load delta index"), + ); + let http = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) + .expect("build delta http fetcher"); + let rsync = PayloadDeltaReplayRsyncFetcher::new(base_index.clone(), delta_index.clone()); + let err = sync_publication_point_replay_delta( + &store, + &delta_index, + Some(¬ify_uri), + &module_uri, + &http, + &rsync, + None, + None, + ) + .unwrap_err(); + assert!(matches!(err, RepoSyncError::Replay(_)), "{err}"); + } +} diff --git a/crates/panda-rpki-validator/src/sync/rrdp.rs b/crates/panda-rpki-validator/src/sync/rrdp.rs new file mode 100644 index 0000000..8b13435 --- /dev/null +++ b/crates/panda-rpki-validator/src/sync/rrdp.rs @@ -0,0 +1,1357 @@ +use crate::analysis::timing::TimingHandle; +use crate::audit::AuditDownloadKind; +use crate::audit_downloads::DownloadLogHandle; +use crate::current_repo_index::CurrentRepoIndexHandle; +use crate::storage::{ + RepositoryViewEntry, RepositoryViewState, RocksStore, RrdpDeltaOp, RrdpSourceSyncState, +}; +mod snapshot_apply; + +use crate::sync::store_projection::{ + build_repository_view_present_entry, build_repository_view_withdrawn_entry, + current_rrdp_owner_is, ensure_rrdp_uri_can_be_owned_by, put_repository_view_present, + put_repository_view_withdrawn, put_rrdp_source_member_present, + put_rrdp_source_member_withdrawn, put_rrdp_uri_owner_active, put_rrdp_uri_owner_withdrawn, + update_rrdp_source_record_on_success, upsert_repo_blob_bytes, +}; +use base64::Engine; +use serde::{Deserialize, Serialize}; +use sha2::Digest; +use std::io::Write; +use uuid::Uuid; + +const RRDP_XMLNS: &str = "http://www.ripe.net/rpki/rrdp"; +const RRDP_SNAPSHOT_APPLY_BATCH_SIZE: usize = 1024; + +#[derive(Debug, thiserror::Error)] +pub enum RrdpError { + #[error("RRDP XML must be US-ASCII encoded (RFC 8182 §3.5.1.3, §3.5.2.3), got non-ASCII bytes")] + NotAscii, + + #[error("RRDP XML parse error: {0} (RFC 8182 §3.5.1.3, §3.5.2.3)")] + Xml(String), + + #[error( + "RRDP root element must be , , or , got <{0}> (RFC 8182 §3.5.1.3, §3.5.2.3, §3.5.3.3)" + )] + UnexpectedRoot(String), + + #[error("RRDP XML namespace must be {RRDP_XMLNS}, got {0} (RFC 8182 §3.5.1.3, §3.5.2.3)")] + InvalidNamespace(String), + + #[error("RRDP version must be 1, got {0} (RFC 8182 §3.5.1.3, §3.5.2.3)")] + InvalidVersion(String), + + #[error("RRDP session_id invalid UUID: {0} (RFC 8182 §3.5.1.3)")] + InvalidSessionId(String), + + #[error("RRDP serial invalid unsigned integer: {0} (RFC 8182 §3.5.1.3)")] + InvalidSerial(String), + + #[error("notification must contain exactly one element (RFC 8182 §3.5.1.3)")] + SnapshotCountInvalid, + + #[error("snapshot/@uri missing (RFC 8182 §3.5.1.3)")] + SnapshotUriMissing, + + #[error("snapshot/@hash missing (RFC 8182 §3.5.1.3)")] + SnapshotHashMissing, + + #[error("snapshot/@hash must be hex encoding of SHA-256, got {0} (RFC 8182 §3.5.1.3)")] + SnapshotHashInvalid(String), + + #[error("delta/@serial missing in notification (RFC 8182 §3.5.1.3)")] + DeltaRefSerialMissing, + + #[error("delta/@uri missing in notification (RFC 8182 §3.5.1.3)")] + DeltaRefUriMissing, + + #[error("delta/@hash missing in notification (RFC 8182 §3.5.1.3)")] + DeltaRefHashMissing, + + #[error("delta/@hash must be hex encoding of SHA-256, got {0} (RFC 8182 §3.5.1.3)")] + DeltaRefHashInvalid(String), + + #[error("delta/@serial duplicates in notification: {0} (RFC 8182 §3.5.1.3)")] + DeltaRefSerialDuplicate(u64), + + #[error( + "delta/@serial must be <= notification/@serial: delta={delta_serial} notification={notification_serial} (RFC 8182 §3.5.1.3)" + )] + DeltaRefSerialTooHigh { + delta_serial: u64, + notification_serial: u64, + }, + + #[error( + "notification contains deltas but does not end at notification/@serial: max_delta={max_delta_serial} notification={notification_serial} (RFC 8182 §3.5.1.3)" + )] + DeltaRefChainDoesNotEndAtNotificationSerial { + max_delta_serial: u64, + notification_serial: u64, + }, + + #[error( + "notification delta chain not contiguous: missing serial={missing_serial} (range {min_serial}..{notification_serial}) (RFC 8182 §3.5.1.3)" + )] + DeltaRefChainNotContiguous { + min_serial: u64, + notification_serial: u64, + missing_serial: u64, + }, + + #[error("snapshot file hash mismatch (RFC 8182 §3.5.1.3)")] + SnapshotHashMismatch, + + #[error("snapshot session_id mismatch: expected {expected}, got {got} (RFC 8182 §3.5.2.3)")] + SnapshotSessionIdMismatch { expected: String, got: String }, + + #[error("snapshot serial mismatch: expected {expected}, got {got} (RFC 8182 §3.5.2.3)")] + SnapshotSerialMismatch { expected: u64, got: u64 }, + + #[error("delta file hash mismatch (RFC 8182 §3.4.2; RFC 8182 §3.5.1.3)")] + DeltaHashMismatch, + + #[error("delta session_id mismatch: expected {expected}, got {got} (RFC 8182 §3.5.3.3)")] + DeltaSessionIdMismatch { expected: String, got: String }, + + #[error("delta serial mismatch: expected {expected}, got {got} (RFC 8182 §3.5.3.3)")] + DeltaSerialMismatch { expected: u64, got: u64 }, + + #[error("notification serial moved backwards: old={old} new={new} (RFC 8182 §3.4.1)")] + NotificationSerialRollback { old: u64, new: u64 }, + + #[error("delta publish without @hash for existing object: {rsync_uri} (RFC 8182 §3.4.2)")] + DeltaPublishWithoutHashForExisting { rsync_uri: String }, + + #[error( + "delta withdraw/replace target not from this repository server: {rsync_uri} (RFC 8182 §3.4.2)" + )] + DeltaTargetNotFromRepository { rsync_uri: String }, + + #[error("delta withdraw/replace target missing in local cache: {rsync_uri} (RFC 8182 §3.4.2)")] + DeltaTargetMissing { rsync_uri: String }, + + #[error("delta withdraw/replace target hash mismatch: {rsync_uri} (RFC 8182 §3.4.2)")] + DeltaTargetHashMismatch { rsync_uri: String }, + + #[error("publish/@uri missing (RFC 8182 §3.5.2.3)")] + PublishUriMissing, + + #[error("publish element missing base64 content (RFC 8182 §3.5.2.3)")] + PublishContentMissing, + + #[error("publish base64 decode failed (RFC 8182 §3.5.2.3): {0}")] + PublishBase64(String), + + #[error("delta file missing @uri (RFC 8182 §3.5.3.3)")] + DeltaPublishUriMissing, + + #[error("delta file base64 content missing (RFC 8182 §3.5.3.3)")] + DeltaPublishContentMissing, + + #[error("delta file base64 decode failed (RFC 8182 §3.5.3.3): {0}")] + DeltaPublishBase64(String), + + #[error( + "delta file @hash must be hex encoding of SHA-256, got {0} (RFC 8182 §3.5.3.3)" + )] + DeltaPublishHashInvalid(String), + + #[error("delta file missing @uri (RFC 8182 §3.5.3.3)")] + DeltaWithdrawUriMissing, + + #[error("delta file missing @hash (RFC 8182 §3.5.3.3)")] + DeltaWithdrawHashMissing, + + #[error( + "delta file @hash must be hex encoding of SHA-256, got {0} (RFC 8182 §3.5.3.3)" + )] + DeltaWithdrawHashInvalid(String), + + #[error("delta file must not contain text content (RFC 8182 §3.5.3.3)")] + DeltaWithdrawUnexpectedContent, + + #[error("delta file must contain at least one publish/withdraw element (RFC 8182 §3.5.3.3)")] + DeltaNoElements, + + #[error("delta file contains unexpected element <{0}> (RFC 8182 §3.5.3.3)")] + DeltaUnexpectedElement(String), +} + +#[derive(Debug, thiserror::Error)] +pub enum RrdpSyncError { + #[error("{0}")] + Rrdp(#[from] RrdpError), + + #[error("fetch failed: {0}")] + Fetch(String), + + #[error("storage error: {0}")] + Storage(String), +} + +pub type RrdpSyncResult = Result; + +pub trait Fetcher: Send + Sync { + fn fetch(&self, uri: &str) -> Result, String>; + + fn fetch_to_writer(&self, uri: &str, out: &mut dyn Write) -> Result { + let bytes = self.fetch(uri)?; + out.write_all(&bytes) + .map_err(|e| format!("write sink failed: {e}"))?; + Ok(bytes.len() as u64) + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RrdpState { + pub session_id: String, + pub serial: u64, +} + +impl RrdpState { + pub fn encode(&self) -> Result, String> { + serde_cbor::to_vec(self).map_err(|e| e.to_string()) + } + + pub fn decode(bytes: &[u8]) -> Result { + serde_cbor::from_slice(bytes).map_err(|e| e.to_string()) + } +} + +pub(crate) fn load_rrdp_local_state( + store: &RocksStore, + notification_uri: &str, +) -> Result, String> { + if let Some(record) = store + .get_rrdp_source_record(notification_uri) + .map_err(|e| e.to_string())? + { + if let (Some(session_id), Some(serial)) = (record.last_session_id, record.last_serial) { + return Ok(Some(RrdpState { session_id, serial })); + } + } + + Ok(None) +} + +pub(crate) fn persist_rrdp_local_state( + store: &RocksStore, + notification_uri: &str, + state: &RrdpState, + sync_state: RrdpSourceSyncState, + last_snapshot_uri: Option<&str>, + last_snapshot_hash_hex: Option<&str>, +) -> Result<(), String> { + update_rrdp_source_record_on_success( + store, + notification_uri, + state.session_id.as_str(), + state.serial, + sync_state, + last_snapshot_uri, + last_snapshot_hash_hex, + ) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NotificationSnapshot { + pub session_id: Uuid, + pub serial: u64, + pub snapshot_uri: String, + pub snapshot_hash_sha256: [u8; 32], +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct NotificationDeltaRef { + pub serial: u64, + pub uri: String, + pub hash_sha256: [u8; 32], +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Notification { + pub session_id: Uuid, + pub serial: u64, + pub snapshot_uri: String, + pub snapshot_hash_sha256: [u8; 32], + /// Deltas referenced by the notification file, sorted by serial ascending. + /// + /// If present, this list is guaranteed to be contiguous and to end at `serial` + /// (RFC 8182 §3.5.1.3). + pub deltas: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum DeltaElement { + Publish { + uri: String, + hash_sha256: Option<[u8; 32]>, + bytes: Vec, + }, + Withdraw { + uri: String, + hash_sha256: [u8; 32], + }, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DeltaFile { + pub session_id: Uuid, + pub serial: u64, + pub elements: Vec, +} + +pub fn parse_notification(xml: &[u8]) -> Result { + let doc = parse_rrdp_xml(xml)?; + let root = doc.root_element(); + if root.tag_name().name() != "notification" { + return Err(RrdpError::UnexpectedRoot( + root.tag_name().name().to_string(), + )); + } + validate_root_common(&root)?; + + let session_id = parse_uuid_attr(&root, "session_id")?; + let serial = parse_u64_attr(&root, "serial")?; + + let snapshots: Vec<_> = root + .children() + .filter(|n| n.is_element() && n.tag_name().name() == "snapshot") + .collect(); + if snapshots.len() != 1 { + return Err(RrdpError::SnapshotCountInvalid); + } + let snapshot = snapshots[0]; + + let snapshot_uri = snapshot + .attribute("uri") + .ok_or(RrdpError::SnapshotUriMissing)? + .to_string(); + let snapshot_hash_hex = snapshot + .attribute("hash") + .ok_or(RrdpError::SnapshotHashMissing)?; + let snapshot_hash_sha256 = parse_sha256_hex_snapshot(snapshot_hash_hex)?; + + let mut deltas: Vec = Vec::new(); + for d in root + .children() + .filter(|n| n.is_element() && n.tag_name().name() == "delta") + { + let delta_serial = d + .attribute("serial") + .ok_or(RrdpError::DeltaRefSerialMissing)?; + let delta_serial = parse_u64_str(delta_serial)?; + if delta_serial > serial { + return Err(RrdpError::DeltaRefSerialTooHigh { + delta_serial, + notification_serial: serial, + }); + } + let uri = d.attribute("uri").ok_or(RrdpError::DeltaRefUriMissing)?; + let hash = d.attribute("hash").ok_or(RrdpError::DeltaRefHashMissing)?; + let hash_sha256 = parse_sha256_hex_delta_ref(hash)?; + + deltas.push(NotificationDeltaRef { + serial: delta_serial, + uri: uri.to_string(), + hash_sha256, + }); + } + + if !deltas.is_empty() { + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + let mut min_serial = u64::MAX; + let mut max_serial = 0u64; + for d in &deltas { + if !seen.insert(d.serial) { + return Err(RrdpError::DeltaRefSerialDuplicate(d.serial)); + } + min_serial = min_serial.min(d.serial); + max_serial = max_serial.max(d.serial); + } + if max_serial != serial { + return Err(RrdpError::DeltaRefChainDoesNotEndAtNotificationSerial { + max_delta_serial: max_serial, + notification_serial: serial, + }); + } + for s in min_serial..=serial { + if !seen.contains(&s) { + return Err(RrdpError::DeltaRefChainNotContiguous { + min_serial, + notification_serial: serial, + missing_serial: s, + }); + } + } + deltas.sort_by_key(|d| d.serial); + } + + Ok(Notification { + session_id, + serial, + snapshot_uri, + snapshot_hash_sha256, + deltas, + }) +} + +pub fn parse_notification_snapshot(xml: &[u8]) -> Result { + let n = parse_notification(xml)?; + Ok(NotificationSnapshot { + session_id: n.session_id, + serial: n.serial, + snapshot_uri: n.snapshot_uri, + snapshot_hash_sha256: n.snapshot_hash_sha256, + }) +} + +pub fn parse_delta_file(xml: &[u8]) -> Result { + let doc = parse_rrdp_xml(xml)?; + let root = doc.root_element(); + if root.tag_name().name() != "delta" { + return Err(RrdpError::UnexpectedRoot( + root.tag_name().name().to_string(), + )); + } + validate_root_common(&root)?; + + let session_id = parse_uuid_attr(&root, "session_id")?; + let serial = parse_u64_attr(&root, "serial")?; + + let mut elements: Vec = Vec::new(); + for child in root.children().filter(|n| n.is_element()) { + match child.tag_name().name() { + "publish" => { + let uri = child + .attribute("uri") + .ok_or(RrdpError::DeltaPublishUriMissing)? + .to_string(); + let hash_sha256 = child + .attribute("hash") + .map(parse_sha256_hex_delta_publish) + .transpose()?; + + let content_b64 = + collect_element_text(&child).ok_or(RrdpError::DeltaPublishContentMissing)?; + let content_b64 = strip_all_ascii_whitespace(&content_b64); + if content_b64.is_empty() { + return Err(RrdpError::DeltaPublishContentMissing); + } + let bytes = base64::engine::general_purpose::STANDARD + .decode(content_b64.as_bytes()) + .map_err(|e| RrdpError::DeltaPublishBase64(e.to_string()))?; + + elements.push(DeltaElement::Publish { + uri, + hash_sha256, + bytes, + }); + } + "withdraw" => { + let uri = child + .attribute("uri") + .ok_or(RrdpError::DeltaWithdrawUriMissing)? + .to_string(); + let hash = child + .attribute("hash") + .ok_or(RrdpError::DeltaWithdrawHashMissing)?; + let hash_sha256 = parse_sha256_hex_delta_withdraw(hash)?; + + if let Some(s) = collect_element_text(&child) { + if !strip_all_ascii_whitespace(&s).is_empty() { + return Err(RrdpError::DeltaWithdrawUnexpectedContent); + } + } + + elements.push(DeltaElement::Withdraw { uri, hash_sha256 }); + } + other => return Err(RrdpError::DeltaUnexpectedElement(other.to_string())), + } + } + + if elements.is_empty() { + return Err(RrdpError::DeltaNoElements); + } + + Ok(DeltaFile { + session_id, + serial, + elements, + }) +} + +pub fn sync_from_notification_snapshot( + store: &RocksStore, + notification_uri: &str, + notification_xml: &[u8], + fetcher: &dyn Fetcher, +) -> RrdpSyncResult { + sync_from_notification_snapshot_inner( + store, + notification_uri, + notification_xml, + fetcher, + None, + None, + ) +} + +pub fn sync_from_notification_snapshot_with_timing( + store: &RocksStore, + notification_uri: &str, + notification_xml: &[u8], + fetcher: &dyn Fetcher, + timing: Option<&TimingHandle>, +) -> RrdpSyncResult { + sync_from_notification_snapshot_inner( + store, + notification_uri, + notification_xml, + fetcher, + timing, + None, + ) +} + +pub fn sync_from_notification_snapshot_with_timing_and_download_log( + store: &RocksStore, + notification_uri: &str, + notification_xml: &[u8], + fetcher: &dyn Fetcher, + timing: Option<&TimingHandle>, + download_log: Option<&DownloadLogHandle>, +) -> RrdpSyncResult { + sync_from_notification_snapshot_inner( + store, + notification_uri, + notification_xml, + fetcher, + timing, + download_log, + ) +} + +fn sync_from_notification_snapshot_inner( + store: &RocksStore, + notification_uri: &str, + notification_xml: &[u8], + fetcher: &dyn Fetcher, + timing: Option<&TimingHandle>, + download_log: Option<&DownloadLogHandle>, +) -> RrdpSyncResult { + let _parse_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "parse_notification_snapshot")); + let _parse_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_parse_notification_total")); + let notif = parse_notification_snapshot(notification_xml)?; + drop(_parse_step); + drop(_parse_total); + + let _fetch_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "fetch_snapshot")); + let _fetch_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_fetch_snapshot_total")); + let mut dl_span = download_log + .map(|dl| dl.span_download(AuditDownloadKind::RrdpSnapshot, ¬if.snapshot_uri)); + let (snapshot_file, _snapshot_bytes) = match fetch_snapshot_into_tempfile( + fetcher, + ¬if.snapshot_uri, + ¬if.snapshot_hash_sha256, + ) { + Ok(v) => { + if let Some(t) = timing.as_ref() { + t.record_count("rrdp_snapshot_fetch_ok_total", 1); + t.record_count("rrdp_snapshot_bytes_total", v.1); + } + if let Some(s) = dl_span.as_mut() { + s.set_bytes(v.1); + s.set_ok(); + } + v + } + Err(RrdpSyncError::Fetch(e)) => { + if let Some(t) = timing.as_ref() { + t.record_count("rrdp_snapshot_fetch_fail_total", 1); + } + if let Some(s) = dl_span.as_mut() { + s.set_err(e.clone()); + } + return Err(RrdpSyncError::Fetch(e)); + } + Err(e) => return Err(e), + }; + drop(_fetch_step); + drop(_fetch_total); + + let _apply_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "apply_snapshot")); + let _apply_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_apply_snapshot_total")); + let published = apply_snapshot_from_bufread( + store, + notification_uri, + None, + std::io::BufReader::new( + snapshot_file + .reopen() + .map_err(|e| RrdpSyncError::Fetch(format!("tempfile reopen failed: {e}")))?, + ), + notif.session_id, + notif.serial, + )?; + if let Some(t) = timing.as_ref() { + t.record_count("rrdp_snapshot_objects_applied_total", published as u64); + } + drop(_apply_step); + drop(_apply_total); + + let _write_state_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "write_state")); + let _write_state_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_write_state_total")); + let state = RrdpState { + session_id: notif.session_id.to_string(), + serial: notif.serial, + }; + persist_rrdp_local_state( + store, + notification_uri, + &state, + RrdpSourceSyncState::SnapshotOnly, + Some(¬if.snapshot_uri), + Some(&hex::encode(notif.snapshot_hash_sha256)), + ) + .map_err(RrdpSyncError::Storage)?; + drop(_write_state_step); + drop(_write_state_total); + + Ok(published) +} + +pub fn sync_from_notification( + store: &RocksStore, + notification_uri: &str, + notification_xml: &[u8], + fetcher: &dyn Fetcher, +) -> RrdpSyncResult { + sync_from_notification_inner( + store, + notification_uri, + None, + notification_xml, + fetcher, + None, + None, + ) +} + +pub fn sync_from_notification_with_timing( + store: &RocksStore, + notification_uri: &str, + notification_xml: &[u8], + fetcher: &dyn Fetcher, + timing: Option<&TimingHandle>, +) -> RrdpSyncResult { + sync_from_notification_inner( + store, + notification_uri, + None, + notification_xml, + fetcher, + timing, + None, + ) +} + +pub fn sync_from_notification_with_timing_and_download_log( + store: &RocksStore, + notification_uri: &str, + current_repo_index: Option<&CurrentRepoIndexHandle>, + notification_xml: &[u8], + fetcher: &dyn Fetcher, + timing: Option<&TimingHandle>, + download_log: Option<&DownloadLogHandle>, +) -> RrdpSyncResult { + sync_from_notification_inner( + store, + notification_uri, + current_repo_index, + notification_xml, + fetcher, + timing, + download_log, + ) +} + +fn sync_from_notification_inner( + store: &RocksStore, + notification_uri: &str, + current_repo_index: Option<&CurrentRepoIndexHandle>, + notification_xml: &[u8], + fetcher: &dyn Fetcher, + timing: Option<&TimingHandle>, + download_log: Option<&DownloadLogHandle>, +) -> RrdpSyncResult { + let _parse_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "parse_notification")); + let _parse_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_parse_notification_total")); + let notif = parse_notification(notification_xml)?; + drop(_parse_step); + drop(_parse_total); + if let Some(t) = timing.as_ref() { + t.record_count( + "rrdp_notification_delta_refs_total", + notif.deltas.len() as u64, + ); + } + + let _read_state_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "read_state")); + let _read_state_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_read_state_total")); + let state = load_rrdp_local_state(store, notification_uri).map_err(RrdpSyncError::Storage)?; + drop(_read_state_step); + drop(_read_state_total); + + let same_session_state = state + .as_ref() + .filter(|s| s.session_id == notif.session_id.to_string()); + + if let Some(s) = same_session_state { + if s.serial == notif.serial { + let _hydrate_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "hydrate_current_index")); + let _hydrate_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_hydrate_current_index_total")); + let hydrated = hydrate_current_repo_index_from_rrdp_members( + store, + notification_uri, + current_repo_index, + )?; + drop(_hydrate_step); + drop(_hydrate_total); + if let Some(t) = timing.as_ref() { + t.record_count("rrdp_current_index_hydrated_objects_total", hydrated as u64); + } + return Ok(0); + } + if s.serial > notif.serial { + return Err(RrdpError::NotificationSerialRollback { + old: s.serial, + new: notif.serial, + } + .into()); + } + } + + if let Some(s) = same_session_state { + // RFC 8182 §3.4.1: if session matches, MAY use deltas when a contiguous chain from the + // last processed serial to the current serial can be processed (i.e. deltas cover the gap). + let want_first = s.serial + 1; + let want_last = notif.serial; + + if want_first <= want_last && !notif.deltas.is_empty() { + let min_serial = notif.deltas[0].serial; + let max_serial = notif.deltas[notif.deltas.len() - 1].serial; + + // `parse_notification` guarantees contiguity and max==notif.serial when deltas exist. + if max_serial == notif.serial && want_first >= min_serial { + // Fetch all required delta files first so a network failure doesn't leave us with + // partially applied deltas and no snapshot fallback. + let _fetch_d_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "fetch_deltas")); + let _fetch_d_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_fetch_deltas_total")); + let mut fetched: Vec<(u64, [u8; 32], Vec)> = + Vec::with_capacity((want_last - want_first + 1) as usize); + let mut fetch_ok = true; + for serial in want_first..=want_last { + if let Some(t) = timing.as_ref() { + t.record_count("rrdp_delta_fetch_attempted_total", 1); + } + let idx = (serial - min_serial) as usize; + let dref = match notif.deltas.get(idx) { + Some(v) if v.serial == serial => v, + _ => { + fetch_ok = false; + break; + } + }; + + let mut dl_span = download_log + .map(|dl| dl.span_download(AuditDownloadKind::RrdpDelta, &dref.uri)); + match fetcher.fetch(&dref.uri) { + Ok(bytes) => { + if let Some(t) = timing.as_ref() { + t.record_count("rrdp_delta_fetch_ok_total", 1); + t.record_count("rrdp_delta_bytes_total", bytes.len() as u64); + } + if let Some(s) = dl_span.as_mut() { + s.set_bytes(bytes.len() as u64); + s.set_ok(); + } + fetched.push((serial, dref.hash_sha256, bytes)) + } + Err(e) => { + if let Some(t) = timing.as_ref() { + t.record_count("rrdp_delta_fetch_fail_total", 1); + } + if let Some(s) = dl_span.as_mut() { + s.set_err(e.clone()); + } + fetch_ok = false; + break; + } + } + } + drop(_fetch_d_step); + drop(_fetch_d_total); + + if fetch_ok { + let _apply_d_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "apply_deltas")); + let _apply_d_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_apply_deltas_total")); + let mut applied_total = 0usize; + let mut ok = true; + for (serial, expected_hash, bytes) in &fetched { + match apply_delta( + store, + notification_uri, + current_repo_index, + bytes.as_slice(), + *expected_hash, + notif.session_id, + *serial, + ) { + Ok(n) => applied_total += n, + Err(_) => { + ok = false; + break; + } + } + } + drop(_apply_d_step); + drop(_apply_d_total); + + if ok { + let _write_state_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "write_state")); + let _write_state_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_write_state_total")); + let new_state = RrdpState { + session_id: notif.session_id.to_string(), + serial: notif.serial, + }; + persist_rrdp_local_state( + store, + notification_uri, + &new_state, + RrdpSourceSyncState::DeltaReady, + Some(¬if.snapshot_uri), + Some(&hex::encode(notif.snapshot_hash_sha256)), + ) + .map_err(RrdpSyncError::Storage)?; + drop(_write_state_step); + drop(_write_state_total); + if let Some(t) = timing.as_ref() { + t.record_count("rrdp_delta_ops_applied_total", applied_total as u64); + } + let _hydrate_step = timing.as_ref().map(|t| { + t.span_rrdp_repo_step(notification_uri, "hydrate_current_index") + }); + let _hydrate_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_hydrate_current_index_total")); + let hydrated = hydrate_current_repo_index_from_rrdp_members( + store, + notification_uri, + current_repo_index, + )?; + drop(_hydrate_step); + drop(_hydrate_total); + if let Some(t) = timing.as_ref() { + t.record_count( + "rrdp_current_index_hydrated_objects_total", + hydrated as u64, + ); + } + return Ok(applied_total); + } + } + } + } + } + + // Snapshot fallback (RFC 8182 §3.4.3). + let _fetch_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "fetch_snapshot")); + let _fetch_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_fetch_snapshot_total")); + let mut dl_span = download_log + .map(|dl| dl.span_download(AuditDownloadKind::RrdpSnapshot, ¬if.snapshot_uri)); + let (snapshot_file, _snapshot_bytes) = match fetch_snapshot_into_tempfile( + fetcher, + ¬if.snapshot_uri, + ¬if.snapshot_hash_sha256, + ) { + Ok(v) => { + if let Some(t) = timing.as_ref() { + t.record_count("rrdp_snapshot_fetch_ok_total", 1); + t.record_count("rrdp_snapshot_bytes_total", v.1); + } + if let Some(s) = dl_span.as_mut() { + s.set_bytes(v.1); + s.set_ok(); + } + v + } + Err(RrdpSyncError::Fetch(e)) => { + if let Some(t) = timing.as_ref() { + t.record_count("rrdp_snapshot_fetch_fail_total", 1); + } + if let Some(s) = dl_span.as_mut() { + s.set_err(e.clone()); + } + return Err(RrdpSyncError::Fetch(e)); + } + Err(e) => return Err(e), + }; + drop(_fetch_step); + drop(_fetch_total); + + let _apply_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "apply_snapshot")); + let _apply_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_apply_snapshot_total")); + let published = apply_snapshot_from_bufread( + store, + notification_uri, + current_repo_index, + std::io::BufReader::new( + snapshot_file + .reopen() + .map_err(|e| RrdpSyncError::Fetch(format!("tempfile reopen failed: {e}")))?, + ), + notif.session_id, + notif.serial, + )?; + if let Some(t) = timing.as_ref() { + t.record_count("rrdp_snapshot_objects_applied_total", published as u64); + } + drop(_apply_step); + drop(_apply_total); + + let _write_state_step = timing + .as_ref() + .map(|t| t.span_rrdp_repo_step(notification_uri, "write_state")); + let _write_state_total = timing + .as_ref() + .map(|t| t.span_phase("rrdp_write_state_total")); + let new_state = RrdpState { + session_id: notif.session_id.to_string(), + serial: notif.serial, + }; + persist_rrdp_local_state( + store, + notification_uri, + &new_state, + RrdpSourceSyncState::SnapshotOnly, + Some(¬if.snapshot_uri), + Some(&hex::encode(notif.snapshot_hash_sha256)), + ) + .map_err(RrdpSyncError::Storage)?; + drop(_write_state_step); + drop(_write_state_total); + + Ok(published) +} + +fn hydrate_current_repo_index_from_rrdp_members( + store: &RocksStore, + notification_uri: &str, + current_repo_index: Option<&CurrentRepoIndexHandle>, +) -> Result { + let Some(index) = current_repo_index else { + return Ok(0); + }; + + let members = store + .list_current_rrdp_source_members(notification_uri) + .map_err(|e| RrdpSyncError::Storage(e.to_string()))?; + if members.is_empty() { + return Ok(0); + } + + let mut entries = Vec::with_capacity(members.len()); + for member in members { + let current_hash = member.current_hash.ok_or_else(|| { + RrdpSyncError::Storage(format!( + "rrdp source member missing current_hash for current object {}", + member.rsync_uri + )) + })?; + entries.push(RepositoryViewEntry { + rsync_uri: member.rsync_uri, + current_hash: Some(current_hash), + repository_source: Some(notification_uri.to_string()), + object_type: member.object_type, + state: RepositoryViewState::Present, + }); + } + + index + .write() + .map_err(|_| RrdpSyncError::Storage("current repo index lock poisoned".to_string()))? + .apply_repository_view_entries(&entries) + .map_err(RrdpSyncError::Storage)?; + Ok(entries.len()) +} + +fn apply_delta( + store: &RocksStore, + notification_uri: &str, + current_repo_index: Option<&CurrentRepoIndexHandle>, + delta_xml: &[u8], + expected_hash_sha256: [u8; 32], + expected_session_id: Uuid, + expected_serial: u64, +) -> Result { + let computed = sha2::Sha256::digest(delta_xml); + if computed.as_slice() != expected_hash_sha256.as_slice() { + return Err(RrdpError::DeltaHashMismatch.into()); + } + + let delta = parse_delta_file(delta_xml)?; + if delta.session_id != expected_session_id { + return Err(RrdpError::DeltaSessionIdMismatch { + expected: expected_session_id.to_string(), + got: delta.session_id.to_string(), + } + .into()); + } + if delta.serial != expected_serial { + return Err(RrdpError::DeltaSerialMismatch { + expected: expected_serial, + got: delta.serial, + } + .into()); + } + + enum DeltaProjectionEffect { + Upsert { + rsync_uri: String, + bytes: Vec, + }, + Delete { + rsync_uri: String, + previous_hash: String, + }, + } + + let session_id = expected_session_id.to_string(); + let mut ops: Vec = Vec::with_capacity(delta.elements.len()); + let mut projection: Vec = Vec::with_capacity(delta.elements.len()); + for e in delta.elements { + match e { + DeltaElement::Publish { + uri, + hash_sha256: Some(old_hash), + bytes, + } => { + let is_member = store + .is_current_rrdp_source_member(notification_uri, uri.as_str()) + .map_err(|e| RrdpSyncError::Storage(e.to_string()))?; + if !is_member { + return Err(RrdpError::DeltaTargetNotFromRepository { rsync_uri: uri }.into()); + } + ensure_rrdp_uri_can_be_owned_by(store, notification_uri, uri.as_str()) + .map_err(RrdpSyncError::Storage)?; + let old_bytes = store + .load_current_object_bytes_by_uri(uri.as_str()) + .map_err(|e| RrdpSyncError::Storage(e.to_string()))? + .ok_or_else(|| RrdpError::DeltaTargetMissing { + rsync_uri: uri.clone(), + })?; + let old_computed = sha2::Sha256::digest(old_bytes.as_slice()); + if old_computed.as_slice() != old_hash.as_slice() { + return Err(RrdpError::DeltaTargetHashMismatch { rsync_uri: uri }.into()); + } + + ops.push(RrdpDeltaOp::Upsert { + rsync_uri: uri.clone(), + bytes: bytes.clone(), + }); + projection.push(DeltaProjectionEffect::Upsert { + rsync_uri: uri, + bytes, + }); + } + DeltaElement::Publish { + uri, + hash_sha256: None, + bytes, + } => { + let is_member = store + .is_current_rrdp_source_member(notification_uri, uri.as_str()) + .map_err(|e| RrdpSyncError::Storage(e.to_string()))?; + if is_member { + return Err( + RrdpError::DeltaPublishWithoutHashForExisting { rsync_uri: uri }.into(), + ); + } + ensure_rrdp_uri_can_be_owned_by(store, notification_uri, uri.as_str()) + .map_err(RrdpSyncError::Storage)?; + ops.push(RrdpDeltaOp::Upsert { + rsync_uri: uri.clone(), + bytes: bytes.clone(), + }); + projection.push(DeltaProjectionEffect::Upsert { + rsync_uri: uri, + bytes, + }); + } + DeltaElement::Withdraw { uri, hash_sha256 } => { + let is_member = store + .is_current_rrdp_source_member(notification_uri, uri.as_str()) + .map_err(|e| RrdpSyncError::Storage(e.to_string()))?; + if !is_member { + return Err(RrdpError::DeltaTargetNotFromRepository { rsync_uri: uri }.into()); + } + ensure_rrdp_uri_can_be_owned_by(store, notification_uri, uri.as_str()) + .map_err(RrdpSyncError::Storage)?; + let old_bytes = store + .load_current_object_bytes_by_uri(uri.as_str()) + .map_err(|e| RrdpSyncError::Storage(e.to_string()))? + .ok_or_else(|| RrdpError::DeltaTargetMissing { + rsync_uri: uri.clone(), + })?; + let old_computed = sha2::Sha256::digest(old_bytes.as_slice()); + if old_computed.as_slice() != hash_sha256.as_slice() { + return Err(RrdpError::DeltaTargetHashMismatch { rsync_uri: uri }.into()); + } + let previous_hash = hex::encode(old_computed); + ops.push(RrdpDeltaOp::Delete { + rsync_uri: uri.clone(), + }); + projection.push(DeltaProjectionEffect::Delete { + rsync_uri: uri, + previous_hash, + }); + } + } + } + + for effect in projection { + match effect { + DeltaProjectionEffect::Upsert { rsync_uri, bytes } => { + let current_hash = + upsert_repo_blob_bytes(store, &bytes).map_err(RrdpSyncError::Storage)?; + put_repository_view_present(store, notification_uri, &rsync_uri, ¤t_hash) + .map_err(RrdpSyncError::Storage)?; + if let Some(index) = current_repo_index { + let entry = build_repository_view_present_entry( + notification_uri, + &rsync_uri, + ¤t_hash, + ); + index + .write() + .map_err(|_| { + RrdpSyncError::Storage("current repo index lock poisoned".to_string()) + })? + .apply_repository_view_entries(&[entry]) + .map_err(RrdpSyncError::Storage)?; + } + put_rrdp_source_member_present( + store, + notification_uri, + &session_id, + expected_serial, + &rsync_uri, + ¤t_hash, + ) + .map_err(RrdpSyncError::Storage)?; + put_rrdp_uri_owner_active( + store, + notification_uri, + &session_id, + expected_serial, + &rsync_uri, + ¤t_hash, + ) + .map_err(RrdpSyncError::Storage)?; + } + DeltaProjectionEffect::Delete { + rsync_uri, + previous_hash, + } => { + put_rrdp_source_member_withdrawn( + store, + notification_uri, + &session_id, + expected_serial, + &rsync_uri, + Some(previous_hash.clone()), + ) + .map_err(RrdpSyncError::Storage)?; + if current_rrdp_owner_is(store, notification_uri, &rsync_uri) + .map_err(RrdpSyncError::Storage)? + { + put_repository_view_withdrawn( + store, + notification_uri, + &rsync_uri, + Some(previous_hash.clone()), + ) + .map_err(RrdpSyncError::Storage)?; + if let Some(index) = current_repo_index { + let entry = build_repository_view_withdrawn_entry( + notification_uri, + &rsync_uri, + Some(previous_hash.clone()), + ); + index + .write() + .map_err(|_| { + RrdpSyncError::Storage( + "current repo index lock poisoned".to_string(), + ) + })? + .apply_repository_view_entries(&[entry]) + .map_err(RrdpSyncError::Storage)?; + } + put_rrdp_uri_owner_withdrawn( + store, + notification_uri, + &session_id, + expected_serial, + &rsync_uri, + Some(previous_hash), + ) + .map_err(RrdpSyncError::Storage)?; + } + } + } + } + + Ok(ops.len()) +} + +#[cfg(test)] +use snapshot_apply::apply_snapshot; +use snapshot_apply::{apply_snapshot_from_bufread, fetch_snapshot_into_tempfile}; + +fn parse_rrdp_xml(xml: &[u8]) -> Result, RrdpError> { + if xml.iter().any(|&b| b > 0x7F) { + return Err(RrdpError::NotAscii); + } + let s = std::str::from_utf8(xml).map_err(|e| RrdpError::Xml(e.to_string()))?; + roxmltree::Document::parse(s).map_err(|e| RrdpError::Xml(e.to_string())) +} + +fn validate_root_common(root: &roxmltree::Node<'_, '_>) -> Result<(), RrdpError> { + let ns = root.default_namespace().unwrap_or("").to_string(); + if ns != RRDP_XMLNS { + return Err(RrdpError::InvalidNamespace(ns)); + } + + let version = root.attribute("version").unwrap_or(""); + if version != "1" { + return Err(RrdpError::InvalidVersion(version.to_string())); + } + + Ok(()) +} + +fn parse_uuid_attr(root: &roxmltree::Node<'_, '_>, name: &'static str) -> Result { + let s = root.attribute(name).unwrap_or(""); + Uuid::parse_str(s).map_err(|_e| RrdpError::InvalidSessionId(s.to_string())) +} + +fn parse_u64_attr(root: &roxmltree::Node<'_, '_>, name: &'static str) -> Result { + let s = root.attribute(name).unwrap_or(""); + parse_u64_str(s) +} + +fn parse_u64_str(s: &str) -> Result { + let v = s + .parse::() + .map_err(|_e| RrdpError::InvalidSerial(s.to_string()))?; + if v == 0 { + return Err(RrdpError::InvalidSerial(s.to_string())); + } + Ok(v) +} + +fn parse_sha256_hex_impl(s: &str, invalid: fn(String) -> RrdpError) -> Result<[u8; 32], RrdpError> { + let bytes = hex::decode(s).map_err(|_e| invalid(s.to_string()))?; + if bytes.len() != 32 { + return Err(invalid(s.to_string())); + } + let mut out = [0u8; 32]; + out.copy_from_slice(&bytes); + Ok(out) +} + +fn parse_sha256_hex_snapshot(s: &str) -> Result<[u8; 32], RrdpError> { + parse_sha256_hex_impl(s, RrdpError::SnapshotHashInvalid) +} + +fn parse_sha256_hex_delta_ref(s: &str) -> Result<[u8; 32], RrdpError> { + parse_sha256_hex_impl(s, RrdpError::DeltaRefHashInvalid) +} + +fn parse_sha256_hex_delta_publish(s: &str) -> Result<[u8; 32], RrdpError> { + parse_sha256_hex_impl(s, RrdpError::DeltaPublishHashInvalid) +} + +fn parse_sha256_hex_delta_withdraw(s: &str) -> Result<[u8; 32], RrdpError> { + parse_sha256_hex_impl(s, RrdpError::DeltaWithdrawHashInvalid) +} + +fn collect_element_text(node: &roxmltree::Node<'_, '_>) -> Option { + let mut out = String::new(); + for child in node.children() { + if child.is_text() { + out.push_str(child.text().unwrap_or("")); + } + } + if out.is_empty() { None } else { Some(out) } +} + +fn strip_all_ascii_whitespace(s: &str) -> String { + s.chars().filter(|c| !c.is_ascii_whitespace()).collect() +} + +#[cfg(test)] +#[path = "rrdp/tests.rs"] +mod tests; diff --git a/crates/panda-rpki-validator/src/sync/rrdp/snapshot_apply.rs b/crates/panda-rpki-validator/src/sync/rrdp/snapshot_apply.rs new file mode 100644 index 0000000..9360b89 --- /dev/null +++ b/crates/panda-rpki-validator/src/sync/rrdp/snapshot_apply.rs @@ -0,0 +1,445 @@ +use base64::Engine; +use quick_xml::Reader; +use quick_xml::events::Event; +use sha2::Digest; +use std::io::{BufRead, Seek, SeekFrom, Write}; +use uuid::Uuid; + +use crate::current_repo_index::CurrentRepoIndexHandle; +use crate::storage::RocksStore; +use crate::sync::store_projection::{ + build_repository_view_present_entry, build_repository_view_withdrawn_entry, + build_rrdp_source_member_present_record, build_rrdp_source_member_withdrawn_record, + build_rrdp_uri_owner_active_record, build_rrdp_uri_owner_withdrawn_record, compute_sha256_hex, + current_rrdp_owner_is, ensure_rrdp_uri_can_be_owned_by, prepare_repo_bytes_batch, +}; + +use super::{ + Fetcher, RRDP_SNAPSHOT_APPLY_BATCH_SIZE, RRDP_XMLNS, RrdpError, RrdpSyncError, parse_u64_str, + strip_all_ascii_whitespace, +}; + +#[cfg(test)] +pub(super) fn apply_snapshot( + store: &RocksStore, + notification_uri: &str, + current_repo_index: Option<&CurrentRepoIndexHandle>, + snapshot_xml: &[u8], + expected_session_id: Uuid, + expected_serial: u64, +) -> Result { + if snapshot_xml.iter().any(|&b| b > 0x7F) { + return Err(RrdpError::NotAscii.into()); + } + apply_snapshot_from_bufread( + store, + notification_uri, + current_repo_index, + std::io::Cursor::new(snapshot_xml), + expected_session_id, + expected_serial, + ) +} + +pub(super) fn apply_snapshot_from_bufread( + store: &RocksStore, + notification_uri: &str, + current_repo_index: Option<&CurrentRepoIndexHandle>, + input: R, + expected_session_id: Uuid, + expected_serial: u64, +) -> Result { + let previous_members: Vec = store + .list_current_rrdp_source_members(notification_uri) + .map_err(|e| RrdpSyncError::Storage(e.to_string()))? + .into_iter() + .map(|record| record.rsync_uri) + .collect(); + let session_id = expected_session_id.to_string(); + let mut new_set: std::collections::HashSet = std::collections::HashSet::new(); + let mut batch_published: Vec<(String, Vec)> = + Vec::with_capacity(RRDP_SNAPSHOT_APPLY_BATCH_SIZE); + let mut published_count = 0usize; + + let mut reader = Reader::from_reader(input); + reader.config_mut().trim_text(false); + let mut buf = Vec::new(); + let mut root_seen = false; + let mut in_publish = false; + let mut publish_nested_depth = 0usize; + let mut current_publish_uri: Option = None; + let mut current_publish_text = String::new(); + + loop { + match reader.read_event_into(&mut buf) { + Ok(Event::Start(e)) => { + let local_name = e.local_name(); + let local_name = local_name.as_ref(); + if !root_seen { + root_seen = true; + if local_name != b"snapshot" { + let got = String::from_utf8_lossy(local_name).to_string(); + return Err(RrdpError::UnexpectedRoot(got).into()); + } + let mut xmlns = String::new(); + let mut version = String::new(); + let mut session_id_attr = String::new(); + let mut serial_attr = String::new(); + for attr in e.attributes().with_checks(false) { + let attr = match attr { + Ok(attr) => attr, + Err(e) => return Err(RrdpError::Xml(e.to_string()).into()), + }; + let key = attr.key.as_ref(); + let value = attr + .decode_and_unescape_value(reader.decoder()) + .map_err(|e| RrdpError::Xml(e.to_string()))? + .into_owned(); + match key { + b"xmlns" => xmlns = value, + b"version" => version = value, + b"session_id" => session_id_attr = value, + b"serial" => serial_attr = value, + _ => {} + } + } + if xmlns != RRDP_XMLNS { + return Err(RrdpError::InvalidNamespace(xmlns).into()); + } + if version != "1" { + return Err(RrdpError::InvalidVersion(version).into()); + } + let got_session_id = Uuid::parse_str(&session_id_attr) + .map_err(|_| RrdpError::InvalidSessionId(session_id_attr.clone()))?; + if got_session_id != expected_session_id { + return Err(RrdpError::SnapshotSessionIdMismatch { + expected: expected_session_id.to_string(), + got: got_session_id.to_string(), + } + .into()); + } + let got_serial = parse_u64_str(&serial_attr)?; + if got_serial != expected_serial { + return Err(RrdpError::SnapshotSerialMismatch { + expected: expected_serial, + got: got_serial, + } + .into()); + } + } else if in_publish { + publish_nested_depth += 1; + } else if local_name == b"publish" { + let mut uri = None; + for attr in e.attributes().with_checks(false) { + let attr = match attr { + Ok(attr) => attr, + Err(e) => return Err(RrdpError::Xml(e.to_string()).into()), + }; + if attr.key.as_ref() == b"uri" { + uri = Some( + attr.decode_and_unescape_value(reader.decoder()) + .map_err(|e| RrdpError::Xml(e.to_string()))? + .into_owned(), + ); + } + } + let uri = uri.ok_or(RrdpError::PublishUriMissing)?; + ensure_rrdp_uri_can_be_owned_by(store, notification_uri, &uri) + .map_err(RrdpSyncError::Storage)?; + in_publish = true; + publish_nested_depth = 0; + current_publish_uri = Some(uri); + current_publish_text.clear(); + } + } + Ok(Event::Empty(e)) => { + let local_name = e.local_name(); + let local_name = local_name.as_ref(); + if !root_seen { + let got = String::from_utf8_lossy(local_name).to_string(); + return Err(RrdpError::UnexpectedRoot(got).into()); + } + if local_name == b"publish" { + let mut has_uri = false; + for attr in e.attributes().with_checks(false) { + let attr = match attr { + Ok(attr) => attr, + Err(e) => return Err(RrdpError::Xml(e.to_string()).into()), + }; + if attr.key.as_ref() == b"uri" { + has_uri = true; + break; + } + } + if !has_uri { + return Err(RrdpError::PublishUriMissing.into()); + } + return Err(RrdpError::PublishContentMissing.into()); + } + } + Ok(Event::Text(e)) => { + if in_publish && publish_nested_depth == 0 { + let text = reader + .decoder() + .decode(e.as_ref()) + .map_err(|e| RrdpError::Xml(e.to_string()))?; + current_publish_text.push_str(&text); + } + } + Ok(Event::CData(e)) => { + if in_publish && publish_nested_depth == 0 { + let text = reader + .decoder() + .decode(e.as_ref()) + .map_err(|e| RrdpError::Xml(e.to_string()))?; + current_publish_text.push_str(&text); + } + } + Ok(Event::End(e)) => { + let local_name = e.local_name(); + let local_name = local_name.as_ref(); + if in_publish { + if publish_nested_depth > 0 { + publish_nested_depth -= 1; + } else if local_name == b"publish" { + let uri = current_publish_uri + .take() + .ok_or_else(|| RrdpError::Xml("publish uri missing in state".into()))?; + let content_b64 = strip_all_ascii_whitespace(¤t_publish_text); + current_publish_text.clear(); + if content_b64.is_empty() { + return Err(RrdpError::PublishContentMissing.into()); + } + let bytes = base64::engine::general_purpose::STANDARD + .decode(content_b64.as_bytes()) + .map_err(|e| RrdpError::PublishBase64(e.to_string()))?; + new_set.insert(uri.clone()); + batch_published.push((uri, bytes)); + published_count += 1; + if batch_published.len() >= RRDP_SNAPSHOT_APPLY_BATCH_SIZE { + flush_snapshot_publish_batch( + store, + notification_uri, + current_repo_index, + &session_id, + expected_serial, + &batch_published, + )?; + batch_published.clear(); + } + in_publish = false; + } + } + } + Ok(Event::Eof) => break, + Ok(Event::Decl(_) | Event::PI(_) | Event::Comment(_) | Event::DocType(_)) => {} + Err(e) => return Err(RrdpError::Xml(e.to_string()).into()), + } + buf.clear(); + } + + if !root_seen { + return Err(RrdpError::Xml("missing root element".to_string()).into()); + } + if in_publish { + return Err(RrdpError::PublishContentMissing.into()); + } + if !batch_published.is_empty() { + flush_snapshot_publish_batch( + store, + notification_uri, + current_repo_index, + &session_id, + expected_serial, + &batch_published, + )?; + batch_published.clear(); + } + + let mut withdrawn: Vec<(String, Option)> = Vec::new(); + for old_uri in &previous_members { + if new_set.contains(old_uri) { + continue; + } + let previous_hash = store + .get_repository_view_entry(old_uri) + .map_err(|e| RrdpSyncError::Storage(e.to_string()))? + .and_then(|entry| entry.current_hash) + .or_else(|| { + store + .load_current_object_bytes_by_uri(old_uri) + .ok() + .flatten() + .map(|bytes| compute_sha256_hex(&bytes)) + }); + withdrawn.push((old_uri.clone(), previous_hash)); + } + + let mut repository_view_entries = Vec::with_capacity(withdrawn.len()); + let mut member_records = Vec::with_capacity(withdrawn.len()); + let mut owner_records = Vec::with_capacity(withdrawn.len()); + for (uri, previous_hash) in withdrawn { + member_records.push(build_rrdp_source_member_withdrawn_record( + notification_uri, + &session_id, + expected_serial, + &uri, + previous_hash.clone(), + )); + if current_rrdp_owner_is(store, notification_uri, &uri).map_err(RrdpSyncError::Storage)? { + repository_view_entries.push(build_repository_view_withdrawn_entry( + notification_uri, + &uri, + previous_hash.clone(), + )); + owner_records.push(build_rrdp_uri_owner_withdrawn_record( + notification_uri, + &session_id, + expected_serial, + &uri, + previous_hash, + )); + } + } + store + .put_projection_batch(&repository_view_entries, &member_records, &owner_records) + .map_err(|e| RrdpSyncError::Storage(e.to_string()))?; + if let Some(index) = current_repo_index { + index + .write() + .map_err(|_| RrdpSyncError::Storage("current repo index lock poisoned".to_string()))? + .apply_repository_view_entries(&repository_view_entries) + .map_err(RrdpSyncError::Storage)?; + } + + Ok(published_count) +} + +fn flush_snapshot_publish_batch( + store: &RocksStore, + notification_uri: &str, + current_repo_index: Option<&CurrentRepoIndexHandle>, + session_id: &str, + serial: u64, + published: &[(String, Vec)], +) -> Result<(), RrdpSyncError> { + let prepared_bytes = prepare_repo_bytes_batch(published).map_err(RrdpSyncError::Storage)?; + let mut repository_view_entries = Vec::with_capacity(published.len()); + let mut member_records = Vec::with_capacity(published.len()); + let mut owner_records = Vec::with_capacity(published.len()); + + for (uri, _bytes) in published { + let current_hash = prepared_bytes + .uri_to_hash + .get(uri) + .cloned() + .ok_or_else(|| { + RrdpSyncError::Storage(format!("missing raw_by_hash mapping for {uri}")) + })?; + repository_view_entries.push(build_repository_view_present_entry( + notification_uri, + uri, + ¤t_hash, + )); + member_records.push(build_rrdp_source_member_present_record( + notification_uri, + session_id, + serial, + uri, + ¤t_hash, + )); + owner_records.push(build_rrdp_uri_owner_active_record( + notification_uri, + session_id, + serial, + uri, + ¤t_hash, + )); + } + + store + .put_blob_bytes_batch(&prepared_bytes.blobs_to_write) + .map_err(|e| RrdpSyncError::Storage(e.to_string()))?; + store + .put_projection_batch(&repository_view_entries, &member_records, &owner_records) + .map_err(|e| RrdpSyncError::Storage(e.to_string()))?; + if let Some(index) = current_repo_index { + index + .write() + .map_err(|_| RrdpSyncError::Storage("current repo index lock poisoned".to_string()))? + .apply_repository_view_entries(&repository_view_entries) + .map_err(RrdpSyncError::Storage)?; + } + + Ok(()) +} + +const SNAPSHOT_NON_ASCII_ERROR: &str = "snapshot body contains non-ASCII bytes"; + +struct SnapshotSpoolWriter<'a, W: Write> { + inner: &'a mut W, + hasher: sha2::Sha256, + bytes: u64, +} + +impl<'a, W: Write> SnapshotSpoolWriter<'a, W> { + fn new(inner: &'a mut W) -> Self { + Self { + inner, + hasher: sha2::Sha256::new(), + bytes: 0, + } + } + + fn finalize_hash(self) -> [u8; 32] { + let digest = self.hasher.finalize(); + let mut out = [0u8; 32]; + out.copy_from_slice(&digest); + out + } +} + +impl Write for SnapshotSpoolWriter<'_, W> { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + if buf.iter().any(|&b| b > 0x7F) { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + SNAPSHOT_NON_ASCII_ERROR, + )); + } + let n = self.inner.write(buf)?; + self.hasher.update(&buf[..n]); + self.bytes += n as u64; + Ok(n) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.inner.flush() + } +} + +pub(super) fn fetch_snapshot_into_tempfile( + fetcher: &dyn Fetcher, + snapshot_uri: &str, + expected_hash_sha256: &[u8; 32], +) -> Result<(tempfile::NamedTempFile, u64), RrdpSyncError> { + let mut tmp = tempfile::NamedTempFile::new() + .map_err(|e| RrdpSyncError::Fetch(format!("tempfile create failed: {e}")))?; + let mut spool = SnapshotSpoolWriter::new(tmp.as_file_mut()); + let bytes_written = match fetcher.fetch_to_writer(snapshot_uri, &mut spool) { + Ok(bytes) => bytes, + Err(e) if e.contains(SNAPSHOT_NON_ASCII_ERROR) => return Err(RrdpError::NotAscii.into()), + Err(e) => return Err(RrdpSyncError::Fetch(e)), + }; + let computed = spool.finalize_hash(); + if computed.as_slice() != expected_hash_sha256.as_slice() { + return Err(RrdpError::SnapshotHashMismatch.into()); + } + tmp.as_file_mut() + .flush() + .map_err(|e| RrdpSyncError::Fetch(format!("tempfile flush failed: {e}")))?; + tmp.as_file_mut() + .seek(SeekFrom::Start(0)) + .map_err(|e| RrdpSyncError::Fetch(format!("tempfile rewind failed: {e}")))?; + Ok((tmp, bytes_written)) +} diff --git a/crates/panda-rpki-validator/src/sync/rrdp/tests.rs b/crates/panda-rpki-validator/src/sync/rrdp/tests.rs new file mode 100644 index 0000000..39a4c85 --- /dev/null +++ b/crates/panda-rpki-validator/src/sync/rrdp/tests.rs @@ -0,0 +1,1316 @@ +use super::*; +use crate::analysis::timing::{TimingHandle, TimingMeta}; +use crate::current_repo_index::CurrentRepoIndex; +use crate::storage::RocksStore; +use std::collections::HashMap; +use std::io::Read; +use std::time::Duration; + +struct MapFetcher { + map: HashMap>, +} + +impl Fetcher for MapFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + self.map + .get(uri) + .cloned() + .ok_or_else(|| format!("not found: {uri}")) + } +} + +struct SleepyFetcher { + inner: MapFetcher, + sleep_uri: String, + sleep: Duration, +} + +impl Fetcher for SleepyFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + if uri == self.sleep_uri { + std::thread::sleep(self.sleep); + } + self.inner.fetch(uri) + } +} + +struct WriterOnlyFetcher { + map: HashMap>, +} + +impl Fetcher for WriterOnlyFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + Err(format!("unexpected buffered fetch: {uri}")) + } + + fn fetch_to_writer(&self, uri: &str, out: &mut dyn std::io::Write) -> Result { + let bytes = self + .map + .get(uri) + .ok_or_else(|| format!("not found: {uri}"))?; + out.write_all(bytes) + .map_err(|e| format!("write sink failed: {e}"))?; + Ok(bytes.len() as u64) + } +} + +struct NonAsciiWriterFetcher; + +impl Fetcher for NonAsciiWriterFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + Err(format!("unexpected buffered fetch: {uri}")) + } + + fn fetch_to_writer(&self, _uri: &str, out: &mut dyn std::io::Write) -> Result { + out.write_all(&[0x80]) + .map_err(|e| format!("write sink failed: {e}"))?; + Err("snapshot body contains non-ASCII bytes".to_string()) + } +} + +fn assert_current_object(store: &RocksStore, uri: &str, expected: &[u8]) { + assert_eq!( + store + .load_current_object_bytes_by_uri(uri) + .expect("load current object"), + Some(expected.to_vec()) + ); +} + +fn notification_xml( + session_id: &str, + serial: u64, + snapshot_uri: &str, + snapshot_hash: &str, +) -> Vec { + format!( + r#""# + ) + .into_bytes() +} + +fn notification_xml_with_deltas( + session_id: &str, + serial: u64, + snapshot_uri: &str, + snapshot_hash: &str, + deltas: &[(&str, u64, &str, &str)], +) -> Vec { + let mut out = format!( + r#""# + ); + for (_name, delta_serial, uri, hash) in deltas { + out.push_str(&format!( + r#""# + )); + } + out.push_str(""); + out.into_bytes() +} + +fn snapshot_xml(session_id: &str, serial: u64, published: &[(&str, &[u8])]) -> Vec { + let mut out = format!( + r#""# + ); + for (uri, bytes) in published { + let b64 = base64::engine::general_purpose::STANDARD.encode(bytes); + out.push_str(&format!(r#"{b64}"#)); + } + out.push_str(""); + out.into_bytes() +} + +#[test] +fn fetch_snapshot_into_tempfile_streams_and_validates_hash() { + let snapshot_uri = "https://example.test/snapshot.xml"; + let snapshot = b"".to_vec(); + let mut expected_hash = [0u8; 32]; + expected_hash.copy_from_slice(&sha2::Sha256::digest(&snapshot)); + let fetcher = WriterOnlyFetcher { + map: HashMap::from([(snapshot_uri.to_string(), snapshot.clone())]), + }; + + let (mut file, bytes_written) = + fetch_snapshot_into_tempfile(&fetcher, snapshot_uri, &expected_hash) + .expect("fetch snapshot into tempfile"); + assert_eq!(bytes_written, snapshot.len() as u64); + let mut got = Vec::new(); + file.as_file_mut() + .read_to_end(&mut got) + .expect("read tempfile"); + assert_eq!(got, snapshot); + + let mut wrong_hash = expected_hash; + wrong_hash[0] ^= 0xff; + let err = fetch_snapshot_into_tempfile(&fetcher, snapshot_uri, &wrong_hash).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::SnapshotHashMismatch) + )); +} + +#[test] +fn fetch_snapshot_into_tempfile_maps_stream_non_ascii_error() { + let err = fetch_snapshot_into_tempfile( + &NonAsciiWriterFetcher, + "https://example.test/snapshot.xml", + &[0u8; 32], + ) + .unwrap_err(); + assert!(matches!(err, RrdpSyncError::Rrdp(RrdpError::NotAscii))); +} + +#[test] +fn timing_rrdp_repo_step_spans_cover_snapshot_fetch_duration() { + let temp = tempfile::tempdir().expect("tempdir"); + let store_dir = temp.path().join("db"); + let store = RocksStore::open(&store_dir).expect("open rocksdb"); + + let notification_uri = "https://example.test/notification.xml"; + let snapshot_uri = "https://example.test/snapshot.xml"; + let published_uri = "rsync://example.test/repo/a.mft"; + let published_bytes = b"x"; + let session_id = "550e8400-e29b-41d4-a716-446655440000"; + + let snapshot = snapshot_xml(session_id, 1, &[(published_uri, published_bytes)]); + let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); + let notif = notification_xml(session_id, 1, snapshot_uri, &snapshot_hash); + + let mut map = HashMap::new(); + map.insert(snapshot_uri.to_string(), snapshot); + let fetcher = SleepyFetcher { + inner: MapFetcher { map }, + sleep_uri: snapshot_uri.to_string(), + sleep: Duration::from_millis(25), + }; + + let timing = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(), + tal_url: None, + db_path: Some(store_dir.to_string_lossy().into_owned()), + }); + + sync_from_notification_snapshot_with_timing( + &store, + notification_uri, + ¬if, + &fetcher, + Some(&timing), + ) + .expect("rrdp snapshot sync ok"); + + let timing_path = temp.path().join("timing.json"); + timing.write_json(&timing_path, 200).expect("write timing"); + let rep: serde_json::Value = + serde_json::from_slice(&std::fs::read(&timing_path).expect("read timing")) + .expect("parse timing"); + + let want = format!("{notification_uri}::fetch_snapshot"); + let steps = rep + .get("top_rrdp_repo_steps") + .and_then(|v| v.as_array()) + .expect("top_rrdp_repo_steps array"); + let entry = steps + .iter() + .find(|e| e.get("key").and_then(|k| k.as_str()) == Some(want.as_str())) + .unwrap_or_else(|| panic!("missing timing step entry for {want}")); + let nanos = entry + .get("total_nanos") + .and_then(|v| v.as_u64()) + .expect("total_nanos"); + assert!( + nanos >= 20_000_000, + "expected fetch_snapshot timing to include the fetch duration; got {nanos}ns" + ); +} + +#[test] +fn parse_notification_snapshot_rejects_non_ascii() { + let mut xml = b"".to_vec(); + xml.push(0x80); + let err = parse_notification_snapshot(&xml).unwrap_err(); + assert!(matches!(err, RrdpError::NotAscii)); +} + +#[test] +fn parse_notification_snapshot_parses_valid_minimal_notification() { + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let snapshot_uri = "https://example.net/snapshot.xml"; + let hash = "00".repeat(32); + let xml = notification_xml(sid, 7, snapshot_uri, &hash); + let n = parse_notification_snapshot(&xml).expect("parse"); + assert_eq!(n.session_id, Uuid::parse_str(sid).unwrap()); + assert_eq!(n.serial, 7); + assert_eq!(n.snapshot_uri, snapshot_uri); + assert_eq!(hex::encode(n.snapshot_hash_sha256), hash); +} + +#[test] +fn parse_notification_parses_deltas_and_validates_contiguity() { + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let snapshot_uri = "https://example.net/snapshot.xml"; + let hash = "00".repeat(32); + let d_hash_2 = "11".repeat(32); + let d_hash_3 = "22".repeat(32); + // Provide deltas in reverse order to ensure we sort. + let xml = notification_xml_with_deltas( + sid, + 3, + snapshot_uri, + &hash, + &[ + ("d3", 3, "https://example.net/delta-3.xml", &d_hash_3), + ("d2", 2, "https://example.net/delta-2.xml", &d_hash_2), + ], + ); + let n = parse_notification(&xml).expect("parse notification"); + assert_eq!(n.serial, 3); + assert_eq!(n.deltas.len(), 2); + assert_eq!(n.deltas[0].serial, 2); + assert_eq!(n.deltas[1].serial, 3); + assert_eq!(n.deltas[0].uri, "https://example.net/delta-2.xml"); + assert_eq!(hex::encode(n.deltas[1].hash_sha256), d_hash_3); +} + +#[test] +fn parse_notification_rejects_non_contiguous_deltas() { + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let snapshot_uri = "https://example.net/snapshot.xml"; + let hash = "00".repeat(32); + let d_hash_1 = "11".repeat(32); + let d_hash_3 = "22".repeat(32); + // Missing delta serial 2. + let xml = notification_xml_with_deltas( + sid, + 3, + snapshot_uri, + &hash, + &[ + ("d3", 3, "https://example.net/delta-3.xml", &d_hash_3), + ("d1", 1, "https://example.net/delta-1.xml", &d_hash_1), + ], + ); + let err = parse_notification(&xml).unwrap_err(); + assert!(matches!(err, RrdpError::DeltaRefChainNotContiguous { .. })); +} + +fn delta_xml(session_id: &str, serial: u64, elements: &[&str]) -> Vec { + let mut out = format!( + r#""# + ); + for e in elements { + out.push_str(e); + } + out.push_str(""); + out.into_bytes() +} + +#[test] +fn parse_delta_file_parses_publish_and_withdraw() { + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let serial = 3u64; + let publish_bytes = b"abc"; + let publish_b64 = base64::engine::general_purpose::STANDARD.encode(publish_bytes); + let withdraw_hash = "33".repeat(32); + + let xml = delta_xml( + sid, + serial, + &[ + &format!(r#"{publish_b64}"#), + &format!(r#""#), + ], + ); + + let d = parse_delta_file(&xml).expect("parse delta"); + assert_eq!(d.session_id, Uuid::parse_str(sid).unwrap()); + assert_eq!(d.serial, serial); + assert_eq!(d.elements.len(), 2); + match &d.elements[0] { + DeltaElement::Publish { + uri, + hash_sha256, + bytes, + } => { + assert_eq!(uri, "rsync://example.net/repo/a.mft"); + assert_eq!(*hash_sha256, None); + assert_eq!(bytes, publish_bytes); + } + _ => panic!("expected publish"), + } + match &d.elements[1] { + DeltaElement::Withdraw { uri, hash_sha256 } => { + assert_eq!(uri, "rsync://example.net/repo/b.cer"); + assert_eq!(hex::encode(hash_sha256), withdraw_hash); + } + _ => panic!("expected withdraw"), + } +} + +#[test] +fn parse_delta_file_rejects_withdraw_with_content() { + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let serial = 1u64; + let withdraw_hash = "33".repeat(32); + let xml = delta_xml( + sid, + serial, + &[&format!( + r#"AA=="# + )], + ); + let err = parse_delta_file(&xml).unwrap_err(); + assert!(matches!(err, RrdpError::DeltaWithdrawUnexpectedContent)); +} + +#[test] +fn apply_delta_applies_publish_replace_and_withdraw_with_membership_checks() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + + // Start from snapshot state with a + b + let snapshot_uri = "https://example.net/snapshot.xml"; + let snapshot = snapshot_xml( + sid, + 1, + &[ + ("rsync://example.net/repo/a.mft", b"a1"), + ("rsync://example.net/repo/b.roa", b"b1"), + ], + ); + let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); + let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash); + let fetcher = MapFetcher { + map: HashMap::from([(snapshot_uri.to_string(), snapshot)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).expect("sync snapshot"); + + let old_b = store + .load_current_object_bytes_by_uri("rsync://example.net/repo/b.roa") + .expect("load current b") + .expect("b present"); + let old_b_hash = hex::encode(sha2::Sha256::digest(old_b.as_slice())); + + let withdraw_a_hash = hex::encode(sha2::Sha256::digest(b"a1".as_slice())); + let publish_c_b64 = base64::engine::general_purpose::STANDARD.encode(b"c2"); + let replace_b_b64 = base64::engine::general_purpose::STANDARD.encode(b"b2"); + + let delta = delta_xml( + sid, + 2, + &[ + &format!( + r#""# + ), + &format!( + r#"{replace_b_b64}"# + ), + &format!(r#"{publish_c_b64}"#), + ], + ); + let delta_hash = sha2::Sha256::digest(&delta); + let mut expected_hash = [0u8; 32]; + expected_hash.copy_from_slice(delta_hash.as_slice()); + + let applied = apply_delta( + &store, + notif_uri, + None, + &delta, + expected_hash, + Uuid::parse_str(sid).unwrap(), + 2, + ) + .expect("apply delta"); + assert_eq!(applied, 3); + + assert_eq!( + store + .load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft") + .expect("load current a"), + None, + "a withdrawn" + ); + let b = store + .load_current_object_bytes_by_uri("rsync://example.net/repo/b.roa") + .expect("load current b") + .expect("b present"); + assert_eq!(b, b"b2"); + let c = store + .load_current_object_bytes_by_uri("rsync://example.net/repo/c.crl") + .expect("load current c") + .expect("c present"); + assert_eq!(c, b"c2"); + + assert!( + !store + .is_current_rrdp_source_member(notif_uri, "rsync://example.net/repo/a.mft") + .expect("is member"), + "a removed from rrdp repo index" + ); + assert!( + store + .is_current_rrdp_source_member(notif_uri, "rsync://example.net/repo/c.crl") + .expect("is member"), + "c added to rrdp repo index" + ); + + let a_view = store + .get_repository_view_entry("rsync://example.net/repo/a.mft") + .expect("get a view") + .expect("a view exists"); + assert_eq!(a_view.state, crate::storage::RepositoryViewState::Withdrawn); + let b_view = store + .get_repository_view_entry("rsync://example.net/repo/b.roa") + .expect("get b view") + .expect("b view exists"); + assert_eq!(b_view.state, crate::storage::RepositoryViewState::Present); + assert_eq!( + b_view.current_hash.as_deref(), + Some(hex::encode(sha2::Sha256::digest(b"b2")).as_str()) + ); + let c_owner = store + .get_rrdp_uri_owner_record("rsync://example.net/repo/c.crl") + .expect("get c owner") + .expect("c owner exists"); + assert_eq!( + c_owner.owner_state, + crate::storage::RrdpUriOwnerState::Active + ); + let a_member = store + .get_rrdp_source_member_record(notif_uri, "rsync://example.net/repo/a.mft") + .expect("get a member") + .expect("a member exists"); + assert!(!a_member.present); + let current_members = store + .list_current_rrdp_source_members(notif_uri) + .expect("list current members"); + assert_eq!( + current_members + .iter() + .map(|record| record.rsync_uri.as_str()) + .collect::>(), + vec![ + "rsync://example.net/repo/b.roa", + "rsync://example.net/repo/c.crl", + ] + ); +} + +#[test] +fn apply_delta_rejects_hash_mismatch() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); + let notif_uri = "https://example.net/notification.xml"; + + let delta = delta_xml( + sid.to_string().as_str(), + 1, + &[r#"QQ=="#], + ); + let mut wrong = [0u8; 32]; + wrong[0] = 1; + let err = apply_delta(&store, notif_uri, None, &delta, wrong, sid, 1).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::DeltaHashMismatch) + )); +} + +#[test] +fn apply_delta_rejects_withdraw_of_non_member() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); + let notif_uri = "https://example.net/notification.xml"; + + let withdraw_hash = "00".repeat(32); + let delta = delta_xml( + sid.to_string().as_str(), + 1, + &[&format!( + r#""# + )], + ); + let delta_hash = sha2::Sha256::digest(&delta); + let mut expected_hash = [0u8; 32]; + expected_hash.copy_from_slice(delta_hash.as_slice()); + + let err = apply_delta(&store, notif_uri, None, &delta, expected_hash, sid, 1).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::DeltaTargetNotFromRepository { .. }) + )); +} + +#[test] +fn apply_delta_rejects_publish_without_hash_for_existing_object() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + + // Seed snapshot with a.mft. + let snapshot_uri = "https://example.net/snapshot.xml"; + let snapshot = snapshot_xml(sid, 1, &[("rsync://example.net/repo/a.mft", b"a1")]); + let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); + let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash); + let fetcher = MapFetcher { + map: HashMap::from([(snapshot_uri.to_string(), snapshot)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).expect("seed"); + + // Replace publish for an existing URI must have @hash. + let publish_b64 = base64::engine::general_purpose::STANDARD.encode(b"a2"); + let delta = delta_xml( + sid, + 2, + &[&format!( + r#"{publish_b64}"# + )], + ); + let delta_hash = sha2::Sha256::digest(&delta); + let mut expected_hash = [0u8; 32]; + expected_hash.copy_from_slice(delta_hash.as_slice()); + + let err = apply_delta( + &store, + notif_uri, + None, + &delta, + expected_hash, + Uuid::parse_str(sid).unwrap(), + 2, + ) + .unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::DeltaPublishWithoutHashForExisting { .. }) + )); +} + +#[test] +fn apply_delta_rejects_target_missing_and_hash_mismatch() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + + // Seed snapshot with a.mft. + let snapshot_uri = "https://example.net/snapshot.xml"; + let snapshot = snapshot_xml(sid, 1, &[("rsync://example.net/repo/a.mft", b"a1")]); + let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); + let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash); + let fetcher = MapFetcher { + map: HashMap::from([(snapshot_uri.to_string(), snapshot)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).expect("seed"); + + let old_bytes = store + .load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft") + .expect("get") + .expect("present"); + let old_hash = hex::encode(sha2::Sha256::digest(old_bytes.as_slice())); + + // Hash mismatch on withdraw. + let wrong_hash = "11".repeat(32); + let delta = delta_xml( + sid, + 2, + &[&format!( + r#""# + )], + ); + let delta_hash = sha2::Sha256::digest(&delta); + let mut expected_hash = [0u8; 32]; + expected_hash.copy_from_slice(delta_hash.as_slice()); + let err = apply_delta( + &store, + notif_uri, + None, + &delta, + expected_hash, + Uuid::parse_str(sid).unwrap(), + 2, + ) + .unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::DeltaTargetHashMismatch { .. }) + )); + + // Target missing in local cache (index still says it's a member). + store + .delete_repository_view_entry("rsync://example.net/repo/a.mft") + .expect("delete current repository view entry"); + let delta = delta_xml( + sid, + 2, + &[&format!( + r#""# + )], + ); + let delta_hash = sha2::Sha256::digest(&delta); + let mut expected_hash = [0u8; 32]; + expected_hash.copy_from_slice(delta_hash.as_slice()); + let err = apply_delta( + &store, + notif_uri, + None, + &delta, + expected_hash, + Uuid::parse_str(sid).unwrap(), + 2, + ) + .unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::DeltaTargetMissing { .. }) + )); +} + +#[test] +fn apply_delta_rejects_session_and_serial_mismatch() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + + let publish_b64 = base64::engine::general_purpose::STANDARD.encode(b"x"); + let delta = delta_xml( + sid, + 2, + &[&format!( + r#"{publish_b64}"# + )], + ); + let delta_hash = sha2::Sha256::digest(&delta); + let mut expected_hash = [0u8; 32]; + expected_hash.copy_from_slice(delta_hash.as_slice()); + + // Session mismatch. + let other_sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440001").unwrap(); + let err = + apply_delta(&store, notif_uri, None, &delta, expected_hash, other_sid, 2).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::DeltaSessionIdMismatch { .. }) + )); + + // Serial mismatch. + let err = apply_delta( + &store, + notif_uri, + None, + &delta, + expected_hash, + Uuid::parse_str(sid).unwrap(), + 3, + ) + .unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::DeltaSerialMismatch { .. }) + )); +} + +#[test] +fn sync_from_notification_snapshot_rejects_cross_source_owner_conflict() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid_a = "550e8400-e29b-41d4-a716-446655440000"; + let sid_b = "550e8400-e29b-41d4-a716-446655440001"; + let uri = "rsync://example.net/repo/a.mft"; + + let notif_a_uri = "https://example.net/a/notification.xml"; + let snapshot_a_uri = "https://example.net/a/snapshot.xml"; + let snapshot_a = snapshot_xml(sid_a, 1, &[(uri, b"a1")]); + let snapshot_a_hash = hex::encode(sha2::Sha256::digest(&snapshot_a)); + let notif_a = notification_xml(sid_a, 1, snapshot_a_uri, &snapshot_a_hash); + let fetcher_a = MapFetcher { + map: HashMap::from([(snapshot_a_uri.to_string(), snapshot_a)]), + }; + sync_from_notification_snapshot(&store, notif_a_uri, ¬if_a, &fetcher_a) + .expect("seed source a"); + + let notif_b_uri = "https://example.net/b/notification.xml"; + let snapshot_b_uri = "https://example.net/b/snapshot.xml"; + let snapshot_b = snapshot_xml(sid_b, 1, &[(uri, b"b1")]); + let snapshot_b_hash = hex::encode(sha2::Sha256::digest(&snapshot_b)); + let notif_b = notification_xml(sid_b, 1, snapshot_b_uri, &snapshot_b_hash); + let fetcher_b = MapFetcher { + map: HashMap::from([(snapshot_b_uri.to_string(), snapshot_b)]), + }; + + let err = sync_from_notification_snapshot(&store, notif_b_uri, ¬if_b, &fetcher_b) + .expect_err("cross-source overwrite must fail"); + assert!(matches!(err, RrdpSyncError::Storage(_))); + assert!(err.to_string().contains("owner conflict"), "{err}"); +} + +#[test] +fn sync_from_notification_snapshot_applies_snapshot_and_stores_state() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let serial = 9u64; + let notif_uri = "https://example.net/notification.xml"; + let snapshot_uri = "https://example.net/snapshot.xml"; + + let snapshot = snapshot_xml( + sid, + serial, + &[ + ("rsync://example.net/repo/a.mft", b"mft-bytes"), + ("rsync://example.net/repo/b.roa", b"roa-bytes"), + ], + ); + let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); + let notif = notification_xml(sid, serial, snapshot_uri, &snapshot_hash); + + let fetcher = MapFetcher { + map: HashMap::from([(snapshot_uri.to_string(), snapshot.clone())]), + }; + + let published = + sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).expect("sync"); + assert_eq!(published, 2); + + assert_current_object(&store, "rsync://example.net/repo/a.mft", b"mft-bytes"); + assert_current_object(&store, "rsync://example.net/repo/b.roa", b"roa-bytes"); + + let state = load_rrdp_local_state(&store, notif_uri) + .expect("get rrdp state") + .expect("state present"); + assert_eq!(state.session_id, sid); + assert_eq!(state.serial, serial); + + let source = store + .get_rrdp_source_record(notif_uri) + .expect("get rrdp source") + .expect("rrdp source exists"); + assert_eq!(source.last_session_id.as_deref(), Some(sid)); + assert_eq!(source.last_serial, Some(serial)); + assert_eq!( + source.sync_state, + crate::storage::RrdpSourceSyncState::SnapshotOnly + ); + + let view = store + .get_repository_view_entry("rsync://example.net/repo/a.mft") + .expect("get repository view") + .expect("repository view exists"); + assert_eq!(view.state, crate::storage::RepositoryViewState::Present); + assert_eq!(view.repository_source.as_deref(), Some(notif_uri)); + + let current_bytes = store + .load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft") + .expect("load current bytes") + .expect("current object bytes exist"); + assert_eq!(current_bytes, b"mft-bytes".to_vec()); + assert!( + store + .get_raw_by_hash_entry(hex::encode(sha2::Sha256::digest(b"mft-bytes")).as_str()) + .expect("get raw_by_hash") + .is_none() + ); + + let member = store + .get_rrdp_source_member_record(notif_uri, "rsync://example.net/repo/a.mft") + .expect("get member") + .expect("member exists"); + assert!(member.present); + let owner = store + .get_rrdp_uri_owner_record("rsync://example.net/repo/a.mft") + .expect("get owner") + .expect("owner exists"); + assert_eq!(owner.notify_uri, notif_uri); + assert_eq!(owner.owner_state, crate::storage::RrdpUriOwnerState::Active); +} + +#[test] +fn sync_from_notification_snapshot_deletes_objects_not_in_new_snapshot() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + + // serial 1: publish a + b + let snapshot_uri_1 = "https://example.net/snapshot-1.xml"; + let snapshot_1 = snapshot_xml( + sid, + 1, + &[ + ("rsync://example.net/repo/a.mft", b"a1"), + ("rsync://example.net/repo/b.roa", b"b1"), + ], + ); + let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1)); + let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1); + + let fetcher_1 = MapFetcher { + map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if_1, &fetcher_1).expect("sync 1"); + + // serial 2: publish b (new bytes) + c, and drop a + let snapshot_uri_2 = "https://example.net/snapshot-2.xml"; + let snapshot_2 = snapshot_xml( + sid, + 2, + &[ + ("rsync://example.net/repo/b.roa", b"b2"), + ("rsync://example.net/repo/c.crl", b"c2"), + ], + ); + let snapshot_hash_2 = hex::encode(sha2::Sha256::digest(&snapshot_2)); + let notif_2 = notification_xml(sid, 2, snapshot_uri_2, &snapshot_hash_2); + + let fetcher_2 = MapFetcher { + map: HashMap::from([(snapshot_uri_2.to_string(), snapshot_2)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if_2, &fetcher_2).expect("sync 2"); + + assert!( + store + .load_current_object_bytes_by_uri("rsync://example.net/repo/a.mft") + .expect("get current a") + .is_none(), + "a should be deleted by full-state snapshot apply" + ); + + assert_current_object(&store, "rsync://example.net/repo/b.roa", b"b2"); + assert_current_object(&store, "rsync://example.net/repo/c.crl", b"c2"); +} + +#[test] +fn sync_from_notification_uses_deltas_when_available_for_local_state() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + + // Seed state with snapshot serial=1 containing a+b. + let snapshot_uri_1 = "https://example.net/snapshot-1.xml"; + let snapshot_1 = snapshot_xml( + sid, + 1, + &[ + ("rsync://example.net/repo/a.mft", b"a1"), + ("rsync://example.net/repo/b.roa", b"b1"), + ], + ); + let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1)); + let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1); + let fetcher_1 = MapFetcher { + map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if_1, &fetcher_1).expect("seed"); + + // Notification serial=3 with deltas 2 and 3. Snapshot URI is intentionally not fetchable + // to assert we really use deltas. + let snapshot_uri_3 = "https://example.net/snapshot-3.xml"; + let snapshot_hash_3 = "00".repeat(32); + + let publish_c_b64 = base64::engine::general_purpose::STANDARD.encode(b"c2"); + let delta_2 = delta_xml( + sid, + 2, + &[&format!( + r#"{publish_c_b64}"# + )], + ); + let delta_2_hash_hex = hex::encode(sha2::Sha256::digest(&delta_2)); + + let c_hash_hex = hex::encode(sha2::Sha256::digest(b"c2".as_slice())); + let delta_3 = delta_xml( + sid, + 3, + &[&format!( + r#""# + )], + ); + let delta_3_hash_hex = hex::encode(sha2::Sha256::digest(&delta_3)); + + let notif_3 = notification_xml_with_deltas( + sid, + 3, + snapshot_uri_3, + &snapshot_hash_3, + &[ + ( + "d3", + 3, + "https://example.net/delta-3.xml", + &delta_3_hash_hex, + ), + ( + "d2", + 2, + "https://example.net/delta-2.xml", + &delta_2_hash_hex, + ), + ], + ); + + let fetcher = MapFetcher { + map: HashMap::from([ + ("https://example.net/delta-2.xml".to_string(), delta_2), + ("https://example.net/delta-3.xml".to_string(), delta_3), + ]), + }; + + let applied = sync_from_notification(&store, notif_uri, ¬if_3, &fetcher).expect("sync"); + assert!(applied > 0); + + // Delta 2 publishes c then delta 3 withdraws it => final state should not contain c. + assert!( + store + .load_current_object_bytes_by_uri("rsync://example.net/repo/c.crl") + .expect("get current") + .is_none() + ); + + let state = load_rrdp_local_state(&store, notif_uri) + .expect("get rrdp state") + .expect("state present"); + assert_eq!(state.session_id, Uuid::parse_str(sid).unwrap().to_string()); + assert_eq!(state.serial, 3); +} + +#[test] +fn sync_from_notification_same_serial_hydrates_current_repo_index() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + let snapshot_uri = "https://example.net/snapshot.xml"; + let uri_a = "rsync://example.net/repo/a.mft"; + let uri_b = "rsync://example.net/repo/b.roa"; + + let snapshot = snapshot_xml(sid, 1, &[(uri_a, b"a1"), (uri_b, b"b1")]); + let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot)); + let notif = notification_xml(sid, 1, snapshot_uri, &snapshot_hash); + let fetcher_1 = MapFetcher { + map: HashMap::from([(snapshot_uri.to_string(), snapshot)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher_1).expect("seed"); + + let index = CurrentRepoIndex::shared(); + let no_fetcher = MapFetcher { + map: HashMap::new(), + }; + let applied = sync_from_notification_with_timing_and_download_log( + &store, + notif_uri, + Some(&index), + ¬if, + &no_fetcher, + None, + None, + ) + .expect("same serial no-op"); + assert_eq!(applied, 0); + + let index = index.read().expect("read-lock index"); + assert_eq!(index.active_uri_count(), 2); + assert!(index.get_by_uri(uri_a).is_some()); + assert!(index.get_by_uri(uri_b).is_some()); +} + +#[test] +fn sync_from_notification_delta_hydrates_unchanged_current_repo_entries() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + let snapshot_uri_1 = "https://example.net/snapshot-1.xml"; + let uri_a = "rsync://example.net/repo/a.mft"; + let uri_b = "rsync://example.net/repo/b.roa"; + let uri_c = "rsync://example.net/repo/c.crl"; + + let snapshot_1 = snapshot_xml(sid, 1, &[(uri_a, b"a1"), (uri_b, b"b1")]); + let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1)); + let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1); + let fetcher_1 = MapFetcher { + map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if_1, &fetcher_1).expect("seed"); + + let publish_c_b64 = base64::engine::general_purpose::STANDARD.encode(b"c2"); + let delta_2 = delta_xml( + sid, + 2, + &[&format!( + r#"{publish_c_b64}"# + )], + ); + let delta_2_hash_hex = hex::encode(sha2::Sha256::digest(&delta_2)); + let notif_2 = notification_xml_with_deltas( + sid, + 2, + "https://example.net/snapshot-2.xml", + &"00".repeat(32), + &[( + "d2", + 2, + "https://example.net/delta-2.xml", + &delta_2_hash_hex, + )], + ); + let fetcher_2 = MapFetcher { + map: HashMap::from([("https://example.net/delta-2.xml".to_string(), delta_2)]), + }; + + let index = CurrentRepoIndex::shared(); + let applied = sync_from_notification_with_timing_and_download_log( + &store, + notif_uri, + Some(&index), + ¬if_2, + &fetcher_2, + None, + None, + ) + .expect("delta sync"); + assert_eq!(applied, 1); + + let index = index.read().expect("read-lock index"); + assert_eq!(index.active_uri_count(), 3); + assert!( + index.get_by_uri(uri_a).is_some(), + "unchanged object from the previous serial must be visible" + ); + assert!( + index.get_by_uri(uri_b).is_some(), + "unchanged object from the previous serial must be visible" + ); + assert!(index.get_by_uri(uri_c).is_some(), "delta publish visible"); +} + +#[test] +fn load_rrdp_local_state_uses_source_record_only() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + let notif_uri = "https://example.net/notification.xml"; + assert_eq!( + load_rrdp_local_state(&store, notif_uri).expect("load empty"), + None + ); + + update_rrdp_source_record_on_success( + &store, + notif_uri, + "source-session", + 9, + crate::storage::RrdpSourceSyncState::DeltaReady, + Some("https://example.net/snapshot.xml"), + Some(&hex::encode([0x11; 32])), + ) + .expect("write source record"); + + let got = load_rrdp_local_state(&store, notif_uri) + .expect("load source preferred") + .expect("source present"); + assert_eq!( + got, + RrdpState { + session_id: "source-session".to_string(), + serial: 9, + } + ); +} + +#[test] +fn sync_from_notification_falls_back_to_snapshot_if_missing_required_deltas() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let notif_uri = "https://example.net/notification.xml"; + + // Seed state serial=1 with a only. + let snapshot_uri_1 = "https://example.net/snapshot-1.xml"; + let snapshot_1 = snapshot_xml(sid, 1, &[("rsync://example.net/repo/a.mft", b"a1")]); + let snapshot_hash_1 = hex::encode(sha2::Sha256::digest(&snapshot_1)); + let notif_1 = notification_xml(sid, 1, snapshot_uri_1, &snapshot_hash_1); + let fetcher_1 = MapFetcher { + map: HashMap::from([(snapshot_uri_1.to_string(), snapshot_1)]), + }; + sync_from_notification_snapshot(&store, notif_uri, ¬if_1, &fetcher_1).expect("seed"); + + // Notification serial=3 only includes delta serial=3 (contiguous per RFC, but missing + // serial=2 relative to our local state, so we must use snapshot). + let snapshot_uri_3 = "https://example.net/snapshot-3.xml"; + let snapshot_3 = snapshot_xml(sid, 3, &[("rsync://example.net/repo/z.roa", b"z3")]); + let snapshot_hash_3 = hex::encode(sha2::Sha256::digest(&snapshot_3)); + let delta_3_hash_hex = "11".repeat(32); + let notif_3 = notification_xml_with_deltas( + sid, + 3, + snapshot_uri_3, + &snapshot_hash_3, + &[( + "d3", + 3, + "https://example.net/delta-3.xml", + &delta_3_hash_hex, + )], + ); + + let fetcher = MapFetcher { + map: HashMap::from([(snapshot_uri_3.to_string(), snapshot_3)]), + }; + + let published = sync_from_notification(&store, notif_uri, ¬if_3, &fetcher).expect("sync"); + assert_eq!(published, 1); + assert!( + store + .load_current_object_bytes_by_uri("rsync://example.net/repo/z.roa") + .expect("get current") + .is_some() + ); +} + +#[test] +fn sync_from_notification_snapshot_rejects_snapshot_hash_mismatch() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + + let sid = "550e8400-e29b-41d4-a716-446655440000"; + let serial = 1u64; + let notif_uri = "https://example.net/notification.xml"; + let snapshot_uri = "https://example.net/snapshot.xml"; + + let snapshot = snapshot_xml(sid, serial, &[("rsync://example.net/repo/a.mft", b"x")]); + let notif = notification_xml(sid, serial, snapshot_uri, &"00".repeat(32)); + + let fetcher = MapFetcher { + map: HashMap::from([(snapshot_uri.to_string(), snapshot)]), + }; + let err = sync_from_notification_snapshot(&store, notif_uri, ¬if, &fetcher).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::SnapshotHashMismatch) + )); +} + +#[test] +fn apply_snapshot_rejects_session_id_and_serial_mismatch() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + let notif_uri = "https://example.net/notification.xml"; + + let expected_sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); + let got_sid = "550e8400-e29b-41d4-a716-446655440001"; + + let snapshot = snapshot_xml(got_sid, 2, &[("rsync://example.net/repo/a.mft", b"x")]); + let err = apply_snapshot(&store, notif_uri, None, &snapshot, expected_sid, 2).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::SnapshotSessionIdMismatch { .. }) + )); + + let snapshot = snapshot_xml( + expected_sid.to_string().as_str(), + 3, + &[("rsync://example.net/repo/a.mft", b"x")], + ); + let err = apply_snapshot(&store, notif_uri, None, &snapshot, expected_sid, 2).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::SnapshotSerialMismatch { .. }) + )); +} + +#[test] +fn strip_all_ascii_whitespace_removes_newlines_and_spaces() { + assert_eq!(strip_all_ascii_whitespace(" a \n b\tc "), "abc"); +} + +#[test] +fn apply_snapshot_reports_publish_errors() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + let notif_uri = "https://example.net/notification.xml"; + let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); + + // Missing publish/@uri + let xml = format!( + r#"AA=="# + ) + .into_bytes(); + let err = apply_snapshot(&store, notif_uri, None, &xml, sid, 1).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::PublishUriMissing) + )); + + // Missing base64 content (no text nodes). + let xml = format!( + r#""# + ) + .into_bytes(); + let err = apply_snapshot(&store, notif_uri, None, &xml, sid, 1).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::PublishContentMissing) + )); + + // Invalid base64 content. + let xml = format!( + r#"!!!"# + ) + .into_bytes(); + let err = apply_snapshot(&store, notif_uri, None, &xml, sid, 1).unwrap_err(); + assert!(matches!( + err, + RrdpSyncError::Rrdp(RrdpError::PublishBase64(_)) + )); +} + +#[test] +fn apply_snapshot_handles_multiple_publish_batches() { + let tmp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(tmp.path()).expect("open rocksdb"); + let notif_uri = "https://example.net/notification.xml"; + let sid = Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap(); + + let total = RRDP_SNAPSHOT_APPLY_BATCH_SIZE + 7; + let mut xml = + format!(r#""#); + for i in 0..total { + let uri = format!("rsync://example.net/repo/{i:04}.roa"); + let bytes = format!("payload-{i}").into_bytes(); + let b64 = base64::engine::general_purpose::STANDARD.encode(bytes); + xml.push_str(&format!(r#"{b64}"#)); + } + xml.push_str(""); + + let published = + apply_snapshot(&store, notif_uri, None, xml.as_bytes(), sid, 1).expect("apply snapshot"); + assert_eq!(published, total); + + for idx in [0usize, RRDP_SNAPSHOT_APPLY_BATCH_SIZE - 1, total - 1] { + let uri = format!("rsync://example.net/repo/{idx:04}.roa"); + let got = store + .load_current_object_bytes_by_uri(&uri) + .expect("load object") + .expect("object exists"); + assert_eq!(got, format!("payload-{idx}").into_bytes()); + } +} diff --git a/crates/panda-rpki-validator/src/sync/store_projection.rs b/crates/panda-rpki-validator/src/sync/store_projection.rs new file mode 100644 index 0000000..c3140b5 --- /dev/null +++ b/crates/panda-rpki-validator/src/sync/store_projection.rs @@ -0,0 +1,510 @@ +use std::collections::BTreeMap; + +use crate::storage::{ + PackTime, RawByHashEntry, RepositoryViewEntry, RepositoryViewState, RocksStore, + RrdpSourceMemberRecord, RrdpSourceRecord, RrdpSourceSyncState, RrdpUriOwnerRecord, + RrdpUriOwnerState, +}; +use sha2::Digest; + +#[allow(dead_code)] +pub struct PreparedRawByHashBatch { + pub uri_to_hash: BTreeMap, + pub entries_to_write: Vec, +} + +pub struct PreparedRepoBytesBatch { + pub uri_to_hash: BTreeMap, + pub blobs_to_write: Vec<(String, Vec)>, +} + +pub fn prepare_repo_bytes_batch( + objects: &[(String, Vec)], +) -> Result { + prepare_repo_bytes_batch_owned(objects.to_vec()) +} + +pub fn prepare_repo_bytes_batch_owned( + objects: Vec<(String, Vec)>, +) -> Result { + let mut uri_to_hash: BTreeMap = BTreeMap::new(); + let mut pending: BTreeMap> = BTreeMap::new(); + + for (uri, bytes) in objects { + if bytes.is_empty() { + return Err(format!("repo bytes for {uri} must not be empty")); + } + let sha256_hex = compute_sha256_hex(&bytes); + uri_to_hash.insert(uri.clone(), sha256_hex.clone()); + match pending.entry(sha256_hex) { + std::collections::btree_map::Entry::Vacant(slot) => { + slot.insert(bytes); + } + std::collections::btree_map::Entry::Occupied(existing) => { + if existing.get() != &bytes { + return Err(format!( + "repo bytes collision for {uri}: same sha256 maps to different bytes" + )); + } + } + } + } + + Ok(PreparedRepoBytesBatch { + uri_to_hash, + blobs_to_write: pending.into_iter().collect(), + }) +} + +#[allow(dead_code)] +pub fn prepare_raw_by_hash_evidence_batch( + store: &RocksStore, + objects: &[(String, Vec)], +) -> Result { + let mut pending: BTreeMap = BTreeMap::new(); + let mut uri_to_hash: BTreeMap = BTreeMap::new(); + + for (uri, bytes) in objects { + let sha256_hex = compute_sha256_hex(bytes); + uri_to_hash.insert(uri.clone(), sha256_hex.clone()); + let entry = pending + .entry(sha256_hex.clone()) + .or_insert_with(|| RawByHashEntry::from_bytes(sha256_hex.clone(), bytes.clone())); + + if entry.bytes != *bytes { + return Err(format!( + "raw_by_hash collision for {uri}: same sha256 maps to different bytes" + )); + } + if !entry.origin_uris.iter().any(|existing| existing == uri) { + entry.origin_uris.push(uri.clone()); + } + if entry.object_type.is_none() { + entry.object_type = infer_object_type_from_uri(uri); + } + } + + let hashes: Vec = pending.keys().cloned().collect(); + let existing_entries = store + .get_raw_by_hash_entries_batch(&hashes) + .map_err(|e| e.to_string())?; + + let mut entries_to_write = Vec::new(); + for (hash, existing_opt) in hashes.into_iter().zip(existing_entries.into_iter()) { + let mut pending_entry = pending.remove(&hash).expect("pending raw_by_hash entry"); + match existing_opt { + Some(mut existing) => { + if existing.bytes != pending_entry.bytes { + return Err(format!( + "raw_by_hash collision for hash {hash}: same sha256 maps to different bytes" + )); + } + let mut changed = false; + for uri in pending_entry.origin_uris.drain(..) { + if !existing + .origin_uris + .iter() + .any(|existing_uri| existing_uri == &uri) + { + existing.origin_uris.push(uri); + changed = true; + } + } + if existing.object_type.is_none() && pending_entry.object_type.is_some() { + existing.object_type = pending_entry.object_type; + changed = true; + } + if changed { + entries_to_write.push(existing); + } + } + None => entries_to_write.push(pending_entry), + } + } + + Ok(PreparedRawByHashBatch { + uri_to_hash, + entries_to_write, + }) +} + +pub fn upsert_repo_blob_bytes(store: &RocksStore, bytes: &[u8]) -> Result { + if bytes.is_empty() { + return Err("repo bytes must not be empty".to_string()); + } + let sha256_hex = compute_sha256_hex(bytes); + store + .put_blob_bytes_batch(&[(sha256_hex.clone(), bytes.to_vec())]) + .map_err(|e| e.to_string())?; + Ok(sha256_hex) +} + +pub fn infer_object_type_from_uri(uri: &str) -> Option { + let ext = uri.rsplit('.').next()?; + let ext = ext.to_ascii_lowercase(); + match ext.as_str() { + "cer" | "crl" | "mft" | "roa" | "asa" | "gbr" | "tal" | "xml" => Some(ext), + _ => None, + } +} + +pub fn build_repository_view_present_entry( + repository_source: &str, + rsync_uri: &str, + current_hash: &str, +) -> RepositoryViewEntry { + RepositoryViewEntry { + rsync_uri: rsync_uri.to_string(), + current_hash: Some(current_hash.to_string()), + repository_source: Some(repository_source.to_string()), + object_type: infer_object_type_from_uri(rsync_uri), + state: RepositoryViewState::Present, + } +} + +pub fn build_repository_view_withdrawn_entry( + repository_source: &str, + rsync_uri: &str, + current_hash: Option, +) -> RepositoryViewEntry { + RepositoryViewEntry { + rsync_uri: rsync_uri.to_string(), + current_hash, + repository_source: Some(repository_source.to_string()), + object_type: infer_object_type_from_uri(rsync_uri), + state: RepositoryViewState::Withdrawn, + } +} + +pub fn build_rrdp_source_member_present_record( + notification_uri: &str, + session_id: &str, + serial: u64, + rsync_uri: &str, + current_hash: &str, +) -> RrdpSourceMemberRecord { + RrdpSourceMemberRecord { + notify_uri: notification_uri.to_string(), + rsync_uri: rsync_uri.to_string(), + current_hash: Some(current_hash.to_string()), + object_type: infer_object_type_from_uri(rsync_uri), + present: true, + last_confirmed_session_id: session_id.to_string(), + last_confirmed_serial: serial, + last_changed_at: now_pack_time(), + } +} + +pub fn build_rrdp_source_member_withdrawn_record( + notification_uri: &str, + session_id: &str, + serial: u64, + rsync_uri: &str, + current_hash: Option, +) -> RrdpSourceMemberRecord { + RrdpSourceMemberRecord { + notify_uri: notification_uri.to_string(), + rsync_uri: rsync_uri.to_string(), + current_hash, + object_type: infer_object_type_from_uri(rsync_uri), + present: false, + last_confirmed_session_id: session_id.to_string(), + last_confirmed_serial: serial, + last_changed_at: now_pack_time(), + } +} + +pub fn build_rrdp_uri_owner_active_record( + notification_uri: &str, + session_id: &str, + serial: u64, + rsync_uri: &str, + current_hash: &str, +) -> RrdpUriOwnerRecord { + RrdpUriOwnerRecord { + rsync_uri: rsync_uri.to_string(), + notify_uri: notification_uri.to_string(), + current_hash: Some(current_hash.to_string()), + last_confirmed_session_id: session_id.to_string(), + last_confirmed_serial: serial, + last_changed_at: now_pack_time(), + owner_state: RrdpUriOwnerState::Active, + } +} + +pub fn build_rrdp_uri_owner_withdrawn_record( + notification_uri: &str, + session_id: &str, + serial: u64, + rsync_uri: &str, + current_hash: Option, +) -> RrdpUriOwnerRecord { + RrdpUriOwnerRecord { + rsync_uri: rsync_uri.to_string(), + notify_uri: notification_uri.to_string(), + current_hash, + last_confirmed_session_id: session_id.to_string(), + last_confirmed_serial: serial, + last_changed_at: now_pack_time(), + owner_state: RrdpUriOwnerState::Withdrawn, + } +} + +#[allow(dead_code)] +pub fn upsert_raw_by_hash_evidence( + store: &RocksStore, + rsync_uri: &str, + bytes: &[u8], +) -> Result { + let prepared = + prepare_raw_by_hash_evidence_batch(store, &[(rsync_uri.to_string(), bytes.to_vec())])?; + store + .put_raw_by_hash_entries_batch_unchecked(&prepared.entries_to_write) + .map_err(|e| e.to_string())?; + let sha256_hex = prepared + .uri_to_hash + .get(rsync_uri) + .cloned() + .expect("raw_by_hash mapping for upsert input"); + Ok(sha256_hex) +} + +pub fn put_repository_view_present( + store: &RocksStore, + repository_source: &str, + rsync_uri: &str, + current_hash: &str, +) -> Result<(), String> { + let entry = build_repository_view_present_entry(repository_source, rsync_uri, current_hash); + store + .put_repository_view_entry(&entry) + .map_err(|e| e.to_string()) +} + +pub fn put_repository_view_withdrawn( + store: &RocksStore, + repository_source: &str, + rsync_uri: &str, + current_hash: Option, +) -> Result<(), String> { + let entry = build_repository_view_withdrawn_entry(repository_source, rsync_uri, current_hash); + store + .put_repository_view_entry(&entry) + .map_err(|e| e.to_string()) +} + +pub fn ensure_rrdp_uri_can_be_owned_by( + store: &RocksStore, + notification_uri: &str, + rsync_uri: &str, +) -> Result<(), String> { + let Some(owner) = store + .get_rrdp_uri_owner_record(rsync_uri) + .map_err(|e| e.to_string())? + else { + return Ok(()); + }; + + if owner.notify_uri != notification_uri && owner.owner_state == RrdpUriOwnerState::Active { + return Err(format!( + "RRDP source owner conflict for {rsync_uri}: current owner {} but incoming source {}", + owner.notify_uri, notification_uri + )); + } + Ok(()) +} + +pub fn current_rrdp_owner_is( + store: &RocksStore, + notification_uri: &str, + rsync_uri: &str, +) -> Result { + Ok(matches!( + store + .get_rrdp_uri_owner_record(rsync_uri) + .map_err(|e| e.to_string())?, + Some(owner) + if owner.notify_uri == notification_uri && owner.owner_state == RrdpUriOwnerState::Active + )) +} + +pub fn put_rrdp_source_member_present( + store: &RocksStore, + notification_uri: &str, + session_id: &str, + serial: u64, + rsync_uri: &str, + current_hash: &str, +) -> Result<(), String> { + let record = build_rrdp_source_member_present_record( + notification_uri, + session_id, + serial, + rsync_uri, + current_hash, + ); + store + .put_rrdp_source_member_record(&record) + .map_err(|e| e.to_string()) +} + +pub fn put_rrdp_source_member_withdrawn( + store: &RocksStore, + notification_uri: &str, + session_id: &str, + serial: u64, + rsync_uri: &str, + current_hash: Option, +) -> Result<(), String> { + let record = build_rrdp_source_member_withdrawn_record( + notification_uri, + session_id, + serial, + rsync_uri, + current_hash, + ); + store + .put_rrdp_source_member_record(&record) + .map_err(|e| e.to_string()) +} + +pub fn put_rrdp_uri_owner_active( + store: &RocksStore, + notification_uri: &str, + session_id: &str, + serial: u64, + rsync_uri: &str, + current_hash: &str, +) -> Result<(), String> { + let record = build_rrdp_uri_owner_active_record( + notification_uri, + session_id, + serial, + rsync_uri, + current_hash, + ); + store + .put_rrdp_uri_owner_record(&record) + .map_err(|e| e.to_string()) +} + +pub fn put_rrdp_uri_owner_withdrawn( + store: &RocksStore, + notification_uri: &str, + session_id: &str, + serial: u64, + rsync_uri: &str, + current_hash: Option, +) -> Result<(), String> { + let record = build_rrdp_uri_owner_withdrawn_record( + notification_uri, + session_id, + serial, + rsync_uri, + current_hash, + ); + store + .put_rrdp_uri_owner_record(&record) + .map_err(|e| e.to_string()) +} + +pub fn update_rrdp_source_record_on_success( + store: &RocksStore, + notification_uri: &str, + session_id: &str, + serial: u64, + sync_state: RrdpSourceSyncState, + last_snapshot_uri: Option<&str>, + last_snapshot_hash_hex: Option<&str>, +) -> Result<(), String> { + let now = now_pack_time(); + let mut record = match store + .get_rrdp_source_record(notification_uri) + .map_err(|e| e.to_string())? + { + Some(existing) => existing, + None => RrdpSourceRecord { + notify_uri: notification_uri.to_string(), + last_session_id: None, + last_serial: None, + first_seen_at: now.clone(), + last_seen_at: now.clone(), + last_sync_at: None, + sync_state, + last_snapshot_uri: None, + last_snapshot_hash: None, + last_error: None, + }, + }; + + record.last_session_id = Some(session_id.to_string()); + record.last_serial = Some(serial); + record.last_seen_at = now.clone(); + record.last_sync_at = Some(now); + record.sync_state = sync_state; + record.last_snapshot_uri = last_snapshot_uri.map(str::to_string); + record.last_snapshot_hash = last_snapshot_hash_hex.map(str::to_string); + record.last_error = None; + + store + .put_rrdp_source_record(&record) + .map_err(|e| e.to_string()) +} + +pub fn compute_sha256_hex(bytes: &[u8]) -> String { + hex::encode(sha2::Sha256::digest(bytes)) +} + +pub fn now_pack_time() -> PackTime { + PackTime::from_utc_offset_datetime(time::OffsetDateTime::now_utc()) +} + +#[cfg(test)] +mod tests { + use super::{prepare_repo_bytes_batch, prepare_repo_bytes_batch_owned}; + use std::collections::BTreeSet; + + #[test] + fn prepare_repo_bytes_batch_deduplicates_by_hash() { + let objects = vec![ + ( + "rsync://example.test/repo/a.roa".to_string(), + b"same".to_vec(), + ), + ( + "rsync://example.test/repo/b.roa".to_string(), + b"same".to_vec(), + ), + ( + "rsync://example.test/repo/c.roa".to_string(), + b"other".to_vec(), + ), + ]; + let prepared = prepare_repo_bytes_batch(&objects).expect("prepare repo bytes"); + assert_eq!(prepared.uri_to_hash.len(), 3); + assert_eq!(prepared.blobs_to_write.len(), 2); + let unique_hashes = prepared + .blobs_to_write + .iter() + .map(|(hash, _)| hash.clone()) + .collect::>(); + assert_eq!(unique_hashes.len(), 2); + } + + #[test] + fn prepare_repo_bytes_batch_owned_deduplicates_without_borrowed_input() { + let objects = vec![ + ( + "rsync://example.test/repo/a.roa".to_string(), + b"same".to_vec(), + ), + ( + "rsync://example.test/repo/b.roa".to_string(), + b"same".to_vec(), + ), + ]; + let prepared = prepare_repo_bytes_batch_owned(objects).expect("prepare repo bytes"); + assert_eq!(prepared.uri_to_hash.len(), 2); + assert_eq!(prepared.blobs_to_write.len(), 1); + } +} diff --git a/crates/panda-rpki-validator/src/ta_constraints.rs b/crates/panda-rpki-validator/src/ta_constraints.rs new file mode 100644 index 0000000..cf9dea1 --- /dev/null +++ b/crates/panda-rpki-validator/src/ta_constraints.rs @@ -0,0 +1,1139 @@ +//! Locally configured constraints for the resources carried by RPKI EE +//! certificates. The configuration format follows +//! draft-ietf-sidrops-constraining-rpki-trust-anchors. + +use std::collections::{BTreeMap, BTreeSet}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::ops::Deref; +use std::path::{Path, PathBuf}; + +use sha2::Digest; + +use crate::data_model::rc::{ + Afi, AsIdOrRange, AsIdentifierChoice, IpAddressChoice, IpAddressOrRange, IpResourceSet, + ResourceCertificate, +}; +use crate::parallel::types::{TalInputSpec, TalSource}; + +const LINEAR_INTERVAL_THRESHOLD: usize = 10; + +/// A normalized rule set with a small-set linear path and a large-set tree path. +/// +/// The parser constructs this only after sorting and merging overlapping or +/// adjacent intervals. The tree lookup therefore only needs to inspect the +/// predecessor of a target interval; normalized intervals are disjoint and +/// sorted by their lower bound. +#[derive(Clone)] +struct IntervalIndex { + rules: Vec, + lookup: IntervalLookup, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum IntervalLookup { + Linear, + Tree(BTreeMap), +} + +impl std::fmt::Debug for IntervalIndex { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Keep derived acceleration data out of Debug so diagnostics continue + // to describe canonical rules rather than their index layout. + self.rules.fmt(formatter) + } +} + +impl PartialEq for IntervalIndex { + fn eq(&self, other: &Self) -> bool { + self.rules == other.rules + } +} + +impl Eq for IntervalIndex {} + +impl Deref for IntervalIndex { + type Target = [I]; + + fn deref(&self) -> &Self::Target { + &self.rules + } +} + +trait IntervalValue { + fn start(&self) -> u128; + fn end(&self) -> u128; + fn overlaps(&self, other: &Self) -> bool; +} + +impl IntervalIndex { + fn new(rules: Vec) -> Self { + debug_assert!( + rules + .windows(2) + .all(|window| { window[0].end().saturating_add(1) < window[1].start() }) + ); + let lookup = if rules.len() > LINEAR_INTERVAL_THRESHOLD { + let tree = rules + .iter() + .map(|rule| (rule.start(), rule.end())) + .collect(); + IntervalLookup::Tree(tree) + } else { + IntervalLookup::Linear + }; + Self { rules, lookup } + } + + fn any_overlaps(&self, target: &I) -> bool { + match &self.lookup { + IntervalLookup::Linear => self.rules.iter().any(|entry| entry.overlaps(target)), + IntervalLookup::Tree(tree) => tree + .range(..=target.end()) + .next_back() + .map(|(_, end)| *end >= target.start()) + .unwrap_or(false), + } + } + + fn fully_covers(&self, target: &I) -> bool { + match &self.lookup { + IntervalLookup::Linear => self.fully_covers_linear(target), + IntervalLookup::Tree(tree) => tree + .range(..=target.start()) + .next_back() + .map(|(_, end)| *end >= target.end()) + .unwrap_or(false), + } + } + + fn fully_covers_linear(&self, target: &I) -> bool { + let mut cursor = target.start(); + for entry in &self.rules { + if entry.end() < cursor { + continue; + } + if entry.start() > cursor { + return false; + } + if entry.end() >= target.end() { + return true; + } + cursor = entry.end().saturating_add(1); + } + false + } + + #[cfg(test)] + fn uses_tree(&self) -> bool { + matches!(self.lookup, IntervalLookup::Tree(_)) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TaConstraintsByTal { + by_tal_id: BTreeMap>, + /// Stable semantic digest of the normalized rules. This is computed once + /// while loading the run policy so cache identity checks do not serialize + /// every rule for every publication point. + fingerprint: [u8; 32], +} + +impl TaConstraintsByTal { + pub fn load_for_tals( + tal_inputs: &[TalInputSpec], + explicit_specs: &[String], + ) -> Result { + let tal_ids = tal_inputs + .iter() + .map(|input| input.tal_id.as_str()) + .collect::>(); + let mut explicit_paths = BTreeMap::::new(); + for spec in explicit_specs { + let (tal_id, path) = spec + .split_once('=') + .ok_or_else(|| format!("--ta-constraints expects =, got '{spec}'"))?; + let tal_id = tal_id.trim(); + let path = path.trim(); + if tal_id.is_empty() || path.is_empty() { + return Err(format!( + "--ta-constraints expects non-empty =, got '{spec}'" + )); + } + if !tal_ids.contains(tal_id) { + return Err(format!( + "--ta-constraints references unknown TAL id '{tal_id}'" + )); + } + if explicit_paths + .insert(tal_id.to_string(), PathBuf::from(path)) + .is_some() + { + return Err(format!( + "--ta-constraints specifies TAL id '{tal_id}' more than once" + )); + } + } + + let mut by_tal_id = BTreeMap::new(); + for input in tal_inputs { + let path = explicit_paths + .get(&input.tal_id) + .cloned() + .or_else(|| adjacent_constraints_path(&input.source).filter(|path| path.is_file())); + if let Some(path) = path { + let constraints = TaConstraints::from_file(&path).map_err(|error| { + format!( + "load TA constraints for '{}' from {} failed: {error}", + input.tal_id, + path.display() + ) + })?; + by_tal_id.insert(input.tal_id.clone(), std::sync::Arc::new(constraints)); + } + } + let fingerprint = constraints_fingerprint(&by_tal_id); + Ok(Self { + by_tal_id, + fingerprint, + }) + } + + pub fn for_tal(&self, tal_id: &str) -> Option<&TaConstraints> { + self.by_tal_id.get(tal_id).map(std::sync::Arc::as_ref) + } + + /// Return the immutable, process-local snapshot for a TAL without cloning + /// the rule trees. Phase-2 object workers own this `Arc` in their task + /// payload, while scoped stage workers borrow the same policy map. + pub(crate) fn shared_for_tal(&self, tal_id: &str) -> Option> { + self.by_tal_id.get(tal_id).cloned() + } + + pub fn is_empty(&self) -> bool { + self.by_tal_id.is_empty() + } + + pub fn configuration_warnings(&self) -> Vec { + self.by_tal_id + .iter() + .flat_map(|(tal_id, constraints)| { + constraints.warnings().iter().map(move |warning| { + format!( + "TA constraints for TAL '{tal_id}' ({}): {warning}", + constraints.source().display() + ) + }) + }) + .collect() + } + + /// Return the precomputed SHA-256 digest of the normalized semantic rules. + /// + /// The returned bytes are stable across source paths and derived index + /// layouts, and are intentionally borrowed so hot-path cache lookups do + /// not allocate. + pub fn fingerprint_bytes(&self) -> &[u8] { + &self.fingerprint + } + + pub fn fingerprint_sha256_hex(&self) -> String { + hex::encode(self.fingerprint) + } +} + +impl Default for TaConstraintsByTal { + fn default() -> Self { + let by_tal_id = BTreeMap::new(); + let fingerprint = constraints_fingerprint(&by_tal_id); + Self { + by_tal_id, + fingerprint, + } + } +} + +const CONSTRAINTS_FINGERPRINT_VERSION: &[u8] = b"ta-constraints-semantic-v1"; + +fn constraints_fingerprint( + by_tal_id: &BTreeMap>, +) -> [u8; 32] { + let mut hasher = sha2::Sha256::new(); + hasher.update(CONSTRAINTS_FINGERPRINT_VERSION); + hasher.update((by_tal_id.len() as u64).to_be_bytes()); + for (tal_id, constraints) in by_tal_id { + update_length_prefixed(&mut hasher, tal_id.as_bytes()); + update_ip_intervals(&mut hasher, b"allow-v4", &constraints.allow_v4); + update_ip_intervals(&mut hasher, b"deny-v4", &constraints.deny_v4); + update_ip_intervals(&mut hasher, b"allow-v6", &constraints.allow_v6); + update_ip_intervals(&mut hasher, b"deny-v6", &constraints.deny_v6); + update_as_intervals(&mut hasher, b"allow-asn", &constraints.allow_asn); + update_as_intervals(&mut hasher, b"deny-asn", &constraints.deny_asn); + } + let digest = hasher.finalize(); + let mut out = [0u8; 32]; + out.copy_from_slice(&digest); + out +} + +fn update_length_prefixed(hasher: &mut sha2::Sha256, value: &[u8]) { + hasher.update((value.len() as u64).to_be_bytes()); + hasher.update(value); +} + +fn update_ip_intervals( + hasher: &mut sha2::Sha256, + label: &[u8], + intervals: &IntervalIndex, +) { + update_length_prefixed(hasher, label); + hasher.update((intervals.rules.len() as u64).to_be_bytes()); + for interval in &intervals.rules { + hasher.update([match interval.afi { + Afi::Ipv4 => 4, + Afi::Ipv6 => 6, + }]); + hasher.update(interval.min.to_be_bytes()); + hasher.update(interval.max.to_be_bytes()); + } +} + +fn update_as_intervals( + hasher: &mut sha2::Sha256, + label: &[u8], + intervals: &IntervalIndex, +) { + update_length_prefixed(hasher, label); + hasher.update((intervals.rules.len() as u64).to_be_bytes()); + for interval in &intervals.rules { + hasher.update(interval.min.to_be_bytes()); + hasher.update(interval.max.to_be_bytes()); + } +} + +fn adjacent_constraints_path(source: &TalSource) -> Option { + match source { + TalSource::FilePath(path) => Some(path.with_extension("constraints")), + TalSource::FilePathWithTa { tal_path, .. } => Some(tal_path.with_extension("constraints")), + TalSource::Url(_) | TalSource::DerBytes { .. } => None, + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TaConstraints { + source: PathBuf, + allow_v4: IntervalIndex, + deny_v4: IntervalIndex, + allow_v6: IntervalIndex, + deny_v6: IntervalIndex, + allow_asn: IntervalIndex, + deny_asn: IntervalIndex, + warnings: Vec, +} + +impl TaConstraints { + pub fn from_file(path: &Path) -> Result { + let contents = std::fs::read_to_string(path) + .map_err(|error| format!("read {}: {error}", path.display()))?; + Self::parse(path.to_path_buf(), &contents) + } + + pub fn warnings(&self) -> &[String] { + &self.warnings + } + + pub fn source(&self) -> &Path { + &self.source + } + + pub fn validate_ee_certificate( + &self, + certificate: &ResourceCertificate, + ) -> Result<(), TaConstraintsViolation> { + if let Some(ip_resources) = certificate.tbs.extensions.ip_resources.as_ref() { + self.validate_ip_resources(ip_resources)?; + } + if let Some(as_resources) = certificate.tbs.extensions.as_resources.as_ref() { + self.validate_as_choice("AS", as_resources.asnum.as_ref())?; + self.validate_as_choice("RDI", as_resources.rdi.as_ref())?; + } + Ok(()) + } + + fn parse(source: PathBuf, contents: &str) -> Result { + let mut allow_v4 = Vec::new(); + let mut deny_v4 = Vec::new(); + let mut allow_v6 = Vec::new(); + let mut deny_v6 = Vec::new(); + let mut allow_asn = Vec::new(); + let mut deny_asn = Vec::new(); + + for (index, raw_line) in contents.lines().enumerate() { + let line_number = index + 1; + let line = raw_line.split('#').next().unwrap_or("").trim(); + if line.is_empty() { + continue; + } + let mut words = line.split_whitespace(); + let action = words.next().expect("non-empty line has first word"); + let resource = words.collect::>().join(" "); + if resource.is_empty() { + return Err(format!( + "line {line_number}: missing resource after '{action}'" + )); + } + let allow = match action { + "allow" => true, + "deny" => false, + _ => { + return Err(format!( + "line {line_number}: expected 'allow' or 'deny', got '{action}'" + )); + } + }; + if looks_like_ip_resource(&resource) { + let interval = parse_ip_interval(&resource).map_err(|error| { + format!("line {line_number}: invalid IP resource '{resource}': {error}") + })?; + match (interval.afi, allow) { + (Afi::Ipv4, true) => allow_v4.push(interval), + (Afi::Ipv4, false) => deny_v4.push(interval), + (Afi::Ipv6, true) => allow_v6.push(interval), + (Afi::Ipv6, false) => deny_v6.push(interval), + } + } else { + let interval = parse_as_interval(&resource).map_err(|error| { + format!("line {line_number}: invalid AS resource '{resource}': {error}") + })?; + if allow { + allow_asn.push(interval); + } else { + deny_asn.push(interval); + } + } + } + + let mut warnings = Vec::new(); + normalize_ip_intervals("allow IPv4", &mut allow_v4, &mut warnings); + normalize_ip_intervals("deny IPv4", &mut deny_v4, &mut warnings); + normalize_ip_intervals("allow IPv6", &mut allow_v6, &mut warnings); + normalize_ip_intervals("deny IPv6", &mut deny_v6, &mut warnings); + normalize_as_intervals("allow AS", &mut allow_asn, &mut warnings); + normalize_as_intervals("deny AS", &mut deny_asn, &mut warnings); + + Ok(Self { + source, + allow_v4: IntervalIndex::new(allow_v4), + deny_v4: IntervalIndex::new(deny_v4), + allow_v6: IntervalIndex::new(allow_v6), + deny_v6: IntervalIndex::new(deny_v6), + allow_asn: IntervalIndex::new(allow_asn), + deny_asn: IntervalIndex::new(deny_asn), + warnings, + }) + } + + fn validate_ip_resources( + &self, + resources: &IpResourceSet, + ) -> Result<(), TaConstraintsViolation> { + for family in &resources.families { + let items = match &family.choice { + // Constraints apply to explicit INR listings. EE profiles for + // constrained signed objects already reject inappropriate inherit. + IpAddressChoice::Inherit => continue, + IpAddressChoice::AddressesOrRanges(items) => items, + }; + let (allow, deny) = match family.afi { + Afi::Ipv4 => (&self.allow_v4, &self.deny_v4), + Afi::Ipv6 => (&self.allow_v6, &self.deny_v6), + }; + for item in items { + let interval = ip_item_to_interval(family.afi, item)?; + if deny.any_overlaps(&interval) { + return Err(TaConstraintsViolation(format!( + "{} {} intersects a deny rule in {}", + afi_name(family.afi), + interval, + self.source.display() + ))); + } + if !allow.fully_covers(&interval) { + return Err(TaConstraintsViolation(format!( + "{} {} is not fully contained in allow rules in {}", + afi_name(family.afi), + interval, + self.source.display() + ))); + } + } + } + Ok(()) + } + + fn validate_as_choice( + &self, + kind: &str, + choice: Option<&AsIdentifierChoice>, + ) -> Result<(), TaConstraintsViolation> { + let Some(choice) = choice else { + return Ok(()); + }; + let items = match choice { + AsIdentifierChoice::Inherit => return Ok(()), + AsIdentifierChoice::AsIdsOrRanges(items) => items, + }; + for item in items { + let interval = match item { + AsIdOrRange::Id(value) => AsInterval::new(*value, *value), + AsIdOrRange::Range { min, max } => AsInterval::new(*min, *max), + }; + if self.deny_asn.any_overlaps(&interval) { + return Err(TaConstraintsViolation(format!( + "{kind} {interval} intersects a deny rule in {}", + self.source.display() + ))); + } + if !self.allow_asn.fully_covers(&interval) { + return Err(TaConstraintsViolation(format!( + "{kind} {interval} is not fully contained in allow rules in {}", + self.source.display() + ))); + } + } + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TaConstraintsViolation(pub String); + +impl std::fmt::Display for TaConstraintsViolation { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(formatter) + } +} + +impl std::error::Error for TaConstraintsViolation {} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct IpInterval { + afi: Afi, + min: u128, + max: u128, +} + +impl IpInterval { + fn new(afi: Afi, min: u128, max: u128) -> Self { + Self { afi, min, max } + } + + fn overlaps(&self, other: &Self) -> bool { + self.afi == other.afi && self.min <= other.max && other.min <= self.max + } +} + +impl IntervalValue for IpInterval { + fn start(&self) -> u128 { + self.min + } + + fn end(&self) -> u128 { + self.max + } + + fn overlaps(&self, other: &Self) -> bool { + IpInterval::overlaps(self, other) + } +} + +impl std::fmt::Display for IpInterval { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let min = u128_to_ip(self.afi, self.min); + let max = u128_to_ip(self.afi, self.max); + if min == max { + write!(formatter, "{min}") + } else { + write!(formatter, "{min} - {max}") + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct AsInterval { + min: u32, + max: u32, +} + +impl AsInterval { + fn new(min: u32, max: u32) -> Self { + Self { min, max } + } + + fn overlaps(&self, other: &Self) -> bool { + self.min <= other.max && other.min <= self.max + } +} + +impl IntervalValue for AsInterval { + fn start(&self) -> u128 { + self.min.into() + } + + fn end(&self) -> u128 { + self.max.into() + } + + fn overlaps(&self, other: &Self) -> bool { + AsInterval::overlaps(self, other) + } +} + +impl std::fmt::Display for AsInterval { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.min == self.max { + write!(formatter, "{}", self.min) + } else { + write!(formatter, "{} - {}", self.min, self.max) + } + } +} + +fn looks_like_ip_resource(resource: &str) -> bool { + resource.contains('.') || resource.contains(':') || resource.contains('/') +} + +fn parse_ip_interval(resource: &str) -> Result { + if let Some((raw_min, raw_max)) = resource.split_once('-') { + let min: IpAddr = raw_min.trim().parse().map_err(|_| "invalid range start")?; + let max: IpAddr = raw_max.trim().parse().map_err(|_| "invalid range end")?; + let (afi, min) = ip_to_u128(min); + let (max_afi, max) = ip_to_u128(max); + if afi != max_afi { + return Err("range endpoints use different address families".to_string()); + } + if min > max { + return Err("range start is greater than range end".to_string()); + } + return Ok(IpInterval::new(afi, min, max)); + } + + let (raw_address, raw_prefix_len) = resource + .split_once('/') + .ok_or_else(|| "expected CIDR prefix or range".to_string())?; + let address: IpAddr = raw_address.trim().parse().map_err(|_| "invalid address")?; + let prefix_len: u16 = raw_prefix_len + .trim() + .parse() + .map_err(|_| "invalid prefix length")?; + let (afi, address) = ip_to_u128(address); + let width = match afi { + Afi::Ipv4 => 32, + Afi::Ipv6 => 128, + }; + if prefix_len > width { + return Err(format!("prefix length must be <= {width}")); + } + let host_bits = width - prefix_len; + let mask = if prefix_len == 0 { + 0 + } else { + width_mask(width) << host_bits + }; + let min = address & mask; + let max = min | (!mask & width_mask(width)); + Ok(IpInterval::new(afi, min, max)) +} + +fn parse_as_interval(resource: &str) -> Result { + let parse_asn = |raw: &str| -> Result { + raw.trim() + .strip_prefix("AS") + .or_else(|| raw.trim().strip_prefix("as")) + .unwrap_or(raw.trim()) + .parse::() + .map_err(|_| "expected an ASN in the range 0..4294967295".to_string()) + }; + if let Some((raw_min, raw_max)) = resource.split_once('-') { + let min = parse_asn(raw_min)?; + let max = parse_asn(raw_max)?; + if min > max { + return Err("range start is greater than range end".to_string()); + } + Ok(AsInterval::new(min, max)) + } else { + let value = parse_asn(resource)?; + Ok(AsInterval::new(value, value)) + } +} + +fn normalize_ip_intervals(label: &str, entries: &mut Vec, warnings: &mut Vec) { + entries.sort_by_key(|entry| (entry.min, entry.max)); + let mut normalized = Vec::with_capacity(entries.len()); + for entry in entries.drain(..) { + let Some(last) = normalized.last_mut() else { + normalized.push(entry); + continue; + }; + if entry.min <= last.max { + warnings.push(format!( + "TA constraints {label} rules overlap; normalized without blocking startup" + )); + last.max = last.max.max(entry.max); + } else if entry.min == last.max.saturating_add(1) { + last.max = entry.max; + } else { + normalized.push(entry); + } + } + warnings.sort(); + warnings.dedup(); + *entries = normalized; +} + +fn normalize_as_intervals(label: &str, entries: &mut Vec, warnings: &mut Vec) { + entries.sort_by_key(|entry| (entry.min, entry.max)); + let mut normalized = Vec::with_capacity(entries.len()); + for entry in entries.drain(..) { + let Some(last) = normalized.last_mut() else { + normalized.push(entry); + continue; + }; + if entry.min <= last.max { + warnings.push(format!( + "TA constraints {label} rules overlap; normalized without blocking startup" + )); + last.max = last.max.max(entry.max); + } else if entry.min == last.max.saturating_add(1) { + last.max = entry.max; + } else { + normalized.push(entry); + } + } + warnings.sort(); + warnings.dedup(); + *entries = normalized; +} + +fn ip_item_to_interval( + afi: Afi, + item: &IpAddressOrRange, +) -> Result { + let (min, max) = match item { + IpAddressOrRange::Prefix(prefix) => { + let width = prefix.afi.ub(); + let address = ip_bytes_to_u128(&prefix.addr); + let prefix_len = prefix.prefix_len.min(width); + let host_bits = width - prefix_len; + let mask = if prefix_len == 0 { + 0 + } else { + width_mask(width) << host_bits + }; + let min = address & mask; + (min, min | (!mask & width_mask(width))) + } + IpAddressOrRange::Range(range) => { + (ip_bytes_to_u128(&range.min), ip_bytes_to_u128(&range.max)) + } + }; + if min > max { + return Err(TaConstraintsViolation( + "EE certificate carries an invalid IP range".to_string(), + )); + } + Ok(IpInterval::new(afi, min, max)) +} + +fn ip_to_u128(address: IpAddr) -> (Afi, u128) { + match address { + IpAddr::V4(address) => (Afi::Ipv4, u32::from(address) as u128), + IpAddr::V6(address) => (Afi::Ipv6, u128::from(address)), + } +} + +fn ip_bytes_to_u128(bytes: &[u8]) -> u128 { + bytes + .iter() + .fold(0u128, |value, byte| (value << 8) | u128::from(*byte)) +} + +fn width_mask(width: u16) -> u128 { + if width == 128 { + u128::MAX + } else { + (1u128 << width) - 1 + } +} + +fn u128_to_ip(afi: Afi, value: u128) -> IpAddr { + match afi { + Afi::Ipv4 => IpAddr::V4(Ipv4Addr::from(value as u32)), + Afi::Ipv6 => IpAddr::V6(Ipv6Addr::from(value)), + } +} + +fn afi_name(afi: Afi) -> &'static str { + match afi { + Afi::Ipv4 => "IPv4", + Afi::Ipv6 => "IPv6", + } +} + +#[cfg(test)] +mod tests { + use super::{IntervalIndex, IntervalLookup, TaConstraints, TaConstraintsByTal}; + + fn parse(body: &str) -> TaConstraints { + TaConstraints::parse("test.constraints".into(), body).expect("parse constraints") + } + + fn indexed_ipv4_rules(action: &str, count: usize) -> String { + (0..count) + .map(|index| format!("{action} 10.{}.0.0/16\n", index * 2)) + .collect() + } + + fn indexed_ipv6_rules(action: &str, count: usize) -> String { + (0..count) + .map(|index| format!("{action} 2001:db8:{}::/48\n", index * 2)) + .collect() + } + + fn indexed_asn_rules(action: &str, count: usize) -> String { + (0..count) + .map(|index| format!("{action} {}\n", 65000 + index * 2)) + .collect() + } + + #[test] + fn deny_has_precedence_and_unlisted_resources_are_denied() { + let constraints = parse("allow 192.0.2.0/24\ndeny 192.0.2.128/25\n"); + assert!( + constraints + .allow_v4 + .fully_covers(&super::parse_ip_interval("192.0.2.0/25").unwrap()) + ); + assert!( + constraints + .deny_v4 + .iter() + .any(|entry| entry.overlaps(&super::parse_ip_interval("192.0.2.128/25").unwrap())) + ); + assert!( + !constraints + .allow_v4 + .fully_covers(&super::parse_ip_interval("198.51.100.0/24").unwrap()) + ); + } + + #[test] + fn same_list_overlap_warns_and_normalizes() { + let constraints = parse("allow 10.0.0.0/8\nallow 10.1.0.0/16\n"); + assert_eq!(constraints.allow_v4.len(), 1); + assert_eq!(constraints.warnings.len(), 1); + assert!(constraints.warnings[0].contains("overlap")); + } + + #[test] + fn adjacent_allow_ranges_cover_one_interval() { + let constraints = parse("allow 64496 - 64500\nallow 64501 - 64511\n"); + assert!( + constraints + .allow_asn + .fully_covers(&super::parse_as_interval("64496 - 64511").unwrap()) + ); + } + + #[test] + fn interval_index_uses_linear_for_at_most_ten_and_tree_above_ten() { + for count in [0, 1, 9, 10] { + let constraints = parse(&indexed_ipv4_rules("deny", count)); + assert_eq!(constraints.deny_v4.len(), count); + assert!(!constraints.deny_v4.uses_tree(), "count={count}"); + } + + let constraints = parse(&indexed_ipv4_rules("deny", 11)); + assert_eq!(constraints.deny_v4.len(), 11); + assert!(constraints.deny_v4.uses_tree()); + } + + #[test] + fn tree_ip_queries_match_linear_reference() { + let body = format!( + "{}{}", + indexed_ipv4_rules("allow", 11), + indexed_ipv4_rules("deny", 11) + ); + let constraints = parse(&body); + assert!(constraints.allow_v4.uses_tree()); + assert!(constraints.deny_v4.uses_tree()); + + let linear_allow = IntervalIndex { + rules: constraints.allow_v4.rules.clone(), + lookup: IntervalLookup::Linear, + }; + let linear_deny = IntervalIndex { + rules: constraints.deny_v4.rules.clone(), + lookup: IntervalLookup::Linear, + }; + let targets = [ + super::parse_ip_interval("10.0.0.0/16").unwrap(), + super::parse_ip_interval("10.0.1.0/24").unwrap(), + super::parse_ip_interval("10.1.0.0/16").unwrap(), + super::parse_ip_interval("10.20.0.0/16").unwrap(), + super::parse_ip_interval("10.21.0.0/16").unwrap(), + super::parse_ip_interval("10.0.0.0 - 10.2.255.255").unwrap(), + super::parse_ip_interval("9.0.0.0/8").unwrap(), + super::parse_ip_interval("11.0.0.0/8").unwrap(), + ]; + for target in targets { + assert_eq!( + constraints.allow_v4.any_overlaps(&target), + linear_allow.any_overlaps(&target), + "allow overlap for {target}" + ); + assert_eq!( + constraints.allow_v4.fully_covers(&target), + linear_allow.fully_covers(&target), + "allow coverage for {target}" + ); + assert_eq!( + constraints.deny_v4.any_overlaps(&target), + linear_deny.any_overlaps(&target), + "deny overlap for {target}" + ); + } + } + + #[test] + fn tree_ipv6_queries_match_linear_reference() { + let body = format!( + "{}{}", + indexed_ipv6_rules("allow", 11), + indexed_ipv6_rules("deny", 11) + ); + let constraints = parse(&body); + assert!(constraints.allow_v6.uses_tree()); + assert!(constraints.deny_v6.uses_tree()); + + let linear_allow = IntervalIndex { + rules: constraints.allow_v6.rules.clone(), + lookup: IntervalLookup::Linear, + }; + let linear_deny = IntervalIndex { + rules: constraints.deny_v6.rules.clone(), + lookup: IntervalLookup::Linear, + }; + let targets = [ + super::parse_ip_interval("2001:db8:0::/48").unwrap(), + super::parse_ip_interval("2001:db8:1::/64").unwrap(), + super::parse_ip_interval("2001:db8:2::/48").unwrap(), + super::parse_ip_interval("2001:db8:20::/48").unwrap(), + super::parse_ip_interval("2001:db8:21::/48").unwrap(), + super::parse_ip_interval("2001:db8:0:: - 2001:db8:2:ffff:ffff:ffff:ffff:ffff").unwrap(), + super::parse_ip_interval("2001:db7::/32").unwrap(), + super::parse_ip_interval("2001:db9::/32").unwrap(), + ]; + for target in targets { + assert_eq!( + constraints.allow_v6.any_overlaps(&target), + linear_allow.any_overlaps(&target), + "allow overlap for {target}" + ); + assert_eq!( + constraints.allow_v6.fully_covers(&target), + linear_allow.fully_covers(&target), + "allow coverage for {target}" + ); + assert_eq!( + constraints.deny_v6.any_overlaps(&target), + linear_deny.any_overlaps(&target), + "deny overlap for {target}" + ); + } + } + + #[test] + fn tree_asn_queries_match_linear_reference() { + let body = format!( + "{}{}", + indexed_asn_rules("allow", 11), + indexed_asn_rules("deny", 11) + ); + let constraints = parse(&body); + assert!(constraints.allow_asn.uses_tree()); + assert!(constraints.deny_asn.uses_tree()); + + let linear_allow = IntervalIndex { + rules: constraints.allow_asn.rules.clone(), + lookup: IntervalLookup::Linear, + }; + let linear_deny = IntervalIndex { + rules: constraints.deny_asn.rules.clone(), + lookup: IntervalLookup::Linear, + }; + let targets = [ + super::AsInterval::new(65000, 65000), + super::AsInterval::new(65001, 65001), + super::AsInterval::new(65000, 65002), + super::AsInterval::new(65020, 65020), + super::AsInterval::new(65021, 65021), + super::AsInterval::new(64999, 65000), + ]; + for target in targets { + assert_eq!( + constraints.allow_asn.any_overlaps(&target), + linear_allow.any_overlaps(&target), + "allow overlap for {target}" + ); + assert_eq!( + constraints.allow_asn.fully_covers(&target), + linear_allow.fully_covers(&target), + "allow coverage for {target}" + ); + assert_eq!( + constraints.deny_asn.any_overlaps(&target), + linear_deny.any_overlaps(&target), + "deny overlap for {target}" + ); + } + } + + #[test] + fn index_equality_and_debug_ignore_derived_lookup() { + let constraints = parse(&indexed_ipv4_rules("deny", 11)); + let tree = constraints.deny_v4.clone(); + let linear = IntervalIndex { + rules: tree.rules.clone(), + lookup: IntervalLookup::Linear, + }; + assert_eq!(tree, linear); + assert_eq!(format!("{tree:?}"), format!("{linear:?}")); + } + + #[test] + fn adjacent_file_is_discovered_by_tal_stem() { + let dir = tempfile::tempdir().expect("tmpdir"); + let tal_path = dir.path().join("example.tal"); + std::fs::write(&tal_path, "placeholder").expect("write TAL"); + std::fs::write( + tal_path.with_extension("constraints"), + "allow 192.0.2.0/24\n", + ) + .expect("write constraints"); + let inputs = vec![crate::parallel::types::TalInputSpec::from_file_path( + tal_path, + )]; + let loaded = TaConstraintsByTal::load_for_tals(&inputs, &[]).expect("load constraints"); + assert!(loaded.for_tal("example").is_some()); + let first = loaded + .shared_for_tal("example") + .expect("shared constraints snapshot"); + let second = loaded + .shared_for_tal("example") + .expect("shared constraints snapshot"); + assert!(std::sync::Arc::ptr_eq(&first, &second)); + } + + #[test] + fn shared_snapshots_remain_tal_specific() { + let dir = tempfile::tempdir().expect("tmpdir"); + let tal_a = dir.path().join("alpha.tal"); + let tal_b = dir.path().join("bravo.tal"); + std::fs::write(&tal_a, "placeholder").expect("write alpha TAL"); + std::fs::write(&tal_b, "placeholder").expect("write bravo TAL"); + std::fs::write(tal_a.with_extension("constraints"), "allow 192.0.2.0/24\n") + .expect("write alpha constraints"); + std::fs::write( + tal_b.with_extension("constraints"), + "allow 198.51.100.0/24\n", + ) + .expect("write bravo constraints"); + + let inputs = vec![ + crate::parallel::types::TalInputSpec::from_file_path(tal_a), + crate::parallel::types::TalInputSpec::from_file_path(tal_b), + ]; + let loaded = TaConstraintsByTal::load_for_tals(&inputs, &[]).expect("load constraints"); + let alpha = loaded.shared_for_tal("alpha").expect("alpha snapshot"); + let bravo = loaded.shared_for_tal("bravo").expect("bravo snapshot"); + assert!(!std::sync::Arc::ptr_eq(&alpha, &bravo)); + assert_ne!(alpha.as_ref(), bravo.as_ref()); + } + + #[test] + fn semantic_fingerprint_is_cached_and_ignores_source_path() { + let first_dir = tempfile::tempdir().expect("first tmpdir"); + let second_dir = tempfile::tempdir().expect("second tmpdir"); + let first_tal = first_dir.path().join("example.tal"); + let second_tal = second_dir.path().join("example.tal"); + for tal_path in [&first_tal, &second_tal] { + std::fs::write(tal_path, "placeholder").expect("write TAL"); + std::fs::write( + tal_path.with_extension("constraints"), + "allow 192.0.2.0/24\ndeny 192.0.2.128/25\n", + ) + .expect("write constraints"); + } + + let first = TaConstraintsByTal::load_for_tals( + &[crate::parallel::types::TalInputSpec::from_file_path( + first_tal, + )], + &[], + ) + .expect("load first constraints"); + let second = TaConstraintsByTal::load_for_tals( + &[crate::parallel::types::TalInputSpec::from_file_path( + second_tal, + )], + &[], + ) + .expect("load second constraints"); + + assert_eq!(first.fingerprint_bytes().len(), 32); + assert_eq!(first.fingerprint_bytes(), second.fingerprint_bytes()); + assert_eq!(first.fingerprint_sha256_hex().len(), 64); + } + + #[test] + fn full_afrinic_ipv4_fixture_is_parseable() { + let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/ta_constraints/afrinic-full-ipv4-deny.constraints"); + let raw = std::fs::read_to_string(&path).expect("read full AFRINIC fixture"); + let source_ipv4_rules = raw + .lines() + .filter(|line| line.trim_start().starts_with("deny ")) + .count(); + assert_eq!(source_ipv4_rules, 383); + + let constraints = TaConstraints::from_file(&path).expect("parse full AFRINIC fixture"); + assert_eq!(constraints.allow_v4.len(), 1); + assert_eq!(constraints.allow_v6.len(), 1); + assert_eq!(constraints.allow_asn.len(), 1); + assert!(!constraints.deny_v4.is_empty()); + } + + #[test] + fn current_afrinic_ipv4_fixture_is_parseable_and_canonical() { + let path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/ta_constraints/afrinic-current-ipv4-deny.constraints"); + let raw = std::fs::read_to_string(&path).expect("read current AFRINIC fixture"); + let source_ipv4_rules = raw + .lines() + .filter(|line| line.trim_start().starts_with("deny ")) + .count(); + assert_eq!(source_ipv4_rules, 1825); + + let constraints = TaConstraints::from_file(&path).expect("parse current AFRINIC fixture"); + assert_eq!(constraints.allow_v4.len(), 1); + assert_eq!(constraints.allow_v6.len(), 1); + assert_eq!(constraints.allow_asn.len(), 1); + assert_eq!(constraints.deny_v4.len(), 815); + assert!(constraints.deny_v4.uses_tree()); + assert!(!constraints.allow_v4.uses_tree()); + assert!(!constraints.allow_v6.uses_tree()); + assert!(!constraints.allow_asn.uses_tree()); + assert!(constraints.warnings.is_empty()); + } +} diff --git a/crates/panda-rpki-validator/src/validation/ca_instance.rs b/crates/panda-rpki-validator/src/validation/ca_instance.rs new file mode 100644 index 0000000..ca72d27 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/ca_instance.rs @@ -0,0 +1,132 @@ +use crate::data_model::oid::{OID_AD_CA_REPOSITORY, OID_AD_RPKI_MANIFEST, OID_AD_RPKI_NOTIFY}; +use crate::data_model::rc::{ResourceCertKind, ResourceCertificate, SubjectInfoAccess}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CaInstanceUris { + /// rsync sync base URI used for live/replay publication point fetches (must end with `/`). + /// + /// This is the parent directory of `manifest_rsync_uri`, not necessarily the full + /// `id-ad-caRepository` URI. Using the manifest parent keeps sync scope narrow while + /// `publication_point_rsync_uri` preserves the full RFC 9286 publication-point value. + pub rsync_base_uri: String, + /// rsync URI for the manifest object (`.mft`). + pub manifest_rsync_uri: String, + /// Publication point rsync URI (RFC 9286 terminology). + pub publication_point_rsync_uri: String, + /// Optional RRDP notification URI (https). + pub rrdp_notification_uri: Option, +} + +#[derive(Debug, thiserror::Error)] +pub enum CaInstanceUrisError { + #[error("certificate must be a CA certificate (RFC 6487 §4.8.1)")] + NotCa, + + #[error( + "CA certificate must contain Subject Information Access extension (RFC 6487 §4.8.8; RFC 5280 §4.2.2.2)" + )] + MissingSia, + + #[error("CA certificate SIA must contain id-ad-caRepository rsync URI (RFC 6487 §4.8.8.1)")] + MissingCaRepository, + + #[error("CA certificate SIA must contain id-ad-rpkiManifest rsync URI (RFC 6487 §4.8.8.2)")] + MissingRpkiManifest, + + #[error( + "SIA id-ad-caRepository accessLocation must be rsync:// URI, got {0} (RFC 6487 §4.8.8.1)" + )] + CaRepositoryNotRsync(String), + + #[error( + "SIA id-ad-rpkiManifest accessLocation must be rsync:// URI, got {0} (RFC 6487 §4.8.8.2)" + )] + RpkiManifestNotRsync(String), + + #[error( + "SIA id-ad-rpkiNotify accessLocation must be https:// URI, got {0} (RFC 8182 §3.4.1; RFC 6487 §4.8.8.3)" + )] + RpkiNotifyNotHttps(String), + + #[error( + "manifest rsync URI must be under CA publication point: manifest={manifest_rsync_uri} publication_point={publication_point_rsync_uri} (RFC 9286 §6.1)" + )] + ManifestNotUnderPublicationPoint { + manifest_rsync_uri: String, + publication_point_rsync_uri: String, + }, +} + +pub fn ca_instance_uris_from_ca_certificate( + cert: &ResourceCertificate, +) -> Result { + if cert.kind != ResourceCertKind::Ca { + return Err(CaInstanceUrisError::NotCa); + } + + let sia = cert + .tbs + .extensions + .subject_info_access + .as_ref() + .ok_or(CaInstanceUrisError::MissingSia)?; + + let access_descriptions = match sia { + SubjectInfoAccess::Ca(ca) => &ca.access_descriptions, + SubjectInfoAccess::Ee(_ee) => return Err(CaInstanceUrisError::MissingSia), + }; + + let mut ca_repo: Option = None; + let mut manifest: Option = None; + let mut notify: Option = None; + + for ad in access_descriptions { + if ad.access_method_oid == OID_AD_CA_REPOSITORY { + let u = ad.access_location.as_str(); + if !u.starts_with("rsync://") { + return Err(CaInstanceUrisError::CaRepositoryNotRsync(u.to_string())); + } + ca_repo.get_or_insert(u.to_string()); + } else if ad.access_method_oid == OID_AD_RPKI_MANIFEST { + let u = ad.access_location.as_str(); + if !u.starts_with("rsync://") { + return Err(CaInstanceUrisError::RpkiManifestNotRsync(u.to_string())); + } + manifest.get_or_insert(u.to_string()); + } else if ad.access_method_oid == OID_AD_RPKI_NOTIFY { + let u = ad.access_location.as_str(); + if !u.starts_with("https://") { + return Err(CaInstanceUrisError::RpkiNotifyNotHttps(u.to_string())); + } + notify.get_or_insert(u.to_string()); + } + } + + let mut publication_point_rsync_uri = + ca_repo.ok_or(CaInstanceUrisError::MissingCaRepository)?; + if !publication_point_rsync_uri.ends_with('/') { + publication_point_rsync_uri.push('/'); + } + + let manifest_rsync_uri = manifest.ok_or(CaInstanceUrisError::MissingRpkiManifest)?; + if !manifest_rsync_uri.starts_with(&publication_point_rsync_uri) { + return Err(CaInstanceUrisError::ManifestNotUnderPublicationPoint { + manifest_rsync_uri, + publication_point_rsync_uri, + }); + } + let manifest_parent = manifest_rsync_uri + .rsplit_once('/') + .map(|(parent, _)| format!("{parent}/")) + .ok_or_else(|| CaInstanceUrisError::ManifestNotUnderPublicationPoint { + manifest_rsync_uri: manifest_rsync_uri.clone(), + publication_point_rsync_uri: publication_point_rsync_uri.clone(), + })?; + + Ok(CaInstanceUris { + rsync_base_uri: manifest_parent, + manifest_rsync_uri, + publication_point_rsync_uri, + rrdp_notification_uri: notify, + }) +} diff --git a/crates/panda-rpki-validator/src/validation/ca_path.rs b/crates/panda-rpki-validator/src/validation/ca_path.rs new file mode 100644 index 0000000..d67c7c2 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/ca_path.rs @@ -0,0 +1,2409 @@ +use crate::data_model::common::BigUnsigned; +use crate::data_model::crl::{CrlDecodeError, CrlVerifyError, RpkixCrl}; +use crate::data_model::oid::OID_KEY_USAGE_RAW; +use crate::data_model::rc::{ + AsIdentifierChoice, AsResourceSet, IpAddressChoice, IpResourceSet, ResourceCertKind, + ResourceCertificate, ResourceCertificateDecodeError, ResourceCertificateProfileError, + ResourceCertificateRole, +}; +use crate::policy::ResourceValidationMode; +use x509_parser::prelude::{FromDer, X509Certificate}; + +use crate::validation::x509_name::x509_names_equivalent; +use std::collections::{BTreeMap, HashMap, HashSet}; +use x509_parser::x509::SubjectPublicKeyInfo; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ValidatedSubordinateCa { + pub child_ca: ResourceCertificate, + pub issuer_ca: ResourceCertificate, + pub issuer_crl: RpkixCrl, + pub effective_ip_resources: Option, + pub effective_as_resources: Option, + pub resource_warnings: ResourceValidationWarnings, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ValidatedSubordinateCaLite { + pub child_ca: ResourceCertificate, + pub effective_ip_resources: Option, + pub effective_as_resources: Option, + pub resource_warnings: ResourceValidationWarnings, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ResourceValidationWarnings { + pub ip_reduced_by_vrs: bool, + pub as_reduced_by_vrs: bool, + pub ip_vrs_empty: bool, + pub as_vrs_empty: bool, +} + +impl ResourceValidationWarnings { + pub fn is_empty(&self) -> bool { + !self.ip_reduced_by_vrs + && !self.as_reduced_by_vrs + && !self.ip_vrs_empty + && !self.as_vrs_empty + } + + pub fn summary(&self) -> String { + let mut parts = Vec::new(); + if self.ip_reduced_by_vrs { + parts.push("ip_reduced_by_vrs"); + } + if self.as_reduced_by_vrs { + parts.push("as_reduced_by_vrs"); + } + if self.ip_vrs_empty { + parts.push("ip_vrs_empty"); + } + if self.as_vrs_empty { + parts.push("as_vrs_empty"); + } + parts.join(",") + } +} + +#[derive(Clone, Debug, Default)] +pub struct IssuerEffectiveResourcesIndex { + parent_ip_by_afi_items: + Option>>, + parent_ip_merged_intervals: HashMap, Vec)>>, + parent_asnum_intervals: Option>, + parent_rdi_intervals: Option>, +} + +impl IssuerEffectiveResourcesIndex { + pub fn from_effective_resources( + issuer_effective_ip: Option<&IpResourceSet>, + issuer_effective_as: Option<&AsResourceSet>, + ) -> Result { + let parent_ip_by_afi_items = issuer_effective_ip + .map(ip_resources_by_afi_items) + .transpose()?; + + let parent_ip_merged_intervals = issuer_effective_ip + .map(ip_resources_to_merged_intervals_by_afi) + .unwrap_or_default(); + + let parent_asnum_intervals = issuer_effective_as + .and_then(|resources| resources.asnum.as_ref().map(as_choice_to_merged_intervals)); + let parent_rdi_intervals = issuer_effective_as + .and_then(|resources| resources.rdi.as_ref().map(as_choice_to_merged_intervals)); + + Ok(Self { + parent_ip_by_afi_items, + parent_ip_merged_intervals, + parent_asnum_intervals, + parent_rdi_intervals, + }) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum CaPathError { + #[error("child CA certificate decode failed: {0} (RFC 6487 §4; RFC 5280 §4.1)")] + ChildDecode(#[from] ResourceCertificateDecodeError), + + #[error("issuer CA certificate decode failed: {0} (RFC 6487 §4; RFC 5280 §4.1)")] + IssuerDecode(ResourceCertificateDecodeError), + + #[error("child CA certificate profile validation failed: {0} (RFC 6487 §4.8)")] + ChildProfile(ResourceCertificateProfileError), + + #[error("issuer CA certificate profile validation failed: {0} (RFC 6487 §4.8)")] + IssuerProfile(ResourceCertificateProfileError), + + #[error("issuer CRL decode failed: {0} (RFC 6487 §5; RFC 9829 §3.1; RFC 5280 §5.1)")] + CrlDecode(#[from] CrlDecodeError), + + #[error( + "child certificate must be a CA resource certificate (RFC 6487 §4.8.1; RFC 5280 §4.2.1.9)" + )] + ChildNotCa, + + #[error( + "issuer certificate must be a CA resource certificate (RFC 6487 §4.8.1; RFC 5280 §4.2.1.9)" + )] + IssuerNotCa, + + #[error( + "child issuer DN does not match issuer CA subject DN: child.issuer={child_issuer_dn} issuer.subject={issuer_subject_dn} (RFC 5280 §6.1)" + )] + IssuerSubjectMismatch { + child_issuer_dn: String, + issuer_subject_dn: String, + }, + + #[error("child CA certificate signature verification failed: {0} (RFC 5280 §6.1)")] + ChildSignatureInvalid(String), + + #[error("issuer SubjectPublicKeyInfo parse error: {0} (RFC 5280 §4.1.2.7)")] + IssuerSpkiParse(String), + + #[error( + "trailing bytes after issuer SubjectPublicKeyInfo DER: {0} bytes (DER; RFC 5280 §4.1.2.7)" + )] + IssuerSpkiTrailingBytes(usize), + + #[error("certificate not valid at validation_time (RFC 5280 §4.1.2.5; RFC 5280 §6.1)")] + CertificateNotValidAtTime, + + #[error("child CA KeyUsage extension missing (RFC 6487 §4.8.4; RFC 5280 §4.2.1.3)")] + KeyUsageMissing, + + #[error("child CA KeyUsage criticality must be critical (RFC 6487 §4.8.4; RFC 5280 §4.2.1.3)")] + KeyUsageNotCritical, + + #[error("child CA KeyUsage must have only keyCertSign and cRLSign set (RFC 6487 §4.8.4)")] + KeyUsageInvalidBits, + + #[error( + "CRL signature/binding verification failed: {0} (RFC 5280 §6.3.3; RFC 6487 §5; RFC 9829 §3.1)" + )] + CrlVerify(#[from] CrlVerifyError), + + #[error( + "CRL not valid at validation_time (RFC 5280 §6.3.3(g); RFC 5280 §5.1.2.4-§5.1.2.5; RFC 6487 §5)" + )] + CrlNotValidAtTime, + + #[error("child CA certificate is revoked by issuer CRL (RFC 5280 §6.3.3; RFC 6487 §5)")] + ChildRevoked, + + #[error( + "child CA certificate must contain at least one RFC 3779 resource extension (IP or AS) (RFC 6487 §4.8.10-§4.8.11)" + )] + ResourcesMissing, + + #[error( + "resource extension inheritance cannot be resolved (parent missing resources) (RFC 6487 §7.2)" + )] + InheritWithoutParentResources, + + #[error("child CA resources are not a subset of issuer resources (RFC 6487 §7.2)")] + ResourcesNotSubset, + + #[error("issuer CA subjectKeyIdentifier missing (RFC 6487 §4.8.2)")] + IssuerSkiMissing, + + #[error("child CA authorityKeyIdentifier missing (RFC 6487 §4.8.3; RFC 5280 §4.2.1.1)")] + ChildAkiMissing, + + #[error( + "child CA authorityKeyIdentifier does not match issuer subjectKeyIdentifier (RFC 6487 §4.8.3)" + )] + ChildAkiMismatch, + + #[error("child CA authorityInfoAccess missing (RFC 6487 §4.8.7; RFC 5280 §4.2.2.1)")] + ChildAiaMissing, + + #[error( + "child CA authorityInfoAccess does not reference issuer certificate rsync URI (RFC 6487 §4.8.7)" + )] + ChildAiaIssuerUriMismatch, + + #[error("child CA CRLDistributionPoints missing (RFC 6487 §4.8.6; RFC 5280 §4.2.1.13)")] + ChildCrlDpMissing, + + #[error( + "child CA CRLDistributionPoints does not reference issuer CRL rsync URI (RFC 6487 §4.8.6)" + )] + ChildCrlDpUriMismatch, +} + +pub fn validate_subordinate_ca_cert( + child_ca_der: &[u8], + issuer_ca_der: &[u8], + issuer_crl_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_crl_rsync_uri: &str, + issuer_effective_ip: Option<&IpResourceSet>, + issuer_effective_as: Option<&AsResourceSet>, + validation_time: time::OffsetDateTime, +) -> Result { + validate_subordinate_ca_cert_with_resource_validation_mode( + child_ca_der, + issuer_ca_der, + issuer_crl_der, + issuer_ca_rsync_uri, + issuer_crl_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + ResourceValidationMode::Rfc6487, + ) +} + +pub fn validate_subordinate_ca_cert_with_resource_validation_mode( + child_ca_der: &[u8], + issuer_ca_der: &[u8], + issuer_crl_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_crl_rsync_uri: &str, + issuer_effective_ip: Option<&IpResourceSet>, + issuer_effective_as: Option<&AsResourceSet>, + validation_time: time::OffsetDateTime, + resource_validation_mode: ResourceValidationMode, +) -> Result { + let child_ca = ResourceCertificate::decode_der(child_ca_der)?; + if child_ca.kind != ResourceCertKind::Ca { + return Err(CaPathError::ChildNotCa); + } + child_ca + .validate_rfc6487_profile(ResourceCertificateRole::Ca) + .map_err(CaPathError::ChildProfile)?; + + let issuer_ca = + ResourceCertificate::decode_der(issuer_ca_der).map_err(CaPathError::IssuerDecode)?; + if issuer_ca.kind != ResourceCertKind::Ca { + return Err(CaPathError::IssuerNotCa); + } + issuer_ca + .validate_rfc6487_profile(ResourceCertificateRole::Ca) + .map_err(CaPathError::IssuerProfile)?; + let issuer_spki = parse_subject_pki_from_der(&issuer_ca.tbs.subject_public_key_info)?; + + if !x509_names_equivalent(&child_ca.tbs.issuer_name, &issuer_ca.tbs.subject_name) { + return Err(CaPathError::IssuerSubjectMismatch { + child_issuer_dn: child_ca.tbs.issuer_name.to_string(), + issuer_subject_dn: issuer_ca.tbs.subject_name.to_string(), + }); + } + + validate_child_aki_matches_issuer_ski(&child_ca, &issuer_ca)?; + if let Some(expected_issuer_uri) = issuer_ca_rsync_uri { + validate_child_aia_points_to_issuer_uri(&child_ca, expected_issuer_uri)?; + } + validate_child_crldp_contains_issuer_crl_uri(&child_ca, issuer_crl_rsync_uri)?; + + if !time_within_validity( + validation_time, + child_ca.tbs.validity_not_before, + child_ca.tbs.validity_not_after, + ) || !time_within_validity( + validation_time, + issuer_ca.tbs.validity_not_before, + issuer_ca.tbs.validity_not_after, + ) { + return Err(CaPathError::CertificateNotValidAtTime); + } + + let child_x509 = parse_x509_cert(child_ca_der)?; + verify_child_signature(&child_x509, &issuer_spki)?; + validate_child_ca_key_usage(&child_x509)?; + + let issuer_crl = RpkixCrl::decode_der(issuer_crl_der)?; + issuer_crl.verify_signature_with_issuer_certificate_der(issuer_ca_der)?; + if !crl_valid_at_time(&issuer_crl, validation_time) { + return Err(CaPathError::CrlNotValidAtTime); + } + + if is_serial_revoked_by_crl(&child_ca, &issuer_crl) { + return Err(CaPathError::ChildRevoked); + } + + let ResourceResolution { + effective_ip_resources, + effective_as_resources, + warnings: resource_warnings, + } = resolve_child_resources( + child_ca.tbs.extensions.ip_resources.as_ref(), + issuer_effective_ip, + child_ca.tbs.extensions.as_resources.as_ref(), + issuer_effective_as, + &IssuerEffectiveResourcesIndex::from_effective_resources( + issuer_effective_ip, + issuer_effective_as, + )?, + resource_validation_mode, + )?; + if effective_ip_resources.is_none() && effective_as_resources.is_none() { + return Err(CaPathError::ResourcesMissing); + } + + Ok(ValidatedSubordinateCa { + child_ca, + issuer_ca, + issuer_crl, + effective_ip_resources, + effective_as_resources, + resource_warnings, + }) +} + +/// Validate a subordinate child CA using *pre-decoded issuer CA* and *pre-decoded+verified issuer CRL*. +/// +/// This avoids repeating issuer CA decode and issuer CRL decode+signature verification for every +/// child CA certificate discovered in a publication point. +pub fn validate_subordinate_ca_cert_with_prevalidated_issuer( + child_ca_der: &[u8], + child_ca: ResourceCertificate, + issuer_ca: &ResourceCertificate, + issuer_spki: &SubjectPublicKeyInfo<'_>, + issuer_crl: &RpkixCrl, + issuer_crl_revoked_serials: &HashSet>, + issuer_ca_rsync_uri: Option<&str>, + issuer_crl_rsync_uri: &str, + issuer_effective_ip: Option<&IpResourceSet>, + issuer_effective_as: Option<&AsResourceSet>, + validation_time: time::OffsetDateTime, +) -> Result { + let issuer_resources_index = IssuerEffectiveResourcesIndex::from_effective_resources( + issuer_effective_ip, + issuer_effective_as, + )?; + validate_subordinate_ca_cert_with_prevalidated_issuer_and_resources( + child_ca_der, + child_ca, + issuer_ca, + issuer_spki, + issuer_crl, + issuer_crl_revoked_serials, + issuer_ca_rsync_uri, + issuer_crl_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + &issuer_resources_index, + validation_time, + ResourceValidationMode::Rfc6487, + ) +} + +pub fn validate_subordinate_ca_cert_with_prevalidated_issuer_and_resources( + child_ca_der: &[u8], + child_ca: ResourceCertificate, + issuer_ca: &ResourceCertificate, + issuer_spki: &SubjectPublicKeyInfo<'_>, + issuer_crl: &RpkixCrl, + issuer_crl_revoked_serials: &HashSet>, + issuer_ca_rsync_uri: Option<&str>, + issuer_crl_rsync_uri: &str, + issuer_effective_ip: Option<&IpResourceSet>, + issuer_effective_as: Option<&AsResourceSet>, + issuer_resources_index: &IssuerEffectiveResourcesIndex, + validation_time: time::OffsetDateTime, + resource_validation_mode: ResourceValidationMode, +) -> Result { + if child_ca.kind != ResourceCertKind::Ca { + return Err(CaPathError::ChildNotCa); + } + if issuer_ca.kind != ResourceCertKind::Ca { + return Err(CaPathError::IssuerNotCa); + } + child_ca + .validate_rfc6487_profile(ResourceCertificateRole::Ca) + .map_err(CaPathError::ChildProfile)?; + issuer_ca + .validate_rfc6487_profile(ResourceCertificateRole::Ca) + .map_err(CaPathError::IssuerProfile)?; + + if !x509_names_equivalent(&child_ca.tbs.issuer_name, &issuer_ca.tbs.subject_name) { + return Err(CaPathError::IssuerSubjectMismatch { + child_issuer_dn: child_ca.tbs.issuer_name.to_string(), + issuer_subject_dn: issuer_ca.tbs.subject_name.to_string(), + }); + } + + validate_child_aki_matches_issuer_ski(&child_ca, issuer_ca)?; + if let Some(expected_issuer_uri) = issuer_ca_rsync_uri { + validate_child_aia_points_to_issuer_uri(&child_ca, expected_issuer_uri)?; + } + validate_child_crldp_contains_issuer_crl_uri(&child_ca, issuer_crl_rsync_uri)?; + + if !time_within_validity( + validation_time, + child_ca.tbs.validity_not_before, + child_ca.tbs.validity_not_after, + ) || !time_within_validity( + validation_time, + issuer_ca.tbs.validity_not_before, + issuer_ca.tbs.validity_not_after, + ) { + return Err(CaPathError::CertificateNotValidAtTime); + } + + let child_x509 = parse_x509_cert(child_ca_der)?; + verify_child_signature(&child_x509, issuer_spki)?; + validate_child_ca_key_usage(&child_x509)?; + + if !crl_valid_at_time(issuer_crl, validation_time) { + return Err(CaPathError::CrlNotValidAtTime); + } + + let serial = BigUnsigned::from_biguint(&child_ca.tbs.serial_number); + if issuer_crl_revoked_serials.contains(&serial.bytes_be) { + return Err(CaPathError::ChildRevoked); + } + + let ResourceResolution { + effective_ip_resources, + effective_as_resources, + warnings: resource_warnings, + } = resolve_child_resources( + child_ca.tbs.extensions.ip_resources.as_ref(), + issuer_effective_ip, + child_ca.tbs.extensions.as_resources.as_ref(), + issuer_effective_as, + issuer_resources_index, + resource_validation_mode, + )?; + if effective_ip_resources.is_none() && effective_as_resources.is_none() { + return Err(CaPathError::ResourcesMissing); + } + + Ok(ValidatedSubordinateCaLite { + child_ca, + effective_ip_resources, + effective_as_resources, + resource_warnings, + }) +} + +fn parse_subject_pki_from_der(der: &[u8]) -> Result, CaPathError> { + let (rem, spki) = SubjectPublicKeyInfo::from_der(der) + .map_err(|e| CaPathError::IssuerSpkiParse(e.to_string()))?; + if !rem.is_empty() { + return Err(CaPathError::IssuerSpkiTrailingBytes(rem.len())); + } + Ok(spki) +} + +fn parse_x509_cert(der: &[u8]) -> Result, CaPathError> { + let (rem, cert) = X509Certificate::from_der(der) + .map_err(|e| CaPathError::ChildSignatureInvalid(e.to_string()))?; + if !rem.is_empty() { + return Err(CaPathError::ChildSignatureInvalid( + "trailing bytes after child certificate".to_string(), + )); + } + Ok(cert) +} + +fn verify_child_signature( + child: &X509Certificate<'_>, + issuer_spki: &SubjectPublicKeyInfo<'_>, +) -> Result<(), CaPathError> { + crate::crypto_sig_cache::verify_with_cache( + crate::crypto_sig_cache::CryptoSigVerifyPoint::ChildCaCert, + child.tbs_certificate.as_ref(), + child.signature_value.data.as_ref(), + issuer_spki.raw, + || { + child + .verify_signature(Some(issuer_spki)) + .map_err(|e| CaPathError::ChildSignatureInvalid(e.to_string())) + }, + ) +} + +fn validate_child_aki_matches_issuer_ski( + child: &ResourceCertificate, + issuer: &ResourceCertificate, +) -> Result<(), CaPathError> { + let Some(issuer_ski) = issuer.tbs.extensions.subject_key_identifier.as_deref() else { + return Err(CaPathError::IssuerSkiMissing); + }; + let Some(child_aki) = child.tbs.extensions.authority_key_identifier.as_deref() else { + return Err(CaPathError::ChildAkiMissing); + }; + if child_aki != issuer_ski { + return Err(CaPathError::ChildAkiMismatch); + } + Ok(()) +} + +fn validate_child_aia_points_to_issuer_uri( + child: &ResourceCertificate, + issuer_ca_rsync_uri: &str, +) -> Result<(), CaPathError> { + let Some(uris) = child.tbs.extensions.ca_issuers_uris.as_ref() else { + return Err(CaPathError::ChildAiaMissing); + }; + if !uris.iter().any(|u| u.as_str() == issuer_ca_rsync_uri) { + return Err(CaPathError::ChildAiaIssuerUriMismatch); + } + Ok(()) +} + +fn validate_child_crldp_contains_issuer_crl_uri( + child: &ResourceCertificate, + issuer_crl_rsync_uri: &str, +) -> Result<(), CaPathError> { + let Some(uris) = child.tbs.extensions.crl_distribution_points_uris.as_ref() else { + return Err(CaPathError::ChildCrlDpMissing); + }; + if !uris.iter().any(|u| u.as_str() == issuer_crl_rsync_uri) { + return Err(CaPathError::ChildCrlDpUriMismatch); + } + Ok(()) +} + +fn validate_child_ca_key_usage(cert: &X509Certificate<'_>) -> Result<(), CaPathError> { + let mut ku_critical: Option = None; + for ext in cert.extensions() { + if ext.oid.as_bytes() == OID_KEY_USAGE_RAW { + ku_critical = Some(ext.critical); + break; + } + } + + let Some(critical) = ku_critical else { + return Err(CaPathError::KeyUsageMissing); + }; + if !critical { + return Err(CaPathError::KeyUsageNotCritical); + } + + let Some(ku) = cert + .key_usage() + .map_err(|e| CaPathError::ChildSignatureInvalid(e.to_string()))? + else { + return Err(CaPathError::KeyUsageMissing); + }; + + let v = &ku.value; + let ok = v.key_cert_sign() + && v.crl_sign() + && !v.digital_signature() + && !v.non_repudiation() + && !v.key_encipherment() + && !v.data_encipherment() + && !v.key_agreement() + && !v.encipher_only() + && !v.decipher_only(); + if !ok { + return Err(CaPathError::KeyUsageInvalidBits); + } + + Ok(()) +} + +fn time_within_validity( + t: time::OffsetDateTime, + not_before: time::OffsetDateTime, + not_after: time::OffsetDateTime, +) -> bool { + let t = t.to_offset(time::UtcOffset::UTC); + let not_before = not_before.to_offset(time::UtcOffset::UTC); + let not_after = not_after.to_offset(time::UtcOffset::UTC); + t >= not_before && t <= not_after +} + +fn crl_valid_at_time(crl: &RpkixCrl, t: time::OffsetDateTime) -> bool { + let t = t.to_offset(time::UtcOffset::UTC); + let this_update = crl.this_update.utc.to_offset(time::UtcOffset::UTC); + let next_update = crl.next_update.utc.to_offset(time::UtcOffset::UTC); + t >= this_update && t < next_update +} + +fn is_serial_revoked_by_crl(cert: &ResourceCertificate, crl: &RpkixCrl) -> bool { + let serial = BigUnsigned::from_biguint(&cert.tbs.serial_number); + crl.revoked_certs + .iter() + .any(|rc| rc.serial_number == serial) +} + +#[derive(Clone, Debug)] +struct ResourceResolution { + effective_ip_resources: Option, + effective_as_resources: Option, + warnings: ResourceValidationWarnings, +} + +fn resolve_child_resources( + child_ip: Option<&IpResourceSet>, + issuer_effective_ip: Option<&IpResourceSet>, + child_as: Option<&AsResourceSet>, + issuer_effective_as: Option<&AsResourceSet>, + issuer_resources_index: &IssuerEffectiveResourcesIndex, + mode: ResourceValidationMode, +) -> Result { + match mode { + ResourceValidationMode::Rfc6487 => Ok(ResourceResolution { + effective_ip_resources: resolve_child_ip_resources_indexed( + child_ip, + issuer_effective_ip, + issuer_resources_index.parent_ip_by_afi_items.as_ref(), + &issuer_resources_index.parent_ip_merged_intervals, + )?, + effective_as_resources: resolve_child_as_resources_indexed( + child_as, + issuer_effective_as, + issuer_resources_index.parent_asnum_intervals.as_deref(), + issuer_resources_index.parent_rdi_intervals.as_deref(), + )?, + warnings: ResourceValidationWarnings::default(), + }), + ResourceValidationMode::ValidationUpdate03 => { + let (effective_ip_resources, ip_reduced_by_vrs, ip_vrs_empty) = + resolve_child_ip_resources_vrs( + child_ip, + issuer_effective_ip, + issuer_resources_index.parent_ip_by_afi_items.as_ref(), + &issuer_resources_index.parent_ip_merged_intervals, + )?; + let (effective_as_resources, as_reduced_by_vrs, as_vrs_empty) = + resolve_child_as_resources_vrs( + child_as, + issuer_effective_as, + issuer_resources_index.parent_asnum_intervals.as_deref(), + issuer_resources_index.parent_rdi_intervals.as_deref(), + )?; + Ok(ResourceResolution { + effective_ip_resources, + effective_as_resources, + warnings: ResourceValidationWarnings { + ip_reduced_by_vrs, + as_reduced_by_vrs, + ip_vrs_empty, + as_vrs_empty, + }, + }) + } + } +} + +fn resolve_child_ip_resources( + child_ip: Option<&IpResourceSet>, + issuer_effective: Option<&IpResourceSet>, +) -> Result, CaPathError> { + let precomputed_parent_by_afi = issuer_effective + .map(ip_resources_by_afi_items) + .transpose()?; + let precomputed_parent_intervals = issuer_effective + .map(ip_resources_to_merged_intervals_by_afi) + .unwrap_or_default(); + resolve_child_ip_resources_indexed( + child_ip, + issuer_effective, + precomputed_parent_by_afi.as_ref(), + &precomputed_parent_intervals, + ) +} + +fn resolve_child_ip_resources_indexed( + child_ip: Option<&IpResourceSet>, + issuer_effective: Option<&IpResourceSet>, + parent_by_afi: Option< + &BTreeMap>, + >, + parent_intervals_by_afi: &HashMap, Vec)>>, +) -> Result, CaPathError> { + let Some(child_ip) = child_ip else { + return Ok(None); + }; + + let Some(_parent) = issuer_effective else { + if child_ip.has_any_inherit() { + return Err(CaPathError::InheritWithoutParentResources); + } + // With no parent effective resources, we cannot validate subset. + return Err(CaPathError::ResourcesNotSubset); + }; + + // Resolve per-AFI inherit, producing an effective set with no inherit. + let parent_by_afi = parent_by_afi.ok_or(CaPathError::InheritWithoutParentResources)?; + let mut out_families: Vec = Vec::new(); + + for fam in &child_ip.families { + match &fam.choice { + IpAddressChoice::Inherit => { + let items = parent_by_afi + .get(&fam.afi) + .ok_or(CaPathError::InheritWithoutParentResources)?; + out_families.push(crate::data_model::rc::IpAddressFamily { + afi: fam.afi, + choice: IpAddressChoice::AddressesOrRanges(items.clone()), + }); + } + IpAddressChoice::AddressesOrRanges(items) => { + // Subset check against parent union for that AFI. + let parent_intervals = parent_intervals_by_afi + .get(&fam.afi) + .map(Vec::as_slice) + .unwrap_or(&[]); + if !ip_family_items_subset_with_parent_intervals(items, parent_intervals) { + return Err(CaPathError::ResourcesNotSubset); + } + out_families.push(crate::data_model::rc::IpAddressFamily { + afi: fam.afi, + choice: IpAddressChoice::AddressesOrRanges(items.clone()), + }); + } + } + } + + Ok(Some(IpResourceSet { + families: out_families, + })) +} + +fn resolve_child_ip_resources_vrs( + child_ip: Option<&IpResourceSet>, + issuer_effective: Option<&IpResourceSet>, + parent_by_afi: Option< + &BTreeMap>, + >, + parent_intervals_by_afi: &HashMap, Vec)>>, +) -> Result<(Option, bool, bool), CaPathError> { + let Some(child_ip) = child_ip else { + return Ok((None, false, false)); + }; + if child_ip.has_any_inherit() && issuer_effective.is_none() { + return Err(CaPathError::InheritWithoutParentResources); + } + + let parent_by_afi = parent_by_afi.unwrap_or_else(|| { + static EMPTY: std::sync::OnceLock< + BTreeMap>, + > = std::sync::OnceLock::new(); + EMPTY.get_or_init(BTreeMap::new) + }); + let mut out_families = Vec::new(); + let mut reduced = false; + let mut saw_declared = false; + + for fam in &child_ip.families { + match &fam.choice { + IpAddressChoice::Inherit => { + let items = parent_by_afi + .get(&fam.afi) + .ok_or(CaPathError::InheritWithoutParentResources)?; + out_families.push(crate::data_model::rc::IpAddressFamily { + afi: fam.afi, + choice: IpAddressChoice::AddressesOrRanges(items.clone()), + }); + } + IpAddressChoice::AddressesOrRanges(items) => { + saw_declared = saw_declared || !items.is_empty(); + let parent_intervals = parent_intervals_by_afi + .get(&fam.afi) + .map(Vec::as_slice) + .unwrap_or(&[]); + let intersections = + intersect_ip_items_with_parent_intervals(items, parent_intervals); + let child_intervals = ip_items_to_merged_intervals(items); + if intervals_changed(&child_intervals, &intersections) { + reduced = true; + } + if !intersections.is_empty() { + out_families.push(crate::data_model::rc::IpAddressFamily { + afi: fam.afi, + choice: IpAddressChoice::AddressesOrRanges(ip_intervals_to_ranges( + fam.afi, + &intersections, + )), + }); + } + } + } + } + + let empty = saw_declared && out_families.is_empty(); + Ok(( + Some(IpResourceSet { + families: out_families, + }), + reduced, + empty, + )) +} + +fn resolve_child_as_resources( + child_as: Option<&AsResourceSet>, + issuer_effective: Option<&AsResourceSet>, +) -> Result, CaPathError> { + let precomputed_asnum = issuer_effective + .and_then(|resources| resources.asnum.as_ref().map(as_choice_to_merged_intervals)); + let precomputed_rdi = issuer_effective + .and_then(|resources| resources.rdi.as_ref().map(as_choice_to_merged_intervals)); + resolve_child_as_resources_indexed( + child_as, + issuer_effective, + precomputed_asnum.as_deref(), + precomputed_rdi.as_deref(), + ) +} + +fn resolve_child_as_resources_indexed( + child_as: Option<&AsResourceSet>, + issuer_effective: Option<&AsResourceSet>, + parent_asnum_intervals: Option<&[(u32, u32)]>, + parent_rdi_intervals: Option<&[(u32, u32)]>, +) -> Result, CaPathError> { + let Some(child_as) = child_as else { + return Ok(None); + }; + let Some(parent) = issuer_effective else { + if matches!(child_as.asnum, Some(AsIdentifierChoice::Inherit)) + || matches!(child_as.rdi, Some(AsIdentifierChoice::Inherit)) + { + return Err(CaPathError::InheritWithoutParentResources); + } + return Err(CaPathError::ResourcesNotSubset); + }; + + let asnum = match child_as.asnum.as_ref() { + None => None, + Some(AsIdentifierChoice::Inherit) => parent + .asnum + .clone() + .ok_or(CaPathError::InheritWithoutParentResources) + .map(Some)?, + Some(_) => { + if !as_choice_subset_with_parent_intervals( + child_as.asnum.as_ref(), + parent.asnum.as_ref(), + parent_asnum_intervals, + ) { + return Err(CaPathError::ResourcesNotSubset); + } + child_as.asnum.clone() + } + }; + + let rdi = match child_as.rdi.as_ref() { + None => None, + Some(AsIdentifierChoice::Inherit) => parent + .rdi + .clone() + .ok_or(CaPathError::InheritWithoutParentResources) + .map(Some)?, + Some(_) => { + if !as_choice_subset_with_parent_intervals( + child_as.rdi.as_ref(), + parent.rdi.as_ref(), + parent_rdi_intervals, + ) { + return Err(CaPathError::ResourcesNotSubset); + } + child_as.rdi.clone() + } + }; + + Ok(Some(AsResourceSet { asnum, rdi })) +} + +fn resolve_child_as_resources_vrs( + child_as: Option<&AsResourceSet>, + issuer_effective: Option<&AsResourceSet>, + parent_asnum_intervals: Option<&[(u32, u32)]>, + parent_rdi_intervals: Option<&[(u32, u32)]>, +) -> Result<(Option, bool, bool), CaPathError> { + let Some(child_as) = child_as else { + return Ok((None, false, false)); + }; + if issuer_effective.is_none() + && (matches!(child_as.asnum, Some(AsIdentifierChoice::Inherit)) + || matches!(child_as.rdi, Some(AsIdentifierChoice::Inherit))) + { + return Err(CaPathError::InheritWithoutParentResources); + } + + let (asnum, asnum_reduced, asnum_empty) = resolve_as_choice_vrs( + child_as.asnum.as_ref(), + issuer_effective.and_then(|p| p.asnum.as_ref()), + parent_asnum_intervals, + )?; + let (rdi, rdi_reduced, rdi_empty) = resolve_as_choice_vrs( + child_as.rdi.as_ref(), + issuer_effective.and_then(|p| p.rdi.as_ref()), + parent_rdi_intervals, + )?; + Ok(( + Some(AsResourceSet { asnum, rdi }), + asnum_reduced || rdi_reduced, + asnum_empty || rdi_empty, + )) +} + +fn resolve_as_choice_vrs( + child: Option<&AsIdentifierChoice>, + parent: Option<&AsIdentifierChoice>, + parent_intervals_hint: Option<&[(u32, u32)]>, +) -> Result<(Option, bool, bool), CaPathError> { + let Some(child) = child else { + return Ok((None, false, false)); + }; + match child { + AsIdentifierChoice::Inherit => { + let parent = parent + .cloned() + .ok_or(CaPathError::InheritWithoutParentResources)?; + Ok((Some(parent), false, false)) + } + AsIdentifierChoice::AsIdsOrRanges(_) => { + let child_intervals = as_choice_to_merged_intervals(child); + let parent_intervals; + let parent_intervals = match parent_intervals_hint { + Some(v) => v, + None => { + parent_intervals = parent + .map(as_choice_to_merged_intervals) + .unwrap_or_default(); + parent_intervals.as_slice() + } + }; + let intersections = intersect_as_intervals(&child_intervals, parent_intervals); + let reduced = child_intervals != intersections; + let empty = !child_intervals.is_empty() && intersections.is_empty(); + Ok(( + Some(AsIdentifierChoice::AsIdsOrRanges(as_intervals_to_items( + &intersections, + ))), + reduced, + empty, + )) + } + } +} + +fn as_choice_subset( + child: Option<&AsIdentifierChoice>, + parent: Option<&AsIdentifierChoice>, +) -> bool { + as_choice_subset_with_parent_intervals(child, parent, None) +} + +fn as_choice_subset_with_parent_intervals( + child: Option<&AsIdentifierChoice>, + parent: Option<&AsIdentifierChoice>, + parent_intervals_hint: Option<&[(u32, u32)]>, +) -> bool { + let Some(child) = child else { + return true; + }; + let Some(parent) = parent else { + return false; + }; + + // Treat inherit as "all of parent" here; actual resolution is handled elsewhere. + if matches!(child, AsIdentifierChoice::Inherit) { + return true; + } + if matches!(parent, AsIdentifierChoice::Inherit) { + return true; + } + + let child_intervals = as_choice_to_merged_intervals(child); + let owned_parent_intervals; + let parent_intervals = match parent_intervals_hint { + Some(intervals) => intervals, + None => { + owned_parent_intervals = as_choice_to_merged_intervals(parent); + owned_parent_intervals.as_slice() + } + }; + for (cmin, cmax) in &child_intervals { + if !as_interval_is_covered(parent_intervals, *cmin, *cmax) { + return false; + } + } + true +} + +fn as_choice_to_merged_intervals(choice: &AsIdentifierChoice) -> Vec<(u32, u32)> { + let mut v = Vec::new(); + match choice { + AsIdentifierChoice::Inherit => {} + AsIdentifierChoice::AsIdsOrRanges(items) => { + for item in items { + match item { + crate::data_model::rc::AsIdOrRange::Id(id) => v.push((*id, *id)), + crate::data_model::rc::AsIdOrRange::Range { min, max } => v.push((*min, *max)), + } + } + } + } + v.sort_by_key(|(a, _b)| *a); + merge_as_intervals(&v) +} + +fn merge_as_intervals(v: &[(u32, u32)]) -> Vec<(u32, u32)> { + let mut out: Vec<(u32, u32)> = Vec::new(); + for (min, max) in v { + let Some(last) = out.last_mut() else { + out.push((*min, *max)); + continue; + }; + if *min <= last.1.saturating_add(1) { + last.1 = last.1.max(*max); + continue; + } + out.push((*min, *max)); + } + out +} + +fn as_interval_is_covered(parent: &[(u32, u32)], min: u32, max: u32) -> bool { + for (pmin, pmax) in parent { + if *pmin <= min && max <= *pmax { + return true; + } + if *pmin > min { + break; + } + } + false +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +enum AfiKey { + V4, + V6, +} + +fn ip_resources_by_afi_items( + set: &IpResourceSet, +) -> Result< + std::collections::BTreeMap< + crate::data_model::rc::Afi, + Vec, + >, + CaPathError, +> { + let mut m: std::collections::BTreeMap< + crate::data_model::rc::Afi, + Vec, + > = std::collections::BTreeMap::new(); + for fam in &set.families { + match &fam.choice { + IpAddressChoice::Inherit => return Err(CaPathError::InheritWithoutParentResources), + IpAddressChoice::AddressesOrRanges(items) => { + m.insert(fam.afi, items.clone()); + } + } + } + Ok(m) +} + +fn ip_resources_single_afi( + parent: &IpResourceSet, + afi: crate::data_model::rc::Afi, + items: Option<&Vec>, +) -> IpResourceSet { + let mut families = Vec::new(); + if let Some(items) = items { + families.push(crate::data_model::rc::IpAddressFamily { + afi, + choice: IpAddressChoice::AddressesOrRanges(items.clone()), + }); + } else { + // If parent doesn't mention this AFI explicitly, treat as empty. + // The subset check will fail. + let _ = parent; + } + IpResourceSet { families } +} + +fn ip_family_items_subset( + child_items: &[crate::data_model::rc::IpAddressOrRange], + parent_set: &IpResourceSet, +) -> bool { + let parent_by_afi = ip_resources_to_merged_intervals(parent_set); + // parent_set should contain exactly one AFI. + let (afi_key, parent_intervals) = match parent_by_afi.into_iter().next() { + None => return false, + Some(v) => v, + }; + + let mut child_intervals: Vec<(Vec, Vec)> = Vec::new(); + for item in child_items { + match item { + crate::data_model::rc::IpAddressOrRange::Prefix(p) => { + child_intervals.push(prefix_to_range(p)) + } + crate::data_model::rc::IpAddressOrRange::Range(r) => { + child_intervals.push((r.min.clone(), r.max.clone())) + } + } + } + child_intervals.sort_by(|(a, _), (b, _)| a.cmp(b)); + let child_intervals = merge_ip_intervals(&child_intervals); + + let _ = afi_key; + for (cmin, cmax) in &child_intervals { + if !interval_is_covered(&parent_intervals, cmin, cmax) { + return false; + } + } + true +} + +fn ip_resources_to_merged_intervals( + set: &IpResourceSet, +) -> std::collections::HashMap, Vec)>> { + let m = ip_resources_to_merged_intervals_by_afi(set); + m.into_iter() + .map(|(afi, ranges)| { + ( + match afi { + crate::data_model::rc::Afi::Ipv4 => AfiKey::V4, + crate::data_model::rc::Afi::Ipv6 => AfiKey::V6, + }, + ranges, + ) + }) + .collect() +} + +fn ip_resources_to_merged_intervals_by_afi( + set: &IpResourceSet, +) -> HashMap, Vec)>> { + let mut m: HashMap, Vec)>> = HashMap::new(); + + for fam in &set.families { + match &fam.choice { + IpAddressChoice::Inherit => { + // When used in subset checks, treat inherit as "all" by leaving it absent. + // Resolution should have happened earlier. + } + IpAddressChoice::AddressesOrRanges(items) => { + let ent = m.entry(fam.afi).or_default(); + for item in items { + match item { + crate::data_model::rc::IpAddressOrRange::Prefix(p) => { + let (min, max) = prefix_to_range(p); + ent.push((min, max)); + } + crate::data_model::rc::IpAddressOrRange::Range(r) => { + ent.push((r.min.clone(), r.max.clone())); + } + } + } + } + } + } + + for (_afi, v) in m.iter_mut() { + v.sort_by(|(a, _), (b, _)| a.cmp(b)); + *v = merge_ip_intervals(v); + } + + m +} + +fn ip_family_items_subset_with_parent_intervals( + child_items: &[crate::data_model::rc::IpAddressOrRange], + parent_intervals: &[(Vec, Vec)], +) -> bool { + if parent_intervals.is_empty() { + return false; + } + + let mut child_intervals: Vec<(Vec, Vec)> = Vec::new(); + for item in child_items { + match item { + crate::data_model::rc::IpAddressOrRange::Prefix(p) => { + child_intervals.push(prefix_to_range(p)) + } + crate::data_model::rc::IpAddressOrRange::Range(r) => { + child_intervals.push((r.min.clone(), r.max.clone())) + } + } + } + child_intervals.sort_by(|(a, _), (b, _)| a.cmp(b)); + let child_intervals = merge_ip_intervals(&child_intervals); + + for (cmin, cmax) in &child_intervals { + if !interval_is_covered(parent_intervals, cmin, cmax) { + return false; + } + } + true +} + +fn ip_items_to_merged_intervals( + items: &[crate::data_model::rc::IpAddressOrRange], +) -> Vec<(Vec, Vec)> { + let mut intervals = Vec::new(); + for item in items { + match item { + crate::data_model::rc::IpAddressOrRange::Prefix(p) => { + intervals.push(prefix_to_range(p)) + } + crate::data_model::rc::IpAddressOrRange::Range(r) => { + intervals.push((r.min.clone(), r.max.clone())) + } + } + } + intervals.sort_by(|(a, _), (b, _)| a.cmp(b)); + merge_ip_intervals(&intervals) +} + +fn intersect_ip_items_with_parent_intervals( + items: &[crate::data_model::rc::IpAddressOrRange], + parent_intervals: &[(Vec, Vec)], +) -> Vec<(Vec, Vec)> { + let child_intervals = ip_items_to_merged_intervals(items); + intersect_ip_intervals(&child_intervals, parent_intervals) +} + +fn intersect_ip_intervals( + child_intervals: &[(Vec, Vec)], + parent_intervals: &[(Vec, Vec)], +) -> Vec<(Vec, Vec)> { + let mut out = Vec::new(); + let mut parent_index = 0usize; + for (child_min, child_max) in child_intervals { + while parent_index < parent_intervals.len() + && parent_intervals[parent_index].1.as_slice() < child_min.as_slice() + { + parent_index += 1; + } + let mut scan = parent_index; + while scan < parent_intervals.len() + && parent_intervals[scan].0.as_slice() <= child_max.as_slice() + { + let (parent_min, parent_max) = &parent_intervals[scan]; + let min = if bytes_leq(child_min, parent_min) { + parent_min.clone() + } else { + child_min.clone() + }; + let max = if bytes_leq(child_max, parent_max) { + child_max.clone() + } else { + parent_max.clone() + }; + if bytes_leq(&min, &max) { + out.push((min, max)); + } + scan += 1; + } + } + merge_ip_intervals(&out) +} + +fn ip_intervals_to_ranges( + afi: crate::data_model::rc::Afi, + intervals: &[(Vec, Vec)], +) -> Vec { + intervals + .iter() + .map(|(min, max)| { + crate::data_model::rc::IpAddressOrRange::Range(crate::data_model::rc::IpAddressRange { + min: normalize_ip_bytes(afi, min), + max: normalize_ip_bytes(afi, max), + }) + }) + .collect() +} + +fn normalize_ip_bytes(afi: crate::data_model::rc::Afi, bytes: &[u8]) -> Vec { + let target_len = afi.octets_len(); + if bytes.len() == target_len { + return bytes.to_vec(); + } + let mut out = vec![0u8; target_len]; + let copy_len = bytes.len().min(target_len); + out[..copy_len].copy_from_slice(&bytes[..copy_len]); + out +} + +fn intervals_changed(left: &[T], right: &[T]) -> bool { + left != right +} + +fn intersect_as_intervals(child: &[(u32, u32)], parent: &[(u32, u32)]) -> Vec<(u32, u32)> { + let mut out = Vec::new(); + let mut parent_index = 0usize; + for (child_min, child_max) in child { + while parent_index < parent.len() && parent[parent_index].1 < *child_min { + parent_index += 1; + } + let mut scan = parent_index; + while scan < parent.len() && parent[scan].0 <= *child_max { + let min = (*child_min).max(parent[scan].0); + let max = (*child_max).min(parent[scan].1); + if min <= max { + out.push((min, max)); + } + scan += 1; + } + } + merge_as_intervals(&out) +} + +fn as_intervals_to_items(intervals: &[(u32, u32)]) -> Vec { + intervals + .iter() + .map(|(min, max)| { + if min == max { + crate::data_model::rc::AsIdOrRange::Id(*min) + } else { + crate::data_model::rc::AsIdOrRange::Range { + min: *min, + max: *max, + } + } + }) + .collect() +} + +fn merge_ip_intervals(v: &[(Vec, Vec)]) -> Vec<(Vec, Vec)> { + let mut out: Vec<(Vec, Vec)> = Vec::new(); + for (min, max) in v { + let Some(last) = out.last_mut() else { + out.push((min.clone(), max.clone())); + continue; + }; + + if bytes_leq(min, &increment_bytes(&last.1)) { + if bytes_leq(&last.1, max) { + last.1 = max.clone(); + } + continue; + } + + out.push((min.clone(), max.clone())); + } + out +} + +fn interval_is_covered(parent: &[(Vec, Vec)], min: &[u8], max: &[u8]) -> bool { + for (pmin, pmax) in parent { + if bytes_leq(pmin, min) && bytes_leq(max, pmax) { + return true; + } + if pmin.as_slice() > min { + break; + } + } + false +} + +fn prefix_to_range(prefix: &crate::data_model::rc::IpPrefix) -> (Vec, Vec) { + let mut min = prefix.addr.clone(); + let mut max = prefix.addr.clone(); + + let bitlen = match prefix.afi { + crate::data_model::rc::Afi::Ipv4 => 32u16, + crate::data_model::rc::Afi::Ipv6 => 128u16, + }; + let plen = prefix.prefix_len.min(bitlen); + for bit in plen..bitlen { + let byte = (bit / 8) as usize; + let offset = 7 - (bit % 8); + let mask = 1u8 << offset; + min[byte] &= !mask; + max[byte] |= mask; + } + (min, max) +} + +fn bytes_leq(a: &[u8], b: &[u8]) -> bool { + a <= b +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::data_model::common::X509NameDer; + use crate::data_model::oid::OID_CP_IPADDR_ASNUMBER; + use crate::data_model::rc::{ + Afi, AsIdOrRange, AsIdentifierChoice, AsResourceSet, IpAddressChoice, IpAddressFamily, + IpAddressOrRange, IpPrefix, IpResourceSet, + }; + use crate::data_model::rc::{ + BasicConstraintsProfile, CertificatePoliciesProfile, RcExtensions, ResourceCertKind, + ResourceCertificate, RpkixTbsCertificate, + }; + use der_parser::num_bigint::BigUint; + use std::process::Command; + fn dummy_cert( + kind: ResourceCertKind, + subject_dn: &str, + issuer_dn: &str, + ski: Option>, + aki: Option>, + aia: Option>, + crldp: Option>, + ) -> ResourceCertificate { + let aia = aia.map(|v| v.into_iter().map(|s| s.to_string()).collect::>()); + let crldp = crldp.map(|v| v.into_iter().map(|s| s.to_string()).collect::>()); + + ResourceCertificate { + raw_der: Vec::new(), + kind, + tbs: RpkixTbsCertificate { + version: 2, + serial_number: BigUint::from(1u8), + signature_algorithm: "1.2.840.113549.1.1.11".to_string(), + issuer_name: X509NameDer(issuer_dn.as_bytes().to_vec()), + subject_name: X509NameDer(subject_dn.as_bytes().to_vec()), + validity_not_before: time::OffsetDateTime::UNIX_EPOCH, + validity_not_after: time::OffsetDateTime::UNIX_EPOCH, + subject_public_key_info: Vec::new(), + extensions: RcExtensions { + basic_constraints_ca: kind == ResourceCertKind::Ca, + basic_constraints: (kind == ResourceCertKind::Ca).then_some( + BasicConstraintsProfile { + ca: true, + critical: true, + path_len_constraint: None, + }, + ), + subject_key_identifier: ski, + authority_key_identifier: aki, + crl_distribution_points_uris: crldp, + ca_issuers_uris: aia, + subject_info_access: None, + certificate_policies_oid: (kind == ResourceCertKind::Ca) + .then_some(OID_CP_IPADDR_ASNUMBER.to_string()), + certificate_policies: (kind == ResourceCertKind::Ca).then_some( + CertificatePoliciesProfile { + policy_oid: OID_CP_IPADDR_ASNUMBER.to_string(), + qualifier_oids: Vec::new(), + }, + ), + extension_oids: Vec::new(), + ip_resources: None, + as_resources: None, + }, + }, + } + } + + fn openssl_available() -> bool { + Command::new("openssl") + .arg("version") + .output() + .map(|o| o.status.success()) + .unwrap_or(false) + } + + fn write_cert_der_with_addext(dir: &std::path::Path, addext: Option<&str>) -> Vec { + assert!(openssl_available(), "openssl is required for this test"); + let key = dir.join("k.pem"); + let cert = dir.join("c.pem"); + let der = dir.join("c.der"); + + let mut cmd = Command::new("openssl"); + cmd.arg("req") + .arg("-x509") + .arg("-newkey") + .arg("rsa:2048") + .arg("-nodes") + .arg("-keyout") + .arg(&key) + .arg("-subj") + .arg("/CN=ku") + .arg("-days") + .arg("1") + .arg("-out") + .arg(&cert); + if let Some(ext) = addext { + cmd.arg("-addext").arg(ext); + } + let out = cmd.output().expect("openssl req"); + assert!( + out.status.success(), + "openssl req failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + + let out = Command::new("openssl") + .arg("x509") + .arg("-in") + .arg(&cert) + .arg("-outform") + .arg("DER") + .arg("-out") + .arg(&der) + .output() + .expect("openssl x509"); + assert!( + out.status.success(), + "openssl x509 failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + std::fs::read(&der).expect("read der") + } + + fn gen_issuer_and_child_der(dir: &std::path::Path) -> (Vec, Vec, Vec) { + assert!(openssl_available(), "openssl is required for this test"); + let issuer_key = dir.join("issuer.key"); + let issuer_csr = dir.join("issuer.csr"); + let issuer_pem = dir.join("issuer.pem"); + let issuer_der = dir.join("issuer.der"); + + let child_key = dir.join("child.key"); + let child_csr = dir.join("child.csr"); + let child_pem = dir.join("child.pem"); + let child_der = dir.join("child.der"); + + let other_key = dir.join("other.key"); + let other_csr = dir.join("other.csr"); + let other_pem = dir.join("other.pem"); + let other_der = dir.join("other.der"); + + let run = |cmd: &mut Command| { + let out = cmd.output().expect("run openssl"); + assert!( + out.status.success(), + "command failed: {:?}\nstderr={}", + cmd, + String::from_utf8_lossy(&out.stderr) + ); + }; + + // Issuer self-signed. + run(Command::new("openssl") + .args(["genrsa", "-out"]) + .arg(&issuer_key) + .arg("2048")); + run(Command::new("openssl") + .args(["req", "-new", "-key"]) + .arg(&issuer_key) + .args(["-subj", "/CN=issuer", "-out"]) + .arg(&issuer_csr)); + run(Command::new("openssl") + .args(["x509", "-req", "-in"]) + .arg(&issuer_csr) + .args(["-signkey"]) + .arg(&issuer_key) + .args(["-days", "1", "-out"]) + .arg(&issuer_pem)); + run(Command::new("openssl") + .args(["x509", "-in"]) + .arg(&issuer_pem) + .args(["-outform", "DER", "-out"]) + .arg(&issuer_der)); + + // Child signed by issuer. + run(Command::new("openssl") + .args(["genrsa", "-out"]) + .arg(&child_key) + .arg("2048")); + run(Command::new("openssl") + .args(["req", "-new", "-key"]) + .arg(&child_key) + .args(["-subj", "/CN=child", "-out"]) + .arg(&child_csr)); + run(Command::new("openssl") + .args(["x509", "-req", "-in"]) + .arg(&child_csr) + .args(["-CA"]) + .arg(&issuer_pem) + .args(["-CAkey"]) + .arg(&issuer_key) + .args(["-CAcreateserial", "-days", "1", "-out"]) + .arg(&child_pem)); + run(Command::new("openssl") + .args(["x509", "-in"]) + .arg(&child_pem) + .args(["-outform", "DER", "-out"]) + .arg(&child_der)); + + // Other self-signed issuer. + run(Command::new("openssl") + .args(["genrsa", "-out"]) + .arg(&other_key) + .arg("2048")); + run(Command::new("openssl") + .args(["req", "-new", "-key"]) + .arg(&other_key) + .args(["-subj", "/CN=other", "-out"]) + .arg(&other_csr)); + run(Command::new("openssl") + .args(["x509", "-req", "-in"]) + .arg(&other_csr) + .args(["-signkey"]) + .arg(&other_key) + .args(["-days", "1", "-out"]) + .arg(&other_pem)); + run(Command::new("openssl") + .args(["x509", "-in"]) + .arg(&other_pem) + .args(["-outform", "DER", "-out"]) + .arg(&other_der)); + + ( + std::fs::read(&issuer_der).expect("read issuer der"), + std::fs::read(&child_der).expect("read child der"), + std::fs::read(&other_der).expect("read other der"), + ) + } + + #[test] + fn resolve_child_ip_resources_rejects_inherit_without_parent_effective_resources() { + let child = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::Inherit, + }], + }; + let err = resolve_child_ip_resources(Some(&child), None).unwrap_err(); + assert!(matches!(err, CaPathError::InheritWithoutParentResources)); + } + + #[test] + fn resolve_child_ip_resources_rejects_non_inherit_without_parent_effective_resources() { + let child = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![]), + }], + }; + let err = resolve_child_ip_resources(Some(&child), None).unwrap_err(); + assert!(matches!(err, CaPathError::ResourcesNotSubset)); + } + + #[test] + fn ip_resources_by_afi_items_rejects_inherit_families() { + let parent = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv6, + choice: IpAddressChoice::Inherit, + }], + }; + let err = ip_resources_by_afi_items(&parent).unwrap_err(); + assert!(matches!(err, CaPathError::InheritWithoutParentResources)); + } + + #[test] + fn resolve_child_as_resources_rejects_inherit_without_parent_effective_resources() { + let child = AsResourceSet { + asnum: Some(AsIdentifierChoice::Inherit), + rdi: None, + }; + let err = resolve_child_as_resources(Some(&child), None).unwrap_err(); + assert!(matches!(err, CaPathError::InheritWithoutParentResources)); + } + + #[test] + fn validation_update_03_intersects_overclaiming_child_ip_resources() { + let issuer = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( + IpPrefix { + afi: Afi::Ipv4, + prefix_len: 24, + addr: vec![10, 0, 0, 0], + }, + )]), + }], + }; + let child = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( + IpPrefix { + afi: Afi::Ipv4, + prefix_len: 23, + addr: vec![10, 0, 0, 0], + }, + )]), + }], + }; + let index = IssuerEffectiveResourcesIndex::from_effective_resources(Some(&issuer), None) + .expect("index"); + let resolved = resolve_child_resources( + Some(&child), + Some(&issuer), + None, + None, + &index, + ResourceValidationMode::ValidationUpdate03, + ) + .expect("vrs resolution"); + assert!(resolved.warnings.ip_reduced_by_vrs); + assert!(!resolved.warnings.ip_vrs_empty); + let effective = resolved.effective_ip_resources.expect("effective ip"); + assert!(effective.families[0].contains_prefix(&IpPrefix { + afi: Afi::Ipv4, + prefix_len: 24, + addr: vec![10, 0, 0, 0], + })); + assert!(!effective.families[0].contains_prefix(&IpPrefix { + afi: Afi::Ipv4, + prefix_len: 24, + addr: vec![10, 0, 1, 0], + })); + } + + #[test] + fn validation_update_03_keeps_empty_vrs_instead_of_rejecting_child_ca() { + let issuer = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( + IpPrefix { + afi: Afi::Ipv4, + prefix_len: 24, + addr: vec![10, 0, 0, 0], + }, + )]), + }], + }; + let child = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( + IpPrefix { + afi: Afi::Ipv4, + prefix_len: 24, + addr: vec![192, 0, 2, 0], + }, + )]), + }], + }; + let index = IssuerEffectiveResourcesIndex::from_effective_resources(Some(&issuer), None) + .expect("index"); + let strict_err = resolve_child_resources( + Some(&child), + Some(&issuer), + None, + None, + &index, + ResourceValidationMode::Rfc6487, + ) + .unwrap_err(); + assert!(matches!(strict_err, CaPathError::ResourcesNotSubset)); + + let resolved = resolve_child_resources( + Some(&child), + Some(&issuer), + None, + None, + &index, + ResourceValidationMode::ValidationUpdate03, + ) + .expect("vrs resolution"); + assert!(resolved.warnings.ip_reduced_by_vrs); + assert!(resolved.warnings.ip_vrs_empty); + assert!( + resolved + .effective_ip_resources + .expect("effective ip") + .families + .is_empty() + ); + } + + #[test] + fn validation_update_03_intersects_overclaiming_child_as_resources() { + let issuer = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { + min: 64500, + max: 64510, + }, + ])), + rdi: None, + }; + let child = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { + min: 64505, + max: 64520, + }, + ])), + rdi: None, + }; + let index = IssuerEffectiveResourcesIndex::from_effective_resources(None, Some(&issuer)) + .expect("index"); + let resolved = resolve_child_resources( + None, + None, + Some(&child), + Some(&issuer), + &index, + ResourceValidationMode::ValidationUpdate03, + ) + .expect("vrs resolution"); + assert!(resolved.warnings.as_reduced_by_vrs); + assert!(!resolved.warnings.as_vrs_empty); + let effective = resolved.effective_as_resources.expect("effective as"); + assert_eq!( + effective.asnum, + Some(AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { + min: 64505, + max: 64510, + }, + ])) + ); + } + + #[test] + fn validation_update_03_records_all_resource_warning_summary_parts() { + let issuer_ip = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( + IpPrefix { + afi: Afi::Ipv4, + prefix_len: 24, + addr: vec![10, 0, 0, 0], + }, + )]), + }], + }; + let child_ip = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( + IpPrefix { + afi: Afi::Ipv4, + prefix_len: 24, + addr: vec![192, 0, 2, 0], + }, + )]), + }], + }; + let issuer_as = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { + min: 64500, + max: 64510, + }, + ])), + rdi: None, + }; + let child_as = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { + min: 64520, + max: 64530, + }, + ])), + rdi: None, + }; + let index = IssuerEffectiveResourcesIndex::from_effective_resources( + Some(&issuer_ip), + Some(&issuer_as), + ) + .expect("index"); + + let resolved = resolve_child_resources( + Some(&child_ip), + Some(&issuer_ip), + Some(&child_as), + Some(&issuer_as), + &index, + ResourceValidationMode::ValidationUpdate03, + ) + .expect("vrs resolution"); + + assert!(resolved.warnings.ip_reduced_by_vrs); + assert!(resolved.warnings.ip_vrs_empty); + assert!(resolved.warnings.as_reduced_by_vrs); + assert!(resolved.warnings.as_vrs_empty); + assert_eq!( + resolved.warnings.summary(), + "ip_reduced_by_vrs,as_reduced_by_vrs,ip_vrs_empty,as_vrs_empty" + ); + assert!(!resolved.warnings.is_empty()); + } + + #[test] + fn child_aki_mismatch_is_rejected() { + let issuer = dummy_cert( + ResourceCertKind::Ca, + "CN=issuer", + "CN=issuer", + Some(vec![1]), + None, + None, + None, + ); + let child = dummy_cert( + ResourceCertKind::Ca, + "CN=child", + "CN=issuer", + Some(vec![2]), + Some(vec![9]), + Some(vec!["rsync://example.test/issuer.cer"]), + Some(vec!["rsync://example.test/issuer.crl"]), + ); + let err = validate_child_aki_matches_issuer_ski(&child, &issuer).unwrap_err(); + assert!(matches!(err, CaPathError::ChildAkiMismatch), "{err}"); + } + + #[test] + fn child_aia_missing_is_rejected() { + let _issuer = dummy_cert( + ResourceCertKind::Ca, + "CN=issuer", + "CN=issuer", + Some(vec![1]), + None, + None, + None, + ); + let child = dummy_cert( + ResourceCertKind::Ca, + "CN=child", + "CN=issuer", + Some(vec![2]), + Some(vec![1]), + None, + Some(vec!["rsync://example.test/issuer.crl"]), + ); + let err = + validate_child_aia_points_to_issuer_uri(&child, "rsync://example.test/issuer.cer") + .unwrap_err(); + assert!(matches!(err, CaPathError::ChildAiaMissing), "{err}"); + + // Also cover issuer ski missing. + let issuer_missing_ski = dummy_cert( + ResourceCertKind::Ca, + "CN=issuer", + "CN=issuer", + None, + None, + None, + None, + ); + let err = validate_child_aki_matches_issuer_ski(&child, &issuer_missing_ski).unwrap_err(); + assert!(matches!(err, CaPathError::IssuerSkiMissing), "{err}"); + } + + #[test] + fn child_aia_issuer_uri_mismatch_is_rejected() { + let _issuer = dummy_cert( + ResourceCertKind::Ca, + "CN=issuer", + "CN=issuer", + Some(vec![1]), + None, + None, + None, + ); + let child = dummy_cert( + ResourceCertKind::Ca, + "CN=child", + "CN=issuer", + Some(vec![2]), + Some(vec![1]), + Some(vec!["rsync://example.test/other.cer"]), + Some(vec!["rsync://example.test/issuer.crl"]), + ); + let err = + validate_child_aia_points_to_issuer_uri(&child, "rsync://example.test/issuer.cer") + .unwrap_err(); + assert!( + matches!(err, CaPathError::ChildAiaIssuerUriMismatch), + "{err}" + ); + } + + #[test] + fn child_crldp_mismatch_is_rejected() { + let issuer = dummy_cert( + ResourceCertKind::Ca, + "CN=issuer", + "CN=issuer", + Some(vec![1]), + None, + None, + None, + ); + let child = dummy_cert( + ResourceCertKind::Ca, + "CN=child", + "CN=issuer", + Some(vec![2]), + Some(vec![1]), + Some(vec!["rsync://example.test/issuer.cer"]), + None, + ); + let err = + validate_child_crldp_contains_issuer_crl_uri(&child, "rsync://example.test/issuer.crl") + .unwrap_err(); + assert!(matches!(err, CaPathError::ChildCrlDpMissing), "{err}"); + + let child = dummy_cert( + ResourceCertKind::Ca, + "CN=child", + "CN=issuer", + Some(vec![2]), + Some(vec![1]), + Some(vec!["rsync://example.test/issuer.cer"]), + Some(vec!["rsync://example.test/other.crl"]), + ); + let err = + validate_child_crldp_contains_issuer_crl_uri(&child, "rsync://example.test/issuer.crl") + .unwrap_err(); + assert!(matches!(err, CaPathError::ChildCrlDpUriMismatch), "{err}"); + + // Cover child AKI missing. + let child_missing_aki = dummy_cert( + ResourceCertKind::Ca, + "CN=child", + "CN=issuer", + Some(vec![2]), + None, + Some(vec!["rsync://example.test/issuer.cer"]), + Some(vec!["rsync://example.test/issuer.crl"]), + ); + let err = validate_child_aki_matches_issuer_ski(&child_missing_aki, &issuer).unwrap_err(); + assert!(matches!(err, CaPathError::ChildAkiMissing), "{err}"); + } + + #[test] + fn child_binding_checks_accept_when_matching() { + let issuer = dummy_cert( + ResourceCertKind::Ca, + "CN=issuer", + "CN=issuer", + Some(vec![1]), + None, + None, + None, + ); + let child = dummy_cert( + ResourceCertKind::Ca, + "CN=child", + "CN=issuer", + Some(vec![2]), + Some(vec![1]), + Some(vec!["rsync://example.test/issuer.cer"]), + Some(vec!["rsync://example.test/issuer.crl"]), + ); + validate_child_aki_matches_issuer_ski(&child, &issuer).expect("aki ok"); + validate_child_aia_points_to_issuer_uri(&child, "rsync://example.test/issuer.cer") + .expect("aia ok"); + validate_child_crldp_contains_issuer_crl_uri(&child, "rsync://example.test/issuer.crl") + .expect("crldp ok"); + } + + #[test] + fn validate_child_ca_key_usage_accepts_only_keycertsign_and_crlsign_critical() { + let td = tempfile::tempdir().expect("tempdir"); + let der = write_cert_der_with_addext( + td.path(), + Some("keyUsage = critical, keyCertSign, cRLSign"), + ); + let cert = parse_x509_cert(&der).expect("x509 parse ok"); + validate_child_ca_key_usage(&cert).expect("key usage ok"); + } + + #[test] + fn validate_child_ca_key_usage_rejects_missing_noncritical_and_invalid_bits() { + let td = tempfile::tempdir().expect("tempdir"); + let missing = write_cert_der_with_addext(td.path(), None); + let cert = parse_x509_cert(&missing).expect("x509 parse ok"); + let err = validate_child_ca_key_usage(&cert).unwrap_err(); + assert!(matches!(err, CaPathError::KeyUsageMissing), "{err}"); + + let td = tempfile::tempdir().expect("tempdir"); + let noncritical = + write_cert_der_with_addext(td.path(), Some("keyUsage = keyCertSign, cRLSign")); + let cert = parse_x509_cert(&noncritical).expect("x509 parse ok"); + let err = validate_child_ca_key_usage(&cert).unwrap_err(); + assert!(matches!(err, CaPathError::KeyUsageNotCritical), "{err}"); + + let td = tempfile::tempdir().expect("tempdir"); + let invalid = write_cert_der_with_addext( + td.path(), + Some("keyUsage = critical, keyCertSign, cRLSign, digitalSignature"), + ); + let cert = parse_x509_cert(&invalid).expect("x509 parse ok"); + let err = validate_child_ca_key_usage(&cert).unwrap_err(); + assert!(matches!(err, CaPathError::KeyUsageInvalidBits), "{err}"); + } + + #[test] + fn verify_cert_signature_with_issuer_accepts_valid_chain_and_rejects_wrong_issuer() { + let td = tempfile::tempdir().expect("tempdir"); + let (issuer, child, other) = gen_issuer_and_child_der(td.path()); + let issuer_cert = parse_x509_cert(&issuer).expect("x509 parse issuer"); + let child_cert = parse_x509_cert(&child).expect("x509 parse child"); + verify_child_signature(&child_cert, &issuer_cert.tbs_certificate.subject_pki) + .expect("signature ok"); + let other_cert = parse_x509_cert(&other).expect("x509 parse other"); + let err = verify_child_signature(&child_cert, &other_cert.tbs_certificate.subject_pki) + .unwrap_err(); + assert!( + matches!(err, CaPathError::ChildSignatureInvalid(_)), + "{err}" + ); + } + + #[test] + fn signature_cache_stores_real_verify_success_and_skips_on_hit() { + let _guard = crate::crypto_sig_cache::GLOBAL_TEST_LOCK + .lock() + .expect("test lock"); + crate::crypto_sig_cache::clear_global(); + let td = tempfile::tempdir().expect("tempdir"); + let (issuer, child, other) = gen_issuer_and_child_der(td.path()); + let issuer_cert = parse_x509_cert(&issuer).expect("x509 parse issuer"); + let child_cert = parse_x509_cert(&child).expect("x509 parse child"); + let other_cert = parse_x509_cert(&other).expect("x509 parse other"); + + let cache_dir = tempfile::tempdir().expect("tempdir"); + let cache = std::sync::Arc::new(crate::crypto_sig_cache::CryptoSigCache::load_or_rebuild( + cache_dir.path().join("work-db.crypto-sig-cache"), + true, + )); + crate::crypto_sig_cache::install_global(std::sync::Arc::clone(&cache)); + + // Miss -> real verification runs and passes -> positive conclusion written. + verify_child_signature(&child_cert, &issuer_cert.tbs_certificate.subject_pki) + .expect("signature ok"); + // Hit -> skipped (a second execution would also pass, so assert via stats). + verify_child_signature(&child_cert, &issuer_cert.tbs_certificate.subject_pki) + .expect("signature ok"); + // Wrong issuer: real verification fails, nothing is written (no negative caching), + // so a retry executes the real verification again instead of inheriting anything. + for _ in 0..2 { + let err = verify_child_signature(&child_cert, &other_cert.tbs_certificate.subject_pki) + .unwrap_err(); + assert!( + matches!(err, CaPathError::ChildSignatureInvalid(_)), + "{err}" + ); + } + + let summary = cache.summary(); + let stats = summary + .per_point + .get("child_ca_cert") + .expect("child_ca_cert stats"); + assert_eq!(stats.calls, 4); + assert_eq!(stats.would_hit, 1); + assert_eq!(stats.new_keys, 3); + assert_eq!(stats.verify_executed, 3); + assert_eq!(stats.verify_skipped, 1); + crate::crypto_sig_cache::clear_global(); + } + + #[test] + fn issuer_effective_resources_index_and_indexed_resolvers_cover_success_and_failure_paths() { + use crate::data_model::rc::{AsIdOrRange, IpPrefix}; + + let parent_ip = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( + IpPrefix { + afi: Afi::Ipv4, + prefix_len: 8, + addr: vec![10, 0, 0, 0], + }, + )]), + }], + }; + let parent_as = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { + min: 64500, + max: 64599, + }, + ])), + rdi: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id( + 65000, + )])), + }; + let idx = IssuerEffectiveResourcesIndex::from_effective_resources( + Some(&parent_ip), + Some(&parent_as), + ) + .expect("index builds"); + assert_eq!( + idx.parent_ip_by_afi_items.as_ref().map(|v| v.len()), + Some(1) + ); + assert_eq!(idx.parent_ip_merged_intervals.len(), 1); + assert_eq!( + idx.parent_asnum_intervals.as_ref().map(|v| v.len()), + Some(1) + ); + assert_eq!(idx.parent_rdi_intervals.as_ref().map(|v| v.len()), Some(1)); + + let child_ip_subset = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( + IpPrefix { + afi: Afi::Ipv4, + prefix_len: 16, + addr: vec![10, 1, 0, 0], + }, + )]), + }], + }; + assert!( + resolve_child_ip_resources_indexed( + Some(&child_ip_subset), + Some(&parent_ip), + idx.parent_ip_by_afi_items.as_ref(), + &idx.parent_ip_merged_intervals, + ) + .expect("subset should resolve") + .is_some() + ); + + let child_ip_bad = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( + IpPrefix { + afi: Afi::Ipv4, + prefix_len: 16, + addr: vec![11, 0, 0, 0], + }, + )]), + }], + }; + let err = resolve_child_ip_resources_indexed( + Some(&child_ip_bad), + Some(&parent_ip), + idx.parent_ip_by_afi_items.as_ref(), + &idx.parent_ip_merged_intervals, + ) + .unwrap_err(); + assert!(matches!(err, CaPathError::ResourcesNotSubset)); + + let child_as_subset = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id( + 64542, + )])), + rdi: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id( + 65000, + )])), + }; + assert!( + resolve_child_as_resources_indexed( + Some(&child_as_subset), + Some(&parent_as), + idx.parent_asnum_intervals.as_deref(), + idx.parent_rdi_intervals.as_deref(), + ) + .expect("subset as resolves") + .is_some() + ); + + let child_as_bad = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id( + 65123, + )])), + rdi: None, + }; + let err = resolve_child_as_resources_indexed( + Some(&child_as_bad), + Some(&parent_as), + idx.parent_asnum_intervals.as_deref(), + idx.parent_rdi_intervals.as_deref(), + ) + .unwrap_err(); + assert!(matches!(err, CaPathError::ResourcesNotSubset)); + } + + #[test] + fn resolve_child_ip_and_as_resources_success_paths() { + use crate::data_model::rc::{AsIdOrRange, IpAddressOrRange, IpPrefix}; + + let parent_ip = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( + IpPrefix { + afi: Afi::Ipv4, + prefix_len: 8, + addr: vec![10, 0, 0, 0], + }, + )]), + }], + }; + + let child_ip_inherit = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::Inherit, + }], + }; + let eff = resolve_child_ip_resources(Some(&child_ip_inherit), Some(&parent_ip)) + .expect("inherit resolves") + .expect("some ip"); + assert_eq!(eff.families.len(), 1); + assert!(matches!( + eff.families[0].choice, + IpAddressChoice::AddressesOrRanges(_) + )); + + let child_ip_subset = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( + IpPrefix { + afi: Afi::Ipv4, + prefix_len: 16, + addr: vec![10, 1, 0, 0], + }, + )]), + }], + }; + resolve_child_ip_resources(Some(&child_ip_subset), Some(&parent_ip)) + .expect("subset ok") + .expect("some"); + + let child_ip_bad = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( + IpPrefix { + afi: Afi::Ipv4, + prefix_len: 16, + addr: vec![11, 0, 0, 0], + }, + )]), + }], + }; + let err = resolve_child_ip_resources(Some(&child_ip_bad), Some(&parent_ip)).unwrap_err(); + assert!(matches!(err, CaPathError::ResourcesNotSubset), "{err}"); + + let parent_as = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { min: 1, max: 100 }, + ])), + rdi: None, + }; + let child_as_inherit = AsResourceSet { + asnum: Some(AsIdentifierChoice::Inherit), + rdi: None, + }; + let eff_as = resolve_child_as_resources(Some(&child_as_inherit), Some(&parent_as)) + .expect("inherit as") + .expect("some"); + assert_eq!(eff_as.asnum, parent_as.asnum); + + let child_as_subset = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id(50)])), + rdi: None, + }; + resolve_child_as_resources(Some(&child_as_subset), Some(&parent_as)) + .expect("subset as") + .expect("some"); + + let child_as_bad = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id( + 200, + )])), + rdi: None, + }; + let err = resolve_child_as_resources(Some(&child_as_bad), Some(&parent_as)).unwrap_err(); + assert!(matches!(err, CaPathError::ResourcesNotSubset), "{err}"); + } +} + +fn increment_bytes(v: &[u8]) -> Vec { + let mut out = v.to_vec(); + for i in (0..out.len()).rev() { + if out[i] != 0xFF { + out[i] += 1; + for j in i + 1..out.len() { + out[j] = 0; + } + return out; + } + } + vec![0u8; out.len()] +} diff --git a/crates/panda-rpki-validator/src/validation/cert_path.rs b/crates/panda-rpki-validator/src/validation/cert_path.rs new file mode 100644 index 0000000..cd90e26 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/cert_path.rs @@ -0,0 +1,880 @@ +use crate::data_model::common::BigUnsigned; +use crate::data_model::crl::{CrlDecodeError, CrlVerifyError, RpkixCrl}; +use crate::data_model::rc::{ + ResourceCertKind, ResourceCertificate, ResourceCertificateDecodeError, + ResourceCertificateProfileError, ResourceCertificateRole, +}; +use crate::data_model::signed_object::{EeKeyUsageSummary, ResourceEeCertificate}; +use ring::signature; +use x509_parser::prelude::{FromDer, X509Certificate}; + +use crate::validation::x509_name::x509_names_equivalent; +use std::collections::HashSet; +use x509_parser::x509::SubjectPublicKeyInfo; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ValidatedEeCertPath { + pub ee: ResourceCertificate, + pub issuer_ca: ResourceCertificate, + pub issuer_crl: RpkixCrl, +} + +#[derive(Debug, thiserror::Error)] +pub enum CertPathError { + #[error("EE certificate decode failed: {0} (RFC 6487 §4; RFC 5280 §4.1)")] + EeDecode(#[from] ResourceCertificateDecodeError), + + #[error("issuer CA certificate decode failed: {0} (RFC 6487 §4; RFC 5280 §4.1)")] + IssuerDecode(ResourceCertificateDecodeError), + + #[error("issuer CRL decode failed: {0} (RFC 6487 §5; RFC 9829 §3.1; RFC 5280 §5.1)")] + CrlDecode(#[from] CrlDecodeError), + + #[error( + "issuer certificate must be a CA resource certificate (RFC 6487 §4.8.1; RFC 5280 §4.2.1.9)" + )] + IssuerNotCa, + + #[error("issuer CA certificate profile validation failed: {0} (RFC 6487 §4.8)")] + IssuerProfile(ResourceCertificateProfileError), + + #[error("EE certificate must be an EE resource certificate (RFC 6487 §4)")] + EeNotEe, + + #[error( + "EE issuer DN does not match issuer CA subject DN: ee.issuer={ee_issuer_dn} issuer.subject={issuer_subject_dn} (RFC 5280 §6.1)" + )] + IssuerSubjectMismatch { + ee_issuer_dn: String, + issuer_subject_dn: String, + }, + + #[error("EE certificate signature verification failed: {0} (RFC 5280 §6.1)")] + EeSignatureInvalid(String), + + #[error("issuer SubjectPublicKeyInfo parse error: {0} (RFC 5280 §4.1.2.7)")] + IssuerSpkiParse(String), + + #[error( + "trailing bytes after issuer SubjectPublicKeyInfo DER: {0} bytes (DER; RFC 5280 §4.1.2.7)" + )] + IssuerSpkiTrailingBytes(usize), + + #[error("EE KeyUsage extension missing (RFC 6487 §4.8.4; RFC 5280 §4.2.1.3)")] + KeyUsageMissing, + + #[error("EE KeyUsage criticality must be critical (RFC 6487 §4.8.4; RFC 5280 §4.2.1.3)")] + KeyUsageNotCritical, + + #[error("EE KeyUsage must have only digitalSignature set (RFC 6487 §4.8.4)")] + KeyUsageInvalidBits, + + #[error("EE KeyUsage parse failed: {0} (RFC 6487 §4.8.4; RFC 5280 §4.2.1.3)")] + KeyUsageParse(String), + + #[error("issuer CA subjectKeyIdentifier missing (RFC 6487 §4.8.2)")] + IssuerSkiMissing, + + #[error("EE authorityKeyIdentifier missing (RFC 6487 §4.8.3; RFC 5280 §4.2.1.1)")] + EeAkiMissing, + + #[error( + "EE authorityKeyIdentifier does not match issuer subjectKeyIdentifier (RFC 6487 §4.8.3)" + )] + EeAkiMismatch, + + #[error("EE authorityInfoAccess missing (RFC 6487 §4.8.7; RFC 5280 §4.2.2.1)")] + EeAiaMissing, + + #[error( + "EE authorityInfoAccess does not reference issuer certificate rsync URI (RFC 6487 §4.8.7)" + )] + EeAiaIssuerUriMismatch, + + #[error("EE CRLDistributionPoints missing (RFC 6487 §4.8.6; RFC 5280 §4.2.1.13)")] + EeCrlDpMissing, + + #[error("EE CRLDistributionPoints does not reference issuer CRL rsync URI (RFC 6487 §4.8.6)")] + EeCrlDpUriMismatch, + + #[error("certificate not valid at validation_time (RFC 5280 §4.1.2.5; RFC 5280 §6.1)")] + CertificateNotValidAtTime, + + #[error( + "CRL signature/binding verification failed: {0} (RFC 5280 §6.3.3; RFC 6487 §5; RFC 9829 §3.1)" + )] + CrlVerify(#[from] CrlVerifyError), + + #[error( + "CRL not valid at validation_time (RFC 5280 §6.3.3(g); RFC 5280 §5.1.2.4-§5.1.2.5; RFC 6487 §5)" + )] + CrlNotValidAtTime, + + #[error("EE certificate is revoked by issuer CRL (RFC 5280 §6.3.3; RFC 6487 §5)")] + EeRevoked, +} + +pub fn validate_ee_cert_path( + ee_cert_der: &[u8], + issuer_ca_der: &[u8], + issuer_crl_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_crl_rsync_uri: Option<&str>, + validation_time: time::OffsetDateTime, +) -> Result { + let ee = ResourceCertificate::decode_der(ee_cert_der)?; + if ee.kind != ResourceCertKind::Ee { + return Err(CertPathError::EeNotEe); + } + + let issuer_ca = + ResourceCertificate::decode_der(issuer_ca_der).map_err(CertPathError::IssuerDecode)?; + if issuer_ca.kind != ResourceCertKind::Ca { + return Err(CertPathError::IssuerNotCa); + } + issuer_ca + .validate_rfc6487_profile(ResourceCertificateRole::Ca) + .map_err(CertPathError::IssuerProfile)?; + let issuer_spki = parse_subject_pki_from_der(&issuer_ca.tbs.subject_public_key_info)?; + + if !x509_names_equivalent(&ee.tbs.issuer_name, &issuer_ca.tbs.subject_name) { + return Err(CertPathError::IssuerSubjectMismatch { + ee_issuer_dn: ee.tbs.issuer_name.to_string(), + issuer_subject_dn: issuer_ca.tbs.subject_name.to_string(), + }); + } + + validate_ee_aki_matches_issuer_ski(&ee, &issuer_ca)?; + if let Some(expected_issuer_uri) = issuer_ca_rsync_uri { + validate_ee_aia_points_to_issuer_uri(&ee, expected_issuer_uri)?; + } + if let Some(expected_crl_uri) = issuer_crl_rsync_uri { + validate_ee_crldp_contains_issuer_crl_uri(&ee, expected_crl_uri)?; + } + + if !time_within_validity( + validation_time, + ee.tbs.validity_not_before, + ee.tbs.validity_not_after, + ) || !time_within_validity( + validation_time, + issuer_ca.tbs.validity_not_before, + issuer_ca.tbs.validity_not_after, + ) { + return Err(CertPathError::CertificateNotValidAtTime); + } + + let ee_x509 = parse_x509_cert(ee_cert_der)?; + verify_ee_cert_signature(&ee_x509, &issuer_spki)?; + validate_ee_key_usage(&ee_x509)?; + + let issuer_crl = RpkixCrl::decode_der(issuer_crl_der)?; + issuer_crl.verify_signature_with_issuer_certificate_der(issuer_ca_der)?; + if !crl_valid_at_time(&issuer_crl, validation_time) { + return Err(CertPathError::CrlNotValidAtTime); + } + + if is_serial_revoked_by_crl(&ee, &issuer_crl) { + return Err(CertPathError::EeRevoked); + } + + Ok(ValidatedEeCertPath { + ee, + issuer_ca, + issuer_crl, + }) +} + +/// Validate the EE certificate path using a *pre-decoded issuer CA* and a *pre-decoded and +/// pre-verified issuer CRL*. +/// +/// This is a performance-oriented helper for stage2 serial runs: it avoids repeating issuer CA +/// decode and issuer CRL decode+signature verification for every signed object in a publication point. +/// +/// The caller must ensure: +/// - `issuer_ca` corresponds to `issuer_ca_der` +/// - `issuer_crl` cryptographic signature/binding has already been verified against `issuer_ca_der` +pub fn validate_ee_cert_path_with_prevalidated_issuer( + ee_cert_der: &[u8], + issuer_ca: &ResourceCertificate, + issuer_spki: &SubjectPublicKeyInfo<'_>, + issuer_crl: &RpkixCrl, + issuer_crl_revoked_serials: &HashSet>, + issuer_ca_rsync_uri: Option<&str>, + issuer_crl_rsync_uri: Option<&str>, + validation_time: time::OffsetDateTime, +) -> Result { + let ee = ResourceCertificate::decode_der(ee_cert_der)?; + validate_ee_cert_path_components( + &ee, + ee_cert_der, + issuer_ca, + issuer_spki, + issuer_crl, + issuer_crl_revoked_serials, + issuer_ca_rsync_uri, + issuer_crl_rsync_uri, + validation_time, + )?; + Ok(ee) +} + +/// Validate the EE certificate path using a pre-decoded EE certificate and a pre-validated issuer. +/// +/// This avoids re-decoding the embedded EE certificate when the caller already parsed it while +/// decoding a signed object (e.g. ROA / ASPA). +pub fn validate_ee_cert_path_with_predecoded_ee( + ee: &ResourceCertificate, + ee_cert_der: &[u8], + issuer_ca: &ResourceCertificate, + issuer_spki: &SubjectPublicKeyInfo<'_>, + issuer_crl: &RpkixCrl, + issuer_crl_revoked_serials: &HashSet>, + issuer_ca_rsync_uri: Option<&str>, + issuer_crl_rsync_uri: Option<&str>, + validation_time: time::OffsetDateTime, +) -> Result<(), CertPathError> { + validate_ee_cert_path_components( + ee, + ee_cert_der, + issuer_ca, + issuer_spki, + issuer_crl, + issuer_crl_revoked_serials, + issuer_ca_rsync_uri, + issuer_crl_rsync_uri, + validation_time, + ) +} + +/// Validate a signed-object embedded EE certificate path using signed-object cached fields. +/// +/// This avoids reparsing the EE certificate as `X509Certificate` and bypasses the +/// x509-parser signature wrapper by verifying the cached TBS/signature bytes directly. +pub fn validate_signed_object_ee_cert_path_fast( + ee: &ResourceEeCertificate, + issuer_ca: &ResourceCertificate, + issuer_spki: &SubjectPublicKeyInfo<'_>, + issuer_crl: &RpkixCrl, + issuer_crl_revoked_serials: &HashSet>, + issuer_ca_rsync_uri: Option<&str>, + issuer_crl_rsync_uri: Option<&str>, + validation_time: time::OffsetDateTime, +) -> Result<(), CertPathError> { + validate_ee_cert_path_bindings_and_status( + &ee.resource_cert, + issuer_ca, + issuer_crl, + issuer_crl_revoked_serials, + issuer_ca_rsync_uri, + issuer_crl_rsync_uri, + validation_time, + )?; + validate_ee_key_usage_summary(&ee.key_usage_summary)?; + verify_ee_cert_signature_fast(&ee.tbs_certificate_der, &ee.signature_bytes, issuer_spki)?; + Ok(()) +} + +fn validate_ee_cert_path_components( + ee: &ResourceCertificate, + ee_cert_der: &[u8], + issuer_ca: &ResourceCertificate, + issuer_spki: &SubjectPublicKeyInfo<'_>, + issuer_crl: &RpkixCrl, + issuer_crl_revoked_serials: &HashSet>, + issuer_ca_rsync_uri: Option<&str>, + issuer_crl_rsync_uri: Option<&str>, + validation_time: time::OffsetDateTime, +) -> Result<(), CertPathError> { + validate_ee_cert_path_bindings_and_status( + ee, + issuer_ca, + issuer_crl, + issuer_crl_revoked_serials, + issuer_ca_rsync_uri, + issuer_crl_rsync_uri, + validation_time, + )?; + + let ee_x509 = parse_x509_cert(ee_cert_der)?; + verify_ee_cert_signature(&ee_x509, issuer_spki)?; + validate_ee_key_usage(&ee_x509)?; + + Ok(()) +} + +fn validate_ee_cert_path_bindings_and_status( + ee: &ResourceCertificate, + issuer_ca: &ResourceCertificate, + issuer_crl: &RpkixCrl, + issuer_crl_revoked_serials: &HashSet>, + issuer_ca_rsync_uri: Option<&str>, + issuer_crl_rsync_uri: Option<&str>, + validation_time: time::OffsetDateTime, +) -> Result<(), CertPathError> { + if ee.kind != ResourceCertKind::Ee { + return Err(CertPathError::EeNotEe); + } + if issuer_ca.kind != ResourceCertKind::Ca { + return Err(CertPathError::IssuerNotCa); + } + issuer_ca + .validate_rfc6487_profile(ResourceCertificateRole::Ca) + .map_err(CertPathError::IssuerProfile)?; + + if !x509_names_equivalent(&ee.tbs.issuer_name, &issuer_ca.tbs.subject_name) { + return Err(CertPathError::IssuerSubjectMismatch { + ee_issuer_dn: ee.tbs.issuer_name.to_string(), + issuer_subject_dn: issuer_ca.tbs.subject_name.to_string(), + }); + } + + validate_ee_aki_matches_issuer_ski(ee, issuer_ca)?; + if let Some(expected_issuer_uri) = issuer_ca_rsync_uri { + validate_ee_aia_points_to_issuer_uri(ee, expected_issuer_uri)?; + } + if let Some(expected_crl_uri) = issuer_crl_rsync_uri { + validate_ee_crldp_contains_issuer_crl_uri(ee, expected_crl_uri)?; + } + + if !time_within_validity( + validation_time, + ee.tbs.validity_not_before, + ee.tbs.validity_not_after, + ) || !time_within_validity( + validation_time, + issuer_ca.tbs.validity_not_before, + issuer_ca.tbs.validity_not_after, + ) { + return Err(CertPathError::CertificateNotValidAtTime); + } + + if !crl_valid_at_time(issuer_crl, validation_time) { + return Err(CertPathError::CrlNotValidAtTime); + } + + let serial = BigUnsigned::from_biguint(&ee.tbs.serial_number); + if issuer_crl_revoked_serials.contains(&serial.bytes_be) { + return Err(CertPathError::EeRevoked); + } + + Ok(()) +} + +fn parse_subject_pki_from_der(der: &[u8]) -> Result, CertPathError> { + let (rem, spki) = SubjectPublicKeyInfo::from_der(der) + .map_err(|e| CertPathError::IssuerSpkiParse(e.to_string()))?; + if !rem.is_empty() { + return Err(CertPathError::IssuerSpkiTrailingBytes(rem.len())); + } + Ok(spki) +} + +fn parse_x509_cert(der: &[u8]) -> Result, CertPathError> { + let (rem, cert) = X509Certificate::from_der(der) + .map_err(|e| CertPathError::EeSignatureInvalid(e.to_string()))?; + if !rem.is_empty() { + return Err(CertPathError::EeSignatureInvalid( + "trailing bytes after EE certificate".to_string(), + )); + } + Ok(cert) +} + +fn verify_ee_cert_signature( + ee: &X509Certificate<'_>, + issuer_spki: &SubjectPublicKeyInfo<'_>, +) -> Result<(), CertPathError> { + crate::crypto_sig_cache::verify_with_cache( + crate::crypto_sig_cache::CryptoSigVerifyPoint::EeCertX509, + ee.tbs_certificate.as_ref(), + ee.signature_value.data.as_ref(), + issuer_spki.raw, + || { + ee.verify_signature(Some(issuer_spki)) + .map_err(|e| CertPathError::EeSignatureInvalid(e.to_string())) + }, + ) +} + +fn verify_ee_cert_signature_fast( + ee_tbs_der: &[u8], + ee_signature_bytes: &[u8], + issuer_spki: &SubjectPublicKeyInfo<'_>, +) -> Result<(), CertPathError> { + crate::crypto_sig_cache::verify_with_cache( + crate::crypto_sig_cache::CryptoSigVerifyPoint::EeCertFast, + ee_tbs_der, + ee_signature_bytes, + issuer_spki.raw, + || { + let key = signature::UnparsedPublicKey::new( + &signature::RSA_PKCS1_2048_8192_SHA256, + &issuer_spki.subject_public_key.data, + ); + key.verify(ee_tbs_der, ee_signature_bytes) + .map_err(|e| CertPathError::EeSignatureInvalid(e.to_string())) + }, + ) +} + +fn validate_ee_aki_matches_issuer_ski( + ee: &ResourceCertificate, + issuer_ca: &ResourceCertificate, +) -> Result<(), CertPathError> { + let Some(issuer_ski) = issuer_ca.tbs.extensions.subject_key_identifier.as_deref() else { + return Err(CertPathError::IssuerSkiMissing); + }; + let Some(ee_aki) = ee.tbs.extensions.authority_key_identifier.as_deref() else { + return Err(CertPathError::EeAkiMissing); + }; + if ee_aki != issuer_ski { + return Err(CertPathError::EeAkiMismatch); + } + Ok(()) +} + +fn validate_ee_aia_points_to_issuer_uri( + ee: &ResourceCertificate, + issuer_ca_rsync_uri: &str, +) -> Result<(), CertPathError> { + let Some(uris) = ee.tbs.extensions.ca_issuers_uris.as_ref() else { + return Err(CertPathError::EeAiaMissing); + }; + if uris.iter().any(|u| u.as_str() == issuer_ca_rsync_uri) { + return Ok(()); + } + if uris.iter().any(|u| u.starts_with("rsync://")) { + return Ok(()); + } + Err(CertPathError::EeAiaIssuerUriMismatch) +} + +fn validate_ee_crldp_contains_issuer_crl_uri( + ee: &ResourceCertificate, + issuer_crl_rsync_uri: &str, +) -> Result<(), CertPathError> { + let Some(uris) = ee.tbs.extensions.crl_distribution_points_uris.as_ref() else { + return Err(CertPathError::EeCrlDpMissing); + }; + if !uris.iter().any(|u| u.as_str() == issuer_crl_rsync_uri) { + return Err(CertPathError::EeCrlDpUriMismatch); + } + Ok(()) +} + +fn validate_ee_key_usage(cert: &X509Certificate<'_>) -> Result<(), CertPathError> { + let mut ku_critical: Option = None; + for ext in cert.extensions() { + if ext.oid.as_bytes() == crate::data_model::oid::OID_KEY_USAGE_RAW { + ku_critical = Some(ext.critical); + break; + } + } + + let Some(critical) = ku_critical else { + return Err(CertPathError::KeyUsageMissing); + }; + if !critical { + return Err(CertPathError::KeyUsageNotCritical); + } + + let Some(ku) = cert + .key_usage() + .map_err(|e| CertPathError::EeSignatureInvalid(e.to_string()))? + else { + return Err(CertPathError::KeyUsageMissing); + }; + + let v = &ku.value; + let ok = v.digital_signature() + && !v.key_cert_sign() + && !v.crl_sign() + && !v.non_repudiation() + && !v.key_encipherment() + && !v.data_encipherment() + && !v.key_agreement() + && !v.encipher_only() + && !v.decipher_only(); + if !ok { + return Err(CertPathError::KeyUsageInvalidBits); + } + Ok(()) +} + +fn validate_ee_key_usage_summary(summary: &EeKeyUsageSummary) -> Result<(), CertPathError> { + match summary { + EeKeyUsageSummary::DigitalSignatureOnly => Ok(()), + EeKeyUsageSummary::Missing => Err(CertPathError::KeyUsageMissing), + EeKeyUsageSummary::NotCritical => Err(CertPathError::KeyUsageNotCritical), + EeKeyUsageSummary::InvalidBits => Err(CertPathError::KeyUsageInvalidBits), + EeKeyUsageSummary::ParseError(err) => Err(CertPathError::KeyUsageParse(err.clone())), + } +} + +fn time_within_validity( + t: time::OffsetDateTime, + not_before: time::OffsetDateTime, + not_after: time::OffsetDateTime, +) -> bool { + let t = t.to_offset(time::UtcOffset::UTC); + let not_before = not_before.to_offset(time::UtcOffset::UTC); + let not_after = not_after.to_offset(time::UtcOffset::UTC); + t >= not_before && t <= not_after +} + +fn crl_valid_at_time(crl: &RpkixCrl, t: time::OffsetDateTime) -> bool { + let t = t.to_offset(time::UtcOffset::UTC); + let this_update = crl.this_update.utc.to_offset(time::UtcOffset::UTC); + let next_update = crl.next_update.utc.to_offset(time::UtcOffset::UTC); + t >= this_update && t < next_update +} + +fn is_serial_revoked_by_crl(ee: &ResourceCertificate, crl: &RpkixCrl) -> bool { + let serial = BigUnsigned::from_biguint(&ee.tbs.serial_number); + crl.revoked_certs + .iter() + .any(|rc| rc.serial_number == serial) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::data_model::common::X509NameDer; + use crate::data_model::oid::OID_CP_IPADDR_ASNUMBER; + use crate::data_model::rc::{ + BasicConstraintsProfile, CertificatePoliciesProfile, RcExtensions, ResourceCertKind, + ResourceCertificate, RpkixTbsCertificate, + }; + use der_parser::num_bigint::BigUint; + + fn dummy_cert( + kind: ResourceCertKind, + subject_dn: &str, + issuer_dn: &str, + ski: Option>, + aki: Option>, + aia: Option>, + crldp: Option>, + ) -> ResourceCertificate { + let aia = aia.map(|v| v.into_iter().map(|s| s.to_string()).collect::>()); + let crldp = crldp.map(|v| v.into_iter().map(|s| s.to_string()).collect::>()); + ResourceCertificate { + raw_der: Vec::new(), + kind, + tbs: RpkixTbsCertificate { + version: 2, + serial_number: BigUint::from(1u8), + signature_algorithm: "1.2.840.113549.1.1.11".to_string(), + issuer_name: X509NameDer(issuer_dn.as_bytes().to_vec()), + subject_name: X509NameDer(subject_dn.as_bytes().to_vec()), + validity_not_before: time::OffsetDateTime::UNIX_EPOCH, + validity_not_after: time::OffsetDateTime::UNIX_EPOCH, + subject_public_key_info: Vec::new(), + extensions: RcExtensions { + basic_constraints_ca: kind == ResourceCertKind::Ca, + basic_constraints: (kind == ResourceCertKind::Ca).then_some( + BasicConstraintsProfile { + ca: true, + critical: true, + path_len_constraint: None, + }, + ), + subject_key_identifier: ski, + authority_key_identifier: aki, + crl_distribution_points_uris: crldp, + ca_issuers_uris: aia, + subject_info_access: None, + certificate_policies_oid: (kind == ResourceCertKind::Ca) + .then_some(OID_CP_IPADDR_ASNUMBER.to_string()), + certificate_policies: (kind == ResourceCertKind::Ca).then_some( + CertificatePoliciesProfile { + policy_oid: OID_CP_IPADDR_ASNUMBER.to_string(), + qualifier_oids: Vec::new(), + }, + ), + extension_oids: Vec::new(), + ip_resources: None, + as_resources: None, + }, + }, + } + } + + #[test] + fn ee_aki_and_aia_and_crldp_binding_checks_report_errors() { + let issuer = dummy_cert( + ResourceCertKind::Ca, + "CN=issuer", + "CN=issuer", + Some(vec![1]), + None, + None, + None, + ); + + let ee = dummy_cert( + ResourceCertKind::Ee, + "CN=ee", + "CN=issuer", + Some(vec![2]), + Some(vec![9]), + Some(vec!["rsync://example.test/issuer.cer"]), + Some(vec!["rsync://example.test/issuer.crl"]), + ); + let err = validate_ee_aki_matches_issuer_ski(&ee, &issuer).unwrap_err(); + assert!(matches!(err, CertPathError::EeAkiMismatch), "{err}"); + + let issuer_missing_ski = dummy_cert( + ResourceCertKind::Ca, + "CN=issuer", + "CN=issuer", + None, + None, + None, + None, + ); + let err = validate_ee_aki_matches_issuer_ski(&ee, &issuer_missing_ski).unwrap_err(); + assert!(matches!(err, CertPathError::IssuerSkiMissing), "{err}"); + + let ee_missing_aki = dummy_cert( + ResourceCertKind::Ee, + "CN=ee", + "CN=issuer", + Some(vec![2]), + None, + Some(vec!["rsync://example.test/issuer.cer"]), + Some(vec!["rsync://example.test/issuer.crl"]), + ); + let err = validate_ee_aki_matches_issuer_ski(&ee_missing_aki, &issuer).unwrap_err(); + assert!(matches!(err, CertPathError::EeAkiMissing), "{err}"); + + let ee_missing_aia = dummy_cert( + ResourceCertKind::Ee, + "CN=ee", + "CN=issuer", + Some(vec![2]), + Some(vec![1]), + None, + Some(vec!["rsync://example.test/issuer.crl"]), + ); + let err = validate_ee_aia_points_to_issuer_uri( + &ee_missing_aia, + "rsync://example.test/issuer.cer", + ) + .unwrap_err(); + assert!(matches!(err, CertPathError::EeAiaMissing), "{err}"); + + let ee_wrong_aia = dummy_cert( + ResourceCertKind::Ee, + "CN=ee", + "CN=issuer", + Some(vec![2]), + Some(vec![1]), + Some(vec!["rsync://example.test/other.cer"]), + Some(vec!["rsync://example.test/issuer.crl"]), + ); + validate_ee_aia_points_to_issuer_uri(&ee_wrong_aia, "rsync://example.test/issuer.cer") + .expect("non-matching rsync AIA is currently accepted"); + + let ee_missing_crldp = dummy_cert( + ResourceCertKind::Ee, + "CN=ee", + "CN=issuer", + Some(vec![2]), + Some(vec![1]), + Some(vec!["rsync://example.test/issuer.cer"]), + None, + ); + let err = validate_ee_crldp_contains_issuer_crl_uri( + &ee_missing_crldp, + "rsync://example.test/issuer.crl", + ) + .unwrap_err(); + assert!(matches!(err, CertPathError::EeCrlDpMissing), "{err}"); + + let ee_wrong_crldp = dummy_cert( + ResourceCertKind::Ee, + "CN=ee", + "CN=issuer", + Some(vec![2]), + Some(vec![1]), + Some(vec!["rsync://example.test/issuer.cer"]), + Some(vec!["rsync://example.test/other.crl"]), + ); + let err = validate_ee_crldp_contains_issuer_crl_uri( + &ee_wrong_crldp, + "rsync://example.test/issuer.crl", + ) + .unwrap_err(); + assert!(matches!(err, CertPathError::EeCrlDpUriMismatch), "{err}"); + } + + #[test] + fn ee_binding_checks_accept_when_matching() { + let issuer = dummy_cert( + ResourceCertKind::Ca, + "CN=issuer", + "CN=issuer", + Some(vec![1]), + None, + None, + None, + ); + let ee = dummy_cert( + ResourceCertKind::Ee, + "CN=ee", + "CN=issuer", + Some(vec![2]), + Some(vec![1]), + Some(vec!["rsync://example.test/issuer.cer"]), + Some(vec!["rsync://example.test/issuer.crl"]), + ); + validate_ee_aki_matches_issuer_ski(&ee, &issuer).expect("aki ok"); + validate_ee_aia_points_to_issuer_uri(&ee, "rsync://example.test/issuer.cer") + .expect("aia ok"); + validate_ee_crldp_contains_issuer_crl_uri(&ee, "rsync://example.test/issuer.crl") + .expect("crldp ok"); + } + + #[test] + fn signature_cache_hit_does_not_skip_crl_revocation_check() { + use crate::crypto_sig_cache::{CryptoSigCache, CryptoSigVerifyPoint, compute_cache_key}; + use crate::data_model::common::{Asn1TimeEncoding, Asn1TimeUtc}; + use crate::data_model::crl::CrlExtensions; + + let _guard = crate::crypto_sig_cache::GLOBAL_TEST_LOCK + .lock() + .expect("test lock"); + crate::crypto_sig_cache::clear_global(); + + let now = time::OffsetDateTime::now_utc(); + let mut issuer = dummy_cert( + ResourceCertKind::Ca, + "CN=issuer", + "CN=issuer", + Some(vec![1]), + None, + None, + None, + ); + issuer.tbs.validity_not_before = now - time::Duration::days(1); + issuer.tbs.validity_not_after = now + time::Duration::days(1); + let mut ee_rc = dummy_cert( + ResourceCertKind::Ee, + "CN=ee", + "CN=issuer", + Some(vec![2]), + Some(vec![1]), + Some(vec!["rsync://example.test/issuer.cer"]), + Some(vec!["rsync://example.test/issuer.crl"]), + ); + ee_rc.tbs.validity_not_before = now - time::Duration::days(1); + ee_rc.tbs.validity_not_after = now + time::Duration::days(1); + + // Tampered "object": these bytes could never pass the real RSA verification, so an + // Ok outcome below proves the crypto step was skipped by the cache hit. + let ee = ResourceEeCertificate { + raw_der: Vec::new(), + subject_key_identifier: vec![2], + spki_der: Vec::new(), + rsa_public_modulus: Vec::new(), + rsa_public_exponent: Vec::new(), + tbs_certificate_der: b"tampered-tbs-bytes".to_vec(), + signature_bytes: b"tampered-signature-bytes".to_vec(), + key_usage_summary: EeKeyUsageSummary::DigitalSignatureOnly, + sia_signed_object_uris: Vec::new(), + resource_cert: ee_rc, + }; + + // Any real RSA SPKI works as the verifier identity (the TA fixture is a valid RSA + // certificate); the SPKI DER only feeds the cache key here. + let ta_der = std::fs::read(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/ta/apnic-ta.cer" + )) + .expect("read ta fixture"); + let (_, ta_x509) = X509Certificate::from_der(&ta_der).expect("parse ta fixture"); + let issuer_spki = &ta_x509.tbs_certificate.subject_pki; + + let crl = RpkixCrl { + raw_der: Vec::new(), + version: 2, + issuer_dn: "CN=issuer".to_string(), + signature_algorithm_oid: "1.2.840.113549.1.1.11".to_string(), + this_update: Asn1TimeUtc { + utc: now - time::Duration::hours(1), + encoding: Asn1TimeEncoding::UtcTime, + }, + next_update: Asn1TimeUtc { + utc: now + time::Duration::hours(1), + encoding: Asn1TimeEncoding::UtcTime, + }, + revoked_certs: Vec::new(), + extensions: CrlExtensions { + authority_key_identifier: vec![1], + crl_number: BigUnsigned { bytes_be: vec![1] }, + }, + }; + + let dir = tempfile::tempdir().expect("tempdir"); + let cache = std::sync::Arc::new(CryptoSigCache::load_or_rebuild( + dir.path().join("work-db.crypto-sig-cache"), + true, + )); + cache.insert_positive(compute_cache_key( + CryptoSigVerifyPoint::EeCertFast, + b"tampered-tbs-bytes", + b"tampered-signature-bytes", + issuer_spki.raw, + )); + crate::crypto_sig_cache::install_global(std::sync::Arc::clone(&cache)); + + // EE serial is 1 (dummy_cert); revoke it. The revocation check runs before the + // signature step in this code path, so the result must be EeRevoked — never Ok — + // even though a positive signature conclusion is cached. + let revoked_serials: HashSet> = HashSet::from([vec![1u8]]); + let err = validate_signed_object_ee_cert_path_fast( + &ee, + &issuer, + issuer_spki, + &crl, + &revoked_serials, + Some("rsync://example.test/issuer.cer"), + Some("rsync://example.test/issuer.crl"), + now, + ) + .unwrap_err(); + assert!( + matches!(err, CertPathError::EeRevoked), + "cached signature conclusion must not bypass revocation: {err}" + ); + + // Not revoked: the cached positive conclusion lets validation pass even though the + // tampered bytes could never verify — proving the crypto step was skipped. + let not_revoked: HashSet> = HashSet::new(); + validate_signed_object_ee_cert_path_fast( + &ee, + &issuer, + issuer_spki, + &crl, + ¬_revoked, + Some("rsync://example.test/issuer.cer"), + Some("rsync://example.test/issuer.crl"), + now, + ) + .expect("cache hit skips the impossible verification"); + + let summary = cache.summary(); + let stats = summary + .per_point + .get("ee_cert_fast") + .expect("ee_cert_fast stats"); + assert_eq!(stats.verify_skipped, 1); + assert_eq!( + stats.verify_executed, 0, + "real verification must never run on cache hits" + ); + crate::crypto_sig_cache::clear_global(); + } +} diff --git a/crates/panda-rpki-validator/src/validation/from_tal.rs b/crates/panda-rpki-validator/src/validation/from_tal.rs new file mode 100644 index 0000000..99abcab --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/from_tal.rs @@ -0,0 +1,556 @@ +use url::Url; + +use crate::data_model::ta::{TrustAnchor, TrustAnchorError}; +use crate::data_model::tal::{Tal, TalDecodeError}; +use crate::fetch::rsync::RsyncFetcher; +use crate::sync::rrdp::Fetcher; +use crate::validation::ca_instance::{ + CaInstanceUris, CaInstanceUrisError, ca_instance_uris_from_ca_certificate, +}; +use crate::validation::run::{RunError, RunOutput, run_publication_point_once}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DiscoveredRootCaInstance { + pub tal_url: Option, + pub trust_anchor: TrustAnchor, + pub ca_instance: CaInstanceUris, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RunFromTalOutput { + pub discovery: DiscoveredRootCaInstance, + pub run: RunOutput, +} + +#[derive(Debug, thiserror::Error)] +pub enum FromTalError { + #[error("TAL fetch failed: {0} (RFC 8630 §2.2)")] + TalFetch(String), + + #[error("TAL decode failed: {0} (RFC 8630 §2.2)")] + TalDecode(#[from] TalDecodeError), + + #[error("failed to fetch TA certificate from TAL: {0} (RFC 8630 §2.3)")] + TaFetch(String), + + #[error("failed to bind TAL and TA certificate: {0} (RFC 8630 §2.3)")] + Bind(#[from] TrustAnchorError), + + #[error("failed to discover CA instance URIs from TA certificate: {0}")] + CaInstanceUris(#[from] CaInstanceUrisError), + + #[error("run failed: {0}")] + Run(#[from] RunError), + + #[error("TAL contains no TA URIs (RFC 8630 §2.2)")] + NoTaUris, +} + +pub fn discover_root_ca_instance_from_tal_url( + http_fetcher: &dyn Fetcher, + tal_url: &str, +) -> Result { + let tal_bytes = http_fetcher + .fetch(tal_url) + .map_err(FromTalError::TalFetch)?; + let tal = Tal::decode_bytes(&tal_bytes)?; + discover_root_ca_instance_from_tal(http_fetcher, tal, Some(tal_url.to_string())) +} + +pub fn discover_root_ca_instance_from_tal_url_with_strict_name( + http_fetcher: &dyn Fetcher, + tal_url: &str, +) -> Result { + let tal_bytes = http_fetcher + .fetch(tal_url) + .map_err(FromTalError::TalFetch)?; + let tal = Tal::decode_bytes(&tal_bytes)?; + discover_root_ca_instance_from_tal_with_strict_name( + http_fetcher, + tal, + Some(tal_url.to_string()), + ) +} + +pub fn discover_root_ca_instance_from_tal_url_with_fetchers( + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn RsyncFetcher, + tal_url: &str, +) -> Result { + let tal_bytes = http_fetcher + .fetch(tal_url) + .map_err(FromTalError::TalFetch)?; + let tal = Tal::decode_bytes(&tal_bytes)?; + discover_root_ca_instance_from_tal_with_fetchers_impl( + http_fetcher, + rsync_fetcher, + tal, + Some(tal_url.to_string()), + false, + TaUriOrder::HttpFirst, + ) +} + +pub fn discover_root_ca_instance_from_tal_url_with_fetchers_strict_name( + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn RsyncFetcher, + tal_url: &str, +) -> Result { + let tal_bytes = http_fetcher + .fetch(tal_url) + .map_err(FromTalError::TalFetch)?; + let tal = Tal::decode_bytes(&tal_bytes)?; + discover_root_ca_instance_from_tal_with_fetchers_impl( + http_fetcher, + rsync_fetcher, + tal, + Some(tal_url.to_string()), + true, + TaUriOrder::HttpFirst, + ) +} + +pub fn discover_root_ca_instance_from_tal( + http_fetcher: &dyn Fetcher, + tal: Tal, + tal_url: Option, +) -> Result { + discover_root_ca_instance_from_tal_impl(http_fetcher, tal, tal_url, false) +} + +pub fn discover_root_ca_instance_from_tal_with_strict_name( + http_fetcher: &dyn Fetcher, + tal: Tal, + tal_url: Option, +) -> Result { + discover_root_ca_instance_from_tal_impl(http_fetcher, tal, tal_url, true) +} + +fn discover_root_ca_instance_from_tal_impl( + http_fetcher: &dyn Fetcher, + tal: Tal, + tal_url: Option, + strict_name: bool, +) -> Result { + if tal.ta_uris.is_empty() { + return Err(FromTalError::NoTaUris); + } + + let mut last_err: Option = None; + for ta_uri in tal.ta_uris.iter() { + let ta_der = match http_fetcher.fetch(ta_uri.as_str()) { + Ok(b) => b, + Err(e) => { + last_err = Some(format!("fetch {ta_uri} failed: {e}")); + continue; + } + }; + + let trust_anchor = + match bind_trust_anchor_der(tal.clone(), &ta_der, Some(ta_uri), strict_name) { + Ok(ta) => ta, + Err(e) => { + last_err = Some(format!("bind {ta_uri} failed: {e}")); + continue; + } + }; + + let ca_instance = + match ca_instance_uris_from_ca_certificate(&trust_anchor.ta_certificate.rc_ca) { + Ok(v) => v, + Err(e) => { + last_err = Some(format!("CA instance discovery failed: {e}")); + continue; + } + }; + + return Ok(DiscoveredRootCaInstance { + tal_url, + trust_anchor, + ca_instance, + }); + } + + Err(FromTalError::TaFetch(last_err.unwrap_or_else(|| { + "unknown TA candidate error".to_string() + }))) +} + +pub fn discover_root_ca_instance_from_tal_with_fetchers( + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn RsyncFetcher, + tal: Tal, + tal_url: Option, +) -> Result { + discover_root_ca_instance_from_tal_with_fetchers_impl( + http_fetcher, + rsync_fetcher, + tal, + tal_url, + false, + TaUriOrder::RsyncFirst, + ) +} + +pub fn discover_root_ca_instance_from_tal_with_fetchers_strict_name( + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn RsyncFetcher, + tal: Tal, + tal_url: Option, +) -> Result { + discover_root_ca_instance_from_tal_with_fetchers_impl( + http_fetcher, + rsync_fetcher, + tal, + tal_url, + true, + TaUriOrder::RsyncFirst, + ) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum TaUriOrder { + HttpFirst, + RsyncFirst, +} + +fn discover_root_ca_instance_from_tal_with_fetchers_impl( + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn RsyncFetcher, + tal: Tal, + tal_url: Option, + strict_name: bool, + ta_uri_order: TaUriOrder, +) -> Result { + if tal.ta_uris.is_empty() { + return Err(FromTalError::NoTaUris); + } + + let mut last_err: Option = None; + let mut ta_uris = tal.ta_uris.clone(); + ta_uris.sort_by_key(|uri| ta_uri_priority(uri, ta_uri_order)); + for ta_uri in ta_uris.iter() { + let ta_der = match fetch_ta_der(http_fetcher, rsync_fetcher, ta_uri) { + Ok(b) => b, + Err(e) => { + last_err = Some(format!("fetch {ta_uri} failed: {e}")); + continue; + } + }; + + let trust_anchor = + match bind_trust_anchor_der(tal.clone(), &ta_der, Some(ta_uri), strict_name) { + Ok(ta) => ta, + Err(e) => { + last_err = Some(format!("bind {ta_uri} failed: {e}")); + continue; + } + }; + + let ca_instance = + match ca_instance_uris_from_ca_certificate(&trust_anchor.ta_certificate.rc_ca) { + Ok(v) => v, + Err(e) => { + last_err = Some(format!("CA instance discovery failed: {e}")); + continue; + } + }; + + return Ok(DiscoveredRootCaInstance { + tal_url, + trust_anchor, + ca_instance, + }); + } + + Err(FromTalError::TaFetch(last_err.unwrap_or_else(|| { + "unknown TA candidate error".to_string() + }))) +} + +fn ta_uri_priority(uri: &Url, order: TaUriOrder) -> u8 { + match (order, uri.scheme()) { + (TaUriOrder::HttpFirst, "https" | "http") => 0, + (TaUriOrder::HttpFirst, "rsync") => 1, + (TaUriOrder::RsyncFirst, "rsync") => 0, + (TaUriOrder::RsyncFirst, "https" | "http") => 1, + _ => 2, + } +} + +fn fetch_ta_der( + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn RsyncFetcher, + ta_uri: &Url, +) -> Result, String> { + match ta_uri.scheme() { + "https" | "http" => http_fetcher.fetch(ta_uri.as_str()), + "rsync" => fetch_ta_der_via_rsync(rsync_fetcher, ta_uri.as_str()), + scheme => Err(format!("unsupported TA URI scheme: {scheme}")), + } +} + +fn bind_trust_anchor_der( + tal: Tal, + ta_der: &[u8], + resolved_uri: Option<&Url>, + strict_name: bool, +) -> Result { + if strict_name { + TrustAnchor::bind_der_with_strict_name(tal, ta_der, resolved_uri) + } else { + TrustAnchor::bind_der(tal, ta_der, resolved_uri) + } +} + +fn fetch_ta_der_via_rsync( + rsync_fetcher: &dyn RsyncFetcher, + ta_rsync_uri: &str, +) -> Result, String> { + rsync_fetcher + .fetch_object(ta_rsync_uri) + .map_err(|e| e.to_string()) +} + +pub fn discover_root_ca_instance_from_tal_and_ta_der( + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&Url>, +) -> Result { + discover_root_ca_instance_from_tal_and_ta_der_impl(tal_bytes, ta_der, resolved_ta_uri, false) +} + +pub fn discover_root_ca_instance_from_tal_and_ta_der_with_strict_name( + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&Url>, +) -> Result { + discover_root_ca_instance_from_tal_and_ta_der_impl(tal_bytes, ta_der, resolved_ta_uri, true) +} + +fn discover_root_ca_instance_from_tal_and_ta_der_impl( + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&Url>, + strict_name: bool, +) -> Result { + let tal = Tal::decode_bytes(tal_bytes)?; + let trust_anchor = bind_trust_anchor_der(tal, ta_der, resolved_ta_uri, strict_name)?; + let ca_instance = ca_instance_uris_from_ca_certificate(&trust_anchor.ta_certificate.rc_ca)?; + Ok(DiscoveredRootCaInstance { + tal_url: None, + trust_anchor, + ca_instance, + }) +} + +pub fn canonical_tal_rsync_uri_from_bytes(tal_bytes: &[u8]) -> Result { + let tal = Tal::decode_bytes(tal_bytes)?; + tal.ta_uris + .iter() + .find(|uri| uri.scheme() == "rsync") + .cloned() + .ok_or_else(|| { + FromTalError::TaFetch("TAL contains no rsync TA URI for offline TA binding".to_string()) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::fetch::rsync::LocalDirRsyncFetcher; + use std::collections::HashMap; + + struct MapHttpFetcher { + map: HashMap>, + } + + impl Fetcher for MapHttpFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + self.map + .get(uri) + .cloned() + .ok_or_else(|| format!("no fixture mapped for {uri}")) + } + } + + struct FailingHttpFetcher; + + impl Fetcher for FailingHttpFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + Err(format!("blocked test HTTP fetch: {uri}")) + } + } + + #[test] + fn discover_root_ca_instance_from_tal_with_fetchers_supports_rsync_ta_uri() { + let tal_bytes = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/tal/apnic-rfc7730-https.tal"), + ) + .unwrap(); + let ta_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/ta/apnic-ta.cer"), + ) + .unwrap(); + let tal = Tal::decode_bytes(&tal_bytes).unwrap(); + let rsync_uri = tal + .ta_uris + .iter() + .find(|uri| uri.scheme() == "rsync") + .unwrap() + .clone(); + + let td = tempfile::tempdir().unwrap(); + let mirror_root = td + .path() + .join(rsync_uri.host_str().unwrap()) + .join("repository"); + std::fs::create_dir_all(&mirror_root).unwrap(); + std::fs::write(mirror_root.join("apnic-rpki-root-iana-origin.cer"), ta_der).unwrap(); + + let rsync = LocalDirRsyncFetcher::new( + td.path() + .join(rsync_uri.host_str().unwrap()) + .join("repository"), + ); + let discovery = discover_root_ca_instance_from_tal_with_fetchers( + &FailingHttpFetcher, + &rsync, + tal, + None, + ) + .expect("discover via rsync TA fallback"); + assert!( + discovery + .trust_anchor + .resolved_ta_uri + .unwrap() + .as_str() + .starts_with("rsync://") + ); + } + + struct PanicRsyncFetcher; + + impl RsyncFetcher for PanicRsyncFetcher { + fn fetch_objects( + &self, + rsync_base_uri: &str, + ) -> crate::fetch::rsync::RsyncFetchResult)>> { + panic!("rsync should not be used when HTTPS TA URI is available: {rsync_base_uri}") + } + } + + #[test] + fn discover_root_ca_instance_from_tal_url_with_fetchers_prefers_https_ta_uri() { + let tal_url = "https://example.test/apnic.tal"; + let tal_bytes = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/tal/apnic-rfc7730-https.tal"), + ) + .unwrap(); + let ta_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/ta/apnic-ta.cer"), + ) + .unwrap(); + let https_uri = "https://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer"; + let mut map = HashMap::new(); + map.insert(tal_url.to_string(), tal_bytes); + map.insert(https_uri.to_string(), ta_der); + let discovery = discover_root_ca_instance_from_tal_url_with_fetchers( + &MapHttpFetcher { map }, + &PanicRsyncFetcher, + tal_url, + ) + .expect("discover via HTTPS TA URI"); + assert_eq!( + discovery.trust_anchor.resolved_ta_uri.unwrap().as_str(), + https_uri + ); + } + + #[test] + fn discover_root_ca_instance_from_tal_url_with_fetchers_supports_rsync_only_tal() { + let tal_url = "https://example.test/apnic-rsync-only.tal"; + let tal_bytes = std::fs::read_to_string( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/tal/apnic-rfc7730-https.tal"), + ) + .unwrap(); + let rsync_uri = "rsync://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer"; + let key_material = tal_bytes + .split_once("\n\n") + .map(|(_, key)| key) + .expect("fixture contains TAL key material"); + let rsync_only_tal = format!("{rsync_uri}\n\n{key_material}"); + let ta_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/ta/apnic-ta.cer"), + ) + .unwrap(); + let td = tempfile::tempdir().unwrap(); + let mirror_root = td.path().join("repository"); + std::fs::create_dir_all(&mirror_root).unwrap(); + std::fs::write(mirror_root.join("apnic-rpki-root-iana-origin.cer"), ta_der).unwrap(); + + let mut map = HashMap::new(); + map.insert(tal_url.to_string(), rsync_only_tal.into_bytes()); + let discovery = discover_root_ca_instance_from_tal_url_with_fetchers( + &MapHttpFetcher { map }, + &LocalDirRsyncFetcher::new(mirror_root), + tal_url, + ) + .expect("discover via rsync-only TAL"); + assert_eq!( + discovery.trust_anchor.resolved_ta_uri.unwrap().as_str(), + rsync_uri + ); + } +} + +pub fn run_root_from_tal_url_once( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_url: &str, + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, + validation_time: time::OffsetDateTime, +) -> Result { + let discovery = + discover_root_ca_instance_from_tal_url_with_fetchers(http_fetcher, rsync_fetcher, tal_url)?; + + let run = run_publication_point_once( + store, + policy, + discovery.ca_instance.rrdp_notification_uri.as_deref(), + &discovery.ca_instance.rsync_base_uri, + &discovery.ca_instance.manifest_rsync_uri, + &discovery.ca_instance.publication_point_rsync_uri, + http_fetcher, + rsync_fetcher, + &discovery.trust_anchor.ta_certificate.raw_der, + None, + discovery + .trust_anchor + .ta_certificate + .rc_ca + .tbs + .extensions + .ip_resources + .as_ref(), + discovery + .trust_anchor + .ta_certificate + .rc_ca + .tbs + .extensions + .as_resources + .as_ref(), + validation_time, + )?; + + Ok(RunFromTalOutput { discovery, run }) +} diff --git a/crates/panda-rpki-validator/src/validation/manifest.rs b/crates/panda-rpki-validator/src/validation/manifest.rs new file mode 100644 index 0000000..802564e --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/manifest.rs @@ -0,0 +1,1824 @@ +use crate::current_repo_index::CurrentRepoIndexHandle; +use crate::data_model::manifest::{ManifestDecodeError, ManifestObject, ManifestValidateError}; +use crate::data_model::signed_object::SignedObjectVerifyError; +use crate::policy::{CaFailedFetchPolicy, Policy}; +use crate::report::{RfcRef, Warning}; +use crate::storage::{PackFile, PackTime, RocksStore, StorageError, VcirArtifactRole}; +use crate::validation::cert_path::{CertPathError, validate_signed_object_ee_cert_path_fast}; +use crate::validation::publication_point::PublicationPointSnapshot; +use sha2::Digest; +use std::cmp::Ordering; +use std::collections::HashSet; +use x509_parser::prelude::FromDer; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum PublicationPointSource { + Fresh, + PublicationPointCache, + VcirCurrentInstance, + FailedFetchNoCache, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PublicationPointResult { + pub source: PublicationPointSource, + pub snapshot: PublicationPointSnapshot, + pub warnings: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct FreshValidatedPublicationPoint { + pub manifest_rsync_uri: String, + pub publication_point_rsync_uri: String, + pub manifest_number_be: Vec, + pub this_update: PackTime, + pub next_update: PackTime, + pub verified_at: PackTime, + pub manifest_bytes: Vec, + pub files: Vec, +} + +pub trait PublicationPointData { + fn manifest_rsync_uri(&self) -> &str; + fn publication_point_rsync_uri(&self) -> &str; + fn manifest_number_be(&self) -> &[u8]; + fn this_update(&self) -> &PackTime; + fn next_update(&self) -> &PackTime; + fn verified_at(&self) -> &PackTime; + fn manifest_bytes(&self) -> &[u8]; + fn files(&self) -> &[PackFile]; +} + +impl FreshValidatedPublicationPoint { + pub fn to_publication_point_snapshot(&self) -> PublicationPointSnapshot { + PublicationPointSnapshot { + format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, + manifest_rsync_uri: self.manifest_rsync_uri.clone(), + publication_point_rsync_uri: self.publication_point_rsync_uri.clone(), + manifest_number_be: self.manifest_number_be.clone(), + this_update: self.this_update.clone(), + next_update: self.next_update.clone(), + verified_at: self.verified_at.clone(), + manifest_bytes: self.manifest_bytes.clone(), + files: self.files.clone(), + } + } +} + +impl PublicationPointData for FreshValidatedPublicationPoint { + fn manifest_rsync_uri(&self) -> &str { + &self.manifest_rsync_uri + } + + fn publication_point_rsync_uri(&self) -> &str { + &self.publication_point_rsync_uri + } + + fn manifest_number_be(&self) -> &[u8] { + self.manifest_number_be.as_slice() + } + + fn this_update(&self) -> &PackTime { + &self.this_update + } + + fn next_update(&self) -> &PackTime { + &self.next_update + } + + fn verified_at(&self) -> &PackTime { + &self.verified_at + } + + fn manifest_bytes(&self) -> &[u8] { + self.manifest_bytes.as_slice() + } + + fn files(&self) -> &[PackFile] { + self.files.as_slice() + } +} + +impl PublicationPointData for PublicationPointSnapshot { + fn manifest_rsync_uri(&self) -> &str { + &self.manifest_rsync_uri + } + + fn publication_point_rsync_uri(&self) -> &str { + &self.publication_point_rsync_uri + } + + fn manifest_number_be(&self) -> &[u8] { + self.manifest_number_be.as_slice() + } + + fn this_update(&self) -> &PackTime { + &self.this_update + } + + fn next_update(&self) -> &PackTime { + &self.next_update + } + + fn verified_at(&self) -> &PackTime { + &self.verified_at + } + + fn manifest_bytes(&self) -> &[u8] { + self.manifest_bytes.as_slice() + } + + fn files(&self) -> &[PackFile] { + self.files.as_slice() + } +} + +#[derive(Debug, thiserror::Error)] +pub enum ManifestFreshError { + #[error("repo sync failed: {detail} (RFC 8182 §3.4.5; RFC 9286 §6.6)")] + RepoSyncFailed { detail: String }, + + #[error( + "manifest not found in current repository view: {manifest_rsync_uri} (RFC 9286 §6.2; RFC 9286 §6.6)" + )] + MissingManifest { manifest_rsync_uri: String }, + + #[error("manifest decode failed: {0} (RFC 9286 §4; RFC 9286 §6.2; RFC 9286 §6.6)")] + Decode(#[from] ManifestDecodeError), + + #[error( + "manifest embedded EE certificate resources invalid: {0} (RFC 9286 §5.1; RFC 9286 §6.2; RFC 9286 §6.6)" + )] + EeResources(#[from] ManifestValidateError), + + #[error( + "manifest CMS signature verification failed: {0} (RFC 6488 §3; RFC 9589 §4; RFC 9286 §6.2; RFC 9286 §6.6)" + )] + Signature(#[from] SignedObjectVerifyError), + + #[error( + "manifest embedded EE certificate path validation failed: {0} (RFC 6488 §3; RFC 9286 §6.2; RFC 9286 §6.6)" + )] + EeCertPath(#[from] CertPathError), + + #[error( + "manifest embedded EE certificate CRLDistributionPoints missing (cannot validate EE certificate) (RFC 6487 §4.8.6; RFC 6488 §3; RFC 9286 §6.2; RFC 9286 §6.6)" + )] + EeCrlDpMissing, + + #[error( + "publication point contains no CRL files (cannot validate manifest EE certificate) (RFC 9286 §7; RFC 6487 §4.8.6; RFC 6488 §3; RFC 9286 §6.2; RFC 9286 §6.6)" + )] + NoCrlFiles, + + #[error( + "CRL referenced by manifest embedded EE certificate CRLDistributionPoints not found at publication point: {0} (RFC 6487 §4.8.6; RFC 9286 §4.2.1; RFC 9286 §6.2; RFC 9286 §6.6)" + )] + EeCrlNotFound(String), + + #[error( + "manifest is not valid at validation_time: this_update={this_update_rfc3339_utc} next_update={next_update_rfc3339_utc} validation_time={validation_time_rfc3339_utc} (RFC 9286 §6.3; RFC 9286 §6.6)" + )] + StaleOrEarly { + this_update_rfc3339_utc: String, + next_update_rfc3339_utc: String, + validation_time_rfc3339_utc: String, + }, + + #[error( + "manifest must reside at the same publication point as id-ad-caRepository: manifest={manifest_rsync_uri} publication_point={publication_point_rsync_uri} (RFC 9286 §6.1; RFC 9286 §6.6)" + )] + ManifestOutsidePublicationPoint { + manifest_rsync_uri: String, + publication_point_rsync_uri: String, + }, + + #[error( + "manifestNumber not higher than previously validated manifest: old={old_hex} new={new_hex} (RFC 9286 §4.2.1; RFC 9286 §6.6)" + )] + ManifestNumberNotIncreasing { old_hex: String, new_hex: String }, + + #[error( + "thisUpdate not more recent than previously validated manifest: old={old_rfc3339_utc} new={new_rfc3339_utc} (RFC 9286 §4.2.1; RFC 9286 §6.6)" + )] + ThisUpdateNotIncreasing { + old_rfc3339_utc: String, + new_rfc3339_utc: String, + }, + + #[error( + "manifest referenced file missing in current repository view: {rsync_uri} (RFC 9286 §6.4; RFC 9286 §6.6)" + )] + MissingFile { rsync_uri: String }, + + #[error("manifest file hash mismatch: {rsync_uri} (RFC 9286 §6.5; RFC 9286 §6.6)")] + HashMismatch { rsync_uri: String }, + + #[error("issuer CA certificate bytes unavailable: {detail} (RFC 6487 §4; RFC 9286 §6.2)")] + IssuerCaLoadFailed { detail: String }, +} + +impl ManifestFreshError { + pub(crate) fn should_warn_when_current_instance_reused(&self) -> bool { + !matches!( + self, + ManifestFreshError::RepoSyncFailed { .. } + | ManifestFreshError::MissingManifest { .. } + | ManifestFreshError::MissingFile { .. } + ) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum ManifestReuseError { + #[error("latest current-instance VCIR missing: {0} (RFC 9286 §6.6)")] + MissingCurrentInstanceVcir(String), + + #[error( + "latest current-instance VCIR is not marked failed-fetch eligible: {0} (RFC 9286 §6.6)" + )] + IneligibleCurrentInstanceVcir(String), + + #[error( + "latest current-instance VCIR instance_gate expired: manifest={manifest_rsync_uri} effective_until={effective_until_rfc3339_utc} validation_time={validation_time_rfc3339_utc} (RFC 9286 §6.6)" + )] + CurrentInstanceVcirExpired { + manifest_rsync_uri: String, + effective_until_rfc3339_utc: String, + validation_time_rfc3339_utc: String, + }, + + #[error( + "current-instance VCIR current_manifest_rsync_uri does not match requested manifest URI: expected={expected} actual={actual}" + )] + ManifestUriMismatch { expected: String, actual: String }, + + #[error("manifest raw bytes missing for current-instance VCIR reconstruction: {0}")] + MissingManifestRaw(String), + + #[error("artifact raw bytes missing for current-instance VCIR reconstruction: {rsync_uri}")] + MissingArtifactRaw { rsync_uri: String }, + + #[error("invalid current-instance VCIR: {0}")] + InvalidCurrentInstanceVcir(String), + + #[error("storage error during current-instance VCIR reuse: {0}")] + Storage(#[from] StorageError), +} + +#[derive(Debug, thiserror::Error)] +pub enum ManifestProcessError { + #[error("manifest processing failed and cache use is disabled: {0}")] + StopAllOutput(#[from] ManifestFreshError), + + #[error( + "manifest processing failed and no reusable current-instance validated result is available: fresh={fresh}; reused={reused}" + )] + NoUsableCache { + fresh: ManifestFreshError, + reused: ManifestReuseError, + }, + + #[error("storage error during manifest processing: {0}")] + Storage(#[from] StorageError), +} + +pub fn process_manifest_publication_point( + store: &RocksStore, + policy: &Policy, + manifest_rsync_uri: &str, + publication_point_rsync_uri: &str, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + validation_time: time::OffsetDateTime, +) -> Result { + process_manifest_publication_point_after_repo_sync( + store, + policy, + manifest_rsync_uri, + publication_point_rsync_uri, + issuer_ca_der, + issuer_ca_rsync_uri, + validation_time, + true, + None, + ) +} + +pub fn process_manifest_publication_point_fresh_after_repo_sync( + store: &RocksStore, + manifest_rsync_uri: &str, + publication_point_rsync_uri: &str, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + validation_time: time::OffsetDateTime, + repo_sync_ok: bool, + repo_sync_error: Option<&str>, +) -> Result { + process_manifest_publication_point_fresh_after_repo_sync_with_timing( + store, + manifest_rsync_uri, + publication_point_rsync_uri, + None, + issuer_ca_der, + issuer_ca_rsync_uri, + validation_time, + repo_sync_ok, + repo_sync_error, + ) + .map(|(fresh, _timing)| fresh) +} + +#[derive(Clone, Debug, Default)] +pub struct FreshPublicationPointTimingBreakdown { + pub current_index_lock_ms: u64, + pub manifest_load_ms: u64, + pub manifest_index_lookup_ms: u64, + pub manifest_blob_load_ms: u64, + pub manifest_decode_ms: u64, + pub replay_guard_ms: u64, + pub replay_meta_hit: bool, + pub replay_meta_miss: bool, + pub manifest_entries_ms: u64, + pub pack_files_ms: u64, + pub pack_files_index_lookup_ms: u64, + pub pack_files_blob_load_ms: u64, + pub ee_path_validate_ms: u64, + pub manifest_file_count: usize, +} + +pub fn process_manifest_publication_point_fresh_after_repo_sync_with_timing( + store: &RocksStore, + manifest_rsync_uri: &str, + publication_point_rsync_uri: &str, + current_repo_index: Option<&CurrentRepoIndexHandle>, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + validation_time: time::OffsetDateTime, + repo_sync_ok: bool, + repo_sync_error: Option<&str>, +) -> Result< + ( + FreshValidatedPublicationPoint, + FreshPublicationPointTimingBreakdown, + ), + ManifestFreshError, +> { + if repo_sync_ok { + try_build_fresh_publication_point_with_timing( + store, + manifest_rsync_uri, + publication_point_rsync_uri, + current_repo_index, + issuer_ca_der, + issuer_ca_rsync_uri, + validation_time, + ) + } else { + Err(ManifestFreshError::RepoSyncFailed { + detail: repo_sync_error.unwrap_or("repo sync failed").to_string(), + }) + } +} + +pub fn process_manifest_publication_point_after_repo_sync( + store: &RocksStore, + policy: &Policy, + manifest_rsync_uri: &str, + publication_point_rsync_uri: &str, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + validation_time: time::OffsetDateTime, + repo_sync_ok: bool, + repo_sync_error: Option<&str>, +) -> Result { + let fresh = if repo_sync_ok { + try_build_fresh_publication_point( + store, + manifest_rsync_uri, + publication_point_rsync_uri, + issuer_ca_der, + issuer_ca_rsync_uri, + validation_time, + ) + } else { + Err(ManifestFreshError::RepoSyncFailed { + detail: repo_sync_error.unwrap_or("repo sync failed").to_string(), + }) + }; + + match fresh { + Ok(fresh_point) => { + let snapshot = fresh_point.to_publication_point_snapshot(); + Ok(PublicationPointResult { + source: PublicationPointSource::Fresh, + snapshot, + warnings: Vec::new(), + }) + } + Err(fresh_err) => match policy.ca_failed_fetch_policy { + CaFailedFetchPolicy::StopAllOutput => { + Err(ManifestProcessError::StopAllOutput(fresh_err)) + } + CaFailedFetchPolicy::ReuseCurrentInstanceVcir => { + match load_current_instance_vcir_publication_point( + store, + manifest_rsync_uri, + publication_point_rsync_uri, + validation_time, + ) { + Ok(snapshot) => { + let mut warnings = Vec::new(); + if fresh_err.should_warn_when_current_instance_reused() { + warnings.push( + Warning::new(format!("manifest failed fetch: {fresh_err}")) + .with_rfc_refs(&[RfcRef("RFC 9286 §6.6")]) + .with_context(manifest_rsync_uri), + ); + } + Ok(PublicationPointResult { + source: PublicationPointSource::VcirCurrentInstance, + snapshot, + warnings, + }) + } + Err(reused) => Err(ManifestProcessError::NoUsableCache { + fresh: fresh_err, + reused, + }), + } + } + }, + } +} + +pub fn load_current_instance_vcir_publication_point( + store: &RocksStore, + manifest_rsync_uri: &str, + publication_point_rsync_uri: &str, + validation_time: time::OffsetDateTime, +) -> Result { + let vcir = store.get_vcir(manifest_rsync_uri)?.ok_or_else(|| { + ManifestReuseError::MissingCurrentInstanceVcir(manifest_rsync_uri.to_string()) + })?; + + if vcir.current_manifest_rsync_uri != manifest_rsync_uri { + return Err(ManifestReuseError::ManifestUriMismatch { + expected: manifest_rsync_uri.to_string(), + actual: vcir.current_manifest_rsync_uri.clone(), + }); + } + + if !vcir.audit_summary.failed_fetch_eligible { + return Err(ManifestReuseError::IneligibleCurrentInstanceVcir( + manifest_rsync_uri.to_string(), + )); + } + + let instance_effective_until = vcir + .instance_gate + .instance_effective_until + .parse() + .map_err(|e| { + ManifestReuseError::InvalidCurrentInstanceVcir(format!( + "instance_gate.instance_effective_until parse failed: {e}" + )) + })?; + if validation_time > instance_effective_until { + use time::format_description::well_known::Rfc3339; + return Err(ManifestReuseError::CurrentInstanceVcirExpired { + manifest_rsync_uri: manifest_rsync_uri.to_string(), + effective_until_rfc3339_utc: instance_effective_until + .to_offset(time::UtcOffset::UTC) + .format(&Rfc3339) + .expect("format VCIR instance_effective_until"), + validation_time_rfc3339_utc: validation_time + .to_offset(time::UtcOffset::UTC) + .format(&Rfc3339) + .expect("format validation_time"), + }); + } + + let manifest_artifact = vcir + .related_artifacts + .iter() + .find(|artifact| { + artifact.artifact_role == VcirArtifactRole::Manifest + && artifact.uri.as_deref() == Some(manifest_rsync_uri) + }) + .ok_or_else(|| { + ManifestReuseError::InvalidCurrentInstanceVcir( + "missing manifest artifact matching manifest_rsync_uri".to_string(), + ) + })?; + + let manifest_bytes = store + .get_blob_bytes(&manifest_artifact.sha256)? + .ok_or_else(|| ManifestReuseError::MissingManifestRaw(manifest_artifact.sha256.clone()))?; + + let mut seen = HashSet::new(); + let mut files = Vec::new(); + for artifact in &vcir.related_artifacts { + let Some(uri) = artifact.uri.as_ref() else { + continue; + }; + if artifact.artifact_role == VcirArtifactRole::Manifest + || artifact.artifact_role == VcirArtifactRole::IssuerCert + || artifact.artifact_role == VcirArtifactRole::Tal + || artifact.artifact_role == VcirArtifactRole::TrustAnchorCert + { + continue; + } + if !seen.insert(uri.clone()) { + continue; + } + let entry_bytes = store.get_blob_bytes(&artifact.sha256)?.ok_or_else(|| { + ManifestReuseError::MissingArtifactRaw { + rsync_uri: uri.clone(), + } + })?; + files.push(PackFile::from_bytes_compute_sha256(uri, entry_bytes)); + } + + Ok(PublicationPointSnapshot { + format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, + manifest_rsync_uri: manifest_rsync_uri.to_string(), + publication_point_rsync_uri: publication_point_rsync_uri.to_string(), + manifest_number_be: vcir + .validated_manifest_meta + .validated_manifest_number + .clone(), + this_update: vcir + .validated_manifest_meta + .validated_manifest_this_update + .clone(), + next_update: vcir + .validated_manifest_meta + .validated_manifest_next_update + .clone(), + verified_at: vcir.last_successful_validation_time.clone(), + manifest_bytes, + files, + }) +} + +fn decode_and_validate_manifest_with_current_time( + manifest_bytes: &[u8], + validation_time: time::OffsetDateTime, +) -> Result { + let manifest = ManifestObject::decode_der(manifest_bytes)?; + manifest.validate_embedded_ee_cert()?; + manifest.signed_object.verify()?; + + let this_update = manifest + .manifest + .this_update + .to_offset(time::UtcOffset::UTC); + let next_update = manifest + .manifest + .next_update + .to_offset(time::UtcOffset::UTC); + let now = validation_time.to_offset(time::UtcOffset::UTC); + if now < this_update || now > next_update { + return Err(ManifestFreshError::StaleOrEarly { + this_update_rfc3339_utc: this_update + .format(&time::format_description::well_known::Rfc3339) + .expect("format thisUpdate"), + next_update_rfc3339_utc: next_update + .format(&time::format_description::well_known::Rfc3339) + .expect("format nextUpdate"), + validation_time_rfc3339_utc: now + .format(&time::format_description::well_known::Rfc3339) + .expect("format validation_time"), + }); + } + + Ok(manifest) +} + +pub(crate) fn try_build_fresh_publication_point( + store: &RocksStore, + manifest_rsync_uri: &str, + publication_point_rsync_uri: &str, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + validation_time: time::OffsetDateTime, +) -> Result { + try_build_fresh_publication_point_with_timing( + store, + manifest_rsync_uri, + publication_point_rsync_uri, + None, + issuer_ca_der, + issuer_ca_rsync_uri, + validation_time, + ) + .map(|(fresh, _timing)| fresh) +} + +pub(crate) fn try_build_fresh_publication_point_with_timing( + store: &RocksStore, + manifest_rsync_uri: &str, + publication_point_rsync_uri: &str, + current_repo_index: Option<&CurrentRepoIndexHandle>, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + validation_time: time::OffsetDateTime, +) -> Result< + ( + FreshValidatedPublicationPoint, + FreshPublicationPointTimingBreakdown, + ), + ManifestFreshError, +> { + let mut timing = FreshPublicationPointTimingBreakdown::default(); + let current_index_lock_started = std::time::Instant::now(); + let current_index_guard = current_repo_index.and_then(|handle| handle.read().ok()); + timing.current_index_lock_ms = current_index_lock_started.elapsed().as_millis() as u64; + + if !rsync_uri_is_under_publication_point(manifest_rsync_uri, publication_point_rsync_uri) { + return Err(ManifestFreshError::ManifestOutsidePublicationPoint { + manifest_rsync_uri: manifest_rsync_uri.to_string(), + publication_point_rsync_uri: publication_point_rsync_uri.to_string(), + }); + } + + let manifest_load_started = std::time::Instant::now(); + let manifest_bytes = if let Some(index) = current_index_guard.as_ref() { + let manifest_lookup_started = std::time::Instant::now(); + let current = index.get_by_uri(manifest_rsync_uri).ok_or_else(|| { + ManifestFreshError::MissingManifest { + manifest_rsync_uri: manifest_rsync_uri.to_string(), + } + })?; + timing.manifest_index_lookup_ms = manifest_lookup_started.elapsed().as_millis() as u64; + let manifest_blob_load_started = std::time::Instant::now(); + store + .get_blob_bytes(¤t.current_hash_hex) + .map_err(|e| ManifestFreshError::MissingManifest { + manifest_rsync_uri: format!("{manifest_rsync_uri} ({e})"), + })? + .ok_or_else(|| ManifestFreshError::MissingManifest { + manifest_rsync_uri: manifest_rsync_uri.to_string(), + }) + .inspect(|_| { + timing.manifest_blob_load_ms = + manifest_blob_load_started.elapsed().as_millis() as u64; + })? + } else { + let manifest_blob_load_started = std::time::Instant::now(); + store + .load_current_object_bytes_by_uri(manifest_rsync_uri) + .map_err(|e| ManifestFreshError::MissingManifest { + manifest_rsync_uri: format!("{manifest_rsync_uri} ({e})"), + })? + .ok_or_else(|| ManifestFreshError::MissingManifest { + manifest_rsync_uri: manifest_rsync_uri.to_string(), + }) + .inspect(|_| { + timing.manifest_blob_load_ms = + manifest_blob_load_started.elapsed().as_millis() as u64; + })? + }; + timing.manifest_load_ms = manifest_load_started.elapsed().as_millis() as u64; + + let manifest_decode_started = std::time::Instant::now(); + let manifest = + decode_and_validate_manifest_with_current_time(&manifest_bytes, validation_time)?; + timing.manifest_decode_ms = manifest_decode_started.elapsed().as_millis() as u64; + + let this_update = manifest + .manifest + .this_update + .to_offset(time::UtcOffset::UTC); + let next_update = manifest + .manifest + .next_update + .to_offset(time::UtcOffset::UTC); + let now = validation_time.to_offset(time::UtcOffset::UTC); + + // RFC 9286 §4.2.1: replay/rollback detection for manifestNumber and thisUpdate. + // + // Important nuance for revalidation across runs: + // - If the manifestNumber is equal to the previously validated manifestNumber *and* the + // manifest bytes are identical, then this is the same manifest being revalidated and MUST + // be accepted (otherwise, RPs would incorrectly treat stable repositories as "failed fetch" + // and fall back to the current-instance VCIR snapshot). + // - If manifestNumber is equal but the manifest bytes differ, treat this as invalid (a + // repository is not allowed to change the manifest while keeping the manifestNumber). + // - If manifestNumber is lower, treat as rollback and reject. + // - If manifestNumber is higher, require thisUpdate to be more recent than the previously + // validated thisUpdate. + let replay_guard_started = std::time::Instant::now(); + if let Some(old_meta) = store + .get_manifest_replay_meta(manifest_rsync_uri) + .ok() + .flatten() + { + timing.replay_meta_hit = true; + if old_meta.manifest_rsync_uri == manifest_rsync_uri { + let new_num = manifest.manifest.manifest_number.bytes_be.as_slice(); + let old_num = old_meta.manifest_number_be.as_slice(); + match cmp_minimal_be_unsigned(new_num, old_num) { + Ordering::Greater => { + let old_this_update = old_meta + .manifest_this_update + .parse() + .expect("manifest replay meta validation ensures thisUpdate parses"); + if this_update <= old_this_update { + use time::format_description::well_known::Rfc3339; + return Err(ManifestFreshError::ThisUpdateNotIncreasing { + old_rfc3339_utc: old_this_update + .to_offset(time::UtcOffset::UTC) + .format(&Rfc3339) + .expect("format old thisUpdate"), + new_rfc3339_utc: this_update + .format(&Rfc3339) + .expect("format new thisUpdate"), + }); + } + } + Ordering::Equal => { + let new_manifest_hash = sha2::Sha256::digest(&manifest_bytes); + if old_meta.manifest_sha256.as_slice() != new_manifest_hash.as_slice() { + return Err(ManifestFreshError::ManifestNumberNotIncreasing { + old_hex: hex::encode_upper(old_num), + new_hex: hex::encode_upper(new_num), + }); + } + } + Ordering::Less => { + return Err(ManifestFreshError::ManifestNumberNotIncreasing { + old_hex: hex::encode_upper(old_num), + new_hex: hex::encode_upper(new_num), + }); + } + } + } + } else { + timing.replay_meta_miss = true; + } + timing.replay_guard_ms = replay_guard_started.elapsed().as_millis() as u64; + + let manifest_entries_started = std::time::Instant::now(); + let entries = manifest + .manifest + .parse_files() + .map_err(ManifestDecodeError::Validate)?; + timing.manifest_entries_ms = manifest_entries_started.elapsed().as_millis() as u64; + timing.manifest_file_count = entries.len(); + let mut files = Vec::with_capacity(manifest.manifest.file_count()); + let pack_files_started = std::time::Instant::now(); + let external_raw_store = store + .external_raw_store_ref() + .cloned() + .map(std::sync::Arc::new); + let external_repo_bytes = store + .external_repo_bytes_ref() + .cloned() + .map(std::sync::Arc::new); + let mut pack_files_index_lookup_duration = std::time::Duration::ZERO; + let mut pack_files_blob_load_duration = std::time::Duration::ZERO; + for entry in &entries { + let rsync_uri = + join_rsync_dir_and_file(publication_point_rsync_uri, entry.file_name.as_str()); + let current_object = if let Some(index) = current_index_guard.as_ref() { + let index_lookup_started = std::time::Instant::now(); + let current = + index + .get_by_uri(&rsync_uri) + .ok_or_else(|| ManifestFreshError::MissingFile { + rsync_uri: rsync_uri.clone(), + })?; + pack_files_index_lookup_duration += index_lookup_started.elapsed(); + crate::storage::CurrentObjectWithHash { + current_hash_hex: current.current_hash_hex.clone(), + current_hash: current.current_hash, + bytes: Vec::new(), + } + } else { + let blob_load_started = std::time::Instant::now(); + store + .load_current_object_with_hash_by_uri(&rsync_uri) + .map_err(|_e| ManifestFreshError::MissingFile { + rsync_uri: rsync_uri.clone(), + })? + .ok_or_else(|| ManifestFreshError::MissingFile { + rsync_uri: rsync_uri.clone(), + }) + .inspect(|_| { + pack_files_blob_load_duration += blob_load_started.elapsed(); + })? + }; + + if current_object.current_hash != entry.hash_bytes { + return Err(ManifestFreshError::HashMismatch { rsync_uri }); + } + + if let (Some(_), Some(repo_bytes)) = + (current_index_guard.as_ref(), external_repo_bytes.as_ref()) + { + files.push(PackFile::from_lazy_repo_bytes( + rsync_uri, + current_object.current_hash_hex, + current_object.current_hash, + repo_bytes.clone(), + )); + } else if let (Some(_), Some(raw_store)) = + (current_index_guard.as_ref(), external_raw_store.as_ref()) + { + files.push(PackFile::from_lazy_external_raw_store( + rsync_uri, + current_object.current_hash_hex, + current_object.current_hash, + raw_store.clone(), + )); + } else { + let bytes = if current_object.bytes.is_empty() { + let blob_load_started = std::time::Instant::now(); + store + .get_blob_bytes(¤t_object.current_hash_hex) + .map_err(|_e| ManifestFreshError::MissingFile { + rsync_uri: rsync_uri.clone(), + })? + .ok_or_else(|| ManifestFreshError::MissingFile { + rsync_uri: rsync_uri.clone(), + }) + .inspect(|_| { + pack_files_blob_load_duration += blob_load_started.elapsed(); + })? + } else { + current_object.bytes + }; + files.push(PackFile::from_bytes_with_sha256( + rsync_uri, + bytes, + current_object.current_hash, + )); + } + } + timing.pack_files_index_lookup_ms = pack_files_index_lookup_duration.as_millis() as u64; + timing.pack_files_blob_load_ms = pack_files_blob_load_duration.as_millis() as u64; + timing.pack_files_ms = pack_files_started.elapsed().as_millis() as u64; + + // RFC 6488 §3: manifest (signed object) validity includes a valid EE cert path. + // We validate this after §6.4/§6.5 so the issuer CRL can be selected from the publication point. + let ee_path_validate_started = std::time::Instant::now(); + validate_manifest_embedded_ee_cert_path( + &manifest, + &files, + issuer_ca_der, + issuer_ca_rsync_uri, + validation_time, + )?; + timing.ee_path_validate_ms = ee_path_validate_started.elapsed().as_millis() as u64; + + Ok(( + FreshValidatedPublicationPoint { + manifest_rsync_uri: manifest_rsync_uri.to_string(), + publication_point_rsync_uri: publication_point_rsync_uri.to_string(), + manifest_number_be: manifest.manifest.manifest_number.bytes_be.clone(), + this_update: PackTime::from_utc_offset_datetime(this_update), + next_update: PackTime::from_utc_offset_datetime(next_update), + verified_at: PackTime::from_utc_offset_datetime(now), + manifest_bytes, + files, + }, + timing, + )) +} + +fn cmp_minimal_be_unsigned(a: &[u8], b: &[u8]) -> Ordering { + // Compare two minimal big-endian byte strings as unsigned integers. + // (Leading zeros are not expected; callers store minimal big-endian.) + a.len().cmp(&b.len()).then_with(|| a.cmp(b)) +} + +fn join_rsync_dir_and_file(base: &str, file_name: &str) -> String { + if base.ends_with('/') { + format!("{base}{file_name}") + } else { + format!("{base}/{file_name}") + } +} + +fn rsync_uri_is_under_publication_point(uri: &str, publication_point_rsync_uri: &str) -> bool { + let pp = if publication_point_rsync_uri.ends_with('/') { + publication_point_rsync_uri.to_string() + } else { + format!("{publication_point_rsync_uri}/") + }; + uri.starts_with(&pp) +} + +fn validate_manifest_embedded_ee_cert_path( + manifest: &ManifestObject, + files: &[crate::storage::PackFile], + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + validation_time: time::OffsetDateTime, +) -> Result<(), ManifestFreshError> { + let ee = &manifest.signed_object.signed_data.certificates[0]; + + let crl_files = files + .iter() + .filter(|f| f.rsync_uri.ends_with(".crl")) + .collect::>(); + if crl_files.is_empty() { + return Err(ManifestFreshError::NoCrlFiles); + } + + let Some(crldp_uris) = ee + .resource_cert + .tbs + .extensions + .crl_distribution_points_uris + .as_ref() + else { + return Err(ManifestFreshError::EeCrlDpMissing); + }; + + for u in crldp_uris { + let s = u.as_str(); + if let Some(f) = crl_files.iter().find(|f| f.rsync_uri == s) { + let crl_bytes = f.bytes().map_err(|e| ManifestFreshError::MissingFile { + rsync_uri: format!("{s} ({e})"), + })?; + let issuer_ca = crate::data_model::rc::ResourceCertificate::decode_der(issuer_ca_der) + .map_err(CertPathError::IssuerDecode)?; + let (rem, issuer_spki) = x509_parser::x509::SubjectPublicKeyInfo::from_der( + &issuer_ca.tbs.subject_public_key_info, + ) + .map_err(|e| CertPathError::IssuerSpkiParse(e.to_string()))?; + if !rem.is_empty() { + return Err(CertPathError::IssuerSpkiTrailingBytes(rem.len()).into()); + } + let issuer_crl = crate::data_model::crl::RpkixCrl::decode_der(crl_bytes) + .map_err(CertPathError::from)?; + let revoked_serials = issuer_crl + .revoked_certs + .iter() + .map(|rc| rc.serial_number.bytes_be.clone()) + .collect::>(); + validate_signed_object_ee_cert_path_fast( + ee, + &issuer_ca, + &issuer_spki, + &issuer_crl, + &revoked_serials, + issuer_ca_rsync_uri, + Some(f.rsync_uri.as_str()), + validation_time, + )?; + return Ok(()); + } + } + + Err(ManifestFreshError::EeCrlNotFound( + crldp_uris + .iter() + .map(|u| u.as_str()) + .collect::>() + .join(", "), + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::current_repo_index::CurrentRepoIndex; + use crate::data_model::manifest::ManifestObject; + use crate::storage::{ + PackFile, PackTime, RawByHashEntry, RocksStore, ValidatedCaInstanceResult, + ValidatedManifestMeta, VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus, + VcirAuditSummary, VcirCcrManifestProjection, VcirInstanceGate, VcirRelatedArtifact, + VcirSummary, + }; + use std::path::Path; + + fn manifest_fixture_path() -> &'static Path { + Path::new( + "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft", + ) + } + + fn issuer_ca_fixture_der() -> Vec { + std::fs::read( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ) + .expect("read issuer ca fixture") + } + + fn issuer_ca_rsync_uri() -> &'static str { + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer" + } + + fn fixture_to_rsync_uri(path: &Path) -> String { + let rel = path + .strip_prefix("tests/fixtures/repository") + .expect("path under fixture repository"); + let mut it = rel.components(); + let host = it + .next() + .expect("host component") + .as_os_str() + .to_string_lossy(); + let rest = it.as_path().to_string_lossy(); + format!("rsync://{host}/{rest}") + } + + fn fixture_dir_to_rsync_uri(dir: &Path) -> String { + let mut s = fixture_to_rsync_uri(dir); + if !s.ends_with('/') { + s.push('/'); + } + s + } + + fn load_manifest_fixture() -> ( + ManifestObject, + Vec, + String, + String, + time::OffsetDateTime, + ) { + let manifest_path = manifest_fixture_path(); + let manifest_bytes = std::fs::read(manifest_path).expect("read manifest fixture"); + let manifest = ManifestObject::decode_der(&manifest_bytes).expect("decode manifest"); + let manifest_rsync_uri = fixture_to_rsync_uri(manifest_path); + let publication_point_rsync_uri = fixture_dir_to_rsync_uri(manifest_path.parent().unwrap()); + let validation_time = manifest.manifest.this_update + time::Duration::seconds(1); + ( + manifest, + manifest_bytes, + manifest_rsync_uri, + publication_point_rsync_uri, + validation_time, + ) + } + + fn raw_by_hash_entry(uri: &str, bytes: Vec, object_type: &str) -> RawByHashEntry { + let mut entry = + RawByHashEntry::from_bytes(hex::encode(sha2::Sha256::digest(&bytes)), bytes); + entry.origin_uris.push(uri.to_string()); + entry.object_type = Some(object_type.to_string()); + entry.encoding = Some("der".to_string()); + entry + } + + fn put_current_object(store: &RocksStore, rsync_uri: &str, bytes: Vec, object_type: &str) { + let hash = hex::encode(sha2::Sha256::digest(&bytes)); + store + .put_raw_by_hash_entry(&raw_by_hash_entry(rsync_uri, bytes, object_type)) + .expect("put raw_by_hash entry"); + store + .put_repository_view_entry(&crate::storage::RepositoryViewEntry { + rsync_uri: rsync_uri.to_string(), + current_hash: Some(hash), + repository_source: Some("https://example.test/notification.xml".to_string()), + object_type: Some(object_type.to_string()), + state: crate::storage::RepositoryViewState::Present, + }) + .expect("put repository view entry"); + } + + fn put_complete_publication_point_current_objects( + store: &RocksStore, + manifest: &ManifestObject, + manifest_rsync_uri: &str, + manifest_bytes: Vec, + publication_point_rsync_uri: &str, + ) { + put_current_object(store, manifest_rsync_uri, manifest_bytes, "mft"); + for entry in manifest.manifest.parse_files().expect("parse files") { + let file_path = manifest_fixture_path() + .parent() + .unwrap() + .join(entry.file_name.as_str()); + let bytes = std::fs::read(&file_path).expect("read fixture file"); + let rsync_uri = format!("{publication_point_rsync_uri}{}", entry.file_name); + let object_type = rsync_uri.rsplit('.').next().unwrap_or("bin"); + put_current_object(store, &rsync_uri, bytes, object_type); + } + } + + fn put_raw_only(store: &RocksStore, rsync_uri: &str, bytes: Vec, object_type: &str) { + store + .put_raw_by_hash_entry(&raw_by_hash_entry(rsync_uri, bytes, object_type)) + .expect("put raw_by_hash entry"); + } + + fn sample_vcir_for_manifest_replay_meta( + manifest: &ManifestObject, + manifest_rsync_uri: &str, + publication_point_rsync_uri: &str, + manifest_bytes: &[u8], + validation_time: time::OffsetDateTime, + ) -> ValidatedCaInstanceResult { + let manifest_hash = hex::encode(sha2::Sha256::digest(manifest_bytes)); + let this_update = manifest + .manifest + .this_update + .to_offset(time::UtcOffset::UTC); + let mut vcir = sample_current_instance_vcir( + manifest_rsync_uri, + publication_point_rsync_uri, + &manifest_hash, + "rsync://example.test/repo/object.roa", + &hex::encode(sha2::Sha256::digest(b"object")), + validation_time, + true, + ); + vcir.validated_manifest_meta.validated_manifest_number = + manifest.manifest.manifest_number.bytes_be.clone(); + vcir.validated_manifest_meta.validated_manifest_this_update = + PackTime::from_utc_offset_datetime(this_update); + vcir.ccr_manifest_projection.manifest_number_be = + manifest.manifest.manifest_number.bytes_be.clone(); + vcir.ccr_manifest_projection.manifest_this_update = + PackTime::from_utc_offset_datetime(this_update); + vcir.ccr_manifest_projection.manifest_sha256 = + hex::decode(manifest_hash).expect("decode manifest hash"); + vcir.ccr_manifest_projection.manifest_size = manifest_bytes.len() as u64; + vcir + } + + fn sample_current_instance_vcir( + manifest_rsync_uri: &str, + publication_point_rsync_uri: &str, + manifest_sha256: &str, + locked_object_uri: &str, + locked_object_sha256: &str, + validation_time: time::OffsetDateTime, + failed_fetch_eligible: bool, + ) -> ValidatedCaInstanceResult { + let gate_time = + PackTime::from_utc_offset_datetime(validation_time + time::Duration::hours(1)); + let ccr_manifest_projection = VcirCcrManifestProjection { + manifest_rsync_uri: manifest_rsync_uri.to_string(), + manifest_sha256: hex::decode(manifest_sha256).expect("decode manifest sha256"), + manifest_size: 2048, + manifest_ee_aki: vec![0x11; 20], + manifest_number_be: vec![1], + manifest_this_update: PackTime::from_utc_offset_datetime(validation_time), + manifest_sia_locations_der: vec![vec![ + 0x30, 0x11, 0x06, 0x08, 0x2b, 0x06, 0x01, 0x05, 0x05, 0x07, 0x30, 0x05, 0x86, 0x05, + b'r', b's', b'y', b'n', b'c', + ]], + subordinate_skis: Vec::new(), + }; + ValidatedCaInstanceResult { + manifest_rsync_uri: manifest_rsync_uri.to_string(), + parent_manifest_rsync_uri: None, + tal_id: "test-tal".to_string(), + ca_subject_name: "CN=test".to_string(), + ca_ski: "00112233445566778899aabbccddeeff00112233".to_string(), + issuer_ski: "00112233445566778899aabbccddeeff00112233".to_string(), + last_successful_validation_time: PackTime::from_utc_offset_datetime(validation_time), + current_manifest_rsync_uri: manifest_rsync_uri.to_string(), + current_crl_rsync_uri: format!("{publication_point_rsync_uri}current.crl"), + validated_manifest_meta: ValidatedManifestMeta { + validated_manifest_number: vec![1], + validated_manifest_this_update: PackTime::from_utc_offset_datetime(validation_time), + validated_manifest_next_update: gate_time.clone(), + }, + ccr_manifest_projection, + instance_gate: VcirInstanceGate { + manifest_next_update: gate_time.clone(), + current_crl_next_update: gate_time.clone(), + self_ca_not_after: gate_time.clone(), + instance_effective_until: gate_time, + }, + child_entries: Vec::new(), + local_outputs: Vec::new(), + related_artifacts: vec![ + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::Manifest, + artifact_kind: VcirArtifactKind::Mft, + uri: Some(manifest_rsync_uri.to_string()), + sha256: manifest_sha256.to_string(), + object_type: Some("mft".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: VcirArtifactKind::Roa, + uri: Some(locked_object_uri.to_string()), + sha256: locked_object_sha256.to_string(), + object_type: Some("roa".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }, + ], + summary: VcirSummary { + local_vrp_count: 0, + local_aspa_count: 0, + local_router_key_count: 0, + child_count: 0, + accepted_object_count: 2, + rejected_object_count: 0, + }, + audit_summary: VcirAuditSummary { + failed_fetch_eligible, + last_failed_fetch_reason: None, + warning_count: 0, + audit_flags: Vec::new(), + }, + } + } + fn locked_files_for_manifest( + manifest: &ManifestObject, + publication_point_rsync_uri: &str, + ) -> Vec { + let manifest_path = manifest_fixture_path(); + manifest + .manifest + .parse_files() + .expect("parse files") + .into_iter() + .map(|entry| { + let file_path = manifest_path + .parent() + .unwrap() + .join(entry.file_name.as_str()); + let bytes = std::fs::read(&file_path).unwrap_or_else(|_| { + panic!("read fixture file referenced by manifest: {file_path:?}") + }); + PackFile::from_bytes_compute_sha256( + format!("{publication_point_rsync_uri}{}", entry.file_name), + bytes, + ) + }) + .collect() + } + + #[test] + fn try_build_fresh_publication_point_rejects_manifest_outside_publication_point() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(temp.path()).expect("open rocksdb"); + let (_, _, manifest_rsync_uri, _, validation_time) = load_manifest_fixture(); + let err = try_build_fresh_publication_point( + &store, + &manifest_rsync_uri, + "rsync://example.test/other/", + &issuer_ca_fixture_der(), + Some(issuer_ca_rsync_uri()), + validation_time, + ) + .unwrap_err(); + assert!( + matches!( + err, + ManifestFreshError::ManifestOutsidePublicationPoint { .. } + ), + "{err}" + ); + } + + #[test] + fn try_build_fresh_publication_point_reports_missing_manifest_when_raw_store_is_empty() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(temp.path()).expect("open rocksdb"); + let (_, _, manifest_rsync_uri, publication_point_rsync_uri, validation_time) = + load_manifest_fixture(); + let err = try_build_fresh_publication_point( + &store, + &manifest_rsync_uri, + &publication_point_rsync_uri, + &issuer_ca_fixture_der(), + Some(issuer_ca_rsync_uri()), + validation_time, + ) + .unwrap_err(); + assert!( + matches!(err, ManifestFreshError::MissingManifest { .. }), + "{err}" + ); + } + + #[test] + fn try_build_fresh_publication_point_reports_missing_locked_file() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(temp.path()).expect("open rocksdb"); + let ( + manifest, + manifest_bytes, + manifest_rsync_uri, + publication_point_rsync_uri, + validation_time, + ) = load_manifest_fixture(); + put_current_object(&store, &manifest_rsync_uri, manifest_bytes, "mft"); + let first_non_crl = manifest + .manifest + .parse_files() + .expect("parse files") + .into_iter() + .find(|entry| !entry.file_name.ends_with(".crl")) + .expect("fixture non-crl entry"); + let file_path = manifest_fixture_path() + .parent() + .unwrap() + .join(first_non_crl.file_name.as_str()); + let bytes = std::fs::read(&file_path).expect("read fixture file"); + let rsync_uri = format!("{publication_point_rsync_uri}{}", first_non_crl.file_name); + let object_type = rsync_uri.rsplit('.').next().unwrap_or("bin"); + put_current_object(&store, &rsync_uri, bytes, object_type); + + let err = try_build_fresh_publication_point( + &store, + &manifest_rsync_uri, + &publication_point_rsync_uri, + &issuer_ca_fixture_der(), + Some(issuer_ca_rsync_uri()), + validation_time, + ) + .unwrap_err(); + assert!( + matches!(err, ManifestFreshError::MissingFile { .. }), + "{err}" + ); + } + + #[test] + fn try_build_fresh_publication_point_detects_hash_mismatch_via_repository_view_hash() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(temp.path()).expect("open rocksdb"); + let ( + manifest, + manifest_bytes, + manifest_rsync_uri, + publication_point_rsync_uri, + validation_time, + ) = load_manifest_fixture(); + put_current_object(&store, &manifest_rsync_uri, manifest_bytes, "mft"); + + let non_crl_entries = manifest + .manifest + .parse_files() + .expect("parse files") + .into_iter() + .filter(|entry| !entry.file_name.ends_with(".crl")) + .collect::>(); + let first = &non_crl_entries[0]; + let second = &non_crl_entries[1]; + + let first_uri = format!("{publication_point_rsync_uri}{}", first.file_name); + let second_path = manifest_fixture_path() + .parent() + .unwrap() + .join(second.file_name.as_str()); + let wrong_bytes = std::fs::read(&second_path).expect("read wrong fixture file"); + let object_type = first_uri.rsplit('.').next().unwrap_or("bin"); + put_current_object(&store, &first_uri, wrong_bytes, object_type); + + for entry in non_crl_entries.iter().skip(1) { + let file_path = manifest_fixture_path() + .parent() + .unwrap() + .join(entry.file_name.as_str()); + let bytes = std::fs::read(&file_path).expect("read fixture file"); + let rsync_uri = format!("{publication_point_rsync_uri}{}", entry.file_name); + let object_type = rsync_uri.rsplit('.').next().unwrap_or("bin"); + put_current_object(&store, &rsync_uri, bytes, object_type); + } + + let err = try_build_fresh_publication_point( + &store, + &manifest_rsync_uri, + &publication_point_rsync_uri, + &issuer_ca_fixture_der(), + Some(issuer_ca_rsync_uri()), + validation_time, + ) + .unwrap_err(); + assert!( + matches!(err, ManifestFreshError::HashMismatch { .. }), + "{err}" + ); + } + + #[test] + fn try_build_fresh_publication_point_uses_current_repo_index_without_repository_view() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(temp.path()).expect("open rocksdb"); + let ( + manifest, + manifest_bytes, + manifest_rsync_uri, + publication_point_rsync_uri, + validation_time, + ) = load_manifest_fixture(); + + put_raw_only(&store, &manifest_rsync_uri, manifest_bytes.clone(), "mft"); + let current_index = CurrentRepoIndex::shared(); + let mut entries = vec![crate::storage::RepositoryViewEntry { + rsync_uri: manifest_rsync_uri.clone(), + current_hash: Some(hex::encode(sha2::Sha256::digest(&manifest_bytes))), + repository_source: Some("https://example.test/notification.xml".to_string()), + object_type: Some("mft".to_string()), + state: crate::storage::RepositoryViewState::Present, + }]; + + for entry in manifest.manifest.parse_files().expect("parse files") { + let file_path = manifest_fixture_path() + .parent() + .unwrap() + .join(entry.file_name.as_str()); + let bytes = std::fs::read(&file_path).expect("read fixture file"); + let rsync_uri = format!("{publication_point_rsync_uri}{}", entry.file_name); + let object_type = rsync_uri.rsplit('.').next().unwrap_or("bin").to_string(); + put_raw_only(&store, &rsync_uri, bytes.clone(), &object_type); + entries.push(crate::storage::RepositoryViewEntry { + rsync_uri, + current_hash: Some(hex::encode(sha2::Sha256::digest(&bytes))), + repository_source: Some("https://example.test/notification.xml".to_string()), + object_type: Some(object_type), + state: crate::storage::RepositoryViewState::Present, + }); + } + + current_index + .write() + .expect("index write lock") + .apply_repository_view_entries(&entries) + .expect("apply current index"); + + assert!( + store + .get_repository_view_entry(&manifest_rsync_uri) + .expect("get repository view") + .is_none() + ); + + let (fresh, _timing) = try_build_fresh_publication_point_with_timing( + &store, + &manifest_rsync_uri, + &publication_point_rsync_uri, + Some(¤t_index), + &issuer_ca_fixture_der(), + Some(issuer_ca_rsync_uri()), + validation_time, + ) + .expect("fresh publication point via current index"); + + assert_eq!(fresh.manifest_rsync_uri, manifest_rsync_uri); + assert_eq!(fresh.files.len(), manifest.manifest.file_count()); + } + + #[test] + fn try_build_fresh_publication_point_records_replay_meta_miss() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(temp.path()).expect("open rocksdb"); + let ( + manifest, + manifest_bytes, + manifest_rsync_uri, + publication_point_rsync_uri, + validation_time, + ) = load_manifest_fixture(); + put_complete_publication_point_current_objects( + &store, + &manifest, + &manifest_rsync_uri, + manifest_bytes, + &publication_point_rsync_uri, + ); + + let (_fresh, timing) = try_build_fresh_publication_point_with_timing( + &store, + &manifest_rsync_uri, + &publication_point_rsync_uri, + None, + &issuer_ca_fixture_der(), + Some(issuer_ca_rsync_uri()), + validation_time, + ) + .expect("fresh publication point without replay meta"); + + assert!(timing.replay_meta_miss); + assert!(!timing.replay_meta_hit); + } + + #[test] + fn try_build_fresh_publication_point_uses_replay_meta_hit_for_same_manifest() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(temp.path()).expect("open rocksdb"); + let ( + manifest, + manifest_bytes, + manifest_rsync_uri, + publication_point_rsync_uri, + validation_time, + ) = load_manifest_fixture(); + let previous_vcir = sample_vcir_for_manifest_replay_meta( + &manifest, + &manifest_rsync_uri, + &publication_point_rsync_uri, + &manifest_bytes, + validation_time, + ); + store.put_vcir(&previous_vcir).expect("put previous vcir"); + put_complete_publication_point_current_objects( + &store, + &manifest, + &manifest_rsync_uri, + manifest_bytes, + &publication_point_rsync_uri, + ); + + let (_fresh, timing) = try_build_fresh_publication_point_with_timing( + &store, + &manifest_rsync_uri, + &publication_point_rsync_uri, + None, + &issuer_ca_fixture_der(), + Some(issuer_ca_rsync_uri()), + validation_time, + ) + .expect("fresh publication point with matching replay meta"); + + assert!(timing.replay_meta_hit); + assert!(!timing.replay_meta_miss); + } + + #[test] + fn validate_manifest_embedded_ee_cert_path_rejects_missing_crl_files() { + let (manifest, _, _, publication_point_rsync_uri, validation_time) = + load_manifest_fixture(); + let files = locked_files_for_manifest(&manifest, &publication_point_rsync_uri) + .into_iter() + .filter(|f| !f.rsync_uri.ends_with(".crl")) + .collect::>(); + + let err = validate_manifest_embedded_ee_cert_path( + &manifest, + &files, + &issuer_ca_fixture_der(), + Some(issuer_ca_rsync_uri()), + validation_time, + ) + .unwrap_err(); + assert!(matches!(err, ManifestFreshError::NoCrlFiles), "{err}"); + } + + #[test] + fn validate_manifest_embedded_ee_cert_path_rejects_missing_ee_crldp() { + let (mut manifest, _, _, publication_point_rsync_uri, validation_time) = + load_manifest_fixture(); + manifest.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .extensions + .crl_distribution_points_uris = None; + let files = locked_files_for_manifest(&manifest, &publication_point_rsync_uri); + + let err = validate_manifest_embedded_ee_cert_path( + &manifest, + &files, + &issuer_ca_fixture_der(), + Some(issuer_ca_rsync_uri()), + validation_time, + ) + .unwrap_err(); + assert!(matches!(err, ManifestFreshError::EeCrlDpMissing), "{err}"); + } + + #[test] + fn validate_manifest_embedded_ee_cert_path_rejects_unlisted_crldp_uri() { + let (manifest, _, _, publication_point_rsync_uri, validation_time) = + load_manifest_fixture(); + let mut files = locked_files_for_manifest(&manifest, &publication_point_rsync_uri) + .into_iter() + .filter(|f| !f.rsync_uri.ends_with(".crl")) + .collect::>(); + files.push(PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/unrelated.crl", + b"dummy".to_vec(), + )); + + let err = validate_manifest_embedded_ee_cert_path( + &manifest, + &files, + &issuer_ca_fixture_der(), + Some(issuer_ca_rsync_uri()), + validation_time, + ) + .unwrap_err(); + assert!(matches!(err, ManifestFreshError::EeCrlNotFound(_)), "{err}"); + } + + #[test] + fn validate_manifest_embedded_ee_cert_path_rejects_expired_crl() { + let (manifest, _, _, publication_point_rsync_uri, _) = load_manifest_fixture(); + let files = locked_files_for_manifest(&manifest, &publication_point_rsync_uri); + let ee = &manifest.signed_object.signed_data.certificates[0]; + let crldp_uri = ee + .resource_cert + .tbs + .extensions + .crl_distribution_points_uris + .as_ref() + .and_then(|uris| uris.first()) + .expect("fixture manifest EE CRLDP") + .as_str() + .to_string(); + let crl_file = files + .iter() + .find(|file| file.rsync_uri == crldp_uri) + .expect("fixture CRL referenced by manifest EE"); + let crl = crate::data_model::crl::RpkixCrl::decode_der( + crl_file.bytes().expect("read fixture crl bytes"), + ) + .expect("decode fixture crl"); + let validation_time = crl.next_update.utc; + + let err = validate_manifest_embedded_ee_cert_path( + &manifest, + &files, + &issuer_ca_fixture_der(), + Some(issuer_ca_rsync_uri()), + validation_time, + ) + .unwrap_err(); + + assert!( + matches!( + err, + ManifestFreshError::EeCertPath( + crate::validation::cert_path::CertPathError::CrlNotValidAtTime + ) + ), + "{err}" + ); + } + + #[test] + fn load_current_instance_vcir_publication_point_returns_manifest_and_locked_files() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(temp.path()).expect("open rocksdb"); + let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::hours(1); + let manifest_uri = "rsync://example.test/repo/current.mft"; + let publication_point_uri = "rsync://example.test/repo/"; + let locked_uri = "rsync://example.test/repo/object.roa"; + let manifest_bytes = vec![0x30, 0x31, 0x32]; + let locked_bytes = vec![0x01, 0x02, 0x03, 0x04]; + let manifest_entry = raw_by_hash_entry(manifest_uri, manifest_bytes.clone(), "mft"); + let locked_entry = raw_by_hash_entry(locked_uri, locked_bytes.clone(), "roa"); + store + .put_raw_by_hash_entry(&manifest_entry) + .expect("put manifest raw_by_hash"); + store + .put_raw_by_hash_entry(&locked_entry) + .expect("put locked raw_by_hash"); + let vcir = sample_current_instance_vcir( + manifest_uri, + publication_point_uri, + &manifest_entry.sha256_hex, + locked_uri, + &locked_entry.sha256_hex, + validation_time, + true, + ); + store.put_vcir(&vcir).expect("put vcir"); + + let point = load_current_instance_vcir_publication_point( + &store, + manifest_uri, + publication_point_uri, + validation_time, + ) + .expect("load current-instance vcir publication point"); + assert_eq!(point.manifest_bytes, manifest_bytes); + assert_eq!(point.files.len(), 1); + assert_eq!(point.files[0].rsync_uri, locked_uri); + assert_eq!( + point.files[0].bytes_cloned().expect("locked bytes"), + locked_bytes + ); + } + + #[test] + fn load_current_instance_vcir_publication_point_rejects_ineligible_vcir() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(temp.path()).expect("open rocksdb"); + let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::hours(1); + let manifest_uri = "rsync://example.test/repo/current.mft"; + let publication_point_uri = "rsync://example.test/repo/"; + let locked_uri = "rsync://example.test/repo/object.roa"; + let manifest_entry = raw_by_hash_entry(manifest_uri, vec![0x30], "mft"); + let locked_entry = raw_by_hash_entry(locked_uri, vec![0x01], "roa"); + store + .put_raw_by_hash_entry(&manifest_entry) + .expect("put manifest raw_by_hash"); + store + .put_raw_by_hash_entry(&locked_entry) + .expect("put locked raw_by_hash"); + let vcir = sample_current_instance_vcir( + manifest_uri, + publication_point_uri, + &manifest_entry.sha256_hex, + locked_uri, + &locked_entry.sha256_hex, + validation_time, + false, + ); + store.put_vcir(&vcir).expect("put vcir"); + + let err = load_current_instance_vcir_publication_point( + &store, + manifest_uri, + publication_point_uri, + validation_time, + ) + .unwrap_err(); + assert!( + matches!(err, ManifestReuseError::IneligibleCurrentInstanceVcir(_)), + "{err}" + ); + } + + #[test] + fn load_current_instance_vcir_publication_point_rejects_expired_vcir() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(temp.path()).expect("open rocksdb"); + let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::hours(2); + let manifest_uri = "rsync://example.test/repo/current.mft"; + let publication_point_uri = "rsync://example.test/repo/"; + let locked_uri = "rsync://example.test/repo/object.roa"; + let manifest_entry = raw_by_hash_entry(manifest_uri, vec![0x30], "mft"); + let locked_entry = raw_by_hash_entry(locked_uri, vec![0x01], "roa"); + store + .put_raw_by_hash_entry(&manifest_entry) + .expect("put manifest raw_by_hash"); + store + .put_raw_by_hash_entry(&locked_entry) + .expect("put locked raw_by_hash"); + let mut vcir = sample_current_instance_vcir( + manifest_uri, + publication_point_uri, + &manifest_entry.sha256_hex, + locked_uri, + &locked_entry.sha256_hex, + validation_time - time::Duration::hours(2), + true, + ); + let expired = + PackTime::from_utc_offset_datetime(validation_time - time::Duration::minutes(1)); + vcir.instance_gate.manifest_next_update = expired.clone(); + vcir.instance_gate.current_crl_next_update = expired.clone(); + vcir.instance_gate.self_ca_not_after = expired.clone(); + vcir.instance_gate.instance_effective_until = expired; + vcir.validated_manifest_meta.validated_manifest_next_update = + vcir.instance_gate.instance_effective_until.clone(); + store.put_vcir(&vcir).expect("put expired vcir"); + + let err = load_current_instance_vcir_publication_point( + &store, + manifest_uri, + publication_point_uri, + validation_time, + ) + .unwrap_err(); + assert!( + matches!(err, ManifestReuseError::CurrentInstanceVcirExpired { .. }), + "{err}" + ); + } + + #[test] + fn load_current_instance_vcir_publication_point_rejects_missing_locked_artifact_raw() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = RocksStore::open(temp.path()).expect("open rocksdb"); + let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::hours(1); + let manifest_uri = "rsync://example.test/repo/current.mft"; + let publication_point_uri = "rsync://example.test/repo/"; + let locked_uri = "rsync://example.test/repo/object.roa"; + let manifest_entry = raw_by_hash_entry(manifest_uri, vec![0x30], "mft"); + store + .put_raw_by_hash_entry(&manifest_entry) + .expect("put manifest raw_by_hash"); + let vcir = sample_current_instance_vcir( + manifest_uri, + publication_point_uri, + &manifest_entry.sha256_hex, + locked_uri, + &hex::encode(sha2::Sha256::digest([0x01, 0x02])), + validation_time, + true, + ); + store.put_vcir(&vcir).expect("put vcir"); + + let err = load_current_instance_vcir_publication_point( + &store, + manifest_uri, + publication_point_uri, + validation_time, + ) + .unwrap_err(); + assert!( + matches!(err, ManifestReuseError::MissingArtifactRaw { .. }), + "{err}" + ); + } +} diff --git a/crates/panda-rpki-validator/src/validation/mod.rs b/crates/panda-rpki-validator/src/validation/mod.rs new file mode 100644 index 0000000..d406b5b --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/mod.rs @@ -0,0 +1,13 @@ +pub mod ca_instance; +pub mod ca_path; +pub mod cert_path; +pub mod from_tal; +pub mod manifest; +pub mod objects; +pub mod publication_point; +pub mod run; +pub mod run_tree_from_tal; +pub mod tree; +pub mod tree_parallel; +pub mod tree_runner; +pub mod x509_name; diff --git a/crates/panda-rpki-validator/src/validation/objects.rs b/crates/panda-rpki-validator/src/validation/objects.rs new file mode 100644 index 0000000..f7ef122 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/objects.rs @@ -0,0 +1,5708 @@ +use crate::analysis::timing::TimingHandle; +use crate::audit::{AuditObjectKind, AuditObjectResult, ObjectAuditEntry, sha256_hex_from_32}; +use crate::data_model::aspa::{AspaDecodeError, AspaObject, AspaValidateError}; +use crate::data_model::common::BigUnsigned; +use crate::data_model::manifest::ManifestObject; +use crate::data_model::rc::{ + AsIdentifierChoice, AsResourceSet, IpAddressChoice, IpAddressOrRange, IpPrefix as RcIpPrefix, + ResourceCertificate, +}; +use crate::data_model::roa::{IpPrefix, RoaAfi, RoaDecodeError, RoaObject, RoaValidateError}; +use crate::data_model::signed_object::{RpkiSignedObject, SignedObjectVerifyError}; +use crate::parallel::config::ParallelPhase2Config; +use crate::parallel::object_worker::{ + ObjectTaskExecutor, ObjectWorkerPool, ObjectWorkerSubmitError, +}; +use crate::policy::{Policy, ResourceValidationMode, SignedObjectFailurePolicy}; +use crate::report::{RfcRef, Warning, WarningCategory}; +use crate::storage::{ + PackFile, PackTime, RoaCacheObjectMeta, RoaCacheProjection, VcirLocalOutput, + VcirLocalOutputPayload, VcirOutputType, VcirSourceObjectType, +}; +use crate::validation::cert_path::{CertPathError, validate_signed_object_ee_cert_path_fast}; +use crate::validation::manifest::PublicationPointData; +use crate::validation::publication_point::PublicationPointSnapshot; +use sha2::Digest; +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; +use x509_parser::prelude::FromDer; +use x509_parser::x509::SubjectPublicKeyInfo; + +const RFC_NONE: &[RfcRef] = &[]; +const RFC_CRLDP: &[RfcRef] = &[RfcRef("RFC 6487 §4.8.6")]; +const RFC_CRLDP_AND_LOCKED_PACK: &[RfcRef] = + &[RfcRef("RFC 6487 §4.8.6"), RfcRef("RFC 9286 §4.2.1")]; + +fn ber_compatible_cms_warning(der: &[u8], rsync_uri: &str, object_kind: &str) -> Option { + let strict_error = RpkiSignedObject::strict_cms_der_error(der)?; + Some( + Warning::new(format!( + "accepted BER-compatible CMS encoding for {object_kind}: {rsync_uri}: {strict_error}" + )) + .with_category(WarningCategory::BerCompatibleCmsEncoding) + .with_rfc_refs(&[RfcRef("X.690 §10"), RfcRef("RFC 6488 §2")]) + .with_context(rsync_uri), + ) +} + +fn ber_compatible_cms_warning_for_file(file: &PackFile, object_kind: &str) -> Option { + ber_compatible_cms_warning(file.bytes().ok()?, &file.rsync_uri, object_kind) +} + +fn sha256_hex_to_32(hex_value: &str) -> [u8; 32] { + let bytes = hex::decode(hex_value).expect("internal sha256 hex should decode"); + let mut out = [0u8; 32]; + out.copy_from_slice(&bytes); + out +} + +fn sha256_hex(bytes: &[u8]) -> String { + hex::encode(sha2::Sha256::digest(bytes)) +} + +fn decode_resource_certificate_with_policy( + der: &[u8], + policy: &Policy, +) -> Result { + if policy.strict.name { + ResourceCertificate::decode_der_with_strict_name(der) + } else { + ResourceCertificate::decode_der(der) + } +} + +#[derive(Clone, Debug)] +pub(crate) struct VerifiedIssuerCrl { + crl: crate::data_model::crl::RpkixCrl, + revoked_serials: std::collections::HashSet>, + sha256_hex: String, +} + +#[derive(Clone, Debug)] +pub(crate) enum CachedIssuerCrl { + Pending { + bytes: Vec, + sha256_hex: Option, + }, + Ok(Arc), +} + +impl CachedIssuerCrl { + fn current_sha256_hex(&mut self) -> &str { + match self { + CachedIssuerCrl::Pending { bytes, sha256_hex } => { + if sha256_hex.is_none() { + *sha256_hex = Some(crate::audit::sha256_hex(bytes)); + } + sha256_hex + .as_deref() + .expect("pending CRL sha256 must be populated") + } + CachedIssuerCrl::Ok(verified) => verified.sha256_hex.as_str(), + } + } +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct IssuerResourcesIndex { + ip_v4: Option, Vec)>>, + ip_v6: Option, Vec)>>, + asnum: Option>, + rdi: Option>, +} + +fn extra_rfc_refs_for_crl_selection(e: &ObjectValidateError) -> &'static [RfcRef] { + match e { + ObjectValidateError::MissingCrlDpUris => RFC_CRLDP, + ObjectValidateError::CrlNotFound(_) => RFC_CRLDP_AND_LOCKED_PACK, + _ => RFC_NONE, + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Vrp { + pub asn: u32, + pub prefix: IpPrefix, + pub max_length: u16, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AspaAttestation { + pub customer_as_id: u32, + pub provider_as_ids: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RouterKeyPayload { + pub as_id: u32, + pub ski: Vec, + pub spki_der: Vec, + pub source_object_uri: String, + pub source_object_hash: String, + pub source_ee_cert_hash: String, + pub item_effective_until: PackTime, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ObjectsOutput { + pub vrps: Vec, + pub aspas: Vec, + pub router_keys: Vec, + pub local_outputs_cache: Vec, + pub warnings: Vec, + pub stats: ObjectsStats, + pub audit: Vec, + pub roa_cache_stats: RoaValidationCacheStats, + pub roa_cache_object_meta: Vec, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ObjectsStats { + pub roa_total: usize, + pub roa_ok: usize, + pub aspa_total: usize, + pub aspa_ok: usize, + /// Whether this publication point was dropped due to an unrecoverable objects-processing error + /// (e.g., missing issuer CRL in the pack, or `signed_object_failure_policy=drop_publication_point`). + pub publication_point_dropped: bool, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize)] +pub struct RoaValidationCacheStats { + pub enabled_publication_points: usize, + pub vcir_hit_publication_points: usize, + pub vcir_miss_publication_points: usize, + pub hit_roas: usize, + pub miss_roas: usize, + pub blocked_roas: usize, + pub fresh_roas: usize, + pub context_blocked_roas: usize, + pub crl_recheck_hit_roas: usize, + pub hash_blocked_roas: usize, + pub expired_blocked_roas: usize, + pub revoked_blocked_roas: usize, + pub metadata_blocked_roas: usize, + pub context_gate_nanos: u64, + pub lookup_nanos: u64, + pub lookup_entry_gate_nanos: u64, + pub lookup_crl_gate_nanos: u64, + pub lookup_materialize_nanos: u64, + pub lookup_crl_gate_verified_crls: usize, + pub lookup_crl_gate_reused_crls: usize, +} + +#[derive(Clone, Copy, Debug)] +pub struct RoaValidationCacheInput<'a> { + enabled: bool, + view: Option<&'a RoaValidationCacheView>, + ca_validation_context_digest: Option<[u8; 32]>, + policy_fingerprint: Option<[u8; 32]>, +} + +impl<'a> RoaValidationCacheInput<'a> { + pub fn disabled() -> Self { + Self { + enabled: false, + view: None, + ca_validation_context_digest: None, + policy_fingerprint: None, + } + } + + pub fn enabled(view: Option<&'a RoaValidationCacheView>) -> Self { + Self { + enabled: true, + view, + ca_validation_context_digest: None, + policy_fingerprint: None, + } + } + + pub fn enabled_with_context( + view: Option<&'a RoaValidationCacheView>, + ca_validation_context_digest: [u8; 32], + policy_fingerprint: [u8; 32], + ) -> Self { + Self { + enabled: true, + view, + ca_validation_context_digest: Some(ca_validation_context_digest), + policy_fingerprint: Some(policy_fingerprint), + } + } +} + +impl RoaValidationCacheStats { + pub fn add_assign(&mut self, other: &Self) { + self.enabled_publication_points += other.enabled_publication_points; + self.vcir_hit_publication_points += other.vcir_hit_publication_points; + self.vcir_miss_publication_points += other.vcir_miss_publication_points; + self.hit_roas += other.hit_roas; + self.miss_roas += other.miss_roas; + self.blocked_roas += other.blocked_roas; + self.fresh_roas += other.fresh_roas; + self.context_blocked_roas += other.context_blocked_roas; + self.crl_recheck_hit_roas += other.crl_recheck_hit_roas; + self.hash_blocked_roas += other.hash_blocked_roas; + self.expired_blocked_roas += other.expired_blocked_roas; + self.revoked_blocked_roas += other.revoked_blocked_roas; + self.metadata_blocked_roas += other.metadata_blocked_roas; + self.context_gate_nanos = self + .context_gate_nanos + .saturating_add(other.context_gate_nanos); + self.lookup_nanos = self.lookup_nanos.saturating_add(other.lookup_nanos); + self.lookup_entry_gate_nanos = self + .lookup_entry_gate_nanos + .saturating_add(other.lookup_entry_gate_nanos); + self.lookup_crl_gate_nanos = self + .lookup_crl_gate_nanos + .saturating_add(other.lookup_crl_gate_nanos); + self.lookup_materialize_nanos = self + .lookup_materialize_nanos + .saturating_add(other.lookup_materialize_nanos); + self.lookup_crl_gate_verified_crls += other.lookup_crl_gate_verified_crls; + self.lookup_crl_gate_reused_crls += other.lookup_crl_gate_reused_crls; + } + + fn for_input(input: RoaValidationCacheInput<'_>, roa_total: usize) -> Self { + let mut stats = Self::default(); + if !input.enabled { + return stats; + } + + stats.enabled_publication_points = 1; + if input.view.is_some() { + stats.vcir_hit_publication_points = 1; + } else { + stats.vcir_miss_publication_points = 1; + stats.miss_roas = roa_total; + stats.fresh_roas = roa_total; + } + stats + } + + fn record_lookup(&mut self, total_nanos: u64, metrics: RoaCacheLookupMetrics) { + self.lookup_nanos = self.lookup_nanos.saturating_add(total_nanos); + self.lookup_entry_gate_nanos = self + .lookup_entry_gate_nanos + .saturating_add(metrics.entry_gate_nanos); + self.lookup_crl_gate_nanos = self + .lookup_crl_gate_nanos + .saturating_add(metrics.crl_gate_nanos); + self.lookup_materialize_nanos = self + .lookup_materialize_nanos + .saturating_add(metrics.materialize_nanos); + self.lookup_crl_gate_verified_crls += metrics.crl_gate_verified_crls; + self.lookup_crl_gate_reused_crls += metrics.crl_gate_reused_crls; + } + + fn record_to_timing(&self, timing: Option<&TimingHandle>) { + let Some(timing) = timing else { + return; + }; + record_non_zero( + timing, + "roa_validation_cache_enabled_publication_points", + self.enabled_publication_points, + ); + record_non_zero( + timing, + "roa_validation_cache_vcir_hit_publication_points", + self.vcir_hit_publication_points, + ); + record_non_zero( + timing, + "roa_validation_cache_vcir_miss_publication_points", + self.vcir_miss_publication_points, + ); + record_non_zero(timing, "roa_validation_cache_hit_roas", self.hit_roas); + record_non_zero(timing, "roa_validation_cache_miss_roas", self.miss_roas); + record_non_zero( + timing, + "roa_validation_cache_blocked_roas", + self.blocked_roas, + ); + record_non_zero(timing, "roa_validation_cache_fresh_roas", self.fresh_roas); + record_non_zero( + timing, + "roa_validation_cache_context_blocked_roas", + self.context_blocked_roas, + ); + record_non_zero( + timing, + "roa_validation_cache_crl_recheck_hit_roas", + self.crl_recheck_hit_roas, + ); + record_non_zero( + timing, + "roa_validation_cache_hash_blocked_roas", + self.hash_blocked_roas, + ); + record_non_zero( + timing, + "roa_validation_cache_expired_blocked_roas", + self.expired_blocked_roas, + ); + record_non_zero( + timing, + "roa_validation_cache_revoked_blocked_roas", + self.revoked_blocked_roas, + ); + record_non_zero( + timing, + "roa_validation_cache_metadata_blocked_roas", + self.metadata_blocked_roas, + ); + record_non_zero( + timing, + "roa_validation_cache_context_gate_nanos", + self.context_gate_nanos as usize, + ); + record_non_zero( + timing, + "roa_validation_cache_lookup_nanos", + self.lookup_nanos as usize, + ); + record_non_zero( + timing, + "roa_validation_cache_lookup_entry_gate_nanos", + self.lookup_entry_gate_nanos as usize, + ); + record_non_zero( + timing, + "roa_validation_cache_lookup_crl_gate_nanos", + self.lookup_crl_gate_nanos as usize, + ); + record_non_zero( + timing, + "roa_validation_cache_lookup_materialize_nanos", + self.lookup_materialize_nanos as usize, + ); + record_non_zero( + timing, + "roa_validation_cache_lookup_crl_gate_verified_crls", + self.lookup_crl_gate_verified_crls, + ); + record_non_zero( + timing, + "roa_validation_cache_lookup_crl_gate_reused_crls", + self.lookup_crl_gate_reused_crls, + ); + timing.record_phase_nanos( + "roa_validation_cache_context_gate_total", + self.context_gate_nanos, + ); + timing.record_phase_nanos("roa_validation_cache_lookup_total", self.lookup_nanos); + timing.record_phase_nanos( + "roa_validation_cache_lookup_entry_gate_total", + self.lookup_entry_gate_nanos, + ); + timing.record_phase_nanos( + "roa_validation_cache_lookup_crl_gate_total", + self.lookup_crl_gate_nanos, + ); + timing.record_phase_nanos( + "roa_validation_cache_lookup_materialize_total", + self.lookup_materialize_nanos, + ); + } +} + +fn record_non_zero(timing: &TimingHandle, key: &'static str, value: usize) { + if value > 0 { + timing.record_count(key, value as u64); + } +} + +fn elapsed_nanos_u64(started: Instant) -> u64 { + started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64 +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct RoaCacheLookupMetrics { + entry_gate_nanos: u64, + crl_gate_nanos: u64, + materialize_nanos: u64, + crl_gate_verified_crls: usize, + crl_gate_reused_crls: usize, +} + +#[derive(Clone, Debug)] +enum RoaCacheCrlGate { + Unchanged, + ChangedValid(Arc), + Expired, + Invalid, + Missing, +} + +#[derive(Debug, Default)] +struct RoaCacheCrlGateSet { + gates_by_uri: HashMap, +} + +impl RoaCacheCrlGateSet { + fn evaluate( + &mut self, + crl_uri: &str, + expected_crl_sha256_by_uri: &HashMap, + crl_cache: &mut std::collections::HashMap, + issuer_ca_der: &[u8], + validation_time: time::OffsetDateTime, + metrics: &mut RoaCacheLookupMetrics, + ) -> RoaCacheCrlGate { + if let Some(gate) = self.gates_by_uri.get(crl_uri) { + metrics.crl_gate_reused_crls += 1; + return gate.clone(); + } + + let gate = evaluate_roa_cache_crl_gate( + crl_uri, + expected_crl_sha256_by_uri, + crl_cache, + issuer_ca_der, + validation_time, + metrics, + ); + self.gates_by_uri.insert(crl_uri.to_string(), gate.clone()); + gate + } +} + +fn evaluate_roa_cache_crl_gate( + crl_uri: &str, + expected_crl_sha256_by_uri: &HashMap, + crl_cache: &mut std::collections::HashMap, + issuer_ca_der: &[u8], + validation_time: time::OffsetDateTime, + metrics: &mut RoaCacheLookupMetrics, +) -> RoaCacheCrlGate { + let crl_unchanged = { + let Some(current_crl_hash) = crl_cache + .get_mut(crl_uri) + .map(CachedIssuerCrl::current_sha256_hex) + else { + return RoaCacheCrlGate::Missing; + }; + expected_crl_sha256_by_uri + .get(crl_uri) + .map(|expected| expected == current_crl_hash) + .unwrap_or(false) + }; + if crl_unchanged { + return RoaCacheCrlGate::Unchanged; + } + + let verified_crl = match ensure_issuer_crl_verified(crl_uri, crl_cache, issuer_ca_der) { + Ok(verified_crl) => { + metrics.crl_gate_verified_crls += 1; + verified_crl + } + Err(_) => return RoaCacheCrlGate::Invalid, + }; + if !crl_valid_at_time(&verified_crl.crl, validation_time) { + return RoaCacheCrlGate::Expired; + } + RoaCacheCrlGate::ChangedValid(verified_crl) +} + +#[derive(Clone, Debug)] +pub struct RoaValidationCacheView { + entries_by_uri: HashMap, + issuer_ca_sha256_hex: Option, + ca_validation_context_digest: Option<[u8; 32]>, + policy_fingerprint: Option<[u8; 32]>, + crl_sha256_by_uri: HashMap, + blocked: bool, +} + +#[derive(Clone, Debug)] +pub struct CachedRoaValidationResult { + source_object_hash: [u8; 32], + ee_serial: Option>, + crl_uri: Option, + earliest_safe_reuse_time_unix: i64, + outputs_effective_until_unix: i64, + outputs: Vec, +} + +impl RoaValidationCacheView { + pub fn from_projection( + projection: &RoaCacheProjection, + validation_time: time::OffsetDateTime, + ) -> Self { + let mut entries_by_uri: HashMap = + HashMap::with_capacity(projection.entries.len()); + let issuer_ca_sha256_hex = projection.issuer_ca_sha256_hex.clone(); + let ca_validation_context_digest = projection.ca_validation_context_digest; + let policy_fingerprint = projection.policy_fingerprint; + let crl_sha256_by_uri = projection + .crl_sha256_by_uri + .iter() + .map(|crl| (crl.uri.clone(), crl.sha256.clone())) + .collect::>(); + let blocked = projection + .instance_effective_until + .parse() + .map(|effective_until| effective_until <= validation_time) + .unwrap_or(true); + + if blocked { + return Self { + entries_by_uri, + issuer_ca_sha256_hex, + ca_validation_context_digest, + policy_fingerprint, + crl_sha256_by_uri, + blocked, + }; + } + + for entry in &projection.entries { + let Some(earliest_safe_reuse_time_unix) = entry.earliest_safe_reuse_time_unix else { + continue; + }; + if time::OffsetDateTime::from_unix_timestamp(earliest_safe_reuse_time_unix).is_err() { + continue; + } + let outputs = entry + .outputs + .iter() + .map(|output| VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: output.item_effective_until.clone(), + source_object_uri: entry.source_object_uri.clone(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: entry.source_object_hash, + source_ee_cert_hash: output.source_ee_cert_hash, + payload: output.payload.clone(), + rule_hash: output.rule_hash, + }) + .collect::>(); + entries_by_uri.insert( + entry.source_object_uri.clone(), + CachedRoaValidationResult { + source_object_hash: entry.source_object_hash, + ee_serial: entry.ee_serial.clone(), + crl_uri: entry.crl_uri.clone(), + earliest_safe_reuse_time_unix, + outputs_effective_until_unix: entry.outputs_effective_until_unix, + outputs, + }, + ); + } + + Self { + entries_by_uri, + issuer_ca_sha256_hex, + ca_validation_context_digest, + policy_fingerprint, + crl_sha256_by_uri, + blocked, + } + } + + fn matches_current_context( + &self, + issuer_ca_der: &[u8], + ca_validation_context_digest: Option<[u8; 32]>, + policy_fingerprint: Option<[u8; 32]>, + ) -> bool { + if self.blocked { + return false; + } + + let Some(expected_issuer_hash) = self.issuer_ca_sha256_hex.as_ref() else { + return false; + }; + if expected_issuer_hash != &sha256_hex(issuer_ca_der) { + return false; + } + + if let Some(expected_ca_validation_context) = self.ca_validation_context_digest { + if Some(expected_ca_validation_context) != ca_validation_context_digest { + return false; + } + } else { + return false; + } + + if let Some(expected_policy) = self.policy_fingerprint { + if Some(expected_policy) != policy_fingerprint { + return false; + } + } else { + return false; + } + + true + } + + fn lookup( + &self, + file: &PackFile, + crl_cache: &mut std::collections::HashMap, + issuer_ca_der: &[u8], + validation_time: time::OffsetDateTime, + ) -> RoaCacheLookupResult { + self.lookup_with_metrics(file, crl_cache, issuer_ca_der, validation_time, None) + .0 + } + + fn lookup_with_metrics( + &self, + file: &PackFile, + crl_cache: &mut std::collections::HashMap, + issuer_ca_der: &[u8], + validation_time: time::OffsetDateTime, + crl_gate_set: Option<&mut RoaCacheCrlGateSet>, + ) -> (RoaCacheLookupResult, RoaCacheLookupMetrics) { + let mut metrics = RoaCacheLookupMetrics::default(); + let entry_gate_started = Instant::now(); + macro_rules! return_entry_gate { + ($result:expr) => {{ + metrics.entry_gate_nanos = metrics + .entry_gate_nanos + .saturating_add(elapsed_nanos_u64(entry_gate_started)); + return ($result, metrics); + }}; + } + macro_rules! return_crl_gate { + ($started:expr, $result:expr) => {{ + metrics.crl_gate_nanos = metrics + .crl_gate_nanos + .saturating_add(elapsed_nanos_u64($started)); + return ($result, metrics); + }}; + } + macro_rules! return_materialize { + ($started:expr, $result:expr) => {{ + metrics.materialize_nanos = metrics + .materialize_nanos + .saturating_add(elapsed_nanos_u64($started)); + return ($result, metrics); + }}; + } + + if self.blocked { + return_entry_gate!(RoaCacheLookupResult::ExpiredBlocked); + } + + let Some(cached) = self.entries_by_uri.get(file.rsync_uri.as_str()) else { + return_entry_gate!(RoaCacheLookupResult::Miss); + }; + if cached.source_object_hash != file.sha256 { + return_entry_gate!(RoaCacheLookupResult::HashBlocked); + } + if cached.outputs.is_empty() { + return_entry_gate!(RoaCacheLookupResult::Miss); + } + if validation_time.unix_timestamp() < cached.earliest_safe_reuse_time_unix + || cached.outputs_effective_until_unix <= validation_time.unix_timestamp() + { + return_entry_gate!(RoaCacheLookupResult::ExpiredBlocked); + } + + let Some(crl_uri) = cached.crl_uri.as_deref() else { + return_entry_gate!(RoaCacheLookupResult::MetadataBlocked); + }; + let Some(ee_serial) = cached.ee_serial.as_ref() else { + return_entry_gate!(RoaCacheLookupResult::MetadataBlocked); + }; + metrics.entry_gate_nanos = metrics + .entry_gate_nanos + .saturating_add(elapsed_nanos_u64(entry_gate_started)); + + let crl_gate_started = Instant::now(); + let crl_gate = if let Some(crl_gate_set) = crl_gate_set { + crl_gate_set.evaluate( + crl_uri, + &self.crl_sha256_by_uri, + crl_cache, + issuer_ca_der, + validation_time, + &mut metrics, + ) + } else { + evaluate_roa_cache_crl_gate( + crl_uri, + &self.crl_sha256_by_uri, + crl_cache, + issuer_ca_der, + validation_time, + &mut metrics, + ) + }; + let crl_rechecked = match crl_gate { + RoaCacheCrlGate::Unchanged => false, + RoaCacheCrlGate::ChangedValid(verified_crl) => { + if verified_crl.revoked_serials.contains(ee_serial) { + return_crl_gate!(crl_gate_started, RoaCacheLookupResult::RevokedBlocked); + } + true + } + RoaCacheCrlGate::Expired => { + return_crl_gate!(crl_gate_started, RoaCacheLookupResult::ExpiredBlocked); + } + RoaCacheCrlGate::Invalid | RoaCacheCrlGate::Missing => { + return_crl_gate!(crl_gate_started, RoaCacheLookupResult::MetadataBlocked); + } + }; + metrics.crl_gate_nanos = metrics + .crl_gate_nanos + .saturating_add(elapsed_nanos_u64(crl_gate_started)); + + let materialize_started = Instant::now(); + let mut vrps = Vec::with_capacity(cached.outputs.len()); + for output in &cached.outputs { + let VcirLocalOutputPayload::Vrp { + asn, + afi, + prefix_len, + addr, + max_length, + } = &output.payload + else { + return_materialize!(materialize_started, RoaCacheLookupResult::MetadataBlocked); + }; + vrps.push(Vrp { + asn: *asn, + prefix: IpPrefix { + afi: *afi, + prefix_len: *prefix_len, + addr: *addr, + }, + max_length: *max_length, + }); + } + + let ok = RoaTaskOk { + vrps, + local_outputs: cached.outputs.clone(), + reused_from_cache: true, + cache_object_meta: Some(RoaCacheObjectMeta { + source_object_uri: file.rsync_uri.clone(), + source_object_hash: file.sha256, + ee_serial: ee_serial.clone(), + crl_uri: crl_uri.to_string(), + earliest_safe_reuse_time: PackTime::from_utc_offset_datetime( + time::OffsetDateTime::from_unix_timestamp(cached.earliest_safe_reuse_time_unix) + .expect("cached ROA safe reuse time must be valid"), + ), + }), + }; + metrics.materialize_nanos = metrics + .materialize_nanos + .saturating_add(elapsed_nanos_u64(materialize_started)); + if crl_rechecked { + (RoaCacheLookupResult::CrlRecheckHit(ok), metrics) + } else { + (RoaCacheLookupResult::Hit(ok), metrics) + } + } +} + +fn active_roa_cache_view<'a>( + roa_cache: RoaValidationCacheInput<'a>, + issuer_ca_der: &[u8], + stats: &mut RoaValidationCacheStats, + roa_total: usize, +) -> Option<&'a RoaValidationCacheView> { + let view = roa_cache.view?; + let gate_started = Instant::now(); + if view.matches_current_context( + issuer_ca_der, + roa_cache.ca_validation_context_digest, + roa_cache.policy_fingerprint, + ) { + stats.context_gate_nanos = stats + .context_gate_nanos + .saturating_add(gate_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); + Some(view) + } else { + stats.context_gate_nanos = stats + .context_gate_nanos + .saturating_add(gate_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); + stats.blocked_roas += roa_total; + stats.context_blocked_roas += roa_total; + stats.fresh_roas += roa_total; + None + } +} + +#[derive(Debug)] +enum RoaCacheLookupResult { + Hit(RoaTaskOk), + CrlRecheckHit(RoaTaskOk), + Miss, + HashBlocked, + ExpiredBlocked, + RevokedBlocked, + MetadataBlocked, +} + +fn crl_valid_at_time( + crl: &crate::data_model::crl::RpkixCrl, + validation_time: time::OffsetDateTime, +) -> bool { + let this_update = crl.this_update.utc.to_offset(time::UtcOffset::UTC); + let next_update = crl.next_update.utc.to_offset(time::UtcOffset::UTC); + validation_time >= this_update && validation_time < next_update +} + +fn roa_cache_earliest_safe_reuse_time( + ee_not_before: time::OffsetDateTime, + crl_this_update: time::OffsetDateTime, + validation_time: time::OffsetDateTime, +) -> PackTime { + PackTime::from_utc_offset_datetime(ee_not_before.max(crl_this_update).max(validation_time)) +} + +fn record_roa_cache_block( + stats: &mut RoaValidationCacheStats, + lookup_result: &RoaCacheLookupResult, +) { + stats.blocked_roas += 1; + stats.fresh_roas += 1; + match lookup_result { + RoaCacheLookupResult::HashBlocked => stats.hash_blocked_roas += 1, + RoaCacheLookupResult::ExpiredBlocked => stats.expired_blocked_roas += 1, + RoaCacheLookupResult::RevokedBlocked => stats.revoked_blocked_roas += 1, + RoaCacheLookupResult::MetadataBlocked => stats.metadata_blocked_roas += 1, + RoaCacheLookupResult::Hit(_) + | RoaCacheLookupResult::CrlRecheckHit(_) + | RoaCacheLookupResult::Miss => {} + } +} + +#[derive(Clone, Copy)] +pub(crate) struct RoaTask<'a> { + pub(crate) index: usize, + pub(crate) file: &'a PackFile, +} + +#[derive(Debug)] +pub(crate) struct RoaTaskOk { + pub(crate) vrps: Vec, + pub(crate) local_outputs: Vec, + pub(crate) reused_from_cache: bool, + pub(crate) cache_object_meta: Option, +} + +#[derive(Debug)] +pub(crate) struct RoaTaskResult { + pub(crate) publication_point_id: u64, + pub(crate) index: usize, + pub(crate) worker_index: usize, + pub(crate) queue_wait_ms: u64, + pub(crate) worker_ms: u64, + pub(crate) outcome: Result, +} + +/// Process objects from a publication point snapshot using a known issuer CA certificate +/// and its effective resources (resolved via the resource-path, RFC 6487 §7.2). +pub fn process_publication_point_for_issuer( + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, +) -> ObjectsOutput { + process_publication_point_for_issuer_with_options( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + true, + ) +} + +pub fn process_publication_point_for_issuer_with_options( + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + collect_vcir_local_outputs: bool, +) -> ObjectsOutput { + process_publication_point_for_issuer_with_cache_options( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + RoaValidationCacheInput::disabled(), + ) +} + +pub fn process_publication_point_for_issuer_with_cache_options( + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + collect_vcir_local_outputs: bool, + roa_cache: RoaValidationCacheInput<'_>, +) -> ObjectsOutput { + process_publication_point_for_issuer_with_cache_options_and_ta_constraints( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + roa_cache, + None, + ) +} + +/// Serial signed-object processing with an optional, locally configured +/// constraint set for the TA that owns this publication-point tree. +pub fn process_publication_point_for_issuer_with_cache_options_and_ta_constraints< + P: PublicationPointData, +>( + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + collect_vcir_local_outputs: bool, + roa_cache: RoaValidationCacheInput<'_>, + ta_constraints: Option<&crate::ta_constraints::TaConstraints>, +) -> ObjectsOutput { + let manifest_rsync_uri = publication_point.manifest_rsync_uri(); + let manifest_bytes = publication_point.manifest_bytes(); + let locked_files = publication_point.files(); + let mut warnings: Vec = Vec::new(); + let mut stats = ObjectsStats::default(); + stats.roa_total = locked_files + .iter() + .filter(|f| f.rsync_uri.ends_with(".roa")) + .count(); + stats.aspa_total = locked_files + .iter() + .filter(|f| f.rsync_uri.ends_with(".asa")) + .count(); + let mut roa_cache_stats = RoaValidationCacheStats::for_input(roa_cache, stats.roa_total); + let mut audit: Vec = Vec::new(); + + // Enforce that `manifest_bytes` is actually a manifest object. + let _manifest = match ManifestObject::decode_der_with_strict_options( + manifest_bytes, + policy.strict.cms_der, + policy.strict.name, + ) { + Ok(manifest) => manifest, + Err(e) => { + stats.publication_point_dropped = true; + warnings.push( + Warning::new(format!( + "dropping publication point: manifest decode failed: {e}" + )) + .with_rfc_refs(&[ + RfcRef("RFC 9286 §4"), + RfcRef("RFC 9286 §6.2"), + RfcRef("RFC 9286 §6.6"), + ]) + .with_context(manifest_rsync_uri), + ); + for f in locked_files { + if f.rsync_uri.ends_with(".roa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Skipped, + detail: Some("skipped: manifest decode failed".to_string()), + }); + } else if f.rsync_uri.ends_with(".asa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Skipped, + detail: Some("skipped: manifest decode failed".to_string()), + }); + } + } + return ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta: Vec::new(), + }; + } + }; + + if let Some(warning) = + ber_compatible_cms_warning(manifest_bytes, manifest_rsync_uri, "manifest") + { + warnings.push(warning); + } + + // Decode issuer CA once; if it fails we cannot validate ROA/ASPA EE certificates. + let issuer_ca = match decode_resource_certificate_with_policy(issuer_ca_der, policy) { + Ok(v) => v, + Err(e) => { + stats.publication_point_dropped = true; + warnings.push( + Warning::new(format!( + "dropping publication point: issuer CA decode failed: {e}" + )) + .with_rfc_refs(&[RfcRef("RFC 6487 §7.2"), RfcRef("RFC 5280 §6.1")]) + .with_context(manifest_rsync_uri), + ); + for f in locked_files { + if f.rsync_uri.ends_with(".roa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Skipped, + detail: Some("skipped: issuer CA decode failed".to_string()), + }); + } else if f.rsync_uri.ends_with(".asa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Skipped, + detail: Some("skipped: issuer CA decode failed".to_string()), + }); + } + } + return ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta: Vec::new(), + }; + } + }; + + // Parse issuer SubjectPublicKeyInfo once and reuse for all EE certificate signature checks. + let issuer_spki = match SubjectPublicKeyInfo::from_der(&issuer_ca.tbs.subject_public_key_info) { + Ok((rem, spki)) if rem.is_empty() => spki, + Ok((rem, _)) => { + stats.publication_point_dropped = true; + warnings.push( + Warning::new(format!( + "dropping publication point: trailing bytes after issuer SPKI DER: {} bytes", + rem.len() + )) + .with_rfc_refs(&[RfcRef("RFC 5280 §4.1.2.7")]) + .with_context(manifest_rsync_uri), + ); + return ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta: Vec::new(), + }; + } + Err(e) => { + stats.publication_point_dropped = true; + warnings.push( + Warning::new(format!( + "dropping publication point: issuer SPKI parse failed: {e}" + )) + .with_rfc_refs(&[RfcRef("RFC 5280 §4.1.2.7")]) + .with_context(manifest_rsync_uri), + ); + return ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta: Vec::new(), + }; + } + }; + + let mut crl_cache: std::collections::HashMap = locked_files + .iter() + .filter(|f| f.rsync_uri.ends_with(".crl")) + .map(|f| { + let bytes = f + .bytes_cloned() + .expect("snapshot CRL bytes must be loadable"); + ( + f.rsync_uri.clone(), + CachedIssuerCrl::Pending { + bytes, + sha256_hex: None, + }, + ) + }) + .collect(); + + let issuer_resources_index = + build_issuer_resources_index(issuer_effective_ip, issuer_effective_as); + + // If the snapshot has signed objects but no CRLs at all, we cannot validate any embedded EE + // certificate paths deterministically (EE CRLDP must reference an rsync URI in the snapshot). + if crl_cache.is_empty() && (stats.roa_total > 0 || stats.aspa_total > 0) { + stats.publication_point_dropped = true; + warnings.push( + Warning::new("dropping publication point: no CRL files in validated publication point") + .with_rfc_refs(&[RfcRef("RFC 6487 §4.8.6"), RfcRef("RFC 9286 §7")]) + .with_context(manifest_rsync_uri), + ); + for f in locked_files { + if f.rsync_uri.ends_with(".roa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Skipped, + detail: Some( + "skipped due to missing CRL files in validated publication point" + .to_string(), + ), + }); + } else if f.rsync_uri.ends_with(".asa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Skipped, + detail: Some( + "skipped due to missing CRL files in validated publication point" + .to_string(), + ), + }); + } + } + return ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta: Vec::new(), + }; + } + + let mut vrps: Vec = Vec::new(); + let mut aspas: Vec = Vec::new(); + let mut local_outputs_cache: Vec = Vec::new(); + let mut roa_cache_object_meta: Vec = Vec::new(); + let active_cache_view = active_roa_cache_view( + roa_cache, + issuer_ca_der, + &mut roa_cache_stats, + stats.roa_total, + ); + let mut crl_gate_set = if active_cache_view.is_some() { + Some(RoaCacheCrlGateSet::default()) + } else { + None + }; + + for (idx, file) in locked_files.iter().enumerate() { + if file.rsync_uri.ends_with(".roa") { + let result = if let Some(cache_view) = active_cache_view { + let lookup_started = Instant::now(); + let (lookup_result, lookup_metrics) = cache_view.lookup_with_metrics( + file, + &mut crl_cache, + issuer_ca_der, + validation_time, + crl_gate_set.as_mut(), + ); + roa_cache_stats.record_lookup(elapsed_nanos_u64(lookup_started), lookup_metrics); + match lookup_result { + RoaCacheLookupResult::Hit(ok) => { + roa_cache_stats.hit_roas += 1; + RoaTaskResult { + publication_point_id: 0, + index: idx, + worker_index: 0, + queue_wait_ms: 0, + worker_ms: 0, + outcome: Ok(ok), + } + } + RoaCacheLookupResult::CrlRecheckHit(ok) => { + roa_cache_stats.hit_roas += 1; + roa_cache_stats.crl_recheck_hit_roas += 1; + RoaTaskResult { + publication_point_id: 0, + index: idx, + worker_index: 0, + queue_wait_ms: 0, + worker_ms: 0, + outcome: Ok(ok), + } + } + RoaCacheLookupResult::Miss => { + roa_cache_stats.miss_roas += 1; + roa_cache_stats.fresh_roas += 1; + let task = RoaTask { index: idx, file }; + let _t = timing.as_ref().map(|t| t.span_phase("objects_roa_total")); + validate_roa_task_serial( + task, + manifest_rsync_uri, + issuer_ca_der, + &issuer_ca, + &issuer_spki, + issuer_ca_rsync_uri, + &mut crl_cache, + &issuer_resources_index, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + policy.strict.cms_der, + policy.strict.name, + policy.resource_validation_mode, + ta_constraints, + ) + } + blocked @ (RoaCacheLookupResult::HashBlocked + | RoaCacheLookupResult::ExpiredBlocked + | RoaCacheLookupResult::RevokedBlocked + | RoaCacheLookupResult::MetadataBlocked) => { + record_roa_cache_block(&mut roa_cache_stats, &blocked); + let task = RoaTask { index: idx, file }; + let _t = timing.as_ref().map(|t| t.span_phase("objects_roa_total")); + validate_roa_task_serial( + task, + manifest_rsync_uri, + issuer_ca_der, + &issuer_ca, + &issuer_spki, + issuer_ca_rsync_uri, + &mut crl_cache, + &issuer_resources_index, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + policy.strict.cms_der, + policy.strict.name, + policy.resource_validation_mode, + ta_constraints, + ) + } + } + } else { + let task = RoaTask { index: idx, file }; + let _t = timing.as_ref().map(|t| t.span_phase("objects_roa_total")); + validate_roa_task_serial( + task, + manifest_rsync_uri, + issuer_ca_der, + &issuer_ca, + &issuer_spki, + issuer_ca_rsync_uri, + &mut crl_cache, + &issuer_resources_index, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + policy.strict.cms_der, + policy.strict.name, + policy.resource_validation_mode, + ta_constraints, + ) + }; + match result.outcome { + Ok(mut ok) => { + stats.roa_ok += 1; + vrps.append(&mut ok.vrps); + if collect_vcir_local_outputs || ok.reused_from_cache { + local_outputs_cache.extend(ok.local_outputs); + } + if let Some(meta) = ok.cache_object_meta.take() { + roa_cache_object_meta.push(meta); + } + audit.push(ObjectAuditEntry { + rsync_uri: file.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&file.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Ok, + detail: None, + }); + if let Some(warning) = ber_compatible_cms_warning_for_file(file, "ROA") { + warnings.push(warning); + } + } + Err(e) => match policy.signed_object_failure_policy { + SignedObjectFailurePolicy::DropObject => { + audit.push(ObjectAuditEntry { + rsync_uri: file.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&file.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Error, + detail: Some(e.to_string()), + }); + let mut refs = vec![RfcRef("RFC 6488 §3"), RfcRef("RFC 9582 §4-§5")]; + refs.extend_from_slice(extra_rfc_refs_for_crl_selection(&e)); + warnings.push( + Warning::new(format!("dropping invalid ROA: {}: {e}", file.rsync_uri)) + .with_rfc_refs(&refs) + .with_context(&file.rsync_uri), + ) + } + SignedObjectFailurePolicy::DropPublicationPoint => { + stats.publication_point_dropped = true; + audit.push(ObjectAuditEntry { + rsync_uri: file.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&file.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Error, + detail: Some(e.to_string()), + }); + for f in locked_files.iter().skip(idx + 1) { + if f.rsync_uri.ends_with(".roa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Skipped, + detail: Some( + "skipped due to policy=signed_object_failure_policy=drop_publication_point" + .to_string(), + ), + }); + } else if f.rsync_uri.ends_with(".asa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Skipped, + detail: Some( + "skipped due to policy=signed_object_failure_policy=drop_publication_point" + .to_string(), + ), + }); + } + } + let mut refs = vec![RfcRef("RFC 6488 §3"), RfcRef("RFC 9582 §4-§5")]; + refs.extend_from_slice(extra_rfc_refs_for_crl_selection(&e)); + warnings.push( + Warning::new(format!( + "dropping publication point due to invalid ROA: {}: {e}", + file.rsync_uri + )) + .with_rfc_refs(&refs) + .with_context(manifest_rsync_uri), + ); + return ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta: Vec::new(), + }; + } + }, + } + } else if file.rsync_uri.ends_with(".asa") { + let _t = timing.as_ref().map(|t| t.span_phase("objects_aspa_total")); + match process_aspa_with_issuer( + file, + manifest_rsync_uri, + issuer_ca_der, + &issuer_ca, + &issuer_spki, + issuer_ca_rsync_uri, + &mut crl_cache, + &issuer_resources_index, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + policy.strict.cms_der, + policy.strict.name, + policy.resource_validation_mode, + ta_constraints, + ) { + Ok((att, local_output)) => { + stats.aspa_ok += 1; + aspas.push(att); + if let Some(local_output) = local_output { + local_outputs_cache.push(local_output); + } + audit.push(ObjectAuditEntry { + rsync_uri: file.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&file.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Ok, + detail: None, + }); + if let Some(warning) = ber_compatible_cms_warning_for_file(file, "ASPA") { + warnings.push(warning); + } + } + Err(e) => match policy.signed_object_failure_policy { + SignedObjectFailurePolicy::DropObject => { + audit.push(ObjectAuditEntry { + rsync_uri: file.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&file.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Error, + detail: Some(e.to_string()), + }); + let mut refs = vec![RfcRef("RFC 6488 §3")]; + refs.extend_from_slice(extra_rfc_refs_for_crl_selection(&e)); + warnings.push( + Warning::new(format!("dropping invalid ASPA: {}: {e}", file.rsync_uri)) + .with_rfc_refs(&refs) + .with_context(&file.rsync_uri), + ) + } + SignedObjectFailurePolicy::DropPublicationPoint => { + stats.publication_point_dropped = true; + audit.push(ObjectAuditEntry { + rsync_uri: file.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&file.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Error, + detail: Some(e.to_string()), + }); + for f in locked_files.iter().skip(idx + 1) { + if f.rsync_uri.ends_with(".roa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Skipped, + detail: Some( + "skipped due to policy=signed_object_failure_policy=drop_publication_point" + .to_string(), + ), + }); + } else if f.rsync_uri.ends_with(".asa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Skipped, + detail: Some( + "skipped due to policy=signed_object_failure_policy=drop_publication_point" + .to_string(), + ), + }); + } + } + let mut refs = vec![RfcRef("RFC 6488 §3")]; + refs.extend_from_slice(extra_rfc_refs_for_crl_selection(&e)); + warnings.push( + Warning::new(format!( + "dropping publication point due to invalid ASPA: {}: {e}", + file.rsync_uri + )) + .with_rfc_refs(&refs) + .with_context(manifest_rsync_uri), + ); + return ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta: Vec::new(), + }; + } + }, + } + } + } + + roa_cache_stats.record_to_timing(timing); + + ObjectsOutput { + vrps, + aspas, + router_keys: Vec::new(), + local_outputs_cache, + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta, + } +} + +pub fn process_publication_point_for_issuer_parallel_roa( + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + config: &ParallelPhase2Config, +) -> ObjectsOutput { + process_publication_point_for_issuer_parallel_roa_with_options( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + config, + true, + ) +} + +pub fn process_publication_point_for_issuer_parallel_roa_with_options( + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + config: &ParallelPhase2Config, + collect_vcir_local_outputs: bool, +) -> ObjectsOutput { + process_publication_point_for_issuer_parallel_roa_with_cache_options( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + config, + collect_vcir_local_outputs, + RoaValidationCacheInput::disabled(), + ) +} + +pub fn process_publication_point_for_issuer_parallel_roa_with_cache_options< + P: PublicationPointData, +>( + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + config: &ParallelPhase2Config, + collect_vcir_local_outputs: bool, + roa_cache: RoaValidationCacheInput<'_>, +) -> ObjectsOutput { + if config.object_workers <= 1 + || policy.signed_object_failure_policy == SignedObjectFailurePolicy::DropPublicationPoint + { + return process_publication_point_for_issuer_with_cache_options( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + roa_cache, + ); + } + + let pool = match ParallelRoaWorkerPool::new(config) { + Ok(pool) => pool, + Err(_) => { + return process_publication_point_for_issuer_with_cache_options( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + roa_cache, + ); + } + }; + + process_publication_point_for_issuer_parallel_roa_with_pool_cache_options( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + &pool, + collect_vcir_local_outputs, + roa_cache, + ) +} + +pub fn process_publication_point_for_issuer_parallel_roa_with_pool( + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + pool: &ParallelRoaWorkerPool, +) -> ObjectsOutput { + process_publication_point_for_issuer_parallel_roa_with_pool_options( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + pool, + true, + ) +} + +pub fn process_publication_point_for_issuer_parallel_roa_with_pool_options< + P: PublicationPointData, +>( + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + pool: &ParallelRoaWorkerPool, + collect_vcir_local_outputs: bool, +) -> ObjectsOutput { + process_publication_point_for_issuer_parallel_roa_with_pool_cache_options( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + pool, + collect_vcir_local_outputs, + RoaValidationCacheInput::disabled(), + ) +} + +pub fn process_publication_point_for_issuer_parallel_roa_with_pool_cache_options< + P: PublicationPointData, +>( + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + pool: &ParallelRoaWorkerPool, + collect_vcir_local_outputs: bool, + roa_cache: RoaValidationCacheInput<'_>, +) -> ObjectsOutput { + if policy.signed_object_failure_policy == SignedObjectFailurePolicy::DropPublicationPoint { + return process_publication_point_for_issuer_with_options( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + ); + } + + process_publication_point_for_issuer_parallel_roa_inner( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + pool, + collect_vcir_local_outputs, + roa_cache, + ) + .unwrap_or_else(|_| { + process_publication_point_for_issuer_with_cache_options( + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + roa_cache, + ) + }) +} + +#[derive(Clone)] +pub(crate) struct RoaTaskShared { + locked_files: Arc<[PackFile]>, + manifest_rsync_uri: Arc, + issuer_ca_der: Arc<[u8]>, + issuer_ca: Arc, + issuer_spki_der: Arc<[u8]>, + issuer_ca_rsync_uri: Option>, + crl_cache: Arc>>, + issuer_resources_index: Arc, + issuer_effective_ip: Option>, + issuer_effective_as: Option>, + resource_validation_mode: ResourceValidationMode, + /// Immutable constraints snapshot selected by the owning TAL. Every + /// ROA task for a publication point shares this Arc, so workers do not + /// need a mutable/global policy lookup or a copy of the interval rules. + ta_constraints: Option>, +} + +#[derive(Clone)] +pub(crate) struct OwnedRoaTask { + pub(crate) publication_point_id: u64, + index: usize, + shared: Arc, + validation_time: time::OffsetDateTime, + collect_vcir_local_outputs: bool, + strict_cms_der: bool, + strict_name: bool, + resource_validation_mode: ResourceValidationMode, + pub(crate) submitted_at: Option, +} + +#[derive(Clone)] +struct RoaTaskExecutor; + +impl ObjectTaskExecutor for RoaTaskExecutor { + fn execute(&self, worker_index: usize, task: OwnedRoaTask) -> RoaTaskResult { + validate_owned_roa_task(worker_index, task) + } +} + +pub struct ParallelRoaWorkerPool { + pool: Mutex>, +} + +impl ParallelRoaWorkerPool { + pub fn new(config: &ParallelPhase2Config) -> Result { + if config.object_workers <= 1 { + return Err("parallel ROA worker pool requires object_workers > 1".to_string()); + } + + Ok(Self { + pool: Mutex::new(ObjectWorkerPool::new( + config.object_workers, + config.worker_queue_capacity, + RoaTaskExecutor, + )?), + }) + } + + pub(crate) fn try_submit_round_robin( + &self, + task: OwnedRoaTask, + ) -> Result> { + self.pool + .lock() + .expect("parallel ROA worker pool lock") + .try_submit_round_robin(task) + } + + pub(crate) fn recv_result_timeout( + &self, + timeout: Duration, + ) -> Result, String> { + self.pool + .lock() + .expect("parallel ROA worker pool lock") + .recv_result_timeout(timeout) + } +} + +fn validate_owned_roa_task(worker_index: usize, task: OwnedRoaTask) -> RoaTaskResult { + let worker_started = Instant::now(); + let queue_wait_ms = task + .submitted_at + .map(|submitted_at| worker_started.saturating_duration_since(submitted_at)) + .map(|duration| duration.as_millis() as u64) + .unwrap_or(0); + let shared = task.shared.as_ref(); + let file = task + .shared + .locked_files + .get(task.index) + .expect("ROA task index must reference locked file"); + let issuer_spki = match SubjectPublicKeyInfo::from_der(shared.issuer_spki_der.as_ref()) { + Ok((rem, spki)) if rem.is_empty() => spki, + Ok((rem, _)) => { + return RoaTaskResult { + publication_point_id: task.publication_point_id, + index: task.index, + worker_index, + queue_wait_ms, + worker_ms: worker_started.elapsed().as_millis() as u64, + outcome: Err(ObjectValidateError::CertPath( + CertPathError::IssuerSpkiTrailingBytes(rem.len()), + )), + }; + } + Err(e) => { + return RoaTaskResult { + publication_point_id: task.publication_point_id, + index: task.index, + worker_index, + queue_wait_ms, + worker_ms: worker_started.elapsed().as_millis() as u64, + outcome: Err(ObjectValidateError::CertPath( + CertPathError::IssuerSpkiParse(e.to_string()), + )), + }; + } + }; + let outcome = process_roa_with_issuer_parallel_cached( + file, + shared.manifest_rsync_uri.as_ref(), + shared.issuer_ca_der.as_ref(), + shared.issuer_ca.as_ref(), + &issuer_spki, + shared.issuer_ca_rsync_uri.as_deref(), + shared.crl_cache.as_ref(), + shared.issuer_resources_index.as_ref(), + shared.issuer_effective_ip.as_deref(), + shared.issuer_effective_as.as_deref(), + task.validation_time, + None, + task.collect_vcir_local_outputs, + task.strict_cms_der, + task.strict_name, + task.resource_validation_mode, + shared.ta_constraints.as_deref(), + ) + .map(|(vrps, local_outputs, cache_object_meta)| RoaTaskOk { + vrps, + local_outputs, + reused_from_cache: false, + cache_object_meta, + }); + + RoaTaskResult { + publication_point_id: task.publication_point_id, + index: task.index, + worker_index, + queue_wait_ms, + worker_ms: worker_started.elapsed().as_millis() as u64, + outcome, + } +} + +pub(crate) enum ParallelObjectsPrepare { + Complete(ObjectsOutput), + Staged(ParallelObjectsStage), +} + +pub(crate) struct ParallelObjectsStage { + pub(crate) publication_point_id: u64, + shared: Arc, + validation_time: time::OffsetDateTime, + collect_vcir_local_outputs: bool, + strict_cms_der: bool, + strict_name: bool, + resource_validation_mode: ResourceValidationMode, + roa_task_indices: Vec, + cached_roa_results: Vec, + roa_cache_stats: RoaValidationCacheStats, + warnings: Vec, + stats: ObjectsStats, + audit: Vec, +} + +impl ParallelObjectsStage { + #[cfg(test)] + pub(crate) fn build_roa_tasks(&self) -> Vec { + let mut tasks = Vec::with_capacity(self.roa_task_count()); + self.extend_roa_tasks(|task| tasks.push(task)); + tasks + } + + pub(crate) fn append_roa_tasks_to( + &self, + pending: &mut std::collections::VecDeque, + ) { + self.extend_roa_tasks(|task| pending.push_back(task)); + } + + fn extend_roa_tasks(&self, mut push: F) + where + F: FnMut(OwnedRoaTask), + { + let shared = self.shared.clone(); + self.roa_task_indices.iter().for_each(|index| { + push(OwnedRoaTask { + publication_point_id: self.publication_point_id, + index: *index, + shared: shared.clone(), + validation_time: self.validation_time, + collect_vcir_local_outputs: self.collect_vcir_local_outputs, + strict_cms_der: self.strict_cms_der, + strict_name: self.strict_name, + resource_validation_mode: self.resource_validation_mode, + submitted_at: None, + }); + }); + } + + pub(crate) fn roa_task_count(&self) -> usize { + self.roa_task_indices.len() + } + + pub(crate) fn aspa_task_count(&self) -> usize { + self.stats.aspa_total + } + + pub(crate) fn locked_file_count(&self) -> usize { + self.shared.locked_files.len() + } +} + +pub(crate) fn prepare_publication_point_for_parallel_roa_with_cache( + publication_point_id: u64, + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + collect_vcir_local_outputs: bool, + roa_cache: RoaValidationCacheInput<'_>, +) -> ParallelObjectsPrepare { + prepare_publication_point_for_parallel_roa_with_cache_and_ta_constraints( + publication_point_id, + publication_point, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + collect_vcir_local_outputs, + roa_cache, + None, + ) +} + +/// Prepare a publication point for parallel ROA validation with the +/// immutable constraints snapshot belonging to its TAL. The snapshot is +/// moved into the stage-owned shared payload and therefore remains available +/// to detached ROA workers after the scoped phase-2 stage worker returns. +pub(crate) fn prepare_publication_point_for_parallel_roa_with_cache_and_ta_constraints< + P: PublicationPointData, +>( + publication_point_id: u64, + publication_point: &P, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + collect_vcir_local_outputs: bool, + roa_cache: RoaValidationCacheInput<'_>, + ta_constraints: Option>, +) -> ParallelObjectsPrepare { + let manifest_rsync_uri = publication_point.manifest_rsync_uri(); + let manifest_bytes = publication_point.manifest_bytes(); + let locked_files = publication_point.files(); + let mut warnings: Vec = Vec::new(); + let mut stats = ObjectsStats::default(); + stats.roa_total = locked_files + .iter() + .filter(|f| f.rsync_uri.ends_with(".roa")) + .count(); + stats.aspa_total = locked_files + .iter() + .filter(|f| f.rsync_uri.ends_with(".asa")) + .count(); + let mut roa_cache_stats = RoaValidationCacheStats::for_input(roa_cache, stats.roa_total); + let mut audit: Vec = Vec::new(); + + let _manifest = match ManifestObject::decode_der_with_strict_options( + manifest_bytes, + policy.strict.cms_der, + policy.strict.name, + ) { + Ok(manifest) => manifest, + Err(e) => { + stats.publication_point_dropped = true; + warnings.push( + Warning::new(format!( + "dropping publication point: manifest decode failed: {e}" + )) + .with_rfc_refs(&[ + RfcRef("RFC 9286 §4"), + RfcRef("RFC 9286 §6.2"), + RfcRef("RFC 9286 §6.6"), + ]) + .with_context(manifest_rsync_uri), + ); + for f in locked_files { + if f.rsync_uri.ends_with(".roa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Skipped, + detail: Some("skipped: manifest decode failed".to_string()), + }); + } else if f.rsync_uri.ends_with(".asa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Skipped, + detail: Some("skipped: manifest decode failed".to_string()), + }); + } + } + return ParallelObjectsPrepare::Complete(ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta: Vec::new(), + }); + } + }; + + if let Some(warning) = + ber_compatible_cms_warning(manifest_bytes, manifest_rsync_uri, "manifest") + { + warnings.push(warning); + } + + let issuer_ca = match decode_resource_certificate_with_policy(issuer_ca_der, policy) { + Ok(v) => v, + Err(e) => { + stats.publication_point_dropped = true; + warnings.push( + Warning::new(format!( + "dropping publication point: issuer CA decode failed: {e}" + )) + .with_rfc_refs(&[RfcRef("RFC 6487 §7.2"), RfcRef("RFC 5280 §6.1")]) + .with_context(manifest_rsync_uri), + ); + for f in locked_files { + if f.rsync_uri.ends_with(".roa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Skipped, + detail: Some("skipped: issuer CA decode failed".to_string()), + }); + } else if f.rsync_uri.ends_with(".asa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Skipped, + detail: Some("skipped: issuer CA decode failed".to_string()), + }); + } + } + return ParallelObjectsPrepare::Complete(ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta: Vec::new(), + }); + } + }; + + match SubjectPublicKeyInfo::from_der(&issuer_ca.tbs.subject_public_key_info) { + Ok((rem, _)) if rem.is_empty() => {} + Ok((rem, _)) => { + stats.publication_point_dropped = true; + warnings.push( + Warning::new(format!( + "dropping publication point: trailing bytes after issuer SPKI DER: {} bytes", + rem.len() + )) + .with_rfc_refs(&[RfcRef("RFC 5280 §4.1.2.7")]) + .with_context(manifest_rsync_uri), + ); + return ParallelObjectsPrepare::Complete(ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta: Vec::new(), + }); + } + Err(e) => { + stats.publication_point_dropped = true; + warnings.push( + Warning::new(format!( + "dropping publication point: issuer SPKI parse failed: {e}" + )) + .with_rfc_refs(&[RfcRef("RFC 5280 §4.1.2.7")]) + .with_context(manifest_rsync_uri), + ); + return ParallelObjectsPrepare::Complete(ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta: Vec::new(), + }); + } + } + + let mut crl_cache: std::collections::HashMap = locked_files + .iter() + .filter(|f| f.rsync_uri.ends_with(".crl")) + .map(|f| { + let bytes = f + .bytes_cloned() + .expect("snapshot CRL bytes must be loadable"); + ( + f.rsync_uri.clone(), + CachedIssuerCrl::Pending { + bytes, + sha256_hex: None, + }, + ) + }) + .collect(); + + if crl_cache.is_empty() && (stats.roa_total > 0 || stats.aspa_total > 0) { + stats.publication_point_dropped = true; + warnings.push( + Warning::new("dropping publication point: no CRL files in validated publication point") + .with_rfc_refs(&[RfcRef("RFC 6487 §4.8.6"), RfcRef("RFC 9286 §7")]) + .with_context(manifest_rsync_uri), + ); + for f in locked_files { + if f.rsync_uri.ends_with(".roa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Skipped, + detail: Some( + "skipped due to missing CRL files in validated publication point" + .to_string(), + ), + }); + } else if f.rsync_uri.ends_with(".asa") { + audit.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Skipped, + detail: Some( + "skipped due to missing CRL files in validated publication point" + .to_string(), + ), + }); + } + } + return ParallelObjectsPrepare::Complete(ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta: Vec::new(), + }); + } + + let active_cache_view = active_roa_cache_view( + roa_cache, + issuer_ca_der, + &mut roa_cache_stats, + stats.roa_total, + ); + let mut crl_gate_set = if active_cache_view.is_some() { + Some(RoaCacheCrlGateSet::default()) + } else { + None + }; + let mut roa_task_indices = Vec::new(); + let mut cached_roa_results = Vec::new(); + for (index, file) in locked_files.iter().enumerate() { + if !file.rsync_uri.ends_with(".roa") { + continue; + } + if let Some(cache_view) = active_cache_view { + let lookup_started = Instant::now(); + let (lookup_result, lookup_metrics) = cache_view.lookup_with_metrics( + file, + &mut crl_cache, + issuer_ca_der, + validation_time, + crl_gate_set.as_mut(), + ); + roa_cache_stats.record_lookup(elapsed_nanos_u64(lookup_started), lookup_metrics); + match lookup_result { + RoaCacheLookupResult::Hit(ok) => { + roa_cache_stats.hit_roas += 1; + cached_roa_results.push(RoaTaskResult { + publication_point_id, + index, + worker_index: usize::MAX, + queue_wait_ms: 0, + worker_ms: 0, + outcome: Ok(ok), + }); + } + RoaCacheLookupResult::CrlRecheckHit(ok) => { + roa_cache_stats.hit_roas += 1; + roa_cache_stats.crl_recheck_hit_roas += 1; + cached_roa_results.push(RoaTaskResult { + publication_point_id, + index, + worker_index: usize::MAX, + queue_wait_ms: 0, + worker_ms: 0, + outcome: Ok(ok), + }); + } + RoaCacheLookupResult::Miss => { + roa_cache_stats.miss_roas += 1; + roa_cache_stats.fresh_roas += 1; + roa_task_indices.push(index); + } + blocked @ (RoaCacheLookupResult::HashBlocked + | RoaCacheLookupResult::ExpiredBlocked + | RoaCacheLookupResult::RevokedBlocked + | RoaCacheLookupResult::MetadataBlocked) => { + record_roa_cache_block(&mut roa_cache_stats, &blocked); + roa_task_indices.push(index); + } + } + } else { + roa_task_indices.push(index); + } + } + + ParallelObjectsPrepare::Staged(ParallelObjectsStage { + publication_point_id, + shared: Arc::new(RoaTaskShared { + locked_files: Arc::<[PackFile]>::from(locked_files.to_vec()), + manifest_rsync_uri: Arc::::from(manifest_rsync_uri), + issuer_ca_der: Arc::<[u8]>::from(issuer_ca_der.to_vec()), + issuer_spki_der: Arc::<[u8]>::from(issuer_ca.tbs.subject_public_key_info.clone()), + issuer_ca: Arc::new(issuer_ca), + issuer_ca_rsync_uri: issuer_ca_rsync_uri.map(Arc::::from), + crl_cache: Arc::new(Mutex::new(crl_cache)), + issuer_resources_index: Arc::new(build_issuer_resources_index( + issuer_effective_ip, + issuer_effective_as, + )), + issuer_effective_ip: issuer_effective_ip.cloned().map(Arc::new), + issuer_effective_as: issuer_effective_as.cloned().map(Arc::new), + resource_validation_mode: policy.resource_validation_mode, + ta_constraints, + }), + validation_time, + collect_vcir_local_outputs, + strict_cms_der: policy.strict.cms_der, + strict_name: policy.strict.name, + resource_validation_mode: policy.resource_validation_mode, + roa_task_indices, + cached_roa_results, + roa_cache_stats, + warnings, + stats, + audit, + }) +} + +pub(crate) fn reduce_parallel_roa_stage( + stage: ParallelObjectsStage, + mut roa_results: Vec, + timing: Option<&TimingHandle>, +) -> Result { + roa_results.extend(stage.cached_roa_results); + roa_results.sort_by_key(|result| result.index); + let mut roa_results = roa_results.into_iter().peekable(); + let shared = stage.shared.clone(); + let mut aspa_crl_cache = shared + .crl_cache + .lock() + .expect("parallel ROA CRL cache lock") + .clone(); + let issuer_spki = SubjectPublicKeyInfo::from_der(shared.issuer_spki_der.as_ref()) + .map_err(|e| e.to_string())? + .1; + let collect_vcir_local_outputs = stage.collect_vcir_local_outputs; + let validation_time = stage.validation_time; + let strict_cms_der = stage.strict_cms_der; + let strict_name = stage.strict_name; + let roa_cache_stats = stage.roa_cache_stats; + let mut stats = stage.stats; + let mut warnings = stage.warnings; + let mut audit = stage.audit; + let mut vrps: Vec = Vec::new(); + let mut aspas: Vec = Vec::new(); + let mut local_outputs_cache: Vec = Vec::new(); + let mut roa_cache_object_meta: Vec = Vec::new(); + + for (idx, file) in shared.locked_files.iter().enumerate() { + if file.rsync_uri.ends_with(".roa") { + let result = match roa_results.peek() { + Some(result) if result.index == idx => roa_results + .next() + .expect("peeked ROA task result must be present"), + Some(result) => { + return Err(format!( + "unexpected ROA task result index {} while reducing {} at index {}", + result.index, file.rsync_uri, idx + )); + } + None => { + return Err(format!("missing ROA task result for {}", file.rsync_uri)); + } + }; + match result.outcome { + Ok(mut ok) => { + stats.roa_ok += 1; + vrps.append(&mut ok.vrps); + if collect_vcir_local_outputs || ok.reused_from_cache { + local_outputs_cache.extend(ok.local_outputs); + } + if let Some(meta) = ok.cache_object_meta.take() { + roa_cache_object_meta.push(meta); + } + audit.push(ObjectAuditEntry { + rsync_uri: file.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&file.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Ok, + detail: None, + }); + if let Some(warning) = ber_compatible_cms_warning_for_file(file, "ROA") { + warnings.push(warning); + } + } + Err(e) => { + audit.push(ObjectAuditEntry { + rsync_uri: file.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&file.sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Error, + detail: Some(e.to_string()), + }); + let mut refs = vec![RfcRef("RFC 6488 §3"), RfcRef("RFC 9582 §4-§5")]; + refs.extend_from_slice(extra_rfc_refs_for_crl_selection(&e)); + warnings.push( + Warning::new(format!("dropping invalid ROA: {}: {e}", file.rsync_uri)) + .with_rfc_refs(&refs) + .with_context(&file.rsync_uri), + ) + } + } + } else if file.rsync_uri.ends_with(".asa") { + let _t = timing.as_ref().map(|t| t.span_phase("objects_aspa_total")); + match process_aspa_with_issuer( + file, + shared.manifest_rsync_uri.as_ref(), + shared.issuer_ca_der.as_ref(), + shared.issuer_ca.as_ref(), + &issuer_spki, + shared.issuer_ca_rsync_uri.as_deref(), + &mut aspa_crl_cache, + shared.issuer_resources_index.as_ref(), + shared.issuer_effective_ip.as_deref(), + shared.issuer_effective_as.as_deref(), + validation_time, + timing, + collect_vcir_local_outputs, + strict_cms_der, + strict_name, + shared.resource_validation_mode, + shared.ta_constraints.as_deref(), + ) { + Ok((att, local_output)) => { + stats.aspa_ok += 1; + aspas.push(att); + if let Some(local_output) = local_output { + local_outputs_cache.push(local_output); + } + audit.push(ObjectAuditEntry { + rsync_uri: file.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&file.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Ok, + detail: None, + }); + if let Some(warning) = ber_compatible_cms_warning_for_file(file, "ASPA") { + warnings.push(warning); + } + } + Err(e) => { + audit.push(ObjectAuditEntry { + rsync_uri: file.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&file.sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Error, + detail: Some(e.to_string()), + }); + let mut refs = vec![RfcRef("RFC 6488 §3")]; + refs.extend_from_slice(extra_rfc_refs_for_crl_selection(&e)); + warnings.push( + Warning::new(format!("dropping invalid ASPA: {}: {e}", file.rsync_uri)) + .with_rfc_refs(&refs) + .with_context(&file.rsync_uri), + ) + } + } + } + } + if let Some(result) = roa_results.next() { + return Err(format!( + "unexpected trailing ROA task result at index {}", + result.index + )); + } + + roa_cache_stats.record_to_timing(timing); + + Ok(ObjectsOutput { + vrps, + aspas, + router_keys: Vec::new(), + local_outputs_cache, + warnings, + stats, + audit, + roa_cache_stats, + roa_cache_object_meta, + }) +} + +fn process_publication_point_for_issuer_parallel_roa_inner( + publication_point: &P, + _policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + pool: &ParallelRoaWorkerPool, + collect_vcir_local_outputs: bool, + roa_cache: RoaValidationCacheInput<'_>, +) -> Result { + let stage = match prepare_publication_point_for_parallel_roa_with_cache( + 0, + publication_point, + _policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + collect_vcir_local_outputs, + roa_cache, + ) { + ParallelObjectsPrepare::Complete(out) => return Ok(out), + ParallelObjectsPrepare::Staged(stage) => stage, + }; + + let roa_task_count = stage.roa_task_count(); + let mut pending = std::collections::VecDeque::with_capacity(roa_task_count); + stage.append_roa_tasks_to(&mut pending); + let mut worker_pool = pool + .pool + .lock() + .map_err(|_| "parallel ROA worker pool lock poisoned".to_string())?; + + while let Some(task) = pending.pop_front() { + match worker_pool.try_submit_round_robin(task) { + Ok(_) => {} + Err(ObjectWorkerSubmitError::QueueFull { task, .. }) => { + pending.push_front(task); + std::thread::yield_now(); + } + Err(ObjectWorkerSubmitError::Disconnected { .. }) => { + return Err("parallel ROA worker queue disconnected".to_string()); + } + } + } + + let mut roa_results = Vec::with_capacity(roa_task_count); + while roa_results.len() < roa_task_count { + let Some(result) = worker_pool.recv_result_timeout(Duration::from_secs(30))? else { + return Err("parallel ROA worker timed out".to_string()); + }; + roa_results.push(result); + } + drop(worker_pool); + + reduce_parallel_roa_stage(stage, roa_results, timing) +} +/// Compatibility wrapper that processes a publication point snapshot. +pub fn process_publication_point_snapshot_for_issuer( + pack: &PublicationPointSnapshot, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, +) -> ObjectsOutput { + process_publication_point_for_issuer( + pack, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + ) +} + +pub fn process_publication_point_snapshot_for_issuer_parallel_roa( + pack: &PublicationPointSnapshot, + policy: &Policy, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + config: &ParallelPhase2Config, +) -> ObjectsOutput { + process_publication_point_for_issuer_parallel_roa( + pack, + policy, + issuer_ca_der, + issuer_ca_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + config, + ) +} + +#[derive(Debug, thiserror::Error)] +pub(crate) enum ObjectValidateError { + #[error("object bytes load failed: {0}")] + BytesLoad(String), + + #[error("ROA decode failed: {0}")] + RoaDecode(#[from] RoaDecodeError), + + #[error("ROA embedded EE resource validation failed: {0}")] + RoaEeResources(#[from] RoaValidateError), + + #[error("ASPA decode failed: {0}")] + AspaDecode(#[from] AspaDecodeError), + + #[error("ASPA embedded EE resource validation failed: {0}")] + AspaEeResources(#[from] AspaValidateError), + + #[error("CMS signature verification failed: {0}")] + Signature(#[from] SignedObjectVerifyError), + + #[error("EE certificate path validation failed: {0}")] + CertPath(#[from] CertPathError), + + #[error( + "certificate CRLDistributionPoints URIs missing (cannot select issuer CRL) (RFC 6487 §4.8.6)" + )] + MissingCrlDpUris, + + #[error( + "no CRL available in publication point snapshot (cannot validate certificates) (RFC 9286 §7; RFC 6487 §4.8.6)" + )] + MissingCrlInPack, + + #[error( + "CRL referenced by CRLDistributionPoints not found in publication point snapshot: {0} (RFC 6487 §4.8.6; RFC 9286 §4.2.1)" + )] + CrlNotFound(String), + + #[error( + "issuer effective IP resources missing (cannot validate EE IP resources subset) (RFC 6487 §7.2; RFC 3779 §2.3)" + )] + MissingIssuerEffectiveIp, + + #[error( + "issuer effective AS resources missing (cannot validate EE AS resources subset) (RFC 6487 §7.2; RFC 3779 §3.3)" + )] + MissingIssuerEffectiveAs, + + #[error( + "EE certificate resources are not a subset of issuer effective resources (RFC 6487 §7.2; RFC 3779)" + )] + EeResourcesNotSubset, + + #[error("EE certificate violates locally configured TA constraints: {0}")] + TaConstraints(#[from] crate::ta_constraints::TaConstraintsViolation), +} + +pub(crate) fn validate_roa_task_serial( + task: RoaTask<'_>, + manifest_rsync_uri: &str, + issuer_ca_der: &[u8], + issuer_ca: &ResourceCertificate, + issuer_spki: &SubjectPublicKeyInfo<'_>, + issuer_ca_rsync_uri: Option<&str>, + crl_cache: &mut std::collections::HashMap, + issuer_resources_index: &IssuerResourcesIndex, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + collect_vcir_local_outputs: bool, + strict_cms_der: bool, + strict_name: bool, + resource_validation_mode: ResourceValidationMode, + ta_constraints: Option<&crate::ta_constraints::TaConstraints>, +) -> RoaTaskResult { + let outcome = process_roa_with_issuer( + task.file, + manifest_rsync_uri, + issuer_ca_der, + issuer_ca, + issuer_spki, + issuer_ca_rsync_uri, + crl_cache, + issuer_resources_index, + issuer_effective_ip, + issuer_effective_as, + validation_time, + timing, + collect_vcir_local_outputs, + strict_cms_der, + strict_name, + resource_validation_mode, + ta_constraints, + ) + .map(|(vrps, local_outputs, cache_object_meta)| RoaTaskOk { + vrps, + local_outputs, + reused_from_cache: false, + cache_object_meta, + }); + + RoaTaskResult { + publication_point_id: 0, + index: task.index, + worker_index: 0, + queue_wait_ms: 0, + worker_ms: 0, + outcome, + } +} + +fn process_roa_with_issuer( + file: &PackFile, + _manifest_rsync_uri: &str, + issuer_ca_der: &[u8], + issuer_ca: &ResourceCertificate, + issuer_spki: &SubjectPublicKeyInfo<'_>, + issuer_ca_rsync_uri: Option<&str>, + crl_cache: &mut std::collections::HashMap, + issuer_resources_index: &IssuerResourcesIndex, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + collect_vcir_local_outputs: bool, + strict_cms_der: bool, + strict_name: bool, + resource_validation_mode: ResourceValidationMode, + ta_constraints: Option<&crate::ta_constraints::TaConstraints>, +) -> Result<(Vec, Vec, Option), ObjectValidateError> { + let _decode = timing + .as_ref() + .map(|t| t.span_phase("objects_roa_decode_and_validate_total")); + let roa = RoaObject::decode_der_with_strict_options( + file.bytes().map_err(ObjectValidateError::BytesLoad)?, + strict_cms_der, + strict_name, + )?; + drop(_decode); + + let _ee_profile = timing + .as_ref() + .map(|t| t.span_phase("objects_roa_validate_embedded_ee_total")); + roa.validate_embedded_ee_cert()?; + drop(_ee_profile); + + let _verify = timing + .as_ref() + .map(|t| t.span_phase("objects_roa_verify_signature_total")); + roa.signed_object.verify()?; + drop(_verify); + + let ee = &roa.signed_object.signed_data.certificates[0]; + let ee_crldp_uris = ee + .resource_cert + .tbs + .extensions + .crl_distribution_points_uris + .as_ref(); + let issuer_crl_rsync_uri = choose_crl_uri_for_certificate(ee_crldp_uris, crl_cache)?; + let verified_crl = ensure_issuer_crl_verified(issuer_crl_rsync_uri, crl_cache, issuer_ca_der)?; + + let _cert_path = timing + .as_ref() + .map(|t| t.span_phase("objects_roa_validate_ee_cert_path_total")); + validate_signed_object_ee_cert_path_fast( + ee, + issuer_ca, + issuer_spki, + &verified_crl.crl, + &verified_crl.revoked_serials, + issuer_ca_rsync_uri, + Some(issuer_crl_rsync_uri), + validation_time, + )?; + drop(_cert_path); + + let _subset = timing + .as_ref() + .map(|t| t.span_phase("objects_roa_validate_ee_resources_subset_total")); + let ee_vrs = validate_ee_resources_for_mode( + &ee.resource_cert, + issuer_effective_ip, + issuer_effective_as, + issuer_resources_index, + resource_validation_mode, + )?; + drop(_subset); + + if let Some(ta_constraints) = ta_constraints { + ta_constraints.validate_ee_certificate(&ee.resource_cert)?; + } + + let vrps = roa_to_vrps_with_vrs(&roa, ee_vrs.ip.as_ref())?; + let cache_object_meta = RoaCacheObjectMeta { + source_object_uri: file.rsync_uri.clone(), + source_object_hash: file.sha256, + ee_serial: BigUnsigned::from_biguint(&ee.resource_cert.tbs.serial_number).bytes_be, + crl_uri: issuer_crl_rsync_uri.to_string(), + earliest_safe_reuse_time: roa_cache_earliest_safe_reuse_time( + ee.resource_cert.tbs.validity_not_before, + verified_crl.crl.this_update.utc, + validation_time, + ), + }; + if !collect_vcir_local_outputs { + return Ok((vrps, Vec::new(), Some(cache_object_meta))); + } + let source_object_hash = sha256_hex_from_32(&file.sha256); + let source_ee_cert_hash = crate::audit::sha256_hex(ee.raw_der.as_slice()); + let item_effective_until = + PackTime::from_utc_offset_datetime(ee.resource_cert.tbs.validity_not_after); + let local_outputs = vrps + .iter() + .map(|vrp| { + let prefix = vrp_prefix_to_string(vrp); + let rule_hash = crate::audit::sha256_hex( + format!( + "roa-rule:{}:{}:{}:{}", + source_object_hash, vrp.asn, prefix, vrp.max_length + ) + .as_bytes(), + ); + VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: item_effective_until.clone(), + source_object_uri: file.rsync_uri.clone(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: file.sha256, + source_ee_cert_hash: sha256_hex_to_32(&source_ee_cert_hash), + payload: VcirLocalOutputPayload::Vrp { + asn: vrp.asn, + afi: vrp.prefix.afi, + prefix_len: vrp.prefix.prefix_len, + addr: vrp.prefix.addr, + max_length: vrp.max_length, + }, + rule_hash: sha256_hex_to_32(&rule_hash), + } + }) + .collect(); + + Ok((vrps, local_outputs, Some(cache_object_meta))) +} + +fn process_roa_with_issuer_parallel_cached( + file: &PackFile, + _manifest_rsync_uri: &str, + issuer_ca_der: &[u8], + issuer_ca: &ResourceCertificate, + issuer_spki: &SubjectPublicKeyInfo<'_>, + issuer_ca_rsync_uri: Option<&str>, + crl_cache: &Mutex>, + issuer_resources_index: &IssuerResourcesIndex, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + collect_vcir_local_outputs: bool, + strict_cms_der: bool, + strict_name: bool, + resource_validation_mode: ResourceValidationMode, + ta_constraints: Option<&crate::ta_constraints::TaConstraints>, +) -> Result<(Vec, Vec, Option), ObjectValidateError> { + let _decode = timing + .as_ref() + .map(|t| t.span_phase("objects_roa_decode_and_validate_total")); + let roa = RoaObject::decode_der_with_strict_options( + file.bytes().map_err(ObjectValidateError::BytesLoad)?, + strict_cms_der, + strict_name, + )?; + drop(_decode); + + let _ee_profile = timing + .as_ref() + .map(|t| t.span_phase("objects_roa_validate_embedded_ee_total")); + roa.validate_embedded_ee_cert()?; + drop(_ee_profile); + + let _verify = timing + .as_ref() + .map(|t| t.span_phase("objects_roa_verify_signature_total")); + roa.signed_object.verify()?; + drop(_verify); + + let ee = &roa.signed_object.signed_data.certificates[0]; + let ee_crldp_uris = ee + .resource_cert + .tbs + .extensions + .crl_distribution_points_uris + .as_ref(); + let (issuer_crl_rsync_uri, verified_crl) = { + let mut crl_cache = crl_cache.lock().expect("parallel ROA CRL cache lock"); + let issuer_crl_rsync_uri = + choose_crl_uri_for_certificate(ee_crldp_uris, &crl_cache)?.to_string(); + let verified_crl = + ensure_issuer_crl_verified(&issuer_crl_rsync_uri, &mut crl_cache, issuer_ca_der)?; + (issuer_crl_rsync_uri, verified_crl) + }; + + let _cert_path = timing + .as_ref() + .map(|t| t.span_phase("objects_roa_validate_ee_cert_path_total")); + validate_signed_object_ee_cert_path_fast( + ee, + issuer_ca, + issuer_spki, + &verified_crl.crl, + &verified_crl.revoked_serials, + issuer_ca_rsync_uri, + Some(issuer_crl_rsync_uri.as_str()), + validation_time, + )?; + drop(_cert_path); + + let _subset = timing + .as_ref() + .map(|t| t.span_phase("objects_roa_validate_ee_resources_subset_total")); + let ee_vrs = validate_ee_resources_for_mode( + &ee.resource_cert, + issuer_effective_ip, + issuer_effective_as, + issuer_resources_index, + resource_validation_mode, + )?; + drop(_subset); + + if let Some(ta_constraints) = ta_constraints { + ta_constraints.validate_ee_certificate(&ee.resource_cert)?; + } + + let vrps = roa_to_vrps_with_vrs(&roa, ee_vrs.ip.as_ref())?; + let cache_object_meta = RoaCacheObjectMeta { + source_object_uri: file.rsync_uri.clone(), + source_object_hash: file.sha256, + ee_serial: BigUnsigned::from_biguint(&ee.resource_cert.tbs.serial_number).bytes_be, + crl_uri: issuer_crl_rsync_uri.clone(), + earliest_safe_reuse_time: roa_cache_earliest_safe_reuse_time( + ee.resource_cert.tbs.validity_not_before, + verified_crl.crl.this_update.utc, + validation_time, + ), + }; + if !collect_vcir_local_outputs { + return Ok((vrps, Vec::new(), Some(cache_object_meta))); + } + let source_object_hash = sha256_hex_from_32(&file.sha256); + let source_ee_cert_hash = crate::audit::sha256_hex(ee.raw_der.as_slice()); + let item_effective_until = + PackTime::from_utc_offset_datetime(ee.resource_cert.tbs.validity_not_after); + let local_outputs = vrps + .iter() + .map(|vrp| { + let prefix = vrp_prefix_to_string(vrp); + let rule_hash = crate::audit::sha256_hex( + format!( + "roa-rule:{}:{}:{}:{}", + source_object_hash, vrp.asn, prefix, vrp.max_length + ) + .as_bytes(), + ); + VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: item_effective_until.clone(), + source_object_uri: file.rsync_uri.clone(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: file.sha256, + source_ee_cert_hash: sha256_hex_to_32(&source_ee_cert_hash), + payload: VcirLocalOutputPayload::Vrp { + asn: vrp.asn, + afi: vrp.prefix.afi, + prefix_len: vrp.prefix.prefix_len, + addr: vrp.prefix.addr, + max_length: vrp.max_length, + }, + rule_hash: sha256_hex_to_32(&rule_hash), + } + }) + .collect(); + + Ok((vrps, local_outputs, Some(cache_object_meta))) +} + +fn process_aspa_with_issuer( + file: &PackFile, + _manifest_rsync_uri: &str, + issuer_ca_der: &[u8], + issuer_ca: &ResourceCertificate, + issuer_spki: &SubjectPublicKeyInfo<'_>, + issuer_ca_rsync_uri: Option<&str>, + crl_cache: &mut std::collections::HashMap, + issuer_resources_index: &IssuerResourcesIndex, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + collect_vcir_local_outputs: bool, + strict_cms_der: bool, + strict_name: bool, + resource_validation_mode: ResourceValidationMode, + ta_constraints: Option<&crate::ta_constraints::TaConstraints>, +) -> Result<(AspaAttestation, Option), ObjectValidateError> { + let _decode = timing + .as_ref() + .map(|t| t.span_phase("objects_aspa_decode_and_validate_total")); + let aspa = AspaObject::decode_der_with_strict_options( + file.bytes().map_err(ObjectValidateError::BytesLoad)?, + strict_cms_der, + strict_name, + )?; + drop(_decode); + + let _ee_profile = timing + .as_ref() + .map(|t| t.span_phase("objects_aspa_validate_embedded_ee_total")); + aspa.validate_embedded_ee_cert()?; + drop(_ee_profile); + + let _verify = timing + .as_ref() + .map(|t| t.span_phase("objects_aspa_verify_signature_total")); + aspa.signed_object.verify()?; + drop(_verify); + + let ee = &aspa.signed_object.signed_data.certificates[0]; + let ee_crldp_uris = ee + .resource_cert + .tbs + .extensions + .crl_distribution_points_uris + .as_ref(); + let issuer_crl_rsync_uri = choose_crl_uri_for_certificate(ee_crldp_uris, crl_cache)?; + let verified_crl = ensure_issuer_crl_verified(issuer_crl_rsync_uri, crl_cache, issuer_ca_der)?; + + let _cert_path = timing + .as_ref() + .map(|t| t.span_phase("objects_aspa_validate_ee_cert_path_total")); + validate_signed_object_ee_cert_path_fast( + ee, + issuer_ca, + issuer_spki, + &verified_crl.crl, + &verified_crl.revoked_serials, + issuer_ca_rsync_uri, + Some(issuer_crl_rsync_uri), + validation_time, + )?; + drop(_cert_path); + + let _subset = timing + .as_ref() + .map(|t| t.span_phase("objects_aspa_validate_ee_resources_subset_total")); + let ee_vrs = validate_ee_resources_for_mode( + &ee.resource_cert, + issuer_effective_ip, + issuer_effective_as, + issuer_resources_index, + resource_validation_mode, + )?; + drop(_subset); + + if let Some(ta_constraints) = ta_constraints { + ta_constraints.validate_ee_certificate(&ee.resource_cert)?; + } + + validate_aspa_customer_in_vrs(&aspa, ee_vrs.asn.as_ref())?; + + let attestation = AspaAttestation { + customer_as_id: aspa.aspa.customer_as_id, + provider_as_ids: aspa.aspa.provider_as_ids.clone(), + }; + if !collect_vcir_local_outputs { + return Ok((attestation, None)); + } + let source_object_hash = sha256_hex_from_32(&file.sha256); + let source_ee_cert_hash = crate::audit::sha256_hex(ee.raw_der.as_slice()); + let item_effective_until = + PackTime::from_utc_offset_datetime(ee.resource_cert.tbs.validity_not_after); + let providers = attestation + .provider_as_ids + .iter() + .map(u32::to_string) + .collect::>() + .join(","); + let rule_hash = crate::audit::sha256_hex( + format!( + "aspa-rule:{}:{}:{}", + source_object_hash, attestation.customer_as_id, providers + ) + .as_bytes(), + ); + let local_output = VcirLocalOutput { + output_type: VcirOutputType::Aspa, + item_effective_until, + source_object_uri: file.rsync_uri.clone(), + source_object_type: VcirSourceObjectType::Aspa, + source_object_hash: file.sha256, + source_ee_cert_hash: sha256_hex_to_32(&source_ee_cert_hash), + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: attestation.customer_as_id, + provider_as_ids: attestation.provider_as_ids.clone(), + }, + rule_hash: sha256_hex_to_32(&rule_hash), + }; + + Ok((attestation, Some(local_output))) +} + +fn vrp_prefix_to_string(vrp: &Vrp) -> String { + let prefix = &vrp.prefix; + match prefix.afi { + RoaAfi::Ipv4 => { + let addr = std::net::Ipv4Addr::new( + prefix.addr[0], + prefix.addr[1], + prefix.addr[2], + prefix.addr[3], + ); + format!("{addr}/{}", prefix.prefix_len) + } + RoaAfi::Ipv6 => { + let mut octets = [0u8; 16]; + octets.copy_from_slice(&prefix.addr[..16]); + let addr = std::net::Ipv6Addr::from(octets); + format!("{addr}/{}", prefix.prefix_len) + } + } +} + +fn choose_crl_uri_for_certificate<'a>( + crldp_uris: Option<&'a Vec>, + crl_cache: &std::collections::HashMap, +) -> Result<&'a str, ObjectValidateError> { + if crl_cache.is_empty() { + return Err(ObjectValidateError::MissingCrlInPack); + } + + let Some(crldp_uris) = crldp_uris else { + return Err(ObjectValidateError::MissingCrlDpUris); + }; + + for u in crldp_uris { + let s = u.as_str(); + if crl_cache.contains_key(s) { + return Ok(s); + } + } + Err(ObjectValidateError::CrlNotFound( + crldp_uris + .iter() + .map(|u| u.as_str()) + .collect::>() + .join(", "), + )) +} + +fn ensure_issuer_crl_verified<'a>( + crl_rsync_uri: &str, + crl_cache: &'a mut std::collections::HashMap, + issuer_ca_der: &[u8], +) -> Result, CertPathError> { + let entry = crl_cache + .get_mut(crl_rsync_uri) + .expect("CRL must exist in cache"); + match entry { + CachedIssuerCrl::Ok(v) => Ok(Arc::clone(v)), + CachedIssuerCrl::Pending { bytes, sha256_hex } => { + let der = std::mem::take(bytes); + let current_sha256_hex = sha256_hex + .take() + .unwrap_or_else(|| crate::audit::sha256_hex(&der)); + let crl = crate::data_model::crl::RpkixCrl::decode_der(&der) + .map_err(CertPathError::CrlDecode)?; + crl.verify_signature_with_issuer_certificate_der(issuer_ca_der) + .map_err(CertPathError::CrlVerify)?; + + let mut revoked_serials: std::collections::HashSet> = + std::collections::HashSet::with_capacity(crl.revoked_certs.len()); + for rc in &crl.revoked_certs { + revoked_serials.insert(rc.serial_number.bytes_be.clone()); + } + + *entry = CachedIssuerCrl::Ok(Arc::new(VerifiedIssuerCrl { + crl, + revoked_serials, + sha256_hex: current_sha256_hex, + })); + match entry { + CachedIssuerCrl::Ok(v) => Ok(Arc::clone(v)), + _ => unreachable!(), + } + } + } +} + +fn validate_ee_resources_subset( + ee: &ResourceCertificate, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + issuer_resources_index: &IssuerResourcesIndex, +) -> Result<(), ObjectValidateError> { + if let Some(child_ip) = ee.tbs.extensions.ip_resources.as_ref() { + let Some(parent_ip) = issuer_effective_ip else { + return Err(ObjectValidateError::MissingIssuerEffectiveIp); + }; + if !ip_resources_is_subset_indexed(child_ip, parent_ip, issuer_resources_index) { + return Err(ObjectValidateError::EeResourcesNotSubset); + } + } + + if let Some(child_as) = ee.tbs.extensions.as_resources.as_ref() { + let Some(parent_as) = issuer_effective_as else { + return Err(ObjectValidateError::MissingIssuerEffectiveAs); + }; + if !as_resources_is_subset_indexed(child_as, parent_as, issuer_resources_index) { + return Err(ObjectValidateError::EeResourcesNotSubset); + } + } + + Ok(()) +} + +#[derive(Debug)] +struct EeVerifiedResources { + ip: Option, + asn: Option, +} + +fn validate_ee_resources_for_mode( + ee: &ResourceCertificate, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + issuer_resources_index: &IssuerResourcesIndex, + mode: ResourceValidationMode, +) -> Result { + match mode { + ResourceValidationMode::Rfc6487 => { + validate_ee_resources_subset( + ee, + issuer_effective_ip, + issuer_effective_as, + issuer_resources_index, + )?; + Ok(EeVerifiedResources { + ip: ee.tbs.extensions.ip_resources.clone(), + asn: ee.tbs.extensions.as_resources.clone(), + }) + } + ResourceValidationMode::ValidationUpdate03 => { + let ip = match ee.tbs.extensions.ip_resources.as_ref() { + Some(child_ip) => Some(intersect_ee_ip_resources_vrs( + child_ip, + issuer_effective_ip, + issuer_resources_index, + )?), + None => None, + }; + let asn = match ee.tbs.extensions.as_resources.as_ref() { + Some(child_as) => Some(intersect_ee_as_resources_vrs( + child_as, + issuer_effective_as, + issuer_resources_index, + )?), + None => None, + }; + Ok(EeVerifiedResources { ip, asn }) + } + } +} + +fn intersect_ee_ip_resources_vrs( + child_ip: &crate::data_model::rc::IpResourceSet, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_resources_index: &IssuerResourcesIndex, +) -> Result { + if child_ip.has_any_inherit() { + return Err(ObjectValidateError::EeResourcesNotSubset); + } + let _ = issuer_effective_ip; + let mut families = Vec::new(); + for fam in &child_ip.families { + let parent_intervals = match fam.afi { + crate::data_model::rc::Afi::Ipv4 => issuer_resources_index.ip_v4.as_deref(), + crate::data_model::rc::Afi::Ipv6 => issuer_resources_index.ip_v6.as_deref(), + } + .unwrap_or(&[]); + let items = match &fam.choice { + IpAddressChoice::Inherit => return Err(ObjectValidateError::EeResourcesNotSubset), + IpAddressChoice::AddressesOrRanges(items) => items, + }; + let intersections = intersect_ip_items_with_parent_intervals(items, parent_intervals); + if !intersections.is_empty() { + families.push(crate::data_model::rc::IpAddressFamily { + afi: fam.afi, + choice: IpAddressChoice::AddressesOrRanges(ip_intervals_to_ranges( + fam.afi, + &intersections, + )), + }); + } + } + Ok(crate::data_model::rc::IpResourceSet { families }) +} + +fn intersect_ee_as_resources_vrs( + child_as: &crate::data_model::rc::AsResourceSet, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + issuer_resources_index: &IssuerResourcesIndex, +) -> Result { + let _ = issuer_effective_as; + if matches!(child_as.asnum, Some(AsIdentifierChoice::Inherit)) + || matches!(child_as.rdi, Some(AsIdentifierChoice::Inherit)) + { + return Err(ObjectValidateError::EeResourcesNotSubset); + } + let asnum = child_as.asnum.as_ref().map(|choice| { + let child_intervals = as_choice_to_merged_intervals(choice); + AsIdentifierChoice::AsIdsOrRanges(as_intervals_to_items(&intersect_as_intervals( + &child_intervals, + issuer_resources_index.asnum.as_deref().unwrap_or(&[]), + ))) + }); + let rdi = child_as.rdi.as_ref().map(|choice| { + let child_intervals = as_choice_to_merged_intervals(choice); + AsIdentifierChoice::AsIdsOrRanges(as_intervals_to_items(&intersect_as_intervals( + &child_intervals, + issuer_resources_index.rdi.as_deref().unwrap_or(&[]), + ))) + }); + Ok(crate::data_model::rc::AsResourceSet { asnum, rdi }) +} + +fn roa_to_vrps_with_vrs( + roa: &RoaObject, + ee_vrs_ip: Option<&crate::data_model::rc::IpResourceSet>, +) -> Result, ObjectValidateError> { + let vrps = roa_to_vrps(roa); + let Some(ee_vrs_ip) = ee_vrs_ip else { + return Err(ObjectValidateError::EeResourcesNotSubset); + }; + for vrp in &vrps { + let rc_prefix = roa_prefix_to_rc_prefix(&vrp.prefix); + if !ee_vrs_ip.contains_prefix(&rc_prefix) { + return Err(ObjectValidateError::EeResourcesNotSubset); + } + } + Ok(vrps) +} + +fn validate_aspa_customer_in_vrs( + aspa: &AspaObject, + ee_vrs_as: Option<&crate::data_model::rc::AsResourceSet>, +) -> Result<(), ObjectValidateError> { + let Some(ee_vrs_as) = ee_vrs_as else { + return Err(ObjectValidateError::EeResourcesNotSubset); + }; + if !as_resource_set_contains_asn(ee_vrs_as, aspa.aspa.customer_as_id) { + return Err(ObjectValidateError::EeResourcesNotSubset); + } + Ok(()) +} + +fn as_resource_set_contains_asn( + resources: &crate::data_model::rc::AsResourceSet, + asn: u32, +) -> bool { + let Some(choice) = resources.asnum.as_ref() else { + return false; + }; + match choice { + AsIdentifierChoice::Inherit => false, + AsIdentifierChoice::AsIdsOrRanges(items) => items.iter().any(|item| match item { + crate::data_model::rc::AsIdOrRange::Id(id) => *id == asn, + crate::data_model::rc::AsIdOrRange::Range { min, max } => *min <= asn && asn <= *max, + }), + } +} + +fn roa_prefix_to_rc_prefix(prefix: &IpPrefix) -> RcIpPrefix { + let afi = match prefix.afi { + RoaAfi::Ipv4 => crate::data_model::rc::Afi::Ipv4, + RoaAfi::Ipv6 => crate::data_model::rc::Afi::Ipv6, + }; + let mut addr = prefix.addr.to_vec(); + addr.truncate(afi.octets_len()); + RcIpPrefix { + afi, + prefix_len: prefix.prefix_len, + addr, + } +} + +fn as_resources_is_subset(child: &AsResourceSet, parent: &AsResourceSet) -> bool { + as_choice_subset(child.asnum.as_ref(), parent.asnum.as_ref()) + && as_choice_subset(child.rdi.as_ref(), parent.rdi.as_ref()) +} + +fn as_resources_is_subset_indexed( + child: &AsResourceSet, + parent: &AsResourceSet, + idx: &IssuerResourcesIndex, +) -> bool { + let _ = parent; + as_choice_subset_indexed(child.asnum.as_ref(), idx.asnum.as_deref()) + && as_choice_subset_indexed(child.rdi.as_ref(), idx.rdi.as_deref()) +} + +fn as_choice_subset_indexed( + child: Option<&AsIdentifierChoice>, + parent_intervals: Option<&[(u32, u32)]>, +) -> bool { + let Some(child) = child else { + return true; + }; + let Some(parent_intervals) = parent_intervals else { + return false; + }; + + if matches!(child, AsIdentifierChoice::Inherit) { + return false; + } + + let child_intervals = as_choice_to_merged_intervals(child); + for (cmin, cmax) in &child_intervals { + if !as_interval_is_covered(parent_intervals, *cmin, *cmax) { + return false; + } + } + true +} + +fn as_choice_subset( + child: Option<&AsIdentifierChoice>, + parent: Option<&AsIdentifierChoice>, +) -> bool { + let Some(child) = child else { + return true; + }; + let Some(parent) = parent else { + return false; + }; + + match (child, parent) { + (AsIdentifierChoice::Inherit, _) => return false, + (_, AsIdentifierChoice::Inherit) => return false, + _ => {} + } + + let child_intervals = as_choice_to_merged_intervals(child); + let parent_intervals = as_choice_to_merged_intervals(parent); + for (cmin, cmax) in &child_intervals { + if !as_interval_is_covered(&parent_intervals, *cmin, *cmax) { + return false; + } + } + true +} + +fn as_choice_to_merged_intervals(choice: &AsIdentifierChoice) -> Vec<(u32, u32)> { + let mut v = Vec::new(); + match choice { + AsIdentifierChoice::Inherit => {} + AsIdentifierChoice::AsIdsOrRanges(items) => { + for item in items { + match item { + crate::data_model::rc::AsIdOrRange::Id(id) => v.push((*id, *id)), + crate::data_model::rc::AsIdOrRange::Range { min, max } => v.push((*min, *max)), + } + } + } + } + v.sort_by_key(|(a, _)| *a); + merge_as_intervals(&v) +} + +fn merge_as_intervals(v: &[(u32, u32)]) -> Vec<(u32, u32)> { + let mut out: Vec<(u32, u32)> = Vec::new(); + for (min, max) in v { + let Some(last) = out.last_mut() else { + out.push((*min, *max)); + continue; + }; + if *min <= last.1.saturating_add(1) { + last.1 = last.1.max(*max); + continue; + } + out.push((*min, *max)); + } + out +} + +fn as_interval_is_covered(parent: &[(u32, u32)], min: u32, max: u32) -> bool { + for (pmin, pmax) in parent { + if *pmin <= min && max <= *pmax { + return true; + } + if *pmin > min { + break; + } + } + false +} + +fn ip_resources_is_subset( + child: &crate::data_model::rc::IpResourceSet, + parent: &crate::data_model::rc::IpResourceSet, +) -> bool { + let parent_by_afi = ip_resources_to_merged_intervals(parent); + let child_by_afi = match ip_resources_to_merged_intervals_strict(child) { + Ok(v) => v, + Err(()) => return false, + }; + + for (afi, child_intervals) in child_by_afi { + let Some(parent_intervals) = parent_by_afi.get(&afi) else { + return false; + }; + for (cmin, cmax) in &child_intervals { + if !interval_is_covered(parent_intervals, cmin, cmax) { + return false; + } + } + } + true +} + +fn ip_resources_is_subset_indexed( + child: &crate::data_model::rc::IpResourceSet, + parent: &crate::data_model::rc::IpResourceSet, + idx: &IssuerResourcesIndex, +) -> bool { + let _ = parent; + + for fam in &child.families { + let parent_intervals = match fam.afi { + crate::data_model::rc::Afi::Ipv4 => idx.ip_v4.as_deref(), + crate::data_model::rc::Afi::Ipv6 => idx.ip_v6.as_deref(), + }; + let Some(parent_intervals) = parent_intervals else { + return false; + }; + let items = match &fam.choice { + IpAddressChoice::Inherit => return false, + IpAddressChoice::AddressesOrRanges(items) => items, + }; + + let mut child_intervals: Vec<(Vec, Vec)> = Vec::new(); + for item in items { + match item { + IpAddressOrRange::Prefix(p) => child_intervals.push(prefix_to_range(p)), + IpAddressOrRange::Range(r) => child_intervals.push((r.min.clone(), r.max.clone())), + } + } + if child_intervals.is_empty() { + continue; + } + child_intervals.sort_by(|(a, _), (b, _)| a.cmp(b)); + merge_ip_intervals_in_place(&mut child_intervals); + if !intervals_are_covered(parent_intervals, &child_intervals) { + return false; + } + } + true +} + +fn ip_items_to_merged_intervals( + items: &[crate::data_model::rc::IpAddressOrRange], +) -> Vec<(Vec, Vec)> { + let mut intervals = Vec::new(); + for item in items { + match item { + IpAddressOrRange::Prefix(p) => intervals.push(prefix_to_range(p)), + IpAddressOrRange::Range(r) => intervals.push((r.min.clone(), r.max.clone())), + } + } + intervals.sort_by(|(a, _), (b, _)| a.cmp(b)); + merge_ip_intervals_in_place(&mut intervals); + intervals +} + +fn intersect_ip_items_with_parent_intervals( + items: &[crate::data_model::rc::IpAddressOrRange], + parent_intervals: &[(Vec, Vec)], +) -> Vec<(Vec, Vec)> { + let child_intervals = ip_items_to_merged_intervals(items); + let mut out = Vec::new(); + let mut parent_index = 0usize; + for (child_min, child_max) in &child_intervals { + while parent_index < parent_intervals.len() + && parent_intervals[parent_index].1.as_slice() < child_min.as_slice() + { + parent_index += 1; + } + let mut scan = parent_index; + while scan < parent_intervals.len() + && parent_intervals[scan].0.as_slice() <= child_max.as_slice() + { + let (parent_min, parent_max) = &parent_intervals[scan]; + let min = if bytes_leq(child_min, parent_min) { + parent_min.clone() + } else { + child_min.clone() + }; + let max = if bytes_leq(child_max, parent_max) { + child_max.clone() + } else { + parent_max.clone() + }; + if bytes_leq(&min, &max) { + out.push((min, max)); + } + scan += 1; + } + } + merge_ip_intervals_in_place(&mut out); + out +} + +fn ip_intervals_to_ranges( + afi: crate::data_model::rc::Afi, + intervals: &[(Vec, Vec)], +) -> Vec { + intervals + .iter() + .map(|(min, max)| { + IpAddressOrRange::Range(crate::data_model::rc::IpAddressRange { + min: normalize_ip_bytes(afi, min), + max: normalize_ip_bytes(afi, max), + }) + }) + .collect() +} + +fn normalize_ip_bytes(afi: crate::data_model::rc::Afi, bytes: &[u8]) -> Vec { + let target_len = afi.octets_len(); + if bytes.len() == target_len { + return bytes.to_vec(); + } + let mut out = vec![0u8; target_len]; + let copy_len = bytes.len().min(target_len); + out[..copy_len].copy_from_slice(&bytes[..copy_len]); + out +} + +fn intersect_as_intervals(child: &[(u32, u32)], parent: &[(u32, u32)]) -> Vec<(u32, u32)> { + let mut out = Vec::new(); + let mut parent_index = 0usize; + for (child_min, child_max) in child { + while parent_index < parent.len() && parent[parent_index].1 < *child_min { + parent_index += 1; + } + let mut scan = parent_index; + while scan < parent.len() && parent[scan].0 <= *child_max { + let min = (*child_min).max(parent[scan].0); + let max = (*child_max).min(parent[scan].1); + if min <= max { + out.push((min, max)); + } + scan += 1; + } + } + merge_as_intervals(&out) +} + +fn as_intervals_to_items(intervals: &[(u32, u32)]) -> Vec { + intervals + .iter() + .map(|(min, max)| { + if min == max { + crate::data_model::rc::AsIdOrRange::Id(*min) + } else { + crate::data_model::rc::AsIdOrRange::Range { + min: *min, + max: *max, + } + } + }) + .collect() +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +enum AfiKey { + V4, + V6, +} + +fn ip_resources_to_merged_intervals( + set: &crate::data_model::rc::IpResourceSet, +) -> std::collections::HashMap, Vec)>> { + let mut m: std::collections::HashMap, Vec)>> = + std::collections::HashMap::new(); + + for fam in &set.families { + let afi = match fam.afi { + crate::data_model::rc::Afi::Ipv4 => AfiKey::V4, + crate::data_model::rc::Afi::Ipv6 => AfiKey::V6, + }; + match &fam.choice { + IpAddressChoice::Inherit => { + // Effective resource sets should not contain inherit, but if they do we treat it + // as "unknown" by leaving it empty here (subset checks will fail). + } + IpAddressChoice::AddressesOrRanges(items) => { + let ent = m.entry(afi).or_default(); + for item in items { + match item { + IpAddressOrRange::Prefix(p) => ent.push(prefix_to_range(p)), + IpAddressOrRange::Range(r) => ent.push((r.min.clone(), r.max.clone())), + } + } + } + } + } + + for (_afi, v) in m.iter_mut() { + v.sort_by(|(a, _), (b, _)| a.cmp(b)); + merge_ip_intervals_in_place(v); + } + + m +} + +fn ip_resources_to_merged_intervals_strict( + set: &crate::data_model::rc::IpResourceSet, +) -> Result, Vec)>>, ()> { + let mut m: std::collections::HashMap, Vec)>> = + std::collections::HashMap::new(); + + for fam in &set.families { + let afi = match fam.afi { + crate::data_model::rc::Afi::Ipv4 => AfiKey::V4, + crate::data_model::rc::Afi::Ipv6 => AfiKey::V6, + }; + match &fam.choice { + IpAddressChoice::Inherit => return Err(()), + IpAddressChoice::AddressesOrRanges(items) => { + let ent = m.entry(afi).or_default(); + for item in items { + match item { + IpAddressOrRange::Prefix(p) => ent.push(prefix_to_range(p)), + IpAddressOrRange::Range(r) => ent.push((r.min.clone(), r.max.clone())), + } + } + } + } + } + + for (_afi, v) in m.iter_mut() { + v.sort_by(|(a, _), (b, _)| a.cmp(b)); + merge_ip_intervals_in_place(v); + } + + Ok(m) +} + +fn merge_ip_intervals_in_place(v: &mut Vec<(Vec, Vec)>) { + if v.is_empty() { + return; + } + let mut out: Vec<(Vec, Vec)> = Vec::with_capacity(v.len()); + for (min, max) in v.drain(..) { + let Some(last) = out.last_mut() else { + out.push((min, max)); + continue; + }; + if bytes_leq(&min, &last.1) || bytes_is_next(&min, &last.1) { + if bytes_leq(&last.1, &max) { + last.1 = max; + } + continue; + } + out.push((min, max)); + } + *v = out; +} + +fn interval_is_covered(parent: &[(Vec, Vec)], min: &[u8], max: &[u8]) -> bool { + for (pmin, pmax) in parent { + if bytes_leq(pmin, min) && bytes_leq(max, pmax) { + return true; + } + if pmin.as_slice() > min { + break; + } + } + false +} + +fn intervals_are_covered(parent: &[(Vec, Vec)], child: &[(Vec, Vec)]) -> bool { + let mut i = 0usize; + for (cmin, cmax) in child { + while i < parent.len() && parent[i].1.as_slice() < cmin.as_slice() { + i += 1; + } + if i >= parent.len() { + return false; + } + let (pmin, pmax) = &parent[i]; + if !bytes_leq(pmin, cmin) || !bytes_leq(cmax, pmax) { + return false; + } + } + true +} + +fn prefix_to_range(prefix: &RcIpPrefix) -> (Vec, Vec) { + let mut min = prefix.addr.clone(); + let mut max = prefix.addr.clone(); + let bitlen = prefix.afi.ub(); + let plen = prefix.prefix_len.min(bitlen); + for bit in plen..bitlen { + let byte = (bit / 8) as usize; + let offset = 7 - (bit % 8); + let mask = 1u8 << offset; + min[byte] &= !mask; + max[byte] |= mask; + } + (min, max) +} + +fn bytes_leq(a: &[u8], b: &[u8]) -> bool { + a <= b +} + +fn increment_bytes(v: &[u8]) -> Vec { + let mut out = v.to_vec(); + for i in (0..out.len()).rev() { + if out[i] != 0xFF { + out[i] += 1; + for j in i + 1..out.len() { + out[j] = 0; + } + return out; + } + } + vec![0u8; out.len()] +} + +fn bytes_is_next(a: &[u8], b: &[u8]) -> bool { + if a.len() != b.len() { + return false; + } + let mut carry: u16 = 1; + for i in (0..b.len()).rev() { + let sum = (b[i] as u16) + carry; + let expected = (sum & 0xFF) as u8; + carry = sum >> 8; + if a[i] != expected { + return false; + } + } + true +} + +fn build_issuer_resources_index( + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, +) -> IssuerResourcesIndex { + let mut idx = IssuerResourcesIndex::default(); + + if let Some(ip) = issuer_effective_ip { + let mut v4: Vec<(Vec, Vec)> = Vec::new(); + let mut v6: Vec<(Vec, Vec)> = Vec::new(); + for fam in &ip.families { + let ent = match fam.afi { + crate::data_model::rc::Afi::Ipv4 => &mut v4, + crate::data_model::rc::Afi::Ipv6 => &mut v6, + }; + match &fam.choice { + IpAddressChoice::Inherit => { + // Effective resources should not contain inherit; leave empty so subset fails. + } + IpAddressChoice::AddressesOrRanges(items) => { + for item in items { + match item { + IpAddressOrRange::Prefix(p) => ent.push(prefix_to_range(p)), + IpAddressOrRange::Range(r) => ent.push((r.min.clone(), r.max.clone())), + } + } + } + } + } + if !v4.is_empty() { + v4.sort_by(|(a, _), (b, _)| a.cmp(b)); + merge_ip_intervals_in_place(&mut v4); + idx.ip_v4 = Some(v4); + } + if !v6.is_empty() { + v6.sort_by(|(a, _), (b, _)| a.cmp(b)); + merge_ip_intervals_in_place(&mut v6); + idx.ip_v6 = Some(v6); + } + } + + if let Some(asr) = issuer_effective_as { + if let Some(choice) = asr.asnum.as_ref() { + if !matches!(choice, AsIdentifierChoice::Inherit) { + idx.asnum = Some(as_choice_to_merged_intervals(choice)); + } + } + if let Some(choice) = asr.rdi.as_ref() { + if !matches!(choice, AsIdentifierChoice::Inherit) { + idx.rdi = Some(as_choice_to_merged_intervals(choice)); + } + } + } + + idx +} + +fn roa_to_vrps(roa: &RoaObject) -> Vec { + let asn = roa.roa.as_id; + let mut out = Vec::new(); + for fam in &roa.roa.ip_addr_blocks { + for entry in &fam.addresses { + let max_length = entry.max_length.unwrap_or(entry.prefix.prefix_len); + out.push(Vrp { + asn, + prefix: entry.prefix.clone(), + max_length, + }); + } + } + out +} + +#[allow(dead_code)] +fn roa_afi_to_string(afi: RoaAfi) -> &'static str { + match afi { + RoaAfi::Ipv4 => "ipv4", + RoaAfi::Ipv6 => "ipv6", + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::analysis::timing::{TimingHandle, TimingMeta}; + use crate::data_model::rc::{ + Afi, AsIdOrRange, AsIdentifierChoice, IpAddressFamily, IpAddressOrRange, IpAddressRange, + IpPrefix, IpResourceSet, + }; + use crate::policy::Policy; + use crate::storage::{ + PackTime, RoaCacheObjectMeta, RoaCacheProjection, RoaCacheProjectionContext, + ValidatedCaInstanceResult, ValidatedManifestMeta, VcirArtifactKind, VcirArtifactRole, + VcirArtifactValidationStatus, VcirAuditSummary, VcirCcrManifestProjection, + VcirInstanceGate, VcirRelatedArtifact, VcirSummary, + }; + use crate::validation::publication_point::PublicationPointSnapshot; + use std::collections::HashMap; + use time::OffsetDateTime; + use time::format_description::well_known::Rfc3339; + + fn fixture_bytes(path: &str) -> Vec { + std::fs::read(std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(path)) + .unwrap_or_else(|e| panic!("read fixture {path}: {e}")) + } + + fn fixed_time(value: &str) -> OffsetDateTime { + OffsetDateTime::parse(value, &Rfc3339).expect("parse fixed test time") + } + + const TEST_CA_VALIDATION_CONTEXT: [u8; 32] = [0x70; 32]; + const TEST_POLICY_FINGERPRINT: [u8; 32] = [0x71; 32]; + const TEST_CRL_URI: &str = "rsync://example.test/repo/current.crl"; + const TEST_ROA_URI: &str = "rsync://example.test/repo/a.roa"; + + fn sha256_32(bytes: &[u8]) -> [u8; 32] { + let digest = sha2::Sha256::digest(bytes); + let mut out = [0u8; 32]; + out.copy_from_slice(&digest); + out + } + + fn sample_roa_cache_projection( + vcir: &ValidatedCaInstanceResult, + roa_hash: [u8; 32], + ) -> RoaCacheProjection { + RoaCacheProjection::from_vcir_with_context( + vcir, + Some(&RoaCacheProjectionContext { + ca_validation_context_digest: TEST_CA_VALIDATION_CONTEXT, + policy_fingerprint: TEST_POLICY_FINGERPRINT, + object_meta: vec![RoaCacheObjectMeta { + source_object_uri: TEST_ROA_URI.to_string(), + source_object_hash: roa_hash, + ee_serial: vec![0x01], + crl_uri: TEST_CRL_URI.to_string(), + earliest_safe_reuse_time: vcir.last_successful_validation_time.clone(), + }], + }), + ) + .expect("build projection") + .expect("projection exists") + } + + fn sample_crl_cache(crl_bytes: Vec) -> HashMap { + HashMap::from([( + TEST_CRL_URI.to_string(), + CachedIssuerCrl::Pending { + bytes: crl_bytes, + sha256_hex: None, + }, + )]) + } + + fn sample_roa_cache_vcir( + issuer_der: &[u8], + crl_hash: [u8; 32], + roa_hash: [u8; 32], + item_effective_until: OffsetDateTime, + instance_effective_until: OffsetDateTime, + ) -> ValidatedCaInstanceResult { + let manifest_time = PackTime::from_utc_offset_datetime(fixed_time("2026-06-04T00:00:00Z")); + let effective_until = PackTime::from_utc_offset_datetime(instance_effective_until); + ValidatedCaInstanceResult { + manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(), + parent_manifest_rsync_uri: Some("rsync://example.test/repo/parent.mft".to_string()), + tal_id: "test-tal".to_string(), + ca_subject_name: "CN=example".to_string(), + ca_ski: "001122".to_string(), + issuer_ski: "334455".to_string(), + last_successful_validation_time: manifest_time.clone(), + current_manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(), + current_crl_rsync_uri: "rsync://example.test/repo/current.crl".to_string(), + validated_manifest_meta: ValidatedManifestMeta { + validated_manifest_number: vec![1], + validated_manifest_this_update: manifest_time.clone(), + validated_manifest_next_update: effective_until.clone(), + }, + ccr_manifest_projection: VcirCcrManifestProjection { + manifest_rsync_uri: "rsync://example.test/repo/current.mft".to_string(), + manifest_sha256: vec![0xaa; 32], + manifest_size: 2048, + manifest_ee_aki: vec![0xbb; 20], + manifest_number_be: vec![1], + manifest_this_update: manifest_time.clone(), + manifest_sia_locations_der: vec![vec![0x30, 0x00]], + subordinate_skis: Vec::new(), + }, + instance_gate: VcirInstanceGate { + manifest_next_update: effective_until.clone(), + current_crl_next_update: effective_until.clone(), + self_ca_not_after: effective_until.clone(), + instance_effective_until: effective_until, + }, + child_entries: Vec::new(), + local_outputs: vec![VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: PackTime::from_utc_offset_datetime(item_effective_until), + source_object_uri: "rsync://example.test/repo/a.roa".to_string(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: roa_hash, + source_ee_cert_hash: [0xcc; 32], + payload: VcirLocalOutputPayload::Vrp { + asn: 64500, + afi: RoaAfi::Ipv4, + prefix_len: 24, + addr: [192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + max_length: 24, + }, + rule_hash: [0xdd; 32], + }], + related_artifacts: vec![ + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::IssuerCert, + artifact_kind: VcirArtifactKind::Cer, + uri: Some("rsync://example.test/repo/ca.cer".to_string()), + sha256: sha256_hex(issuer_der), + object_type: Some("cer".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::CurrentCrl, + artifact_kind: VcirArtifactKind::Crl, + uri: Some("rsync://example.test/repo/current.crl".to_string()), + sha256: sha256_hex_from_32(&crl_hash), + object_type: Some("crl".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }, + ], + summary: VcirSummary { + local_vrp_count: 1, + local_aspa_count: 0, + local_router_key_count: 0, + child_count: 0, + accepted_object_count: 2, + rejected_object_count: 0, + }, + audit_summary: VcirAuditSummary { + failed_fetch_eligible: true, + last_failed_fetch_reason: None, + warning_count: 0, + audit_flags: Vec::new(), + }, + } + } + + #[test] + fn roa_validation_cache_view_hits_when_context_and_hash_match() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_bytes = b"current-crl".to_vec(); + let crl_hash = sha256_32(&crl_bytes); + let roa_hash = [0x11; 32]; + let vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + let projection = sample_roa_cache_projection(&vcir, roa_hash); + let view = RoaValidationCacheView::from_projection(&projection, validation_time); + let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); + let mut crl_cache = sample_crl_cache(crl_bytes); + + assert!(view.matches_current_context( + issuer_der, + Some(TEST_CA_VALIDATION_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); + let hit = view.lookup(&file, &mut crl_cache, issuer_der, validation_time); + let RoaCacheLookupResult::Hit(ok) = hit else { + panic!("expected cache hit, got {hit:?}"); + }; + assert!(ok.reused_from_cache); + assert_eq!(ok.vrps.len(), 1); + assert_eq!(ok.vrps[0].asn, 64500); + assert_eq!(ok.local_outputs.len(), 1); + assert!(ok.cache_object_meta.is_some()); + } + + #[test] + fn roa_validation_cache_lookup_memoizes_current_crl_hash() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_bytes = b"current-crl".to_vec(); + let crl_hash = sha256_32(&crl_bytes); + let crl_hash_hex = sha256_hex(&crl_bytes); + let roa_hash = [0x11; 32]; + let vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + let projection = sample_roa_cache_projection(&vcir, roa_hash); + let view = RoaValidationCacheView::from_projection(&projection, validation_time); + let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); + let mut crl_cache = sample_crl_cache(crl_bytes); + + let first = view.lookup(&file, &mut crl_cache, issuer_der, validation_time); + assert!(matches!(first, RoaCacheLookupResult::Hit(_))); + match crl_cache.get(TEST_CRL_URI).expect("test CRL cache entry") { + CachedIssuerCrl::Pending { + sha256_hex: Some(cached), + .. + } => assert_eq!(cached, &crl_hash_hex), + other => panic!("expected pending CRL with memoized hash, got {other:?}"), + } + + let second = view.lookup(&file, &mut crl_cache, issuer_der, validation_time); + assert!(matches!(second, RoaCacheLookupResult::Hit(_))); + } + + #[test] + fn roa_validation_cache_lookup_reuses_publication_point_crl_gate() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_bytes = b"current-crl".to_vec(); + let crl_hash = sha256_32(&crl_bytes); + let roa_hash = [0x11; 32]; + let vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + let projection = sample_roa_cache_projection(&vcir, roa_hash); + let view = RoaValidationCacheView::from_projection(&projection, validation_time); + let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); + let mut crl_cache = sample_crl_cache(crl_bytes); + let mut gate_set = RoaCacheCrlGateSet::default(); + + let (first, first_metrics) = view.lookup_with_metrics( + &file, + &mut crl_cache, + issuer_der, + validation_time, + Some(&mut gate_set), + ); + assert!(matches!(first, RoaCacheLookupResult::Hit(_))); + assert_eq!(first_metrics.crl_gate_reused_crls, 0); + assert_eq!(gate_set.gates_by_uri.len(), 1); + + let (second, second_metrics) = view.lookup_with_metrics( + &file, + &mut crl_cache, + issuer_der, + validation_time, + Some(&mut gate_set), + ); + assert!(matches!(second, RoaCacheLookupResult::Hit(_))); + assert_eq!(second_metrics.crl_gate_reused_crls, 1); + assert_eq!(second_metrics.crl_gate_verified_crls, 0); + assert_eq!(gate_set.gates_by_uri.len(), 1); + } + + #[test] + fn cached_verified_crl_reports_hash_and_validity_window() { + let crl_bytes = fixture_bytes( + "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.crl", + ); + let crl = crate::data_model::crl::RpkixCrl::decode_der(&crl_bytes).expect("decode CRL"); + let verified = Arc::new(VerifiedIssuerCrl { + crl, + revoked_serials: std::collections::HashSet::new(), + sha256_hex: sha256_hex(&crl_bytes), + }); + let mut cached = CachedIssuerCrl::Ok(Arc::clone(&verified)); + + assert_eq!(cached.current_sha256_hex(), verified.sha256_hex); + assert!(crl_valid_at_time( + &verified.crl, + fixed_time("2026-01-21T00:00:00Z") + )); + assert!(!crl_valid_at_time( + &verified.crl, + fixed_time("2026-01-22T00:00:00Z") + )); + } + + #[test] + fn roa_validation_cache_view_from_projection_preserves_metadata() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_bytes = b"current-crl".to_vec(); + let crl_hash = sha256_32(&crl_bytes); + let roa_hash = [0x11; 32]; + let vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + let projection = sample_roa_cache_projection(&vcir, roa_hash); + assert_eq!( + projection.ca_validation_context_digest, + Some(TEST_CA_VALIDATION_CONTEXT) + ); + assert_eq!(projection.policy_fingerprint, Some(TEST_POLICY_FINGERPRINT)); + assert_eq!( + projection.entries[0].outputs_effective_until_unix, + fixed_time("2026-06-07T00:00:00Z").unix_timestamp() + ); + assert_eq!( + projection.entries[0].ee_serial.as_deref(), + Some(&[0x01][..]) + ); + assert_eq!(projection.entries[0].crl_uri.as_deref(), Some(TEST_CRL_URI)); + + let view = RoaValidationCacheView::from_projection(&projection, validation_time); + let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); + let mut crl_cache = sample_crl_cache(crl_bytes); + let hit = view.lookup(&file, &mut crl_cache, issuer_der, validation_time); + let RoaCacheLookupResult::Hit(ok) = hit else { + panic!("expected projection cache hit, got {hit:?}"); + }; + assert_eq!(ok.local_outputs[0].source_object_uri, TEST_ROA_URI); + } + + #[test] + fn roa_validation_cache_view_blocks_on_context_hash_and_expiry_gates() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_bytes = b"current-crl".to_vec(); + let crl_hash = sha256_32(&crl_bytes); + let roa_hash = [0x11; 32]; + let vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); + + let mut projection = sample_roa_cache_projection(&vcir, roa_hash); + projection.issuer_ca_sha256_hex = Some("00".repeat(32)); + let issuer_changed = RoaValidationCacheView::from_projection(&projection, validation_time); + assert!(!issuer_changed.matches_current_context( + issuer_der, + Some(TEST_CA_VALIDATION_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); + + let projection = sample_roa_cache_projection(&vcir, roa_hash); + let parent_changed = RoaValidationCacheView::from_projection(&projection, validation_time); + assert!(!parent_changed.matches_current_context( + issuer_der, + Some([0x99; 32]), + Some(TEST_POLICY_FINGERPRINT) + )); + assert!(!parent_changed.matches_current_context( + issuer_der, + Some(TEST_CA_VALIDATION_CONTEXT), + Some([0x98; 32]) + )); + + let mut projection = sample_roa_cache_projection(&vcir, roa_hash); + projection.entries[0].source_object_hash = [0xff; 32]; + let roa_changed = RoaValidationCacheView::from_projection(&projection, validation_time); + let mut crl_cache = sample_crl_cache(crl_bytes); + assert!(matches!( + roa_changed.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::HashBlocked + )); + + let expired_vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-04T00:00:00Z"), + ); + let expired_projection = sample_roa_cache_projection(&expired_vcir, roa_hash); + let expired_view = + RoaValidationCacheView::from_projection(&expired_projection, validation_time); + assert!(!expired_view.matches_current_context( + issuer_der, + Some(TEST_CA_VALIDATION_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); + let mut crl_cache = sample_crl_cache(b"current-crl".to_vec()); + assert!(matches!( + expired_view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::ExpiredBlocked + )); + } + + #[test] + fn active_roa_cache_view_records_context_block() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_hash = [0x22; 32]; + let roa_hash = [0x11; 32]; + let vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + let projection = sample_roa_cache_projection(&vcir, roa_hash); + let view = RoaValidationCacheView::from_projection(&projection, validation_time); + let input = RoaValidationCacheInput::enabled_with_context( + Some(&view), + [0x99; 32], + TEST_POLICY_FINGERPRINT, + ); + let mut stats = RoaValidationCacheStats::for_input(input, 1); + + assert!(active_roa_cache_view(input, issuer_der, &mut stats, 1).is_none()); + assert_eq!(stats.blocked_roas, 1); + assert_eq!(stats.context_blocked_roas, 1); + assert_eq!(stats.fresh_roas, 1); + } + + #[test] + fn roa_validation_cache_view_blocks_expired_output() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_bytes = b"current-crl".to_vec(); + let crl_hash = sha256_32(&crl_bytes); + let roa_hash = [0x11; 32]; + let vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-04T01:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + let projection = sample_roa_cache_projection(&vcir, roa_hash); + let view = RoaValidationCacheView::from_projection(&projection, validation_time); + let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); + let mut crl_cache = sample_crl_cache(crl_bytes); + + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::ExpiredBlocked + )); + } + + #[test] + fn roa_validation_cache_view_blocks_before_safe_reuse_time() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_bytes = b"current-crl".to_vec(); + let crl_hash = sha256_32(&crl_bytes); + let roa_hash = [0x11; 32]; + let vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + let mut projection = sample_roa_cache_projection(&vcir, roa_hash); + projection.entries[0].earliest_safe_reuse_time_unix = + Some(fixed_time("2026-06-06T00:00:00Z").unix_timestamp()); + let view = RoaValidationCacheView::from_projection(&projection, validation_time); + let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); + let mut crl_cache = sample_crl_cache(crl_bytes); + + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::ExpiredBlocked + )); + } + + #[test] + fn roa_validation_cache_view_treats_legacy_entry_without_lower_bound_as_miss() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_hash = sha256_32(b"current-crl"); + let roa_hash = [0x11; 32]; + let vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + let mut projection = sample_roa_cache_projection(&vcir, roa_hash); + projection.entries[0].earliest_safe_reuse_time_unix = None; + let view = RoaValidationCacheView::from_projection(&projection, validation_time); + let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); + let mut crl_cache = sample_crl_cache(b"current-crl".to_vec()); + + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::Miss + )); + } + + #[test] + fn roa_validation_cache_stats_records_vcir_miss_to_timing() { + let stats = RoaValidationCacheStats::for_input(RoaValidationCacheInput::enabled(None), 3); + assert_eq!(stats.enabled_publication_points, 1); + assert_eq!(stats.vcir_miss_publication_points, 1); + assert_eq!(stats.miss_roas, 3); + assert_eq!(stats.fresh_roas, 3); + + let timing = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339: "2026-06-05T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-06-05T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + stats.record_to_timing(Some(&timing)); + let dir = tempfile::tempdir().expect("timing dir"); + let path = dir.path().join("timing.json"); + timing.write_json(&path, 10).expect("write timing"); + let report: serde_json::Value = + serde_json::from_slice(&std::fs::read(path).expect("read timing")) + .expect("parse timing"); + + assert_eq!( + report["counts"]["roa_validation_cache_enabled_publication_points"], + 1 + ); + assert_eq!( + report["counts"]["roa_validation_cache_vcir_miss_publication_points"], + 1 + ); + assert_eq!(report["counts"]["roa_validation_cache_miss_roas"], 3); + assert_eq!(report["counts"]["roa_validation_cache_fresh_roas"], 3); + } + + #[test] + fn roa_validation_cache_stats_records_block_breakdown_to_timing() { + let mut stats = RoaValidationCacheStats::default(); + record_roa_cache_block(&mut stats, &RoaCacheLookupResult::HashBlocked); + record_roa_cache_block(&mut stats, &RoaCacheLookupResult::ExpiredBlocked); + record_roa_cache_block(&mut stats, &RoaCacheLookupResult::RevokedBlocked); + record_roa_cache_block(&mut stats, &RoaCacheLookupResult::MetadataBlocked); + stats.crl_recheck_hit_roas = 2; + stats.context_blocked_roas = 3; + stats.context_gate_nanos = 4; + stats.lookup_nanos = 5; + stats.lookup_entry_gate_nanos = 6; + stats.lookup_crl_gate_nanos = 7; + stats.lookup_materialize_nanos = 8; + stats.lookup_crl_gate_verified_crls = 9; + stats.lookup_crl_gate_reused_crls = 10; + + let timing = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339: "2026-06-05T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-06-05T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + stats.record_to_timing(Some(&timing)); + let dir = tempfile::tempdir().expect("timing dir"); + let path = dir.path().join("timing.json"); + timing.write_json(&path, 10).expect("write timing"); + let report: serde_json::Value = + serde_json::from_slice(&std::fs::read(path).expect("read timing")) + .expect("parse timing"); + + assert_eq!(stats.blocked_roas, 4); + assert_eq!(stats.fresh_roas, 4); + assert_eq!(report["counts"]["roa_validation_cache_blocked_roas"], 4); + assert_eq!( + report["counts"]["roa_validation_cache_hash_blocked_roas"], + 1 + ); + assert_eq!( + report["counts"]["roa_validation_cache_expired_blocked_roas"], + 1 + ); + assert_eq!( + report["counts"]["roa_validation_cache_revoked_blocked_roas"], + 1 + ); + assert_eq!( + report["counts"]["roa_validation_cache_metadata_blocked_roas"], + 1 + ); + assert_eq!( + report["counts"]["roa_validation_cache_crl_recheck_hit_roas"], + 2 + ); + assert_eq!( + report["counts"]["roa_validation_cache_context_blocked_roas"], + 3 + ); + assert_eq!( + report["counts"]["roa_validation_cache_context_gate_nanos"], + 4 + ); + assert_eq!(report["counts"]["roa_validation_cache_lookup_nanos"], 5); + assert_eq!( + report["counts"]["roa_validation_cache_lookup_entry_gate_nanos"], + 6 + ); + assert_eq!( + report["counts"]["roa_validation_cache_lookup_crl_gate_nanos"], + 7 + ); + assert_eq!( + report["counts"]["roa_validation_cache_lookup_materialize_nanos"], + 8 + ); + assert_eq!( + report["counts"]["roa_validation_cache_lookup_crl_gate_verified_crls"], + 9 + ); + assert_eq!( + report["counts"]["roa_validation_cache_lookup_crl_gate_reused_crls"], + 10 + ); + } + + #[test] + fn roa_validation_cache_view_ignores_rejected_artifacts_and_non_roa_outputs() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_bytes = b"current-crl".to_vec(); + let crl_hash = sha256_32(&crl_bytes); + let roa_hash = [0x11; 32]; + let mut vcir = sample_roa_cache_vcir( + issuer_der, + crl_hash, + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + vcir.related_artifacts.insert( + 0, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::IssuerCert, + artifact_kind: VcirArtifactKind::Cer, + uri: Some("rsync://example.test/repo/rejected.cer".to_string()), + sha256: "00".repeat(32), + object_type: Some("cer".to_string()), + validation_status: VcirArtifactValidationStatus::Rejected, + reject_reason: None, + }, + ); + vcir.local_outputs.push(VcirLocalOutput { + output_type: VcirOutputType::Aspa, + item_effective_until: PackTime::from_utc_offset_datetime(fixed_time( + "2026-06-07T00:00:00Z", + )), + source_object_uri: "rsync://example.test/repo/a.asa".to_string(), + source_object_type: VcirSourceObjectType::Aspa, + source_object_hash: [0x44; 32], + source_ee_cert_hash: [0x55; 32], + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: 64500, + provider_as_ids: vec![64501], + }, + rule_hash: [0x66; 32], + }); + let projection = sample_roa_cache_projection(&vcir, roa_hash); + let view = RoaValidationCacheView::from_projection(&projection, validation_time); + let mut crl_cache = sample_crl_cache(crl_bytes); + + assert!(view.matches_current_context( + issuer_der, + Some(TEST_CA_VALIDATION_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); + assert!(matches!( + view.lookup( + &PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash), + &mut crl_cache, + issuer_der, + validation_time + ), + RoaCacheLookupResult::Hit(_) + )); + assert!(matches!( + view.lookup( + &PackFile::from_bytes_with_sha256( + "rsync://example.test/repo/a.asa", + vec![0x03], + [0x44; 32], + ), + &mut crl_cache, + issuer_der, + validation_time + ), + RoaCacheLookupResult::Miss + )); + } + + #[test] + fn roa_validation_cache_context_requires_issuer_parent_and_policy() { + let mut view = RoaValidationCacheView { + entries_by_uri: HashMap::new(), + issuer_ca_sha256_hex: None, + ca_validation_context_digest: Some(TEST_CA_VALIDATION_CONTEXT), + policy_fingerprint: Some(TEST_POLICY_FINGERPRINT), + crl_sha256_by_uri: HashMap::new(), + blocked: false, + }; + assert!(!view.matches_current_context( + b"issuer-ca", + Some(TEST_CA_VALIDATION_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); + + view.issuer_ca_sha256_hex = Some("00".repeat(32)); + assert!(!view.matches_current_context( + b"issuer-ca", + Some(TEST_CA_VALIDATION_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); + + view.issuer_ca_sha256_hex = Some(sha256_hex(b"issuer-ca")); + assert!(view.matches_current_context( + b"issuer-ca", + Some(TEST_CA_VALIDATION_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); + assert!(!view.matches_current_context(b"issuer-ca", None, Some(TEST_POLICY_FINGERPRINT))); + + view.ca_validation_context_digest = None; + assert!(!view.matches_current_context( + b"issuer-ca", + Some(TEST_CA_VALIDATION_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); + view.ca_validation_context_digest = Some(TEST_CA_VALIDATION_CONTEXT); + view.policy_fingerprint = None; + assert!(!view.matches_current_context( + b"issuer-ca", + Some(TEST_CA_VALIDATION_CONTEXT), + Some(TEST_POLICY_FINGERPRINT) + )); + } + + #[test] + fn roa_validation_cache_lookup_classifies_hash_empty_payload_and_missing_uri() { + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let issuer_der = b"issuer-ca"; + let crl_bytes = b"current-crl".to_vec(); + let crl_hash_hex = sha256_hex(&crl_bytes); + let roa_hash = [0x11; 32]; + let file = PackFile::from_bytes_with_sha256(TEST_ROA_URI, vec![0x01], roa_hash); + let mut output = sample_roa_cache_vcir( + issuer_der, + sha256_32(&crl_bytes), + roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ) + .local_outputs + .remove(0); + let outputs_effective_until_unix = output + .item_effective_until + .parse() + .expect("parse output effective until") + .unix_timestamp(); + let mut view = RoaValidationCacheView { + entries_by_uri: HashMap::new(), + issuer_ca_sha256_hex: Some(sha256_hex(issuer_der)), + ca_validation_context_digest: Some(TEST_CA_VALIDATION_CONTEXT), + policy_fingerprint: Some(TEST_POLICY_FINGERPRINT), + crl_sha256_by_uri: HashMap::from([(TEST_CRL_URI.to_string(), crl_hash_hex.clone())]), + blocked: false, + }; + let mut crl_cache = sample_crl_cache(crl_bytes.clone()); + + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::Miss + )); + + view.entries_by_uri.insert( + file.rsync_uri.clone(), + CachedRoaValidationResult { + source_object_hash: roa_hash, + ee_serial: Some(vec![0x01]), + crl_uri: None, + earliest_safe_reuse_time_unix: validation_time.unix_timestamp(), + outputs_effective_until_unix, + outputs: vec![output.clone()], + }, + ); + let mut crl_cache = sample_crl_cache(crl_bytes.clone()); + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::MetadataBlocked + )); + + view.entries_by_uri.insert( + file.rsync_uri.clone(), + CachedRoaValidationResult { + source_object_hash: roa_hash, + ee_serial: None, + crl_uri: Some(TEST_CRL_URI.to_string()), + earliest_safe_reuse_time_unix: validation_time.unix_timestamp(), + outputs_effective_until_unix, + outputs: vec![output.clone()], + }, + ); + let mut crl_cache = sample_crl_cache(crl_bytes.clone()); + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::MetadataBlocked + )); + + view.entries_by_uri.insert( + file.rsync_uri.clone(), + CachedRoaValidationResult { + source_object_hash: roa_hash, + ee_serial: Some(vec![0x01]), + crl_uri: Some(TEST_CRL_URI.to_string()), + earliest_safe_reuse_time_unix: validation_time.unix_timestamp(), + outputs_effective_until_unix, + outputs: vec![output.clone()], + }, + ); + let mut crl_cache = HashMap::new(); + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::MetadataBlocked + )); + + view.crl_sha256_by_uri + .insert(TEST_CRL_URI.to_string(), "00".repeat(32)); + let mut crl_cache = sample_crl_cache(crl_bytes.clone()); + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::MetadataBlocked + )); + view.crl_sha256_by_uri + .insert(TEST_CRL_URI.to_string(), crl_hash_hex); + + view.entries_by_uri.insert( + file.rsync_uri.clone(), + CachedRoaValidationResult { + source_object_hash: [0xff; 32], + ee_serial: Some(vec![0x01]), + crl_uri: Some(TEST_CRL_URI.to_string()), + earliest_safe_reuse_time_unix: validation_time.unix_timestamp(), + outputs_effective_until_unix, + outputs: vec![output.clone()], + }, + ); + let mut crl_cache = sample_crl_cache(crl_bytes.clone()); + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::HashBlocked + )); + + view.entries_by_uri.insert( + file.rsync_uri.clone(), + CachedRoaValidationResult { + source_object_hash: roa_hash, + ee_serial: Some(vec![0x01]), + crl_uri: Some(TEST_CRL_URI.to_string()), + earliest_safe_reuse_time_unix: validation_time.unix_timestamp(), + outputs_effective_until_unix, + outputs: Vec::new(), + }, + ); + let mut crl_cache = sample_crl_cache(crl_bytes.clone()); + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::Miss + )); + + output.payload = VcirLocalOutputPayload::Aspa { + customer_as_id: 64500, + provider_as_ids: vec![64501], + }; + view.entries_by_uri.insert( + file.rsync_uri.clone(), + CachedRoaValidationResult { + source_object_hash: roa_hash, + ee_serial: Some(vec![0x01]), + crl_uri: Some(TEST_CRL_URI.to_string()), + earliest_safe_reuse_time_unix: validation_time.unix_timestamp(), + outputs_effective_until_unix, + outputs: vec![output], + }, + ); + let mut crl_cache = sample_crl_cache(crl_bytes); + assert!(matches!( + view.lookup(&file, &mut crl_cache, issuer_der, validation_time), + RoaCacheLookupResult::MetadataBlocked + )); + } + + #[test] + fn parallel_roa_cache_blocked_falls_back_to_single_fresh_task() { + let manifest_bytes = fixture_bytes( + "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft", + ); + let issuer_ca_der = fixture_bytes( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ); + let crl_bytes = fixture_bytes( + "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.crl", + ); + let crl_hash = sha256_32(&crl_bytes); + let actual_roa_hash = [0x11; 32]; + let cached_roa_hash = [0x33; 32]; + let validation_time = fixed_time("2026-06-05T00:00:00Z"); + let publication_point = PublicationPointSnapshot { + format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, + manifest_rsync_uri: + "rsync://rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft" + .to_string(), + publication_point_rsync_uri: "rsync://rpki.cernet.net/repo/cernet/0/".to_string(), + manifest_number_be: vec![1], + this_update: PackTime::from_utc_offset_datetime(validation_time), + next_update: PackTime::from_utc_offset_datetime( + validation_time + time::Duration::days(1), + ), + verified_at: PackTime::from_utc_offset_datetime(validation_time), + manifest_bytes, + files: vec![ + PackFile::from_bytes_with_sha256(TEST_CRL_URI, crl_bytes, crl_hash), + PackFile::from_bytes_with_sha256( + "rsync://example.test/repo/a.roa", + vec![0x01], + actual_roa_hash, + ), + ], + }; + let vcir = sample_roa_cache_vcir( + &issuer_ca_der, + crl_hash, + cached_roa_hash, + fixed_time("2026-06-07T00:00:00Z"), + fixed_time("2026-06-08T00:00:00Z"), + ); + let projection = sample_roa_cache_projection(&vcir, cached_roa_hash); + let view = RoaValidationCacheView::from_projection(&projection, validation_time); + + let stage = match prepare_publication_point_for_parallel_roa_with_cache( + 7, + &publication_point, + &Policy::default(), + &issuer_ca_der, + None, + None, + None, + validation_time, + false, + RoaValidationCacheInput::enabled_with_context( + Some(&view), + TEST_CA_VALIDATION_CONTEXT, + TEST_POLICY_FINGERPRINT, + ), + ) { + ParallelObjectsPrepare::Staged(stage) => stage, + ParallelObjectsPrepare::Complete(_) => panic!("expected staged ROA fallback"), + }; + + assert_eq!(stage.roa_cache_stats.blocked_roas, 1); + assert_eq!(stage.roa_cache_stats.fresh_roas, 1); + assert_eq!(stage.roa_task_indices, vec![1]); + assert_eq!(stage.roa_task_count(), 1); + } + + #[test] + fn merge_as_intervals_merges_overlapping_and_adjacent() { + let v = vec![(1, 2), (3, 5), (10, 10), (11, 12)]; + let merged = merge_as_intervals(&v); + assert_eq!(merged, vec![(1, 5), (10, 12)]); + } + + #[test] + fn as_choice_subset_rejects_inherit() { + let child = Some(&AsIdentifierChoice::Inherit); + let parent = Some(&AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { min: 1, max: 10 }, + ])); + assert!(!as_choice_subset(child, parent)); + } + + #[test] + fn as_choice_subset_checks_ranges() { + let child = Some(&AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Id(5), + AsIdOrRange::Range { min: 7, max: 9 }, + ])); + let parent = Some(&AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { min: 1, max: 10 }, + ])); + assert!(as_choice_subset(child, parent)); + } + + #[test] + fn ip_resources_is_subset_accepts_prefixes_and_ranges() { + let parent = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![ + IpAddressOrRange::Prefix(IpPrefix { + afi: Afi::Ipv4, + prefix_len: 8, + addr: vec![10, 0, 0, 0], + }), + IpAddressOrRange::Range(IpAddressRange { + min: vec![192, 0, 2, 0], + max: vec![192, 0, 2, 255], + }), + ]), + }], + }; + let child = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![ + IpAddressOrRange::Prefix(IpPrefix { + afi: Afi::Ipv4, + prefix_len: 16, + addr: vec![10, 1, 0, 0], + }), + IpAddressOrRange::Range(IpAddressRange { + min: vec![192, 0, 2, 10], + max: vec![192, 0, 2, 20], + }), + ]), + }], + }; + + assert!(ip_resources_is_subset(&child, &parent)); + } + + #[test] + fn ip_resources_is_subset_rejects_inherit_in_child() { + let parent = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv6, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( + IpPrefix { + afi: Afi::Ipv6, + prefix_len: 32, + addr: vec![0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + }, + )]), + }], + }; + let child = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv6, + choice: IpAddressChoice::Inherit, + }], + }; + assert!(!ip_resources_is_subset(&child, &parent)); + } + + #[test] + fn increment_bytes_wraps_all_ff_to_zero() { + assert_eq!(increment_bytes(&[0xFF, 0xFF]), vec![0x00, 0x00]); + } + + #[test] + fn merge_ip_intervals_merges_contiguous() { + let mut v = vec![ + (vec![0, 0, 0, 0], vec![0, 0, 0, 10]), + (vec![0, 0, 0, 11], vec![0, 0, 0, 20]), + ]; + merge_ip_intervals_in_place(&mut v); + assert_eq!(v, vec![(vec![0, 0, 0, 0], vec![0, 0, 0, 20])]); + } + + #[test] + fn choose_crl_for_certificate_reports_missing_crl_in_snapshot() { + let roa_der = + fixture_bytes("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"); + let roa = RoaObject::decode_der(&roa_der).expect("decode roa"); + let ee_crldp_uris = roa.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .extensions + .crl_distribution_points_uris + .as_ref(); + let crl_cache: HashMap = HashMap::new(); + let err = choose_crl_uri_for_certificate(ee_crldp_uris, &crl_cache).unwrap_err(); + assert!(matches!(err, ObjectValidateError::MissingCrlInPack)); + } + + #[test] + fn choose_crl_for_certificate_reports_missing_crldp_uris() { + let mut crl_cache: HashMap = HashMap::new(); + crl_cache.insert( + "rsync://example.test/a.crl".to_string(), + CachedIssuerCrl::Pending { + bytes: vec![0x01], + sha256_hex: None, + }, + ); + let err = choose_crl_uri_for_certificate(None, &crl_cache).unwrap_err(); + assert!(matches!(err, ObjectValidateError::MissingCrlDpUris)); + } + + #[test] + fn choose_crl_for_certificate_prefers_matching_crldp_uri_in_order() { + let roa_der = + fixture_bytes("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"); + let roa = RoaObject::decode_der(&roa_der).expect("decode roa"); + let ee_crldp_uris = roa.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .extensions + .crl_distribution_points_uris + .as_ref() + .expect("fixture ee has crldp"); + + let matching_uri = ee_crldp_uris[0].as_str().to_string(); + let mut crl_cache: HashMap = HashMap::new(); + crl_cache.insert( + "rsync://example.test/other.crl".to_string(), + CachedIssuerCrl::Pending { + bytes: vec![0x00], + sha256_hex: None, + }, + ); + crl_cache.insert( + matching_uri.clone(), + CachedIssuerCrl::Pending { + bytes: vec![0x01], + sha256_hex: None, + }, + ); + + let uri = choose_crl_uri_for_certificate(Some(ee_crldp_uris), &crl_cache).unwrap(); + assert_eq!(uri, matching_uri); + } + + #[test] + fn choose_crl_for_certificate_reports_not_found_when_crldp_does_not_match_snapshot() { + let roa_der = + fixture_bytes("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"); + let roa = RoaObject::decode_der(&roa_der).expect("decode roa"); + let ee_crldp_uris = roa.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .extensions + .crl_distribution_points_uris + .as_ref(); + + let mut crl_cache: HashMap = HashMap::new(); + crl_cache.insert( + "rsync://example.test/other.crl".to_string(), + CachedIssuerCrl::Pending { + bytes: vec![0x01], + sha256_hex: None, + }, + ); + let err = choose_crl_uri_for_certificate(ee_crldp_uris, &crl_cache).unwrap_err(); + assert!(matches!(err, ObjectValidateError::CrlNotFound(_))); + } + + #[test] + fn validate_ee_resources_subset_reports_missing_issuer_effective_ip() { + let roa_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"); + let roa_der = std::fs::read(roa_path).expect("read roa"); + let roa = RoaObject::decode_der(&roa_der).expect("decode roa"); + let ee = &roa.signed_object.signed_data.certificates[0].resource_cert; + + let idx = IssuerResourcesIndex::default(); + let err = validate_ee_resources_subset(ee, None, None, &idx).unwrap_err(); + assert!(matches!(err, ObjectValidateError::MissingIssuerEffectiveIp)); + } + + #[test] + fn validate_ee_resources_subset_reports_missing_issuer_effective_as() { + let aspa_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/5m80fwYws_3FiFD7JiQjAqZ1RYQ.asa", + ); + let aspa_der = std::fs::read(aspa_path).expect("read aspa"); + let aspa = AspaObject::decode_der(&aspa_der).expect("decode aspa"); + let ee = &aspa.signed_object.signed_data.certificates[0].resource_cert; + + let issuer_ip = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv6, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( + IpPrefix { + afi: Afi::Ipv6, + prefix_len: 32, + addr: vec![0x20, 0x01, 0x0d, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + }, + )]), + }], + }; + + let idx = build_issuer_resources_index(Some(&issuer_ip), None); + let err = validate_ee_resources_subset(ee, Some(&issuer_ip), None, &idx).unwrap_err(); + assert!(matches!(err, ObjectValidateError::MissingIssuerEffectiveAs)); + } + + #[test] + fn validate_ee_resources_subset_reports_not_subset() { + let roa_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"); + let roa_der = std::fs::read(roa_path).expect("read roa"); + let roa = RoaObject::decode_der(&roa_der).expect("decode roa"); + let ee = &roa.signed_object.signed_data.certificates[0].resource_cert; + + // Unrelated parent resources. + let issuer_ip = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv6, + choice: IpAddressChoice::AddressesOrRanges(vec![IpAddressOrRange::Prefix( + IpPrefix { + afi: Afi::Ipv6, + prefix_len: 32, + addr: vec![0x26, 0x20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + }, + )]), + }], + }; + + let idx = build_issuer_resources_index(Some(&issuer_ip), None); + let err = validate_ee_resources_subset(ee, Some(&issuer_ip), None, &idx).unwrap_err(); + assert!(matches!(err, ObjectValidateError::EeResourcesNotSubset)); + } + + #[test] + fn validation_update_03_ee_resources_reduce_to_vrs_for_roa_checks() { + let roa_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"); + let roa_der = std::fs::read(roa_path).expect("read roa"); + let roa = RoaObject::decode_der(&roa_der).expect("decode roa"); + let ee = &roa.signed_object.signed_data.certificates[0].resource_cert; + let ee_ip = ee + .tbs + .extensions + .ip_resources + .as_ref() + .expect("fixture EE has IP resources"); + let first_family = ee_ip.families.first().expect("family"); + let first_item = match &first_family.choice { + IpAddressChoice::AddressesOrRanges(items) => items.first().expect("item").clone(), + IpAddressChoice::Inherit => panic!("fixture should not inherit"), + }; + let issuer_ip = IpResourceSet { + families: vec![IpAddressFamily { + afi: first_family.afi, + choice: IpAddressChoice::AddressesOrRanges(vec![first_item]), + }], + }; + let idx = build_issuer_resources_index(Some(&issuer_ip), None); + + let strict_err = validate_ee_resources_for_mode( + ee, + Some(&issuer_ip), + None, + &idx, + ResourceValidationMode::Rfc6487, + ) + .unwrap_err(); + assert!(matches!( + strict_err, + ObjectValidateError::EeResourcesNotSubset + )); + + let vrs = validate_ee_resources_for_mode( + ee, + Some(&issuer_ip), + None, + &idx, + ResourceValidationMode::ValidationUpdate03, + ) + .expect("vrs resource resolution"); + assert_eq!(vrs.ip.expect("vrs ip").families.len(), 1); + } + + #[test] + fn validation_update_03_ee_as_vrs_filters_aspa_customer_resources() { + let issuer_as = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { + min: 64500, + max: 64510, + }, + ])), + rdi: None, + }; + let child_as = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { + min: 64505, + max: 64520, + }, + ])), + rdi: None, + }; + let idx = build_issuer_resources_index(None, Some(&issuer_as)); + let vrs = intersect_ee_as_resources_vrs(&child_as, Some(&issuer_as), &idx) + .expect("intersect AS resources"); + assert!(as_resource_set_contains_asn(&vrs, 64505)); + assert!(as_resource_set_contains_asn(&vrs, 64510)); + assert!(!as_resource_set_contains_asn(&vrs, 64511)); + } + + #[test] + fn validation_update_03_empty_ee_vrs_rejects_roa_and_aspa_outputs() { + let roa_der = + fixture_bytes("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"); + let roa = RoaObject::decode_der(&roa_der).expect("decode roa"); + let empty_ip_vrs = IpResourceSet { + families: Vec::new(), + }; + let roa_err = roa_to_vrps_with_vrs(&roa, Some(&empty_ip_vrs)).unwrap_err(); + assert!(matches!(roa_err, ObjectValidateError::EeResourcesNotSubset)); + let missing_ip_err = roa_to_vrps_with_vrs(&roa, None).unwrap_err(); + assert!(matches!( + missing_ip_err, + ObjectValidateError::EeResourcesNotSubset + )); + + let aspa_der = fixture_bytes( + "tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/5m80fwYws_3FiFD7JiQjAqZ1RYQ.asa", + ); + let aspa = AspaObject::decode_der(&aspa_der).expect("decode aspa"); + let empty_as_vrs = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(Vec::new())), + rdi: None, + }; + let aspa_err = validate_aspa_customer_in_vrs(&aspa, Some(&empty_as_vrs)).unwrap_err(); + assert!(matches!( + aspa_err, + ObjectValidateError::EeResourcesNotSubset + )); + let missing_as_err = validate_aspa_customer_in_vrs(&aspa, None).unwrap_err(); + assert!(matches!( + missing_as_err, + ObjectValidateError::EeResourcesNotSubset + )); + } + + #[test] + fn extra_rfc_refs_for_crl_selection_distinguishes_crl_errors() { + assert_eq!( + extra_rfc_refs_for_crl_selection(&ObjectValidateError::MissingCrlDpUris), + RFC_CRLDP + ); + assert_eq!( + extra_rfc_refs_for_crl_selection(&ObjectValidateError::CrlNotFound( + "rsync://example.test/x.crl".to_string(), + )), + RFC_CRLDP_AND_LOCKED_PACK + ); + assert!( + extra_rfc_refs_for_crl_selection(&ObjectValidateError::MissingCrlInPack).is_empty() + ); + } + + #[test] + fn as_subset_helpers_cover_success_and_failure_paths() { + let child = AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Id(5), + AsIdOrRange::Range { min: 7, max: 9 }, + ]); + let parent = + AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Range { min: 1, max: 10 }]); + let parent_intervals = [(1, 10)]; + + assert!(as_choice_subset(None, Some(&parent))); + assert!(!as_choice_subset(Some(&child), None)); + assert!(!as_choice_subset( + Some(&AsIdentifierChoice::Inherit), + Some(&parent) + )); + assert!(!as_choice_subset( + Some(&child), + Some(&AsIdentifierChoice::Inherit) + )); + assert!(as_choice_subset(Some(&child), Some(&parent))); + + assert!(as_choice_subset_indexed(None, Some(&parent_intervals))); + assert!(!as_choice_subset_indexed(Some(&child), None)); + assert!(!as_choice_subset_indexed( + Some(&AsIdentifierChoice::Inherit), + Some(&parent_intervals), + )); + assert!(as_choice_subset_indexed( + Some(&child), + Some(&parent_intervals) + )); + assert!(!as_choice_subset_indexed( + Some(&AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { min: 11, max: 12 } + ])), + Some(&parent_intervals), + )); + + let child_set = AsResourceSet { + asnum: Some(child.clone()), + rdi: Some(AsIdentifierChoice::AsIdsOrRanges(vec![AsIdOrRange::Id(42)])), + }; + let parent_set = AsResourceSet { + asnum: Some(parent.clone()), + rdi: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { min: 40, max: 50 }, + ])), + }; + assert!(as_resources_is_subset(&child_set, &parent_set)); + assert!(as_resources_is_subset_indexed( + &child_set, + &parent_set, + &IssuerResourcesIndex { + asnum: Some(vec![(1, 10)]), + rdi: Some(vec![(40, 50)]), + ..IssuerResourcesIndex::default() + }, + )); + } + + #[test] + fn ip_subset_helpers_cover_strict_and_indexed_paths() { + let parent = IpResourceSet { + families: vec![ + IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![ + IpAddressOrRange::Prefix(IpPrefix { + afi: Afi::Ipv4, + prefix_len: 8, + addr: vec![10, 0, 0, 0], + }), + IpAddressOrRange::Range(IpAddressRange { + min: vec![192, 0, 2, 0], + max: vec![192, 0, 2, 255], + }), + ]), + }, + IpAddressFamily { + afi: Afi::Ipv6, + choice: IpAddressChoice::Inherit, + }, + ], + }; + let child = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::AddressesOrRanges(vec![ + IpAddressOrRange::Prefix(IpPrefix { + afi: Afi::Ipv4, + prefix_len: 16, + addr: vec![10, 1, 0, 0], + }), + IpAddressOrRange::Range(IpAddressRange { + min: vec![192, 0, 2, 10], + max: vec![192, 0, 2, 20], + }), + ]), + }], + }; + let strict_bad = IpResourceSet { + families: vec![IpAddressFamily { + afi: Afi::Ipv4, + choice: IpAddressChoice::Inherit, + }], + }; + + assert!(ip_resources_is_subset(&child, &parent)); + assert!(ip_resources_to_merged_intervals(&parent).contains_key(&AfiKey::V4)); + assert!(ip_resources_to_merged_intervals_strict(&child).is_ok()); + assert!(ip_resources_to_merged_intervals_strict(&strict_bad).is_err()); + + let idx = build_issuer_resources_index( + Some(&parent), + Some(&AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { + min: 64500, + max: 64510, + }, + ])), + rdi: Some(AsIdentifierChoice::Inherit), + }), + ); + assert!(idx.ip_v4.is_some()); + assert!(idx.ip_v6.is_none()); + assert!(idx.asnum.is_some()); + assert!(idx.rdi.is_none()); + assert!(ip_resources_is_subset_indexed(&child, &parent, &idx)); + assert!(!ip_resources_is_subset_indexed(&strict_bad, &parent, &idx)); + } + + #[test] + fn interval_and_byte_helpers_cover_edge_cases() { + let parent = vec![(vec![0, 0, 0, 0], vec![0, 0, 0, 10])]; + assert!(interval_is_covered(&parent, &[0, 0, 0, 1], &[0, 0, 0, 2])); + assert!(!interval_is_covered( + &parent, + &[0, 0, 0, 11], + &[0, 0, 0, 12] + )); + assert!(intervals_are_covered( + &parent, + &[(vec![0, 0, 0, 1], vec![0, 0, 0, 2])] + )); + assert!(!intervals_are_covered( + &parent, + &[(vec![0, 0, 0, 9], vec![0, 0, 0, 11])], + )); + + let prefix = RcIpPrefix { + afi: Afi::Ipv4, + prefix_len: 24, + addr: vec![203, 0, 113, 7], + }; + assert_eq!( + prefix_to_range(&prefix), + (vec![203, 0, 113, 0], vec![203, 0, 113, 255]) + ); + assert_eq!(increment_bytes(&[0, 0, 0, 255]), vec![0, 0, 1, 0]); + assert!(bytes_is_next(&[0, 0, 1, 0], &[0, 0, 0, 255])); + assert!(!bytes_is_next(&[1, 2], &[1])); + } + + #[test] + fn merged_interval_helpers_cover_empty_and_break_paths() { + let mut empty: Vec<(Vec, Vec)> = Vec::new(); + merge_ip_intervals_in_place(&mut empty); + assert!(empty.is_empty()); + + let mut v = vec![ + (vec![0, 0, 0, 20], vec![0, 0, 0, 30]), + (vec![0, 0, 0, 0], vec![0, 0, 0, 10]), + (vec![0, 0, 0, 11], vec![0, 0, 0, 19]), + ]; + v.sort_by(|(a, _), (b, _)| a.cmp(b)); + merge_ip_intervals_in_place(&mut v); + assert_eq!(v, vec![(vec![0, 0, 0, 0], vec![0, 0, 0, 30])]); + + assert_eq!( + as_choice_to_merged_intervals(&AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Id(1), + AsIdOrRange::Range { min: 2, max: 3 }, + AsIdOrRange::Range { min: 7, max: 9 }, + ])), + vec![(1, 3), (7, 9)] + ); + assert!(as_interval_is_covered(&[(1, 3), (7, 9)], 2, 3)); + assert!(!as_interval_is_covered(&[(7, 9)], 2, 3)); + } + + #[test] + fn roa_output_helpers_cover_vrps_and_afi_strings() { + let roa_der = + fixture_bytes("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"); + let roa = RoaObject::decode_der(&roa_der).expect("decode roa"); + let vrps = roa_to_vrps(&roa); + assert!(!vrps.is_empty()); + assert!(vrps.iter().all(|vrp| vrp.asn == roa.roa.as_id)); + assert_eq!(roa_afi_to_string(RoaAfi::Ipv4), "ipv4"); + assert_eq!(roa_afi_to_string(RoaAfi::Ipv6), "ipv6"); + } + + #[test] + fn parallel_stage_roa_tasks_share_stage_owned_payloads() { + let ta_constraints = Arc::new( + crate::ta_constraints::TaConstraints::from_file( + &std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/ta_constraints/afrinic-current-ipv4-deny.constraints"), + ) + .expect("load constraints fixture"), + ); + let stage = ParallelObjectsStage { + publication_point_id: 7, + shared: Arc::new(RoaTaskShared { + locked_files: Arc::<[PackFile]>::from(vec![ + PackFile::from_bytes_with_sha256( + "rsync://example.test/repo/a.roa", + vec![1, 2, 3], + [1u8; 32], + ), + PackFile::from_bytes_with_sha256( + "rsync://example.test/repo/b.roa", + vec![4, 5, 6], + [2u8; 32], + ), + ]), + manifest_rsync_uri: Arc::::from("rsync://example.test/repo/manifest.mft"), + issuer_ca_der: Arc::from([0x01u8].as_slice()), + issuer_ca: Arc::new( + ResourceCertificate::decode_der(&fixture_bytes( + "tests/fixtures/ta/apnic-ta.cer", + )) + .expect("decode fixture CA certificate"), + ), + issuer_spki_der: Arc::from([0x02u8].as_slice()), + issuer_ca_rsync_uri: Some(Arc::::from("rsync://example.test/repo/ca.cer")), + crl_cache: Arc::new(Mutex::new(HashMap::new())), + issuer_resources_index: Arc::new(IssuerResourcesIndex::default()), + issuer_effective_ip: None, + issuer_effective_as: None, + resource_validation_mode: ResourceValidationMode::default(), + ta_constraints: Some(Arc::clone(&ta_constraints)), + }), + validation_time: OffsetDateTime::now_utc(), + collect_vcir_local_outputs: false, + strict_cms_der: false, + strict_name: false, + resource_validation_mode: ResourceValidationMode::default(), + roa_task_indices: vec![0, 1], + cached_roa_results: Vec::new(), + roa_cache_stats: RoaValidationCacheStats::default(), + warnings: Vec::new(), + stats: ObjectsStats { + roa_total: 2, + ..ObjectsStats::default() + }, + audit: Vec::new(), + }; + let tasks = stage.build_roa_tasks(); + assert_eq!(tasks.len(), 2); + assert!(Arc::ptr_eq(&tasks[0].shared, &tasks[1].shared)); + assert!(Arc::ptr_eq( + tasks[0] + .shared + .ta_constraints + .as_ref() + .expect("task constraints snapshot"), + &ta_constraints + )); + } + + #[test] + fn strict_name_manifest_decode_failure_drops_publication_point() { + let publication_point = PublicationPointSnapshot { + format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, + manifest_rsync_uri: "rsync://example.test/repo/manifest.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), + manifest_number_be: vec![0x01], + this_update: PackTime::from_utc_offset_datetime(OffsetDateTime::now_utc()), + next_update: PackTime::from_utc_offset_datetime(OffsetDateTime::now_utc()), + verified_at: PackTime::from_utc_offset_datetime(OffsetDateTime::now_utc()), + manifest_bytes: vec![0x01, 0x02, 0x03], + files: vec![], + }; + let policy = Policy::default(); + let output = process_publication_point_for_issuer_with_options( + &publication_point, + &policy, + &[], + None, + None, + None, + OffsetDateTime::now_utc(), + None, + false, + ); + assert!(output.stats.publication_point_dropped); + assert!(output.vrps.is_empty()); + assert!( + output + .warnings + .iter() + .any(|warning| warning.message.contains("manifest decode failed")), + "{:?}", + output.warnings + ); + } +} diff --git a/crates/panda-rpki-validator/src/validation/publication_point.rs b/crates/panda-rpki-validator/src/validation/publication_point.rs new file mode 100644 index 0000000..b490b7b --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/publication_point.rs @@ -0,0 +1,18 @@ +use crate::storage::{PackFile, PackTime}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PublicationPointSnapshot { + pub format_version: u32, + pub manifest_rsync_uri: String, + pub publication_point_rsync_uri: String, + pub manifest_number_be: Vec, + pub this_update: PackTime, + pub next_update: PackTime, + pub verified_at: PackTime, + pub manifest_bytes: Vec, + pub files: Vec, +} + +impl PublicationPointSnapshot { + pub const FORMAT_VERSION_V1: u32 = 1; +} diff --git a/crates/panda-rpki-validator/src/validation/run.rs b/crates/panda-rpki-validator/src/validation/run.rs new file mode 100644 index 0000000..5ac5a17 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/run.rs @@ -0,0 +1,97 @@ +use crate::data_model::rc::{AsResourceSet, IpResourceSet}; +use crate::fetch::rsync::RsyncFetcher; +use crate::policy::Policy; +use crate::report::Warning; +use crate::storage::RocksStore; +use crate::sync::rrdp::Fetcher as HttpFetcher; +use crate::validation::manifest::PublicationPointSource; +use crate::validation::objects::ObjectsOutput; +use crate::validation::tree::{CaCertificateRef, CaInstanceHandle, PublicationPointRunner}; +use crate::validation::tree_runner::Rpkiv1PublicationPointRunner; +use std::collections::HashMap; +use std::sync::Mutex; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RunOutput { + pub publication_point_source: PublicationPointSource, + pub publication_point_warnings: Vec, + pub objects: ObjectsOutput, +} + +#[derive(Debug, thiserror::Error)] +pub enum RunError { + #[error("publication point runner failed: {0}")] + Runner(String), +} + +/// Offline-friendly end-to-end execution for a single publication point. +/// +/// This reuses the same fresh-first runtime path as the tree runner: +/// 1) repository sync (RRDP or rsync fallback) into the current repository view +/// 2) manifest processing into a validated publication point result +/// 3) signed object processing (ROA/ASPA) and latest-VCIR persistence +pub fn run_publication_point_once( + store: &RocksStore, + policy: &Policy, + rrdp_notification_uri: Option<&str>, + rsync_base_uri: &str, + manifest_rsync_uri: &str, + publication_point_rsync_uri: &str, + http_fetcher: &dyn HttpFetcher, + rsync_fetcher: &dyn RsyncFetcher, + issuer_ca_der: &[u8], + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&IpResourceSet>, + issuer_effective_as: Option<&AsResourceSet>, + validation_time: time::OffsetDateTime, +) -> Result { + let handle = CaInstanceHandle { + depth: 0, + tal_id: "single-publication-point".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der.to_vec()), + ca_certificate_rsync_uri: issuer_ca_rsync_uri.map(str::to_string), + effective_ip_resources: issuer_effective_ip.cloned(), + effective_as_resources: issuer_effective_as.cloned(), + rsync_base_uri: rsync_base_uri.to_string(), + manifest_rsync_uri: manifest_rsync_uri.to_string(), + publication_point_rsync_uri: publication_point_rsync_uri.to_string(), + rrdp_notification_uri: rrdp_notification_uri.map(str::to_string), + }; + + let runner = Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher, + rsync_fetcher, + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + }; + + let result = runner + .run_publication_point(&handle) + .map_err(RunError::Runner)?; + + Ok(RunOutput { + publication_point_source: result.source, + publication_point_warnings: result.warnings, + objects: result.objects, + }) +} diff --git a/crates/panda-rpki-validator/src/validation/run_tree_from_tal.rs b/crates/panda-rpki-validator/src/validation/run_tree_from_tal.rs new file mode 100644 index 0000000..1cf2605 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/run_tree_from_tal.rs @@ -0,0 +1,3045 @@ +use url::Url; + +use crate::analysis::timing::TimingHandle; +use crate::audit::PublicationPointAudit; +use crate::audit_downloads::DownloadLogHandle; +use crate::ccr::CcrAccumulator; +use crate::current_repo_index::{CurrentRepoIndexHandle, CurrentRepoObject}; +use crate::data_model::ta::TrustAnchor; +use crate::parallel::config::{ParallelPhase1Config, ParallelPhase2Config}; +use crate::parallel::repo_runtime::{Phase1RepoSyncRuntime, RepoSyncRuntime}; +use crate::parallel::repo_worker::{ + LiveRepoTransportExecutor, RepoTransportWorkerPool, RepoWorkerPoolConfig, +}; +use crate::parallel::run_coordinator::GlobalRunCoordinator; +use crate::parallel::types::{TalInputSpec, TalSource}; +use crate::policy::SyncPreference; +use crate::replay::archive::ReplayArchiveIndex; +use crate::replay::delta_archive::ReplayDeltaArchiveIndex; +use crate::replay::delta_fetch_http::PayloadDeltaReplayHttpFetcher; +use crate::replay::delta_fetch_rsync::{ + PayloadDeltaReplayCurrentStoreRsyncFetcher, PayloadDeltaReplayRsyncFetcher, +}; +use crate::replay::fetch_http::PayloadReplayHttpFetcher; +use crate::replay::fetch_rsync::PayloadReplayRsyncFetcher; +use crate::sync::rrdp::Fetcher; +use crate::validation::from_tal::{ + DiscoveredRootCaInstance, FromTalError, canonical_tal_rsync_uri_from_bytes, + discover_root_ca_instance_from_tal_and_ta_der, + discover_root_ca_instance_from_tal_and_ta_der_with_strict_name, + discover_root_ca_instance_from_tal_url_with_fetchers, + discover_root_ca_instance_from_tal_url_with_fetchers_strict_name, + discover_root_ca_instance_from_tal_with_fetchers, + discover_root_ca_instance_from_tal_with_fetchers_strict_name, +}; +use crate::validation::objects::ParallelRoaWorkerPool; +use crate::validation::tree::CaCertificateRef; +use crate::validation::tree::{ + CaInstanceHandle, TreeRunAuditOutput, TreeRunConfig, TreeRunError, TreeRunOutput, + run_tree_serial, run_tree_serial_audit, run_tree_serial_audit_multi_root, +}; +use crate::validation::tree_parallel::{ + run_tree_parallel_phase2_audit, run_tree_parallel_phase2_audit_multi_root, +}; +use crate::validation::tree_runner::Rpkiv1PublicationPointRunner; +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; + +fn tal_id_from_url_like(s: &str) -> Option { + let url = Url::parse(s).ok()?; + if let Some(last) = url + .path_segments() + .and_then(|segments| segments.filter(|seg| !seg.is_empty()).next_back()) + { + let stem = last.rsplit_once('.').map(|(stem, _)| stem).unwrap_or(last); + let trimmed = stem.trim(); + if !trimmed.is_empty() { + return Some(trimmed.to_string()); + } + } + url.host_str().map(|host| host.to_string()) +} + +fn derive_tal_id(discovery: &DiscoveredRootCaInstance) -> String { + discovery + .tal_url + .as_deref() + .and_then(tal_id_from_url_like) + .or_else(|| { + discovery + .trust_anchor + .resolved_ta_uri + .as_ref() + .and_then(|uri| tal_id_from_url_like(uri.as_str())) + }) + .or_else(|| { + discovery + .trust_anchor + .tal + .ta_uris + .first() + .and_then(|uri| tal_id_from_url_like(uri.as_str())) + }) + .unwrap_or_else(|| "unknown-tal".to_string()) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RunTreeFromTalOutput { + pub discovery: DiscoveredRootCaInstance, + pub tree: TreeRunOutput, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RunTreeFromTalAuditOutput { + pub discovery: DiscoveredRootCaInstance, + pub discoveries: Vec, + pub successful_tal_inputs: Vec, + pub tree: TreeRunOutput, + pub publication_points: Vec, + pub roa_cache_stats: crate::validation::objects::RoaValidationCacheStats, + pub cir_input: crate::cir::CirInputSnapshot, + pub downloads: Vec, + pub download_stats: crate::audit::AuditDownloadStats, + pub current_repo_objects: Vec, + pub ccr_accumulator: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TalRootDiscovery { + pub tal_input: TalInputSpec, + pub discovery: DiscoveredRootCaInstance, + pub root_handle: CaInstanceHandle, +} + +fn snapshot_current_repo_objects( + current_repo_index: Option<&CurrentRepoIndexHandle>, + collect: bool, +) -> Vec { + if !collect { + return Vec::new(); + } + current_repo_index + .and_then(|handle| handle.read().ok().map(|idx| idx.snapshot_objects())) + .unwrap_or_default() +} + +fn make_live_runner<'a>( + store: &'a crate::storage::RocksStore, + policy: &'a crate::policy::Policy, + http_fetcher: &'a dyn Fetcher, + rsync_fetcher: &'a dyn crate::fetch::rsync::RsyncFetcher, + validation_time: time::OffsetDateTime, + timing: Option, + download_log: Option, + current_repo_index: Option, + repo_sync_runtime: Option>, + parallel_phase2_config: Option, + ccr_accumulator: Option, + persist_vcir: bool, + enable_roa_validation_cache: bool, + enable_child_certificate_validation_cache: bool, + publication_point_cache_observe_only: bool, + enable_publication_point_validation_cache: bool, +) -> Rpkiv1PublicationPointRunner<'a> { + let parallel_roa_worker_pool = parallel_phase2_config + .as_ref() + .and_then(|config| ParallelRoaWorkerPool::new(config).ok()); + Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher, + rsync_fetcher, + validation_time, + timing, + download_log, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index, + repo_sync_runtime, + parallel_phase2_config, + parallel_roa_worker_pool, + ccr_accumulator: ccr_accumulator.map(Mutex::new), + persist_vcir, + enable_roa_validation_cache, + enable_child_certificate_validation_cache, + publication_point_cache_observe_only, + enable_publication_point_validation_cache, + } +} + +fn build_phase1_repo_sync_runtime( + store: Arc, + policy: &crate::policy::Policy, + http_fetcher: &H, + rsync_fetcher: &R, + parallel_config: ParallelPhase1Config, + timing: Option, + download_log: Option, + tal_inputs: Vec, + record_transport_prefetch_requests: bool, +) -> Result<(Arc, CurrentRepoIndexHandle), RunTreeFromTalError> +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + let coordinator = GlobalRunCoordinator::new(parallel_config.clone(), tal_inputs); + let current_repo_index = coordinator.current_repo_index_handle(); + let rsync_fetcher_arc = Arc::new(rsync_fetcher.clone()); + let executor = LiveRepoTransportExecutor::new( + Arc::clone(&store), + current_repo_index.clone(), + Arc::new(http_fetcher.clone()), + Arc::clone(&rsync_fetcher_arc), + timing.clone(), + download_log, + ); + let pool = RepoTransportWorkerPool::new(RepoWorkerPoolConfig::from(¶llel_config), executor) + .map_err(RunTreeFromTalError::Replay)?; + let resolver_rsync_fetcher_arc = Arc::clone(&rsync_fetcher_arc); + let resolver: Arc String + Send + Sync> = + Arc::new(move |base: &str| resolver_rsync_fetcher_arc.dedup_key(base)); + let failure_rsync_fetcher_arc = Arc::clone(&rsync_fetcher_arc); + let failure_resolver: Arc Option + Send + Sync> = + Arc::new(move |base: &str| failure_rsync_fetcher_arc.failure_dedup_key(base)); + let _ = policy; // policy reserved for later runtime-level decisions + let runtime = Arc::new( + Phase1RepoSyncRuntime::new_with_failure_scope_and_prefetch_recording( + coordinator, + pool, + resolver, + failure_resolver, + policy.sync_preference, + record_transport_prefetch_requests, + ), + ); + Ok((runtime, current_repo_index)) +} + +fn record_transport_prefetch_count(timing: Option<&TimingHandle>, key: &'static str, value: u64) { + if value == 0 { + return; + } + if let Some(timing) = timing { + timing.record_count(key, value); + } +} + +fn apply_transport_request_prefetch( + store: &crate::storage::RocksStore, + runtime: &Arc, + sync_preference: SyncPreference, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + timing: Option<&TimingHandle>, +) -> Result<(), RunTreeFromTalError> { + if !config.enable_transport_request_prefetch { + return Ok(()); + } + + let snapshot = match store.get_transport_prefetch_snapshot() { + Ok(Some(snapshot)) => snapshot, + Ok(None) => { + crate::progress_log::emit( + "phase1_repo_prefetch_missing_snapshot", + serde_json::json!({ "status": "missing" }), + ); + return Ok(()); + } + Err(err) => { + record_transport_prefetch_count(timing, "transport_prefetch_load_errors", 1); + crate::progress_log::emit( + "phase1_repo_prefetch_load_error", + serde_json::json!({ "error": err.to_string() }), + ); + return Ok(()); + } + }; + + if !snapshot.is_compatible_with(sync_preference) { + record_transport_prefetch_count( + timing, + "transport_prefetch_skipped_incompatible_snapshots", + 1, + ); + crate::progress_log::emit( + "phase1_repo_prefetch_skipped", + serde_json::json!({ + "reason": "incompatible_snapshot", + "schema_version": snapshot.schema_version, + "request_count": snapshot.requests.len(), + }), + ); + return Ok(()); + } + + let stats = runtime + .prefetch_transport_requests(&snapshot, validation_time) + .map_err(TreeRunError::Runner)?; + record_transport_prefetch_count( + timing.clone(), + "transport_prefetch_loaded_requests", + stats.loaded_requests, + ); + record_transport_prefetch_count( + timing.clone(), + "transport_prefetch_enqueued_tasks", + stats.enqueued_tasks, + ); + record_transport_prefetch_count( + timing, + "transport_prefetch_waiting_requests", + stats.waiting_requests, + ); + record_transport_prefetch_count( + timing, + "transport_prefetch_reused_results", + stats.reused_results, + ); + record_transport_prefetch_count( + timing, + "transport_prefetch_skipped_incompatible_requests", + stats.skipped_incompatible, + ); + crate::progress_log::emit( + "phase1_repo_prefetch_applied", + serde_json::json!({ + "loaded_requests": stats.loaded_requests, + "enqueued_tasks": stats.enqueued_tasks, + "waiting_requests": stats.waiting_requests, + "reused_results": stats.reused_results, + "skipped_incompatible": stats.skipped_incompatible, + }), + ); + Ok(()) +} + +fn persist_transport_request_prefetch_snapshot( + store: &crate::storage::RocksStore, + runtime: &Arc, + config: &TreeRunConfig, + timing: Option<&TimingHandle>, +) -> Result<(), RunTreeFromTalError> { + if !config.enable_transport_request_prefetch { + return Ok(()); + } + let snapshot = runtime.transport_prefetch_snapshot(); + let recorded = snapshot.requests.len() as u64; + let persist_started = std::time::Instant::now(); + store + .put_transport_prefetch_snapshot(&snapshot) + .map_err(|err| TreeRunError::Runner(err.to_string()))?; + let persist_ms = persist_started.elapsed().as_millis() as u64; + record_transport_prefetch_count(timing, "transport_prefetch_recorded_requests", recorded); + crate::progress_log::emit( + "phase1_repo_prefetch_persisted", + serde_json::json!({ + "recorded_requests": recorded, + "schema_version": snapshot.schema_version, + "persist_ms": persist_ms, + }), + ); + Ok(()) +} + +/// Persist the dead-repo blacklist working copy at run end (#141). Failures +/// degrade to a progress event; a broken blacklist file must never fail a run. +fn persist_dead_repo_blacklist( + runtime: &Arc, + timing: Option<&TimingHandle>, +) -> Result<(), RunTreeFromTalError> { + let Some((blacklist_config, blacklist)) = runtime.dead_repo_blacklist_state() else { + return Ok(()); + }; + let persist_started = std::time::Instant::now(); + let now_unix = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.as_secs()) + .unwrap_or(0); + match blacklist.store_atomic(&blacklist_config.path, now_unix) { + Ok(()) => { + let persist_ms = persist_started.elapsed().as_millis() as u64; + record_transport_prefetch_count( + timing, + "dead_repo_blacklist_entries", + blacklist.len() as u64, + ); + record_transport_prefetch_count( + timing, + "dead_repo_blacklist_blacklisted", + blacklist.blacklisted_len() as u64, + ); + crate::progress_log::emit( + "dead_repo_blacklist_persisted", + serde_json::json!({ + "path": blacklist_config.path.display().to_string(), + "entries": blacklist.len(), + "blacklisted": blacklist.blacklisted_len(), + "persist_ms": persist_ms, + }), + ); + } + Err(err) => { + crate::progress_log::emit( + "dead_repo_blacklist_persist_error", + serde_json::json!({ + "path": blacklist_config.path.display().to_string(), + "error": err.to_string(), + }), + ); + } + } + Ok(()) +} + +fn root_discovery_from_tal_input( + tal_input: &TalInputSpec, + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, + strict_name: bool, +) -> Result { + match &tal_input.source { + TalSource::Url(url) => { + if strict_name { + discover_root_ca_instance_from_tal_url_with_fetchers_strict_name( + http_fetcher, + rsync_fetcher, + url, + ) + } else { + discover_root_ca_instance_from_tal_url_with_fetchers( + http_fetcher, + rsync_fetcher, + url, + ) + } + } + TalSource::DerBytes { + tal_bytes, ta_der, .. + } => { + if strict_name { + discover_root_ca_instance_from_tal_and_ta_der_with_strict_name( + tal_bytes, ta_der, None, + ) + } else { + discover_root_ca_instance_from_tal_and_ta_der(tal_bytes, ta_der, None) + } + } + TalSource::FilePath(path) => { + let tal_bytes = std::fs::read(path).map_err(|e| { + FromTalError::TalFetch(format!("read TAL file failed: {}: {e}", path.display())) + })?; + let tal = crate::data_model::tal::Tal::decode_bytes(&tal_bytes) + .map_err(FromTalError::from)?; + if strict_name { + discover_root_ca_instance_from_tal_with_fetchers_strict_name( + http_fetcher, + rsync_fetcher, + tal, + None, + ) + } else { + discover_root_ca_instance_from_tal_with_fetchers( + http_fetcher, + rsync_fetcher, + tal, + None, + ) + } + } + TalSource::FilePathWithTa { tal_path, ta_path } => { + let tal_bytes = std::fs::read(tal_path).map_err(|e| { + FromTalError::TalFetch(format!("read TAL file failed: {}: {e}", tal_path.display())) + })?; + let ta_der = std::fs::read(ta_path).map_err(|e| { + FromTalError::TaFetch(format!("read TA file failed: {}: {e}", ta_path.display())) + })?; + let resolved_ta_uri = canonical_tal_rsync_uri_from_bytes(&tal_bytes)?; + if strict_name { + discover_root_ca_instance_from_tal_and_ta_der_with_strict_name( + &tal_bytes, + &ta_der, + Some(&resolved_ta_uri), + ) + } else { + discover_root_ca_instance_from_tal_and_ta_der( + &tal_bytes, + &ta_der, + Some(&resolved_ta_uri), + ) + } + } + } +} + +fn discover_root_ca_instance_from_tal_url_with_policy( + policy: &crate::policy::Policy, + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, + tal_url: &str, +) -> Result { + if policy.strict.name { + discover_root_ca_instance_from_tal_url_with_fetchers_strict_name( + http_fetcher, + rsync_fetcher, + tal_url, + ) + } else { + discover_root_ca_instance_from_tal_url_with_fetchers(http_fetcher, rsync_fetcher, tal_url) + } +} + +fn discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&Url>, +) -> Result { + if policy.strict.name { + discover_root_ca_instance_from_tal_and_ta_der_with_strict_name( + tal_bytes, + ta_der, + resolved_ta_uri, + ) + } else { + discover_root_ca_instance_from_tal_and_ta_der(tal_bytes, ta_der, resolved_ta_uri) + } +} + +fn discover_multiple_roots_from_tal_inputs( + tal_inputs: &[TalInputSpec], + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, + strict_name: bool, +) -> Result, RunTreeFromTalError> { + let mut roots = Vec::with_capacity(tal_inputs.len()); + for tal_input in tal_inputs { + let discovery = match root_discovery_from_tal_input( + tal_input, + http_fetcher, + rsync_fetcher, + strict_name, + ) { + Ok(discovery) => discovery, + Err(error) + if should_isolate_multi_tal_strict_name_failure( + tal_inputs.len(), + strict_name, + &error, + ) => + { + eprintln!( + "warning: skipping TAL '{}' because strict name validation failed during trust anchor discovery: {error}", + tal_input.tal_id + ); + continue; + } + Err(error) => return Err(error.into()), + }; + let root_handle = root_handle_from_trust_anchor( + &discovery.trust_anchor, + tal_input.tal_id.clone(), + None, + &discovery.ca_instance, + ); + roots.push(TalRootDiscovery { + tal_input: tal_input.clone(), + discovery, + root_handle, + }); + } + if roots.is_empty() { + return Err(RunTreeFromTalError::Replay( + "multi-TAL root discovery returned no usable roots after strict name filtering" + .to_string(), + )); + } + Ok(roots) +} + +fn should_isolate_multi_tal_strict_name_failure( + tal_input_count: usize, + strict_name: bool, + error: &FromTalError, +) -> bool { + strict_name + && tal_input_count > 1 + && error.to_string().contains("Name strict validation failed") +} + +#[derive(Debug, thiserror::Error)] +pub enum RunTreeFromTalError { + #[error("{0}")] + FromTal(#[from] FromTalError), + + #[error("payload replay setup failed: {0}")] + Replay(String), + + #[error("{0}")] + Tree(#[from] TreeRunError), +} + +pub fn root_handle_from_trust_anchor( + trust_anchor: &TrustAnchor, + tal_id: String, + ca_certificate_rsync_uri: Option, + ca_instance: &crate::validation::ca_instance::CaInstanceUris, +) -> CaInstanceHandle { + let ta_rc = trust_anchor.ta_certificate.rc_ca.clone(); + CaInstanceHandle { + depth: 0, + tal_id, + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(trust_anchor.ta_certificate.raw_der.clone()), + ca_certificate_rsync_uri, + effective_ip_resources: ta_rc.tbs.extensions.ip_resources.clone(), + effective_as_resources: ta_rc.tbs.extensions.as_resources.clone(), + rsync_base_uri: ca_instance.rsync_base_uri.clone(), + manifest_rsync_uri: ca_instance.manifest_rsync_uri.clone(), + publication_point_rsync_uri: ca_instance.publication_point_rsync_uri.clone(), + rrdp_notification_uri: ca_instance.rrdp_notification_uri.clone(), + } +} + +pub fn run_tree_from_tal_url_serial( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_url: &str, + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, +) -> Result { + let discovery = discover_root_ca_instance_from_tal_url_with_policy( + policy, + http_fetcher, + rsync_fetcher, + tal_url, + )?; + + let runner = make_live_runner( + store, + policy, + http_fetcher, + rsync_fetcher, + validation_time, + None, + None, + None, + None, + None, + None, + config.persist_vcir, + config.enable_roa_validation_cache, + config.enable_child_certificate_validation_cache, + config.publication_point_cache_observe_only, + config.enable_publication_point_validation_cache, + ); + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + let tree = run_tree_serial(root, &runner, config)?; + + Ok(RunTreeFromTalOutput { discovery, tree }) +} + +pub fn run_tree_from_tal_url_serial_audit( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_url: &str, + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, +) -> Result { + let discovery = discover_root_ca_instance_from_tal_url_with_policy( + policy, + http_fetcher, + rsync_fetcher, + tal_url, + )?; + + let download_log = DownloadLogHandle::new(); + let runner = make_live_runner( + store, + policy, + http_fetcher, + rsync_fetcher, + validation_time, + None, + Some(download_log.clone()), + None, + None, + None, + None, + config.persist_vcir, + config.enable_roa_validation_cache, + config.enable_child_certificate_validation_cache, + config.publication_point_cache_observe_only, + config.enable_publication_point_validation_cache, + ); + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = run_tree_serial_audit(root, &runner, config)?; + + let downloads = download_log.snapshot_events(); + let download_stats = DownloadLogHandle::stats_from_events(&downloads); + Ok(RunTreeFromTalAuditOutput { + discovery: discovery.clone(), + discoveries: vec![discovery], + successful_tal_inputs: Vec::new(), + tree, + publication_points, + roa_cache_stats, + cir_input, + downloads, + download_stats, + current_repo_objects: Vec::new(), + ccr_accumulator: None, + }) +} + +pub fn run_tree_from_tal_url_serial_audit_with_timing( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_url: &str, + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + timing: &TimingHandle, +) -> Result { + let _tal = timing.span_phase("tal_bootstrap"); + let discovery = discover_root_ca_instance_from_tal_url_with_policy( + policy, + http_fetcher, + rsync_fetcher, + tal_url, + )?; + drop(_tal); + + let download_log = DownloadLogHandle::new(); + let runner = make_live_runner( + store, + policy, + http_fetcher, + rsync_fetcher, + validation_time, + Some(timing.clone()), + Some(download_log.clone()), + None, + None, + None, + None, + config.persist_vcir, + config.enable_roa_validation_cache, + config.enable_child_certificate_validation_cache, + config.publication_point_cache_observe_only, + config.enable_publication_point_validation_cache, + ); + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + let _tree = timing.span_phase("tree_run_total"); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = run_tree_serial_audit(root, &runner, config)?; + + let downloads = download_log.snapshot_events(); + let download_stats = DownloadLogHandle::stats_from_events(&downloads); + Ok(RunTreeFromTalAuditOutput { + discovery: discovery.clone(), + discoveries: vec![discovery], + successful_tal_inputs: Vec::new(), + tree, + publication_points, + roa_cache_stats, + cir_input, + downloads, + download_stats, + current_repo_objects: Vec::new(), + ccr_accumulator: None, + }) +} + +fn run_single_root_parallel_audit_inner( + store: Arc, + policy: &crate::policy::Policy, + discovery: DiscoveredRootCaInstance, + tal_inputs: Vec, + http_fetcher: &H, + rsync_fetcher: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + parallel_config: ParallelPhase1Config, + phase2_config: Option, + collect_current_repo_objects: bool, + timing: Option, +) -> Result +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + let phase2_enabled = phase2_config.is_some(); + let download_log = DownloadLogHandle::new(); + let (runtime, current_repo_index) = build_phase1_repo_sync_runtime( + Arc::clone(&store), + policy, + http_fetcher, + rsync_fetcher, + parallel_config, + timing.clone(), + Some(download_log.clone()), + tal_inputs, + config.enable_transport_request_prefetch, + )?; + apply_transport_request_prefetch( + store.as_ref(), + &runtime, + policy.sync_preference, + validation_time, + config, + timing.as_ref(), + )?; + let current_repo_index_for_output = current_repo_index.clone(); + let runner = make_live_runner( + store.as_ref(), + policy, + http_fetcher, + rsync_fetcher, + validation_time, + timing.clone(), + Some(download_log.clone()), + Some(current_repo_index), + Some(Arc::clone(&runtime)), + phase2_config, + (phase2_enabled && config.build_ccr_accumulator) + .then(|| CcrAccumulator::new(vec![discovery.trust_anchor.clone()])), + config.persist_vcir, + config.enable_roa_validation_cache, + config.enable_child_certificate_validation_cache, + config.publication_point_cache_observe_only, + config.enable_publication_point_validation_cache, + ); + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = if phase2_enabled { + run_tree_parallel_phase2_audit(root, &runner, config)? + } else { + run_tree_serial_audit(root, &runner, config)? + }; + persist_transport_request_prefetch_snapshot(store.as_ref(), &runtime, config, timing.as_ref())?; + persist_dead_repo_blacklist(&runtime, timing.as_ref())?; + let downloads = download_log.snapshot_events(); + let download_stats = DownloadLogHandle::stats_from_events(&downloads); + Ok(RunTreeFromTalAuditOutput { + discovery: discovery.clone(), + discoveries: vec![discovery], + successful_tal_inputs: Vec::new(), + tree, + publication_points, + roa_cache_stats, + cir_input, + downloads, + download_stats, + current_repo_objects: snapshot_current_repo_objects( + Some(¤t_repo_index_for_output), + collect_current_repo_objects, + ), + ccr_accumulator: runner.ccr_accumulator_snapshot(), + }) +} + +fn run_multi_root_parallel_audit_inner( + store: Arc, + policy: &crate::policy::Policy, + tal_inputs: Vec, + http_fetcher: &H, + rsync_fetcher: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + parallel_config: ParallelPhase1Config, + phase2_config: Option, + collect_current_repo_objects: bool, + timing: Option, +) -> Result +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + // Constraints are an immutable per-run policy snapshot. The phase-2 + // ready-stage binds the snapshot to each CA's TAL and moves an Arc into + // ROA/ASPA worker state, so constrained multi-TAL runs retain the same + // parallel scheduler as unconstrained runs. + let phase2_enabled = phase2_config.is_some(); + if tal_inputs.is_empty() { + return Err(RunTreeFromTalError::Replay( + "multi-TAL run requires at least one TAL input".to_string(), + )); + } + let roots = discover_multiple_roots_from_tal_inputs( + &tal_inputs, + http_fetcher, + rsync_fetcher, + policy.strict.name, + )?; + let primary = roots.first().cloned().ok_or_else(|| { + RunTreeFromTalError::Replay("multi-TAL root discovery returned no roots".to_string()) + })?; + let discoveries = roots + .iter() + .map(|item| item.discovery.clone()) + .collect::>(); + let successful_tal_inputs = roots + .iter() + .map(|item| item.tal_input.clone()) + .collect::>(); + let root_handles = roots + .iter() + .map(|item| item.root_handle.clone()) + .collect::>(); + + let download_log = DownloadLogHandle::new(); + let (runtime, current_repo_index) = build_phase1_repo_sync_runtime( + Arc::clone(&store), + policy, + http_fetcher, + rsync_fetcher, + parallel_config, + timing.clone(), + Some(download_log.clone()), + successful_tal_inputs.clone(), + config.enable_transport_request_prefetch, + )?; + apply_transport_request_prefetch( + store.as_ref(), + &runtime, + policy.sync_preference, + validation_time, + config, + timing.as_ref(), + )?; + let current_repo_index_for_output = current_repo_index.clone(); + let runner = make_live_runner( + store.as_ref(), + policy, + http_fetcher, + rsync_fetcher, + validation_time, + timing.clone(), + Some(download_log.clone()), + Some(current_repo_index), + Some(Arc::clone(&runtime)), + phase2_config, + (phase2_enabled && config.build_ccr_accumulator).then(|| { + CcrAccumulator::new( + discoveries + .iter() + .map(|item| item.trust_anchor.clone()) + .collect::>(), + ) + }), + config.persist_vcir, + config.enable_roa_validation_cache, + config.enable_child_certificate_validation_cache, + config.publication_point_cache_observe_only, + config.enable_publication_point_validation_cache, + ); + + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = if phase2_enabled { + run_tree_parallel_phase2_audit_multi_root(root_handles, &runner, config)? + } else { + run_tree_serial_audit_multi_root(root_handles, &runner, config)? + }; + persist_transport_request_prefetch_snapshot(store.as_ref(), &runtime, config, timing.as_ref())?; + persist_dead_repo_blacklist(&runtime, timing.as_ref())?; + let downloads = download_log.snapshot_events(); + let download_stats = DownloadLogHandle::stats_from_events(&downloads); + Ok(RunTreeFromTalAuditOutput { + discovery: primary.discovery.clone(), + discoveries, + successful_tal_inputs, + tree, + publication_points, + roa_cache_stats, + cir_input, + downloads, + download_stats, + current_repo_objects: snapshot_current_repo_objects( + Some(¤t_repo_index_for_output), + collect_current_repo_objects, + ), + ccr_accumulator: runner.ccr_accumulator_snapshot(), + }) +} + +pub fn run_tree_from_tal_url_parallel_phase1_audit( + store: Arc, + policy: &crate::policy::Policy, + tal_url: &str, + http_fetcher: &H, + rsync_fetcher: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + parallel_config: ParallelPhase1Config, + collect_current_repo_objects: bool, +) -> Result +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + let discovery = discover_root_ca_instance_from_tal_url_with_policy( + policy, + http_fetcher, + rsync_fetcher, + tal_url, + )?; + run_single_root_parallel_audit_inner( + store, + policy, + discovery, + vec![TalInputSpec::from_url(tal_url.to_string())], + http_fetcher, + rsync_fetcher, + validation_time, + config, + parallel_config, + None, + collect_current_repo_objects, + None, + ) +} + +pub fn run_tree_from_tal_and_ta_der_parallel_phase1_audit( + store: Arc, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&url::Url>, + http_fetcher: &H, + rsync_fetcher: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + parallel_config: ParallelPhase1Config, + collect_current_repo_objects: bool, +) -> Result +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy, + tal_bytes, + ta_der, + resolved_ta_uri, + )?; + let derived_tal_id = derive_tal_id(&discovery); + let tal_inputs = vec![TalInputSpec { + tal_id: derived_tal_id.clone(), + rir_id: derived_tal_id, + source: TalSource::DerBytes { + tal_url: discovery + .tal_url + .clone() + .unwrap_or_else(|| "embedded-tal".to_string()), + tal_bytes: tal_bytes.to_vec(), + ta_der: ta_der.to_vec(), + }, + }]; + run_single_root_parallel_audit_inner( + store, + policy, + discovery, + tal_inputs, + http_fetcher, + rsync_fetcher, + validation_time, + config, + parallel_config, + None, + collect_current_repo_objects, + None, + ) +} + +pub fn run_tree_from_multiple_tals_parallel_phase1_audit( + store: Arc, + policy: &crate::policy::Policy, + tal_inputs: Vec, + http_fetcher: &H, + rsync_fetcher: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + parallel_config: ParallelPhase1Config, + collect_current_repo_objects: bool, +) -> Result +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + run_multi_root_parallel_audit_inner( + store, + policy, + tal_inputs, + http_fetcher, + rsync_fetcher, + validation_time, + config, + parallel_config, + None, + collect_current_repo_objects, + None, + ) +} + +pub fn run_tree_from_tal_url_parallel_phase2_audit( + store: Arc, + policy: &crate::policy::Policy, + tal_url: &str, + http_fetcher: &H, + rsync_fetcher: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + parallel_config: ParallelPhase1Config, + phase2_config: ParallelPhase2Config, + collect_current_repo_objects: bool, +) -> Result +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + let discovery = discover_root_ca_instance_from_tal_url_with_policy( + policy, + http_fetcher, + rsync_fetcher, + tal_url, + )?; + run_single_root_parallel_audit_inner( + store, + policy, + discovery, + vec![TalInputSpec::from_url(tal_url.to_string())], + http_fetcher, + rsync_fetcher, + validation_time, + config, + parallel_config, + Some(phase2_config), + collect_current_repo_objects, + None, + ) +} + +pub fn run_tree_from_tal_url_parallel_phase2_audit_with_timing( + store: Arc, + policy: &crate::policy::Policy, + tal_url: &str, + http_fetcher: &H, + rsync_fetcher: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + parallel_config: ParallelPhase1Config, + phase2_config: ParallelPhase2Config, + collect_current_repo_objects: bool, + timing: &TimingHandle, +) -> Result +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + let discovery = discover_root_ca_instance_from_tal_url_with_policy( + policy, + http_fetcher, + rsync_fetcher, + tal_url, + )?; + run_single_root_parallel_audit_inner( + store, + policy, + discovery, + vec![TalInputSpec::from_url(tal_url.to_string())], + http_fetcher, + rsync_fetcher, + validation_time, + config, + parallel_config, + Some(phase2_config), + collect_current_repo_objects, + Some(timing.clone()), + ) +} + +pub fn run_tree_from_tal_and_ta_der_parallel_phase2_audit( + store: Arc, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&url::Url>, + http_fetcher: &H, + rsync_fetcher: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + parallel_config: ParallelPhase1Config, + phase2_config: ParallelPhase2Config, + collect_current_repo_objects: bool, +) -> Result +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy, + tal_bytes, + ta_der, + resolved_ta_uri, + )?; + let derived_tal_id = derive_tal_id(&discovery); + let tal_inputs = vec![TalInputSpec { + tal_id: derived_tal_id.clone(), + rir_id: derived_tal_id, + source: TalSource::DerBytes { + tal_url: discovery + .tal_url + .clone() + .unwrap_or_else(|| "embedded-tal".to_string()), + tal_bytes: tal_bytes.to_vec(), + ta_der: ta_der.to_vec(), + }, + }]; + run_single_root_parallel_audit_inner( + store, + policy, + discovery, + tal_inputs, + http_fetcher, + rsync_fetcher, + validation_time, + config, + parallel_config, + Some(phase2_config), + collect_current_repo_objects, + None, + ) +} + +pub fn run_tree_from_tal_and_ta_der_parallel_phase2_audit_with_timing( + store: Arc, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&url::Url>, + http_fetcher: &H, + rsync_fetcher: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + parallel_config: ParallelPhase1Config, + phase2_config: ParallelPhase2Config, + collect_current_repo_objects: bool, + timing: &TimingHandle, +) -> Result +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy, + tal_bytes, + ta_der, + resolved_ta_uri, + )?; + let derived_tal_id = derive_tal_id(&discovery); + let tal_inputs = vec![TalInputSpec { + tal_id: derived_tal_id.clone(), + rir_id: derived_tal_id, + source: TalSource::DerBytes { + tal_url: discovery + .tal_url + .clone() + .unwrap_or_else(|| "embedded-tal".to_string()), + tal_bytes: tal_bytes.to_vec(), + ta_der: ta_der.to_vec(), + }, + }]; + run_single_root_parallel_audit_inner( + store, + policy, + discovery, + tal_inputs, + http_fetcher, + rsync_fetcher, + validation_time, + config, + parallel_config, + Some(phase2_config), + collect_current_repo_objects, + Some(timing.clone()), + ) +} + +pub fn run_tree_from_multiple_tals_parallel_phase2_audit( + store: Arc, + policy: &crate::policy::Policy, + tal_inputs: Vec, + http_fetcher: &H, + rsync_fetcher: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + parallel_config: ParallelPhase1Config, + phase2_config: ParallelPhase2Config, + collect_current_repo_objects: bool, +) -> Result +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + run_multi_root_parallel_audit_inner( + store, + policy, + tal_inputs, + http_fetcher, + rsync_fetcher, + validation_time, + config, + parallel_config, + Some(phase2_config), + collect_current_repo_objects, + None, + ) +} + +pub fn run_tree_from_multiple_tals_parallel_phase2_audit_with_timing( + store: Arc, + policy: &crate::policy::Policy, + tal_inputs: Vec, + http_fetcher: &H, + rsync_fetcher: &R, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + parallel_config: ParallelPhase1Config, + phase2_config: ParallelPhase2Config, + collect_current_repo_objects: bool, + timing: &TimingHandle, +) -> Result +where + H: Fetcher + Clone + 'static, + R: crate::fetch::rsync::RsyncFetcher + Clone + 'static, +{ + run_multi_root_parallel_audit_inner( + store, + policy, + tal_inputs, + http_fetcher, + rsync_fetcher, + validation_time, + config, + parallel_config, + Some(phase2_config), + collect_current_repo_objects, + Some(timing.clone()), + ) +} + +pub fn run_tree_from_tal_and_ta_der_serial( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&Url>, + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, +) -> Result { + let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy, + tal_bytes, + ta_der, + resolved_ta_uri, + )?; + + let runner = Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher, + rsync_fetcher, + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: config.enable_roa_validation_cache, + enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache, + publication_point_cache_observe_only: config.publication_point_cache_observe_only, + enable_publication_point_validation_cache: config.enable_publication_point_validation_cache, + }; + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + let tree = run_tree_serial(root, &runner, config)?; + + Ok(RunTreeFromTalOutput { discovery, tree }) +} + +pub fn run_tree_from_tal_bytes_serial_audit( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + tal_uri: Option, + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, +) -> Result { + let tal = crate::data_model::tal::Tal::decode_bytes(tal_bytes).map_err(FromTalError::from)?; + let discovery = discover_root_ca_instance_from_tal_with_fetchers( + http_fetcher, + rsync_fetcher, + tal, + tal_uri, + )?; + + let download_log = DownloadLogHandle::new(); + let runner = Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher, + rsync_fetcher, + validation_time, + timing: None, + download_log: Some(download_log.clone()), + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: config.enable_roa_validation_cache, + enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache, + publication_point_cache_observe_only: config.publication_point_cache_observe_only, + enable_publication_point_validation_cache: config.enable_publication_point_validation_cache, + }; + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = run_tree_serial_audit(root, &runner, config)?; + + let downloads = download_log.snapshot_events(); + let download_stats = DownloadLogHandle::stats_from_events(&downloads); + Ok(RunTreeFromTalAuditOutput { + discovery: discovery.clone(), + discoveries: vec![discovery], + successful_tal_inputs: Vec::new(), + tree, + publication_points, + roa_cache_stats, + cir_input, + downloads, + download_stats, + current_repo_objects: Vec::new(), + ccr_accumulator: None, + }) +} + +pub fn run_tree_from_tal_bytes_serial_audit_with_timing( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + tal_uri: Option, + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + timing: &TimingHandle, +) -> Result { + let _tal = timing.span_phase("tal_bootstrap"); + let tal = crate::data_model::tal::Tal::decode_bytes(tal_bytes).map_err(FromTalError::from)?; + let discovery = discover_root_ca_instance_from_tal_with_fetchers( + http_fetcher, + rsync_fetcher, + tal, + tal_uri, + )?; + drop(_tal); + + let download_log = DownloadLogHandle::new(); + let runner = Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher, + rsync_fetcher, + validation_time, + timing: Some(timing.clone()), + download_log: Some(download_log.clone()), + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: config.enable_roa_validation_cache, + enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache, + publication_point_cache_observe_only: config.publication_point_cache_observe_only, + enable_publication_point_validation_cache: config.enable_publication_point_validation_cache, + }; + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + let _tree = timing.span_phase("tree_run"); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = run_tree_serial_audit(root, &runner, config)?; + drop(_tree); + + let downloads = download_log.snapshot_events(); + let download_stats = DownloadLogHandle::stats_from_events(&downloads); + Ok(RunTreeFromTalAuditOutput { + discovery: discovery.clone(), + discoveries: vec![discovery], + successful_tal_inputs: Vec::new(), + tree, + publication_points, + roa_cache_stats, + cir_input, + downloads, + download_stats, + current_repo_objects: Vec::new(), + ccr_accumulator: None, + }) +} + +pub fn run_tree_from_tal_and_ta_der_serial_audit( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&Url>, + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, +) -> Result { + let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy, + tal_bytes, + ta_der, + resolved_ta_uri, + )?; + + let download_log = DownloadLogHandle::new(); + let runner = Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher, + rsync_fetcher, + validation_time, + timing: None, + download_log: Some(download_log.clone()), + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: config.enable_roa_validation_cache, + enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache, + publication_point_cache_observe_only: config.publication_point_cache_observe_only, + enable_publication_point_validation_cache: config.enable_publication_point_validation_cache, + }; + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = run_tree_serial_audit(root, &runner, config)?; + + let downloads = download_log.snapshot_events(); + let download_stats = DownloadLogHandle::stats_from_events(&downloads); + Ok(RunTreeFromTalAuditOutput { + discovery: discovery.clone(), + discoveries: vec![discovery], + successful_tal_inputs: Vec::new(), + tree, + publication_points, + roa_cache_stats, + cir_input, + downloads, + download_stats, + current_repo_objects: Vec::new(), + ccr_accumulator: None, + }) +} + +pub fn run_tree_from_tal_and_ta_der_serial_audit_with_timing( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&Url>, + http_fetcher: &dyn Fetcher, + rsync_fetcher: &dyn crate::fetch::rsync::RsyncFetcher, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + timing: &TimingHandle, +) -> Result { + let _tal = timing.span_phase("tal_bootstrap"); + let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy, + tal_bytes, + ta_der, + resolved_ta_uri, + )?; + drop(_tal); + + let download_log = DownloadLogHandle::new(); + let runner = Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher, + rsync_fetcher, + validation_time, + timing: Some(timing.clone()), + download_log: Some(download_log.clone()), + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: config.enable_roa_validation_cache, + enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache, + publication_point_cache_observe_only: config.publication_point_cache_observe_only, + enable_publication_point_validation_cache: config.enable_publication_point_validation_cache, + }; + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + let _tree = timing.span_phase("tree_run_total"); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = run_tree_serial_audit(root, &runner, config)?; + + let downloads = download_log.snapshot_events(); + let download_stats = DownloadLogHandle::stats_from_events(&downloads); + Ok(RunTreeFromTalAuditOutput { + discovery: discovery.clone(), + discoveries: vec![discovery], + successful_tal_inputs: Vec::new(), + tree, + publication_points, + roa_cache_stats, + cir_input, + downloads, + download_stats, + current_repo_objects: Vec::new(), + ccr_accumulator: None, + }) +} + +pub fn run_tree_from_tal_and_ta_der_payload_replay_serial( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&Url>, + payload_archive_root: &std::path::Path, + payload_locks_path: &std::path::Path, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, +) -> Result { + let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy, + tal_bytes, + ta_der, + resolved_ta_uri, + )?; + let replay_index = Arc::new( + ReplayArchiveIndex::load_allow_missing_rsync_modules( + payload_archive_root, + payload_locks_path, + ) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?, + ); + let http_fetcher = PayloadReplayHttpFetcher::new(replay_index.clone()) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?; + let rsync_fetcher = PayloadReplayRsyncFetcher::new(replay_index.clone()); + + let runner = Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher: &http_fetcher, + rsync_fetcher: &rsync_fetcher, + validation_time, + timing: None, + download_log: None, + replay_archive_index: Some(replay_index), + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: config.enable_roa_validation_cache, + enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache, + publication_point_cache_observe_only: config.publication_point_cache_observe_only, + enable_publication_point_validation_cache: config.enable_publication_point_validation_cache, + }; + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + let tree = run_tree_serial(root, &runner, config)?; + + Ok(RunTreeFromTalOutput { discovery, tree }) +} + +pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&Url>, + payload_archive_root: &std::path::Path, + payload_locks_path: &std::path::Path, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, +) -> Result { + let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy, + tal_bytes, + ta_der, + resolved_ta_uri, + )?; + let replay_index = Arc::new( + ReplayArchiveIndex::load_allow_missing_rsync_modules( + payload_archive_root, + payload_locks_path, + ) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?, + ); + let http_fetcher = PayloadReplayHttpFetcher::new(replay_index.clone()) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?; + let rsync_fetcher = PayloadReplayRsyncFetcher::new(replay_index.clone()); + let download_log = DownloadLogHandle::new(); + + let runner = Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher: &http_fetcher, + rsync_fetcher: &rsync_fetcher, + validation_time, + timing: None, + download_log: Some(download_log.clone()), + replay_archive_index: Some(replay_index), + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: config.enable_roa_validation_cache, + enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache, + publication_point_cache_observe_only: config.publication_point_cache_observe_only, + enable_publication_point_validation_cache: config.enable_publication_point_validation_cache, + }; + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = run_tree_serial_audit(root, &runner, config)?; + + let downloads = download_log.snapshot_events(); + let download_stats = DownloadLogHandle::stats_from_events(&downloads); + Ok(RunTreeFromTalAuditOutput { + discovery: discovery.clone(), + discoveries: vec![discovery], + successful_tal_inputs: Vec::new(), + tree, + publication_points, + roa_cache_stats, + cir_input, + downloads, + download_stats, + current_repo_objects: Vec::new(), + ccr_accumulator: None, + }) +} + +pub fn run_tree_from_tal_and_ta_der_payload_replay_serial_audit_with_timing( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&Url>, + payload_archive_root: &std::path::Path, + payload_locks_path: &std::path::Path, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + timing: &TimingHandle, +) -> Result { + let _tal = timing.span_phase("tal_bootstrap"); + let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy, + tal_bytes, + ta_der, + resolved_ta_uri, + )?; + drop(_tal); + let replay_index = Arc::new( + ReplayArchiveIndex::load_allow_missing_rsync_modules( + payload_archive_root, + payload_locks_path, + ) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?, + ); + let http_fetcher = PayloadReplayHttpFetcher::new(replay_index.clone()) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?; + let rsync_fetcher = PayloadReplayRsyncFetcher::new(replay_index.clone()); + let download_log = DownloadLogHandle::new(); + + let runner = Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher: &http_fetcher, + rsync_fetcher: &rsync_fetcher, + validation_time, + timing: Some(timing.clone()), + download_log: Some(download_log.clone()), + replay_archive_index: Some(replay_index), + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: config.enable_roa_validation_cache, + enable_child_certificate_validation_cache: config.enable_child_certificate_validation_cache, + publication_point_cache_observe_only: config.publication_point_cache_observe_only, + enable_publication_point_validation_cache: config.enable_publication_point_validation_cache, + }; + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + let _tree = timing.span_phase("tree_run_total"); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = run_tree_serial_audit(root, &runner, config)?; + + let downloads = download_log.snapshot_events(); + let download_stats = DownloadLogHandle::stats_from_events(&downloads); + Ok(RunTreeFromTalAuditOutput { + discovery: discovery.clone(), + discoveries: vec![discovery], + successful_tal_inputs: Vec::new(), + tree, + publication_points, + roa_cache_stats, + cir_input, + downloads, + download_stats, + current_repo_objects: Vec::new(), + ccr_accumulator: None, + }) +} + +fn build_payload_replay_runner<'a>( + store: &'a crate::storage::RocksStore, + policy: &'a crate::policy::Policy, + replay_index: Arc, + http_fetcher: &'a PayloadReplayHttpFetcher, + rsync_fetcher: &'a PayloadReplayRsyncFetcher, + validation_time: time::OffsetDateTime, + timing: Option, + download_log: Option, + enable_roa_validation_cache: bool, +) -> Rpkiv1PublicationPointRunner<'a> { + Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher, + rsync_fetcher, + validation_time, + timing, + download_log, + replay_archive_index: Some(replay_index), + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + } +} + +fn build_payload_delta_replay_runner<'a>( + store: &'a crate::storage::RocksStore, + policy: &'a crate::policy::Policy, + delta_index: Arc, + http_fetcher: &'a PayloadDeltaReplayHttpFetcher, + rsync_fetcher: &'a PayloadDeltaReplayRsyncFetcher, + validation_time: time::OffsetDateTime, + timing: Option, + download_log: Option, + enable_roa_validation_cache: bool, +) -> Rpkiv1PublicationPointRunner<'a> { + Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher, + rsync_fetcher, + validation_time, + timing, + download_log, + replay_archive_index: None, + replay_delta_index: Some(delta_index), + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + } +} + +fn build_payload_delta_replay_current_store_runner<'a>( + store: &'a crate::storage::RocksStore, + policy: &'a crate::policy::Policy, + delta_index: Arc, + http_fetcher: &'a PayloadDeltaReplayHttpFetcher, + rsync_fetcher: &'a PayloadDeltaReplayCurrentStoreRsyncFetcher<'a>, + validation_time: time::OffsetDateTime, + timing: Option, + download_log: Option, + enable_roa_validation_cache: bool, +) -> Rpkiv1PublicationPointRunner<'a> { + Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher, + rsync_fetcher, + validation_time, + timing, + download_log, + replay_archive_index: None, + replay_delta_index: Some(delta_index), + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + } +} + +fn run_payload_delta_replay_audit_inner( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + discovery: DiscoveredRootCaInstance, + base_payload_archive_root: &std::path::Path, + base_locks_path: &std::path::Path, + delta_payload_archive_root: &std::path::Path, + delta_locks_path: &std::path::Path, + base_validation_time: time::OffsetDateTime, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + timing: Option, +) -> Result { + let base_index = Arc::new( + ReplayArchiveIndex::load_allow_missing_rsync_modules( + base_payload_archive_root, + base_locks_path, + ) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?, + ); + let delta_index = Arc::new( + ReplayDeltaArchiveIndex::load(delta_payload_archive_root, delta_locks_path) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?, + ); + delta_index + .validate_base_locks_sha256_file(base_locks_path) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?; + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + + let base_http_fetcher = PayloadReplayHttpFetcher::new(base_index.clone()) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?; + let base_rsync_fetcher = PayloadReplayRsyncFetcher::new(base_index.clone()); + if let Some(t) = timing.as_ref() { + let _phase = t.span_phase("payload_delta_replay_base_total"); + let base_runner = build_payload_replay_runner( + store, + policy, + base_index.clone(), + &base_http_fetcher, + &base_rsync_fetcher, + base_validation_time, + Some(t.clone()), + None, + config.enable_roa_validation_cache, + ); + let _base = run_tree_serial(root.clone(), &base_runner, config)?; + } else { + let base_runner = build_payload_replay_runner( + store, + policy, + base_index.clone(), + &base_http_fetcher, + &base_rsync_fetcher, + base_validation_time, + None, + None, + config.enable_roa_validation_cache, + ); + let _base = run_tree_serial(root.clone(), &base_runner, config)?; + } + + let delta_http_fetcher = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?; + let delta_rsync_fetcher = PayloadDeltaReplayRsyncFetcher::new(base_index, delta_index.clone()); + let download_log = DownloadLogHandle::new(); + let (tree, publication_points, roa_cache_stats, cir_input) = if let Some(t) = timing.as_ref() { + let _phase = t.span_phase("payload_delta_replay_target_total"); + let delta_runner = build_payload_delta_replay_runner( + store, + policy, + delta_index, + &delta_http_fetcher, + &delta_rsync_fetcher, + base_validation_time, + Some(t.clone()), + Some(download_log.clone()), + config.enable_roa_validation_cache, + ); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = run_tree_serial_audit(root, &delta_runner, config)?; + (tree, publication_points, roa_cache_stats, cir_input) + } else { + let delta_runner = build_payload_delta_replay_runner( + store, + policy, + delta_index, + &delta_http_fetcher, + &delta_rsync_fetcher, + validation_time, + None, + Some(download_log.clone()), + config.enable_roa_validation_cache, + ); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = run_tree_serial_audit(root, &delta_runner, config)?; + (tree, publication_points, roa_cache_stats, cir_input) + }; + let downloads = download_log.snapshot_events(); + let download_stats = DownloadLogHandle::stats_from_events(&downloads); + Ok(RunTreeFromTalAuditOutput { + discovery: discovery.clone(), + discoveries: vec![discovery], + successful_tal_inputs: Vec::new(), + tree, + publication_points, + roa_cache_stats, + cir_input, + downloads, + download_stats, + current_repo_objects: Vec::new(), + ccr_accumulator: None, + }) +} + +pub fn run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&Url>, + base_payload_archive_root: &std::path::Path, + base_locks_path: &std::path::Path, + delta_payload_archive_root: &std::path::Path, + delta_locks_path: &std::path::Path, + base_validation_time: time::OffsetDateTime, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, +) -> Result { + let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy, + tal_bytes, + ta_der, + resolved_ta_uri, + )?; + run_payload_delta_replay_audit_inner( + store, + policy, + discovery, + base_payload_archive_root, + base_locks_path, + delta_payload_archive_root, + delta_locks_path, + base_validation_time, + validation_time, + config, + None, + ) +} + +pub fn run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit_with_timing( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&Url>, + base_payload_archive_root: &std::path::Path, + base_locks_path: &std::path::Path, + delta_payload_archive_root: &std::path::Path, + delta_locks_path: &std::path::Path, + base_validation_time: time::OffsetDateTime, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + timing: &TimingHandle, +) -> Result { + let _tal = timing.span_phase("tal_bootstrap"); + let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy, + tal_bytes, + ta_der, + resolved_ta_uri, + )?; + drop(_tal); + run_payload_delta_replay_audit_inner( + store, + policy, + discovery, + base_payload_archive_root, + base_locks_path, + delta_payload_archive_root, + delta_locks_path, + base_validation_time, + validation_time, + config, + Some(timing.clone()), + ) +} + +fn run_payload_delta_replay_step_audit_inner( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + discovery: DiscoveredRootCaInstance, + delta_payload_archive_root: &std::path::Path, + previous_locks_path: &std::path::Path, + delta_locks_path: &std::path::Path, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, + timing: Option, +) -> Result { + let delta_index = Arc::new( + ReplayDeltaArchiveIndex::load(delta_payload_archive_root, delta_locks_path) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?, + ); + delta_index + .validate_base_locks_sha256_file(previous_locks_path) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?; + + let root = root_handle_from_trust_anchor( + &discovery.trust_anchor, + derive_tal_id(&discovery), + None, + &discovery.ca_instance, + ); + + let delta_http_fetcher = PayloadDeltaReplayHttpFetcher::from_index(delta_index.clone()) + .map_err(|e| RunTreeFromTalError::Replay(e.to_string()))?; + let delta_rsync_fetcher = + PayloadDeltaReplayCurrentStoreRsyncFetcher::new(store, delta_index.clone()); + let download_log = DownloadLogHandle::new(); + + let (tree, publication_points, roa_cache_stats, cir_input) = if let Some(t) = timing.as_ref() { + let _phase = t.span_phase("payload_delta_replay_step_total"); + let delta_runner = build_payload_delta_replay_current_store_runner( + store, + policy, + delta_index, + &delta_http_fetcher, + &delta_rsync_fetcher, + validation_time, + Some(t.clone()), + Some(download_log.clone()), + config.enable_roa_validation_cache, + ); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = run_tree_serial_audit(root, &delta_runner, config)?; + (tree, publication_points, roa_cache_stats, cir_input) + } else { + let delta_runner = build_payload_delta_replay_current_store_runner( + store, + policy, + delta_index, + &delta_http_fetcher, + &delta_rsync_fetcher, + validation_time, + None, + Some(download_log.clone()), + config.enable_roa_validation_cache, + ); + let TreeRunAuditOutput { + tree, + publication_points, + roa_cache_stats, + cir_input, + } = run_tree_serial_audit(root, &delta_runner, config)?; + (tree, publication_points, roa_cache_stats, cir_input) + }; + let downloads = download_log.snapshot_events(); + let download_stats = DownloadLogHandle::stats_from_events(&downloads); + Ok(RunTreeFromTalAuditOutput { + discovery: discovery.clone(), + discoveries: vec![discovery], + successful_tal_inputs: Vec::new(), + tree, + publication_points, + roa_cache_stats, + cir_input, + downloads, + download_stats, + current_repo_objects: Vec::new(), + ccr_accumulator: None, + }) +} + +pub fn run_tree_from_tal_and_ta_der_payload_delta_replay_step_serial_audit( + store: &crate::storage::RocksStore, + policy: &crate::policy::Policy, + tal_bytes: &[u8], + ta_der: &[u8], + resolved_ta_uri: Option<&Url>, + delta_payload_archive_root: &std::path::Path, + previous_locks_path: &std::path::Path, + delta_locks_path: &std::path::Path, + validation_time: time::OffsetDateTime, + config: &TreeRunConfig, +) -> Result { + let discovery = discover_root_ca_instance_from_tal_and_ta_der_with_policy( + policy, + tal_bytes, + ta_der, + resolved_ta_uri, + )?; + run_payload_delta_replay_step_audit_inner( + store, + policy, + discovery, + delta_payload_archive_root, + previous_locks_path, + delta_locks_path, + validation_time, + config, + None, + ) +} + +#[cfg(test)] +mod multi_tal_tests { + use super::*; + use crate::current_repo_index::CurrentRepoIndex; + use crate::storage::{RepositoryViewEntry, RepositoryViewState}; + + struct RejectingHttpFetcher; + + impl Fetcher for RejectingHttpFetcher { + fn fetch(&self, uri: &str) -> Result, String> { + Err(format!("unexpected http fetch: {uri}")) + } + } + + struct RejectingRsyncFetcher; + + impl crate::fetch::rsync::RsyncFetcher for RejectingRsyncFetcher { + fn fetch_objects( + &self, + _rsync_base_uri: &str, + ) -> crate::fetch::rsync::RsyncFetchResult)>> { + Err(crate::fetch::rsync::RsyncFetchError::Fetch( + "unexpected rsync fetch".to_string(), + )) + } + + fn dedup_key(&self, base_uri: &str) -> String { + base_uri.to_string() + } + } + + #[test] + fn snapshot_current_repo_objects_is_on_demand() { + let handle = CurrentRepoIndex::shared(); + handle + .write() + .expect("write-lock index") + .apply_repository_view_entries(&[RepositoryViewEntry { + rsync_uri: "rsync://example.test/repo/a.roa".to_string(), + current_hash: Some("11".repeat(32)), + repository_source: Some("rsync://example.test/repo/".to_string()), + object_type: Some("roa".to_string()), + state: RepositoryViewState::Present, + }]) + .expect("apply present entry"); + + assert!( + snapshot_current_repo_objects(Some(&handle), false).is_empty(), + "collection should be skipped when disabled" + ); + + let collected = snapshot_current_repo_objects(Some(&handle), true); + assert_eq!(collected.len(), 1); + assert_eq!(collected[0].rsync_uri, "rsync://example.test/repo/a.roa"); + } + + #[test] + fn discover_multiple_roots_from_tal_inputs_builds_multiple_root_handles() { + let apnic_tal = + std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal").expect("read apnic tal"); + let apnic_ta = std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta"); + let arin_tal = std::fs::read("tests/fixtures/tal/arin.tal").expect("read arin tal"); + let arin_ta = std::fs::read("tests/fixtures/ta/arin-ta.cer").expect("read arin ta"); + + let tal_inputs = vec![ + TalInputSpec::from_ta_der("https://example.test/apnic.tal", apnic_tal, apnic_ta), + TalInputSpec::from_ta_der("https://example.test/arin.tal", arin_tal, arin_ta), + ]; + + let roots = discover_multiple_roots_from_tal_inputs( + &tal_inputs, + &RejectingHttpFetcher, + &RejectingRsyncFetcher, + false, + ) + .expect("discover roots"); + + assert_eq!(roots.len(), 2); + assert_eq!(roots[0].tal_input.tal_id, "apnic"); + assert_eq!(roots[1].tal_input.tal_id, "arin"); + assert_eq!(roots[0].root_handle.tal_id, "apnic"); + assert_eq!(roots[1].root_handle.tal_id, "arin"); + assert_ne!( + roots[0].root_handle.manifest_rsync_uri, + roots[1].root_handle.manifest_rsync_uri + ); + } + + #[test] + fn discover_multiple_roots_isolates_strict_name_failure() { + let apnic_tal = + std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal").expect("read apnic tal"); + let apnic_ta = std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta"); + let arin_tal = std::fs::read("tests/fixtures/tal/arin.tal").expect("read arin tal"); + let arin_ta = std::fs::read("tests/fixtures/ta/arin-ta.cer").expect("read arin ta"); + + let tal_inputs = vec![ + TalInputSpec::from_ta_der("https://example.test/apnic.tal", apnic_tal, apnic_ta), + TalInputSpec::from_ta_der("https://example.test/arin.tal", arin_tal, arin_ta), + ]; + + let roots = discover_multiple_roots_from_tal_inputs( + &tal_inputs, + &RejectingHttpFetcher, + &RejectingRsyncFetcher, + true, + ) + .expect("strict discovery should keep usable roots"); + + assert_eq!(roots.len(), 1); + assert_eq!(roots[0].tal_input.tal_id, "arin"); + assert_eq!(roots[0].root_handle.tal_id, "arin"); + } + + #[test] + fn discover_single_root_keeps_strict_name_failure_fatal() { + let apnic_tal = + std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal").expect("read apnic tal"); + let apnic_ta = std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta"); + let tal_inputs = vec![TalInputSpec::from_ta_der( + "https://example.test/apnic.tal", + apnic_tal, + apnic_ta, + )]; + + let error = discover_multiple_roots_from_tal_inputs( + &tal_inputs, + &RejectingHttpFetcher, + &RejectingRsyncFetcher, + true, + ) + .expect_err("single-TAL strict failure should remain fatal"); + + assert!(error.to_string().contains("Name strict validation failed")); + } +} + +#[cfg(test)] +mod replay_api_tests { + use super::*; + use crate::analysis::timing::{TimingHandle, TimingMeta}; + use time::format_description::well_known::Rfc3339; + + fn apnic_replay_inputs() -> ( + Vec, + Vec, + std::path::PathBuf, + std::path::PathBuf, + time::OffsetDateTime, + ) { + let tal_bytes = std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal") + .expect("read apnic tal fixture"); + let ta_der = + std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta fixture"); + let archive_root = std::path::PathBuf::from("target/live/payload_replay/payload-archive"); + let locks_path = std::path::PathBuf::from("target/live/payload_replay/locks.json"); + let validation_time = time::OffsetDateTime::parse("2026-03-13T02:30:00Z", &Rfc3339) + .expect("parse validation time"); + (tal_bytes, ta_der, archive_root, locks_path, validation_time) + } + + fn apnic_multi_rir_replay_inputs() -> ( + Vec, + Vec, + std::path::PathBuf, + std::path::PathBuf, + time::OffsetDateTime, + ) { + let tal_bytes = std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal") + .expect("read apnic tal fixture"); + let ta_der = + std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta fixture"); + let archive_root = std::path::PathBuf::from( + "../../rpki/target/live/20260316-112341-multi-final3/apnic/base-payload-archive", + ); + let locks_path = std::path::PathBuf::from( + "../../rpki/target/live/20260316-112341-multi-final3/apnic/base-locks.json", + ); + let validation_time = time::OffsetDateTime::parse("2026-03-16T11:49:48+08:00", &Rfc3339) + .expect("parse validation time"); + (tal_bytes, ta_der, archive_root, locks_path, validation_time) + } + + fn apnic_delta_replay_inputs() -> ( + Vec, + Vec, + std::path::PathBuf, + std::path::PathBuf, + std::path::PathBuf, + std::path::PathBuf, + time::OffsetDateTime, + ) { + let tal_bytes = std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal") + .expect("read apnic tal fixture"); + let ta_der = + std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta fixture"); + let root = + std::path::PathBuf::from("target/live/apnic_delta_demo/20260315-170223-autoplay"); + let base_archive = root.join("base-payload-archive"); + let base_locks = root.join("base-locks.json"); + let delta_archive = root.join("payload-delta-archive"); + let delta_locks = root.join("locks-delta.json"); + let validation_time = time::OffsetDateTime::parse("2026-03-15T10:00:00Z", &Rfc3339) + .expect("parse validation time"); + ( + tal_bytes, + ta_der, + base_archive, + base_locks, + delta_archive, + delta_locks, + validation_time, + ) + } + + #[test] + fn payload_replay_api_reports_setup_error_for_missing_archive() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = crate::storage::RocksStore::open(&temp.path().join("db")).expect("open db"); + let tal_bytes = std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal") + .expect("read apnic tal fixture"); + let ta_der = + std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta fixture"); + let err = run_tree_from_tal_and_ta_der_payload_replay_serial_audit( + &store, + &crate::policy::Policy::default(), + &tal_bytes, + &ta_der, + None, + std::path::Path::new("tests/fixtures/missing-payload-archive"), + std::path::Path::new("tests/fixtures/missing-locks.json"), + time::OffsetDateTime::now_utc(), + &TreeRunConfig { + max_depth: Some(0), + max_instances: Some(1), + compact_audit: false, + persist_vcir: true, + build_ccr_accumulator: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + enable_transport_request_prefetch: false, + }, + ) + .unwrap_err(); + assert!(matches!(err, RunTreeFromTalError::Replay(_)), "{err}"); + } + + #[test] + fn payload_replay_api_root_only_apnic_archive_runs() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = crate::storage::RocksStore::open(&temp.path().join("db")).expect("open db"); + let (tal_bytes, ta_der, archive_root, locks_path, validation_time) = apnic_replay_inputs(); + if !archive_root.is_dir() || !locks_path.is_file() { + eprintln!( + "skipping payload replay api test; missing fixtures: archive={} locks={}", + archive_root.display(), + locks_path.display() + ); + return; + } + + let out = run_tree_from_tal_and_ta_der_payload_replay_serial_audit( + &store, + &crate::policy::Policy::default(), + &tal_bytes, + &ta_der, + None, + &archive_root, + &locks_path, + validation_time, + &TreeRunConfig { + max_depth: Some(0), + max_instances: Some(1), + compact_audit: false, + persist_vcir: true, + build_ccr_accumulator: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + enable_transport_request_prefetch: false, + }, + ) + .expect("run replay root-only audit"); + + assert_eq!(out.tree.instances_processed, 1); + assert_eq!(out.tree.instances_failed, 0); + assert_eq!(out.publication_points.len(), 1); + assert_eq!(out.discovery.trust_anchor.resolved_ta_uri, None); + assert!(!out.downloads.is_empty()); + assert!( + out.downloads.iter().all(|d| d.success), + "expected successful replay downloads" + ); + } + + #[test] + fn payload_replay_api_root_only_apnic_multi_rir_bundle_runs_with_lenient_rsync_modules() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = crate::storage::RocksStore::open(&temp.path().join("db")).expect("open db"); + let (tal_bytes, ta_der, archive_root, locks_path, validation_time) = + apnic_multi_rir_replay_inputs(); + if !archive_root.is_dir() || !locks_path.is_file() { + eprintln!( + "skipping multi-rir payload replay api test; missing fixtures: archive={} locks={}", + archive_root.display(), + locks_path.display() + ); + return; + } + + let out = run_tree_from_tal_and_ta_der_payload_replay_serial_audit( + &store, + &crate::policy::Policy::default(), + &tal_bytes, + &ta_der, + None, + &archive_root, + &locks_path, + validation_time, + &TreeRunConfig { + max_depth: Some(0), + max_instances: Some(1), + compact_audit: false, + persist_vcir: true, + build_ccr_accumulator: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + enable_transport_request_prefetch: false, + }, + ) + .expect("run replay root-only audit"); + + assert_eq!(out.tree.instances_processed, 1); + assert_eq!(out.tree.instances_failed, 0); + assert_eq!(out.publication_points.len(), 1); + } + + #[test] + fn payload_replay_api_root_only_apnic_archive_runs_with_timing() { + let temp = tempfile::tempdir().expect("tempdir"); + let db_path = temp.path().join("db"); + let store = crate::storage::RocksStore::open(&db_path).expect("open db"); + let (tal_bytes, ta_der, archive_root, locks_path, validation_time) = apnic_replay_inputs(); + if !archive_root.is_dir() || !locks_path.is_file() { + eprintln!( + "skipping payload replay api timing test; missing fixtures: archive={} locks={}", + archive_root.display(), + locks_path.display() + ); + return; + } + + let timing = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339: "2026-03-13T03:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-03-13T02:30:00Z".to_string(), + tal_url: None, + db_path: Some(db_path.to_string_lossy().into_owned()), + }); + + let out = run_tree_from_tal_and_ta_der_payload_replay_serial_audit_with_timing( + &store, + &crate::policy::Policy::default(), + &tal_bytes, + &ta_der, + None, + &archive_root, + &locks_path, + validation_time, + &TreeRunConfig { + max_depth: Some(0), + max_instances: Some(1), + compact_audit: false, + persist_vcir: true, + build_ccr_accumulator: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + enable_transport_request_prefetch: false, + }, + &timing, + ) + .expect("run replay root-only audit with timing"); + + assert_eq!(out.tree.instances_processed, 1); + let timing_json = temp.path().join("timing_replay.json"); + timing + .write_json(&timing_json, 20) + .expect("write timing json"); + let json: serde_json::Value = + serde_json::from_slice(&std::fs::read(&timing_json).expect("read timing json")) + .expect("parse timing json"); + let counts = json.get("counts").expect("counts"); + assert!( + counts + .get("repo_sync_rrdp_ok_total") + .and_then(|v| v.as_u64()) + .unwrap_or(0) + >= 1 + ); + } + + #[test] + fn payload_delta_replay_api_rejects_base_locks_sha_mismatch() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = crate::storage::RocksStore::open(&temp.path().join("db")).expect("open db"); + let ( + tal_bytes, + ta_der, + base_archive, + _base_locks, + delta_archive, + delta_locks, + validation_time, + ) = apnic_delta_replay_inputs(); + let wrong_base_locks = temp.path().join("wrong-base-locks.json"); + std::fs::write(&wrong_base_locks, b"wrong-base-locks").expect("write wrong base locks"); + let err = run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit( + &store, + &crate::policy::Policy::default(), + &tal_bytes, + &ta_der, + None, + &base_archive, + &wrong_base_locks, + &delta_archive, + &delta_locks, + validation_time, + validation_time, + &TreeRunConfig { + max_depth: Some(0), + max_instances: Some(1), + compact_audit: false, + persist_vcir: true, + build_ccr_accumulator: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + enable_transport_request_prefetch: false, + }, + ) + .unwrap_err(); + assert!(matches!(err, RunTreeFromTalError::Replay(_)), "{err}"); + } + + #[test] + fn payload_delta_replay_api_reports_setup_error_for_missing_inputs() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = crate::storage::RocksStore::open(&temp.path().join("db")).expect("open db"); + let tal_bytes = std::fs::read("tests/fixtures/tal/apnic-rfc7730-https.tal") + .expect("read apnic tal fixture"); + let ta_der = + std::fs::read("tests/fixtures/ta/apnic-ta.cer").expect("read apnic ta fixture"); + let err = run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit( + &store, + &crate::policy::Policy::default(), + &tal_bytes, + &ta_der, + None, + std::path::Path::new("tests/fixtures/missing-base-archive"), + std::path::Path::new("tests/fixtures/missing-base-locks.json"), + std::path::Path::new("tests/fixtures/missing-delta-archive"), + std::path::Path::new("tests/fixtures/missing-delta-locks.json"), + time::OffsetDateTime::now_utc(), + time::OffsetDateTime::now_utc(), + &TreeRunConfig { + max_depth: Some(0), + max_instances: Some(1), + compact_audit: false, + persist_vcir: true, + build_ccr_accumulator: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + enable_transport_request_prefetch: false, + }, + ) + .unwrap_err(); + assert!(matches!(err, RunTreeFromTalError::Replay(_)), "{err}"); + } + + #[test] + fn payload_delta_replay_api_root_only_apnic_bundle_runs() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = crate::storage::RocksStore::open(&temp.path().join("db")).expect("open db"); + let ( + tal_bytes, + ta_der, + base_archive, + base_locks, + delta_archive, + delta_locks, + validation_time, + ) = apnic_delta_replay_inputs(); + if !base_archive.is_dir() + || !base_locks.is_file() + || !delta_archive.is_dir() + || !delta_locks.is_file() + { + eprintln!( + "skipping payload delta replay api test; missing fixtures: base_archive={} base_locks={} delta_archive={} delta_locks={}", + base_archive.display(), + base_locks.display(), + delta_archive.display(), + delta_locks.display() + ); + return; + } + + let out = run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit( + &store, + &crate::policy::Policy::default(), + &tal_bytes, + &ta_der, + None, + &base_archive, + &base_locks, + &delta_archive, + &delta_locks, + validation_time, + validation_time, + &TreeRunConfig { + max_depth: Some(0), + max_instances: Some(1), + compact_audit: false, + persist_vcir: true, + build_ccr_accumulator: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + enable_transport_request_prefetch: false, + }, + ) + .expect("run delta replay root-only audit"); + + assert_eq!(out.tree.instances_processed, 1); + assert_eq!(out.tree.instances_failed, 0); + assert_eq!(out.publication_points.len(), 1); + } + + #[test] + fn payload_delta_replay_api_root_only_apnic_bundle_runs_with_timing() { + let temp = tempfile::tempdir().expect("tempdir"); + let db_path = temp.path().join("db"); + let store = crate::storage::RocksStore::open(&db_path).expect("open db"); + let ( + tal_bytes, + ta_der, + base_archive, + base_locks, + delta_archive, + delta_locks, + validation_time, + ) = apnic_delta_replay_inputs(); + if !base_archive.is_dir() + || !base_locks.is_file() + || !delta_archive.is_dir() + || !delta_locks.is_file() + { + eprintln!( + "skipping payload delta replay timing test; missing fixtures: base_archive={} base_locks={} delta_archive={} delta_locks={}", + base_archive.display(), + base_locks.display(), + delta_archive.display(), + delta_locks.display() + ); + return; + } + let timing = TimingHandle::new(TimingMeta { + recorded_at_utc_rfc3339: "2026-03-16T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-03-15T10:00:00Z".to_string(), + tal_url: None, + db_path: Some(db_path.to_string_lossy().into_owned()), + }); + let out = run_tree_from_tal_and_ta_der_payload_delta_replay_serial_audit_with_timing( + &store, + &crate::policy::Policy::default(), + &tal_bytes, + &ta_der, + None, + &base_archive, + &base_locks, + &delta_archive, + &delta_locks, + validation_time, + validation_time, + &TreeRunConfig { + max_depth: Some(0), + max_instances: Some(1), + compact_audit: false, + persist_vcir: true, + build_ccr_accumulator: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + enable_transport_request_prefetch: false, + }, + &timing, + ) + .expect("run delta replay root-only audit with timing"); + assert_eq!(out.tree.instances_processed, 1); + let timing_json = temp.path().join("timing_delta_replay.json"); + timing + .write_json(&timing_json, 20) + .expect("write timing json"); + let json: serde_json::Value = + serde_json::from_slice(&std::fs::read(&timing_json).expect("read timing json")) + .expect("parse timing json"); + assert_eq!( + json["phases"]["payload_delta_replay_base_total"]["count"].as_u64(), + Some(1) + ); + assert_eq!( + json["phases"]["payload_delta_replay_target_total"]["count"].as_u64(), + Some(1) + ); + } +} diff --git a/crates/panda-rpki-validator/src/validation/tree.rs b/crates/panda-rpki-validator/src/validation/tree.rs new file mode 100644 index 0000000..3561a2b --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree.rs @@ -0,0 +1,694 @@ +use crate::audit::{DiscoveredFrom, ObjectAuditEntry, PublicationPointAudit}; +use crate::cir::{CirInputAccumulator, CirInputSnapshot}; +use crate::data_model::rc::{AsResourceSet, IpResourceSet}; +use crate::report::Warning; +use crate::validation::manifest::PublicationPointSource; +use crate::validation::objects::{ + AspaAttestation, ObjectsOutput, RoaValidationCacheStats, RouterKeyPayload, Vrp, +}; +use crate::validation::publication_point::PublicationPointSnapshot; +use std::borrow::Cow; +use std::sync::{Arc, OnceLock}; + +pub const DEFAULT_MAX_CA_DEPTH: usize = 32; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TreeRunConfig { + /// Max CA instance depth to process (0 = root only). + pub max_depth: Option, + /// Max number of CA instances to process. + pub max_instances: Option, + /// Drop per-object audit payload when the caller only needs validation outputs. + pub compact_audit: bool, + /// Persist per-CA VCIR state for future reuse/delta runs. + pub persist_vcir: bool, + /// Build online CCR manifest projections during phase2 validation. + pub build_ccr_accumulator: bool, + /// Reuse accepted ROA validation outputs from previous VCIR when explicitly enabled. + pub enable_roa_validation_cache: bool, + /// Reuse validated child certificate discovery results when explicitly enabled. + pub enable_child_certificate_validation_cache: bool, + /// Evaluate publication-point cache eligibility without changing validation results. + pub publication_point_cache_observe_only: bool, + /// Reuse complete publication-point validation projections when explicitly enabled. + pub enable_publication_point_validation_cache: bool, + /// Prefetch transport requests from the previous run before normal tree traversal. + pub enable_transport_request_prefetch: bool, +} + +impl Default for TreeRunConfig { + fn default() -> Self { + Self { + max_depth: Some(DEFAULT_MAX_CA_DEPTH), + max_instances: None, + compact_audit: false, + persist_vcir: true, + build_ccr_accumulator: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + enable_transport_request_prefetch: false, + } + } +} + +pub(crate) fn ca_depth_is_allowed(config: &TreeRunConfig, depth: usize) -> bool { + config.max_depth.is_none_or(|max_depth| depth <= max_depth) +} + +pub(crate) fn next_allowed_ca_depth(config: &TreeRunConfig, parent_depth: usize) -> Option { + let child_depth = parent_depth.checked_add(1)?; + ca_depth_is_allowed(config, child_depth).then_some(child_depth) +} + +#[derive(Clone, Debug)] +pub enum CaCertificateRef { + InlineDer(Vec), + RepoBytes { + sha256_hex: String, + cached_der: Arc>>, + }, +} + +impl PartialEq for CaCertificateRef { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::InlineDer(left), Self::InlineDer(right)) => left == right, + ( + Self::RepoBytes { + sha256_hex: left, .. + }, + Self::RepoBytes { + sha256_hex: right, .. + }, + ) => left == right, + _ => false, + } + } +} + +impl Eq for CaCertificateRef {} + +impl CaCertificateRef { + pub fn inline_der(bytes: Vec) -> Self { + Self::InlineDer(bytes) + } + + pub fn repo_bytes(sha256_hex: String) -> Self { + Self::RepoBytes { + sha256_hex, + cached_der: Arc::new(OnceLock::new()), + } + } + + pub fn sha256_hex(&self) -> Option<&str> { + match self { + Self::InlineDer(_) => None, + Self::RepoBytes { sha256_hex, .. } => Some(sha256_hex.as_str()), + } + } + + pub fn der<'a>(&'a self, store: &crate::storage::RocksStore) -> Result, String> { + match self { + Self::InlineDer(bytes) => Ok(Cow::Borrowed(bytes.as_slice())), + Self::RepoBytes { + sha256_hex, + cached_der, + } => { + if cached_der.get().is_none() { + let bytes = store + .get_blob_bytes(sha256_hex) + .map_err(|e| format!("load CA certificate bytes failed: {e}"))? + .ok_or_else(|| { + format!("missing CA certificate repo bytes for sha256={sha256_hex}") + })?; + let _ = cached_der.set(Arc::from(bytes)); + } + let bytes = cached_der.get().ok_or_else(|| { + format!("missing cached CA certificate bytes for sha256={sha256_hex}") + })?; + Ok(Cow::Borrowed(bytes.as_ref())) + } + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct CaInstanceHandle { + pub depth: usize, + pub tal_id: String, + pub parent_manifest_rsync_uri: Option, + /// CA certificate bytes or a lazy repo-bytes reference for this CA instance. + pub ca_certificate: CaCertificateRef, + /// rsync URI of this CA certificate object (where it is published). + /// + /// This is used for strict AIA binding checks (RFC 6487 §4.8.7) when validating + /// subordinate CA and EE certificates. Trust anchor certificates downloaded from + /// TAL URIs may not have an rsync publication URI, so this can be `None`. + pub ca_certificate_rsync_uri: Option, + /// Effective (fully resolved) resources for this CA instance, used for resource-path + /// validation of subordinate CA certificates. + pub effective_ip_resources: Option, + pub effective_as_resources: Option, + pub rsync_base_uri: String, + pub manifest_rsync_uri: String, + pub publication_point_rsync_uri: String, + pub rrdp_notification_uri: Option, +} + +impl CaInstanceHandle { + pub fn with_depth(mut self, depth: usize) -> Self { + self.depth = depth; + self + } + + pub fn ca_certificate_der<'a>( + &'a self, + store: &crate::storage::RocksStore, + ) -> Result, String> { + self.ca_certificate.der(store) + } + + pub fn ca_certificate_sha256_hex(&self) -> Option<&str> { + self.ca_certificate.sha256_hex() + } + + pub fn ca_certificate_sha256_32(&self) -> Option<[u8; 32]> { + match &self.ca_certificate { + CaCertificateRef::InlineDer(bytes) => { + use sha2::Digest as _; + let digest = sha2::Sha256::digest(bytes); + Some(digest.into()) + } + CaCertificateRef::RepoBytes { sha256_hex, .. } => { + let mut out = [0u8; 32]; + hex::decode_to_slice(sha256_hex, &mut out).ok()?; + Some(out) + } + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PublicationPointRunResult { + pub source: PublicationPointSource, + pub snapshot: Option, + pub warnings: Vec, + pub objects: ObjectsOutput, + pub audit: PublicationPointAudit, + pub cir_fresh_objects: Vec, + pub cir_cached_objects: Vec, + /// Candidate child CA instances to enqueue after this publication point completes. + /// + /// - For `Fresh`, these are discovered from the current validated publication point. + /// - For `VcirCurrentInstance`, these are restored from the current instance VCIR and then + /// processed fresh-first when their own turn arrives. + pub discovered_children: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DiscoveredChildEntryProjection { + pub child_ski: String, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct DiscoveredChildCaInstance { + pub handle: CaInstanceHandle, + pub discovered_from: DiscoveredFrom, + pub child_entry_projection: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TreeRunOutput { + pub instances_processed: usize, + pub instances_failed: usize, + pub warnings: Vec, + pub vrps: Vec, + pub aspas: Vec, + pub router_keys: Vec, +} + +#[derive(Debug, thiserror::Error)] +pub enum TreeRunError { + #[error("publication point runner failed: {0}")] + Runner(String), +} + +pub trait PublicationPointRunner { + fn run_publication_point( + &self, + ca: &CaInstanceHandle, + ) -> Result; + + fn prefetch_discovered_children( + &self, + _children: &[DiscoveredChildCaInstance], + ) -> Result<(), String> { + Ok(()) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TreeRunAuditOutput { + pub tree: TreeRunOutput, + pub publication_points: Vec, + pub roa_cache_stats: RoaValidationCacheStats, + pub cir_input: CirInputSnapshot, +} + +pub(crate) fn submit_publication_point_cir_input( + cir_input: &mut CirInputAccumulator, + source: PublicationPointSource, + audit: &PublicationPointAudit, + cir_fresh_objects: &[ObjectAuditEntry], + cir_cached_objects: &[ObjectAuditEntry], +) -> Result<(), TreeRunError> { + match source { + PublicationPointSource::Fresh => { + cir_input + .submit_audit_entries(crate::cir::CirInputSection::Fresh, &audit.objects) + .map_err(|e| TreeRunError::Runner(e.to_string()))?; + } + PublicationPointSource::FailedFetchNoCache => { + let entries = if cir_fresh_objects.is_empty() { + &audit.objects + } else { + cir_fresh_objects + }; + cir_input + .submit_audit_entries(crate::cir::CirInputSection::Fresh, entries) + .map_err(|e| TreeRunError::Runner(e.to_string()))?; + } + PublicationPointSource::VcirCurrentInstance + | PublicationPointSource::PublicationPointCache => { + cir_input + .submit_audit_entries(crate::cir::CirInputSection::Fresh, cir_fresh_objects) + .map_err(|e| TreeRunError::Runner(e.to_string()))?; + cir_input + .submit_audit_entries(crate::cir::CirInputSection::Cached, cir_cached_objects) + .map_err(|e| TreeRunError::Runner(e.to_string()))?; + } + } + Ok(()) +} + +pub fn run_tree_serial( + root: CaInstanceHandle, + runner: &dyn PublicationPointRunner, + config: &TreeRunConfig, +) -> Result { + Ok(run_tree_serial_audit(root, runner, config)?.tree) +} + +pub fn run_tree_serial_audit( + root: CaInstanceHandle, + runner: &dyn PublicationPointRunner, + config: &TreeRunConfig, +) -> Result { + run_tree_serial_audit_multi_root(vec![root], runner, config) +} + +pub fn run_tree_serial_audit_multi_root( + roots: Vec, + runner: &dyn PublicationPointRunner, + config: &TreeRunConfig, +) -> Result { + #[derive(Clone, Debug)] + struct QueuedCaInstance { + id: u64, + handle: CaInstanceHandle, + parent_id: Option, + discovered_from: Option, + } + + let mut next_id: u64 = 0; + let mut queue: std::collections::VecDeque = std::collections::VecDeque::new(); + for root in roots { + queue.push_back(QueuedCaInstance { + id: next_id, + handle: root, + parent_id: None, + discovered_from: None, + }); + next_id += 1; + } + + let mut visited_manifest_uris: std::collections::HashSet = + std::collections::HashSet::new(); + let mut instances_processed = 0usize; + let mut instances_failed = 0usize; + let mut warnings: Vec = Vec::new(); + let mut vrps: Vec = Vec::new(); + let mut aspas: Vec = Vec::new(); + let mut router_keys: Vec = Vec::new(); + let mut publication_points: Vec = Vec::new(); + let mut roa_cache_stats = RoaValidationCacheStats::default(); + let mut cir_input = CirInputAccumulator::default(); + + while let Some(node) = queue.pop_front() { + let ca = &node.handle; + if !visited_manifest_uris.insert(ca.manifest_rsync_uri.clone()) { + continue; + } + + if let Some(max_instances) = config.max_instances { + if instances_processed >= max_instances { + break; + } + } + + if !ca_depth_is_allowed(config, ca.depth) { + continue; + } + + let res = match runner.run_publication_point(ca) { + Ok(v) => v, + Err(e) => { + instances_failed += 1; + warnings.push( + Warning::new(format!("publication point failed: {e}")) + .with_context(&ca.manifest_rsync_uri), + ); + continue; + } + }; + + instances_processed += 1; + warnings.extend(res.warnings); + warnings.extend(res.objects.warnings); + roa_cache_stats.add_assign(&res.objects.roa_cache_stats); + vrps.extend(res.objects.vrps); + aspas.extend(res.objects.aspas); + router_keys.extend(res.objects.router_keys); + + let source = res.source; + let cir_fresh_objects = res.cir_fresh_objects; + let cir_cached_objects = res.cir_cached_objects; + let mut audit = res.audit; + audit.node_id = Some(node.id); + audit.parent_node_id = node.parent_id; + audit.discovered_from = node.discovered_from.clone(); + submit_publication_point_cir_input( + &mut cir_input, + source, + &audit, + &cir_fresh_objects, + &cir_cached_objects, + )?; + if config.compact_audit { + audit.objects.clear(); + audit.warnings.clear(); + } + publication_points.push(audit); + + let Some(child_depth) = next_allowed_ca_depth(config, ca.depth) else { + continue; + }; + let mut children = res.discovered_children; + children.sort_by(|a, b| { + a.handle + .manifest_rsync_uri + .cmp(&b.handle.manifest_rsync_uri) + .then_with(|| { + a.discovered_from + .child_ca_certificate_rsync_uri + .cmp(&b.discovered_from.child_ca_certificate_rsync_uri) + }) + }); + if let Err(e) = runner.prefetch_discovered_children(&children) { + warnings.push( + Warning::new(format!("prefetch discovered children failed: {e}")) + .with_context(&ca.manifest_rsync_uri), + ); + } + for child in children { + queue.push_back(QueuedCaInstance { + id: next_id, + handle: child.handle.with_depth(child_depth), + parent_id: Some(node.id), + discovered_from: Some(child.discovered_from), + }); + next_id += 1; + } + } + + Ok(TreeRunAuditOutput { + tree: TreeRunOutput { + instances_processed, + instances_failed, + warnings, + vrps, + aspas, + router_keys, + }, + publication_points, + roa_cache_stats, + cir_input: cir_input.finalize(), + }) +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use crate::audit::{ + AuditObjectKind, AuditObjectResult, DiscoveredFrom, ObjectAuditEntry, PublicationPointAudit, + }; + use crate::validation::objects::{ObjectsOutput, ObjectsStats}; + + use super::{ + CaCertificateRef, CaInstanceHandle, DiscoveredChildCaInstance, PublicationPointRunResult, + PublicationPointRunner, TreeRunConfig, run_tree_serial_audit, + run_tree_serial_audit_multi_root, + }; + + fn sample_handle(manifest: &str) -> CaInstanceHandle { + CaInstanceHandle { + depth: 0, + tal_id: "arin".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(vec![1, 2, 3]), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/".to_string(), + manifest_rsync_uri: manifest.to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), + rrdp_notification_uri: None, + } + } + + struct PrefetchRecordingRunner { + seen_prefetch_children: Arc>>, + } + + impl PublicationPointRunner for PrefetchRecordingRunner { + fn prefetch_discovered_children( + &self, + children: &[DiscoveredChildCaInstance], + ) -> Result<(), String> { + let mut seen = self + .seen_prefetch_children + .lock() + .expect("prefetch lock poisoned"); + seen.extend( + children + .iter() + .map(|child| child.handle.manifest_rsync_uri.clone()), + ); + Ok(()) + } + + fn run_publication_point( + &self, + ca: &CaInstanceHandle, + ) -> Result { + let children = if ca.manifest_rsync_uri.ends_with("root.mft") { + vec![DiscoveredChildCaInstance { + handle: sample_handle("rsync://example.test/repo/child.mft"), + discovered_from: DiscoveredFrom { + parent_manifest_rsync_uri: ca.manifest_rsync_uri.clone(), + child_ca_certificate_rsync_uri: "rsync://example.test/repo/child.cer" + .to_string(), + child_ca_certificate_sha256_hex: "00".repeat(32), + }, + child_entry_projection: None, + }] + } else { + Vec::new() + }; + + Ok(PublicationPointRunResult { + source: crate::validation::manifest::PublicationPointSource::Fresh, + snapshot: None, + warnings: Vec::new(), + objects: ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: Vec::new(), + stats: ObjectsStats::default(), + audit: Vec::new(), + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + }, + audit: PublicationPointAudit::default(), + cir_fresh_objects: Vec::new(), + cir_cached_objects: Vec::new(), + discovered_children: children, + }) + } + } + + #[test] + fn run_tree_serial_audit_invokes_prefetch_hook_for_discovered_children() { + let seen = Arc::new(Mutex::new(Vec::new())); + let runner = PrefetchRecordingRunner { + seen_prefetch_children: Arc::clone(&seen), + }; + let out = run_tree_serial_audit( + sample_handle("rsync://example.test/repo/root.mft"), + &runner, + &TreeRunConfig::default(), + ) + .expect("tree run"); + assert_eq!(out.tree.instances_processed, 2); + assert_eq!( + seen.lock().expect("lock").as_slice(), + ["rsync://example.test/repo/child.mft"] + ); + } + + #[test] + fn run_tree_serial_audit_multi_root_processes_all_roots() { + let seen = Arc::new(Mutex::new(Vec::new())); + let runner = PrefetchRecordingRunner { + seen_prefetch_children: Arc::clone(&seen), + }; + let out = run_tree_serial_audit_multi_root( + vec![ + sample_handle("rsync://example.test/repo/root-a.mft"), + sample_handle("rsync://example.test/repo/root-b.mft"), + ], + &runner, + &TreeRunConfig::default(), + ) + .expect("tree run"); + assert_eq!(out.tree.instances_processed, 2); + assert_eq!(out.publication_points.len(), 2); + } + + #[test] + fn run_tree_serial_audit_accumulates_fresh_and_cached_cir_input() { + struct CirRunner; + + impl PublicationPointRunner for CirRunner { + fn run_publication_point( + &self, + ca: &CaInstanceHandle, + ) -> Result { + if ca.manifest_rsync_uri.ends_with("cached.mft") { + let fresh_reject = ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/cached/current.mft".to_string(), + sha256_hex: "11".repeat(32), + kind: AuditObjectKind::Manifest, + result: AuditObjectResult::Error, + detail: Some("fresh manifest rejected".to_string()), + }; + let cached = ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/cached/old.roa".to_string(), + sha256_hex: "22".repeat(32), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Ok, + detail: None, + }; + return Ok(PublicationPointRunResult { + source: + crate::validation::manifest::PublicationPointSource::VcirCurrentInstance, + snapshot: None, + warnings: Vec::new(), + objects: ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: Vec::new(), + stats: ObjectsStats::default(), + audit: Vec::new(), + roa_cache_stats: + crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + }, + audit: PublicationPointAudit { + objects: vec![fresh_reject.clone(), cached.clone()], + ..PublicationPointAudit::default() + }, + cir_fresh_objects: vec![fresh_reject], + cir_cached_objects: vec![cached], + discovered_children: Vec::new(), + }); + } + + let fresh = ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/fresh/a.roa".to_string(), + sha256_hex: "33".repeat(32), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Ok, + detail: None, + }; + Ok(PublicationPointRunResult { + source: crate::validation::manifest::PublicationPointSource::Fresh, + snapshot: None, + warnings: Vec::new(), + objects: ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: Vec::new(), + stats: ObjectsStats::default(), + audit: Vec::new(), + roa_cache_stats: + crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + }, + audit: PublicationPointAudit { + objects: vec![fresh], + ..PublicationPointAudit::default() + }, + cir_fresh_objects: Vec::new(), + cir_cached_objects: Vec::new(), + discovered_children: Vec::new(), + }) + } + } + + let out = run_tree_serial_audit_multi_root( + vec![ + sample_handle("rsync://example.test/repo/fresh/root.mft"), + sample_handle("rsync://example.test/repo/cached/cached.mft"), + ], + &CirRunner, + &TreeRunConfig::default(), + ) + .expect("tree run"); + + assert_eq!(out.cir_input.fresh_validated_objects.len(), 2); + assert_eq!(out.cir_input.cached_validated_objects.len(), 1); + assert_eq!(out.cir_input.fresh_rejected_objects.len(), 1); + assert!( + out.cir_input + .fresh_validated_objects + .iter() + .any(|item| item.rsync_uri == "rsync://example.test/repo/fresh/a.roa") + ); + assert!( + out.cir_input + .cached_validated_objects + .iter() + .any(|item| item.rsync_uri == "rsync://example.test/repo/cached/old.roa") + ); + } +} diff --git a/crates/panda-rpki-validator/src/validation/tree_parallel.rs b/crates/panda-rpki-validator/src/validation/tree_parallel.rs new file mode 100644 index 0000000..f7858f6 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_parallel.rs @@ -0,0 +1,3843 @@ +use std::collections::{HashMap, HashSet, VecDeque}; +use std::sync::mpsc::{self, Receiver, SyncSender, TryRecvError, TrySendError}; +use std::time::{Duration, Instant}; + +use crate::audit::{DiscoveredFrom, PublicationPointAudit}; +use crate::cir::CirInputAccumulator; +use crate::parallel::object_worker::{ + ObjectTaskExecutor, ObjectWorkerSubmitError, ScopedObjectWorkerPool, +}; +use crate::parallel::repo_runtime::{RepoSyncRequestStatus, RepoSyncRuntimeOutcome}; +use crate::parallel::types::RepoIdentity; +use crate::policy::SignedObjectFailurePolicy; +use crate::report::Warning; +use crate::storage::VcirReplaceTimingBreakdown; +use crate::validation::manifest::PublicationPointData; +use crate::validation::manifest::PublicationPointSource; +use crate::validation::objects::{ + ObjectsOutput, OwnedRoaTask, ParallelObjectsPrepare, ParallelObjectsStage, + RoaValidationCacheInput, + prepare_publication_point_for_parallel_roa_with_cache_and_ta_constraints, + reduce_parallel_roa_stage, +}; +use crate::validation::tree::{ + CaInstanceHandle, DiscoveredChildCaInstance, PublicationPointRunResult, PublicationPointRunner, + TreeRunAuditOutput, TreeRunConfig, TreeRunError, TreeRunOutput, ca_depth_is_allowed, + next_allowed_ca_depth, run_tree_serial_audit_multi_root, +}; +use crate::validation::tree_runner::{ + FreshPublicationPointFinalizeOutput, FreshPublicationPointStage, Rpkiv1PublicationPointRunner, +}; + +#[derive(Clone, Debug)] +struct QueuedCaInstance { + id: u64, + handle: CaInstanceHandle, + parent_id: Option, + discovered_from: Option, +} + +#[derive(Clone, Debug)] +struct ReadyCaInstance { + node: QueuedCaInstance, + repo_outcome: RepoSyncRuntimeOutcome, + ready_enqueued_at: Instant, +} + +struct InflightPublicationPoint { + node: QueuedCaInstance, + fresh_stage: FreshPublicationPointStage, + objects_prepare: ParallelObjectsPrepare, + repo_outcome: RepoSyncRuntimeOutcome, + warnings: Vec, + started_at: Instant, + objects_started_at: Instant, + task_count: usize, + tasks_submitted: usize, + first_task_submitted_at: Option, + last_task_submitted_at: Option, + first_result_at: Option, + last_result_at: Option, + worker_ms_total: u64, + worker_ms_max: u64, + queue_wait_ms_total: u64, + queue_wait_ms_max: u64, + finalize_enqueued_at: Option, + results: Vec, +} + +struct FinishedPublicationPoint { + node: FinishedPublicationPointNode, + result: FinishedPublicationPointResult, +} + +#[derive(Clone, Debug)] +struct FinishedPublicationPointNode { + id: u64, + parent_id: Option, + discovered_from: Option, + manifest_rsync_uri: String, +} + +impl FinishedPublicationPointNode { + fn from_queued(node: QueuedCaInstance) -> Self { + Self { + id: node.id, + parent_id: node.parent_id, + discovered_from: node.discovered_from, + manifest_rsync_uri: node.handle.manifest_rsync_uri, + } + } +} + +#[derive(Debug)] +enum FinishedPublicationPointResult { + Ok { + source: PublicationPointSource, + warnings: Vec, + objects: ObjectsOutput, + audit: PublicationPointAudit, + cir_fresh_objects: Vec, + cir_cached_objects: Vec, + }, + Err(String), +} + +struct FinalizeTask { + state: InflightPublicationPoint, +} + +/// Outcome of the pure compute phase for one ready publication point. +/// +/// `compute_ready_publication_point_stage` only performs read-only validation +/// work (publication point cache lookup, fresh snapshot staging, ROA prepare) +/// and returns this enum. `apply_ready_publication_point_stage` then performs +/// every write to control-loop state (`ca_queue`/`next_id`, `finished`, +/// `pending_roa_dispatch`, `pending_finalization`, `inflight_publication_points`) +/// in the same per-publication-point order the monolithic staging function did. +/// Each variant payload is boxed so the enum itself stays small. +enum StageOutcome { + /// Publication point cache hit: the run result is fully built in memory and + /// only needs child enqueueing plus a `finished` entry. + CacheHit(Box), + /// Fresh staging failed: apply runs the existing blocking + /// `run_publication_point` fallback inline on the control thread. + FreshError(Box), + /// Fresh staging succeeded and ROA prepare returned complete objects (no + /// ROA tasks to dispatch): apply hands the publication point to the + /// finalize worker through `pending_finalization` instead of running the + /// finalize synchronously on the control thread. + Complete(Box), + /// Fresh staging succeeded with a staged objects plan that contains zero + /// ROA tasks: apply queues the finalize task directly. + ZeroTask(Box), + /// Fresh staging succeeded with ROA tasks to dispatch: apply appends the + /// tasks to `pending_roa_dispatch` and registers the inflight publication + /// point. + Fresh(Box), +} + +struct CacheHitOutcome { + ready: ReadyCaInstance, + publication_point_started: Instant, + result: PublicationPointRunResult, +} + +struct FreshErrorOutcome { + ready: ReadyCaInstance, + publication_point_started: Instant, +} + +struct CompleteOutcome { + ready: ReadyCaInstance, + publication_point_started: Instant, + fresh_stage: FreshPublicationPointStage, + warnings: Vec, + objects: ObjectsOutput, +} + +struct StagedOutcome { + ready: ReadyCaInstance, + publication_point_started: Instant, + fresh_stage: FreshPublicationPointStage, + warnings: Vec, + objects_stage: ParallelObjectsStage, +} + +struct FinalizeWorkerResult { + finished: FinishedPublicationPoint, + metrics: FinalizePublicationPointMetrics, +} + +/// Task submitted to the experimental ready-stage worker pool: everything +/// `compute_ready_publication_point_stage` needs for one ready publication +/// point. +struct ReadyStageTask { + ready: ReadyCaInstance, + ready_queue_len_after_pop: usize, + submitted_at: Instant, +} + +/// Result drained from the ready-stage worker pool: the compute outcome and +/// its metrics plus per-task pool timing for the `phase2_stage_pool_stats` +/// observability event. +struct ReadyStageWorkerResult { + outcome: StageOutcome, + metrics: ReadyStageMetrics, + queue_wait_ms: u64, + worker_ms: u64, +} + +/// Executor borrowing the publication point runner so stage workers can run +/// the read-only compute phase off the control thread. The runner is shared +/// with the finalize worker and the ROA pool in the same way; the scoped pool +/// guarantees all borrows end before the enclosing `std::thread::scope`. +struct ReadyStageTaskExecutor<'a> { + runner: &'a Rpkiv1PublicationPointRunner<'a>, +} + +impl<'a> ObjectTaskExecutor for ReadyStageTaskExecutor<'a> { + fn execute(&self, _worker_index: usize, task: ReadyStageTask) -> ReadyStageWorkerResult { + let worker_started = Instant::now(); + let queue_wait_ms = worker_started + .saturating_duration_since(task.submitted_at) + .as_millis() as u64; + let (outcome, metrics) = compute_ready_publication_point_stage( + self.runner, + task.ready, + task.ready_queue_len_after_pop, + ); + ReadyStageWorkerResult { + outcome, + metrics, + queue_wait_ms, + worker_ms: elapsed_ms(worker_started), + } + } +} + +type ReadyStagePool<'scope, 'env> = ScopedObjectWorkerPool< + 'scope, + 'env, + ReadyStageTask, + ReadyStageWorkerResult, + ReadyStageTaskExecutor<'env>, +>; + +#[derive(Default)] +struct StageDispatchMetrics { + submitted: usize, + queue_full: bool, + duration_ms: u64, +} + +#[derive(Default)] +struct StageDrainMetrics { + results_drained: usize, + queue_wait_ms_total: u64, + queue_wait_ms_max: u64, + worker_ms_total: u64, + worker_ms_max: u64, + duration_ms: u64, +} + +#[derive(Default)] +struct ReadyStageMetrics { + manifest_rsync_uri: Option, + publication_point_rsync_uri: Option, + ready_count: usize, + fallback_count: usize, + complete_count: usize, + staged_count: usize, + zero_task_count: usize, + error_count: usize, + discovered_children: usize, + locked_files: usize, + roa_tasks: usize, + aspa_objects: usize, + stage_fresh_ms: u64, + snapshot_prepare_ms: u64, + snapshot_current_index_lock_ms: u64, + snapshot_manifest_load_ms: u64, + snapshot_manifest_index_lookup_ms: u64, + snapshot_manifest_blob_load_ms: u64, + snapshot_manifest_decode_ms: u64, + snapshot_replay_guard_ms: u64, + replay_meta_hit_count: usize, + replay_meta_miss_count: usize, + snapshot_manifest_entries_ms: u64, + snapshot_pack_files_ms: u64, + snapshot_pack_files_index_lookup_ms: u64, + snapshot_pack_files_blob_load_ms: u64, + snapshot_ee_path_validate_ms: u64, + snapshot_manifest_file_count: usize, + child_discovery_ms: u64, + child_enqueue_ms: u64, + ready_queue_wait_ms: u64, + ready_queue_len_after_pop: usize, + roa_presence_scan_ms: u64, + roa_cache_view_ms: u64, + direct_finalize_ms: u64, + fallback_full_run_ms: u64, + prepare_ms: u64, + build_roa_tasks_ms: u64, + total_ms: u64, +} + +#[derive(Default)] +struct ReadyStageBatchMetrics { + ready_count: usize, + fallback_count: usize, + complete_count: usize, + staged_count: usize, + zero_task_count: usize, + error_count: usize, + discovered_children: usize, + locked_files: usize, + roa_tasks: usize, + aspa_objects: usize, + stage_fresh_ms_total: u64, + stage_fresh_ms_max: u64, + stage_fresh_ms_max_manifest_rsync_uri: Option, + stage_fresh_ms_max_publication_point_rsync_uri: Option, + snapshot_prepare_ms_total: u64, + snapshot_prepare_ms_max: u64, + snapshot_current_index_lock_ms_total: u64, + snapshot_current_index_lock_ms_max: u64, + snapshot_manifest_load_ms_total: u64, + snapshot_manifest_load_ms_max: u64, + snapshot_manifest_index_lookup_ms_total: u64, + snapshot_manifest_index_lookup_ms_max: u64, + snapshot_manifest_blob_load_ms_total: u64, + snapshot_manifest_blob_load_ms_max: u64, + snapshot_manifest_decode_ms_total: u64, + snapshot_manifest_decode_ms_max: u64, + snapshot_replay_guard_ms_total: u64, + snapshot_replay_guard_ms_max: u64, + replay_meta_hit_count: usize, + replay_meta_miss_count: usize, + snapshot_manifest_entries_ms_total: u64, + snapshot_manifest_entries_ms_max: u64, + snapshot_pack_files_ms_total: u64, + snapshot_pack_files_ms_max: u64, + snapshot_pack_files_index_lookup_ms_total: u64, + snapshot_pack_files_index_lookup_ms_max: u64, + snapshot_pack_files_blob_load_ms_total: u64, + snapshot_pack_files_blob_load_ms_max: u64, + snapshot_ee_path_validate_ms_total: u64, + snapshot_ee_path_validate_ms_max: u64, + snapshot_manifest_file_count_total: usize, + snapshot_manifest_file_count_max: usize, + child_discovery_ms_total: u64, + child_discovery_ms_max: u64, + child_enqueue_ms_total: u64, + child_enqueue_ms_max: u64, + ready_queue_wait_ms_total: u64, + ready_queue_wait_ms_max: u64, + roa_presence_scan_ms_total: u64, + roa_presence_scan_ms_max: u64, + roa_cache_view_ms_total: u64, + roa_cache_view_ms_max: u64, + direct_finalize_ms_total: u64, + direct_finalize_ms_max: u64, + fallback_full_run_ms_total: u64, + fallback_full_run_ms_max: u64, + prepare_ms_total: u64, + prepare_ms_max: u64, + build_roa_tasks_ms_total: u64, + build_roa_tasks_ms_max: u64, + total_ms: u64, +} + +impl ReadyStageBatchMetrics { + fn record(&mut self, metrics: ReadyStageMetrics) { + self.ready_count += metrics.ready_count; + self.fallback_count += metrics.fallback_count; + self.complete_count += metrics.complete_count; + self.staged_count += metrics.staged_count; + self.zero_task_count += metrics.zero_task_count; + self.error_count += metrics.error_count; + self.discovered_children += metrics.discovered_children; + self.locked_files += metrics.locked_files; + self.roa_tasks += metrics.roa_tasks; + self.aspa_objects += metrics.aspa_objects; + if metrics.stage_fresh_ms >= self.stage_fresh_ms_max { + self.stage_fresh_ms_max_manifest_rsync_uri = metrics.manifest_rsync_uri.clone(); + self.stage_fresh_ms_max_publication_point_rsync_uri = + metrics.publication_point_rsync_uri.clone(); + } + self.stage_fresh_ms_total += metrics.stage_fresh_ms; + self.stage_fresh_ms_max = self.stage_fresh_ms_max.max(metrics.stage_fresh_ms); + self.snapshot_prepare_ms_total += metrics.snapshot_prepare_ms; + self.snapshot_prepare_ms_max = self + .snapshot_prepare_ms_max + .max(metrics.snapshot_prepare_ms); + self.snapshot_current_index_lock_ms_total += metrics.snapshot_current_index_lock_ms; + self.snapshot_current_index_lock_ms_max = self + .snapshot_current_index_lock_ms_max + .max(metrics.snapshot_current_index_lock_ms); + self.snapshot_manifest_load_ms_total += metrics.snapshot_manifest_load_ms; + self.snapshot_manifest_load_ms_max = self + .snapshot_manifest_load_ms_max + .max(metrics.snapshot_manifest_load_ms); + self.snapshot_manifest_index_lookup_ms_total += metrics.snapshot_manifest_index_lookup_ms; + self.snapshot_manifest_index_lookup_ms_max = self + .snapshot_manifest_index_lookup_ms_max + .max(metrics.snapshot_manifest_index_lookup_ms); + self.snapshot_manifest_blob_load_ms_total += metrics.snapshot_manifest_blob_load_ms; + self.snapshot_manifest_blob_load_ms_max = self + .snapshot_manifest_blob_load_ms_max + .max(metrics.snapshot_manifest_blob_load_ms); + self.snapshot_manifest_decode_ms_total += metrics.snapshot_manifest_decode_ms; + self.snapshot_manifest_decode_ms_max = self + .snapshot_manifest_decode_ms_max + .max(metrics.snapshot_manifest_decode_ms); + self.snapshot_replay_guard_ms_total += metrics.snapshot_replay_guard_ms; + self.snapshot_replay_guard_ms_max = self + .snapshot_replay_guard_ms_max + .max(metrics.snapshot_replay_guard_ms); + self.replay_meta_hit_count += metrics.replay_meta_hit_count; + self.replay_meta_miss_count += metrics.replay_meta_miss_count; + self.snapshot_manifest_entries_ms_total += metrics.snapshot_manifest_entries_ms; + self.snapshot_manifest_entries_ms_max = self + .snapshot_manifest_entries_ms_max + .max(metrics.snapshot_manifest_entries_ms); + self.snapshot_pack_files_ms_total += metrics.snapshot_pack_files_ms; + self.snapshot_pack_files_ms_max = self + .snapshot_pack_files_ms_max + .max(metrics.snapshot_pack_files_ms); + self.snapshot_pack_files_index_lookup_ms_total += + metrics.snapshot_pack_files_index_lookup_ms; + self.snapshot_pack_files_index_lookup_ms_max = self + .snapshot_pack_files_index_lookup_ms_max + .max(metrics.snapshot_pack_files_index_lookup_ms); + self.snapshot_pack_files_blob_load_ms_total += metrics.snapshot_pack_files_blob_load_ms; + self.snapshot_pack_files_blob_load_ms_max = self + .snapshot_pack_files_blob_load_ms_max + .max(metrics.snapshot_pack_files_blob_load_ms); + self.snapshot_ee_path_validate_ms_total += metrics.snapshot_ee_path_validate_ms; + self.snapshot_ee_path_validate_ms_max = self + .snapshot_ee_path_validate_ms_max + .max(metrics.snapshot_ee_path_validate_ms); + self.snapshot_manifest_file_count_total += metrics.snapshot_manifest_file_count; + self.snapshot_manifest_file_count_max = self + .snapshot_manifest_file_count_max + .max(metrics.snapshot_manifest_file_count); + self.child_discovery_ms_total += metrics.child_discovery_ms; + self.child_discovery_ms_max = self.child_discovery_ms_max.max(metrics.child_discovery_ms); + self.child_enqueue_ms_total += metrics.child_enqueue_ms; + self.child_enqueue_ms_max = self.child_enqueue_ms_max.max(metrics.child_enqueue_ms); + self.ready_queue_wait_ms_total += metrics.ready_queue_wait_ms; + self.ready_queue_wait_ms_max = self + .ready_queue_wait_ms_max + .max(metrics.ready_queue_wait_ms); + self.roa_presence_scan_ms_total += metrics.roa_presence_scan_ms; + self.roa_presence_scan_ms_max = self + .roa_presence_scan_ms_max + .max(metrics.roa_presence_scan_ms); + self.roa_cache_view_ms_total += metrics.roa_cache_view_ms; + self.roa_cache_view_ms_max = self.roa_cache_view_ms_max.max(metrics.roa_cache_view_ms); + self.direct_finalize_ms_total += metrics.direct_finalize_ms; + self.direct_finalize_ms_max = self.direct_finalize_ms_max.max(metrics.direct_finalize_ms); + self.fallback_full_run_ms_total += metrics.fallback_full_run_ms; + self.fallback_full_run_ms_max = self + .fallback_full_run_ms_max + .max(metrics.fallback_full_run_ms); + self.prepare_ms_total += metrics.prepare_ms; + self.prepare_ms_max = self.prepare_ms_max.max(metrics.prepare_ms); + self.build_roa_tasks_ms_total += metrics.build_roa_tasks_ms; + self.build_roa_tasks_ms_max = self.build_roa_tasks_ms_max.max(metrics.build_roa_tasks_ms); + self.total_ms += metrics.total_ms; + } +} + +#[derive(Default)] +struct RoaDispatchMetrics { + attempted: usize, + submitted: usize, + queue_full: bool, + pending_remaining: usize, + duration_ms: u64, +} + +#[derive(Default)] +struct ObjectDrainMetrics { + results_drained: usize, + publication_points_completed: usize, + worker_ms_total: u64, + worker_ms_max: u64, + queue_wait_ms_total: u64, + queue_wait_ms_max: u64, + result_budget_exhausted: bool, + duration_ms: u64, +} + +#[derive(Default)] +struct FinalizeSubmitMetrics { + submitted: usize, + queue_full: bool, + duration_ms: u64, +} + +#[derive(Default)] +struct FinalizePublicationPointMetrics { + reduce_ms: u64, + finalize_ms: u64, + finalize_queue_wait_ms: Option, + finalize_worker_ms: u64, + snapshot_pack_ms: u64, + persist_vcir_ms: u64, + persist_build_vcir_ms: u64, + persist_replace_vcir_ms: u64, + persist_replace_breakdown: VcirReplaceTimingBreakdown, + ccr_projection_build_ms: u64, + ccr_append_ms: u64, + audit_build_ms: u64, + locked_files: usize, + child_count: usize, + warning_count: usize, + vrp_count: usize, + vap_count: usize, + router_key_count: usize, + audit_object_count: usize, +} + +#[derive(Default)] +struct FinalizeResultsDrainMetrics { + results_drained: usize, + reduce_ms_total: u64, + reduce_ms_max: u64, + finalize_ms_total: u64, + finalize_ms_max: u64, + finalize_queue_wait_ms_max: u64, + finalize_worker_ms_total: u64, + finalize_worker_ms_max: u64, + snapshot_pack_ms_total: u64, + snapshot_pack_ms_max: u64, + persist_vcir_ms_total: u64, + persist_vcir_ms_max: u64, + persist_build_vcir_ms_total: u64, + persist_build_vcir_ms_max: u64, + persist_replace_vcir_ms_total: u64, + persist_replace_vcir_ms_max: u64, + ccr_projection_build_ms_total: u64, + ccr_projection_build_ms_max: u64, + ccr_append_ms_total: u64, + ccr_append_ms_max: u64, + audit_build_ms_total: u64, + audit_build_ms_max: u64, + duration_ms: u64, +} + +#[derive(Default)] +struct RepoDrainMetrics { + event_count: usize, + completions: usize, + ready_enqueued: usize, + duration_ms: u64, +} + +const REPO_RESULT_DRAIN_MAX_EVENTS: usize = 64; + +fn elapsed_ms(started: Instant) -> u64 { + started.elapsed().as_millis() as u64 +} + +fn emit_control_loop_slow( + duration_ms: u64, + repo_poll_timeout: Duration, + repo_metrics: &RepoDrainMetrics, + ready_batch_metrics: &ReadyStageBatchMetrics, + ca_queue_len: usize, + ready_queue_len: usize, + ca_waiting_repo_identities: usize, + pending_roa_dispatch_len: usize, + inflight_publication_points_len: usize, + pending_finalization_len: usize, + finalize_inflight: usize, +) { + let threshold_ms = crate::progress_log::control_loop_slow_threshold_ms(); + if duration_ms < threshold_ms { + return; + } + crate::progress_log::emit( + "phase2_control_loop_slow", + serde_json::json!({ + "duration_ms": duration_ms, + "slow_threshold_ms": threshold_ms, + "repo_poll_timeout_ms": repo_poll_timeout.as_millis() as u64, + "repo_event_count": repo_metrics.event_count, + "repo_completions": repo_metrics.completions, + "repo_ready_enqueued": repo_metrics.ready_enqueued, + "repo_drain_duration_ms": repo_metrics.duration_ms, + "ready_count": ready_batch_metrics.ready_count, + "ready_batch_duration_ms": ready_batch_metrics.total_ms, + "ready_batch_stage_fresh_ms_total": ready_batch_metrics.stage_fresh_ms_total, + "ready_batch_stage_fresh_ms_max": ready_batch_metrics.stage_fresh_ms_max, + "ready_batch_stage_fresh_ms_max_manifest_rsync_uri": ready_batch_metrics.stage_fresh_ms_max_manifest_rsync_uri, + "ready_batch_child_discovery_ms_total": ready_batch_metrics.child_discovery_ms_total, + "ready_batch_child_discovery_ms_max": ready_batch_metrics.child_discovery_ms_max, + "ready_batch_prepare_ms_total": ready_batch_metrics.prepare_ms_total, + "ready_batch_prepare_ms_max": ready_batch_metrics.prepare_ms_max, + "ready_batch_direct_finalize_ms_total": ready_batch_metrics.direct_finalize_ms_total, + "ready_batch_direct_finalize_ms_max": ready_batch_metrics.direct_finalize_ms_max, + "ca_queue_len": ca_queue_len, + "ready_queue_len": ready_queue_len, + "ca_waiting_repo_identities": ca_waiting_repo_identities, + "pending_roa_dispatch_len": pending_roa_dispatch_len, + "inflight_publication_points_len": inflight_publication_points_len, + "pending_finalization_len": pending_finalization_len, + "finalize_inflight": finalize_inflight, + }), + ); +} + +fn compact_phase2_finished_result( + mut result: PublicationPointRunResult, + compact_audit: bool, +) -> FinishedPublicationPointResult { + result.objects.audit.clear(); + result.objects.local_outputs_cache.clear(); + let cir_fresh_objects = if compact_audit && result.source == PublicationPointSource::Fresh { + result.audit.objects.clone() + } else { + result.cir_fresh_objects + }; + if compact_audit { + result.audit.objects.clear(); + result.audit.warnings.clear(); + } + FinishedPublicationPointResult::Ok { + source: result.source, + warnings: result.warnings, + objects: result.objects, + audit: result.audit, + cir_fresh_objects, + cir_cached_objects: result.cir_cached_objects, + } +} + +fn compact_phase2_finished_result_result( + result: Result, + compact_audit: bool, +) -> FinishedPublicationPointResult { + match result { + Ok(result) => compact_phase2_finished_result(result, compact_audit), + Err(err) => FinishedPublicationPointResult::Err(err), + } +} + +pub fn run_tree_parallel_phase2_audit_multi_root( + roots: Vec, + runner: &Rpkiv1PublicationPointRunner<'_>, + config: &TreeRunConfig, +) -> Result { + if runner.policy.signed_object_failure_policy == SignedObjectFailurePolicy::DropPublicationPoint + { + return run_tree_serial_audit_multi_root(roots, runner, config); + } + + let Some(repo_runtime) = runner.repo_sync_runtime.as_ref() else { + return run_tree_serial_audit_multi_root(roots, runner, config); + }; + if runner.parallel_roa_worker_pool.is_none() { + return run_tree_serial_audit_multi_root(roots, runner, config); + } + + let mut next_id: u64 = 0; + let mut ca_queue: VecDeque = VecDeque::new(); + for root in roots { + ca_queue.push_back(QueuedCaInstance { + id: next_id, + handle: root, + parent_id: None, + discovered_from: None, + }); + next_id += 1; + } + + let mut visited_manifest_uris: HashSet = HashSet::new(); + let mut ca_waiting_repo_by_identity: HashMap> = + HashMap::new(); + let mut ready_queue: VecDeque = VecDeque::new(); + let mut inflight_publication_points: HashMap = HashMap::new(); + let mut pending_finalization: VecDeque = VecDeque::new(); + let mut pending_roa_dispatch: VecDeque = VecDeque::new(); + let mut finished: Vec = Vec::new(); + let mut instances_started = 0usize; + let phase2_config = runner.parallel_phase2_config.as_ref(); + let ready_batch_size = phase2_config + .map(|cfg| cfg.ready_batch_size) + .unwrap_or(256) + .max(1); + let ready_batch_wall_time_budget_ms = phase2_config + .map(|cfg| cfg.ready_batch_wall_time_budget_ms) + .unwrap_or(100) + .max(1); + let ready_batch_wall_time_budget = Duration::from_millis(ready_batch_wall_time_budget_ms); + let object_result_drain_batch_size = phase2_config + .map(|cfg| cfg.object_result_drain_batch_size) + .unwrap_or(2048) + .max(1); + let publication_point_finalize_queue_capacity = phase2_config + .map(|cfg| cfg.publication_point_finalize_queue_capacity) + .unwrap_or(32768) + .max(1); + // Experimental ready-stage pool: `stage_workers == 0` keeps the inline + // compute+apply staging path byte-for-byte; any positive value moves the + // compute phase to a scoped worker pool. + let stage_worker_count = phase2_config.map(|cfg| cfg.stage_workers).unwrap_or(0); + let stage_queue_capacity = phase2_config + .map(|cfg| cfg.worker_queue_capacity) + .unwrap_or(256) + .max(1); + + let (finalize_task_tx, finalize_task_rx) = + mpsc::sync_channel::(publication_point_finalize_queue_capacity); + let (finalize_result_tx, finalize_result_rx) = mpsc::channel::(); + let mut finalize_inflight = 0usize; + + return std::thread::scope(|scope| { + let finalize_worker = scope.spawn(move || { + run_finalize_worker( + runner, + finalize_task_rx, + finalize_result_tx, + config.compact_audit, + ) + }); + + // The stage pool borrows the runner like the finalize worker does; it + // lives inside this scope and is dropped before the scope joins. + let mut stage_pool = if stage_worker_count > 0 { + Some( + ReadyStagePool::new( + scope, + stage_worker_count, + stage_queue_capacity, + ReadyStageTaskExecutor { runner }, + ) + .map_err(TreeRunError::Runner)?, + ) + } else { + None + }; + // Submitted-but-not-yet-drained stage tasks. The drain loop collects + // every available result each turn, so `staging_inflight == 0` also + // implies the stage result channel is empty. + let mut staging_inflight = 0usize; + + let run_result: Result<(), TreeRunError> = (|| { + loop { + let control_loop_started = Instant::now(); + // With the stage pool enabled the batch wall clock covers the + // whole turn (turn-head drain + dispatch + apply); the inline + // path keeps its historical start point at the ready batch. + let turn_stage_started = Instant::now(); + let mut ready_batch_metrics = ReadyStageBatchMetrics::default(); + let mut stage_drain_metrics = StageDrainMetrics::default(); + drain_finalize_results_with_progress( + &finalize_result_rx, + &mut finished, + &mut finalize_inflight, + pending_finalization.len(), + pending_roa_dispatch.len(), + inflight_publication_points.len(), + )?; + flush_pending_roa_dispatch_with_progress( + runner, + &mut pending_roa_dispatch, + &mut inflight_publication_points, + &pending_finalization, + )?; + drain_object_results_with_progress( + runner, + &mut inflight_publication_points, + &mut pending_finalization, + pending_roa_dispatch.len(), + object_result_drain_batch_size, + )?; + if let Some(pool) = stage_pool.as_ref() { + drain_stage_results( + pool, + runner, + &mut next_id, + &mut ca_queue, + &mut pending_roa_dispatch, + &mut inflight_publication_points, + &mut pending_finalization, + &mut finished, + &mut staging_inflight, + &mut ready_batch_metrics, + &mut stage_drain_metrics, + config, + )?; + } + submit_pending_finalization_with_progress( + &finalize_task_tx, + &mut pending_finalization, + &mut finalize_inflight, + publication_point_finalize_queue_capacity, + pending_roa_dispatch.len(), + inflight_publication_points.len(), + )?; + + start_queued_ca_instances( + repo_runtime.as_ref(), + &mut ca_queue, + &mut ready_queue, + &mut ca_waiting_repo_by_identity, + &mut finished, + &mut visited_manifest_uris, + &mut instances_started, + config, + ); + + let repo_poll_timeout = event_poll_timeout( + &ca_queue, + &ready_queue, + &pending_roa_dispatch, + &inflight_publication_points, + &pending_finalization, + finalize_inflight, + staging_inflight, + instances_started, + config, + ); + let repo_metrics = drain_repo_events( + repo_runtime.as_ref(), + &mut ca_waiting_repo_by_identity, + &mut ready_queue, + repo_poll_timeout, + )?; + if repo_metrics.event_count > 0 { + crate::progress_log::emit( + "phase2_repo_events_drain", + serde_json::json!({ + "event_count": repo_metrics.event_count, + "completions": repo_metrics.completions, + "ready_enqueued": repo_metrics.ready_enqueued, + "duration_ms": repo_metrics.duration_ms, + "ready_queue_len": ready_queue.len(), + "ca_waiting_repo_identities": ca_waiting_repo_by_identity.len(), + }), + ); + } + + let ready_batch_started = Instant::now(); + let mut ready_time_budget_exhausted = false; + let mut stage_dispatch_metrics = StageDispatchMetrics::default(); + if let Some(pool) = stage_pool.as_mut() { + // Pool path: the ready batch becomes a dispatch loop that + // submits compute tasks and applies whatever results are + // already available; backpressure requeues for next turn. + stage_dispatch_metrics = submit_ready_batch_to_stage_pool( + pool, + &mut ready_queue, + &mut staging_inflight, + ready_batch_size, + ready_batch_wall_time_budget, + )?; + ready_time_budget_exhausted = stage_dispatch_metrics.queue_full + || (!ready_queue.is_empty() + && ready_batch_started.elapsed() >= ready_batch_wall_time_budget); + drain_stage_results( + pool, + runner, + &mut next_id, + &mut ca_queue, + &mut pending_roa_dispatch, + &mut inflight_publication_points, + &mut pending_finalization, + &mut finished, + &mut staging_inflight, + &mut ready_batch_metrics, + &mut stage_drain_metrics, + config, + )?; + } else { + while ready_batch_metrics.ready_count < ready_batch_size { + let Some(ready) = ready_queue.pop_front() else { + break; + }; + let ready_queue_len_after_pop = ready_queue.len(); + let (outcome, metrics) = compute_ready_publication_point_stage( + runner, + ready, + ready_queue_len_after_pop, + ); + let metrics = apply_ready_publication_point_stage( + runner, + &mut next_id, + &mut ca_queue, + &mut pending_roa_dispatch, + &mut inflight_publication_points, + &mut pending_finalization, + &mut finished, + outcome, + metrics, + config, + config.compact_audit, + ); + ready_batch_metrics.record(metrics); + if ready_batch_metrics.ready_count > 0 + && ready_batch_started.elapsed() >= ready_batch_wall_time_budget + { + ready_time_budget_exhausted = !ready_queue.is_empty(); + break; + } + } + } + if ready_batch_metrics.ready_count > 0 { + ready_batch_metrics.total_ms = if stage_pool.is_some() { + elapsed_ms(turn_stage_started) + } else { + elapsed_ms(ready_batch_started) + }; + let ready_count_budget_exhausted = ready_batch_metrics.ready_count + >= ready_batch_size + && !ready_queue.is_empty(); + ready_time_budget_exhausted = ready_time_budget_exhausted + || (!ready_queue.is_empty() + && ready_batch_metrics.total_ms >= ready_batch_wall_time_budget_ms); + emit_ready_queue_batch_progress( + &ready_batch_metrics, + ready_batch_size, + ready_batch_wall_time_budget_ms, + ready_queue.len(), + ready_count_budget_exhausted, + ready_time_budget_exhausted, + ca_queue.len(), + pending_roa_dispatch.len(), + inflight_publication_points.len(), + pending_finalization.len(), + finalize_inflight, + ); + crate::progress_log::emit( + "phase2_ready_queue_stage_fresh_breakdown", + serde_json::json!({ + "ready_count": ready_batch_metrics.ready_count, + "stage_fresh_ms_total": ready_batch_metrics.stage_fresh_ms_total, + "stage_fresh_ms_max": ready_batch_metrics.stage_fresh_ms_max, + "stage_fresh_ms_max_manifest_rsync_uri": ready_batch_metrics.stage_fresh_ms_max_manifest_rsync_uri, + "stage_fresh_ms_max_publication_point_rsync_uri": ready_batch_metrics.stage_fresh_ms_max_publication_point_rsync_uri, + "snapshot_prepare_ms_total": ready_batch_metrics.snapshot_prepare_ms_total, + "snapshot_prepare_ms_max": ready_batch_metrics.snapshot_prepare_ms_max, + "snapshot_current_index_lock_ms_total": ready_batch_metrics.snapshot_current_index_lock_ms_total, + "snapshot_current_index_lock_ms_max": ready_batch_metrics.snapshot_current_index_lock_ms_max, + "snapshot_manifest_load_ms_total": ready_batch_metrics.snapshot_manifest_load_ms_total, + "snapshot_manifest_load_ms_max": ready_batch_metrics.snapshot_manifest_load_ms_max, + "snapshot_manifest_index_lookup_ms_total": ready_batch_metrics.snapshot_manifest_index_lookup_ms_total, + "snapshot_manifest_index_lookup_ms_max": ready_batch_metrics.snapshot_manifest_index_lookup_ms_max, + "snapshot_manifest_blob_load_ms_total": ready_batch_metrics.snapshot_manifest_blob_load_ms_total, + "snapshot_manifest_blob_load_ms_max": ready_batch_metrics.snapshot_manifest_blob_load_ms_max, + "snapshot_manifest_decode_ms_total": ready_batch_metrics.snapshot_manifest_decode_ms_total, + "snapshot_manifest_decode_ms_max": ready_batch_metrics.snapshot_manifest_decode_ms_max, + "snapshot_replay_guard_ms_total": ready_batch_metrics.snapshot_replay_guard_ms_total, + "snapshot_replay_guard_ms_max": ready_batch_metrics.snapshot_replay_guard_ms_max, + "replay_meta_hit_count": ready_batch_metrics.replay_meta_hit_count, + "replay_meta_miss_count": ready_batch_metrics.replay_meta_miss_count, + "snapshot_manifest_entries_ms_total": ready_batch_metrics.snapshot_manifest_entries_ms_total, + "snapshot_manifest_entries_ms_max": ready_batch_metrics.snapshot_manifest_entries_ms_max, + "snapshot_pack_files_ms_total": ready_batch_metrics.snapshot_pack_files_ms_total, + "snapshot_pack_files_ms_max": ready_batch_metrics.snapshot_pack_files_ms_max, + "snapshot_pack_files_index_lookup_ms_total": ready_batch_metrics.snapshot_pack_files_index_lookup_ms_total, + "snapshot_pack_files_index_lookup_ms_max": ready_batch_metrics.snapshot_pack_files_index_lookup_ms_max, + "snapshot_pack_files_blob_load_ms_total": ready_batch_metrics.snapshot_pack_files_blob_load_ms_total, + "snapshot_pack_files_blob_load_ms_max": ready_batch_metrics.snapshot_pack_files_blob_load_ms_max, + "snapshot_ee_path_validate_ms_total": ready_batch_metrics.snapshot_ee_path_validate_ms_total, + "snapshot_ee_path_validate_ms_max": ready_batch_metrics.snapshot_ee_path_validate_ms_max, + "snapshot_manifest_file_count_total": ready_batch_metrics.snapshot_manifest_file_count_total, + "snapshot_manifest_file_count_max": ready_batch_metrics.snapshot_manifest_file_count_max, + "child_discovery_ms_total": ready_batch_metrics.child_discovery_ms_total, + "child_discovery_ms_max": ready_batch_metrics.child_discovery_ms_max, + "batch_duration_ms": ready_batch_metrics.total_ms, + }), + ); + } + if stage_pool.is_some() + && (stage_dispatch_metrics.submitted > 0 + || stage_drain_metrics.results_drained > 0 + || stage_dispatch_metrics.queue_full) + { + crate::progress_log::emit( + "phase2_stage_pool_stats", + serde_json::json!({ + "stage_workers": stage_worker_count, + "submitted": stage_dispatch_metrics.submitted, + "results_drained": stage_drain_metrics.results_drained, + "queue_full": stage_dispatch_metrics.queue_full, + "staging_inflight": staging_inflight, + "ready_queue_len": ready_queue.len(), + "queue_wait_ms_total": stage_drain_metrics.queue_wait_ms_total, + "queue_wait_ms_max": stage_drain_metrics.queue_wait_ms_max, + "worker_ms_total": stage_drain_metrics.worker_ms_total, + "worker_ms_max": stage_drain_metrics.worker_ms_max, + "dispatch_duration_ms": stage_dispatch_metrics.duration_ms, + "drain_duration_ms": stage_drain_metrics.duration_ms, + }), + ); + } + + flush_pending_roa_dispatch_with_progress( + runner, + &mut pending_roa_dispatch, + &mut inflight_publication_points, + &pending_finalization, + )?; + drain_object_results_with_progress( + runner, + &mut inflight_publication_points, + &mut pending_finalization, + pending_roa_dispatch.len(), + object_result_drain_batch_size, + )?; + submit_pending_finalization_with_progress( + &finalize_task_tx, + &mut pending_finalization, + &mut finalize_inflight, + publication_point_finalize_queue_capacity, + pending_roa_dispatch.len(), + inflight_publication_points.len(), + )?; + drain_finalize_results_with_progress( + &finalize_result_rx, + &mut finished, + &mut finalize_inflight, + pending_finalization.len(), + pending_roa_dispatch.len(), + inflight_publication_points.len(), + )?; + + emit_control_loop_slow( + elapsed_ms(control_loop_started), + repo_poll_timeout, + &repo_metrics, + &ready_batch_metrics, + ca_queue.len(), + ready_queue.len(), + ca_waiting_repo_by_identity.len(), + pending_roa_dispatch.len(), + inflight_publication_points.len(), + pending_finalization.len(), + finalize_inflight, + ); + + if is_complete( + &ca_queue, + &ready_queue, + &ca_waiting_repo_by_identity, + &pending_roa_dispatch, + &inflight_publication_points, + &pending_finalization, + finalize_inflight, + staging_inflight, + instances_started, + config, + ) { + break; + } + } + + repo_runtime + .reset_run_state() + .map_err(TreeRunError::Runner)?; + Ok(()) + })(); + + drop(finalize_task_tx); + // Dropping the pool closes the stage task queues so the scoped stage + // workers exit before the scope joins them below. + drop(stage_pool); + let worker_result = finalize_worker + .join() + .map_err(|_| TreeRunError::Runner("phase2 finalize worker panicked".to_string()))?; + run_result?; + worker_result?; + drain_finalize_results_with_progress( + &finalize_result_rx, + &mut finished, + &mut finalize_inflight, + pending_finalization.len(), + pending_roa_dispatch.len(), + inflight_publication_points.len(), + )?; + if finalize_inflight != 0 || !pending_finalization.is_empty() { + return Err(TreeRunError::Runner(format!( + "phase2 finalize worker stopped with pending work: queued={} inflight={}", + pending_finalization.len(), + finalize_inflight + ))); + } + Ok(build_tree_output(finished)) + }); +} + +fn emit_ready_queue_batch_progress( + metrics: &ReadyStageBatchMetrics, + ready_batch_size: usize, + ready_batch_wall_time_budget_ms: u64, + ready_queue_len_after_batch: usize, + ready_count_budget_exhausted: bool, + ready_time_budget_exhausted: bool, + ca_queue_len_after_batch: usize, + pending_roa_dispatch_len_after_batch: usize, + inflight_publication_points_after_batch: usize, + pending_finalization_len_after_batch: usize, + finalize_inflight_after_batch: usize, +) { + crate::progress_log::emit( + "phase2_ready_queue_batch", + serde_json::json!({ + "ready_count": metrics.ready_count, + "fallback_count": metrics.fallback_count, + "complete_count": metrics.complete_count, + "staged_count": metrics.staged_count, + "zero_task_count": metrics.zero_task_count, + "error_count": metrics.error_count, + "discovered_children": metrics.discovered_children, + "locked_files": metrics.locked_files, + "roa_tasks": metrics.roa_tasks, + "aspa_objects": metrics.aspa_objects, + "stage_fresh_ms_total": metrics.stage_fresh_ms_total, + "stage_fresh_ms_max": metrics.stage_fresh_ms_max, + "stage_fresh_ms_max_manifest_rsync_uri": metrics.stage_fresh_ms_max_manifest_rsync_uri, + "stage_fresh_ms_max_publication_point_rsync_uri": metrics.stage_fresh_ms_max_publication_point_rsync_uri, + "prepare_ms_total": metrics.prepare_ms_total, + "prepare_ms_max": metrics.prepare_ms_max, + "build_roa_tasks_ms_total": metrics.build_roa_tasks_ms_total, + "build_roa_tasks_ms_max": metrics.build_roa_tasks_ms_max, + "batch_duration_ms": metrics.total_ms, + "ready_batch_size": ready_batch_size, + "ready_batch_wall_time_budget_ms": ready_batch_wall_time_budget_ms, + "ready_queue_len_after_batch": ready_queue_len_after_batch, + "ready_queue_budget_exhausted": ready_queue_len_after_batch > 0, + "ready_count_budget_exhausted": ready_count_budget_exhausted, + "ready_time_budget_exhausted": ready_time_budget_exhausted, + "ca_queue_len_after_batch": ca_queue_len_after_batch, + "pending_roa_dispatch_len_after_batch": pending_roa_dispatch_len_after_batch, + "inflight_publication_points_after_batch": inflight_publication_points_after_batch, + "pending_finalization_len_after_batch": pending_finalization_len_after_batch, + "finalize_inflight_after_batch": finalize_inflight_after_batch, + }), + ); + crate::progress_log::emit( + "phase2_ready_queue_control_breakdown", + serde_json::json!({ + "ready_count": metrics.ready_count, + "ready_queue_wait_ms_total": metrics.ready_queue_wait_ms_total, + "ready_queue_wait_ms_max": metrics.ready_queue_wait_ms_max, + "child_enqueue_ms_total": metrics.child_enqueue_ms_total, + "child_enqueue_ms_max": metrics.child_enqueue_ms_max, + "roa_presence_scan_ms_total": metrics.roa_presence_scan_ms_total, + "roa_presence_scan_ms_max": metrics.roa_presence_scan_ms_max, + "roa_cache_view_ms_total": metrics.roa_cache_view_ms_total, + "roa_cache_view_ms_max": metrics.roa_cache_view_ms_max, + "direct_finalize_ms_total": metrics.direct_finalize_ms_total, + "direct_finalize_ms_max": metrics.direct_finalize_ms_max, + "fallback_full_run_ms_total": metrics.fallback_full_run_ms_total, + "fallback_full_run_ms_max": metrics.fallback_full_run_ms_max, + "batch_duration_ms": metrics.total_ms, + }), + ); +} + +fn can_start_more(instances_started: usize, config: &TreeRunConfig) -> bool { + config + .max_instances + .map(|max| instances_started < max) + .unwrap_or(true) +} + +fn start_queued_ca_instances( + repo_runtime: &dyn crate::parallel::repo_runtime::RepoSyncRuntime, + ca_queue: &mut VecDeque, + ready_queue: &mut VecDeque, + ca_waiting_repo_by_identity: &mut HashMap>, + finished: &mut Vec, + visited_manifest_uris: &mut HashSet, + instances_started: &mut usize, + config: &TreeRunConfig, +) { + while can_start_more(*instances_started, config) { + let Some(node) = ca_queue.pop_front() else { + break; + }; + if !visited_manifest_uris.insert(node.handle.manifest_rsync_uri.clone()) { + continue; + } + if !ca_depth_is_allowed(config, node.handle.depth) { + continue; + } + *instances_started += 1; + match repo_runtime.request_publication_point_repo(&node.handle, 0) { + Ok(RepoSyncRequestStatus::Ready { mut outcome, .. }) => { + // Ready here means this CA is reusing repo work that has already completed + // (often due to child prefetch). Do not add the transport duration again. + outcome.repo_sync_duration_ms = 0; + ready_queue.push_back(ReadyCaInstance { + node, + repo_outcome: outcome, + ready_enqueued_at: Instant::now(), + }); + } + Ok(RepoSyncRequestStatus::Pending { identity, .. }) => { + ca_waiting_repo_by_identity + .entry(identity) + .or_default() + .push(node); + } + Err(err) => { + finished.push(FinishedPublicationPoint { + node: FinishedPublicationPointNode::from_queued(node), + result: FinishedPublicationPointResult::Err(err), + }); + } + } + } +} + +fn compute_ready_publication_point_stage( + runner: &Rpkiv1PublicationPointRunner<'_>, + ready: ReadyCaInstance, + ready_queue_len_after_pop: usize, +) -> (StageOutcome, ReadyStageMetrics) { + let publication_point_started = Instant::now(); + let ready_queue_wait_ms = publication_point_started + .saturating_duration_since(ready.ready_enqueued_at) + .as_millis() as u64; + let mut metrics = ReadyStageMetrics { + ready_count: 1, + manifest_rsync_uri: Some(ready.node.handle.manifest_rsync_uri.clone()), + publication_point_rsync_uri: Some(ready.node.handle.publication_point_rsync_uri.clone()), + ready_queue_wait_ms, + ready_queue_len_after_pop, + ..ReadyStageMetrics::default() + }; + let mut warnings = ready.repo_outcome.warnings.clone(); + let repo_outcome = ready.repo_outcome.clone(); + if let Some(result) = runner.observe_or_reuse_publication_point_cache( + &ready.node.handle, + repo_outcome.repo_sync_source.as_deref(), + repo_outcome.repo_sync_phase.as_deref(), + repo_outcome.repo_sync_duration_ms, + repo_outcome.repo_sync_err.as_deref(), + &warnings, + ) { + metrics.complete_count = 1; + metrics.discovered_children = result.discovered_children.len(); + return ( + StageOutcome::CacheHit(Box::new(CacheHitOutcome { + ready, + publication_point_started, + result, + })), + metrics, + ); + } + + let stage_fresh_started = Instant::now(); + let stage = runner.stage_fresh_publication_point_after_repo_ready( + &ready.node.handle, + repo_outcome.repo_sync_ok, + repo_outcome.repo_sync_err.as_deref(), + ); + metrics.stage_fresh_ms = elapsed_ms(stage_fresh_started); + + let fresh_stage = match stage { + Ok(stage) => stage, + Err(err) => { + if metrics.stage_fresh_ms >= crate::progress_log::stage_fresh_slow_threshold_ms() { + crate::progress_log::emit( + "phase2_stage_fresh_slow", + serde_json::json!({ + "manifest_rsync_uri": ready.node.handle.manifest_rsync_uri.as_str(), + "publication_point_rsync_uri": ready.node.handle.publication_point_rsync_uri.as_str(), + "status": "error", + "error": err.error.to_string(), + "stage_fresh_ms": metrics.stage_fresh_ms, + "snapshot_prepare_ms": err.snapshot_prepare_ms, + "repo_sync_source": repo_outcome.repo_sync_source.as_deref(), + "repo_sync_phase": repo_outcome.repo_sync_phase.as_deref(), + "repo_sync_duration_ms": repo_outcome.repo_sync_duration_ms, + }), + ); + } + // The blocking `run_publication_point` fallback stays on the control + // thread; it is executed by the apply phase for this outcome. + return ( + StageOutcome::FreshError(Box::new(FreshErrorOutcome { + ready, + publication_point_started, + })), + metrics, + ); + } + }; + metrics.snapshot_prepare_ms = fresh_stage.snapshot_prepare_ms; + metrics.snapshot_current_index_lock_ms = + fresh_stage.snapshot_prepare_timing.current_index_lock_ms; + metrics.snapshot_manifest_load_ms = fresh_stage.snapshot_prepare_timing.manifest_load_ms; + metrics.snapshot_manifest_index_lookup_ms = + fresh_stage.snapshot_prepare_timing.manifest_index_lookup_ms; + metrics.snapshot_manifest_blob_load_ms = + fresh_stage.snapshot_prepare_timing.manifest_blob_load_ms; + metrics.snapshot_manifest_decode_ms = fresh_stage.snapshot_prepare_timing.manifest_decode_ms; + metrics.snapshot_replay_guard_ms = fresh_stage.snapshot_prepare_timing.replay_guard_ms; + metrics.replay_meta_hit_count = fresh_stage.snapshot_prepare_timing.replay_meta_hit as usize; + metrics.replay_meta_miss_count = fresh_stage.snapshot_prepare_timing.replay_meta_miss as usize; + metrics.snapshot_manifest_entries_ms = fresh_stage.snapshot_prepare_timing.manifest_entries_ms; + metrics.snapshot_pack_files_ms = fresh_stage.snapshot_prepare_timing.pack_files_ms; + metrics.snapshot_pack_files_index_lookup_ms = fresh_stage + .snapshot_prepare_timing + .pack_files_index_lookup_ms; + metrics.snapshot_pack_files_blob_load_ms = + fresh_stage.snapshot_prepare_timing.pack_files_blob_load_ms; + metrics.snapshot_ee_path_validate_ms = fresh_stage.snapshot_prepare_timing.ee_path_validate_ms; + metrics.snapshot_manifest_file_count = fresh_stage.snapshot_prepare_timing.manifest_file_count; + metrics.child_discovery_ms = fresh_stage.child_discovery_ms; + if metrics.stage_fresh_ms >= crate::progress_log::stage_fresh_slow_threshold_ms() { + crate::progress_log::emit( + "phase2_stage_fresh_slow", + serde_json::json!({ + "manifest_rsync_uri": ready.node.handle.manifest_rsync_uri.as_str(), + "publication_point_rsync_uri": ready.node.handle.publication_point_rsync_uri.as_str(), + "status": "ok", + "stage_fresh_ms": metrics.stage_fresh_ms, + "snapshot_prepare_ms": fresh_stage.snapshot_prepare_ms, + "snapshot_current_index_lock_ms": fresh_stage.snapshot_prepare_timing.current_index_lock_ms, + "snapshot_manifest_load_ms": fresh_stage.snapshot_prepare_timing.manifest_load_ms, + "snapshot_manifest_index_lookup_ms": fresh_stage.snapshot_prepare_timing.manifest_index_lookup_ms, + "snapshot_manifest_blob_load_ms": fresh_stage.snapshot_prepare_timing.manifest_blob_load_ms, + "snapshot_manifest_decode_ms": fresh_stage.snapshot_prepare_timing.manifest_decode_ms, + "snapshot_replay_guard_ms": fresh_stage.snapshot_prepare_timing.replay_guard_ms, + "replay_meta_hit": fresh_stage.snapshot_prepare_timing.replay_meta_hit, + "replay_meta_miss": fresh_stage.snapshot_prepare_timing.replay_meta_miss, + "snapshot_manifest_entries_ms": fresh_stage.snapshot_prepare_timing.manifest_entries_ms, + "snapshot_pack_files_ms": fresh_stage.snapshot_prepare_timing.pack_files_ms, + "snapshot_pack_files_index_lookup_ms": fresh_stage.snapshot_prepare_timing.pack_files_index_lookup_ms, + "snapshot_pack_files_blob_load_ms": fresh_stage.snapshot_prepare_timing.pack_files_blob_load_ms, + "snapshot_ee_path_validate_ms": fresh_stage.snapshot_prepare_timing.ee_path_validate_ms, + "snapshot_manifest_file_count": fresh_stage.snapshot_prepare_timing.manifest_file_count, + "child_discovery_ms": fresh_stage.child_discovery_ms, + "child_count": fresh_stage.discovered_children.len(), + "repo_sync_source": repo_outcome.repo_sync_source.as_deref(), + "repo_sync_phase": repo_outcome.repo_sync_phase.as_deref(), + "repo_sync_duration_ms": repo_outcome.repo_sync_duration_ms, + }), + ); + } + warnings.extend(fresh_stage.warnings.clone()); + + metrics.discovered_children = fresh_stage.discovered_children.len(); + + let prepare_started = Instant::now(); + let roa_presence_scan_started = Instant::now(); + let has_roa = fresh_stage + .fresh_point + .files() + .iter() + .any(|file| file.rsync_uri.ends_with(".roa")); + metrics.roa_presence_scan_ms = elapsed_ms(roa_presence_scan_started); + if runner.enable_roa_validation_cache { + if let Some(timing) = runner.timing.as_ref() { + if has_roa { + timing.record_count("roa_validation_cache_roa_candidate_publication_points", 1); + } else { + timing.record_count("roa_validation_cache_skipped_no_roa_publication_points", 1); + } + } + } + let roa_cache_view = if has_roa { + let roa_cache_view_started = Instant::now(); + let view = runner + .roa_validation_cache_view_for_fresh_point(&fresh_stage.fresh_point.manifest_rsync_uri); + metrics.roa_cache_view_ms = elapsed_ms(roa_cache_view_started); + view + } else { + None + }; + let roa_cache = if runner.enable_roa_validation_cache && has_roa { + RoaValidationCacheInput::enabled_with_context( + roa_cache_view.as_ref(), + crate::validation::tree_runner::ca_validation_context_digest_for_ca(&ready.node.handle), + crate::validation::tree_runner::publication_point_cache_policy_fingerprint( + runner.policy, + ), + ) + } else { + RoaValidationCacheInput::disabled() + }; + let ta_constraints = runner + .policy + .ta_constraints + .shared_for_tal(&ready.node.handle.tal_id); + if ta_constraints.is_some() { + if let Some(timing) = runner.timing.as_ref() { + timing.record_count("ta_constraints_parallel_publication_points", 1); + } + } + match prepare_publication_point_for_parallel_roa_with_cache_and_ta_constraints( + ready.node.id, + &fresh_stage.fresh_point, + runner.policy, + fresh_stage.issuer_ca_der.as_ref(), + ready.node.handle.ca_certificate_rsync_uri.as_deref(), + ready.node.handle.effective_ip_resources.as_ref(), + ready.node.handle.effective_as_resources.as_ref(), + runner.validation_time, + runner.persist_vcir, + roa_cache, + ta_constraints, + ) { + ParallelObjectsPrepare::Complete(mut objects) => { + metrics.prepare_ms = elapsed_ms(prepare_started); + runner.record_publication_point_step_ms( + &ready.node.handle.manifest_rsync_uri, + "fresh_objects_prepare", + metrics.prepare_ms, + ); + metrics.complete_count = 1; + metrics.roa_tasks = objects.stats.roa_total; + metrics.aspa_objects = objects.stats.aspa_total; + objects + .router_keys + .extend(fresh_stage.discovered_router_keys.clone()); + objects.local_outputs_cache.extend( + crate::validation::tree_runner::build_router_key_local_outputs( + &ready.node.handle, + &objects.router_keys, + ), + ); + ( + StageOutcome::Complete(Box::new(CompleteOutcome { + ready, + publication_point_started, + fresh_stage, + warnings, + objects, + })), + metrics, + ) + } + ParallelObjectsPrepare::Staged(objects_stage) => { + metrics.prepare_ms = elapsed_ms(prepare_started); + runner.record_publication_point_step_ms( + &ready.node.handle.manifest_rsync_uri, + "fresh_objects_prepare", + metrics.prepare_ms, + ); + metrics.staged_count = 1; + metrics.locked_files = objects_stage.locked_file_count(); + metrics.aspa_objects = objects_stage.aspa_task_count(); + let task_count = objects_stage.roa_task_count(); + metrics.roa_tasks = task_count; + let outcome = StagedOutcome { + ready, + publication_point_started, + fresh_stage, + warnings, + objects_stage, + }; + if task_count == 0 { + metrics.zero_task_count = 1; + (StageOutcome::ZeroTask(Box::new(outcome)), metrics) + } else { + (StageOutcome::Fresh(Box::new(outcome)), metrics) + } + } + } +} + +fn apply_ready_publication_point_stage( + runner: &Rpkiv1PublicationPointRunner<'_>, + next_id: &mut u64, + ca_queue: &mut VecDeque, + pending_roa_dispatch: &mut VecDeque, + inflight_publication_points: &mut HashMap, + pending_finalization: &mut VecDeque, + finished: &mut Vec, + outcome: StageOutcome, + mut metrics: ReadyStageMetrics, + config: &TreeRunConfig, + compact_audit: bool, +) -> ReadyStageMetrics { + match outcome { + StageOutcome::CacheHit(outcome) => { + let CacheHitOutcome { + ready, + publication_point_started, + result, + } = *outcome; + let repo_outcome = ready.repo_outcome.clone(); + let child_enqueue_started = Instant::now(); + enqueue_discovered_children( + runner, + next_id, + ca_queue, + &ready.node, + config, + result.discovered_children.clone(), + ); + metrics.child_enqueue_ms = elapsed_ms(child_enqueue_started); + runner.record_publication_point_step_ms( + metrics.manifest_rsync_uri.as_deref().unwrap_or_default(), + "publication_point_cache_child_enqueue", + metrics.child_enqueue_ms, + ); + finished.push(FinishedPublicationPoint { + node: FinishedPublicationPointNode::from_queued(ready.node), + result: compact_phase2_finished_result(result, compact_audit), + }); + metrics.total_ms = elapsed_ms(publication_point_started); + emit_ready_publication_point_control_slow( + metrics.manifest_rsync_uri.as_deref().unwrap_or_default(), + metrics + .publication_point_rsync_uri + .as_deref() + .unwrap_or_default(), + &repo_outcome, + &metrics, + "publication_point_cache", + false, + ); + metrics + } + StageOutcome::FreshError(outcome) => { + let FreshErrorOutcome { + ready, + publication_point_started, + } = *outcome; + let repo_outcome = ready.repo_outcome.clone(); + metrics.fallback_count = 1; + let fallback_started = Instant::now(); + let fallback = runner.run_publication_point(&ready.node.handle); + metrics.fallback_full_run_ms = elapsed_ms(fallback_started); + if let Ok(result) = fallback.as_ref() { + metrics.discovered_children = result.discovered_children.len(); + let child_enqueue_started = Instant::now(); + enqueue_discovered_children( + runner, + next_id, + ca_queue, + &ready.node, + config, + result.discovered_children.clone(), + ); + metrics.child_enqueue_ms = elapsed_ms(child_enqueue_started); + } + finished.push(FinishedPublicationPoint { + node: FinishedPublicationPointNode::from_queued(ready.node), + result: compact_phase2_finished_result_result(fallback, compact_audit), + }); + metrics.total_ms = elapsed_ms(publication_point_started); + emit_ready_publication_point_control_slow( + metrics.manifest_rsync_uri.as_deref().unwrap_or_default(), + metrics + .publication_point_rsync_uri + .as_deref() + .unwrap_or_default(), + &repo_outcome, + &metrics, + "fallback", + true, + ); + metrics + } + StageOutcome::Complete(outcome) => { + let CompleteOutcome { + ready, + publication_point_started, + fresh_stage, + warnings, + objects, + } = *outcome; + let repo_outcome = ready.repo_outcome.clone(); + let child_enqueue_started = Instant::now(); + enqueue_discovered_children( + runner, + next_id, + ca_queue, + &ready.node, + config, + fresh_stage.discovered_children.clone(), + ); + metrics.child_enqueue_ms = elapsed_ms(child_enqueue_started); + // The finalize no longer runs synchronously here: the publication + // point is queued for the shared finalize worker through the same + // pending_finalization path (queue capacity backpressure and + // finalize_inflight accounting included) as zero-task staging. The + // per-publication-point total timing is recorded by the finalize + // worker, exactly like the zero-task and staged paths. + let direct_finalize_started = Instant::now(); + pending_finalization.push_back(FinalizeTask { + state: InflightPublicationPoint { + node: ready.node, + fresh_stage, + objects_prepare: ParallelObjectsPrepare::Complete(objects), + repo_outcome: repo_outcome.clone(), + warnings, + started_at: publication_point_started, + objects_started_at: Instant::now(), + task_count: 0, + tasks_submitted: 0, + first_task_submitted_at: None, + last_task_submitted_at: None, + first_result_at: None, + last_result_at: None, + worker_ms_total: 0, + worker_ms_max: 0, + queue_wait_ms_total: 0, + queue_wait_ms_max: 0, + finalize_enqueued_at: Some(Instant::now()), + results: Vec::new(), + }, + }); + metrics.direct_finalize_ms = elapsed_ms(direct_finalize_started); + runner.record_publication_point_step_ms( + &metrics.manifest_rsync_uri.clone().unwrap_or_default(), + "fresh_direct_finalize", + metrics.direct_finalize_ms, + ); + metrics.total_ms = elapsed_ms(publication_point_started); + emit_ready_publication_point_control_slow( + metrics.manifest_rsync_uri.as_deref().unwrap_or_default(), + metrics + .publication_point_rsync_uri + .as_deref() + .unwrap_or_default(), + &repo_outcome, + &metrics, + "complete", + false, + ); + metrics + } + StageOutcome::ZeroTask(outcome) => { + let StagedOutcome { + ready, + publication_point_started, + fresh_stage, + warnings, + objects_stage, + } = *outcome; + let repo_outcome = ready.repo_outcome.clone(); + let child_enqueue_started = Instant::now(); + enqueue_discovered_children( + runner, + next_id, + ca_queue, + &ready.node, + config, + fresh_stage.discovered_children.clone(), + ); + metrics.child_enqueue_ms = elapsed_ms(child_enqueue_started); + let build_tasks_started = Instant::now(); + objects_stage.append_roa_tasks_to(pending_roa_dispatch); + metrics.build_roa_tasks_ms = elapsed_ms(build_tasks_started); + runner.record_publication_point_step_ms( + &ready.node.handle.manifest_rsync_uri, + "fresh_build_roa_tasks", + metrics.build_roa_tasks_ms, + ); + let task_count = objects_stage.roa_task_count(); + pending_finalization.push_back(FinalizeTask { + state: InflightPublicationPoint { + node: ready.node, + fresh_stage, + objects_prepare: ParallelObjectsPrepare::Staged(objects_stage), + repo_outcome: repo_outcome.clone(), + warnings, + started_at: publication_point_started, + objects_started_at: Instant::now(), + task_count, + tasks_submitted: 0, + first_task_submitted_at: None, + last_task_submitted_at: None, + first_result_at: None, + last_result_at: None, + worker_ms_total: 0, + worker_ms_max: 0, + queue_wait_ms_total: 0, + queue_wait_ms_max: 0, + finalize_enqueued_at: Some(Instant::now()), + results: Vec::new(), + }, + }); + metrics.total_ms = elapsed_ms(publication_point_started); + emit_ready_publication_point_control_slow( + metrics.manifest_rsync_uri.as_deref().unwrap_or_default(), + metrics + .publication_point_rsync_uri + .as_deref() + .unwrap_or_default(), + &repo_outcome, + &metrics, + "zero_task", + false, + ); + metrics + } + StageOutcome::Fresh(outcome) => { + let StagedOutcome { + ready, + publication_point_started, + fresh_stage, + warnings, + objects_stage, + } = *outcome; + let repo_outcome = ready.repo_outcome.clone(); + let child_enqueue_started = Instant::now(); + enqueue_discovered_children( + runner, + next_id, + ca_queue, + &ready.node, + config, + fresh_stage.discovered_children.clone(), + ); + metrics.child_enqueue_ms = elapsed_ms(child_enqueue_started); + let build_tasks_started = Instant::now(); + objects_stage.append_roa_tasks_to(pending_roa_dispatch); + metrics.build_roa_tasks_ms = elapsed_ms(build_tasks_started); + runner.record_publication_point_step_ms( + &ready.node.handle.manifest_rsync_uri, + "fresh_build_roa_tasks", + metrics.build_roa_tasks_ms, + ); + let task_count = objects_stage.roa_task_count(); + inflight_publication_points.insert( + ready.node.id, + InflightPublicationPoint { + node: ready.node, + fresh_stage, + objects_prepare: ParallelObjectsPrepare::Staged(objects_stage), + repo_outcome: repo_outcome.clone(), + warnings, + started_at: publication_point_started, + objects_started_at: Instant::now(), + task_count, + tasks_submitted: 0, + first_task_submitted_at: None, + last_task_submitted_at: None, + first_result_at: None, + last_result_at: None, + worker_ms_total: 0, + worker_ms_max: 0, + queue_wait_ms_total: 0, + queue_wait_ms_max: 0, + finalize_enqueued_at: None, + results: Vec::with_capacity(task_count), + }, + ); + metrics.total_ms = elapsed_ms(publication_point_started); + emit_ready_publication_point_control_slow( + metrics.manifest_rsync_uri.as_deref().unwrap_or_default(), + metrics + .publication_point_rsync_uri + .as_deref() + .unwrap_or_default(), + &repo_outcome, + &metrics, + "staged", + false, + ); + metrics + } + } +} + +fn emit_ready_publication_point_control_slow( + manifest_rsync_uri: &str, + publication_point_rsync_uri: &str, + repo_outcome: &RepoSyncRuntimeOutcome, + metrics: &ReadyStageMetrics, + status: &str, + force_error_path: bool, +) { + let threshold_ms = crate::progress_log::pp_control_slow_threshold_ms(); + if !force_error_path && metrics.total_ms < threshold_ms { + return; + } + crate::progress_log::emit( + "phase2_ready_publication_point_control_slow", + serde_json::json!({ + "manifest_rsync_uri": manifest_rsync_uri, + "publication_point_rsync_uri": publication_point_rsync_uri, + "status": status, + "repo_sync_source": repo_outcome.repo_sync_source.as_deref(), + "repo_sync_phase": repo_outcome.repo_sync_phase.as_deref(), + "repo_sync_duration_ms": repo_outcome.repo_sync_duration_ms, + "repo_sync_ok": repo_outcome.repo_sync_ok, + "repo_sync_err": repo_outcome.repo_sync_err.as_deref(), + "ready_queue_wait_ms": metrics.ready_queue_wait_ms, + "ready_queue_len_after_pop": metrics.ready_queue_len_after_pop, + "stage_fresh_ms": metrics.stage_fresh_ms, + "child_discovery_ms": metrics.child_discovery_ms, + "child_enqueue_ms": metrics.child_enqueue_ms, + "discovered_children": metrics.discovered_children, + "roa_presence_scan_ms": metrics.roa_presence_scan_ms, + "roa_cache_view_ms": metrics.roa_cache_view_ms, + "prepare_ms": metrics.prepare_ms, + "build_roa_tasks_ms": metrics.build_roa_tasks_ms, + "direct_finalize_ms": metrics.direct_finalize_ms, + "fallback_full_run_ms": metrics.fallback_full_run_ms, + "locked_files": metrics.locked_files, + "roa_tasks": metrics.roa_tasks, + "aspa_objects": metrics.aspa_objects, + "complete_count": metrics.complete_count, + "staged_count": metrics.staged_count, + "zero_task_count": metrics.zero_task_count, + "fallback_count": metrics.fallback_count, + "total_ms": metrics.total_ms, + "slow_threshold_ms": threshold_ms, + }), + ); + crate::progress_log::emit( + "phase2_ready_publication_point_control_snapshot_breakdown", + serde_json::json!({ + "manifest_rsync_uri": manifest_rsync_uri, + "publication_point_rsync_uri": publication_point_rsync_uri, + "status": status, + "snapshot_prepare_ms": metrics.snapshot_prepare_ms, + "snapshot_current_index_lock_ms": metrics.snapshot_current_index_lock_ms, + "snapshot_manifest_load_ms": metrics.snapshot_manifest_load_ms, + "snapshot_manifest_index_lookup_ms": metrics.snapshot_manifest_index_lookup_ms, + "snapshot_manifest_blob_load_ms": metrics.snapshot_manifest_blob_load_ms, + "snapshot_manifest_decode_ms": metrics.snapshot_manifest_decode_ms, + "snapshot_replay_guard_ms": metrics.snapshot_replay_guard_ms, + "replay_meta_hit_count": metrics.replay_meta_hit_count, + "replay_meta_miss_count": metrics.replay_meta_miss_count, + "snapshot_manifest_entries_ms": metrics.snapshot_manifest_entries_ms, + "snapshot_pack_files_ms": metrics.snapshot_pack_files_ms, + "snapshot_pack_files_index_lookup_ms": metrics.snapshot_pack_files_index_lookup_ms, + "snapshot_pack_files_blob_load_ms": metrics.snapshot_pack_files_blob_load_ms, + "snapshot_ee_path_validate_ms": metrics.snapshot_ee_path_validate_ms, + "snapshot_manifest_file_count": metrics.snapshot_manifest_file_count, + "total_ms": metrics.total_ms, + "slow_threshold_ms": threshold_ms, + }), + ); +} + +fn enqueue_discovered_children( + runner: &Rpkiv1PublicationPointRunner<'_>, + next_id: &mut u64, + ca_queue: &mut VecDeque, + parent: &QueuedCaInstance, + config: &TreeRunConfig, + mut children: Vec, +) { + let Some(child_depth) = next_allowed_ca_depth(config, parent.handle.depth) else { + return; + }; + + children.sort_by(|a, b| { + a.handle + .manifest_rsync_uri + .cmp(&b.handle.manifest_rsync_uri) + .then_with(|| { + a.discovered_from + .child_ca_certificate_rsync_uri + .cmp(&b.discovered_from.child_ca_certificate_rsync_uri) + }) + }); + if let Some(runtime) = runner.repo_sync_runtime.as_ref() { + let _ = runtime.prefetch_discovered_children(&children); + } + for child in children { + let mut handle = child.handle.with_depth(child_depth); + handle.parent_manifest_rsync_uri = Some(parent.handle.manifest_rsync_uri.clone()); + ca_queue.push_back(QueuedCaInstance { + id: *next_id, + handle, + parent_id: Some(parent.id), + discovered_from: Some(child.discovered_from), + }); + *next_id += 1; + } +} + +fn finalize_metrics_from_output( + output: &FreshPublicationPointFinalizeOutput, + reduce_ms: u64, + finalize_ms: u64, + finalize_queue_wait_ms: Option, + finalize_worker_ms: u64, + locked_files: usize, +) -> FinalizePublicationPointMetrics { + FinalizePublicationPointMetrics { + reduce_ms, + finalize_ms, + finalize_queue_wait_ms, + finalize_worker_ms, + snapshot_pack_ms: output.snapshot_pack_ms, + persist_vcir_ms: output.persist_vcir_ms, + persist_build_vcir_ms: output.persist_vcir_timing.build_vcir_ms, + persist_replace_vcir_ms: output.persist_vcir_timing.replace_vcir_ms, + persist_replace_breakdown: output.persist_vcir_timing.replace_vcir.clone(), + ccr_projection_build_ms: output.ccr_projection_build_ms, + ccr_append_ms: output.ccr_append_ms, + audit_build_ms: output.audit_build_ms, + locked_files, + child_count: output.result.discovered_children.len(), + warning_count: output.result.warnings.len(), + vrp_count: output.result.objects.vrps.len(), + vap_count: output.result.objects.aspas.len(), + router_key_count: output.result.objects.router_keys.len(), + audit_object_count: output.result.audit.objects.len(), + } +} + +fn emit_finalize_breakdown( + event_name: &str, + manifest_rsync_uri: &str, + publication_point_rsync_uri: &str, + metrics: &FinalizePublicationPointMetrics, +) { + crate::progress_log::emit( + event_name, + serde_json::json!({ + "manifest_rsync_uri": manifest_rsync_uri, + "publication_point_rsync_uri": publication_point_rsync_uri, + "reduce_ms": metrics.reduce_ms, + "finalize_ms": metrics.finalize_ms, + "finalize_queue_wait_ms": metrics.finalize_queue_wait_ms, + "finalize_worker_ms": metrics.finalize_worker_ms, + "snapshot_pack_ms": metrics.snapshot_pack_ms, + "persist_vcir_ms": metrics.persist_vcir_ms, + "persist_build_vcir_ms": metrics.persist_build_vcir_ms, + "persist_replace_vcir_ms": metrics.persist_replace_vcir_ms, + "persist_replace_breakdown": &metrics.persist_replace_breakdown, + "ccr_projection_build_ms": metrics.ccr_projection_build_ms, + "ccr_append_ms": metrics.ccr_append_ms, + "audit_build_ms": metrics.audit_build_ms, + "locked_files": metrics.locked_files, + "child_count": metrics.child_count, + "warning_count": metrics.warning_count, + "vrp_count": metrics.vrp_count, + "vap_count": metrics.vap_count, + "router_key_count": metrics.router_key_count, + "audit_object_count": metrics.audit_object_count, + }), + ); +} + +fn flush_pending_roa_dispatch( + runner: &Rpkiv1PublicationPointRunner<'_>, + pending_roa_dispatch: &mut VecDeque, + inflight_publication_points: &mut HashMap, +) -> Result { + let started = Instant::now(); + let mut metrics = RoaDispatchMetrics::default(); + let Some(pool) = runner.parallel_roa_worker_pool.as_ref() else { + return Ok(metrics); + }; + while let Some(mut task) = pending_roa_dispatch.pop_front() { + metrics.attempted += 1; + let pp_id = task.publication_point_id; + task.submitted_at = Some(Instant::now()); + match pool.try_submit_round_robin(task) { + Ok(_) => { + metrics.submitted += 1; + if let Some(state) = inflight_publication_points.get_mut(&pp_id) { + let now = Instant::now(); + state.tasks_submitted += 1; + if state.first_task_submitted_at.is_none() { + state.first_task_submitted_at = Some(now); + } + state.last_task_submitted_at = Some(now); + } + } + Err(ObjectWorkerSubmitError::QueueFull { task, .. }) => { + pending_roa_dispatch.push_front(task); + metrics.queue_full = true; + break; + } + Err(ObjectWorkerSubmitError::Disconnected { .. }) => { + return Err(TreeRunError::Runner( + "parallel ROA worker queue disconnected".to_string(), + )); + } + } + } + metrics.pending_remaining = pending_roa_dispatch.len(); + metrics.duration_ms = elapsed_ms(started); + Ok(metrics) +} + +fn flush_pending_roa_dispatch_with_progress( + runner: &Rpkiv1PublicationPointRunner<'_>, + pending_roa_dispatch: &mut VecDeque, + inflight_publication_points: &mut HashMap, + pending_finalization: &VecDeque, +) -> Result<(), TreeRunError> { + let dispatch_metrics = + flush_pending_roa_dispatch(runner, pending_roa_dispatch, inflight_publication_points)?; + if dispatch_metrics.attempted > 0 || dispatch_metrics.queue_full { + crate::progress_log::emit( + "phase2_roa_dispatch_batch", + serde_json::json!({ + "attempted": dispatch_metrics.attempted, + "submitted": dispatch_metrics.submitted, + "queue_full": dispatch_metrics.queue_full, + "pending_remaining": dispatch_metrics.pending_remaining, + "duration_ms": dispatch_metrics.duration_ms, + "inflight_publication_points": inflight_publication_points.len(), + "pending_finalization_len": pending_finalization.len(), + }), + ); + } + Ok(()) +} + +fn drain_object_results( + runner: &Rpkiv1PublicationPointRunner<'_>, + inflight_publication_points: &mut HashMap, + pending_finalization: &mut VecDeque, + result_budget: usize, +) -> Result { + let started = Instant::now(); + let mut metrics = ObjectDrainMetrics::default(); + let Some(pool) = runner.parallel_roa_worker_pool.as_ref() else { + return Ok(metrics); + }; + let result_budget = result_budget.max(1); + while metrics.results_drained < result_budget { + let Some(result) = pool + .recv_result_timeout(Duration::from_millis(0)) + .map_err(TreeRunError::Runner)? + else { + break; + }; + metrics.results_drained += 1; + let pp_id = result.publication_point_id; + let _worker_index = result.worker_index; + metrics.worker_ms_total += result.worker_ms; + metrics.worker_ms_max = metrics.worker_ms_max.max(result.worker_ms); + metrics.queue_wait_ms_total += result.queue_wait_ms; + metrics.queue_wait_ms_max = metrics.queue_wait_ms_max.max(result.queue_wait_ms); + let should_finalize = if let Some(state) = inflight_publication_points.get_mut(&pp_id) { + let now = Instant::now(); + if state.first_result_at.is_none() { + state.first_result_at = Some(now); + } + state.last_result_at = Some(now); + state.worker_ms_total += result.worker_ms; + state.worker_ms_max = state.worker_ms_max.max(result.worker_ms); + state.queue_wait_ms_total += result.queue_wait_ms; + state.queue_wait_ms_max = state.queue_wait_ms_max.max(result.queue_wait_ms); + state.results.push(result); + state.results.len() == state.task_count + } else { + false + }; + if should_finalize { + let mut state = inflight_publication_points + .remove(&pp_id) + .expect("inflight publication point must exist"); + state.finalize_enqueued_at = Some(Instant::now()); + metrics.publication_points_completed += 1; + pending_finalization.push_back(FinalizeTask { state }); + } + } + metrics.result_budget_exhausted = metrics.results_drained == result_budget; + metrics.duration_ms = elapsed_ms(started); + Ok(metrics) +} + +fn drain_object_results_with_progress( + runner: &Rpkiv1PublicationPointRunner<'_>, + inflight_publication_points: &mut HashMap, + pending_finalization: &mut VecDeque, + pending_roa_dispatch_len: usize, + result_budget: usize, +) -> Result<(), TreeRunError> { + let drain_metrics = drain_object_results( + runner, + inflight_publication_points, + pending_finalization, + result_budget, + )?; + if drain_metrics.results_drained > 0 || drain_metrics.result_budget_exhausted { + crate::progress_log::emit( + "phase2_object_results_drain", + serde_json::json!({ + "results_drained": drain_metrics.results_drained, + "publication_points_completed": drain_metrics.publication_points_completed, + "result_budget_exhausted": drain_metrics.result_budget_exhausted, + "result_drain_batch_size": result_budget, + "worker_ms_total": drain_metrics.worker_ms_total, + "worker_ms_max": drain_metrics.worker_ms_max, + "queue_wait_ms_total": drain_metrics.queue_wait_ms_total, + "queue_wait_ms_max": drain_metrics.queue_wait_ms_max, + "duration_ms": drain_metrics.duration_ms, + "pending_roa_dispatch_len": pending_roa_dispatch_len, + "inflight_publication_points": inflight_publication_points.len(), + "pending_finalization_len": pending_finalization.len(), + }), + ); + } + Ok(()) +} + +/// Submission endpoint of the stage pool, abstracted so the dispatch loop can +/// be tested with a deterministic backpressure source. +trait ReadyStageSubmitter { + // The error must hand the task back so the dispatch loop can requeue it; + // boxing it away would only shuffle the same bytes around. + #[allow(clippy::result_large_err)] + fn try_submit_ready_stage( + &mut self, + task: ReadyStageTask, + ) -> Result<(), ObjectWorkerSubmitError>; +} + +impl ReadyStageSubmitter for ReadyStagePool<'_, '_> { + fn try_submit_ready_stage( + &mut self, + task: ReadyStageTask, + ) -> Result<(), ObjectWorkerSubmitError> { + self.try_submit_round_robin(task).map(|_| ()) + } +} + +/// Dispatch loop of the pool-enabled ready batch: pop ready publication +/// points and submit their compute tasks to the stage pool, following the +/// `flush_pending_roa_dispatch` backpressure pattern — on a full worker queue +/// the publication point goes back to the head of the ready queue so the next +/// turn retries it first (nothing lost, nothing duplicated). +fn submit_ready_batch_to_stage_pool( + stage_pool: &mut impl ReadyStageSubmitter, + ready_queue: &mut VecDeque, + staging_inflight: &mut usize, + ready_batch_size: usize, + ready_batch_wall_time_budget: Duration, +) -> Result { + let started = Instant::now(); + let mut metrics = StageDispatchMetrics::default(); + while metrics.submitted < ready_batch_size { + let Some(ready) = ready_queue.pop_front() else { + break; + }; + let task = ReadyStageTask { + ready_queue_len_after_pop: ready_queue.len(), + ready, + submitted_at: Instant::now(), + }; + match stage_pool.try_submit_ready_stage(task) { + Ok(_) => { + *staging_inflight += 1; + metrics.submitted += 1; + } + Err(ObjectWorkerSubmitError::QueueFull { task, .. }) => { + ready_queue.push_front(task.ready); + metrics.queue_full = true; + break; + } + Err(ObjectWorkerSubmitError::Disconnected { .. }) => { + return Err(TreeRunError::Runner( + "ready stage worker queue disconnected".to_string(), + )); + } + } + if metrics.submitted > 0 && started.elapsed() >= ready_batch_wall_time_budget { + break; + } + } + metrics.duration_ms = elapsed_ms(started); + Ok(metrics) +} + +/// Collect every available stage result without blocking and run the apply +/// phase for each on the control thread, exactly like the inline path would +/// have done right after compute. Drained metrics aggregate into the same +/// `ReadyStageBatchMetrics`, keeping the `phase2_ready_queue_*` events +/// unchanged. +#[allow(clippy::too_many_arguments)] +fn drain_stage_results( + stage_pool: &ReadyStagePool<'_, '_>, + runner: &Rpkiv1PublicationPointRunner<'_>, + next_id: &mut u64, + ca_queue: &mut VecDeque, + pending_roa_dispatch: &mut VecDeque, + inflight_publication_points: &mut HashMap, + pending_finalization: &mut VecDeque, + finished: &mut Vec, + staging_inflight: &mut usize, + batch_metrics: &mut ReadyStageBatchMetrics, + pool_metrics: &mut StageDrainMetrics, + config: &TreeRunConfig, +) -> Result<(), TreeRunError> { + let started = Instant::now(); + loop { + let Some(result) = stage_pool + .recv_result_timeout(Duration::from_millis(0)) + .map_err(TreeRunError::Runner)? + else { + break; + }; + pool_metrics.results_drained += 1; + pool_metrics.queue_wait_ms_total += result.queue_wait_ms; + pool_metrics.queue_wait_ms_max = pool_metrics.queue_wait_ms_max.max(result.queue_wait_ms); + pool_metrics.worker_ms_total += result.worker_ms; + pool_metrics.worker_ms_max = pool_metrics.worker_ms_max.max(result.worker_ms); + *staging_inflight = staging_inflight.saturating_sub(1); + let metrics = apply_ready_publication_point_stage( + runner, + next_id, + ca_queue, + pending_roa_dispatch, + inflight_publication_points, + pending_finalization, + finished, + result.outcome, + result.metrics, + config, + config.compact_audit, + ); + batch_metrics.record(metrics); + } + pool_metrics.duration_ms += elapsed_ms(started); + Ok(()) +} + +fn submit_pending_finalization( + finalize_task_tx: &SyncSender, + pending_finalization: &mut VecDeque, + finalize_inflight: &mut usize, +) -> Result { + let started = Instant::now(); + let mut metrics = FinalizeSubmitMetrics::default(); + while let Some(task) = pending_finalization.pop_front() { + match finalize_task_tx.try_send(task) { + Ok(()) => { + metrics.submitted += 1; + *finalize_inflight += 1; + } + Err(TrySendError::Full(task)) => { + pending_finalization.push_front(task); + metrics.queue_full = true; + break; + } + Err(TrySendError::Disconnected(_task)) => { + return Err(TreeRunError::Runner( + "phase2 finalize worker queue disconnected".to_string(), + )); + } + } + } + metrics.duration_ms = elapsed_ms(started); + Ok(metrics) +} + +fn submit_pending_finalization_with_progress( + finalize_task_tx: &SyncSender, + pending_finalization: &mut VecDeque, + finalize_inflight: &mut usize, + finalize_queue_capacity: usize, + pending_roa_dispatch_len: usize, + inflight_publication_points_len: usize, +) -> Result<(), TreeRunError> { + let submit_metrics = + submit_pending_finalization(finalize_task_tx, pending_finalization, finalize_inflight)?; + if submit_metrics.submitted > 0 || submit_metrics.queue_full { + crate::progress_log::emit( + "phase2_finalize_task_submit", + serde_json::json!({ + "submitted": submit_metrics.submitted, + "queue_full": submit_metrics.queue_full, + "duration_ms": submit_metrics.duration_ms, + "finalize_queue_capacity": finalize_queue_capacity, + "pending_finalization_len": pending_finalization.len(), + "finalize_inflight": *finalize_inflight, + "pending_roa_dispatch_len": pending_roa_dispatch_len, + "inflight_publication_points": inflight_publication_points_len, + }), + ); + } + Ok(()) +} + +fn drain_finalize_results( + finalize_result_rx: &Receiver, + finished: &mut Vec, + finalize_inflight: &mut usize, +) -> Result { + let started = Instant::now(); + let mut metrics = FinalizeResultsDrainMetrics::default(); + loop { + match finalize_result_rx.try_recv() { + Ok(result) => { + metrics.results_drained += 1; + metrics.reduce_ms_total += result.metrics.reduce_ms; + metrics.reduce_ms_max = metrics.reduce_ms_max.max(result.metrics.reduce_ms); + metrics.finalize_ms_total += result.metrics.finalize_ms; + metrics.finalize_ms_max = metrics.finalize_ms_max.max(result.metrics.finalize_ms); + metrics.finalize_queue_wait_ms_max = metrics + .finalize_queue_wait_ms_max + .max(result.metrics.finalize_queue_wait_ms.unwrap_or(0)); + metrics.finalize_worker_ms_total += result.metrics.finalize_worker_ms; + metrics.finalize_worker_ms_max = metrics + .finalize_worker_ms_max + .max(result.metrics.finalize_worker_ms); + metrics.snapshot_pack_ms_total += result.metrics.snapshot_pack_ms; + metrics.snapshot_pack_ms_max = metrics + .snapshot_pack_ms_max + .max(result.metrics.snapshot_pack_ms); + metrics.persist_vcir_ms_total += result.metrics.persist_vcir_ms; + metrics.persist_vcir_ms_max = metrics + .persist_vcir_ms_max + .max(result.metrics.persist_vcir_ms); + metrics.persist_build_vcir_ms_total += result.metrics.persist_build_vcir_ms; + metrics.persist_build_vcir_ms_max = metrics + .persist_build_vcir_ms_max + .max(result.metrics.persist_build_vcir_ms); + metrics.persist_replace_vcir_ms_total += result.metrics.persist_replace_vcir_ms; + metrics.persist_replace_vcir_ms_max = metrics + .persist_replace_vcir_ms_max + .max(result.metrics.persist_replace_vcir_ms); + metrics.ccr_projection_build_ms_total += result.metrics.ccr_projection_build_ms; + metrics.ccr_projection_build_ms_max = metrics + .ccr_projection_build_ms_max + .max(result.metrics.ccr_projection_build_ms); + metrics.ccr_append_ms_total += result.metrics.ccr_append_ms; + metrics.ccr_append_ms_max = + metrics.ccr_append_ms_max.max(result.metrics.ccr_append_ms); + metrics.audit_build_ms_total += result.metrics.audit_build_ms; + metrics.audit_build_ms_max = metrics + .audit_build_ms_max + .max(result.metrics.audit_build_ms); + *finalize_inflight = finalize_inflight.saturating_sub(1); + finished.push(result.finished); + } + Err(TryRecvError::Empty) => break, + Err(TryRecvError::Disconnected) => { + if *finalize_inflight == 0 { + break; + } + return Err(TreeRunError::Runner( + "phase2 finalize result channel disconnected".to_string(), + )); + } + } + } + metrics.duration_ms = elapsed_ms(started); + Ok(metrics) +} + +fn drain_finalize_results_with_progress( + finalize_result_rx: &Receiver, + finished: &mut Vec, + finalize_inflight: &mut usize, + pending_finalization_len: usize, + pending_roa_dispatch_len: usize, + inflight_publication_points_len: usize, +) -> Result<(), TreeRunError> { + let drain_metrics = drain_finalize_results(finalize_result_rx, finished, finalize_inflight)?; + if drain_metrics.results_drained >= 64 + || (drain_metrics.results_drained > 0 + && (*finalize_inflight == 0 || pending_finalization_len > 0)) + { + crate::progress_log::emit( + "phase2_finalize_results_drain", + serde_json::json!({ + "results_drained": drain_metrics.results_drained, + "reduce_ms_total": drain_metrics.reduce_ms_total, + "reduce_ms_max": drain_metrics.reduce_ms_max, + "finalize_ms_total": drain_metrics.finalize_ms_total, + "finalize_ms_max": drain_metrics.finalize_ms_max, + "finalize_queue_wait_ms_max": drain_metrics.finalize_queue_wait_ms_max, + "finalize_worker_ms_total": drain_metrics.finalize_worker_ms_total, + "finalize_worker_ms_max": drain_metrics.finalize_worker_ms_max, + "snapshot_pack_ms_total": drain_metrics.snapshot_pack_ms_total, + "snapshot_pack_ms_max": drain_metrics.snapshot_pack_ms_max, + "persist_vcir_ms_total": drain_metrics.persist_vcir_ms_total, + "persist_vcir_ms_max": drain_metrics.persist_vcir_ms_max, + "persist_build_vcir_ms_total": drain_metrics.persist_build_vcir_ms_total, + "persist_build_vcir_ms_max": drain_metrics.persist_build_vcir_ms_max, + "persist_replace_vcir_ms_total": drain_metrics.persist_replace_vcir_ms_total, + "persist_replace_vcir_ms_max": drain_metrics.persist_replace_vcir_ms_max, + "ccr_projection_build_ms_total": drain_metrics.ccr_projection_build_ms_total, + "ccr_projection_build_ms_max": drain_metrics.ccr_projection_build_ms_max, + "ccr_append_ms_total": drain_metrics.ccr_append_ms_total, + "ccr_append_ms_max": drain_metrics.ccr_append_ms_max, + "audit_build_ms_total": drain_metrics.audit_build_ms_total, + "audit_build_ms_max": drain_metrics.audit_build_ms_max, + "duration_ms": drain_metrics.duration_ms, + "pending_finalization_len": pending_finalization_len, + "finalize_inflight": *finalize_inflight, + "pending_roa_dispatch_len": pending_roa_dispatch_len, + "inflight_publication_points": inflight_publication_points_len, + }), + ); + } + Ok(()) +} + +fn run_finalize_worker( + runner: &Rpkiv1PublicationPointRunner<'_>, + finalize_task_rx: Receiver, + finalize_result_tx: mpsc::Sender, + compact_audit: bool, +) -> Result<(), TreeRunError> { + while let Ok(task) = finalize_task_rx.recv() { + let result = finalize_publication_point_state(runner, task.state, compact_audit); + if finalize_result_tx.send(result).is_err() { + return Err(TreeRunError::Runner( + "phase2 finalize result receiver disconnected".to_string(), + )); + } + } + Ok(()) +} + +fn finalize_publication_point_state( + runner: &Rpkiv1PublicationPointRunner<'_>, + state: InflightPublicationPoint, + compact_audit: bool, +) -> FinalizeWorkerResult { + let finalize_worker_started = Instant::now(); + let InflightPublicationPoint { + node, + fresh_stage, + objects_prepare, + repo_outcome, + warnings, + started_at, + objects_started_at, + task_count, + tasks_submitted, + first_task_submitted_at, + last_task_submitted_at, + first_result_at, + last_result_at, + worker_ms_total, + worker_ms_max, + queue_wait_ms_total, + queue_wait_ms_max, + finalize_enqueued_at, + results, + } = state; + let finalize_queue_wait_ms = finalize_enqueued_at.or(last_result_at).map(|ready_at| { + Instant::now() + .saturating_duration_since(ready_at) + .as_millis() as u64 + }); + let objects_processing_ms = objects_started_at.elapsed().as_millis() as u64; + runner.record_publication_point_step_ms( + &node.handle.manifest_rsync_uri, + "fresh_objects_processing_lifetime", + objects_processing_ms, + ); + + let (result, mut metrics, reduce_ms) = match objects_prepare { + ParallelObjectsPrepare::Staged(objects_stage) => { + let reduce_started = Instant::now(); + let locked_files = objects_stage.locked_file_count(); + let reduce_result = + reduce_parallel_roa_stage(objects_stage, results, runner.timing.as_ref()); + let reduce_ms = elapsed_ms(reduce_started); + runner.record_publication_point_step_ms( + &node.handle.manifest_rsync_uri, + "fresh_roa_reduce", + reduce_ms, + ); + + let (result, metrics) = match reduce_result { + Ok(mut objects) => { + let finalize_started = Instant::now(); + objects + .router_keys + .extend(fresh_stage.discovered_router_keys.clone()); + objects.local_outputs_cache.extend( + crate::validation::tree_runner::build_router_key_local_outputs( + &node.handle, + &objects.router_keys, + ), + ); + let finalized = runner.finalize_fresh_publication_point_from_reducer( + &node.handle, + &fresh_stage.fresh_point, + warnings, + objects, + fresh_stage.child_audits, + fresh_stage.discovered_children, + repo_outcome.repo_sync_source.as_deref(), + repo_outcome.repo_sync_phase.as_deref(), + repo_outcome.repo_sync_duration_ms, + repo_outcome.repo_sync_err.as_deref(), + ); + let finalize_ms = elapsed_ms(finalize_started); + match finalized { + Ok(output) => { + let metrics = finalize_metrics_from_output( + &output, + reduce_ms, + finalize_ms, + finalize_queue_wait_ms, + 0, + locked_files, + ); + ( + compact_phase2_finished_result(output.result, compact_audit), + metrics, + ) + } + Err(err) => ( + FinishedPublicationPointResult::Err(err), + FinalizePublicationPointMetrics { + reduce_ms, + finalize_ms, + finalize_queue_wait_ms, + locked_files, + ..FinalizePublicationPointMetrics::default() + }, + ), + } + } + Err(err) => ( + FinishedPublicationPointResult::Err(err), + FinalizePublicationPointMetrics { + reduce_ms, + finalize_queue_wait_ms, + locked_files, + ..FinalizePublicationPointMetrics::default() + }, + ), + }; + (result, metrics, reduce_ms) + } + ParallelObjectsPrepare::Complete(objects) => { + // ROA prepare already produced complete objects for this publication + // point, so there is nothing to reduce; finalize directly. This is + // the former control-thread "direct finalize", now running on the + // finalize worker through the regular task queue. + let locked_files = fresh_stage.fresh_point.files().len(); + let finalize_started = Instant::now(); + let finalized = runner.finalize_fresh_publication_point_from_reducer( + &node.handle, + &fresh_stage.fresh_point, + warnings, + objects, + fresh_stage.child_audits, + fresh_stage.discovered_children, + repo_outcome.repo_sync_source.as_deref(), + repo_outcome.repo_sync_phase.as_deref(), + repo_outcome.repo_sync_duration_ms, + repo_outcome.repo_sync_err.as_deref(), + ); + let finalize_ms = elapsed_ms(finalize_started); + let (result, metrics) = match finalized { + Ok(output) => { + let metrics = finalize_metrics_from_output( + &output, + 0, + finalize_ms, + finalize_queue_wait_ms, + 0, + locked_files, + ); + ( + compact_phase2_finished_result(output.result, compact_audit), + metrics, + ) + } + Err(err) => ( + FinishedPublicationPointResult::Err(err), + FinalizePublicationPointMetrics { + finalize_ms, + finalize_queue_wait_ms, + locked_files, + ..FinalizePublicationPointMetrics::default() + }, + ), + }; + (result, metrics, 0) + } + }; + let finalize_worker_ms = elapsed_ms(finalize_worker_started); + metrics.finalize_worker_ms = finalize_worker_ms; + runner.record_publication_point_step_ms( + &node.handle.manifest_rsync_uri, + "fresh_finalize_worker", + finalize_worker_ms, + ); + runner.record_publication_point_step_ms( + &node.handle.manifest_rsync_uri, + "fresh_finalize_queue_wait", + finalize_queue_wait_ms.unwrap_or(0), + ); + runner.record_publication_point_step_ms( + &node.handle.manifest_rsync_uri, + "fresh_finalize", + metrics.finalize_ms, + ); + runner.record_publication_point_total_ms( + &node.handle.manifest_rsync_uri, + started_at.elapsed().as_millis() as u64, + ); + emit_finalize_breakdown( + "phase2_finalize_worker_breakdown", + node.handle.manifest_rsync_uri.as_str(), + node.handle.publication_point_rsync_uri.as_str(), + &metrics, + ); + crate::progress_log::emit( + "phase2_publication_point_reduced", + serde_json::json!({ + "manifest_rsync_uri": node.handle.manifest_rsync_uri.as_str(), + "publication_point_rsync_uri": node.handle.publication_point_rsync_uri.as_str(), + "objects_processing_ms": objects_processing_ms, + "task_count": task_count, + "tasks_submitted": tasks_submitted, + "first_task_submitted_ms": first_task_submitted_at.map(|t| t.saturating_duration_since(objects_started_at).as_millis() as u64), + "last_task_submitted_ms": last_task_submitted_at.map(|t| t.saturating_duration_since(objects_started_at).as_millis() as u64), + "task_submit_span_ms": match (first_task_submitted_at, last_task_submitted_at) { + (Some(first), Some(last)) => Some(last.saturating_duration_since(first).as_millis() as u64), + _ => None, + }, + "first_result_ms": first_result_at.map(|t| t.saturating_duration_since(objects_started_at).as_millis() as u64), + "last_result_ms": last_result_at.map(|t| t.saturating_duration_since(objects_started_at).as_millis() as u64), + "all_results_ready_ms": last_result_at.map(|t| t.saturating_duration_since(objects_started_at).as_millis() as u64), + "finalize_queue_wait_ms": finalize_queue_wait_ms, + "result_span_ms": match (first_result_at, last_result_at) { + (Some(first), Some(last)) => Some(last.saturating_duration_since(first).as_millis() as u64), + _ => None, + }, + "worker_ms_total": worker_ms_total, + "worker_ms_max": worker_ms_max, + "worker_ms_avg": if task_count > 0 { worker_ms_total / task_count as u64 } else { 0 }, + "queue_wait_ms_total": queue_wait_ms_total, + "queue_wait_ms_max": queue_wait_ms_max, + "queue_wait_ms_avg": if task_count > 0 { queue_wait_ms_total / task_count as u64 } else { 0 }, + "reduce_ms": reduce_ms, + "finalize_ms": metrics.finalize_ms, + "finalize_worker_ms": finalize_worker_ms, + "snapshot_pack_ms": metrics.snapshot_pack_ms, + "persist_vcir_ms": metrics.persist_vcir_ms, + "persist_build_vcir_ms": metrics.persist_build_vcir_ms, + "persist_replace_vcir_ms": metrics.persist_replace_vcir_ms, + "ccr_projection_build_ms": metrics.ccr_projection_build_ms, + "ccr_append_ms": metrics.ccr_append_ms, + "audit_build_ms": metrics.audit_build_ms, + "locked_files": metrics.locked_files, + "child_count": metrics.child_count, + "warning_count": metrics.warning_count, + "vrp_count": metrics.vrp_count, + "vap_count": metrics.vap_count, + "router_key_count": metrics.router_key_count, + "audit_object_count": metrics.audit_object_count, + "total_duration_ms": started_at.elapsed().as_millis() as u64, + }), + ); + FinalizeWorkerResult { + finished: FinishedPublicationPoint { + node: FinishedPublicationPointNode::from_queued(node), + result, + }, + metrics, + } +} + +fn drain_repo_events( + repo_runtime: &dyn crate::parallel::repo_runtime::RepoSyncRuntime, + ca_waiting_repo_by_identity: &mut HashMap>, + ready_queue: &mut VecDeque, + timeout: Duration, +) -> Result { + let started = Instant::now(); + let mut metrics = RepoDrainMetrics::default(); + let events = repo_runtime + .drain_repo_results_timeout(timeout, REPO_RESULT_DRAIN_MAX_EVENTS) + .map_err(TreeRunError::Runner)?; + for event in events { + metrics.event_count += 1; + metrics.completions += event.completions.len(); + for completion in event.completions { + let mut outcome = completion.outcome; + if completion.identity != event.transport_identity { + // Shared RRDP/rsync transports release many publication points, but the transport + // wall time should only be counted once in per-PP stage timing aggregation. + outcome.repo_sync_duration_ms = 0; + } + if let Some(waiters) = ca_waiting_repo_by_identity.remove(&completion.identity) { + metrics.ready_enqueued += waiters.len(); + for node in waiters { + ready_queue.push_back(ReadyCaInstance { + node, + repo_outcome: outcome.clone(), + ready_enqueued_at: Instant::now(), + }); + } + } + } + } + metrics.duration_ms = elapsed_ms(started); + Ok(metrics) +} + +fn event_poll_timeout( + ca_queue: &VecDeque, + ready_queue: &VecDeque, + pending_roa_dispatch: &VecDeque, + inflight_publication_points: &HashMap, + pending_finalization: &VecDeque, + finalize_inflight: usize, + staging_inflight: usize, + instances_started: usize, + config: &TreeRunConfig, +) -> Duration { + // Stage results pending collection must not let the loop sleep: with the + // stage pool enabled a 50ms nap per turn would collapse throughput. + if !ready_queue.is_empty() + || !pending_roa_dispatch.is_empty() + || !inflight_publication_points.is_empty() + || !pending_finalization.is_empty() + || staging_inflight > 0 + || (!ca_queue.is_empty() && can_start_more(instances_started, config)) + { + Duration::from_millis(0) + } else if finalize_inflight > 0 { + Duration::from_millis(10) + } else { + Duration::from_millis(50) + } +} + +fn is_complete( + ca_queue: &VecDeque, + ready_queue: &VecDeque, + ca_waiting_repo_by_identity: &HashMap>, + pending_roa_dispatch: &VecDeque, + inflight_publication_points: &HashMap, + pending_finalization: &VecDeque, + finalize_inflight: usize, + staging_inflight: usize, + instances_started: usize, + config: &TreeRunConfig, +) -> bool { + // `staging_inflight == 0` additionally implies the stage result channel is + // drained empty: the drain loop collects every available result each turn + // and only decrements the counter while collecting. + let ca_queue_done = ca_queue.is_empty() || !can_start_more(instances_started, config); + ca_queue_done + && ready_queue.is_empty() + && ca_waiting_repo_by_identity.is_empty() + && pending_roa_dispatch.is_empty() + && inflight_publication_points.is_empty() + && pending_finalization.is_empty() + && finalize_inflight == 0 + && staging_inflight == 0 +} + +/// Minimum publication points per reduction shard; smaller runs stay +/// single-threaded to avoid thread-spawn overhead dominating the reduction. +const TREE_OUTPUT_MIN_SHARD_LEN: usize = 4096; +/// Hard cap on reduction shards regardless of core count. +const TREE_OUTPUT_MAX_SHARDS: usize = 8; + +fn tree_output_shard_count(len: usize) -> usize { + if len < TREE_OUTPUT_MIN_SHARD_LEN * 2 { + return 1; + } + let parallel = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1); + parallel + .min(TREE_OUTPUT_MAX_SHARDS) + .min(len / TREE_OUTPUT_MIN_SHARD_LEN) + .max(1) +} + +/// Per-shard reduction result of the phase2 output merge. Shards cover +/// contiguous ranges of the id-sorted finished list, so concatenating shard +/// outputs in shard order reproduces the sequential single-threaded order +/// exactly. +#[derive(Default)] +struct TreeOutputShardReduction { + instances_processed: usize, + instances_failed: usize, + warnings: Vec, + vrps: Vec, + aspas: Vec, + router_keys: Vec, + publication_points: Vec, + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats, + cir_input: CirInputAccumulator, +} + +fn reduce_finished_shard(items: Vec) -> TreeOutputShardReduction { + let mut reduction = TreeOutputShardReduction::default(); + for item in items { + match item.result { + FinishedPublicationPointResult::Ok { + source, + warnings: result_warnings, + objects, + audit, + cir_fresh_objects, + cir_cached_objects, + } => { + reduction.instances_processed += 1; + reduction.warnings.extend(result_warnings); + reduction.warnings.extend(objects.warnings); + reduction + .roa_cache_stats + .add_assign(&objects.roa_cache_stats); + reduction.vrps.extend(objects.vrps); + reduction.aspas.extend(objects.aspas); + reduction.router_keys.extend(objects.router_keys); + + let mut audit: PublicationPointAudit = audit; + audit.node_id = Some(item.node.id); + audit.parent_node_id = item.node.parent_id; + audit.discovered_from = item.node.discovered_from; + crate::validation::tree::submit_publication_point_cir_input( + &mut reduction.cir_input, + source, + &audit, + &cir_fresh_objects, + &cir_cached_objects, + ) + .expect("CIR input collection from validated audit must not fail"); + reduction.publication_points.push(audit); + } + FinishedPublicationPointResult::Err(err) => { + reduction.instances_failed += 1; + reduction.warnings.push( + Warning::new(format!("publication point failed: {err}")) + .with_context(&item.node.manifest_rsync_uri), + ); + } + } + } + reduction +} + +fn merge_shard_reductions(reductions: Vec) -> TreeOutputShardReduction { + let mut iter = reductions.into_iter(); + let mut merged = iter.next().unwrap_or_default(); + for mut other in iter { + merged.instances_processed += other.instances_processed; + merged.instances_failed += other.instances_failed; + merged.warnings.append(&mut other.warnings); + merged.vrps.append(&mut other.vrps); + merged.aspas.append(&mut other.aspas); + merged.router_keys.append(&mut other.router_keys); + merged + .publication_points + .append(&mut other.publication_points); + merged.roa_cache_stats.add_assign(&other.roa_cache_stats); + merged + .cir_input + .merge(std::mem::take(&mut other.cir_input)) + .expect("CIR input merge from validated audits must not fail"); + } + merged +} + +fn build_tree_output(mut finished: Vec) -> TreeRunAuditOutput { + let total_started = Instant::now(); + finished.sort_by_key(|item| item.node.id); + let sort_ms = total_started.elapsed().as_millis() as u64; + let shard_count = tree_output_shard_count(finished.len()); + + let reduce_started = Instant::now(); + let reductions = if shard_count <= 1 { + vec![reduce_finished_shard(finished)] + } else { + let chunk_len = finished.len().div_ceil(shard_count); + let mut shards: Vec> = Vec::new(); + let mut rest = finished; + while !rest.is_empty() { + let split_at = chunk_len.min(rest.len()); + let tail = rest.split_off(split_at); + shards.push(std::mem::replace(&mut rest, tail)); + } + std::thread::scope(|scope| { + let handles: Vec<_> = shards + .into_iter() + .map(|shard| scope.spawn(move || reduce_finished_shard(shard))) + .collect(); + handles + .into_iter() + .map(|handle| handle.join().expect("tree output reduction shard panicked")) + .collect() + }) + }; + let reduce_ms = reduce_started.elapsed().as_millis() as u64; + + let merge_started = Instant::now(); + let merged = merge_shard_reductions(reductions); + let merge_ms = merge_started.elapsed().as_millis() as u64; + + let finalize_started = Instant::now(); + let output = TreeRunAuditOutput { + tree: TreeRunOutput { + instances_processed: merged.instances_processed, + instances_failed: merged.instances_failed, + warnings: merged.warnings, + vrps: merged.vrps, + aspas: merged.aspas, + router_keys: merged.router_keys, + }, + publication_points: merged.publication_points, + roa_cache_stats: merged.roa_cache_stats, + cir_input: merged.cir_input.finalize(), + }; + let finalize_ms = finalize_started.elapsed().as_millis() as u64; + + crate::progress_log::emit( + "phase2_build_tree_output", + serde_json::json!({ + "sort_ms": sort_ms, + "shard_count": shard_count, + "reduce_ms": reduce_ms, + "merge_ms": merge_ms, + "finalize_ms": finalize_ms, + "total_ms": total_started.elapsed().as_millis() as u64, + "publication_points": output.publication_points.len(), + "instances_processed": output.tree.instances_processed, + "instances_failed": output.tree.instances_failed, + }), + ); + output +} + +pub fn run_tree_parallel_phase2_audit( + root: CaInstanceHandle, + runner: &Rpkiv1PublicationPointRunner<'_>, + config: &TreeRunConfig, +) -> Result { + run_tree_parallel_phase2_audit_multi_root(vec![root], runner, config) +} + +#[cfg(test)] +mod tests { + use super::{ + CacheHitOutcome, CompleteOutcome, FinishedPublicationPoint, FinishedPublicationPointNode, + FinishedPublicationPointResult, InflightPublicationPoint, QueuedCaInstance, + ReadyCaInstance, ReadyStageMetrics, ReadyStageSubmitter, StageOutcome, + TREE_OUTPUT_MAX_SHARDS, TREE_OUTPUT_MIN_SHARD_LEN, apply_ready_publication_point_stage, + build_tree_output, compact_phase2_finished_result, compact_phase2_finished_result_result, + compute_ready_publication_point_stage, event_poll_timeout, finalize_metrics_from_output, + finalize_publication_point_state, is_complete, merge_shard_reductions, + reduce_finished_shard, submit_ready_batch_to_stage_pool, tree_output_shard_count, + }; + use crate::audit::{ + AuditObjectKind, AuditObjectResult, DiscoveredFrom, ObjectAuditEntry, PublicationPointAudit, + }; + use crate::fetch::rsync::{RsyncFetchError, RsyncFetcher}; + use crate::parallel::repo_runtime::RepoSyncRuntimeOutcome; + use crate::policy::{CaFailedFetchPolicy, Policy, SyncPreference}; + use crate::storage::{PackTime, RocksStore}; + use crate::sync::rrdp::Fetcher; + use crate::validation::manifest::{ + FreshPublicationPointTimingBreakdown, FreshValidatedPublicationPoint, + PublicationPointSource, + }; + use crate::validation::objects::{ObjectsOutput, ObjectsStats, ParallelObjectsPrepare}; + use crate::validation::publication_point::PublicationPointSnapshot; + use crate::validation::tree::{ + CaCertificateRef, CaInstanceHandle, DiscoveredChildCaInstance, PublicationPointRunResult, + TreeRunConfig, + }; + use crate::validation::tree_runner::{ + BuildVcirTimingBreakdown, FreshPublicationPointFinalizeOutput, FreshPublicationPointStage, + PersistVcirTimingBreakdown, Rpkiv1PublicationPointRunner, + }; + use std::collections::{HashMap, VecDeque}; + use std::sync::Mutex; + use std::time::{Duration, Instant}; + + fn sample_snapshot() -> PublicationPointSnapshot { + PublicationPointSnapshot { + format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, + manifest_rsync_uri: "rsync://example.test/repo/example.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), + manifest_number_be: vec![1], + this_update: PackTime { + rfc3339_utc: "2026-04-21T00:00:00Z".to_string(), + }, + next_update: PackTime { + rfc3339_utc: "2026-04-22T00:00:00Z".to_string(), + }, + verified_at: PackTime { + rfc3339_utc: "2026-04-21T00:00:01Z".to_string(), + }, + manifest_bytes: vec![1, 2, 3], + files: Vec::new(), + } + } + + fn sample_result() -> PublicationPointRunResult { + PublicationPointRunResult { + source: PublicationPointSource::Fresh, + snapshot: Some(sample_snapshot()), + warnings: Vec::new(), + objects: ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: Vec::new(), + stats: ObjectsStats::default(), + audit: Vec::new(), + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + }, + audit: PublicationPointAudit::default(), + cir_fresh_objects: Vec::new(), + cir_cached_objects: Vec::new(), + discovered_children: Vec::new(), + } + } + + #[test] + fn compact_phase2_finished_result_drops_snapshot() { + let result = compact_phase2_finished_result(sample_result(), false); + match result { + FinishedPublicationPointResult::Ok { warnings, .. } => { + assert!(warnings.is_empty()); + } + FinishedPublicationPointResult::Err(err) => panic!("unexpected error: {err}"), + } + } + + #[test] + fn compact_phase2_finished_result_can_drop_audit_payload() { + let mut sample = sample_result(); + sample.audit.objects.push(crate::audit::ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/a.roa".to_string(), + sha256_hex: "11".repeat(32), + kind: crate::audit::AuditObjectKind::Roa, + result: crate::audit::AuditObjectResult::Ok, + detail: None, + }); + sample.audit.warnings.push(crate::audit::AuditWarning { + message: "warning".to_string(), + category: "unclassified".to_string(), + rfc_refs: Vec::new(), + context: None, + }); + sample.objects.audit.push(crate::audit::ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/b.roa".to_string(), + sha256_hex: "22".repeat(32), + kind: crate::audit::AuditObjectKind::Roa, + result: crate::audit::AuditObjectResult::Ok, + detail: None, + }); + let result = compact_phase2_finished_result(sample, true); + match result { + FinishedPublicationPointResult::Ok { + objects, + audit, + cir_fresh_objects, + .. + } => { + assert!(audit.objects.is_empty()); + assert!(audit.warnings.is_empty()); + assert!(objects.audit.is_empty()); + assert_eq!(cir_fresh_objects.len(), 1); + } + FinishedPublicationPointResult::Err(err) => panic!("unexpected error: {err}"), + } + } + + #[test] + fn compact_phase2_finished_result_result_preserves_err() { + match compact_phase2_finished_result_result(Err("boom".to_string()), false) { + FinishedPublicationPointResult::Err(err) => assert_eq!(err, "boom"), + FinishedPublicationPointResult::Ok { .. } => panic!("error should be preserved"), + } + } + + fn finished_ok_item(id: u64) -> FinishedPublicationPoint { + let mut result = sample_result(); + result.warnings.push(crate::report::Warning::new(format!( + "result-warning-{id:05}" + ))); + result + .objects + .warnings + .push(crate::report::Warning::new(format!( + "objects-warning-{id:05}" + ))); + result.objects.vrps.push(crate::validation::objects::Vrp { + asn: 64496 + id as u32, + prefix: crate::data_model::roa::IpPrefix { + afi: crate::data_model::roa::RoaAfi::Ipv4, + addr: [192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + prefix_len: 24, + }, + max_length: 24, + }); + result.audit.objects.push(crate::audit::ObjectAuditEntry { + rsync_uri: format!("rsync://example.test/repo/{id:05}.roa"), + sha256_hex: format!("{id:064x}"), + kind: crate::audit::AuditObjectKind::Roa, + result: crate::audit::AuditObjectResult::Ok, + detail: None, + }); + FinishedPublicationPoint { + node: FinishedPublicationPointNode { + id, + parent_id: None, + discovered_from: None, + manifest_rsync_uri: format!("rsync://example.test/repo/{id:05}.mft"), + }, + result: compact_phase2_finished_result(result, false), + } + } + + fn finished_err_item(id: u64) -> FinishedPublicationPoint { + FinishedPublicationPoint { + node: FinishedPublicationPointNode { + id, + parent_id: None, + discovered_from: None, + manifest_rsync_uri: format!("rsync://example.test/repo/{id:05}.mft"), + }, + result: FinishedPublicationPointResult::Err(format!("boom-{id:05}")), + } + } + + fn mixed_finished_items(n: u64) -> Vec { + (0..n) + .map(|id| { + if id % 97 == 0 { + finished_err_item(id) + } else { + finished_ok_item(id) + } + }) + .collect() + } + + #[test] + fn tree_output_shard_count_respects_thresholds() { + assert_eq!(tree_output_shard_count(0), 1); + assert_eq!( + tree_output_shard_count(TREE_OUTPUT_MIN_SHARD_LEN * 2 - 1), + 1 + ); + assert!(tree_output_shard_count(TREE_OUTPUT_MIN_SHARD_LEN * 2) >= 1); + let parallel = std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1); + assert_eq!( + tree_output_shard_count(1_000_000), + parallel.min(TREE_OUTPUT_MAX_SHARDS) + ); + } + + #[test] + fn tree_output_sharded_merge_matches_single_reduction() { + let single = merge_shard_reductions(vec![reduce_finished_shard(mixed_finished_items(300))]); + + let mut chunks: Vec> = Vec::new(); + let mut rest = mixed_finished_items(300); + while !rest.is_empty() { + let split_at = 100.min(rest.len()); + let tail = rest.split_off(split_at); + chunks.push(std::mem::replace(&mut rest, tail)); + } + let merged = + merge_shard_reductions(chunks.into_iter().map(reduce_finished_shard).collect()); + + assert_eq!(single.instances_processed, merged.instances_processed); + assert_eq!(single.instances_failed, merged.instances_failed); + assert_eq!( + format!("{:?}", single.warnings), + format!("{:?}", merged.warnings) + ); + assert_eq!(format!("{:?}", single.vrps), format!("{:?}", merged.vrps)); + assert_eq!( + format!("{:?}", single.publication_points), + format!("{:?}", merged.publication_points) + ); + assert_eq!( + format!("{:?}", single.roa_cache_stats), + format!("{:?}", merged.roa_cache_stats) + ); + assert_eq!(single.cir_input.finalize(), merged.cir_input.finalize()); + } + + #[test] + fn build_tree_output_orders_results_by_node_id() { + let mut items = mixed_finished_items(50); + items.reverse(); + let output = build_tree_output(items); + let got: Vec = output + .publication_points + .iter() + .map(|pp| pp.node_id.expect("node id set")) + .collect(); + let want: Vec = (1..50).filter(|id| id % 97 != 0).collect(); + assert_eq!(got, want); + assert_eq!(output.tree.instances_failed, 1); + let first_warning = format!("{:?}", output.tree.warnings[0]); + assert!( + first_warning.contains("publication point failed: boom-00000"), + "failed publication point warning keeps id order: {first_warning}" + ); + assert_eq!(output.cir_input.fresh_validated_objects.len(), 49); + assert_eq!( + output.cir_input.fresh_validated_objects[0].rsync_uri, + "rsync://example.test/repo/00001.roa" + ); + } + + #[test] + fn finalize_metrics_from_output_captures_breakdown_and_counts() { + let mut result = sample_result(); + result.objects.vrps.push(crate::validation::objects::Vrp { + asn: 64496, + prefix: crate::data_model::roa::IpPrefix { + afi: crate::data_model::roa::RoaAfi::Ipv4, + addr: [192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + prefix_len: 24, + }, + max_length: 24, + }); + result + .objects + .aspas + .push(crate::validation::objects::AspaAttestation { + customer_as_id: 64497, + provider_as_ids: vec![64498], + }); + result.audit.objects.push(ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/a.roa".to_string(), + sha256_hex: "11".repeat(32), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Ok, + detail: None, + }); + result + .discovered_children + .push(crate::validation::tree::DiscoveredChildCaInstance { + handle: crate::validation::tree::CaInstanceHandle { + tal_id: "test".to_string(), + ca_certificate: crate::validation::tree::CaCertificateRef::inline_der(vec![1]), + ca_certificate_rsync_uri: Some( + "rsync://example.test/repo/child.cer".to_string(), + ), + effective_ip_resources: None, + effective_as_resources: None, + manifest_rsync_uri: "rsync://example.test/repo/child.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), + rsync_base_uri: "rsync://example.test/repo/".to_string(), + rrdp_notification_uri: None, + parent_manifest_rsync_uri: None, + depth: 1, + }, + discovered_from: crate::audit::DiscoveredFrom { + parent_manifest_rsync_uri: "rsync://example.test/repo/example.mft".to_string(), + child_ca_certificate_rsync_uri: "rsync://example.test/repo/child.cer" + .to_string(), + child_ca_certificate_sha256_hex: "55".repeat(32), + }, + child_entry_projection: None, + }); + let output = FreshPublicationPointFinalizeOutput { + result, + snapshot_pack_ms: 1, + persist_vcir_ms: 2, + persist_vcir_timing: PersistVcirTimingBreakdown { + build_vcir_ms: 3, + replace_vcir_ms: 4, + build_vcir: BuildVcirTimingBreakdown { + related_artifacts_ms: 5, + ..BuildVcirTimingBreakdown::default() + }, + ..PersistVcirTimingBreakdown::default() + }, + ccr_projection_build_ms: 6, + ccr_append_ms: 7, + audit_build_ms: 8, + }; + + let metrics = finalize_metrics_from_output(&output, 9, 10, Some(11), 12, 13); + + assert_eq!(metrics.snapshot_pack_ms, 1); + assert_eq!(metrics.persist_vcir_ms, 2); + assert_eq!(metrics.persist_build_vcir_ms, 3); + assert_eq!(metrics.persist_replace_vcir_ms, 4); + assert_eq!( + metrics.persist_replace_breakdown, + crate::storage::VcirReplaceTimingBreakdown::default() + ); + assert_eq!(metrics.ccr_projection_build_ms, 6); + assert_eq!(metrics.ccr_append_ms, 7); + assert_eq!(metrics.audit_build_ms, 8); + assert_eq!(metrics.reduce_ms, 9); + assert_eq!(metrics.finalize_ms, 10); + assert_eq!(metrics.finalize_queue_wait_ms, Some(11)); + assert_eq!(metrics.finalize_worker_ms, 12); + assert_eq!(metrics.locked_files, 13); + assert_eq!(metrics.child_count, 1); + assert_eq!(metrics.vrp_count, 1); + assert_eq!(metrics.vap_count, 1); + assert_eq!(metrics.audit_object_count, 1); + } + + struct NeverHttpFetcher; + impl Fetcher for NeverHttpFetcher { + fn fetch(&self, _uri: &str) -> Result, String> { + Err("http fetch disabled in test".to_string()) + } + } + + struct FailingRsyncFetcher; + impl RsyncFetcher for FailingRsyncFetcher { + fn fetch_objects( + &self, + _rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + Err(RsyncFetchError::Fetch("rsync disabled in test".to_string())) + } + } + + fn stage_test_runner<'a>( + store: &'a RocksStore, + policy: &'a Policy, + ) -> Rpkiv1PublicationPointRunner<'a> { + Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &FailingRsyncFetcher, + validation_time: time::OffsetDateTime::now_utc(), + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + } + } + + fn stage_test_handle(manifest_rsync_uri: &str) -> CaInstanceHandle { + CaInstanceHandle { + tal_id: "test".to_string(), + ca_certificate: CaCertificateRef::inline_der(vec![1]), + ca_certificate_rsync_uri: Some("rsync://example.test/repo/ca.cer".to_string()), + effective_ip_resources: None, + effective_as_resources: None, + manifest_rsync_uri: manifest_rsync_uri.to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), + rsync_base_uri: "rsync://example.test/repo/".to_string(), + rrdp_notification_uri: None, + parent_manifest_rsync_uri: None, + depth: 0, + } + } + + fn stage_test_ready(id: u64) -> ReadyCaInstance { + ReadyCaInstance { + node: QueuedCaInstance { + id, + handle: stage_test_handle("rsync://example.test/repo/example.mft"), + parent_id: None, + discovered_from: None, + }, + repo_outcome: stage_test_repo_outcome(), + ready_enqueued_at: Instant::now(), + } + } + + fn stage_test_repo_outcome() -> RepoSyncRuntimeOutcome { + RepoSyncRuntimeOutcome { + repo_sync_ok: true, + repo_sync_err: None, + repo_sync_source: Some("rsync".to_string()), + repo_sync_phase: Some("rsync".to_string()), + repo_sync_duration_ms: 0, + warnings: Vec::new(), + } + } + + fn stage_test_child( + manifest_rsync_uri: &str, + ca_certificate_rsync_uri: &str, + ) -> DiscoveredChildCaInstance { + DiscoveredChildCaInstance { + handle: CaInstanceHandle { + ca_certificate_rsync_uri: Some(ca_certificate_rsync_uri.to_string()), + ..stage_test_handle(manifest_rsync_uri) + }, + discovered_from: DiscoveredFrom { + parent_manifest_rsync_uri: "rsync://example.test/repo/example.mft".to_string(), + child_ca_certificate_rsync_uri: ca_certificate_rsync_uri.to_string(), + child_ca_certificate_sha256_hex: "55".repeat(32), + }, + child_entry_projection: None, + } + } + + fn stage_test_fresh_stage() -> FreshPublicationPointStage { + FreshPublicationPointStage { + fresh_point: FreshValidatedPublicationPoint { + manifest_rsync_uri: "rsync://example.test/repo/example.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/".to_string(), + manifest_number_be: vec![1], + this_update: PackTime { + rfc3339_utc: "2026-04-21T00:00:00Z".to_string(), + }, + next_update: PackTime { + rfc3339_utc: "2026-04-22T00:00:00Z".to_string(), + }, + verified_at: PackTime { + rfc3339_utc: "2026-04-21T00:00:01Z".to_string(), + }, + manifest_bytes: vec![1, 2, 3], + files: Vec::new(), + }, + issuer_ca_der: vec![1u8].into(), + snapshot_prepare_timing: FreshPublicationPointTimingBreakdown::default(), + snapshot_prepare_ms: 0, + discovered_children: Vec::new(), + child_audits: Vec::new(), + discovered_router_keys: Vec::new(), + child_discovery_ms: 0, + warnings: Vec::new(), + } + } + + fn stage_test_metrics() -> ReadyStageMetrics { + ReadyStageMetrics { + ready_count: 1, + manifest_rsync_uri: Some("rsync://example.test/repo/example.mft".to_string()), + publication_point_rsync_uri: Some("rsync://example.test/repo/".to_string()), + ..ReadyStageMetrics::default() + } + } + + #[test] + fn compute_apply_fresh_error_runs_inline_fallback_and_finishes_err() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy { + sync_preference: SyncPreference::RsyncOnly, + ca_failed_fetch_policy: CaFailedFetchPolicy::StopAllOutput, + ..Policy::default() + }; + let runner = stage_test_runner(&store, &policy); + let config = TreeRunConfig::default(); + let mut next_id = 1u64; + let mut ca_queue = VecDeque::new(); + let mut pending_roa_dispatch = VecDeque::new(); + let mut inflight_publication_points = HashMap::new(); + let mut pending_finalization = VecDeque::new(); + let mut finished = Vec::new(); + + let (outcome, metrics) = + compute_ready_publication_point_stage(&runner, stage_test_ready(0), 0); + assert!( + matches!(outcome, StageOutcome::FreshError(_)), + "fresh staging against an empty store must fail" + ); + let metrics = apply_ready_publication_point_stage( + &runner, + &mut next_id, + &mut ca_queue, + &mut pending_roa_dispatch, + &mut inflight_publication_points, + &mut pending_finalization, + &mut finished, + outcome, + metrics, + &config, + false, + ); + + assert_eq!(metrics.ready_count, 1); + assert_eq!(metrics.fallback_count, 1); + assert_eq!(metrics.complete_count, 0); + assert_eq!(metrics.staged_count, 0); + assert_eq!(metrics.zero_task_count, 0); + assert_eq!(finished.len(), 1); + match &finished[0].result { + FinishedPublicationPointResult::Err(_) => {} + FinishedPublicationPointResult::Ok { .. } => { + panic!("fallback without repository data must fail") + } + } + assert!(ca_queue.is_empty()); + assert!(pending_roa_dispatch.is_empty()); + assert!(pending_finalization.is_empty()); + assert!(inflight_publication_points.is_empty()); + assert_eq!(next_id, 1); + } + + #[test] + fn apply_cache_hit_enqueues_children_sorted_and_finishes() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let runner = stage_test_runner(&store, &policy); + let config = TreeRunConfig::default(); + let mut next_id = 42u64; + let mut ca_queue = VecDeque::new(); + let mut pending_roa_dispatch = VecDeque::new(); + let mut inflight_publication_points = HashMap::new(); + let mut pending_finalization = VecDeque::new(); + let mut finished = Vec::new(); + + let mut result = sample_result(); + result.discovered_children = vec![ + stage_test_child( + "rsync://example.test/repo/z.mft", + "rsync://example.test/repo/z.cer", + ), + stage_test_child( + "rsync://example.test/repo/a.mft", + "rsync://example.test/repo/a.cer", + ), + ]; + let outcome = StageOutcome::CacheHit(Box::new(CacheHitOutcome { + ready: stage_test_ready(7), + publication_point_started: Instant::now(), + result, + })); + let mut metrics = stage_test_metrics(); + metrics.complete_count = 1; + metrics.discovered_children = 2; + let metrics = apply_ready_publication_point_stage( + &runner, + &mut next_id, + &mut ca_queue, + &mut pending_roa_dispatch, + &mut inflight_publication_points, + &mut pending_finalization, + &mut finished, + outcome, + metrics, + &config, + false, + ); + + assert_eq!(finished.len(), 1); + match &finished[0].result { + FinishedPublicationPointResult::Ok { .. } => {} + FinishedPublicationPointResult::Err(err) => { + panic!("cache hit must finish ok: {err}") + } + } + assert_eq!(finished[0].node.id, 7); + // Children are enqueued sorted by manifest URI with sequential ids + // taken from next_id, exactly like the monolithic staging did. + assert_eq!(ca_queue.len(), 2); + assert_eq!( + ca_queue[0].handle.manifest_rsync_uri, + "rsync://example.test/repo/a.mft" + ); + assert_eq!( + ca_queue[1].handle.manifest_rsync_uri, + "rsync://example.test/repo/z.mft" + ); + assert_eq!(ca_queue[0].id, 42); + assert_eq!(ca_queue[1].id, 43); + assert_eq!(ca_queue[0].parent_id, Some(7)); + assert_eq!(ca_queue[1].parent_id, Some(7)); + assert_eq!(next_id, 44); + assert!(pending_roa_dispatch.is_empty()); + assert!(pending_finalization.is_empty()); + assert!(inflight_publication_points.is_empty()); + assert_eq!(metrics.complete_count, 1); + assert_eq!(metrics.discovered_children, 2); + } + + #[test] + fn apply_complete_enqueues_finalize_task_instead_of_finishing() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let runner = stage_test_runner(&store, &policy); + let config = TreeRunConfig::default(); + let mut next_id = 1u64; + let mut ca_queue = VecDeque::new(); + let mut pending_roa_dispatch = VecDeque::new(); + let mut inflight_publication_points = HashMap::new(); + let mut pending_finalization = VecDeque::new(); + let mut finished = Vec::new(); + + let outcome = StageOutcome::Complete(Box::new(CompleteOutcome { + ready: stage_test_ready(9), + publication_point_started: Instant::now(), + fresh_stage: stage_test_fresh_stage(), + warnings: Vec::new(), + objects: sample_result().objects, + })); + let mut metrics = stage_test_metrics(); + metrics.complete_count = 1; + let metrics = apply_ready_publication_point_stage( + &runner, + &mut next_id, + &mut ca_queue, + &mut pending_roa_dispatch, + &mut inflight_publication_points, + &mut pending_finalization, + &mut finished, + outcome, + metrics, + &config, + false, + ); + + assert!( + finished.is_empty(), + "complete staging must defer the finalize to the worker queue" + ); + assert_eq!(pending_finalization.len(), 1); + let task = pending_finalization.pop_front().expect("finalize task"); + let state = task.state; + assert_eq!(state.node.id, 9); + assert_eq!(state.task_count, 0); + assert!(state.finalize_enqueued_at.is_some()); + assert!(state.results.is_empty()); + assert!(matches!( + state.objects_prepare, + ParallelObjectsPrepare::Complete(_) + )); + assert!(inflight_publication_points.is_empty()); + assert!(pending_roa_dispatch.is_empty()); + assert!(ca_queue.is_empty()); + assert_eq!(next_id, 1); + assert_eq!(metrics.complete_count, 1); + } + + #[test] + fn finalize_worker_complete_arm_finalizes_without_reduce() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let mut runner = stage_test_runner(&store, &policy); + runner.persist_vcir = false; + + let state = InflightPublicationPoint { + node: QueuedCaInstance { + id: 3, + handle: stage_test_handle("rsync://example.test/repo/example.mft"), + parent_id: None, + discovered_from: None, + }, + fresh_stage: stage_test_fresh_stage(), + objects_prepare: ParallelObjectsPrepare::Complete(sample_result().objects), + repo_outcome: stage_test_repo_outcome(), + warnings: Vec::new(), + started_at: Instant::now(), + objects_started_at: Instant::now(), + task_count: 0, + tasks_submitted: 0, + first_task_submitted_at: None, + last_task_submitted_at: None, + first_result_at: None, + last_result_at: None, + worker_ms_total: 0, + worker_ms_max: 0, + queue_wait_ms_total: 0, + queue_wait_ms_max: 0, + finalize_enqueued_at: Some(Instant::now()), + results: Vec::new(), + }; + let result = finalize_publication_point_state(&runner, state, false); + + match result.finished.result { + FinishedPublicationPointResult::Ok { source, .. } => { + assert_eq!(source, PublicationPointSource::Fresh); + } + FinishedPublicationPointResult::Err(err) => { + panic!("complete finalize must succeed: {err}") + } + } + assert_eq!(result.finished.node.id, 3); + assert_eq!(result.metrics.reduce_ms, 0); + assert_eq!(result.metrics.locked_files, 0); + } + + #[test] + fn is_complete_waits_for_staging_inflight() { + let ca_queue = VecDeque::new(); + let ready_queue = VecDeque::new(); + let ca_waiting_repo_by_identity = HashMap::new(); + let pending_roa_dispatch = VecDeque::new(); + let inflight_publication_points = HashMap::new(); + let pending_finalization = VecDeque::new(); + let config = TreeRunConfig::default(); + + // Everything drained and no staging in flight: the loop may exit. + assert!(is_complete( + &ca_queue, + &ready_queue, + &ca_waiting_repo_by_identity, + &pending_roa_dispatch, + &inflight_publication_points, + &pending_finalization, + 0, + 0, + 0, + &config, + )); + // A submitted stage task whose result has not been collected yet must + // keep the loop alive, otherwise its publication point would be lost. + assert!(!is_complete( + &ca_queue, + &ready_queue, + &ca_waiting_repo_by_identity, + &pending_roa_dispatch, + &inflight_publication_points, + &pending_finalization, + 0, + 1, + 0, + &config, + )); + } + + #[test] + fn event_poll_timeout_stays_awake_while_staging() { + let ca_queue = VecDeque::new(); + let ready_queue = VecDeque::new(); + let pending_roa_dispatch = VecDeque::new(); + let inflight_publication_points = HashMap::new(); + let pending_finalization = VecDeque::new(); + let config = TreeRunConfig::default(); + + // Staging in flight: poll must not sleep, or stage results pile up. + assert_eq!( + event_poll_timeout( + &ca_queue, + &ready_queue, + &pending_roa_dispatch, + &inflight_publication_points, + &pending_finalization, + 0, + 1, + 0, + &config, + ), + Duration::from_millis(0) + ); + // Without staging the historical tiers are unchanged. + assert_eq!( + event_poll_timeout( + &ca_queue, + &ready_queue, + &pending_roa_dispatch, + &inflight_publication_points, + &pending_finalization, + 1, + 0, + 0, + &config, + ), + Duration::from_millis(10) + ); + assert_eq!( + event_poll_timeout( + &ca_queue, + &ready_queue, + &pending_roa_dispatch, + &inflight_publication_points, + &pending_finalization, + 0, + 0, + 0, + &config, + ), + Duration::from_millis(50) + ); + } + + struct MockStageSubmitter { + capacity: usize, + submitted_ids: Vec, + } + + impl ReadyStageSubmitter for MockStageSubmitter { + fn try_submit_ready_stage( + &mut self, + task: super::ReadyStageTask, + ) -> Result< + (), + crate::parallel::object_worker::ObjectWorkerSubmitError, + > { + if self.submitted_ids.len() < self.capacity { + self.submitted_ids.push(task.ready.node.id); + Ok(()) + } else { + Err( + crate::parallel::object_worker::ObjectWorkerSubmitError::QueueFull { + worker_index: 0, + task, + }, + ) + } + } + } + + #[test] + fn submit_ready_batch_requeues_on_backpressure_without_loss_or_duplication() { + let mut ready_queue = VecDeque::new(); + for id in [10, 11, 12] { + ready_queue.push_back(stage_test_ready(id)); + } + let mut staging_inflight = 0usize; + + // First turn: only two tasks fit, the third must be returned to the + // head of the ready queue. + let mut submitter = MockStageSubmitter { + capacity: 2, + submitted_ids: Vec::new(), + }; + let metrics = submit_ready_batch_to_stage_pool( + &mut submitter, + &mut ready_queue, + &mut staging_inflight, + 256, + Duration::from_secs(60), + ) + .expect("dispatch"); + assert_eq!(metrics.submitted, 2); + assert!(metrics.queue_full); + assert_eq!(staging_inflight, 2); + assert_eq!(submitter.submitted_ids, vec![10, 11]); + assert_eq!(ready_queue.len(), 1); + assert_eq!(ready_queue[0].node.id, 12); + + // Next turn retries the requeued publication point; across both turns + // every id is submitted exactly once. + let mut submitter = MockStageSubmitter { + capacity: 8, + submitted_ids: Vec::new(), + }; + let metrics = submit_ready_batch_to_stage_pool( + &mut submitter, + &mut ready_queue, + &mut staging_inflight, + 256, + Duration::from_secs(60), + ) + .expect("dispatch"); + assert_eq!(metrics.submitted, 1); + assert!(!metrics.queue_full); + assert_eq!(staging_inflight, 3); + assert_eq!(submitter.submitted_ids, vec![12]); + assert!(ready_queue.is_empty()); + } +} diff --git a/crates/panda-rpki-validator/src/validation/tree_runner.rs b/crates/panda-rpki-validator/src/validation/tree_runner.rs new file mode 100644 index 0000000..ef2d16c --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_runner.rs @@ -0,0 +1,6195 @@ +use crate::analysis::timing::TimingHandle; +use crate::audit::{ + AuditObjectKind, AuditObjectResult, AuditWarning, ObjectAuditEntry, PublicationPointAudit, + sha256_hex, sha256_hex_from_32, +}; +use crate::audit_downloads::DownloadLogHandle; +use crate::ccr::CcrAccumulator; +use crate::current_repo_index::CurrentRepoIndexHandle; +use crate::data_model::aspa::AspaObject; +use crate::data_model::common::BigUnsigned; +use crate::data_model::crl::RpkixCrl; +use crate::data_model::manifest::ManifestObject; +use crate::data_model::rc::{ResourceCertificate, SubjectInfoAccess}; +use crate::data_model::roa::{RoaAfi, RoaObject}; +use crate::data_model::router_cert::{ + BgpsecRouterCertificate, BgpsecRouterCertificateDecodeError, BgpsecRouterCertificatePathError, + BgpsecRouterCertificateProfileError, +}; +use crate::fetch::rsync::RsyncFetcher; +use crate::parallel::config::ParallelPhase2Config; +use crate::parallel::repo_runtime::{RepoSyncRuntime, RepoSyncRuntimeOutcome}; +use crate::policy::{Policy, ResourceValidationMode}; +use crate::replay::archive::ReplayArchiveIndex; +use crate::replay::delta_archive::ReplayDeltaArchiveIndex; +use crate::report::{RfcRef, Warning}; +use crate::storage::{ + CHILD_CERTIFICATE_CACHE_ALGORITHM_VERSION, CHILD_CERTIFICATE_CACHE_SCHEMA_VERSION, + ChildCertificateCachePayload, ChildCertificateCacheProjection, + ChildCertificateCacheRouterKeyProjection, PackFile, PackTime, PublicationPointCacheChild, + PublicationPointCacheOutput, PublicationPointCacheProjection, + PublicationPointCacheProjectionWriteAction, RawByHashEntry, RoaCacheProjectionContext, + RocksStore, ValidatedCaInstanceResult, VcirArtifactKind, VcirArtifactRole, + VcirArtifactValidationStatus, VcirAuditSummary, VcirCcrManifestProjection, VcirChildEntry, + VcirFailedFetchReuseIdentity, VcirInstanceGate, VcirLocalOutput, VcirLocalOutputPayload, + VcirOutputType, VcirRelatedArtifact, VcirReplaceTimingBreakdown, VcirSourceObjectType, + VcirSummary, +}; +use crate::sync::repo::{ + sync_publication_point, sync_publication_point_replay, sync_publication_point_replay_delta, +}; +use crate::sync::rrdp::Fetcher; +use crate::validation::ca_instance::ca_instance_uris_from_ca_certificate; +use crate::validation::ca_path::{ + CaPathError, IssuerEffectiveResourcesIndex, ValidatedSubordinateCaLite, + validate_subordinate_ca_cert_with_prevalidated_issuer_and_resources, +}; +use crate::validation::manifest::{ + FreshPublicationPointTimingBreakdown, FreshValidatedPublicationPoint, ManifestFreshError, + PublicationPointData, PublicationPointSource, + process_manifest_publication_point_fresh_after_repo_sync_with_timing, +}; +use crate::validation::objects::{ + AspaAttestation, ParallelRoaWorkerPool, RoaValidationCacheInput, RoaValidationCacheView, + RouterKeyPayload, Vrp, process_publication_point_for_issuer_parallel_roa_with_cache_options, + process_publication_point_for_issuer_parallel_roa_with_pool_cache_options, + process_publication_point_for_issuer_with_cache_options_and_ta_constraints, +}; +use crate::validation::publication_point::PublicationPointSnapshot; +use crate::validation::tree::{ + CaCertificateRef, CaInstanceHandle, DiscoveredChildCaInstance, DiscoveredChildEntryProjection, + PublicationPointRunResult, PublicationPointRunner, +}; +use sha2::Digest; +use std::collections::{HashMap, HashSet}; +use std::sync::{Arc, Mutex}; + +use base64::Engine as _; +use x509_parser::prelude::FromDer; +use x509_parser::x509::SubjectPublicKeyInfo; + +use crate::ccr::manifest_location::select_manifest_signed_object_location; + +const PUBLICATION_POINT_CACHE_CHILD_RESTORE_PARALLEL_MIN_CHILDREN: usize = 256; +const PUBLICATION_POINT_CACHE_CHILD_RESTORE_MAX_WORKERS: usize = 16; +const CHILD_CERTIFICATE_CACHE_MMAP_MIN_CER_COUNT: usize = 2048; + +fn sha256_hex_to_32(hex_value: &str) -> [u8; 32] { + let mut out = [0u8; 32]; + hex::decode_to_slice(hex_value, &mut out).expect("internal sha256 hex should decode"); + out +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct BuildVcirTimingBreakdown { + pub(crate) select_crl_ms: u64, + pub(crate) current_ca_decode_ms: u64, + pub(crate) local_outputs_ms: u64, + pub(crate) child_entries_ms: u64, + pub(crate) related_artifacts_ms: u64, + pub(crate) struct_build_ms: u64, +} + +#[derive(Clone, Debug, Default)] +pub(crate) struct PersistVcirTimingBreakdown { + pub(crate) embedded_collect_ms: u64, + pub(crate) embedded_store_ms: u64, + pub(crate) build_vcir_ms: u64, + pub(crate) replace_vcir_ms: u64, + pub(crate) publication_point_cache_future_notbefore_guarded: bool, + pub(crate) build_vcir: BuildVcirTimingBreakdown, + pub(crate) replace_vcir: VcirReplaceTimingBreakdown, +} + +#[derive(Clone, Debug)] +pub(crate) struct FreshPublicationPointStage { + pub(crate) fresh_point: FreshValidatedPublicationPoint, + pub(crate) issuer_ca_der: Arc<[u8]>, + pub(crate) snapshot_prepare_timing: FreshPublicationPointTimingBreakdown, + pub(crate) snapshot_prepare_ms: u64, + pub(crate) discovered_children: Vec, + pub(crate) child_audits: Vec, + pub(crate) discovered_router_keys: Vec, + pub(crate) child_discovery_ms: u64, + pub(crate) warnings: Vec, +} + +#[derive(Debug)] +pub(crate) struct FreshPublicationPointStageError { + pub(crate) error: ManifestFreshError, + pub(crate) snapshot_prepare_ms: u64, +} + +#[derive(Clone, Debug)] +pub(crate) struct FreshPublicationPointFinalizeOutput { + pub(crate) result: PublicationPointRunResult, + pub(crate) snapshot_pack_ms: u64, + pub(crate) persist_vcir_ms: u64, + pub(crate) persist_vcir_timing: PersistVcirTimingBreakdown, + pub(crate) ccr_projection_build_ms: u64, + pub(crate) ccr_append_ms: u64, + pub(crate) audit_build_ms: u64, +} + +pub struct Rpkiv1PublicationPointRunner<'a> { + pub store: &'a RocksStore, + pub policy: &'a Policy, + pub http_fetcher: &'a dyn Fetcher, + pub rsync_fetcher: &'a dyn RsyncFetcher, + pub validation_time: time::OffsetDateTime, + pub timing: Option, + pub download_log: Option, + pub replay_archive_index: Option>, + pub replay_delta_index: Option>, + /// In-run RRDP dedup: when RRDP is enabled, only sync each `rrdp_notification_uri` once per run. + /// + /// - If RRDP succeeded for a repo, later publication points referencing that same RRDP repo + /// skip network fetches and reuse the already-populated current repository view. + /// - If RRDP failed for a repo, later publication points skip RRDP attempts and go straight + /// to rsync for their own `rsync_base_uri` (still per-publication-point). + pub rrdp_dedup: bool, + pub rrdp_repo_cache: Mutex>, // notification_uri -> rrdp_ok + + /// In-run rsync dedup: when rsync is used, only sync each `rsync_base_uri` once per run. + /// + /// This reduces duplicate rsync network fetches when multiple publication points share the + /// same `rsync_base_uri` (observed in APNIC full sync timing reports). + pub rsync_dedup: bool, + pub rsync_repo_cache: Mutex>, // rsync_base_uri -> rsync_ok + pub current_repo_index: Option, + pub repo_sync_runtime: Option>, + pub parallel_phase2_config: Option, + pub parallel_roa_worker_pool: Option, + pub ccr_accumulator: Option>, + /// When false, skip VCIR persistence and per-output VCIR projection building. + /// + /// This is intended for replay/compare-only runs where the caller does not need + /// the resulting DB to be reused by a later delta run. + pub persist_vcir: bool, + pub enable_roa_validation_cache: bool, + pub enable_child_certificate_validation_cache: bool, + pub publication_point_cache_observe_only: bool, + pub enable_publication_point_validation_cache: bool, +} + +impl<'a> Rpkiv1PublicationPointRunner<'a> { + pub(crate) fn roa_validation_cache_view_for_fresh_point( + &self, + manifest_rsync_uri: &str, + ) -> Option { + if !self.enable_roa_validation_cache { + return None; + } + let load_started = std::time::Instant::now(); + let loaded_projection = self.store.get_roa_cache_projection(manifest_rsync_uri); + if let Some(timing) = self.timing.as_ref() { + timing.record_phase_nanos( + "roa_validation_cache_projection_load_total", + load_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64, + ); + } + match loaded_projection { + Ok(Some(projection)) => { + if let Some(timing) = self.timing.as_ref() { + timing + .record_count("roa_validation_cache_projection_hit_publication_points", 1); + } + let view_started = std::time::Instant::now(); + let view = + RoaValidationCacheView::from_projection(&projection, self.validation_time); + if let Some(timing) = self.timing.as_ref() { + timing.record_phase_nanos( + "roa_validation_cache_projection_build_total", + view_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64, + ); + } + Some(view) + } + Ok(None) => { + if let Some(timing) = self.timing.as_ref() { + timing.record_count( + "roa_validation_cache_projection_missing_publication_points", + 1, + ); + } + None + } + Err(err) => { + if let Some(timing) = self.timing.as_ref() { + timing.record_count("roa_validation_cache_projection_load_errors", 1); + } + crate::progress_log::emit( + "roa_validation_cache_projection_load_error", + serde_json::json!({ + "manifest_rsync_uri": manifest_rsync_uri, + "error": err.to_string(), + }), + ); + None + } + } + } + + pub(crate) fn observe_or_reuse_publication_point_cache( + &self, + ca: &CaInstanceHandle, + repo_sync_source: Option<&str>, + repo_sync_phase: Option<&str>, + repo_sync_duration_ms: u64, + repo_sync_err: Option<&str>, + warnings: &[Warning], + ) -> Option { + if !self.publication_point_cache_observe_only + && !self.enable_publication_point_validation_cache + { + return None; + } + + let lookup_started = std::time::Instant::now(); + if let Some(timing) = self.timing.as_ref() { + timing.record_count("publication_point_cache_lookup_total", 1); + } + let projection = match self + .store + .get_publication_point_cache_projection_cached(&ca.manifest_rsync_uri) + { + Ok(Some(projection)) => projection, + Ok(None) => { + self.finish_publication_point_cache_miss(ca, "missing_projection", lookup_started); + return None; + } + Err(e) => { + self.finish_publication_point_cache_miss( + ca, + "projection_load_error", + lookup_started, + ); + crate::progress_log::emit( + "publication_point_cache_lookup_error", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "error": e.to_string(), + }), + ); + return None; + } + }; + + let current_identity = match self.current_publication_point_cache_identity(ca) { + Ok(identity) => identity, + Err(reason) => { + self.finish_publication_point_cache_miss(ca, reason.as_str(), lookup_started); + return None; + } + }; + + if projection.ca_cert_uri != ca.ca_certificate_rsync_uri { + self.finish_publication_point_cache_miss(ca, "ca_uri_mismatch", lookup_started); + return None; + } + if projection.ca_cert_sha256 != current_identity.ca_cert_sha256 { + self.finish_publication_point_cache_miss(ca, "ca_hash_mismatch", lookup_started); + return None; + } + if projection.manifest_sha256 != current_identity.manifest_sha256 { + self.finish_publication_point_cache_miss(ca, "manifest_hash_mismatch", lookup_started); + return None; + } + if projection.tal_id != ca.tal_id { + self.finish_publication_point_cache_miss(ca, "tal_mismatch", lookup_started); + return None; + } + if projection.ta_context_digest != current_identity.ta_context_digest { + self.finish_publication_point_cache_miss(ca, "ta_context_mismatch", lookup_started); + return None; + } + if projection.ca_validation_context_digest != current_identity.ca_validation_context_digest + { + self.finish_publication_point_cache_miss(ca, "parent_context_mismatch", lookup_started); + return None; + } + if projection.validation_policy_fingerprint != current_identity.policy_fingerprint { + self.finish_publication_point_cache_miss(ca, "policy_mismatch", lookup_started); + return None; + } + + let instance_not_before = + match parse_snapshot_time_value(&projection.instance_effective_not_before) { + Ok(value) => value, + Err(_) => { + self.finish_publication_point_cache_miss( + ca, + "instance_not_before_invalid", + lookup_started, + ); + return None; + } + }; + let instance_until = match parse_snapshot_time_value(&projection.instance_effective_until) { + Ok(value) => value, + Err(_) => { + self.finish_publication_point_cache_miss( + ca, + "instance_until_invalid", + lookup_started, + ); + return None; + } + }; + if self.validation_time < instance_not_before || self.validation_time >= instance_until { + self.finish_publication_point_cache_miss(ca, "instance_time_gate_miss", lookup_started); + return None; + } + if let Err(reason) = + publication_point_cache_projection_items_valid(&projection, self.validation_time) + { + self.finish_publication_point_cache_miss(ca, reason, lookup_started); + return None; + } + + if let Some(timing) = self.timing.as_ref() { + let lookup_nanos = lookup_started + .elapsed() + .as_nanos() + .min(u128::from(u64::MAX)) as u64; + timing.record_count("publication_point_cache_theoretical_hits", 1); + timing.record_phase_nanos("publication_point_cache_lookup_total", lookup_nanos); + timing.record_phase_nanos("publication_point_cache_lookup_hit_total", lookup_nanos); + timing.record_phase_nanos( + "publication_point_cache_lookup_duration_total", + lookup_nanos, + ); + } + if self.publication_point_cache_observe_only { + return None; + } + + match self.build_publication_point_cache_result( + ca, + projection, + repo_sync_source, + repo_sync_phase, + repo_sync_duration_ms, + repo_sync_err, + warnings, + ) { + Ok(result) => { + if let Some(timing) = self.timing.as_ref() { + timing.record_count("publication_point_cache_reuse_hits", 1); + } + Some(result) + } + Err(e) => { + self.record_publication_point_cache_miss("reuse_build_error"); + crate::progress_log::emit( + "publication_point_cache_reuse_error", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "error": e, + }), + ); + None + } + } + } + + fn finish_publication_point_cache_miss( + &self, + ca: &CaInstanceHandle, + reason: &str, + lookup_started: std::time::Instant, + ) { + self.record_publication_point_cache_miss(reason); + let lookup_nanos = lookup_started + .elapsed() + .as_nanos() + .min(u128::from(u64::MAX)) as u64; + if let Some(timing) = self.timing.as_ref() { + timing.record_phase_nanos("publication_point_cache_lookup_miss_total", lookup_nanos); + timing.record_phase_nanos( + "publication_point_cache_lookup_duration_total", + lookup_nanos, + ); + } + let elapsed_ms = lookup_nanos / 1_000_000; + if elapsed_ms >= crate::progress_log::pp_cache_slow_threshold_ms() { + crate::progress_log::emit( + "publication_point_cache_miss_slow", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri.as_str(), + "publication_point_rsync_uri": ca.publication_point_rsync_uri.as_str(), + "ca_certificate_rsync_uri": ca.ca_certificate_rsync_uri.as_deref(), + "reason": reason, + "elapsed_ms": elapsed_ms, + "slow_threshold_ms": crate::progress_log::pp_cache_slow_threshold_ms(), + }), + ); + } + } + + fn record_publication_point_cache_miss(&self, reason: &str) { + if let Some(timing) = self.timing.as_ref() { + timing.record_count("publication_point_cache_miss_total", 1); + match reason { + "missing_projection" => { + timing.record_count("publication_point_cache_miss_missing_projection", 1) + } + "projection_load_error" => { + timing.record_count("publication_point_cache_miss_projection_load_error", 1) + } + "current_manifest_missing" => { + timing.record_count("publication_point_cache_miss_current_manifest_missing", 1) + } + "ca_uri_mismatch" => { + timing.record_count("publication_point_cache_miss_ca_uri_mismatch", 1) + } + "ca_hash_mismatch" => { + timing.record_count("publication_point_cache_miss_ca_hash_mismatch", 1) + } + "manifest_hash_mismatch" => { + timing.record_count("publication_point_cache_miss_manifest_hash_mismatch", 1) + } + "tal_mismatch" => { + timing.record_count("publication_point_cache_miss_tal_mismatch", 1) + } + "ta_context_mismatch" => { + timing.record_count("publication_point_cache_miss_ta_context_mismatch", 1) + } + "parent_context_mismatch" => { + timing.record_count("publication_point_cache_miss_parent_context_mismatch", 1) + } + "policy_mismatch" => { + timing.record_count("publication_point_cache_miss_policy_mismatch", 1) + } + "instance_not_before_invalid" => timing.record_count( + "publication_point_cache_miss_instance_not_before_invalid", + 1, + ), + "instance_until_invalid" => { + timing.record_count("publication_point_cache_miss_instance_until_invalid", 1) + } + "instance_time_gate_miss" => { + timing.record_count("publication_point_cache_miss_instance_time_gate", 1) + } + "output_time_gate_miss" => { + timing.record_count("publication_point_cache_miss_output_time_gate", 1) + } + "child_time_gate_miss" => { + timing.record_count("publication_point_cache_miss_child_time_gate", 1) + } + "reuse_build_error" => { + timing.record_count("publication_point_cache_miss_reuse_build_error", 1) + } + _ => timing.record_count("publication_point_cache_miss_other", 1), + } + } + } + + fn current_publication_point_cache_identity( + &self, + ca: &CaInstanceHandle, + ) -> Result { + let ca_cert_sha256 = match ca.ca_certificate_rsync_uri.as_deref() { + Some(uri) => self + .current_hash_for_uri(uri) + .or_else(|| ca.ca_certificate_sha256_32()) + .ok_or_else(|| "current_ca_certificate_hash_missing".to_string())?, + None => ca + .ca_certificate_sha256_32() + .ok_or_else(|| "current_ca_certificate_hash_missing".to_string())?, + }; + let manifest_sha256 = self + .current_hash_for_uri(&ca.manifest_rsync_uri) + .ok_or_else(|| "current_manifest_missing".to_string())?; + Ok(PublicationPointCacheIdentity { + ca_cert_sha256, + manifest_sha256, + ta_context_digest: ta_context_digest_for_ca(ca), + ca_validation_context_digest: ca_validation_context_digest_for_ca(ca), + policy_fingerprint: publication_point_cache_policy_fingerprint(self.policy), + }) + } + + fn current_hash_for_uri(&self, uri: &str) -> Option<[u8; 32]> { + if let Some(index) = self.current_repo_index.as_ref() { + if let Ok(index) = index.read() { + if let Some(entry) = index.get_by_uri(uri) { + return Some(entry.current_hash); + } + } + } + self.store + .load_current_object_with_hash_by_uri(uri) + .ok() + .flatten() + .map(|entry| entry.current_hash) + } + + fn build_publication_point_cache_result( + &self, + ca: &CaInstanceHandle, + projection: PublicationPointCacheProjection, + repo_sync_source: Option<&str>, + repo_sync_phase: Option<&str>, + repo_sync_duration_ms: u64, + repo_sync_err: Option<&str>, + warnings: &[Warning], + ) -> Result { + let build_started = std::time::Instant::now(); + let mut warnings = warnings.to_vec(); + let output_reuse_count = projection.outputs.len() as u64; + let child_reuse_count = projection.children.len() as u64; + let related_object_reuse_count = projection.related_objects.len() as u64; + let build_objects_started = std::time::Instant::now(); + let mut objects = build_objects_output_from_publication_point_cache_projection( + &projection, + self.validation_time, + &mut warnings, + ); + let build_objects_ms = self.record_publication_point_cache_phase_ms( + "publication_point_cache_build_objects_total", + build_objects_started, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "publication_point_cache_build_objects", + build_objects_ms, + ); + let restore_children_started = std::time::Instant::now(); + let child_restore_workers = self.publication_point_cache_child_restore_worker_count(); + let (discovered_children, child_audits) = restore_children_from_publication_point_cache( + self.store, + ca, + &projection, + self.validation_time, + &mut warnings, + child_restore_workers, + self.timing.as_ref(), + ); + let restore_children_ms = self.record_publication_point_cache_phase_ms( + "publication_point_cache_restore_children_total", + restore_children_started, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "publication_point_cache_restore_children", + restore_children_ms, + ); + let ccr_projection = projection.ccr_manifest_projection.clone(); + let ccr_append_started = std::time::Instant::now(); + self.append_ccr_manifest_projection(&ccr_projection)?; + let ccr_append_ms = self.record_publication_point_cache_phase_ms( + "publication_point_cache_ccr_append_total", + ccr_append_started, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "publication_point_cache_ccr_append", + ccr_append_ms, + ); + let audit_build_started = std::time::Instant::now(); + let audit = build_publication_point_audit_from_publication_point_cache_projection( + ca, + PublicationPointSource::PublicationPointCache, + repo_sync_source, + repo_sync_phase, + Some(repo_sync_duration_ms), + repo_sync_err, + &projection, + self.validation_time, + &warnings, + &objects, + &child_audits, + ); + let audit_build_ms = self.record_publication_point_cache_phase_ms( + "publication_point_cache_audit_build_total", + audit_build_started, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "publication_point_cache_audit_build", + audit_build_ms, + ); + let audit_object_count = audit.objects.len() as u64; + let cir_cached_objects = audit.objects.clone(); + let cir_cached_objects_count = cir_cached_objects.len() as u64; + objects.local_outputs_cache.clear(); + let total_ms = self.record_publication_point_cache_phase_ms( + "publication_point_cache_reuse_build_total", + build_started, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "publication_point_cache_reuse_build", + total_ms, + ); + if let Some(timing) = self.timing.as_ref() { + timing.record_count("publication_point_cache_outputs_reused", output_reuse_count); + timing.record_count("publication_point_cache_children_reused", child_reuse_count); + timing.record_count( + "publication_point_cache_related_objects_reused", + related_object_reuse_count, + ); + timing.record_count( + "publication_point_cache_audit_objects_reused", + audit_object_count, + ); + if total_ms > 0 { + timing.record_count("publication_point_cache_reuse_nonzero_ms_hits", 1); + timing.record_count("publication_point_cache_reuse_nonzero_ms_total", total_ms); + } + } + if total_ms >= crate::progress_log::pp_cache_slow_threshold_ms() { + crate::progress_log::emit( + "publication_point_cache_reuse_slow", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri.as_str(), + "publication_point_rsync_uri": ca.publication_point_rsync_uri.as_str(), + "repo_sync_source": repo_sync_source, + "repo_sync_phase": repo_sync_phase, + "repo_sync_duration_ms": repo_sync_duration_ms, + "repo_sync_err": repo_sync_err, + "outputs_reused": output_reuse_count, + "children_reused": child_reuse_count, + "related_objects_reused": related_object_reuse_count, + "audit_objects_reused": audit_object_count, + "build_objects_ms": build_objects_ms, + "restore_children_ms": restore_children_ms, + "restore_children_workers": child_restore_workers, + "ccr_append_ms": ccr_append_ms, + "audit_build_ms": audit_build_ms, + "total_ms": total_ms, + "cir_cached_objects": cir_cached_objects_count, + "slow_threshold_ms": crate::progress_log::pp_cache_slow_threshold_ms(), + }), + ); + } + Ok(PublicationPointRunResult { + source: PublicationPointSource::PublicationPointCache, + snapshot: None, + warnings, + objects, + audit, + cir_fresh_objects: Vec::new(), + cir_cached_objects, + discovered_children, + }) + } + + fn record_publication_point_cache_phase_ms( + &self, + phase: &'static str, + started: std::time::Instant, + ) -> u64 { + let nanos = started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64; + if let Some(timing) = self.timing.as_ref() { + timing.record_phase_nanos(phase, nanos); + } + nanos / 1_000_000 + } + + pub(crate) fn record_publication_point_total_ms(&self, manifest_rsync_uri: &str, ms: u64) { + if let Some(timing) = self.timing.as_ref() { + timing.record_publication_point_nanos(manifest_rsync_uri, ms.saturating_mul(1_000_000)); + } + } + + pub(crate) fn record_publication_point_step_ms( + &self, + manifest_rsync_uri: &str, + step: &'static str, + ms: u64, + ) { + if let Some(timing) = self.timing.as_ref() { + timing.record_publication_point_step_nanos( + manifest_rsync_uri, + step, + ms.saturating_mul(1_000_000), + ); + } + } + + fn publication_point_cache_child_restore_worker_count(&self) -> usize { + self.parallel_phase2_config + .as_ref() + .map(|config| config.object_workers) + .unwrap_or(1) + .clamp(1, PUBLICATION_POINT_CACHE_CHILD_RESTORE_MAX_WORKERS) + } + + pub(crate) fn ccr_accumulator_snapshot(&self) -> Option { + self.ccr_accumulator + .as_ref() + .and_then(|accumulator| accumulator.lock().ok().map(|guard| guard.clone())) + } + + pub(crate) fn append_ccr_manifest_projection( + &self, + projection: &VcirCcrManifestProjection, + ) -> Result<(), String> { + if let Some(accumulator) = self.ccr_accumulator.as_ref() { + accumulator + .lock() + .map_err(|_| "lock CCR accumulator failed".to_string())? + .append_manifest_projection(projection)?; + } + Ok(()) + } + + fn append_ccr_manifest_projection_from_reuse( + &self, + projection: &VcirReuseProjection, + ) -> Result<(), String> { + match projection.source { + PublicationPointSource::Fresh => Err( + "invalid reuse projection source: fresh does not belong to failed-fetch reuse" + .to_string(), + ), + PublicationPointSource::PublicationPointCache => self.append_ccr_manifest_projection( + projection.ccr_manifest_projection.as_ref().ok_or_else(|| { + "publication-point cache reuse is missing CCR manifest projection".to_string() + })?, + ), + PublicationPointSource::VcirCurrentInstance => self.append_ccr_manifest_projection( + projection.ccr_manifest_projection.as_ref().ok_or_else(|| { + "vcir current-instance reuse is missing CCR manifest projection".to_string() + })?, + ), + PublicationPointSource::FailedFetchNoCache => Ok(()), + } + } + + fn current_manifest_hash_hex_for_audit(&self, ca: &CaInstanceHandle) -> Option { + if let Some(index_handle) = self.current_repo_index.as_ref() + && let Ok(index) = index_handle.read() + && let Some(entry) = index.get_by_uri(&ca.manifest_rsync_uri) + { + return Some(entry.current_hash_hex.clone()); + } + + self.store + .load_current_object_with_hash_by_uri(&ca.manifest_rsync_uri) + .ok() + .flatten() + .map(|current| current.current_hash_hex) + } + + fn rejected_manifest_audit_entry_for_failed_fetch( + &self, + ca: &CaInstanceHandle, + fresh_err: &ManifestFreshError, + ) -> Option { + let sha256_hex = self.current_manifest_hash_hex_for_audit(ca)?; + Some(ObjectAuditEntry { + rsync_uri: ca.manifest_rsync_uri.clone(), + sha256_hex, + kind: AuditObjectKind::Manifest, + result: AuditObjectResult::Error, + detail: Some(fresh_err.to_string()), + }) + } + + fn fresh_failure_audit_entries_for_cir( + &self, + ca: &CaInstanceHandle, + fresh_err: &ManifestFreshError, + ) -> Vec { + if !fresh_err.should_warn_when_current_instance_reused() { + return Vec::new(); + } + self.rejected_manifest_audit_entry_for_failed_fetch(ca, fresh_err) + .into_iter() + .collect() + } + + pub(crate) fn stage_fresh_publication_point_after_repo_ready( + &self, + ca: &CaInstanceHandle, + repo_sync_ok: bool, + repo_sync_err: Option<&str>, + ) -> Result { + let snapshot_prepare_started = std::time::Instant::now(); + let issuer_ca_der = ca_certificate_der_for_validation(ca, self.store, self.timing.as_ref()) + .map_err(|detail| FreshPublicationPointStageError { + error: ManifestFreshError::IssuerCaLoadFailed { detail }, + snapshot_prepare_ms: snapshot_prepare_started.elapsed().as_millis() as u64, + })?; + let issuer_ca_der: Arc<[u8]> = Arc::from(issuer_ca_der.as_ref()); + let fresh_publication_point = { + let _manifest_total = self + .timing + .as_ref() + .map(|t| t.span_phase("manifest_processing_total")); + process_manifest_publication_point_fresh_after_repo_sync_with_timing( + self.store, + &ca.manifest_rsync_uri, + &ca.publication_point_rsync_uri, + self.current_repo_index.as_ref(), + issuer_ca_der.as_ref(), + ca.ca_certificate_rsync_uri.as_deref(), + self.validation_time, + repo_sync_ok, + repo_sync_err, + ) + }; + let snapshot_prepare_ms = snapshot_prepare_started.elapsed().as_millis() as u64; + let (fresh_point, snapshot_prepare_timing) = + fresh_publication_point.map_err(|error| FreshPublicationPointStageError { + error, + snapshot_prepare_ms, + })?; + if let Some(timing) = self.timing.as_ref() { + timing.record_phase_nanos( + "fresh_snapshot_prepare_total", + snapshot_prepare_ms.saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_snapshot_manifest_load_total", + snapshot_prepare_timing + .manifest_load_ms + .saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_snapshot_manifest_decode_total", + snapshot_prepare_timing + .manifest_decode_ms + .saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_snapshot_replay_guard_total", + snapshot_prepare_timing + .replay_guard_ms + .saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_snapshot_manifest_entries_total", + snapshot_prepare_timing + .manifest_entries_ms + .saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_snapshot_pack_files_total", + snapshot_prepare_timing + .pack_files_ms + .saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_snapshot_ee_path_validate_total", + snapshot_prepare_timing + .ee_path_validate_ms + .saturating_mul(1_000_000), + ); + timing.record_count("fresh_publication_points", 1); + timing.record_count( + "fresh_manifest_files_total", + snapshot_prepare_timing.manifest_file_count as u64, + ); + } + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_snapshot_prepare", + snapshot_prepare_ms, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_snapshot_manifest_load", + snapshot_prepare_timing.manifest_load_ms, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_snapshot_manifest_decode", + snapshot_prepare_timing.manifest_decode_ms, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_snapshot_replay_guard", + snapshot_prepare_timing.replay_guard_ms, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_snapshot_manifest_entries", + snapshot_prepare_timing.manifest_entries_ms, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_snapshot_pack_files", + snapshot_prepare_timing.pack_files_ms, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_snapshot_ee_path_validate", + snapshot_prepare_timing.ee_path_validate_ms, + ); + + let child_discovery_started = std::time::Instant::now(); + let out = { + let _child_disc_total = self + .timing + .as_ref() + .map(|t| t.span_phase("child_discovery_total")); + discover_children_from_fresh_snapshot_with_audit_cached_with_issuer_der( + ca, + issuer_ca_der.as_ref(), + &fresh_point, + self.validation_time, + self.timing.as_ref(), + self.policy, + if self.enable_child_certificate_validation_cache { + Some(ChildCertificateValidationCacheContext { + store: self.store, + issuer_ca_sha256: sha256_digest_32(issuer_ca_der.as_ref()), + ca_validation_context_digest: ca_validation_context_digest_for_ca(ca), + policy_fingerprint: publication_point_cache_policy_fingerprint(self.policy), + }) + } else { + None + }, + ) + }; + let (discovered_children, child_audits, discovered_router_keys, warnings) = match out { + Ok(out) => (out.children, out.audits, out.router_keys, Vec::new()), + Err(e) => ( + Vec::new(), + Vec::new(), + Vec::new(), + vec![ + Warning::new(format!("child CA discovery failed: {e}")) + .with_rfc_refs(&[RfcRef("RFC 6487 §7.2")]) + .with_context(&ca.manifest_rsync_uri), + ], + ), + }; + let child_discovery_ms = child_discovery_started.elapsed().as_millis() as u64; + if let Some(timing) = self.timing.as_ref() { + timing.record_phase_nanos( + "fresh_child_discovery_total", + child_discovery_ms.saturating_mul(1_000_000), + ); + timing.record_count( + "fresh_children_discovered", + discovered_children.len() as u64, + ); + timing.record_count("fresh_child_audits", child_audits.len() as u64); + timing.record_count( + "fresh_router_keys_discovered", + discovered_router_keys.len() as u64, + ); + } + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_child_discovery", + child_discovery_ms, + ); + + Ok(FreshPublicationPointStage { + fresh_point, + issuer_ca_der, + snapshot_prepare_timing, + snapshot_prepare_ms, + discovered_children, + child_audits, + discovered_router_keys, + child_discovery_ms, + warnings, + }) + } + + pub(crate) fn finalize_fresh_publication_point_from_reducer( + &self, + ca: &CaInstanceHandle, + fresh_point: &FreshValidatedPublicationPoint, + warnings: Vec, + mut objects: crate::validation::objects::ObjectsOutput, + child_audits: Vec, + discovered_children: Vec, + repo_sync_source: Option<&str>, + repo_sync_phase: Option<&str>, + repo_sync_duration_ms: u64, + repo_sync_err: Option<&str>, + ) -> Result { + let snapshot_pack_started = std::time::Instant::now(); + let pack = fresh_point.to_publication_point_snapshot(); + let snapshot_pack_ms = snapshot_pack_started.elapsed().as_millis() as u64; + if let Some(timing) = self.timing.as_ref() { + timing.record_phase_nanos( + "fresh_snapshot_pack_total", + snapshot_pack_ms.saturating_mul(1_000_000), + ); + } + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_snapshot_pack", + snapshot_pack_ms, + ); + + let persist_vcir_started = std::time::Instant::now(); + let persist_vcir_timing = if self.persist_vcir { + persist_vcir_for_fresh_result_with_timing( + self.store, + self.policy, + ca, + &pack, + &mut objects, + &warnings, + &child_audits, + &discovered_children, + self.validation_time, + self.publication_point_cache_observe_only + || self.enable_publication_point_validation_cache, + ) + .map_err(|e| format!("persist VCIR failed: {e}"))? + } else { + PersistVcirTimingBreakdown::default() + }; + let persist_vcir_ms = persist_vcir_started.elapsed().as_millis() as u64; + if let Some(timing) = self.timing.as_ref() { + timing.record_phase_nanos( + "fresh_persist_vcir_total", + persist_vcir_ms.saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_persist_embedded_store_total", + persist_vcir_timing + .embedded_store_ms + .saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_persist_build_vcir_total", + persist_vcir_timing.build_vcir_ms.saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_persist_replace_vcir_total", + persist_vcir_timing + .replace_vcir_ms + .saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_persist_local_outputs_total", + persist_vcir_timing + .build_vcir + .local_outputs_ms + .saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_persist_child_entries_total", + persist_vcir_timing + .build_vcir + .child_entries_ms + .saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_persist_related_artifacts_total", + persist_vcir_timing + .build_vcir + .related_artifacts_ms + .saturating_mul(1_000_000), + ); + if persist_vcir_timing.publication_point_cache_future_notbefore_guarded { + timing.record_count("publication_point_cache_future_notbefore_guarded", 1); + } + } + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_persist_vcir", + persist_vcir_ms, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_persist_build_vcir", + persist_vcir_timing.build_vcir_ms, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_persist_replace_vcir", + persist_vcir_timing.replace_vcir_ms, + ); + + // local_outputs_cache only exists to build/persist VCIR. Release it before the + // publication point result is retained for the rest of the run. + let _released_local_outputs = std::mem::take(&mut objects.local_outputs_cache); + let _released_roa_cache_object_meta = std::mem::take(&mut objects.roa_cache_object_meta); + + let mut ccr_projection_build_ms = 0; + let mut ccr_append_ms = 0; + if self.ccr_accumulator.is_some() { + let ccr_projection_build_started = std::time::Instant::now(); + let child_entries = + build_vcir_child_entries(self.store, &discovered_children, self.validation_time)?; + let ccr_manifest_projection = + build_vcir_ccr_manifest_projection_from_fresh(ca, &pack, &child_entries)?; + ccr_projection_build_ms = ccr_projection_build_started.elapsed().as_millis() as u64; + let ccr_append_started = std::time::Instant::now(); + self.append_ccr_manifest_projection(&ccr_manifest_projection)?; + ccr_append_ms = ccr_append_started.elapsed().as_millis() as u64; + } + if let Some(timing) = self.timing.as_ref() { + timing.record_phase_nanos( + "fresh_ccr_projection_build_total", + ccr_projection_build_ms.saturating_mul(1_000_000), + ); + timing.record_phase_nanos( + "fresh_ccr_append_total", + ccr_append_ms.saturating_mul(1_000_000), + ); + } + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_ccr_projection_build", + ccr_projection_build_ms, + ); + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_ccr_append", + ccr_append_ms, + ); + + let audit_build_started = std::time::Instant::now(); + let audit = build_publication_point_audit_from_snapshot( + ca, + PublicationPointSource::Fresh, + repo_sync_source, + repo_sync_phase, + Some(repo_sync_duration_ms), + repo_sync_err, + &pack, + &warnings, + &objects, + &child_audits, + ); + let audit_build_ms = audit_build_started.elapsed().as_millis() as u64; + if let Some(timing) = self.timing.as_ref() { + timing.record_phase_nanos( + "fresh_audit_build_total", + audit_build_ms.saturating_mul(1_000_000), + ); + } + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_audit_build", + audit_build_ms, + ); + + Ok(FreshPublicationPointFinalizeOutput { + result: PublicationPointRunResult { + source: PublicationPointSource::Fresh, + snapshot: Some(pack), + warnings, + objects, + audit, + cir_fresh_objects: Vec::new(), + cir_cached_objects: Vec::new(), + discovered_children, + }, + snapshot_pack_ms, + persist_vcir_ms, + persist_vcir_timing, + ccr_projection_build_ms, + ccr_append_ms, + audit_build_ms, + }) + } +} + +impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> { + fn prefetch_discovered_children( + &self, + children: &[DiscoveredChildCaInstance], + ) -> Result<(), String> { + if let Some(runtime) = self.repo_sync_runtime.as_ref() { + runtime.prefetch_discovered_children(children)?; + } + Ok(()) + } + + fn run_publication_point( + &self, + ca: &CaInstanceHandle, + ) -> Result { + let publication_point_started = std::time::Instant::now(); + let _pp_total = self + .timing + .as_ref() + .map(|t| t.span_publication_point(&ca.manifest_rsync_uri)); + if let Some(t) = self.timing.as_ref() { + t.record_count("publication_points_seen", 1); + if ca.rrdp_notification_uri.is_some() { + t.record_count("publication_points_rrdp_notify_present_total", 1); + } else { + t.record_count("publication_points_rrdp_notify_missing_total", 1); + } + } + + let mut warnings: Vec = Vec::new(); + crate::progress_log::emit( + "publication_point_start", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "rsync_base_uri": ca.rsync_base_uri, + "rrdp_notification_uri": ca.rrdp_notification_uri, + }), + ); + + let attempted_rrdp = + self.policy.sync_preference == crate::policy::SyncPreference::RrdpThenRsync; + let original_notification_uri = ca.rrdp_notification_uri.as_deref(); + let mut effective_notification_uri = if attempted_rrdp { + original_notification_uri + } else { + None + }; + let mut skip_sync_due_to_dedup = false; + + if attempted_rrdp && self.rrdp_dedup { + if let Some(notification_uri) = original_notification_uri { + if let Some(rrdp_ok) = self + .rrdp_repo_cache + .lock() + .expect("rrdp_repo_cache lock") + .get(notification_uri) + .copied() + { + if let Some(t) = self.timing.as_ref() { + t.record_count("rrdp_repo_dedup_hits", 1); + } + if rrdp_ok { + if let Some(t) = self.timing.as_ref() { + t.record_count("rrdp_repo_dedup_rrdp_ok_skip", 1); + } + skip_sync_due_to_dedup = true; + } else { + if let Some(t) = self.timing.as_ref() { + t.record_count("rrdp_repo_dedup_rrdp_failed_skip", 1); + } + effective_notification_uri = None; + } + } else if let Some(t) = self.timing.as_ref() { + t.record_count("rrdp_repo_dedup_misses", 1); + } + } + } + + if !skip_sync_due_to_dedup && effective_notification_uri.is_none() && self.rsync_dedup { + let base = self.rsync_fetcher.dedup_key(&ca.rsync_base_uri); + let hit_ok = self + .rsync_repo_cache + .lock() + .expect("rsync_repo_cache lock") + .get(&base) + .copied() + .unwrap_or(false); + if hit_ok { + if let Some(t) = self.timing.as_ref() { + t.record_count("rsync_repo_dedup_hits", 1); + t.record_count("rsync_repo_dedup_skipped_sync", 1); + } + skip_sync_due_to_dedup = true; + } else if let Some(t) = self.timing.as_ref() { + t.record_count("rsync_repo_dedup_misses", 1); + } + } + + let repo_sync_started = std::time::Instant::now(); + let mut runtime_repo_sync_duration_ms = None; + let (repo_sync_ok, repo_sync_err, repo_sync_source, repo_sync_phase): ( + bool, + Option, + Option, + Option, + ) = if let Some(runtime) = self.repo_sync_runtime.as_ref() { + let RepoSyncRuntimeOutcome { + repo_sync_ok, + repo_sync_err, + repo_sync_source, + repo_sync_phase, + repo_sync_duration_ms, + warnings: repo_warnings, + } = runtime.sync_publication_point_repo(ca)?; + runtime_repo_sync_duration_ms = Some(repo_sync_duration_ms); + warnings.extend(repo_warnings); + ( + repo_sync_ok, + repo_sync_err, + repo_sync_source, + repo_sync_phase, + ) + } else if skip_sync_due_to_dedup { + let source = if effective_notification_uri.is_some() { + Some("rrdp_dedup_skip".to_string()) + } else { + Some("rsync_dedup_skip".to_string()) + }; + let phase = source.clone(); + (true, None, source, phase) + } else { + let repo_key = effective_notification_uri.unwrap_or_else(|| ca.rsync_base_uri.as_str()); + let _repo_total = self + .timing + .as_ref() + .map(|t| t.span_phase("repo_sync_total")); + let _repo_span = self.timing.as_ref().map(|t| t.span_rrdp_repo(repo_key)); + + match if let Some(delta_index) = self.replay_delta_index.as_ref() { + sync_publication_point_replay_delta( + self.store, + delta_index, + effective_notification_uri, + &ca.rsync_base_uri, + self.http_fetcher, + self.rsync_fetcher, + self.timing.as_ref(), + self.download_log.as_ref(), + ) + } else if let Some(replay_index) = self.replay_archive_index.as_ref() { + sync_publication_point_replay( + self.store, + replay_index, + effective_notification_uri, + &ca.rsync_base_uri, + self.http_fetcher, + self.rsync_fetcher, + self.timing.as_ref(), + self.download_log.as_ref(), + ) + } else { + sync_publication_point( + self.store, + self.policy, + effective_notification_uri, + &ca.rsync_base_uri, + self.http_fetcher, + self.rsync_fetcher, + self.timing.as_ref(), + self.download_log.as_ref(), + ) + } { + Ok(res) => { + if self.rsync_dedup && res.source == crate::sync::repo::RepoSyncSource::Rsync { + let base = self.rsync_fetcher.dedup_key(&ca.rsync_base_uri); + self.rsync_repo_cache + .lock() + .expect("rsync_repo_cache lock") + .insert(base, true); + if let Some(t) = self.timing.as_ref() { + t.record_count("rsync_repo_dedup_mark_ok", 1); + } + } + + if attempted_rrdp && self.rrdp_dedup { + if let Some(notification_uri) = original_notification_uri { + if effective_notification_uri.is_some() { + let rrdp_ok = res.source == crate::sync::repo::RepoSyncSource::Rrdp; + self.rrdp_repo_cache + .lock() + .expect("rrdp_repo_cache lock") + .insert(notification_uri.to_string(), rrdp_ok); + if let Some(t) = self.timing.as_ref() { + if rrdp_ok { + t.record_count("rrdp_repo_dedup_mark_ok", 1); + } else { + t.record_count("rrdp_repo_dedup_mark_failed", 1); + } + } + } + } + } + + warnings.extend(res.warnings); + ( + true, + None, + Some(repo_sync_source_label(res.source).to_string()), + Some(repo_sync_phase_label(res.phase).to_string()), + ) + } + Err(e) => { + if attempted_rrdp && self.rrdp_dedup { + if let Some(notification_uri) = original_notification_uri { + if effective_notification_uri.is_some() { + self.rrdp_repo_cache + .lock() + .expect("rrdp_repo_cache lock") + .insert(notification_uri.to_string(), false); + } + } + } + warnings.push( + Warning::new(format!("repo sync failed (fresh processing stopped): {e}")) + .with_rfc_refs(&[RfcRef("RFC 8182 §3.4.5"), RfcRef("RFC 9286 §6.6")]) + .with_context(&ca.rsync_base_uri), + ); + ( + false, + Some(e.to_string()), + None, + Some( + repo_sync_failure_phase_label( + attempted_rrdp, + original_notification_uri, + effective_notification_uri, + ) + .to_string(), + ), + ) + } + } + }; + let repo_sync_duration_ms = effective_repo_sync_duration_ms( + repo_sync_started.elapsed().as_millis() as u64, + runtime_repo_sync_duration_ms, + repo_sync_ok, + ); + crate::progress_log::emit( + "publication_point_repo_sync_done", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "repo_sync_ok": repo_sync_ok, + "repo_sync_source": repo_sync_source, + "repo_sync_phase": repo_sync_phase, + "repo_sync_error": repo_sync_err, + "repo_sync_duration_ms": repo_sync_duration_ms, + }), + ); + + if let Some(result) = self.observe_or_reuse_publication_point_cache( + ca, + repo_sync_source.as_deref(), + repo_sync_phase.as_deref(), + repo_sync_duration_ms, + repo_sync_err.as_deref(), + &warnings, + ) { + let total_duration_ms = publication_point_started.elapsed().as_millis() as u64; + crate::progress_log::emit( + "publication_point_finish", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "source": source_label(result.source), + "repo_sync_source": repo_sync_source, + "repo_sync_phase": repo_sync_phase, + "repo_sync_duration_ms": repo_sync_duration_ms, + "total_duration_ms": total_duration_ms, + "post_repo_duration_ms": total_duration_ms.saturating_sub(repo_sync_duration_ms), + "warning_count": result.warnings.len(), + "vrp_count": result.objects.vrps.len(), + "vap_count": result.objects.aspas.len(), + "router_key_count": result.objects.router_keys.len(), + "child_count": result.discovered_children.len(), + }), + ); + return Ok(result); + } + + let fresh_stage = self.stage_fresh_publication_point_after_repo_ready( + ca, + repo_sync_ok, + repo_sync_err.as_deref(), + ); + + match fresh_stage { + Ok(stage) => { + let FreshPublicationPointStage { + fresh_point, + issuer_ca_der, + snapshot_prepare_timing, + snapshot_prepare_ms, + discovered_children, + child_audits, + discovered_router_keys, + child_discovery_ms, + warnings: stage_warnings, + } = stage; + warnings.extend(stage_warnings); + + let has_roa = fresh_point + .files() + .iter() + .any(|file| file.rsync_uri.ends_with(".roa")); + if self.enable_roa_validation_cache { + if let Some(timing) = self.timing.as_ref() { + if has_roa { + timing.record_count( + "roa_validation_cache_roa_candidate_publication_points", + 1, + ); + } else { + timing.record_count( + "roa_validation_cache_skipped_no_roa_publication_points", + 1, + ); + } + } + } + let roa_cache_view = if has_roa { + self.roa_validation_cache_view_for_fresh_point(fresh_point.manifest_rsync_uri()) + } else { + None + }; + let roa_cache = if self.enable_roa_validation_cache && has_roa { + RoaValidationCacheInput::enabled_with_context( + roa_cache_view.as_ref(), + ca_validation_context_digest_for_ca(ca), + publication_point_cache_policy_fingerprint(self.policy), + ) + } else { + RoaValidationCacheInput::disabled() + }; + let objects_processing_started = std::time::Instant::now(); + let ta_constraints = self.policy.ta_constraints.for_tal(&ca.tal_id); + let mut objects = { + let _objects_total = self + .timing + .as_ref() + .map(|t| t.span_phase("objects_processing_total")); + if let Some(ta_constraints) = ta_constraints { + // This method is the serial/fallback publication-point path. The + // phase-2 scheduler uses the stage-specific parallel prepare path, + // which carries the same immutable per-TAL snapshot into workers. + process_publication_point_for_issuer_with_cache_options_and_ta_constraints( + &fresh_point, + self.policy, + issuer_ca_der.as_ref(), + ca.ca_certificate_rsync_uri.as_deref(), + ca.effective_ip_resources.as_ref(), + ca.effective_as_resources.as_ref(), + self.validation_time, + self.timing.as_ref(), + false, + roa_cache, + Some(ta_constraints), + ) + } else if let Some(phase2_pool) = self.parallel_roa_worker_pool.as_ref() { + process_publication_point_for_issuer_parallel_roa_with_pool_cache_options( + &fresh_point, + self.policy, + issuer_ca_der.as_ref(), + ca.ca_certificate_rsync_uri.as_deref(), + ca.effective_ip_resources.as_ref(), + ca.effective_as_resources.as_ref(), + self.validation_time, + self.timing.as_ref(), + phase2_pool, + false, + roa_cache, + ) + } else if let Some(phase2_config) = self.parallel_phase2_config.as_ref() { + process_publication_point_for_issuer_parallel_roa_with_cache_options( + &fresh_point, + self.policy, + issuer_ca_der.as_ref(), + ca.ca_certificate_rsync_uri.as_deref(), + ca.effective_ip_resources.as_ref(), + ca.effective_as_resources.as_ref(), + self.validation_time, + self.timing.as_ref(), + phase2_config, + false, + roa_cache, + ) + } else { + crate::validation::objects::process_publication_point_for_issuer_with_cache_options( + &fresh_point, + self.policy, + issuer_ca_der.as_ref(), + ca.ca_certificate_rsync_uri.as_deref(), + ca.effective_ip_resources.as_ref(), + ca.effective_as_resources.as_ref(), + self.validation_time, + self.timing.as_ref(), + false, + roa_cache, + ) + } + }; + let objects_processing_ms = objects_processing_started.elapsed().as_millis() as u64; + self.record_publication_point_step_ms( + &ca.manifest_rsync_uri, + "fresh_objects_processing", + objects_processing_ms, + ); + + objects.router_keys.extend(discovered_router_keys); + objects + .local_outputs_cache + .extend(build_router_key_local_outputs(ca, &objects.router_keys)); + + let finalized = self.finalize_fresh_publication_point_from_reducer( + ca, + &fresh_point, + warnings, + objects, + child_audits, + discovered_children, + repo_sync_source.as_deref(), + repo_sync_phase.as_deref(), + repo_sync_duration_ms, + repo_sync_err.as_deref(), + )?; + let FreshPublicationPointFinalizeOutput { + result, + snapshot_pack_ms, + persist_vcir_ms, + persist_vcir_timing, + ccr_projection_build_ms, + ccr_append_ms, + audit_build_ms, + } = finalized; + let total_duration_ms = publication_point_started.elapsed().as_millis() as u64; + crate::progress_log::emit( + "publication_point_finish", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "source": "fresh", + "repo_sync_source": repo_sync_source, + "repo_sync_phase": repo_sync_phase, + "repo_sync_duration_ms": repo_sync_duration_ms, + "total_duration_ms": total_duration_ms, + "post_repo_duration_ms": total_duration_ms.saturating_sub(repo_sync_duration_ms), + "snapshot_prepare_ms": snapshot_prepare_ms, + "snapshot_manifest_load_ms": snapshot_prepare_timing.manifest_load_ms, + "snapshot_manifest_decode_ms": snapshot_prepare_timing.manifest_decode_ms, + "snapshot_replay_guard_ms": snapshot_prepare_timing.replay_guard_ms, + "snapshot_manifest_entries_ms": snapshot_prepare_timing.manifest_entries_ms, + "snapshot_pack_files_ms": snapshot_prepare_timing.pack_files_ms, + "snapshot_ee_path_validate_ms": snapshot_prepare_timing.ee_path_validate_ms, + "objects_processing_ms": objects_processing_ms, + "child_discovery_ms": child_discovery_ms, + "snapshot_pack_ms": snapshot_pack_ms, + "persist_vcir_ms": persist_vcir_ms, + "persist_embedded_collect_ms": persist_vcir_timing.embedded_collect_ms, + "persist_embedded_store_ms": persist_vcir_timing.embedded_store_ms, + "persist_build_vcir_ms": persist_vcir_timing.build_vcir_ms, + "persist_replace_vcir_ms": persist_vcir_timing.replace_vcir_ms, + "persist_select_crl_ms": persist_vcir_timing.build_vcir.select_crl_ms, + "persist_current_ca_decode_ms": persist_vcir_timing.build_vcir.current_ca_decode_ms, + "persist_local_outputs_ms": persist_vcir_timing.build_vcir.local_outputs_ms, + "persist_child_entries_ms": persist_vcir_timing.build_vcir.child_entries_ms, + "persist_related_artifacts_ms": persist_vcir_timing.build_vcir.related_artifacts_ms, + "persist_vcir_struct_ms": persist_vcir_timing.build_vcir.struct_build_ms, + "persist_replace_breakdown": &persist_vcir_timing.replace_vcir, + "publication_point_cache_future_notbefore_guarded": persist_vcir_timing.publication_point_cache_future_notbefore_guarded, + "ccr_projection_build_ms": ccr_projection_build_ms, + "ccr_append_ms": ccr_append_ms, + "audit_build_ms": audit_build_ms, + "warning_count": result.warnings.len(), + "vrp_count": result.objects.vrps.len(), + "vap_count": result.objects.aspas.len(), + "router_key_count": result.objects.router_keys.len(), + "child_count": result.discovered_children.len(), + }), + ); + if (total_duration_ms as f64) / 1000.0 >= crate::progress_log::slow_threshold_secs() + { + crate::progress_log::emit( + "publication_point_slow", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "source": "fresh", + "repo_sync_source": repo_sync_source, + "repo_sync_phase": repo_sync_phase, + "repo_sync_duration_ms": repo_sync_duration_ms, + "total_duration_ms": total_duration_ms, + "post_repo_duration_ms": total_duration_ms.saturating_sub(repo_sync_duration_ms), + "snapshot_prepare_ms": snapshot_prepare_ms, + "snapshot_manifest_load_ms": snapshot_prepare_timing.manifest_load_ms, + "snapshot_manifest_decode_ms": snapshot_prepare_timing.manifest_decode_ms, + "snapshot_replay_guard_ms": snapshot_prepare_timing.replay_guard_ms, + "snapshot_manifest_entries_ms": snapshot_prepare_timing.manifest_entries_ms, + "snapshot_pack_files_ms": snapshot_prepare_timing.pack_files_ms, + "snapshot_ee_path_validate_ms": snapshot_prepare_timing.ee_path_validate_ms, + "objects_processing_ms": objects_processing_ms, + "child_discovery_ms": child_discovery_ms, + "snapshot_pack_ms": snapshot_pack_ms, + "persist_vcir_ms": persist_vcir_ms, + "persist_embedded_collect_ms": persist_vcir_timing.embedded_collect_ms, + "persist_embedded_store_ms": persist_vcir_timing.embedded_store_ms, + "persist_build_vcir_ms": persist_vcir_timing.build_vcir_ms, + "persist_replace_vcir_ms": persist_vcir_timing.replace_vcir_ms, + "persist_select_crl_ms": persist_vcir_timing.build_vcir.select_crl_ms, + "persist_current_ca_decode_ms": persist_vcir_timing.build_vcir.current_ca_decode_ms, + "persist_local_outputs_ms": persist_vcir_timing.build_vcir.local_outputs_ms, + "persist_child_entries_ms": persist_vcir_timing.build_vcir.child_entries_ms, + "persist_related_artifacts_ms": persist_vcir_timing.build_vcir.related_artifacts_ms, + "persist_vcir_struct_ms": persist_vcir_timing.build_vcir.struct_build_ms, + "persist_replace_breakdown": &persist_vcir_timing.replace_vcir, + "publication_point_cache_future_notbefore_guarded": persist_vcir_timing.publication_point_cache_future_notbefore_guarded, + "ccr_projection_build_ms": ccr_projection_build_ms, + "ccr_append_ms": ccr_append_ms, + "audit_build_ms": audit_build_ms, + }), + ); + } + Ok(result) + } + Err(stage_err) => { + let snapshot_prepare_ms = stage_err.snapshot_prepare_ms; + let fresh_err = stage_err.error; + match self.policy.ca_failed_fetch_policy { + crate::policy::CaFailedFetchPolicy::StopAllOutput => { + let total_duration_ms = + publication_point_started.elapsed().as_millis() as u64; + crate::progress_log::emit( + "publication_point_finish", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "source": "error", + "repo_sync_source": repo_sync_source, + "repo_sync_phase": repo_sync_phase, + "repo_sync_duration_ms": repo_sync_duration_ms, + "total_duration_ms": total_duration_ms, + "post_repo_duration_ms": total_duration_ms.saturating_sub(repo_sync_duration_ms), + "snapshot_prepare_ms": snapshot_prepare_ms, + "projection_ms": 0, + "audit_build_ms": 0, + "error": fresh_err.to_string(), + }), + ); + crate::progress_log::emit( + "repo_terminal_failure", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "repo_sync_source": repo_sync_source, + "repo_sync_phase": repo_sync_phase, + "repo_sync_error": repo_sync_err, + "repo_sync_duration_ms": repo_sync_duration_ms, + "terminal_state": "stop_all_output", + "error": fresh_err.to_string(), + }), + ); + Err(format!("{fresh_err}")) + } + crate::policy::CaFailedFetchPolicy::ReuseCurrentInstanceVcir => { + let projection_started = std::time::Instant::now(); + let projection = project_current_instance_vcir_on_failed_fetch( + self.store, + ca, + &fresh_err, + self.policy, + self.validation_time, + ) + .map_err(|e| format!("failed fetch VCIR projection failed: {e}"))?; + let fresh_failure_audits = + self.fresh_failure_audit_entries_for_cir(ca, &fresh_err); + self.append_ccr_manifest_projection_from_reuse(&projection)?; + let projection_ms = projection_started.elapsed().as_millis() as u64; + warnings.extend(projection.warnings.clone()); + let audit_build_started = std::time::Instant::now(); + let audit = build_publication_point_audit_from_vcir( + ca, + projection.source, + repo_sync_source.as_deref(), + repo_sync_phase.as_deref(), + Some(repo_sync_duration_ms), + repo_sync_err.as_deref(), + projection.vcir.as_ref(), + projection.snapshot.as_ref(), + &warnings, + &projection.objects, + &projection.child_audits, + &fresh_failure_audits, + ); + let audit_build_ms = audit_build_started.elapsed().as_millis() as u64; + let cir_cached_objects = + if projection.source == PublicationPointSource::VcirCurrentInstance { + audit + .objects + .iter() + .filter(|entry| { + !fresh_failure_audits.iter().any(|fresh| fresh == *entry) + }) + .cloned() + .collect() + } else { + Vec::new() + }; + let result = PublicationPointRunResult { + source: projection.source, + snapshot: projection.snapshot, + warnings, + objects: projection.objects, + audit, + cir_fresh_objects: if projection.source + == PublicationPointSource::VcirCurrentInstance + { + fresh_failure_audits + } else { + Vec::new() + }, + cir_cached_objects, + discovered_children: projection.discovered_children, + }; + let total_duration_ms = + publication_point_started.elapsed().as_millis() as u64; + crate::progress_log::emit( + "publication_point_finish", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "source": source_label(result.source), + "repo_sync_source": repo_sync_source, + "repo_sync_phase": repo_sync_phase, + "repo_sync_duration_ms": repo_sync_duration_ms, + "total_duration_ms": total_duration_ms, + "post_repo_duration_ms": total_duration_ms.saturating_sub(repo_sync_duration_ms), + "snapshot_prepare_ms": snapshot_prepare_ms, + "projection_ms": projection_ms, + "audit_build_ms": audit_build_ms, + "warning_count": result.warnings.len(), + "vrp_count": result.objects.vrps.len(), + "vap_count": result.objects.aspas.len(), + "router_key_count": result.objects.router_keys.len(), + "child_count": result.discovered_children.len(), + }), + ); + match result.source { + PublicationPointSource::VcirCurrentInstance if !repo_sync_ok => { + crate::progress_log::emit( + "rsync_failed_fallback_current_instance", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "repo_sync_source": repo_sync_source, + "repo_sync_phase": repo_sync_phase, + "repo_sync_error": repo_sync_err, + "repo_sync_duration_ms": repo_sync_duration_ms, + "terminal_state": "fallback_current_instance", + }), + ); + } + PublicationPointSource::FailedFetchNoCache => { + if !repo_sync_ok { + crate::progress_log::emit( + "rsync_failed_no_cache", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "repo_sync_source": repo_sync_source, + "repo_sync_phase": repo_sync_phase, + "repo_sync_error": repo_sync_err, + "repo_sync_duration_ms": repo_sync_duration_ms, + "terminal_state": "failed_no_cache", + }), + ); + } + crate::progress_log::emit( + "repo_terminal_failure", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "repo_sync_source": repo_sync_source, + "repo_sync_phase": repo_sync_phase, + "repo_sync_error": repo_sync_err, + "repo_sync_duration_ms": repo_sync_duration_ms, + "terminal_state": "failed_no_cache", + }), + ); + } + PublicationPointSource::Fresh => {} + PublicationPointSource::PublicationPointCache => {} + PublicationPointSource::VcirCurrentInstance => {} + } + if (total_duration_ms as f64) / 1000.0 + >= crate::progress_log::slow_threshold_secs() + { + crate::progress_log::emit( + "publication_point_slow", + serde_json::json!({ + "manifest_rsync_uri": ca.manifest_rsync_uri, + "publication_point_rsync_uri": ca.publication_point_rsync_uri, + "source": source_label(result.source), + "repo_sync_source": repo_sync_source, + "repo_sync_phase": repo_sync_phase, + "repo_sync_duration_ms": repo_sync_duration_ms, + "total_duration_ms": total_duration_ms, + "post_repo_duration_ms": total_duration_ms.saturating_sub(repo_sync_duration_ms), + "snapshot_prepare_ms": snapshot_prepare_ms, + "projection_ms": projection_ms, + "audit_build_ms": audit_build_ms, + }), + ); + } + Ok(result) + } + } + } + } + } +} + +struct ChildDiscoveryOutput { + children: Vec, + audits: Vec, + router_keys: Vec, +} + +#[derive(Clone, Debug)] +struct VerifiedIssuerCrl { + crl: crate::data_model::crl::RpkixCrl, + revoked_serials: std::collections::HashSet>, + sha256_hex: String, +} + +#[derive(Clone, Debug)] +enum CachedIssuerCrl { + Pending { + bytes: Vec, + sha256_hex: Option, + }, + Ok(VerifiedIssuerCrl), +} + +impl CachedIssuerCrl { + fn current_sha256_hex(&mut self) -> &str { + match self { + CachedIssuerCrl::Pending { bytes, sha256_hex } => { + if sha256_hex.is_none() { + *sha256_hex = Some(crate::audit::sha256_hex(bytes)); + } + sha256_hex + .as_deref() + .expect("pending CRL sha256 must be populated") + } + CachedIssuerCrl::Ok(verified) => verified.sha256_hex.as_str(), + } + } +} + +struct PublicationPointCacheIdentity { + ca_cert_sha256: [u8; 32], + manifest_sha256: [u8; 32], + ta_context_digest: [u8; 32], + ca_validation_context_digest: [u8; 32], + policy_fingerprint: [u8; 32], +} + +fn sha256_digest_32(bytes: impl AsRef<[u8]>) -> [u8; 32] { + let digest = sha2::Sha256::digest(bytes.as_ref()); + let mut out = [0u8; 32]; + out.copy_from_slice(&digest); + out +} + +fn hash_serialized_parts(parts: &[(&str, &[u8])]) -> [u8; 32] { + let mut hasher = sha2::Sha256::new(); + for (label, value) in parts { + hasher.update((label.len() as u64).to_be_bytes()); + hasher.update(label.as_bytes()); + hasher.update((value.len() as u64).to_be_bytes()); + hasher.update(*value); + } + let digest = hasher.finalize(); + let mut out = [0u8; 32]; + out.copy_from_slice(&digest); + out +} + +fn cbor_or_debug_bytes(value: &T) -> Vec { + serde_cbor::to_vec(value).unwrap_or_else(|_| format!("{value:?}").into_bytes()) +} + +fn ta_context_digest_for_ca(ca: &CaInstanceHandle) -> [u8; 32] { + hash_serialized_parts(&[ + ("version", b"publication-point-cache-ta-v1"), + ("tal_id", ca.tal_id.as_bytes()), + ]) +} + +pub(crate) fn ca_validation_context_digest_for_ca(ca: &CaInstanceHandle) -> [u8; 32] { + let parent_manifest = ca + .parent_manifest_rsync_uri + .as_deref() + .unwrap_or("") + .as_bytes(); + let effective_ip = cbor_or_debug_bytes(&ca.effective_ip_resources); + let effective_as = cbor_or_debug_bytes(&ca.effective_as_resources); + hash_serialized_parts(&[ + ("version", b"publication-point-cache-parent-context-v1"), + ("tal_id", ca.tal_id.as_bytes()), + ("parent_manifest", parent_manifest), + ("effective_ip", effective_ip.as_slice()), + ("effective_as", effective_as.as_slice()), + ]) +} + +pub(crate) fn publication_point_cache_policy_fingerprint(policy: &Policy) -> [u8; 32] { + let policy_bytes = cbor_or_debug_bytes(policy); + hash_serialized_parts(&[ + ("version", b"publication-point-cache-policy-v3"), + ("policy", policy_bytes.as_slice()), + ("ta_constraints", policy.ta_constraints.fingerprint_bytes()), + ]) +} + +fn publication_point_cache_projection_items_valid( + projection: &PublicationPointCacheProjection, + validation_time: time::OffsetDateTime, +) -> Result<(), &'static str> { + for output in &projection.outputs { + if !pack_time_window_contains( + &output.item_effective_not_before, + &output.item_effective_until, + validation_time, + ) { + return Err("output_time_gate_miss"); + } + } + for child in &projection.children { + if !pack_time_window_contains( + &child.child_effective_not_before, + &child.child_effective_until, + validation_time, + ) { + return Err("child_time_gate_miss"); + } + } + Ok(()) +} + +fn pack_time_window_contains( + not_before: &PackTime, + until: &PackTime, + validation_time: time::OffsetDateTime, +) -> bool { + let Ok(not_before) = parse_snapshot_time_value(not_before) else { + return false; + }; + let Ok(until) = parse_snapshot_time_value(until) else { + return false; + }; + validation_time >= not_before && validation_time < until +} + +#[derive(Clone, Copy)] +struct ChildCertificateValidationCacheContext<'a> { + store: &'a RocksStore, + issuer_ca_sha256: [u8; 32], + ca_validation_context_digest: [u8; 32], + policy_fingerprint: [u8; 32], +} + +fn child_certificate_cache_key_sha256_hex( + child_cert_uri: &str, + child_cert_sha256: &[u8; 32], + issuer_ca_sha256: &[u8; 32], + ca_validation_context_digest: &[u8; 32], + policy_fingerprint: &[u8; 32], +) -> String { + let digest = hash_serialized_parts(&[ + ("version", b"child-certificate-cache-key-v1"), + ("child_cert_uri", child_cert_uri.as_bytes()), + ("child_cert_sha256", child_cert_sha256), + ("issuer_ca_sha256", issuer_ca_sha256), + // Keep the persisted cache-key label stable; this change only renames Rust identifiers. + ("parent_context_digest", ca_validation_context_digest), + ("policy_fingerprint", policy_fingerprint), + ]); + sha256_hex_from_32(&digest) +} + +#[derive(Clone, Debug)] +struct ChildCertificateCacheCandidate { + projection: Option, +} + +fn remember_child_certificate_cache_dirty_projection( + dirty_projections: &mut Option>, + projection: &ChildCertificateCacheProjection, +) { + if let Some(dirty_projections) = dirty_projections { + dirty_projections.insert(projection.cache_key_sha256_hex.clone(), projection.clone()); + } +} + +fn load_child_certificate_der_for_discovery<'a>( + file: &'a PackFile, + elapsed_nanos: &mut u64, + count: &mut u64, +) -> Result<&'a [u8], String> { + let started = std::time::Instant::now(); + let bytes = file + .bytes() + .map_err(|e| format!("child certificate bytes load failed: {e}"))?; + *elapsed_nanos = + elapsed_nanos.saturating_add(started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); + *count = count.saturating_add(1); + Ok(bytes) +} + +fn ca_certificate_der_for_validation<'a>( + ca: &'a CaInstanceHandle, + store: &RocksStore, + timing: Option<&TimingHandle>, +) -> Result, String> { + let started = std::time::Instant::now(); + let was_lazy = ca.ca_certificate_sha256_hex().is_some(); + let der = ca.ca_certificate_der(store)?; + if was_lazy { + let elapsed = started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64; + if let Some(timing) = timing { + timing.record_count("ca_certificate_lazy_load_count", 1); + timing.record_count("ca_certificate_lazy_load_bytes", der.len() as u64); + timing.record_phase_nanos("ca_certificate_lazy_load_total", elapsed); + } + } + Ok(der) +} + +fn child_certificate_cache_certificate_window( + child_not_before: time::OffsetDateTime, + child_not_after: time::OffsetDateTime, + issuer_not_before: time::OffsetDateTime, + issuer_not_after: time::OffsetDateTime, +) -> (PackTime, PackTime) { + let effective_not_before = child_not_before.max(issuer_not_before); + let effective_until = child_not_after.min(issuer_not_after); + ( + PackTime::from_utc_offset_datetime(effective_not_before), + PackTime::from_utc_offset_datetime(effective_until), + ) +} + +fn get_current_crl_sha256_hex( + crl_rsync_uri: &str, + crl_cache: &mut std::collections::HashMap, +) -> Option { + crl_cache + .get_mut(crl_rsync_uri) + .map(|entry| entry.current_sha256_hex().to_string()) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ChildCertificateCacheCrlGate { + Unchanged, + ChangedValid, + Expired, + Invalid, + Missing, +} + +#[derive(Default)] +struct ChildCertificateCacheCrlGateSet { + gates_by_uri_and_expected_hash: + std::collections::HashMap<(String, String), ChildCertificateCacheCrlGate>, +} + +impl ChildCertificateCacheCrlGateSet { + fn evaluate( + &mut self, + projection: &ChildCertificateCacheProjection, + crl_cache: &mut std::collections::HashMap, + issuer_ca_der: &[u8], + validation_time: time::OffsetDateTime, + ) -> (ChildCertificateCacheCrlGate, bool) { + let key = ( + projection.issuer_crl_uri.clone(), + projection.issuer_crl_sha256_hex.clone(), + ); + if let Some(gate) = self.gates_by_uri_and_expected_hash.get(&key) { + return (*gate, true); + } + + let gate = evaluate_child_certificate_cache_crl_gate( + projection, + crl_cache, + issuer_ca_der, + validation_time, + ); + self.gates_by_uri_and_expected_hash.insert(key, gate); + (gate, false) + } +} + +fn evaluate_child_certificate_cache_crl_gate( + projection: &ChildCertificateCacheProjection, + crl_cache: &mut std::collections::HashMap, + issuer_ca_der: &[u8], + validation_time: time::OffsetDateTime, +) -> ChildCertificateCacheCrlGate { + let Some(current_crl_hash) = get_current_crl_sha256_hex(&projection.issuer_crl_uri, crl_cache) + else { + return ChildCertificateCacheCrlGate::Missing; + }; + if current_crl_hash == projection.issuer_crl_sha256_hex { + let verified_crl = match ensure_issuer_crl_verified( + &projection.issuer_crl_uri, + crl_cache, + issuer_ca_der, + ) { + Ok(verified_crl) => verified_crl, + Err(_) => return ChildCertificateCacheCrlGate::Invalid, + }; + if !crl_valid_at_time_for_cache(&verified_crl.crl, validation_time) { + return ChildCertificateCacheCrlGate::Expired; + } + return ChildCertificateCacheCrlGate::Unchanged; + } + + let verified_crl = + match ensure_issuer_crl_verified(&projection.issuer_crl_uri, crl_cache, issuer_ca_der) { + Ok(verified_crl) => verified_crl, + Err(_) => return ChildCertificateCacheCrlGate::Invalid, + }; + if !crl_valid_at_time_for_cache(&verified_crl.crl, validation_time) { + return ChildCertificateCacheCrlGate::Expired; + } + ChildCertificateCacheCrlGate::ChangedValid +} + +fn crl_valid_at_time_for_cache( + crl: &crate::data_model::crl::RpkixCrl, + validation_time: time::OffsetDateTime, +) -> bool { + let this_update = crl.this_update.utc.to_offset(time::UtcOffset::UTC); + let next_update = crl.next_update.utc.to_offset(time::UtcOffset::UTC); + validation_time >= this_update && validation_time < next_update +} + +#[derive(Clone, Debug)] +struct VcirReuseProjection { + source: PublicationPointSource, + vcir: Option, + ccr_manifest_projection: Option, + snapshot: Option, + objects: crate::validation::objects::ObjectsOutput, + child_audits: Vec, + discovered_children: Vec, + warnings: Vec, +} + +fn discover_children_from_fresh_snapshot_with_audit( + issuer: &CaInstanceHandle, + publication_point: &P, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, +) -> Result { + let issuer_ca_der = match &issuer.ca_certificate { + CaCertificateRef::InlineDer(bytes) => bytes.as_slice(), + CaCertificateRef::RepoBytes { .. } => { + return Err("lazy CA certificate requires store-backed child discovery".to_string()); + } + }; + let default_policy = Policy::default(); + discover_children_from_fresh_snapshot_with_audit_cached_with_issuer_der( + issuer, + issuer_ca_der, + publication_point, + validation_time, + timing, + &default_policy, + None, + ) +} + +fn discover_children_from_fresh_snapshot_with_audit_cached( + issuer: &CaInstanceHandle, + publication_point: &P, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + cache_context: Option>, +) -> Result { + let issuer_ca_der = match &issuer.ca_certificate { + CaCertificateRef::InlineDer(bytes) => bytes.as_slice(), + CaCertificateRef::RepoBytes { .. } => { + return Err("lazy CA certificate requires store-backed child discovery".to_string()); + } + }; + let default_policy = Policy::default(); + discover_children_from_fresh_snapshot_with_audit_cached_with_issuer_der( + issuer, + issuer_ca_der, + publication_point, + validation_time, + timing, + &default_policy, + cache_context, + ) +} + +fn discover_children_from_fresh_snapshot_with_audit_cached_with_issuer_der< + P: PublicationPointData, +>( + issuer: &CaInstanceHandle, + issuer_ca_der: &[u8], + publication_point: &P, + validation_time: time::OffsetDateTime, + timing: Option<&TimingHandle>, + policy: &Policy, + cache_context: Option>, +) -> Result { + let locked_files = publication_point.files(); + // Issuer CA is only required when we actually attempt to validate a subordinate CA. For some + // audit-only error paths (e.g., missing CRL in the snapshot), we still want discovery to succeed. + let issuer_ca_decode_error: Option; + let issuer_ca = match crate::data_model::rc::ResourceCertificate::decode_der(issuer_ca_der) { + Ok(v) => { + match v.validate_rfc6487_profile(crate::data_model::rc::ResourceCertificateRole::Ca) { + Ok(()) => { + issuer_ca_decode_error = None; + Some(v) + } + Err(e) => { + issuer_ca_decode_error = Some(format!( + "issuer CA profile validation failed: {e} (RFC 6487 §4.8)" + )); + None + } + } + } + Err(e) => { + issuer_ca_decode_error = Some(format!( + "issuer CA decode failed: {e} (RFC 5280 §4.1; RFC 6487 §4)" + )); + None + } + }; + + let issuer_spki_error: Option; + let issuer_spki: Option> = if let Some(ca) = issuer_ca.as_ref() { + match SubjectPublicKeyInfo::from_der(&ca.tbs.subject_public_key_info) { + Ok((rem, spki)) if rem.is_empty() => { + issuer_spki_error = None; + Some(spki) + } + Ok((rem, _)) => { + issuer_spki_error = Some(format!( + "trailing bytes after issuer SubjectPublicKeyInfo DER: {} bytes (DER; RFC 5280 §4.1.2.7)", + rem.len() + )); + None + } + Err(e) => { + issuer_spki_error = Some(format!( + "issuer SubjectPublicKeyInfo parse error: {e} (RFC 5280 §4.1.2.7)" + )); + None + } + } + } else { + issuer_spki_error = issuer_ca_decode_error.clone(); + None + }; + + let mut crl_cache: std::collections::HashMap = locked_files + .iter() + .filter(|f| f.rsync_uri.ends_with(".crl")) + .map(|f| -> Result<(String, CachedIssuerCrl), String> { + let bytes = f + .bytes_cloned() + .map_err(|e| format!("snapshot CRL bytes load failed: {e}"))?; + Ok(( + f.rsync_uri.clone(), + CachedIssuerCrl::Pending { + bytes, + sha256_hex: Some(sha256_hex_from_32(&f.sha256)), + }, + )) + }) + .collect::>()?; + + let mut out: Vec = Vec::new(); + let mut audits: Vec = Vec::new(); + let mut router_keys: Vec = Vec::new(); + let issuer_resources_index = IssuerEffectiveResourcesIndex::from_effective_resources( + issuer.effective_ip_resources.as_ref(), + issuer.effective_as_resources.as_ref(), + ) + .map_err(|e| format!("build issuer effective resources index failed: {e}"))?; + + let mut cer_seen: u64 = 0; + let mut ca_skipped_not_ca: u64 = 0; + let mut ca_ok: u64 = 0; + let mut ca_error: u64 = 0; + let mut router_ok: u64 = 0; + let mut router_error: u64 = 0; + let mut router_skipped_non_router: u64 = 0; + let mut crl_select_error: u64 = 0; + let mut uri_discovery_error: u64 = 0; + let mut child_cert_cache_lookup: u64 = 0; + let mut child_cert_cache_hit: u64 = 0; + let mut child_cert_cache_hit_ca: u64 = 0; + let mut child_cert_cache_hit_router: u64 = 0; + let mut child_cert_cache_miss_not_found: u64 = 0; + let mut child_cert_cache_miss_time_gate: u64 = 0; + let mut child_cert_cache_miss_crl_missing: u64 = 0; + let mut child_cert_cache_miss_crl_invalid: u64 = 0; + let mut child_cert_cache_miss_crl_expired: u64 = 0; + let mut child_cert_cache_miss_revoked: u64 = 0; + let mut child_cert_cache_crl_recheck_hit: u64 = 0; + let mut child_cert_cache_crl_gate_reused: u64 = 0; + let mut child_cert_cache_load_error: u64 = 0; + let mut child_cert_cache_write_ok: u64 = 0; + let mut child_cert_cache_write_error: u64 = 0; + + let mut select_crl_nanos: u64 = 0; + let mut child_decode_nanos: u64 = 0; + let mut validate_sub_ca_nanos: u64 = 0; + let mut validate_router_nanos: u64 = 0; + let mut uri_discovery_nanos: u64 = 0; + let mut enqueue_nanos: u64 = 0; + let mut child_cert_cache_lookup_nanos: u64 = 0; + let mut child_cert_cache_write_nanos: u64 = 0; + let child_cert_der_load_cache_hit_nanos: u64 = 0; + let mut child_cert_der_load_fresh_nanos: u64 = 0; + let child_cert_der_load_cache_hit_count: u64 = 0; + let mut child_cert_der_load_fresh_count: u64 = 0; + + let mut eff_ip_items_bucket_le_10: u64 = 0; + let mut eff_ip_items_bucket_le_100: u64 = 0; + let mut eff_ip_items_bucket_gt_100: u64 = 0; + let mut eff_as_items_bucket_le_10: u64 = 0; + let mut eff_as_items_bucket_le_100: u64 = 0; + let mut eff_as_items_bucket_gt_100: u64 = 0; + + let mut child_cache_candidates: HashMap = + HashMap::new(); + let mut child_cache_segment_keys = Vec::::new(); + let mut child_cache_segment_dirty_projections: Option< + HashMap, + > = None; + let mut child_cert_cache_batch_lookup_publication_points: u64 = 0; + let mut child_cert_cache_batch_lookup_entries: u64 = 0; + let mut child_cert_cache_batch_lookup_errors: u64 = 0; + let mut child_cert_cache_batch_lookup_nanos: u64 = 0; + let mut child_cert_cache_mmap_lookup_publication_points: u64 = 0; + let mut child_cert_cache_mmap_lookup_entries: u64 = 0; + let mut child_cert_cache_mmap_lookup_hits: u64 = 0; + let mut child_cert_cache_mmap_lookup_misses: u64 = 0; + let mut child_cert_cache_mmap_lookup_missing_segments: u64 = 0; + let mut child_cert_cache_mmap_lookup_errors: u64 = 0; + let mut child_cert_cache_mmap_lookup_file_bytes: u64 = 0; + let mut child_cert_cache_mmap_lookup_nanos: u64 = 0; + let mut child_cert_cache_mmap_write_entries: u64 = 0; + let mut child_cert_cache_mmap_write_errors: u64 = 0; + let mut child_cert_cache_mmap_write_file_bytes: u64 = 0; + let mut child_cert_cache_mmap_write_nanos: u64 = 0; + + if let Some(cache) = cache_context { + let mut uris = Vec::new(); + let mut keys = Vec::new(); + for f in locked_files + .iter() + .filter(|file| file.rsync_uri.ends_with(".cer")) + { + uris.push(f.rsync_uri.clone()); + keys.push(child_certificate_cache_key_sha256_hex( + &f.rsync_uri, + &f.sha256, + &cache.issuer_ca_sha256, + &cache.ca_validation_context_digest, + &cache.policy_fingerprint, + )); + } + let use_mmap_segment = keys.len() >= CHILD_CERTIFICATE_CACHE_MMAP_MIN_CER_COUNT; + if use_mmap_segment { + child_cache_segment_keys = keys.clone(); + child_cache_segment_dirty_projections = Some(HashMap::new()); + } + + let mut db_lookup_indices: Vec = Vec::new(); + if !keys.is_empty() { + if use_mmap_segment { + let mmap_lookup_started = std::time::Instant::now(); + child_cert_cache_mmap_lookup_publication_points = 1; + child_cert_cache_mmap_lookup_entries = keys.len() as u64; + match cache + .store + .get_child_certificate_cache_projections_mmap_segment( + publication_point.manifest_rsync_uri(), + &keys, + ) { + Ok(Some(lookup)) => { + child_cert_cache_mmap_lookup_hits = lookup.hits as u64; + child_cert_cache_mmap_lookup_misses = lookup.misses as u64; + child_cert_cache_mmap_lookup_file_bytes = lookup.file_bytes; + for (idx, ((uri, _key), projection)) in uris + .iter() + .zip(keys.iter()) + .zip(lookup.projections.into_iter()) + .enumerate() + { + if let Some(projection) = projection { + child_cache_candidates.insert( + uri.clone(), + ChildCertificateCacheCandidate { + projection: Some(projection), + }, + ); + } else { + db_lookup_indices.push(idx); + } + } + } + Ok(None) => { + child_cert_cache_mmap_lookup_missing_segments = 1; + db_lookup_indices.extend(0..keys.len()); + } + Err(_) => { + child_cert_cache_mmap_lookup_errors = + child_cert_cache_mmap_lookup_errors.saturating_add(keys.len() as u64); + db_lookup_indices.extend(0..keys.len()); + } + } + child_cert_cache_mmap_lookup_nanos = mmap_lookup_started + .elapsed() + .as_nanos() + .min(u128::from(u64::MAX)) + as u64; + } else { + db_lookup_indices.extend(0..keys.len()); + } + + if !db_lookup_indices.is_empty() { + let batch_lookup_started = std::time::Instant::now(); + child_cert_cache_batch_lookup_publication_points = 1; + child_cert_cache_batch_lookup_entries = db_lookup_indices.len() as u64; + let db_keys = db_lookup_indices + .iter() + .map(|idx| keys[*idx].clone()) + .collect::>(); + match cache + .store + .get_child_certificate_cache_projections_batch(&db_keys) + { + Ok(projections) => { + for (idx, projection) in db_lookup_indices + .iter() + .copied() + .zip(projections.into_iter()) + { + if let Some(projection) = projection { + remember_child_certificate_cache_dirty_projection( + &mut child_cache_segment_dirty_projections, + &projection, + ); + child_cache_candidates.insert( + uris[idx].clone(), + ChildCertificateCacheCandidate { + projection: Some(projection), + }, + ); + } + } + } + Err(_) => { + child_cert_cache_batch_lookup_errors = child_cert_cache_batch_lookup_errors + .saturating_add(child_cert_cache_batch_lookup_entries); + } + } + child_cert_cache_batch_lookup_nanos = child_cert_cache_batch_lookup_nanos + .saturating_add( + batch_lookup_started + .elapsed() + .as_nanos() + .min(u128::from(u64::MAX)) as u64, + ); + } + } + } + + fn bucketize(v: usize) -> u8 { + if v <= 10 { + 0 + } else if v <= 100 { + 1 + } else { + 2 + } + } + + fn ip_item_count(ip: Option<&crate::data_model::rc::IpResourceSet>) -> usize { + let Some(ip) = ip else { return 0 }; + ip.families + .iter() + .map(|f| match &f.choice { + crate::data_model::rc::IpAddressChoice::Inherit => 0usize, + crate::data_model::rc::IpAddressChoice::AddressesOrRanges(items) => items.len(), + }) + .sum() + } + + fn as_item_count(asr: Option<&crate::data_model::rc::AsResourceSet>) -> usize { + let Some(asr) = asr else { return 0 }; + let mut n = 0usize; + if let Some(c) = asr.asnum.as_ref() { + if let crate::data_model::rc::AsIdentifierChoice::AsIdsOrRanges(items) = c { + n = n.saturating_add(items.len()); + } + } + if let Some(c) = asr.rdi.as_ref() { + if let crate::data_model::rc::AsIdentifierChoice::AsIdsOrRanges(items) = c { + n = n.saturating_add(items.len()); + } + } + n + } + + let mut child_cert_crl_gate_set = ChildCertificateCacheCrlGateSet::default(); + + for f in locked_files { + if !f.rsync_uri.ends_with(".cer") { + continue; + } + cer_seen = cer_seen.saturating_add(1); + let child_cert_sha256_hex = sha256_hex_from_32(&f.sha256); + + if cache_context.is_some() { + let lookup_started = std::time::Instant::now(); + child_cert_cache_lookup = child_cert_cache_lookup.saturating_add(1); + let cache_lookup_failed = child_cert_cache_batch_lookup_errors > 0 + && !child_cache_candidates.contains_key(&f.rsync_uri); + if cache_lookup_failed { + child_cert_cache_load_error = child_cert_cache_load_error.saturating_add(1); + } else { + match child_cache_candidates + .get(&f.rsync_uri) + .and_then(|candidate| candidate.projection.as_ref()) + { + Some(projection) => { + let time_gate_ok = pack_time_window_contains( + &projection.effective_not_before, + &projection.effective_until, + validation_time, + ); + if !time_gate_ok { + child_cert_cache_miss_time_gate = + child_cert_cache_miss_time_gate.saturating_add(1); + } else { + let (crl_gate, gate_reused) = child_cert_crl_gate_set.evaluate( + projection, + &mut crl_cache, + issuer_ca_der, + validation_time, + ); + if gate_reused { + child_cert_cache_crl_gate_reused = + child_cert_cache_crl_gate_reused.saturating_add(1); + } + let mut crl_gate_allows_reuse = false; + match crl_gate { + ChildCertificateCacheCrlGate::Unchanged => { + crl_gate_allows_reuse = true; + } + ChildCertificateCacheCrlGate::ChangedValid => { + match ensure_issuer_crl_verified( + &projection.issuer_crl_uri, + &mut crl_cache, + issuer_ca_der, + ) { + Ok(verified_crl) => { + if verified_crl + .revoked_serials + .contains(&projection.child_cert_serial) + { + child_cert_cache_miss_revoked = + child_cert_cache_miss_revoked.saturating_add(1); + } else { + child_cert_cache_crl_recheck_hit = + child_cert_cache_crl_recheck_hit + .saturating_add(1); + crl_gate_allows_reuse = true; + } + } + Err(_) => { + child_cert_cache_miss_crl_invalid = + child_cert_cache_miss_crl_invalid.saturating_add(1); + } + } + } + ChildCertificateCacheCrlGate::Expired => { + child_cert_cache_miss_crl_expired = + child_cert_cache_miss_crl_expired.saturating_add(1); + } + ChildCertificateCacheCrlGate::Invalid => { + child_cert_cache_miss_crl_invalid = + child_cert_cache_miss_crl_invalid.saturating_add(1); + } + ChildCertificateCacheCrlGate::Missing => { + child_cert_cache_miss_crl_missing = + child_cert_cache_miss_crl_missing.saturating_add(1); + } + } + + if crl_gate_allows_reuse { + match &projection.payload { + ChildCertificateCachePayload::ChildCa { + child_manifest_rsync_uri, + child_ski, + child_rsync_base_uri, + child_publication_point_rsync_uri, + child_rrdp_notification_uri, + child_effective_ip_resources, + child_effective_as_resources, + .. + } => { + child_cert_cache_lookup_nanos = + child_cert_cache_lookup_nanos.saturating_add( + lookup_started + .elapsed() + .as_nanos() + .min(u128::from(u64::MAX)) + as u64, + ); + out.push(DiscoveredChildCaInstance { + handle: CaInstanceHandle { + depth: 0, + tal_id: issuer.tal_id.clone(), + parent_manifest_rsync_uri: Some( + issuer.manifest_rsync_uri.clone(), + ), + ca_certificate: CaCertificateRef::repo_bytes( + child_cert_sha256_hex.clone(), + ), + ca_certificate_rsync_uri: Some(f.rsync_uri.clone()), + effective_ip_resources: + child_effective_ip_resources.clone(), + effective_as_resources: + child_effective_as_resources.clone(), + rsync_base_uri: child_rsync_base_uri.clone(), + manifest_rsync_uri: child_manifest_rsync_uri + .clone(), + publication_point_rsync_uri: + child_publication_point_rsync_uri.clone(), + rrdp_notification_uri: child_rrdp_notification_uri + .clone(), + }, + discovered_from: crate::audit::DiscoveredFrom { + parent_manifest_rsync_uri: issuer + .manifest_rsync_uri + .clone(), + child_ca_certificate_rsync_uri: f.rsync_uri.clone(), + child_ca_certificate_sha256_hex: + child_cert_sha256_hex.clone(), + }, + child_entry_projection: Some( + DiscoveredChildEntryProjection { + child_ski: child_ski.clone(), + }, + ), + }); + ca_ok = ca_ok.saturating_add(1); + child_cert_cache_hit = + child_cert_cache_hit.saturating_add(1); + child_cert_cache_hit_ca = + child_cert_cache_hit_ca.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: child_cert_sha256_hex.clone(), + kind: AuditObjectKind::Certificate, + result: AuditObjectResult::Ok, + detail: Some( + "restored subordinate CA discovery from child certificate validation cache" + .to_string(), + ), + }); + continue; + } + ChildCertificateCachePayload::Router { + router_keys: cached_keys, + } => { + for cached_key in cached_keys { + router_keys.push(RouterKeyPayload { + as_id: cached_key.as_id, + ski: cached_key.ski.clone(), + spki_der: cached_key.spki_der.clone(), + source_object_uri: f.rsync_uri.clone(), + source_object_hash: child_cert_sha256_hex.clone(), + source_ee_cert_hash: child_cert_sha256_hex.clone(), + item_effective_until: cached_key + .item_effective_until + .clone(), + }); + } + router_ok = router_ok.saturating_add(1); + child_cert_cache_hit = + child_cert_cache_hit.saturating_add(1); + child_cert_cache_hit_router = + child_cert_cache_hit_router.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: child_cert_sha256_hex.clone(), + kind: AuditObjectKind::RouterCertificate, + result: AuditObjectResult::Ok, + detail: Some( + "restored BGPsec router certificate from child certificate validation cache" + .to_string(), + ), + }); + child_cert_cache_lookup_nanos = + child_cert_cache_lookup_nanos.saturating_add( + lookup_started + .elapsed() + .as_nanos() + .min(u128::from(u64::MAX)) + as u64, + ); + continue; + } + } + } + } + } + None => { + child_cert_cache_miss_not_found = + child_cert_cache_miss_not_found.saturating_add(1); + } + } + } + child_cert_cache_lookup_nanos = child_cert_cache_lookup_nanos.saturating_add( + lookup_started + .elapsed() + .as_nanos() + .min(u128::from(u64::MAX)) as u64, + ); + } + + let child_der = load_child_certificate_der_for_discovery( + f, + &mut child_cert_der_load_fresh_nanos, + &mut child_cert_der_load_fresh_count, + )?; + + let tdecode = std::time::Instant::now(); + let child_cert = match crate::data_model::rc::ResourceCertificate::decode_der(child_der) { + Ok(v) => v, + Err(e) => { + ca_error = ca_error.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Certificate, + result: AuditObjectResult::Error, + detail: Some(format!("child certificate decode failed: {e}")), + }); + continue; + } + }; + child_decode_nanos = child_decode_nanos + .saturating_add(tdecode.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); + + let t0 = std::time::Instant::now(); + let issuer_crl_uri = match select_issuer_crl_uri_for_child(&child_cert, &crl_cache) { + Ok(v) => v.to_string(), + Err(e) => { + crl_select_error = crl_select_error.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Certificate, + result: AuditObjectResult::Error, + detail: Some(format!( + "cannot select issuer CRL for child certificate: {e}" + )), + }); + continue; + } + }; + select_crl_nanos = select_crl_nanos + .saturating_add(t0.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); + + let t1 = std::time::Instant::now(); + let Some(issuer_ca_ref) = issuer_ca.as_ref() else { + ca_error = ca_error.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Certificate, + result: AuditObjectResult::Error, + detail: Some( + issuer_ca_decode_error + .clone() + .unwrap_or_else(|| "issuer CA decode failed".to_string()), + ), + }); + continue; + }; + let Some(issuer_spki_ref) = issuer_spki.as_ref() else { + ca_error = ca_error.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Certificate, + result: AuditObjectResult::Error, + detail: Some( + issuer_spki_error + .clone() + .unwrap_or_else(|| "issuer SubjectPublicKeyInfo unavailable".to_string()), + ), + }); + continue; + }; + let validated = match validate_subordinate_ca_cert_with_cached_issuer( + child_der, + child_cert, + issuer_ca_der, + issuer_ca_ref, + issuer_spki_ref, + issuer_crl_uri.as_str(), + &mut crl_cache, + issuer.ca_certificate_rsync_uri.as_deref(), + issuer.effective_ip_resources.as_ref(), + issuer.effective_as_resources.as_ref(), + &issuer_resources_index, + validation_time, + policy.resource_validation_mode, + ) { + Ok(v) => v, + Err(CaPathError::ChildNotCa) => { + let tr = std::time::Instant::now(); + let router_result = match ensure_issuer_crl_verified( + issuer_crl_uri.as_str(), + &mut crl_cache, + issuer_ca_der, + ) { + Ok(verified_crl) => { + BgpsecRouterCertificate::validate_path_with_prevalidated_issuer( + child_der, + issuer_ca_ref, + issuer_spki_ref, + &verified_crl.crl, + &verified_crl.revoked_serials, + issuer.ca_certificate_rsync_uri.as_deref(), + Some(issuer_crl_uri.as_str()), + validation_time, + ) + } + Err(err) => { + validate_router_nanos = validate_router_nanos.saturating_add( + tr.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64, + ); + router_error = router_error.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::RouterCertificate, + result: AuditObjectResult::Error, + detail: Some(format!( + "router certificate issuer CRL validation failed: {err}" + )), + }); + continue; + } + }; + validate_router_nanos = validate_router_nanos + .saturating_add(tr.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); + + match router_result { + Ok(router) => { + if let Some(ta_constraints) = policy.ta_constraints.for_tal(&issuer.tal_id) + { + if let Err(error) = + ta_constraints.validate_ee_certificate(&router.resource_cert) + { + router_error = router_error.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::RouterCertificate, + result: AuditObjectResult::Error, + detail: Some(format!( + "router certificate violates TA constraints: {error}" + )), + }); + continue; + } + } + let router_asns = match router_asns_for_resource_mode( + &router.asns, + issuer.effective_as_resources.as_ref(), + policy.resource_validation_mode, + ) { + Ok(v) => v, + Err(detail) => { + router_error = router_error.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::RouterCertificate, + result: AuditObjectResult::Error, + detail: Some(format!( + "router certificate AS resource validation failed: {detail}" + )), + }); + continue; + } + }; + router_ok = router_ok.saturating_add(1); + let source_object_hash = sha256_hex_from_32(&f.sha256); + let item_effective_until = PackTime::from_utc_offset_datetime( + router.resource_cert.tbs.validity_not_after, + ); + for as_id in &router_asns { + router_keys.push(RouterKeyPayload { + as_id: *as_id, + ski: router.subject_key_identifier.clone(), + spki_der: router.spki_der.clone(), + source_object_uri: f.rsync_uri.clone(), + source_object_hash: source_object_hash.clone(), + source_ee_cert_hash: source_object_hash.clone(), + item_effective_until: item_effective_until.clone(), + }); + } + if let Some(cache) = cache_context { + let write_started = std::time::Instant::now(); + if let Ok(verified_crl) = ensure_issuer_crl_verified( + issuer_crl_uri.as_str(), + &mut crl_cache, + issuer_ca_der, + ) { + let (effective_not_before, effective_until) = + child_certificate_cache_certificate_window( + router.resource_cert.tbs.validity_not_before, + router.resource_cert.tbs.validity_not_after, + issuer_ca_ref.tbs.validity_not_before, + issuer_ca_ref.tbs.validity_not_after, + ); + let cache_key_sha256_hex = child_certificate_cache_key_sha256_hex( + &f.rsync_uri, + &f.sha256, + &cache.issuer_ca_sha256, + &cache.ca_validation_context_digest, + &cache.policy_fingerprint, + ); + let projection = ChildCertificateCacheProjection { + schema_version: CHILD_CERTIFICATE_CACHE_SCHEMA_VERSION, + algorithm_version: CHILD_CERTIFICATE_CACHE_ALGORITHM_VERSION, + cache_key_sha256_hex, + child_cert_uri: f.rsync_uri.clone(), + child_cert_sha256_hex: child_cert_sha256_hex.clone(), + child_cert_serial: BigUnsigned::from_biguint( + &router.resource_cert.tbs.serial_number, + ) + .bytes_be, + issuer_ca_sha256_hex: sha256_hex_from_32( + &cache.issuer_ca_sha256, + ), + issuer_crl_uri: issuer_crl_uri.clone(), + issuer_crl_sha256_hex: verified_crl.sha256_hex.clone(), + ca_validation_context_digest: cache + .ca_validation_context_digest, + validation_policy_fingerprint: cache.policy_fingerprint, + effective_not_before, + effective_until, + payload: ChildCertificateCachePayload::Router { + router_keys: router_asns + .iter() + .map(|as_id| ChildCertificateCacheRouterKeyProjection { + as_id: *as_id, + ski: router.subject_key_identifier.clone(), + spki_der: router.spki_der.clone(), + item_effective_until: item_effective_until.clone(), + }) + .collect(), + }, + }; + if cache + .store + .put_child_certificate_cache_projection(&projection) + .is_ok() + { + remember_child_certificate_cache_dirty_projection( + &mut child_cache_segment_dirty_projections, + &projection, + ); + child_cert_cache_write_ok = + child_cert_cache_write_ok.saturating_add(1); + } else { + child_cert_cache_write_error = + child_cert_cache_write_error.saturating_add(1); + } + } else { + child_cert_cache_write_error = + child_cert_cache_write_error.saturating_add(1); + } + child_cert_cache_write_nanos = child_cert_cache_write_nanos + .saturating_add( + write_started.elapsed().as_nanos().min(u128::from(u64::MAX)) + as u64, + ); + } + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::RouterCertificate, + result: AuditObjectResult::Ok, + detail: Some( + "validated BGPsec router certificate (RFC 8209); no child CA instance enqueued" + .to_string(), + ), + }); + } + Err(err) if is_non_router_certificate(&err) => { + ca_skipped_not_ca = ca_skipped_not_ca.saturating_add(1); + router_skipped_non_router = router_skipped_non_router.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Certificate, + result: AuditObjectResult::Skipped, + detail: Some( + "skipped: not a CA resource certificate or BGPsec router certificate" + .to_string(), + ), + }); + } + Err(err) => { + router_error = router_error.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::RouterCertificate, + result: AuditObjectResult::Error, + detail: Some(format!("router certificate validation failed: {err}")), + }); + } + } + continue; + } + Err(e) => { + ca_error = ca_error.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Certificate, + result: AuditObjectResult::Error, + detail: Some(format!("child CA validation failed: {e}")), + }); + continue; + } + }; + validate_sub_ca_nanos = validate_sub_ca_nanos + .saturating_add(t1.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); + + if !validated.resource_warnings.is_empty() { + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Certificate, + result: AuditObjectResult::Ok, + detail: Some(format!( + "resource validation warning ({:?}): {}", + policy.resource_validation_mode, + validated.resource_warnings.summary() + )), + }); + } + + let eff_ip_items = ip_item_count(validated.effective_ip_resources.as_ref()); + match bucketize(eff_ip_items) { + 0 => eff_ip_items_bucket_le_10 = eff_ip_items_bucket_le_10.saturating_add(1), + 1 => eff_ip_items_bucket_le_100 = eff_ip_items_bucket_le_100.saturating_add(1), + _ => eff_ip_items_bucket_gt_100 = eff_ip_items_bucket_gt_100.saturating_add(1), + } + let eff_as_items = as_item_count(validated.effective_as_resources.as_ref()); + match bucketize(eff_as_items) { + 0 => eff_as_items_bucket_le_10 = eff_as_items_bucket_le_10.saturating_add(1), + 1 => eff_as_items_bucket_le_100 = eff_as_items_bucket_le_100.saturating_add(1), + _ => eff_as_items_bucket_gt_100 = eff_as_items_bucket_gt_100.saturating_add(1), + } + + let t2 = std::time::Instant::now(); + let uris = match ca_instance_uris_from_ca_certificate(&validated.child_ca) { + Ok(v) => v, + Err(e) => { + uri_discovery_error = uri_discovery_error.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Certificate, + result: AuditObjectResult::Error, + detail: Some(format!("CA instance URI discovery failed: {e}")), + }); + continue; + } + }; + uri_discovery_nanos = uri_discovery_nanos + .saturating_add(t2.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); + + let t3 = std::time::Instant::now(); + let child_rsync_base_uri = uris.rsync_base_uri.clone(); + let child_manifest_rsync_uri = uris.manifest_rsync_uri.clone(); + let child_publication_point_rsync_uri = uris.publication_point_rsync_uri.clone(); + let child_rrdp_notification_uri = uris.rrdp_notification_uri.clone(); + out.push(DiscoveredChildCaInstance { + handle: CaInstanceHandle { + depth: 0, + tal_id: issuer.tal_id.clone(), + parent_manifest_rsync_uri: Some(issuer.manifest_rsync_uri.clone()), + ca_certificate: CaCertificateRef::inline_der(child_der.to_vec()), + ca_certificate_rsync_uri: Some(f.rsync_uri.clone()), + effective_ip_resources: validated.effective_ip_resources.clone(), + effective_as_resources: validated.effective_as_resources.clone(), + rsync_base_uri: child_rsync_base_uri.clone(), + manifest_rsync_uri: child_manifest_rsync_uri.clone(), + publication_point_rsync_uri: child_publication_point_rsync_uri.clone(), + rrdp_notification_uri: child_rrdp_notification_uri.clone(), + }, + discovered_from: crate::audit::DiscoveredFrom { + parent_manifest_rsync_uri: issuer.manifest_rsync_uri.clone(), + child_ca_certificate_rsync_uri: f.rsync_uri.clone(), + child_ca_certificate_sha256_hex: child_cert_sha256_hex.clone(), + }, + child_entry_projection: validated + .child_ca + .tbs + .extensions + .subject_key_identifier + .as_ref() + .map(|child_ski| DiscoveredChildEntryProjection { + child_ski: hex::encode(child_ski), + }), + }); + enqueue_nanos = + enqueue_nanos.saturating_add(t3.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64); + + if let Some(cache) = cache_context { + let write_started = std::time::Instant::now(); + if let Ok(verified_crl) = + ensure_issuer_crl_verified(issuer_crl_uri.as_str(), &mut crl_cache, issuer_ca_der) + { + if let Some(child_ski) = validated + .child_ca + .tbs + .extensions + .subject_key_identifier + .as_ref() + { + let (effective_not_before, effective_until) = + child_certificate_cache_certificate_window( + validated.child_ca.tbs.validity_not_before, + validated.child_ca.tbs.validity_not_after, + issuer_ca_ref.tbs.validity_not_before, + issuer_ca_ref.tbs.validity_not_after, + ); + let cache_key_sha256_hex = child_certificate_cache_key_sha256_hex( + &f.rsync_uri, + &f.sha256, + &cache.issuer_ca_sha256, + &cache.ca_validation_context_digest, + &cache.policy_fingerprint, + ); + let projection = ChildCertificateCacheProjection { + schema_version: CHILD_CERTIFICATE_CACHE_SCHEMA_VERSION, + algorithm_version: CHILD_CERTIFICATE_CACHE_ALGORITHM_VERSION, + cache_key_sha256_hex, + child_cert_uri: f.rsync_uri.clone(), + child_cert_sha256_hex: child_cert_sha256_hex.clone(), + child_cert_serial: BigUnsigned::from_biguint( + &validated.child_ca.tbs.serial_number, + ) + .bytes_be, + issuer_ca_sha256_hex: sha256_hex_from_32(&cache.issuer_ca_sha256), + issuer_crl_uri: issuer_crl_uri.clone(), + issuer_crl_sha256_hex: verified_crl.sha256_hex.clone(), + ca_validation_context_digest: cache.ca_validation_context_digest, + validation_policy_fingerprint: cache.policy_fingerprint, + effective_not_before, + effective_until, + payload: ChildCertificateCachePayload::ChildCa { + child_manifest_rsync_uri, + child_ski: hex::encode(child_ski), + child_rsync_base_uri, + child_publication_point_rsync_uri, + child_rrdp_notification_uri, + child_effective_ip_resources: validated.effective_ip_resources.clone(), + child_effective_as_resources: validated.effective_as_resources.clone(), + }, + }; + if cache + .store + .put_child_certificate_cache_projection(&projection) + .is_ok() + { + remember_child_certificate_cache_dirty_projection( + &mut child_cache_segment_dirty_projections, + &projection, + ); + child_cert_cache_write_ok = child_cert_cache_write_ok.saturating_add(1); + } else { + child_cert_cache_write_error = + child_cert_cache_write_error.saturating_add(1); + } + } else { + child_cert_cache_write_error = child_cert_cache_write_error.saturating_add(1); + } + } else { + child_cert_cache_write_error = child_cert_cache_write_error.saturating_add(1); + } + child_cert_cache_write_nanos = + child_cert_cache_write_nanos.saturating_add( + write_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64, + ); + } + + ca_ok = ca_ok.saturating_add(1); + audits.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: child_cert_sha256_hex.clone(), + kind: AuditObjectKind::Certificate, + result: AuditObjectResult::Ok, + detail: Some("validated subordinate CA certificate; enqueued CA instance".to_string()), + }); + } + + if let (Some(cache), Some(dirty_projections)) = + (cache_context, child_cache_segment_dirty_projections.take()) + { + if !dirty_projections.is_empty() { + let write_started = std::time::Instant::now(); + let projections = dirty_projections.into_values().collect::>(); + child_cert_cache_mmap_write_entries = projections.len() as u64; + match cache + .store + .write_child_certificate_cache_mmap_segment_overlay( + publication_point.manifest_rsync_uri(), + &child_cache_segment_keys, + &projections, + ) { + Ok(stats) => { + child_cert_cache_mmap_write_file_bytes = stats.file_bytes; + } + Err(_) => { + child_cert_cache_mmap_write_errors = + child_cert_cache_mmap_write_errors.saturating_add(1); + } + } + child_cert_cache_mmap_write_nanos = + write_started.elapsed().as_nanos().min(u128::from(u64::MAX)) as u64; + } + } + + if let Some(t) = timing { + t.record_count("child_cer_seen", cer_seen); + t.record_count("child_ca_ok", ca_ok); + t.record_count("child_ca_error", ca_error); + t.record_count("child_ca_skipped_not_ca", ca_skipped_not_ca); + t.record_count("child_router_ok", router_ok); + t.record_count("child_router_error", router_error); + t.record_count("child_router_skipped_non_router", router_skipped_non_router); + t.record_count("child_crl_select_error", crl_select_error); + t.record_count("child_uri_discovery_error", uri_discovery_error); + t.record_count("child_certificate_cache_lookup", child_cert_cache_lookup); + t.record_count("child_certificate_cache_hit", child_cert_cache_hit); + t.record_count("child_certificate_cache_hit_ca", child_cert_cache_hit_ca); + t.record_count( + "child_certificate_cache_hit_router", + child_cert_cache_hit_router, + ); + t.record_count( + "child_certificate_cache_miss_not_found", + child_cert_cache_miss_not_found, + ); + t.record_count( + "child_certificate_cache_miss_time_gate", + child_cert_cache_miss_time_gate, + ); + t.record_count( + "child_certificate_cache_miss_crl_missing", + child_cert_cache_miss_crl_missing, + ); + t.record_count( + "child_certificate_cache_miss_crl_invalid", + child_cert_cache_miss_crl_invalid, + ); + t.record_count( + "child_certificate_cache_miss_crl_expired", + child_cert_cache_miss_crl_expired, + ); + t.record_count( + "child_certificate_cache_miss_revoked", + child_cert_cache_miss_revoked, + ); + t.record_count( + "child_certificate_cache_crl_recheck_hit", + child_cert_cache_crl_recheck_hit, + ); + t.record_count( + "child_certificate_cache_crl_gate_reused", + child_cert_cache_crl_gate_reused, + ); + t.record_count( + "child_certificate_cache_load_error", + child_cert_cache_load_error, + ); + t.record_count( + "child_certificate_cache_write_ok", + child_cert_cache_write_ok, + ); + t.record_count( + "child_certificate_cache_write_error", + child_cert_cache_write_error, + ); + t.record_count( + "child_certificate_cache_batch_lookup_publication_points", + child_cert_cache_batch_lookup_publication_points, + ); + t.record_count( + "child_certificate_cache_batch_lookup_entries", + child_cert_cache_batch_lookup_entries, + ); + t.record_count( + "child_certificate_cache_batch_lookup_errors", + child_cert_cache_batch_lookup_errors, + ); + t.record_count( + "child_certificate_cache_mmap_lookup_publication_points", + child_cert_cache_mmap_lookup_publication_points, + ); + t.record_count( + "child_certificate_cache_mmap_lookup_entries", + child_cert_cache_mmap_lookup_entries, + ); + t.record_count( + "child_certificate_cache_mmap_lookup_hits", + child_cert_cache_mmap_lookup_hits, + ); + t.record_count( + "child_certificate_cache_mmap_lookup_misses", + child_cert_cache_mmap_lookup_misses, + ); + t.record_count( + "child_certificate_cache_mmap_lookup_missing_segments", + child_cert_cache_mmap_lookup_missing_segments, + ); + t.record_count( + "child_certificate_cache_mmap_lookup_errors", + child_cert_cache_mmap_lookup_errors, + ); + t.record_count( + "child_certificate_cache_mmap_lookup_file_bytes", + child_cert_cache_mmap_lookup_file_bytes, + ); + t.record_count( + "child_certificate_cache_mmap_write_entries", + child_cert_cache_mmap_write_entries, + ); + t.record_count( + "child_certificate_cache_mmap_write_errors", + child_cert_cache_mmap_write_errors, + ); + t.record_count( + "child_certificate_cache_mmap_write_file_bytes", + child_cert_cache_mmap_write_file_bytes, + ); + t.record_count( + "child_certificate_der_load_cache_hit_count", + child_cert_der_load_cache_hit_count, + ); + t.record_count( + "child_certificate_der_load_fresh_count", + child_cert_der_load_fresh_count, + ); + + t.record_count("child_effective_ip_items_le_10", eff_ip_items_bucket_le_10); + t.record_count( + "child_effective_ip_items_le_100", + eff_ip_items_bucket_le_100, + ); + t.record_count( + "child_effective_ip_items_gt_100", + eff_ip_items_bucket_gt_100, + ); + t.record_count("child_effective_as_items_le_10", eff_as_items_bucket_le_10); + t.record_count( + "child_effective_as_items_le_100", + eff_as_items_bucket_le_100, + ); + t.record_count( + "child_effective_as_items_gt_100", + eff_as_items_bucket_gt_100, + ); + + t.record_phase_nanos("child_select_issuer_crl_total", select_crl_nanos); + t.record_phase_nanos("child_decode_certificate_total", child_decode_nanos); + t.record_phase_nanos("child_validate_subordinate_total", validate_sub_ca_nanos); + t.record_phase_nanos( + "child_validate_router_certificate_total", + validate_router_nanos, + ); + t.record_phase_nanos("child_ca_instance_uri_discovery_total", uri_discovery_nanos); + t.record_phase_nanos("child_enqueue_total", enqueue_nanos); + t.record_phase_nanos( + "child_certificate_cache_lookup_total", + child_cert_cache_lookup_nanos, + ); + t.record_phase_nanos( + "child_certificate_cache_write_total", + child_cert_cache_write_nanos, + ); + t.record_phase_nanos( + "child_certificate_cache_batch_lookup_total", + child_cert_cache_batch_lookup_nanos, + ); + t.record_phase_nanos( + "child_certificate_cache_mmap_lookup_total", + child_cert_cache_mmap_lookup_nanos, + ); + t.record_phase_nanos( + "child_certificate_cache_mmap_write_total", + child_cert_cache_mmap_write_nanos, + ); + t.record_phase_nanos( + "child_certificate_der_load_cache_hit_total", + child_cert_der_load_cache_hit_nanos, + ); + t.record_phase_nanos( + "child_certificate_der_load_fresh_total", + child_cert_der_load_fresh_nanos, + ); + t.record_phase_nanos( + "child_certificate_der_load_total", + child_cert_der_load_cache_hit_nanos.saturating_add(child_cert_der_load_fresh_nanos), + ); + } + + Ok(ChildDiscoveryOutput { + children: out, + audits, + router_keys, + }) +} + +fn is_non_router_certificate(err: &BgpsecRouterCertificatePathError) -> bool { + matches!( + err, + BgpsecRouterCertificatePathError::Decode(BgpsecRouterCertificateDecodeError::Validate( + BgpsecRouterCertificateProfileError::NotEe + | BgpsecRouterCertificateProfileError::MissingExtendedKeyUsage + | BgpsecRouterCertificateProfileError::MissingBgpsecRouterEku + )) + ) +} + +fn router_asns_for_resource_mode( + router_asns: &[u32], + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + mode: ResourceValidationMode, +) -> Result, String> { + let Some(issuer_effective_as) = issuer_effective_as else { + return Err("issuer has no effective AS resources".to_string()); + }; + + match mode { + ResourceValidationMode::Rfc6487 => { + let outside: Vec = router_asns + .iter() + .copied() + .filter(|asn| !as_resource_set_contains_asn(issuer_effective_as, *asn)) + .collect(); + if outside.is_empty() { + Ok(router_asns.to_vec()) + } else { + Err(format!( + "router AS resources are not a subset of issuer effective AS resources: {outside:?}" + )) + } + } + ResourceValidationMode::ValidationUpdate03 => { + let filtered: Vec = router_asns + .iter() + .copied() + .filter(|asn| as_resource_set_contains_asn(issuer_effective_as, *asn)) + .collect(); + if filtered.is_empty() { + Err("router AS resources have empty validated resource set".to_string()) + } else { + Ok(filtered) + } + } + } +} + +fn as_resource_set_contains_asn( + resources: &crate::data_model::rc::AsResourceSet, + asn: u32, +) -> bool { + let Some(choice) = resources.asnum.as_ref() else { + return false; + }; + match choice { + crate::data_model::rc::AsIdentifierChoice::Inherit => false, + crate::data_model::rc::AsIdentifierChoice::AsIdsOrRanges(items) => { + items.iter().any(|item| match item { + crate::data_model::rc::AsIdOrRange::Id(id) => *id == asn, + crate::data_model::rc::AsIdOrRange::Range { min, max } => { + *min <= asn && asn <= *max + } + }) + } + } +} + +fn select_issuer_crl_uri_for_child<'a>( + child: &'a crate::data_model::rc::ResourceCertificate, + crl_cache: &std::collections::HashMap, +) -> Result<&'a str, String> { + if crl_cache.is_empty() { + return Err( + "no CRL available in publication point snapshot (cannot validate certificates) (RFC 9286 §7; RFC 6487 §4.8.6)" + .to_string(), + ); + } + let Some(crldp_uris) = child.tbs.extensions.crl_distribution_points_uris.as_ref() else { + return Err( + "child certificate CRLDistributionPoints missing (RFC 6487 §4.8.6)".to_string(), + ); + }; + + for u in crldp_uris { + let s = u.as_str(); + if crl_cache.contains_key(s) { + return Ok(s); + } + } + + Err(format!( + "CRL referenced by child certificate CRLDistributionPoints not found in publication point snapshot: {} (RFC 6487 §4.8.6; RFC 9286 §4.2.1)", + crldp_uris + .iter() + .map(|u| u.as_str()) + .collect::>() + .join(", ") + )) +} + +fn ensure_issuer_crl_verified<'a>( + crl_rsync_uri: &str, + crl_cache: &'a mut std::collections::HashMap, + issuer_ca_der: &[u8], +) -> Result<&'a VerifiedIssuerCrl, CaPathError> { + let entry = crl_cache + .get_mut(crl_rsync_uri) + .expect("CRL must exist in cache"); + match entry { + CachedIssuerCrl::Ok(v) => Ok(v), + CachedIssuerCrl::Pending { + bytes, + sha256_hex: cached_sha256_hex, + } => { + let der = std::mem::take(bytes); + let crl = crate::data_model::crl::RpkixCrl::decode_der(&der)?; + crl.verify_signature_with_issuer_certificate_der(issuer_ca_der)?; + let sha256_hex = cached_sha256_hex + .take() + .unwrap_or_else(|| crate::audit::sha256_hex(&der)); + + let mut revoked_serials: std::collections::HashSet> = + std::collections::HashSet::with_capacity(crl.revoked_certs.len()); + for rc in &crl.revoked_certs { + revoked_serials.insert(rc.serial_number.bytes_be.clone()); + } + + *entry = CachedIssuerCrl::Ok(VerifiedIssuerCrl { + crl, + revoked_serials, + sha256_hex, + }); + match entry { + CachedIssuerCrl::Ok(v) => Ok(v), + _ => unreachable!(), + } + } + } +} + +fn validate_subordinate_ca_cert_with_cached_issuer( + child_ca_der: &[u8], + child_ca: crate::data_model::rc::ResourceCertificate, + issuer_ca_der: &[u8], + issuer_ca: &crate::data_model::rc::ResourceCertificate, + issuer_spki: &SubjectPublicKeyInfo<'_>, + issuer_crl_rsync_uri: &str, + crl_cache: &mut std::collections::HashMap, + issuer_ca_rsync_uri: Option<&str>, + issuer_effective_ip: Option<&crate::data_model::rc::IpResourceSet>, + issuer_effective_as: Option<&crate::data_model::rc::AsResourceSet>, + issuer_resources_index: &IssuerEffectiveResourcesIndex, + validation_time: time::OffsetDateTime, + resource_validation_mode: crate::policy::ResourceValidationMode, +) -> Result { + let verified_crl = ensure_issuer_crl_verified(issuer_crl_rsync_uri, crl_cache, issuer_ca_der)?; + + validate_subordinate_ca_cert_with_prevalidated_issuer_and_resources( + child_ca_der, + child_ca, + issuer_ca, + issuer_spki, + &verified_crl.crl, + &verified_crl.revoked_serials, + issuer_ca_rsync_uri, + issuer_crl_rsync_uri, + issuer_effective_ip, + issuer_effective_as, + issuer_resources_index, + validation_time, + resource_validation_mode, + ) +} + +#[cfg(test)] +fn select_issuer_crl_from_snapshot<'a>( + child_cert_der: &[u8], + pack: &'a PublicationPointSnapshot, +) -> Result<(&'a str, &'a [u8]), String> { + let child = crate::data_model::rc::ResourceCertificate::decode_der(child_cert_der) + .map_err(|e| format!("child certificate decode failed: {e}"))?; + let Some(crldp_uris) = child.tbs.extensions.crl_distribution_points_uris.as_ref() else { + return Err( + "child certificate CRLDistributionPoints missing (RFC 6487 §4.8.6)".to_string(), + ); + }; + + for u in crldp_uris { + let s = u.as_str(); + if let Some(f) = pack.files.iter().find(|f| f.rsync_uri == s) { + let bytes = f + .bytes() + .map_err(|e| format!("snapshot CRL bytes load failed: {e}"))?; + return Ok((f.rsync_uri.as_str(), bytes)); + } + } + + Err(format!( + "CRL referenced by child certificate CRLDistributionPoints not found in publication point snapshot: {} (RFC 6487 §4.8.6; RFC 9286 §4.2.1)", + crldp_uris + .iter() + .map(|u| u.as_str()) + .collect::>() + .join(", ") + )) +} + +fn kind_from_rsync_uri(uri: &str) -> AuditObjectKind { + if uri.ends_with(".crl") { + AuditObjectKind::Crl + } else if uri.ends_with(".cer") { + AuditObjectKind::Certificate + } else if uri.ends_with(".roa") { + AuditObjectKind::Roa + } else if uri.ends_with(".asa") { + AuditObjectKind::Aspa + } else { + AuditObjectKind::Other + } +} + +fn source_label(source: PublicationPointSource) -> String { + match source { + PublicationPointSource::Fresh => "fresh".to_string(), + PublicationPointSource::PublicationPointCache => "publication_point_cache".to_string(), + PublicationPointSource::VcirCurrentInstance => "vcir_current_instance".to_string(), + PublicationPointSource::FailedFetchNoCache => "failed_fetch_no_cache".to_string(), + } +} + +fn repo_sync_phase_label(phase: crate::sync::repo::RepoSyncPhase) -> &'static str { + match phase { + crate::sync::repo::RepoSyncPhase::RrdpOk => "rrdp_ok", + crate::sync::repo::RepoSyncPhase::RrdpFailedRsyncOk => "rrdp_failed_rsync_ok", + crate::sync::repo::RepoSyncPhase::RsyncOnlyOk => "rsync_only_ok", + crate::sync::repo::RepoSyncPhase::ReplayRrdpOk => "replay_rrdp_ok", + crate::sync::repo::RepoSyncPhase::ReplayRsyncOk => "replay_rsync_ok", + crate::sync::repo::RepoSyncPhase::ReplayNoopRrdp => "replay_noop_rrdp", + crate::sync::repo::RepoSyncPhase::ReplayNoopRsync => "replay_noop_rsync", + } +} + +fn repo_sync_failure_phase_label( + attempted_rrdp: bool, + original_notification_uri: Option<&str>, + effective_notification_uri: Option<&str>, +) -> &'static str { + if attempted_rrdp && original_notification_uri.is_some() && effective_notification_uri.is_some() + { + "rrdp_failed_rsync_failed" + } else if attempted_rrdp + && original_notification_uri.is_some() + && effective_notification_uri.is_none() + { + "rsync_only_failed_after_rrdp_dedup" + } else { + "rsync_only_failed" + } +} + +fn terminal_state_label(source: PublicationPointSource) -> &'static str { + match source { + PublicationPointSource::Fresh => "fresh", + PublicationPointSource::PublicationPointCache => "publication_point_cache", + PublicationPointSource::VcirCurrentInstance => "fallback_current_instance", + PublicationPointSource::FailedFetchNoCache => "failed_no_cache", + } +} + +fn repo_sync_source_label(source: crate::sync::repo::RepoSyncSource) -> &'static str { + match source { + crate::sync::repo::RepoSyncSource::Rrdp => "rrdp", + crate::sync::repo::RepoSyncSource::Rsync => "rsync", + } +} + +fn effective_repo_sync_duration_ms( + elapsed_ms: u64, + runtime_reported_duration_ms: Option, + repo_sync_ok: bool, +) -> u64 { + if repo_sync_ok { + return elapsed_ms; + } + runtime_reported_duration_ms + .map(|runtime_ms| elapsed_ms.max(runtime_ms)) + .unwrap_or(elapsed_ms) +} + +fn kind_from_vcir_artifact_kind(kind: VcirArtifactKind) -> AuditObjectKind { + match kind { + VcirArtifactKind::Mft => AuditObjectKind::Manifest, + VcirArtifactKind::Crl => AuditObjectKind::Crl, + VcirArtifactKind::Cer => AuditObjectKind::Certificate, + VcirArtifactKind::Roa => AuditObjectKind::Roa, + VcirArtifactKind::Aspa => AuditObjectKind::Aspa, + VcirArtifactKind::Gbr | VcirArtifactKind::Tal | VcirArtifactKind::Other => { + AuditObjectKind::Other + } + } +} + +fn audit_result_from_vcir_status(status: VcirArtifactValidationStatus) -> AuditObjectResult { + match status { + VcirArtifactValidationStatus::Accepted => AuditObjectResult::Ok, + VcirArtifactValidationStatus::Rejected => AuditObjectResult::Error, + VcirArtifactValidationStatus::WarningOnly => AuditObjectResult::Skipped, + } +} + +/// Fallback detail for cached artifacts rejected by an earlier run whose cache +/// entry predates reject-reason recording (field added 2026-07-27). +const CACHED_REJECT_REASON_NOT_RECORDED: &str = + "rejected in an earlier validation run (reject reason not recorded in cache)"; + +/// Detail for an audit entry rebuilt from a cached artifact: the real reject +/// reason when the cache recorded one, an explicit fallback for legacy cache +/// entries, and `None` for non-rejected artifacts. +fn audit_detail_from_vcir_status( + status: VcirArtifactValidationStatus, + reject_reason: Option<&str>, +) -> Option { + match status { + VcirArtifactValidationStatus::Rejected => Some( + reject_reason + .map(str::to_string) + .unwrap_or_else(|| CACHED_REJECT_REASON_NOT_RECORDED.to_string()), + ), + VcirArtifactValidationStatus::Accepted | VcirArtifactValidationStatus::WarningOnly => None, + } +} + +fn build_publication_point_audit_from_snapshot( + ca: &CaInstanceHandle, + source: PublicationPointSource, + repo_sync_source: Option<&str>, + repo_sync_phase: Option<&str>, + repo_sync_duration_ms: Option, + repo_sync_error: Option<&str>, + pack: &PublicationPointSnapshot, + runner_warnings: &[Warning], + objects: &crate::validation::objects::ObjectsOutput, + child_audits: &[ObjectAuditEntry], +) -> PublicationPointAudit { + use crate::data_model::crl::RpkixCrl; + use std::collections::HashMap; + + let locked_files = &pack.files; + let mut audit_by_uri: HashMap = HashMap::new(); + for f in locked_files { + audit_by_uri.insert( + f.rsync_uri.clone(), + ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: kind_from_rsync_uri(&f.rsync_uri), + result: AuditObjectResult::Skipped, + detail: Some("skipped: not processed in stage2".to_string()), + }, + ); + } + + for f in locked_files { + if !f.rsync_uri.ends_with(".crl") { + continue; + } + let ok = f + .bytes() + .ok() + .and_then(|bytes| RpkixCrl::decode_der(bytes).ok()) + .is_some(); + audit_by_uri.insert( + f.rsync_uri.clone(), + ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: AuditObjectKind::Crl, + result: if ok { + AuditObjectResult::Ok + } else { + AuditObjectResult::Error + }, + detail: if ok { + None + } else { + Some("CRL decode failed".to_string()) + }, + }, + ); + } + + for e in child_audits { + audit_by_uri.insert(e.rsync_uri.clone(), e.clone()); + } + for e in &objects.audit { + audit_by_uri.insert(e.rsync_uri.clone(), e.clone()); + } + + let mut objects_out: Vec = Vec::with_capacity(pack.files.len() + 1); + objects_out.push(ObjectAuditEntry { + rsync_uri: pack.manifest_rsync_uri.clone(), + sha256_hex: sha256_hex(&pack.manifest_bytes), + kind: AuditObjectKind::Manifest, + result: AuditObjectResult::Ok, + detail: None, + }); + for f in locked_files { + if let Some(e) = audit_by_uri.remove(&f.rsync_uri) { + objects_out.push(e); + } else { + objects_out.push(ObjectAuditEntry { + rsync_uri: f.rsync_uri.clone(), + sha256_hex: sha256_hex_from_32(&f.sha256), + kind: kind_from_rsync_uri(&f.rsync_uri), + result: AuditObjectResult::Skipped, + detail: Some("skipped: no audit entry".to_string()), + }); + } + } + + let mut warnings = Vec::new(); + warnings.extend(runner_warnings.iter().map(AuditWarning::from)); + warnings.extend(objects.warnings.iter().map(AuditWarning::from)); + + PublicationPointAudit { + node_id: None, + parent_node_id: None, + discovered_from: None, + rsync_base_uri: ca.rsync_base_uri.clone(), + manifest_rsync_uri: ca.manifest_rsync_uri.clone(), + publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(), + rrdp_notification_uri: ca.rrdp_notification_uri.clone(), + source: source_label(source), + repo_sync_source: repo_sync_source.map(ToString::to_string), + repo_sync_phase: repo_sync_phase.map(ToString::to_string), + repo_sync_duration_ms, + repo_sync_error: repo_sync_error.map(ToString::to_string), + repo_terminal_state: terminal_state_label(source).to_string(), + this_update_rfc3339_utc: pack.this_update.rfc3339_utc.clone(), + next_update_rfc3339_utc: pack.next_update.rfc3339_utc.clone(), + verified_at_rfc3339_utc: pack.verified_at.rfc3339_utc.clone(), + warnings, + objects: objects_out, + } +} + +fn build_publication_point_audit_from_vcir( + ca: &CaInstanceHandle, + source: PublicationPointSource, + repo_sync_source: Option<&str>, + repo_sync_phase: Option<&str>, + repo_sync_duration_ms: Option, + repo_sync_error: Option<&str>, + vcir: Option<&ValidatedCaInstanceResult>, + pack: Option<&PublicationPointSnapshot>, + runner_warnings: &[Warning], + objects: &crate::validation::objects::ObjectsOutput, + child_audits: &[ObjectAuditEntry], + fresh_failure_audits: &[ObjectAuditEntry], +) -> PublicationPointAudit { + if let Some(pack) = pack { + return build_publication_point_audit_from_snapshot( + ca, + source, + repo_sync_source, + repo_sync_phase, + repo_sync_duration_ms, + repo_sync_error, + pack, + runner_warnings, + objects, + child_audits, + ); + } + + let mut warnings = Vec::new(); + warnings.extend(runner_warnings.iter().map(AuditWarning::from)); + warnings.extend(objects.warnings.iter().map(AuditWarning::from)); + + let Some(vcir) = vcir else { + return PublicationPointAudit { + node_id: None, + parent_node_id: None, + discovered_from: None, + rsync_base_uri: ca.rsync_base_uri.clone(), + manifest_rsync_uri: ca.manifest_rsync_uri.clone(), + publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(), + rrdp_notification_uri: ca.rrdp_notification_uri.clone(), + source: source_label(source), + repo_sync_source: repo_sync_source.map(ToString::to_string), + repo_sync_phase: repo_sync_phase.map(ToString::to_string), + repo_sync_duration_ms, + repo_sync_error: repo_sync_error.map(ToString::to_string), + repo_terminal_state: terminal_state_label(source).to_string(), + this_update_rfc3339_utc: String::new(), + next_update_rfc3339_utc: String::new(), + verified_at_rfc3339_utc: String::new(), + warnings, + objects: fresh_failure_audits.to_vec(), + }; + }; + + if source == PublicationPointSource::FailedFetchNoCache { + let mut objects_out = Vec::with_capacity( + objects.audit.len() + child_audits.len() + fresh_failure_audits.len(), + ); + objects_out.extend(child_audits.iter().cloned()); + objects_out.extend(objects.audit.iter().cloned()); + objects_out.extend(fresh_failure_audits.iter().cloned()); + return PublicationPointAudit { + node_id: None, + parent_node_id: None, + discovered_from: None, + rsync_base_uri: ca.rsync_base_uri.clone(), + manifest_rsync_uri: ca.manifest_rsync_uri.clone(), + publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(), + rrdp_notification_uri: ca.rrdp_notification_uri.clone(), + source: source_label(source), + repo_sync_source: repo_sync_source.map(ToString::to_string), + repo_sync_phase: repo_sync_phase.map(ToString::to_string), + repo_sync_duration_ms, + repo_sync_error: repo_sync_error.map(ToString::to_string), + repo_terminal_state: terminal_state_label(source).to_string(), + this_update_rfc3339_utc: vcir + .validated_manifest_meta + .validated_manifest_this_update + .rfc3339_utc + .clone(), + next_update_rfc3339_utc: vcir + .validated_manifest_meta + .validated_manifest_next_update + .rfc3339_utc + .clone(), + verified_at_rfc3339_utc: vcir.last_successful_validation_time.rfc3339_utc.clone(), + warnings, + objects: objects_out, + }; + } + + let mut audit_by_uri: HashMap = HashMap::new(); + for artifact in &vcir.related_artifacts { + let Some(uri) = artifact.uri.as_ref() else { + continue; + }; + audit_by_uri.insert( + uri.clone(), + ObjectAuditEntry { + rsync_uri: uri.clone(), + sha256_hex: artifact.sha256.clone(), + kind: kind_from_vcir_artifact_kind(artifact.artifact_kind), + result: audit_result_from_vcir_status(artifact.validation_status), + detail: audit_detail_from_vcir_status( + artifact.validation_status, + artifact.reject_reason.as_deref(), + ), + }, + ); + } + for e in child_audits { + audit_by_uri.insert(e.rsync_uri.clone(), e.clone()); + } + for e in &objects.audit { + audit_by_uri.insert(e.rsync_uri.clone(), e.clone()); + } + + let mut ordered_uris: Vec = vcir + .related_artifacts + .iter() + .filter_map(|artifact| artifact.uri.clone()) + .collect(); + ordered_uris.sort(); + ordered_uris.dedup(); + + let mut objects_out: Vec = Vec::new(); + if let Some(entry) = audit_by_uri.remove(&vcir.current_manifest_rsync_uri) { + objects_out.push(entry); + } else { + objects_out.push(ObjectAuditEntry { + rsync_uri: vcir.current_manifest_rsync_uri.clone(), + sha256_hex: vcir + .related_artifacts + .iter() + .find(|artifact| { + artifact.artifact_role == VcirArtifactRole::Manifest + && artifact.uri.as_deref() == Some(vcir.current_manifest_rsync_uri.as_str()) + }) + .map(|artifact| artifact.sha256.clone()) + .unwrap_or_default(), + kind: AuditObjectKind::Manifest, + result: AuditObjectResult::Ok, + detail: None, + }); + } + + for uri in ordered_uris { + if uri == vcir.current_manifest_rsync_uri { + continue; + } + if let Some(entry) = audit_by_uri.remove(&uri) { + objects_out.push(entry); + } + } + + let mut audit_objects = objects_out.clone(); + audit_objects.extend(fresh_failure_audits.iter().cloned()); + + PublicationPointAudit { + node_id: None, + parent_node_id: None, + discovered_from: None, + rsync_base_uri: ca.rsync_base_uri.clone(), + manifest_rsync_uri: ca.manifest_rsync_uri.clone(), + publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(), + rrdp_notification_uri: ca.rrdp_notification_uri.clone(), + source: source_label(source), + repo_sync_source: repo_sync_source.map(ToString::to_string), + repo_sync_phase: repo_sync_phase.map(ToString::to_string), + repo_sync_duration_ms, + repo_sync_error: repo_sync_error.map(ToString::to_string), + repo_terminal_state: terminal_state_label(source).to_string(), + this_update_rfc3339_utc: vcir + .validated_manifest_meta + .validated_manifest_this_update + .rfc3339_utc + .clone(), + next_update_rfc3339_utc: vcir + .validated_manifest_meta + .validated_manifest_next_update + .rfc3339_utc + .clone(), + verified_at_rfc3339_utc: vcir.last_successful_validation_time.rfc3339_utc.clone(), + warnings, + objects: audit_objects, + } +} + +fn build_publication_point_audit_from_publication_point_cache_projection( + ca: &CaInstanceHandle, + source: PublicationPointSource, + repo_sync_source: Option<&str>, + repo_sync_phase: Option<&str>, + repo_sync_duration_ms: Option, + repo_sync_error: Option<&str>, + projection: &PublicationPointCacheProjection, + validation_time: time::OffsetDateTime, + runner_warnings: &[Warning], + objects: &crate::validation::objects::ObjectsOutput, + child_audits: &[ObjectAuditEntry], +) -> PublicationPointAudit { + let mut warnings = Vec::new(); + warnings.extend(runner_warnings.iter().map(AuditWarning::from)); + warnings.extend(objects.warnings.iter().map(AuditWarning::from)); + + let mut audit_by_uri: HashMap = HashMap::new(); + for artifact in &projection.related_objects { + let Some(uri) = artifact.uri.as_ref() else { + continue; + }; + audit_by_uri.insert( + uri.clone(), + ObjectAuditEntry { + rsync_uri: uri.clone(), + sha256_hex: artifact.sha256.clone(), + kind: kind_from_vcir_artifact_kind(artifact.artifact_kind), + result: audit_result_from_vcir_status(artifact.validation_status), + detail: audit_detail_from_vcir_status( + artifact.validation_status, + artifact.reject_reason.as_deref(), + ), + }, + ); + } + for entry in child_audits { + audit_by_uri.insert(entry.rsync_uri.clone(), entry.clone()); + } + for entry in &objects.audit { + audit_by_uri.insert(entry.rsync_uri.clone(), entry.clone()); + } + + let mut ordered_uris: Vec = projection + .related_objects + .iter() + .filter_map(|artifact| artifact.uri.clone()) + .collect(); + ordered_uris.sort(); + ordered_uris.dedup(); + + let mut objects_out: Vec = Vec::with_capacity(ordered_uris.len().max(1)); + if let Some(entry) = audit_by_uri.remove(&projection.manifest_rsync_uri) { + objects_out.push(entry); + } else { + objects_out.push(ObjectAuditEntry { + rsync_uri: projection.manifest_rsync_uri.clone(), + sha256_hex: hex::encode(projection.manifest_sha256), + kind: AuditObjectKind::Manifest, + result: AuditObjectResult::Ok, + detail: None, + }); + } + + for uri in ordered_uris { + if uri == projection.manifest_rsync_uri { + continue; + } + if let Some(entry) = audit_by_uri.remove(&uri) { + objects_out.push(entry); + } + } + + PublicationPointAudit { + node_id: None, + parent_node_id: None, + discovered_from: None, + rsync_base_uri: ca.rsync_base_uri.clone(), + manifest_rsync_uri: ca.manifest_rsync_uri.clone(), + publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(), + rrdp_notification_uri: ca.rrdp_notification_uri.clone(), + source: source_label(source), + repo_sync_source: repo_sync_source.map(ToString::to_string), + repo_sync_phase: repo_sync_phase.map(ToString::to_string), + repo_sync_duration_ms, + repo_sync_error: repo_sync_error.map(ToString::to_string), + repo_terminal_state: terminal_state_label(source).to_string(), + this_update_rfc3339_utc: projection.manifest_this_update.rfc3339_utc.clone(), + next_update_rfc3339_utc: projection.manifest_next_update.rfc3339_utc.clone(), + verified_at_rfc3339_utc: PackTime::from_utc_offset_datetime(validation_time).rfc3339_utc, + warnings, + objects: objects_out, + } +} + +fn parse_snapshot_time_value(pack_time: &PackTime) -> Result { + time::OffsetDateTime::parse( + &pack_time.rfc3339_utc, + &time::format_description::well_known::Rfc3339, + ) + .map_err(|e| format!("invalid RFC3339 time '{}': {e}", pack_time.rfc3339_utc)) +} + +fn empty_objects_output() -> crate::validation::objects::ObjectsOutput { + crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: Vec::new(), + stats: crate::validation::objects::ObjectsStats::default(), + audit: Vec::new(), + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + } +} + +fn reuse_ccr_manifest_projection_from_vcir( + ca: &CaInstanceHandle, + vcir: &ValidatedCaInstanceResult, +) -> Result { + if vcir.ccr_manifest_projection.manifest_rsync_uri != ca.manifest_rsync_uri { + return Err(format!( + "vcir CCR manifest projection URI mismatch: expected {}, got {}", + ca.manifest_rsync_uri, vcir.ccr_manifest_projection.manifest_rsync_uri + )); + } + Ok(vcir.ccr_manifest_projection.clone()) +} + +fn project_current_instance_vcir_on_failed_fetch( + store: &RocksStore, + ca: &CaInstanceHandle, + fresh_err: &ManifestFreshError, + policy: &Policy, + validation_time: time::OffsetDateTime, +) -> Result { + let mut warnings = Vec::new(); + + let Some(vcir) = store + .get_vcir(&ca.manifest_rsync_uri) + .map_err(|e| format!("load VCIR failed: {e}"))? + else { + return Ok(failed_fetch_no_cache_projection( + ca, + fresh_err, + None, + "no latest validated result for current CA instance; no cached output reused", + )); + }; + + if !vcir.audit_summary.failed_fetch_eligible { + return Ok(failed_fetch_no_cache_projection( + ca, + fresh_err, + Some(vcir), + "latest VCIR is not marked failed-fetch eligible; no cached output reused", + )); + } + + let reuse_identity = match store.get_vcir_failed_fetch_reuse_identity(&ca.manifest_rsync_uri) { + Ok(Some(identity)) => identity, + Ok(None) => { + return Ok(failed_fetch_no_cache_projection( + ca, + fresh_err, + Some(vcir), + "latest VCIR reuse identity is missing; no cached output or child reused", + )); + } + Err(error) => { + return Ok(failed_fetch_no_cache_projection( + ca, + fresh_err, + Some(vcir), + &format!( + "latest VCIR reuse identity is invalid ({error}); no cached output or child reused" + ), + )); + } + }; + if !failed_fetch_reuse_identity_matches_current( + &reuse_identity, + &vcir, + ca, + policy, + validation_time, + ) { + return Ok(failed_fetch_no_cache_projection( + ca, + fresh_err, + Some(vcir), + "latest VCIR reuse identity does not match the current CA context or time window; no cached output or child reused", + )); + } + + let ccr_manifest_projection = reuse_ccr_manifest_projection_from_vcir(ca, &vcir)?; + if fresh_err.should_warn_when_current_instance_reused() { + warnings.push( + Warning::new(format!("manifest failed fetch: {fresh_err}")) + .with_rfc_refs(&[RfcRef("RFC 9286 §6.6")]) + .with_context(&ca.manifest_rsync_uri), + ); + } + // Current-instance reuse is fully described by VCIR projections; rebuilding a + // byte-backed snapshot here only duplicates repo-byte I/O and creates warning noise. + let snapshot = None; + let objects = build_objects_output_from_vcir(&vcir, validation_time, &mut warnings); + let (discovered_children, child_audits) = + restore_children_from_vcir(store, ca, &vcir, &mut warnings); + + Ok(VcirReuseProjection { + source: PublicationPointSource::VcirCurrentInstance, + vcir: Some(vcir), + ccr_manifest_projection: Some(ccr_manifest_projection), + snapshot, + objects, + child_audits, + discovered_children, + warnings, + }) +} + +fn failed_fetch_no_cache_projection( + ca: &CaInstanceHandle, + fresh_err: &ManifestFreshError, + vcir: Option, + reason: &str, +) -> VcirReuseProjection { + let warnings = vec![ + Warning::new(format!("manifest failed fetch: {fresh_err}")) + .with_rfc_refs(&[RfcRef("RFC 9286 §6.6")]) + .with_context(&ca.manifest_rsync_uri), + Warning::new(reason) + .with_rfc_refs(&[RfcRef("RFC 9286 §6.6")]) + .with_context(&ca.manifest_rsync_uri), + ]; + VcirReuseProjection { + source: PublicationPointSource::FailedFetchNoCache, + vcir, + ccr_manifest_projection: None, + snapshot: None, + objects: empty_objects_output(), + child_audits: Vec::new(), + discovered_children: Vec::new(), + warnings, + } +} + +fn failed_fetch_reuse_identity_for_fresh_result( + ca: &CaInstanceHandle, + policy: &Policy, + validation_time: time::OffsetDateTime, + effective_until: PackTime, +) -> Result { + let current_ca_sha256 = ca.ca_certificate_sha256_32().ok_or_else(|| { + "current CA certificate hash unavailable for VCIR reuse identity".to_string() + })?; + let identity = VcirFailedFetchReuseIdentity { + current_ca_sha256, + ta_context_digest: ta_context_digest_for_ca(ca), + ca_validation_context_digest: ca_validation_context_digest_for_ca(ca), + policy_fingerprint: publication_point_cache_policy_fingerprint(policy), + effective_not_before: PackTime::from_utc_offset_datetime(validation_time), + effective_until, + }; + identity + .validate_internal() + .map_err(|error| error.to_string())?; + Ok(identity) +} + +fn failed_fetch_reuse_identity_matches_current( + cached: &VcirFailedFetchReuseIdentity, + vcir: &ValidatedCaInstanceResult, + ca: &CaInstanceHandle, + policy: &Policy, + validation_time: time::OffsetDateTime, +) -> bool { + let Some(current_ca_sha256) = ca.ca_certificate_sha256_32() else { + return false; + }; + cached.current_ca_sha256 == current_ca_sha256 + && cached.ta_context_digest == ta_context_digest_for_ca(ca) + && cached.ca_validation_context_digest == ca_validation_context_digest_for_ca(ca) + && cached.policy_fingerprint == publication_point_cache_policy_fingerprint(policy) + && cached.effective_until == vcir.instance_gate.instance_effective_until + && cached.contains_validation_time(validation_time) +} + +#[cfg(test)] +fn reconstruct_snapshot_from_vcir( + store: &RocksStore, + ca: &CaInstanceHandle, + vcir: &ValidatedCaInstanceResult, + warnings: &mut Vec, +) -> Option { + let manifest_artifact = vcir.related_artifacts.iter().find(|artifact| { + artifact.artifact_role == VcirArtifactRole::Manifest + && artifact.uri.as_deref() == Some(ca.manifest_rsync_uri.as_str()) + })?; + let manifest_bytes = match store.get_blob_bytes(&manifest_artifact.sha256) { + Ok(Some(bytes)) => bytes, + Ok(None) => { + warnings.push( + Warning::new("manifest raw bytes missing for VCIR audit reconstruction") + .with_context(&ca.manifest_rsync_uri), + ); + return None; + } + Err(e) => { + warnings.push( + Warning::new(format!( + "manifest raw bytes load failed for VCIR audit reconstruction: {e}" + )) + .with_context(&ca.manifest_rsync_uri), + ); + return None; + } + }; + + let mut seen = HashSet::new(); + let mut files = Vec::new(); + for artifact in &vcir.related_artifacts { + let Some(uri) = artifact.uri.as_ref() else { + continue; + }; + if artifact.artifact_role == VcirArtifactRole::Manifest + || artifact.artifact_role == VcirArtifactRole::IssuerCert + || artifact.artifact_role == VcirArtifactRole::TrustAnchorCert + || artifact.artifact_role == VcirArtifactRole::Tal + { + continue; + } + if !seen.insert(uri.clone()) { + continue; + } + match store.get_blob_bytes(&artifact.sha256) { + Ok(Some(bytes)) => files.push(PackFile::from_bytes_compute_sha256(uri, bytes)), + Ok(None) => warnings.push( + Warning::new("related artifact raw bytes missing for VCIR audit reconstruction") + .with_context(uri), + ), + Err(e) => warnings.push( + Warning::new(format!( + "related artifact raw bytes load failed for VCIR audit reconstruction: {e}" + )) + .with_context(uri), + ), + } + } + + Some(PublicationPointSnapshot { + format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, + publication_point_rsync_uri: ca.publication_point_rsync_uri.clone(), + manifest_rsync_uri: ca.manifest_rsync_uri.clone(), + manifest_number_be: vcir + .validated_manifest_meta + .validated_manifest_number + .clone(), + this_update: vcir + .validated_manifest_meta + .validated_manifest_this_update + .clone(), + next_update: vcir + .validated_manifest_meta + .validated_manifest_next_update + .clone(), + verified_at: vcir.last_successful_validation_time.clone(), + manifest_bytes, + files, + }) +} + +fn audit_kind_for_vcir_output_type(output_type: VcirOutputType) -> AuditObjectKind { + match output_type { + VcirOutputType::Vrp => AuditObjectKind::Roa, + VcirOutputType::Aspa => AuditObjectKind::Aspa, + VcirOutputType::RouterKey => AuditObjectKind::RouterCertificate, + } +} + +fn build_objects_output_from_vcir( + vcir: &ValidatedCaInstanceResult, + validation_time: time::OffsetDateTime, + warnings: &mut Vec, +) -> crate::validation::objects::ObjectsOutput { + let mut output = empty_objects_output(); + let mut audit_by_uri: HashMap = HashMap::new(); + let mut roa_total: HashSet = HashSet::new(); + let mut aspa_total: HashSet = HashSet::new(); + let mut roa_ok: HashSet = HashSet::new(); + let mut aspa_ok: HashSet = HashSet::new(); + + for artifact in &vcir.related_artifacts { + if artifact.artifact_role != VcirArtifactRole::SignedObject { + continue; + } + if let Some(uri) = artifact.uri.as_ref() { + match artifact.artifact_kind { + VcirArtifactKind::Roa => { + roa_total.insert(uri.clone()); + } + VcirArtifactKind::Aspa => { + aspa_total.insert(uri.clone()); + } + _ => {} + } + } + } + + for local in &vcir.local_outputs { + let effective_until = match parse_snapshot_time_value(&local.item_effective_until) { + Ok(v) => v, + Err(e) => { + warnings.push( + Warning::new(format!( + "cached local output has invalid item_effective_until: {e}" + )) + .with_context(&local.source_object_uri), + ); + audit_by_uri.insert( + local.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: local.source_object_uri.clone(), + sha256_hex: local.source_object_hash_hex(), + kind: audit_kind_for_vcir_output_type(local.output_type), + result: AuditObjectResult::Error, + detail: Some( + "cached local output has invalid item_effective_until".to_string(), + ), + }, + ); + continue; + } + }; + if validation_time > effective_until { + audit_by_uri + .entry(local.source_object_uri.clone()) + .or_insert_with(|| ObjectAuditEntry { + rsync_uri: local.source_object_uri.clone(), + sha256_hex: local.source_object_hash_hex(), + kind: audit_kind_for_vcir_output_type(local.output_type), + result: AuditObjectResult::Skipped, + detail: Some("skipped: cached local output expired".to_string()), + }); + continue; + } + + match local.output_type { + VcirOutputType::Vrp => match parse_vcir_vrp_output(local) { + Ok(vrp) => { + roa_ok.insert(local.source_object_uri.clone()); + output.vrps.push(vrp); + audit_by_uri.insert( + local.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: local.source_object_uri.clone(), + sha256_hex: local.source_object_hash_hex(), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Ok, + detail: None, + }, + ); + } + Err(e) => { + warnings.push( + Warning::new(format!("cached ROA local output parse failed: {e}")) + .with_context(&local.source_object_uri), + ); + audit_by_uri.insert( + local.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: local.source_object_uri.clone(), + sha256_hex: local.source_object_hash_hex(), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Error, + detail: Some(format!("cached ROA local output parse failed: {e}")), + }, + ); + } + }, + VcirOutputType::Aspa => match parse_vcir_aspa_output(local) { + Ok(aspa) => { + aspa_ok.insert(local.source_object_uri.clone()); + output.aspas.push(aspa); + audit_by_uri.insert( + local.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: local.source_object_uri.clone(), + sha256_hex: local.source_object_hash_hex(), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Ok, + detail: None, + }, + ); + } + Err(e) => { + warnings.push( + Warning::new(format!("cached ASPA local output parse failed: {e}")) + .with_context(&local.source_object_uri), + ); + audit_by_uri.insert( + local.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: local.source_object_uri.clone(), + sha256_hex: local.source_object_hash_hex(), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Error, + detail: Some(format!("cached ASPA local output parse failed: {e}")), + }, + ); + } + }, + VcirOutputType::RouterKey => match parse_vcir_router_key_output(local) { + Ok(router_key) => { + output.router_keys.push(router_key); + audit_by_uri.insert( + local.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: local.source_object_uri.clone(), + sha256_hex: local.source_object_hash_hex(), + kind: AuditObjectKind::RouterCertificate, + result: AuditObjectResult::Ok, + detail: Some("cached Router Key local output restored".to_string()), + }, + ); + } + Err(e) => { + warnings.push( + Warning::new(format!("cached Router Key local output parse failed: {e}")) + .with_context(&local.source_object_uri), + ); + audit_by_uri.insert( + local.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: local.source_object_uri.clone(), + sha256_hex: local.source_object_hash_hex(), + kind: AuditObjectKind::RouterCertificate, + result: AuditObjectResult::Error, + detail: Some(format!( + "cached Router Key local output parse failed: {e}" + )), + }, + ); + } + }, + } + } + + output.stats.roa_total = roa_total.len(); + output.stats.roa_ok = roa_ok.len(); + output.stats.aspa_total = aspa_total.len(); + output.stats.aspa_ok = aspa_ok.len(); + let mut audit: Vec<_> = audit_by_uri.into_values().collect(); + audit.sort_by(|a, b| a.rsync_uri.cmp(&b.rsync_uri)); + output.audit = audit; + output +} + +fn build_objects_output_from_publication_point_cache_projection( + projection: &PublicationPointCacheProjection, + validation_time: time::OffsetDateTime, + warnings: &mut Vec, +) -> crate::validation::objects::ObjectsOutput { + let mut output = empty_objects_output(); + let mut audit_by_uri: HashMap = HashMap::new(); + let mut roa_total: HashSet = HashSet::new(); + let mut aspa_total: HashSet = HashSet::new(); + let mut roa_ok: HashSet = HashSet::new(); + let mut aspa_ok: HashSet = HashSet::new(); + + for artifact in &projection.related_objects { + if artifact.artifact_role != VcirArtifactRole::SignedObject { + continue; + } + if let Some(uri) = artifact.uri.as_ref() { + match artifact.artifact_kind { + VcirArtifactKind::Roa => { + roa_total.insert(uri.clone()); + } + VcirArtifactKind::Aspa => { + aspa_total.insert(uri.clone()); + } + _ => {} + } + } + } + + for projected in &projection.outputs { + let effective_until = match parse_snapshot_time_value(&projected.item_effective_until) { + Ok(value) => value, + Err(err) => { + warnings.push( + Warning::new(format!( + "publication-point cached local output has invalid item_effective_until: {err}" + )) + .with_context(&projected.source_object_uri), + ); + audit_by_uri.insert( + projected.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: projected.source_object_uri.clone(), + sha256_hex: hex::encode(projected.source_object_hash), + kind: audit_kind_for_vcir_output_type(projected.output_type), + result: AuditObjectResult::Error, + detail: Some( + "publication-point cached local output has invalid item_effective_until" + .to_string(), + ), + }, + ); + continue; + } + }; + if validation_time > effective_until { + audit_by_uri + .entry(projected.source_object_uri.clone()) + .or_insert_with(|| ObjectAuditEntry { + rsync_uri: projected.source_object_uri.clone(), + sha256_hex: hex::encode(projected.source_object_hash), + kind: audit_kind_for_vcir_output_type(projected.output_type), + result: AuditObjectResult::Skipped, + detail: Some( + "skipped: publication-point cached local output expired".to_string(), + ), + }); + continue; + } + + match projected.output_type { + VcirOutputType::Vrp => match parse_publication_point_cache_vrp_output(projected) { + Ok(vrp) => { + roa_ok.insert(projected.source_object_uri.clone()); + output.vrps.push(vrp); + audit_by_uri.insert( + projected.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: projected.source_object_uri.clone(), + sha256_hex: hex::encode(projected.source_object_hash), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Ok, + detail: None, + }, + ); + } + Err(err) => { + warnings.push( + Warning::new(format!( + "publication-point cached ROA local output parse failed: {err}" + )) + .with_context(&projected.source_object_uri), + ); + audit_by_uri.insert( + projected.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: projected.source_object_uri.clone(), + sha256_hex: hex::encode(projected.source_object_hash), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Error, + detail: Some(format!( + "publication-point cached ROA local output parse failed: {err}" + )), + }, + ); + } + }, + VcirOutputType::Aspa => match parse_publication_point_cache_aspa_output(projected) { + Ok(aspa) => { + aspa_ok.insert(projected.source_object_uri.clone()); + output.aspas.push(aspa); + audit_by_uri.insert( + projected.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: projected.source_object_uri.clone(), + sha256_hex: hex::encode(projected.source_object_hash), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Ok, + detail: None, + }, + ); + } + Err(err) => { + warnings.push( + Warning::new(format!( + "publication-point cached ASPA local output parse failed: {err}" + )) + .with_context(&projected.source_object_uri), + ); + audit_by_uri.insert( + projected.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: projected.source_object_uri.clone(), + sha256_hex: hex::encode(projected.source_object_hash), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Error, + detail: Some(format!( + "publication-point cached ASPA local output parse failed: {err}" + )), + }, + ); + } + }, + VcirOutputType::RouterKey => { + match parse_publication_point_cache_router_key_output(projected) { + Ok(router_key) => { + output.router_keys.push(router_key); + audit_by_uri.insert( + projected.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: projected.source_object_uri.clone(), + sha256_hex: hex::encode(projected.source_object_hash), + kind: AuditObjectKind::RouterCertificate, + result: AuditObjectResult::Ok, + detail: Some( + "publication-point cached Router Key local output restored" + .to_string(), + ), + }, + ); + } + Err(err) => { + warnings.push( + Warning::new(format!( + "publication-point cached Router Key local output parse failed: {err}" + )) + .with_context(&projected.source_object_uri), + ); + audit_by_uri.insert( + projected.source_object_uri.clone(), + ObjectAuditEntry { + rsync_uri: projected.source_object_uri.clone(), + sha256_hex: hex::encode(projected.source_object_hash), + kind: AuditObjectKind::RouterCertificate, + result: AuditObjectResult::Error, + detail: Some(format!( + "publication-point cached Router Key local output parse failed: {err}" + )), + }, + ); + } + } + } + } + } + + output.stats.roa_total = roa_total.len(); + output.stats.roa_ok = roa_ok.len(); + output.stats.aspa_total = aspa_total.len(); + output.stats.aspa_ok = aspa_ok.len(); + let mut audit: Vec<_> = audit_by_uri.into_values().collect(); + audit.sort_by(|left, right| left.rsync_uri.cmp(&right.rsync_uri)); + output.audit = audit; + output +} + +fn parse_vcir_vrp_output(local: &VcirLocalOutput) -> Result { + match &local.payload { + VcirLocalOutputPayload::Vrp { + asn, + afi, + prefix_len, + addr, + max_length, + } => Ok(Vrp { + asn: *asn, + prefix: crate::data_model::roa::IpPrefix { + afi: *afi, + prefix_len: *prefix_len, + addr: *addr, + }, + max_length: *max_length, + }), + _ => Err("VCIR local output payload is not VRP".to_string()), + } +} + +fn parse_vcir_aspa_output(local: &VcirLocalOutput) -> Result { + match &local.payload { + VcirLocalOutputPayload::Aspa { + customer_as_id, + provider_as_ids, + } => Ok(AspaAttestation { + customer_as_id: *customer_as_id, + provider_as_ids: provider_as_ids.clone(), + }), + _ => Err("VCIR local output payload is not ASPA".to_string()), + } +} + +fn parse_vcir_router_key_output(local: &VcirLocalOutput) -> Result { + match &local.payload { + VcirLocalOutputPayload::RouterKey { + as_id, + ski, + spki_der, + } => Ok(RouterKeyPayload { + as_id: *as_id, + ski: ski.clone(), + spki_der: spki_der.clone(), + source_object_uri: local.source_object_uri.clone(), + source_object_hash: local.source_object_hash_hex(), + source_ee_cert_hash: local.source_ee_cert_hash_hex(), + item_effective_until: local.item_effective_until.clone(), + }), + _ => Err("VCIR local output payload is not Router Key".to_string()), + } +} + +fn parse_publication_point_cache_vrp_output( + projected: &PublicationPointCacheOutput, +) -> Result { + match &projected.payload { + VcirLocalOutputPayload::Vrp { + asn, + afi, + prefix_len, + addr, + max_length, + } => Ok(Vrp { + asn: *asn, + prefix: crate::data_model::roa::IpPrefix { + afi: *afi, + prefix_len: *prefix_len, + addr: *addr, + }, + max_length: *max_length, + }), + _ => Err("publication-point cache output payload is not VRP".to_string()), + } +} + +fn parse_publication_point_cache_aspa_output( + projected: &PublicationPointCacheOutput, +) -> Result { + match &projected.payload { + VcirLocalOutputPayload::Aspa { + customer_as_id, + provider_as_ids, + } => Ok(AspaAttestation { + customer_as_id: *customer_as_id, + provider_as_ids: provider_as_ids.clone(), + }), + _ => Err("publication-point cache output payload is not ASPA".to_string()), + } +} + +fn parse_publication_point_cache_router_key_output( + projected: &PublicationPointCacheOutput, +) -> Result { + match &projected.payload { + VcirLocalOutputPayload::RouterKey { + as_id, + ski, + spki_der, + } => Ok(RouterKeyPayload { + as_id: *as_id, + ski: ski.clone(), + spki_der: spki_der.clone(), + source_object_uri: projected.source_object_uri.clone(), + source_object_hash: hex::encode(projected.source_object_hash), + source_ee_cert_hash: hex::encode(projected.source_ee_cert_hash), + item_effective_until: projected.item_effective_until.clone(), + }), + _ => Err("publication-point cache output payload is not Router Key".to_string()), + } +} + +fn restore_children_from_vcir( + _store: &RocksStore, + ca: &CaInstanceHandle, + vcir: &ValidatedCaInstanceResult, + _warnings: &mut Vec, +) -> (Vec, Vec) { + let mut children = Vec::new(); + let mut audits = Vec::new(); + for child in &vcir.child_entries { + children.push(DiscoveredChildCaInstance { + handle: CaInstanceHandle { + depth: 0, + tal_id: ca.tal_id.clone(), + parent_manifest_rsync_uri: Some(ca.manifest_rsync_uri.clone()), + ca_certificate: CaCertificateRef::repo_bytes(child.child_cert_hash.clone()), + ca_certificate_rsync_uri: Some(child.child_cert_rsync_uri.clone()), + effective_ip_resources: child.child_effective_ip_resources.clone(), + effective_as_resources: child.child_effective_as_resources.clone(), + rsync_base_uri: child.child_rsync_base_uri.clone(), + manifest_rsync_uri: child.child_manifest_rsync_uri.clone(), + publication_point_rsync_uri: child.child_publication_point_rsync_uri.clone(), + rrdp_notification_uri: child.child_rrdp_notification_uri.clone(), + }, + discovered_from: crate::audit::DiscoveredFrom { + parent_manifest_rsync_uri: ca.manifest_rsync_uri.clone(), + child_ca_certificate_rsync_uri: child.child_cert_rsync_uri.clone(), + child_ca_certificate_sha256_hex: child.child_cert_hash.clone(), + }, + child_entry_projection: Some(DiscoveredChildEntryProjection { + child_ski: child.child_ski.clone(), + }), + }); + audits.push(ObjectAuditEntry { + rsync_uri: child.child_cert_rsync_uri.clone(), + sha256_hex: child.child_cert_hash.clone(), + kind: AuditObjectKind::Certificate, + result: AuditObjectResult::Ok, + detail: Some("restored child CA instance from VCIR".to_string()), + }); + } + (children, audits) +} + +fn restore_children_from_publication_point_cache( + store: &RocksStore, + ca: &CaInstanceHandle, + projection: &PublicationPointCacheProjection, + validation_time: time::OffsetDateTime, + warnings: &mut Vec, + worker_count: usize, + timing: Option<&TimingHandle>, +) -> (Vec, Vec) { + let worker_count = worker_count + .clamp(1, PUBLICATION_POINT_CACHE_CHILD_RESTORE_MAX_WORKERS) + .min(projection.children.len().max(1)); + let outcomes = if worker_count > 1 + && projection.children.len() >= PUBLICATION_POINT_CACHE_CHILD_RESTORE_PARALLEL_MIN_CHILDREN + { + if let Some(timing) = timing { + timing.record_count( + "publication_point_cache_restore_children_parallel_publication_points", + 1, + ); + timing.record_count( + "publication_point_cache_restore_children_parallel_children", + projection.children.len() as u64, + ); + timing.record_count( + "publication_point_cache_restore_children_workers_total", + worker_count as u64, + ); + } + restore_publication_point_cache_children_parallel( + store, + ca, + &projection.children, + validation_time, + worker_count, + ) + } else { + if let Some(timing) = timing { + timing.record_count( + "publication_point_cache_restore_children_batch_publication_points", + 1, + ); + timing.record_count( + "publication_point_cache_restore_children_batch_children", + projection.children.len() as u64, + ); + } + restore_publication_point_cache_children_chunk( + store, + ca, + &projection.children, + validation_time, + ) + }; + collect_publication_point_cache_child_restore_outcomes(outcomes, warnings) +} + +fn restore_publication_point_cache_children_parallel( + _store: &RocksStore, + ca: &CaInstanceHandle, + children: &[PublicationPointCacheChild], + validation_time: time::OffsetDateTime, + worker_count: usize, +) -> Vec { + let chunk_size = children.len().div_ceil(worker_count).max(1); + let mut chunk_results = Vec::new(); + std::thread::scope(|scope| { + let mut handles = Vec::new(); + for chunk in children.chunks(chunk_size) { + handles.push(scope.spawn(move || { + restore_publication_point_cache_children_chunk(_store, ca, chunk, validation_time) + })); + } + for handle in handles { + chunk_results.extend( + handle + .join() + .expect("publication-point cache child restore worker panicked"), + ); + } + }); + chunk_results +} + +fn restore_publication_point_cache_children_chunk( + _store: &RocksStore, + ca: &CaInstanceHandle, + children: &[PublicationPointCacheChild], + validation_time: time::OffsetDateTime, +) -> Vec { + let mut outcomes = vec![None; children.len()]; + for (position, child) in children.iter().enumerate() { + let effective_not_before = + match parse_snapshot_time_value(&child.child_effective_not_before) { + Ok(value) => value, + Err(e) => { + outcomes[position] = Some(PublicationPointCacheChildRestoreOutcome { + child: None, + audit: None, + warning: Some( + Warning::new(format!( + "publication-point cache child has invalid effective notBefore: {e}" + )) + .with_context(&child.child_cert_rsync_uri), + ), + }); + continue; + } + }; + let effective_until = match parse_snapshot_time_value(&child.child_effective_until) { + Ok(value) => value, + Err(e) => { + outcomes[position] = Some(PublicationPointCacheChildRestoreOutcome { + child: None, + audit: None, + warning: Some( + Warning::new(format!( + "publication-point cache child has invalid effective until: {e}" + )) + .with_context(&child.child_cert_rsync_uri), + ), + }); + continue; + } + }; + if validation_time < effective_not_before || validation_time > effective_until { + outcomes[position] = Some(PublicationPointCacheChildRestoreOutcome { + child: None, + warning: None, + audit: Some(publication_point_cache_child_audit( + child, + AuditObjectResult::Skipped, + Some("skipped: publication-point cache child expired".to_string()), + )), + }); + continue; + } + outcomes[position] = Some(PublicationPointCacheChildRestoreOutcome { + child: Some(publication_point_cache_discovered_child(ca, child)), + warning: None, + audit: Some(publication_point_cache_child_audit( + child, + AuditObjectResult::Ok, + Some("restored child CA instance from publication-point cache".to_string()), + )), + }); + } + + outcomes + .into_iter() + .flatten() + .collect::>() +} + +fn collect_publication_point_cache_child_restore_outcomes( + outcomes: Vec, + warnings: &mut Vec, +) -> (Vec, Vec) { + let mut children = Vec::new(); + let mut audits = Vec::new(); + for outcome in outcomes { + if let Some(warning) = outcome.warning { + warnings.push(warning); + } + if let Some(audit) = outcome.audit { + audits.push(audit); + } + if let Some(child) = outcome.child { + children.push(child); + } + } + (children, audits) +} + +#[derive(Clone)] +struct PublicationPointCacheChildRestoreOutcome { + child: Option, + audit: Option, + warning: Option, +} + +fn publication_point_cache_discovered_child( + ca: &CaInstanceHandle, + child: &PublicationPointCacheChild, +) -> DiscoveredChildCaInstance { + DiscoveredChildCaInstance { + handle: CaInstanceHandle { + depth: 0, + tal_id: ca.tal_id.clone(), + parent_manifest_rsync_uri: Some(ca.manifest_rsync_uri.clone()), + ca_certificate: CaCertificateRef::repo_bytes(child.child_cert_hash.clone()), + ca_certificate_rsync_uri: Some(child.child_cert_rsync_uri.clone()), + effective_ip_resources: child.child_effective_ip_resources.clone(), + effective_as_resources: child.child_effective_as_resources.clone(), + rsync_base_uri: child.child_rsync_base_uri.clone(), + manifest_rsync_uri: child.child_manifest_rsync_uri.clone(), + publication_point_rsync_uri: child.child_publication_point_rsync_uri.clone(), + rrdp_notification_uri: child.child_rrdp_notification_uri.clone(), + }, + discovered_from: crate::audit::DiscoveredFrom { + parent_manifest_rsync_uri: ca.manifest_rsync_uri.clone(), + child_ca_certificate_rsync_uri: child.child_cert_rsync_uri.clone(), + child_ca_certificate_sha256_hex: child.child_cert_hash.clone(), + }, + child_entry_projection: Some(DiscoveredChildEntryProjection { + child_ski: child.child_ski.clone(), + }), + } +} + +fn publication_point_cache_child_audit( + child: &PublicationPointCacheChild, + result: AuditObjectResult, + detail: Option, +) -> ObjectAuditEntry { + ObjectAuditEntry { + rsync_uri: child.child_cert_rsync_uri.clone(), + sha256_hex: child.child_cert_hash.clone(), + kind: AuditObjectKind::Certificate, + result, + detail, + } +} + +fn persist_vcir_for_fresh_result_with_timing( + store: &RocksStore, + policy: &Policy, + ca: &CaInstanceHandle, + pack: &PublicationPointSnapshot, + objects: &mut crate::validation::objects::ObjectsOutput, + warnings: &[Warning], + child_audits: &[ObjectAuditEntry], + discovered_children: &[DiscoveredChildCaInstance], + validation_time: time::OffsetDateTime, + write_publication_point_cache_projection: bool, +) -> Result { + let mut timing = PersistVcirTimingBreakdown::default(); + + if objects.stats.publication_point_dropped { + return Ok(timing); + } + + let embedded_store_started = std::time::Instant::now(); + persist_vcir_non_repository_evidence(store, ca) + .map_err(|e| format!("store VCIR audit evidence failed: {e}"))?; + timing.embedded_store_ms = embedded_store_started.elapsed().as_millis() as u64; + + let build_vcir_started = std::time::Instant::now(); + let (vcir, build_vcir_timing) = build_vcir_from_fresh_result_with_timing( + store, + ca, + pack, + objects, + warnings, + child_audits, + discovered_children, + validation_time, + )?; + timing.build_vcir_ms = build_vcir_started.elapsed().as_millis() as u64; + timing.build_vcir = build_vcir_timing; + + let replace_vcir_started = std::time::Instant::now(); + let future_not_before_cache_guard = write_publication_point_cache_projection + && publication_point_cache_has_future_not_before_risk( + pack, + objects, + child_audits, + validation_time, + policy, + ); + timing.publication_point_cache_future_notbefore_guarded = future_not_before_cache_guard; + let publication_point_cache_projection = + if write_publication_point_cache_projection && !future_not_before_cache_guard { + Some(build_publication_point_cache_projection_from_fresh( + policy, ca, pack, &vcir, + )?) + } else { + None + }; + let publication_point_cache_projection_action = if !write_publication_point_cache_projection { + PublicationPointCacheProjectionWriteAction::Keep + } else if future_not_before_cache_guard { + PublicationPointCacheProjectionWriteAction::Delete { + manifest_rsync_uri: &vcir.manifest_rsync_uri, + } + } else { + PublicationPointCacheProjectionWriteAction::Write( + publication_point_cache_projection + .as_ref() + .expect("publication point projection must exist when guard is not active"), + ) + }; + let failed_fetch_reuse_identity = failed_fetch_reuse_identity_for_fresh_result( + ca, + policy, + validation_time, + vcir.instance_gate.instance_effective_until.clone(), + )?; + let replace_timing = store + .replace_vcir_manifest_replay_meta_and_projection_action_with_failed_fetch_reuse_identity( + &vcir, + Some(&RoaCacheProjectionContext { + ca_validation_context_digest: ca_validation_context_digest_for_ca(ca), + policy_fingerprint: publication_point_cache_policy_fingerprint(policy), + object_meta: objects.roa_cache_object_meta.clone(), + }), + publication_point_cache_projection_action, + Some(&failed_fetch_reuse_identity), + ) + .map_err(|e| format!("store VCIR and manifest replay meta failed: {e}"))?; + timing.replace_vcir_ms = replace_vcir_started.elapsed().as_millis() as u64; + timing.replace_vcir = replace_timing; + + Ok(timing) +} + +fn build_publication_point_cache_projection_from_fresh( + policy: &Policy, + ca: &CaInstanceHandle, + pack: &PublicationPointSnapshot, + vcir: &ValidatedCaInstanceResult, +) -> Result { + let ca_cert_sha256 = ca + .ca_certificate_sha256_32() + .ok_or_else(|| "current CA certificate hash unavailable".to_string())?; + let manifest_sha256 = sha256_digest_32(&pack.manifest_bytes); + PublicationPointCacheProjection::from_vcir_with_context( + vcir, + pack.publication_point_rsync_uri.clone(), + ca.ca_certificate_rsync_uri.clone(), + ca_cert_sha256, + manifest_sha256, + ta_context_digest_for_ca(ca), + ca_validation_context_digest_for_ca(ca), + publication_point_cache_policy_fingerprint(policy), + ) + .map_err(|e| e.to_string()) +} + +fn publication_point_cache_has_future_not_before_risk( + pack: &PublicationPointSnapshot, + objects: &crate::validation::objects::ObjectsOutput, + child_audits: &[ObjectAuditEntry], + validation_time: time::OffsetDateTime, + policy: &Policy, +) -> bool { + let mut files_by_uri: HashMap<&str, &PackFile> = HashMap::new(); + for file in &pack.files { + files_by_uri.insert(file.rsync_uri.as_str(), file); + } + + objects + .audit + .iter() + .chain(child_audits.iter()) + .any(|entry| { + audit_entry_has_certificate_time_error(entry) + && files_by_uri + .get(entry.rsync_uri.as_str()) + .map(|file| { + audit_entry_has_future_not_before(entry, file, validation_time, policy) + }) + .unwrap_or(true) + }) +} + +fn audit_entry_has_certificate_time_error(entry: &ObjectAuditEntry) -> bool { + entry.result == AuditObjectResult::Error + && entry + .detail + .as_deref() + .is_some_and(|detail| detail.contains("certificate not valid at validation_time")) +} + +fn audit_entry_has_future_not_before( + entry: &ObjectAuditEntry, + file: &PackFile, + validation_time: time::OffsetDateTime, + policy: &Policy, +) -> bool { + match entry.kind { + AuditObjectKind::Roa => signed_object_ee_not_before(file, policy, SignedObjectKind::Roa) + .map(|not_before| validation_time < not_before) + .unwrap_or(true), + AuditObjectKind::Aspa => signed_object_ee_not_before(file, policy, SignedObjectKind::Aspa) + .map(|not_before| validation_time < not_before) + .unwrap_or(true), + AuditObjectKind::Certificate | AuditObjectKind::RouterCertificate => file + .bytes() + .ok() + .and_then(|bytes| ResourceCertificate::decode_der(bytes).ok()) + .map(|cert| validation_time < cert.tbs.validity_not_before) + .unwrap_or(true), + _ => true, + } +} + +#[derive(Clone, Copy)] +enum SignedObjectKind { + Roa, + Aspa, +} + +fn signed_object_ee_not_before( + file: &PackFile, + policy: &Policy, + kind: SignedObjectKind, +) -> Option { + let bytes = file.bytes().ok()?; + match kind { + SignedObjectKind::Roa => { + let object = RoaObject::decode_der_with_strict_options( + bytes, + policy.strict.cms_der, + policy.strict.name, + ) + .ok()?; + Some( + object.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .validity_not_before, + ) + } + SignedObjectKind::Aspa => { + let object = AspaObject::decode_der_with_strict_options( + bytes, + policy.strict.cms_der, + policy.strict.name, + ) + .ok()?; + Some( + object.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .validity_not_before, + ) + } + } +} + +fn build_vcir_from_fresh_result_with_timing( + store: &RocksStore, + ca: &CaInstanceHandle, + pack: &PublicationPointSnapshot, + objects: &mut crate::validation::objects::ObjectsOutput, + warnings: &[Warning], + child_audits: &[ObjectAuditEntry], + discovered_children: &[DiscoveredChildCaInstance], + validation_time: time::OffsetDateTime, +) -> Result<(ValidatedCaInstanceResult, BuildVcirTimingBreakdown), String> { + let mut timing = BuildVcirTimingBreakdown::default(); + + let select_crl_started = std::time::Instant::now(); + let current_crl = select_manifest_current_crl_from_snapshot(pack)?; + timing.select_crl_ms = select_crl_started.elapsed().as_millis() as u64; + + let current_ca_decode_started = std::time::Instant::now(); + let ca_der = ca.ca_certificate_der(store)?; + let ca_cert = ResourceCertificate::decode_der(ca_der.as_ref()) + .map_err(|e| format!("decode current CA certificate failed: {e}"))?; + timing.current_ca_decode_ms = current_ca_decode_started.elapsed().as_millis() as u64; + + let local_outputs_started = std::time::Instant::now(); + let local_outputs = take_or_build_vcir_local_outputs(ca, pack, objects)?; + timing.local_outputs_ms = local_outputs_started.elapsed().as_millis() as u64; + + let child_entries_started = std::time::Instant::now(); + let child_entries = build_vcir_child_entries(store, discovered_children, validation_time)?; + timing.child_entries_ms = child_entries_started.elapsed().as_millis() as u64; + + let related_artifacts_started = std::time::Instant::now(); + let related_artifacts = build_vcir_related_artifacts( + store, + ca, + pack, + current_crl.file.rsync_uri.as_str(), + objects, + child_audits, + ); + timing.related_artifacts_ms = related_artifacts_started.elapsed().as_millis() as u64; + let ccr_manifest_projection = + build_vcir_ccr_manifest_projection_from_fresh(ca, pack, &child_entries)?; + let local_vrp_count = local_outputs + .iter() + .filter(|output| output.output_type == VcirOutputType::Vrp) + .count() as u32; + let local_aspa_count = local_outputs + .iter() + .filter(|output| output.output_type == VcirOutputType::Aspa) + .count() as u32; + let local_router_key_count = local_outputs + .iter() + .filter(|output| output.output_type == VcirOutputType::RouterKey) + .count() as u32; + let accepted_object_count = related_artifacts + .iter() + .filter(|artifact| artifact.validation_status == VcirArtifactValidationStatus::Accepted) + .count() as u32; + let rejected_object_count = related_artifacts + .iter() + .filter(|artifact| artifact.validation_status == VcirArtifactValidationStatus::Rejected) + .count() as u32; + let ca_ski = hex::encode( + ca_cert + .tbs + .extensions + .subject_key_identifier + .as_ref() + .ok_or_else(|| "current CA certificate missing SubjectKeyIdentifier".to_string())?, + ); + let issuer_ski = hex::encode( + ca_cert + .tbs + .extensions + .authority_key_identifier + .as_ref() + .or(ca_cert.tbs.extensions.subject_key_identifier.as_ref()) + .ok_or_else(|| "current CA certificate missing AuthorityKeyIdentifier".to_string())?, + ); + + let struct_build_started = std::time::Instant::now(); + let vcir = ValidatedCaInstanceResult { + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + parent_manifest_rsync_uri: ca.parent_manifest_rsync_uri.clone(), + tal_id: ca.tal_id.clone(), + ca_subject_name: ca_cert.tbs.subject_name.to_string(), + ca_ski, + issuer_ski, + last_successful_validation_time: PackTime::from_utc_offset_datetime(validation_time), + current_manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + current_crl_rsync_uri: current_crl.file.rsync_uri.clone(), + validated_manifest_meta: crate::storage::ValidatedManifestMeta { + validated_manifest_number: pack.manifest_number_be.clone(), + validated_manifest_this_update: pack.this_update.clone(), + validated_manifest_next_update: pack.next_update.clone(), + }, + ccr_manifest_projection, + instance_gate: VcirInstanceGate { + manifest_next_update: pack.next_update.clone(), + current_crl_next_update: PackTime::from_utc_offset_datetime( + current_crl.crl.next_update.utc, + ), + self_ca_not_after: PackTime::from_utc_offset_datetime(ca_cert.tbs.validity_not_after), + instance_effective_until: PackTime::from_utc_offset_datetime( + pack.next_update + .parse() + .map_err(|e| format!("parse snapshot next_update failed: {e}"))? + .min(current_crl.crl.next_update.utc) + .min(ca_cert.tbs.validity_not_after), + ), + }, + child_entries, + local_outputs, + related_artifacts, + summary: VcirSummary { + local_vrp_count, + local_aspa_count, + local_router_key_count, + child_count: discovered_children.len() as u32, + accepted_object_count, + rejected_object_count, + }, + audit_summary: VcirAuditSummary { + failed_fetch_eligible: true, + last_failed_fetch_reason: None, + warning_count: (warnings.len() + objects.warnings.len()) as u32, + audit_flags: Vec::new(), + }, + }; + vcir.validate_internal().map_err(|e| e.to_string())?; + timing.struct_build_ms = struct_build_started.elapsed().as_millis() as u64; + Ok((vcir, timing)) +} + +fn take_or_build_vcir_local_outputs( + ca: &CaInstanceHandle, + pack: &PublicationPointSnapshot, + objects: &mut crate::validation::objects::ObjectsOutput, +) -> Result, String> { + let mut cached_outputs = std::mem::take(&mut objects.local_outputs_cache); + if cached_outputs.is_empty() { + return build_vcir_local_outputs(ca, pack, objects); + } + + let covered_roa_uris: HashSet = cached_outputs + .iter() + .filter(|output| output.source_object_type == VcirSourceObjectType::Roa) + .map(|output| output.source_object_uri.clone()) + .collect(); + let covered_aspa_uris: HashSet = cached_outputs + .iter() + .filter(|output| output.source_object_type == VcirSourceObjectType::Aspa) + .map(|output| output.source_object_uri.clone()) + .collect(); + cached_outputs.extend(build_vcir_local_outputs_excluding( + ca, + pack, + objects, + &covered_roa_uris, + &covered_aspa_uris, + )?); + Ok(cached_outputs) +} + +fn build_vcir_ccr_manifest_projection_from_fresh( + ca: &CaInstanceHandle, + pack: &PublicationPointSnapshot, + child_entries: &[VcirChildEntry], +) -> Result { + let manifest = ManifestObject::decode_der(&pack.manifest_bytes) + .map_err(|e| format!("decode manifest for VCIR CCR projection failed: {e}"))?; + let ee = &manifest.signed_object.signed_data.certificates[0].resource_cert; + let manifest_ee_aki = ee + .tbs + .extensions + .authority_key_identifier + .clone() + .ok_or_else(|| "manifest EE certificate missing AuthorityKeyIdentifier".to_string())?; + let manifest_sia_locations_der = match ee + .tbs + .extensions + .subject_info_access + .as_ref() + .ok_or_else(|| "manifest EE certificate missing Subject Information Access".to_string())? + { + SubjectInfoAccess::Ee(ee_sia) => vec![select_manifest_signed_object_location( + &ca.manifest_rsync_uri, + &ee_sia.access_descriptions, + )?], + SubjectInfoAccess::Ca(_) => { + return Err( + "manifest EE certificate Subject Information Access has CA variant".to_string(), + ); + } + }; + + let mut subordinate_skis = child_entries + .iter() + .map(|child| { + hex::decode(&child.child_ski) + .map_err(|e| format!("decode child_ski for VCIR CCR projection failed: {e}")) + }) + .collect::, _>>()?; + subordinate_skis.sort(); + subordinate_skis.dedup(); + + Ok(VcirCcrManifestProjection { + manifest_rsync_uri: ca.manifest_rsync_uri.clone(), + manifest_sha256: sha2::Sha256::digest(&pack.manifest_bytes).to_vec(), + manifest_size: pack.manifest_bytes.len() as u64, + manifest_ee_aki, + manifest_number_be: pack.manifest_number_be.clone(), + manifest_this_update: pack.this_update.clone(), + manifest_sia_locations_der, + subordinate_skis, + }) +} + +struct CurrentCrlRef<'a> { + file: &'a PackFile, + crl: RpkixCrl, +} + +fn select_manifest_current_crl_from_snapshot( + pack: &PublicationPointSnapshot, +) -> Result, String> { + let manifest = ManifestObject::decode_der(&pack.manifest_bytes) + .map_err(|e| format!("decode snapshot manifest for VCIR failed: {e}"))?; + let ee = &manifest.signed_object.signed_data.certificates[0].resource_cert; + let crldp_uris = ee + .tbs + .extensions + .crl_distribution_points_uris + .as_ref() + .ok_or_else(|| "manifest EE certificate missing CRLDistributionPoints".to_string())?; + for uri in crldp_uris { + if let Some(file) = pack + .files + .iter() + .find(|candidate| candidate.rsync_uri == *uri) + { + let crl = RpkixCrl::decode_der( + file.bytes() + .map_err(|e| format!("load current CRL bytes for VCIR failed: {e}"))?, + ) + .map_err(|e| format!("decode current CRL for VCIR failed: {e}"))?; + return Ok(CurrentCrlRef { file, crl }); + } + } + Err(format!( + "manifest EE certificate CRLDistributionPoints not found in pack: {}", + crldp_uris.join(", ") + )) +} + +fn build_vcir_local_outputs( + _ca: &CaInstanceHandle, + pack: &PublicationPointSnapshot, + objects: &crate::validation::objects::ObjectsOutput, +) -> Result, String> { + build_vcir_local_outputs_excluding(_ca, pack, objects, &HashSet::new(), &HashSet::new()) +} + +fn build_vcir_local_outputs_excluding( + _ca: &CaInstanceHandle, + pack: &PublicationPointSnapshot, + objects: &crate::validation::objects::ObjectsOutput, + covered_roa_uris: &HashSet, + covered_aspa_uris: &HashSet, +) -> Result, String> { + let accepted_roa_uris: HashSet<&str> = objects + .audit + .iter() + .filter(|entry| entry.kind == AuditObjectKind::Roa && entry.result == AuditObjectResult::Ok) + .map(|entry| entry.rsync_uri.as_str()) + .collect(); + let accepted_aspa_uris: HashSet<&str> = objects + .audit + .iter() + .filter(|entry| { + entry.kind == AuditObjectKind::Aspa && entry.result == AuditObjectResult::Ok + }) + .map(|entry| entry.rsync_uri.as_str()) + .collect(); + + let mut out = Vec::new(); + for file in &pack.files { + let source_object_hash = sha256_hex_from_32(&file.sha256); + if accepted_roa_uris.contains(file.rsync_uri.as_str()) + && !covered_roa_uris.contains(file.rsync_uri.as_str()) + { + let roa = RoaObject::decode_der( + file.bytes() + .map_err(|e| format!("load accepted ROA bytes for VCIR failed: {e}"))?, + ) + .map_err(|e| format!("decode accepted ROA for VCIR failed: {e}"))?; + let ee = &roa.signed_object.signed_data.certificates[0]; + let source_ee_cert_hash = sha256_hex(ee.raw_der.as_slice()); + let item_effective_until = + PackTime::from_utc_offset_datetime(ee.resource_cert.tbs.validity_not_after); + for vrp in roa_to_vrps_for_vcir(&roa) { + let prefix = vrp_prefix_to_string(&vrp); + let rule_hash = sha256_hex( + format!( + "roa-rule:{}:{}:{}:{}", + source_object_hash, vrp.asn, prefix, vrp.max_length + ) + .as_bytes(), + ); + out.push(VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: item_effective_until.clone(), + source_object_uri: file.rsync_uri.clone(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: file.sha256, + source_ee_cert_hash: sha256_hex_to_32(&source_ee_cert_hash), + payload: VcirLocalOutputPayload::Vrp { + asn: vrp.asn, + afi: vrp.prefix.afi, + prefix_len: vrp.prefix.prefix_len, + addr: vrp.prefix.addr, + max_length: vrp.max_length, + }, + rule_hash: sha256_hex_to_32(&rule_hash), + }); + } + } else if accepted_aspa_uris.contains(file.rsync_uri.as_str()) + && !covered_aspa_uris.contains(file.rsync_uri.as_str()) + { + let aspa = AspaObject::decode_der( + file.bytes() + .map_err(|e| format!("load accepted ASPA bytes for VCIR failed: {e}"))?, + ) + .map_err(|e| format!("decode accepted ASPA for VCIR failed: {e}"))?; + let ee = &aspa.signed_object.signed_data.certificates[0]; + let source_ee_cert_hash = sha256_hex(ee.raw_der.as_slice()); + let item_effective_until = + PackTime::from_utc_offset_datetime(ee.resource_cert.tbs.validity_not_after); + let providers = aspa + .aspa + .provider_as_ids + .iter() + .map(u32::to_string) + .collect::>() + .join(","); + let rule_hash = sha256_hex( + format!( + "aspa-rule:{}:{}:{}", + source_object_hash, aspa.aspa.customer_as_id, providers + ) + .as_bytes(), + ); + out.push(VcirLocalOutput { + output_type: VcirOutputType::Aspa, + item_effective_until, + source_object_uri: file.rsync_uri.clone(), + source_object_type: VcirSourceObjectType::Aspa, + source_object_hash: file.sha256, + source_ee_cert_hash: sha256_hex_to_32(&source_ee_cert_hash), + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: aspa.aspa.customer_as_id, + provider_as_ids: aspa.aspa.provider_as_ids.clone(), + }, + rule_hash: sha256_hex_to_32(&rule_hash), + }); + } + } + Ok(out) +} + +pub(crate) fn build_router_key_local_outputs( + _ca: &CaInstanceHandle, + router_keys: &[RouterKeyPayload], +) -> Vec { + router_keys + .iter() + .map(|router_key| { + let ski_hex = hex::encode(&router_key.ski); + let spki_der_base64 = + base64::engine::general_purpose::STANDARD.encode(&router_key.spki_der); + let rule_hash = sha256_hex( + format!( + "router-key-rule:{}:{}:{}:{}", + router_key.source_object_hash, router_key.as_id, ski_hex, spki_der_base64 + ) + .as_bytes(), + ); + VcirLocalOutput { + output_type: VcirOutputType::RouterKey, + item_effective_until: router_key.item_effective_until.clone(), + source_object_uri: router_key.source_object_uri.clone(), + source_object_type: VcirSourceObjectType::RouterKey, + source_object_hash: sha256_hex_to_32(&router_key.source_object_hash), + source_ee_cert_hash: sha256_hex_to_32(&router_key.source_ee_cert_hash), + payload: VcirLocalOutputPayload::RouterKey { + as_id: router_key.as_id, + ski: router_key.ski.clone(), + spki_der: router_key.spki_der.clone(), + }, + rule_hash: sha256_hex_to_32(&rule_hash), + } + }) + .collect() +} + +fn build_vcir_child_entries( + store: &RocksStore, + discovered_children: &[DiscoveredChildCaInstance], + validation_time: time::OffsetDateTime, +) -> Result, String> { + let mut out = Vec::with_capacity(discovered_children.len()); + for child in discovered_children { + let child_ski = match child.child_entry_projection.as_ref() { + Some(projection) => projection.child_ski.clone(), + None => { + let child_der = child.handle.ca_certificate_der(store)?; + let child_cert = ResourceCertificate::decode_der(child_der.as_ref()) + .map_err(|e| format!("decode child certificate for VCIR failed: {e}"))?; + let child_ski = child_cert + .tbs + .extensions + .subject_key_identifier + .as_ref() + .ok_or_else(|| "child certificate missing SubjectKeyIdentifier".to_string())?; + hex::encode(child_ski) + } + }; + out.push(VcirChildEntry { + child_manifest_rsync_uri: child.handle.manifest_rsync_uri.clone(), + child_cert_rsync_uri: child.discovered_from.child_ca_certificate_rsync_uri.clone(), + child_cert_hash: child + .discovered_from + .child_ca_certificate_sha256_hex + .clone(), + child_ski, + child_rsync_base_uri: child.handle.rsync_base_uri.clone(), + child_publication_point_rsync_uri: child.handle.publication_point_rsync_uri.clone(), + child_rrdp_notification_uri: child.handle.rrdp_notification_uri.clone(), + child_effective_ip_resources: child.handle.effective_ip_resources.clone(), + child_effective_as_resources: child.handle.effective_as_resources.clone(), + accepted_at_validation_time: PackTime::from_utc_offset_datetime(validation_time), + }); + } + Ok(out) +} + +fn persist_vcir_non_repository_evidence( + store: &RocksStore, + ca: &CaInstanceHandle, +) -> Result<(), String> { + let ca_der = ca.ca_certificate_der(store)?; + let current_ca_hash = sha256_hex(ca_der.as_ref()); + let mut current_ca_entry = RawByHashEntry::from_bytes(current_ca_hash, ca_der.to_vec()); + if let Some(uri) = ca.ca_certificate_rsync_uri.as_ref() { + current_ca_entry.origin_uris.push(uri.clone()); + } + current_ca_entry.object_type = Some("cer".to_string()); + current_ca_entry.encoding = Some("der".to_string()); + upsert_raw_by_hash_entry(store, current_ca_entry)?; + Ok(()) +} + +fn upsert_raw_by_hash_entry(store: &RocksStore, entry: RawByHashEntry) -> Result<(), String> { + match store.get_raw_by_hash_entry(&entry.sha256_hex) { + Ok(Some(existing)) => { + if existing.bytes != entry.bytes { + return Err(format!( + "raw_by_hash collision for sha256 {} while storing VCIR audit evidence", + entry.sha256_hex + )); + } + let mut merged = existing; + let mut changed = false; + for uri in entry.origin_uris { + if !merged + .origin_uris + .iter() + .any(|existing_uri| existing_uri == &uri) + { + merged.origin_uris.push(uri); + changed = true; + } + } + if merged.object_type.is_none() && entry.object_type.is_some() { + merged.object_type = entry.object_type; + changed = true; + } + if merged.encoding.is_none() && entry.encoding.is_some() { + merged.encoding = entry.encoding; + changed = true; + } + if changed { + store + .put_raw_by_hash_entry(&merged) + .map_err(|e| format!("update raw_by_hash entry failed: {e}"))?; + } + Ok(()) + } + Ok(None) => store + .put_raw_by_hash_entry(&entry) + .map_err(|e| format!("store raw_by_hash entry failed: {e}")), + Err(e) => Err(format!("load raw_by_hash entry failed: {e}")), + } +} + +fn build_vcir_related_artifacts( + store: &RocksStore, + ca: &CaInstanceHandle, + pack: &PublicationPointSnapshot, + current_crl_rsync_uri: &str, + objects: &crate::validation::objects::ObjectsOutput, + child_audits: &[ObjectAuditEntry], +) -> Vec { + let mut audit_by_uri: HashMap<&str, &ObjectAuditEntry> = HashMap::new(); + for entry in child_audits.iter().chain(objects.audit.iter()) { + audit_by_uri.insert(entry.rsync_uri.as_str(), entry); + } + + let mut artifacts = Vec::with_capacity(pack.files.len() + 2); + artifacts.push(VcirRelatedArtifact { + artifact_role: VcirArtifactRole::Manifest, + artifact_kind: VcirArtifactKind::Mft, + uri: Some(pack.manifest_rsync_uri.clone()), + sha256: sha256_hex(&pack.manifest_bytes), + object_type: Some("mft".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }); + artifacts.push(VcirRelatedArtifact { + artifact_role: if ca.parent_manifest_rsync_uri.is_none() { + VcirArtifactRole::TrustAnchorCert + } else { + VcirArtifactRole::IssuerCert + }, + artifact_kind: VcirArtifactKind::Cer, + uri: ca.ca_certificate_rsync_uri.clone(), + sha256: ca + .ca_certificate_sha256_hex() + .map(str::to_string) + .unwrap_or_else(|| { + ca.ca_certificate_der(store) + .map(|bytes| sha256_hex(bytes.as_ref())) + .unwrap_or_default() + }), + object_type: Some("cer".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }); + + for file in &pack.files { + let audit_entry = audit_by_uri.get(file.rsync_uri.as_str()).copied(); + let result = audit_entry + .map(|entry| entry.result.clone()) + .unwrap_or(AuditObjectResult::Ok); + let validation_status = audit_result_to_vcir_status(&result); + let reject_reason = if validation_status == VcirArtifactValidationStatus::Rejected { + audit_entry.and_then(|entry| entry.detail.clone()) + } else { + None + }; + let (artifact_role, artifact_kind) = artifact_role_and_kind(file, current_crl_rsync_uri); + artifacts.push(VcirRelatedArtifact { + artifact_role, + artifact_kind, + uri: Some(file.rsync_uri.clone()), + sha256: sha256_hex_from_32(&file.sha256), + object_type: object_type_from_uri(file.rsync_uri.as_str()), + validation_status, + reject_reason, + }); + } + + artifacts +} + +fn artifact_role_and_kind( + file: &PackFile, + current_crl_rsync_uri: &str, +) -> (VcirArtifactRole, VcirArtifactKind) { + if file.rsync_uri == current_crl_rsync_uri { + (VcirArtifactRole::CurrentCrl, VcirArtifactKind::Crl) + } else if file.rsync_uri.ends_with(".cer") { + (VcirArtifactRole::ChildCaCert, VcirArtifactKind::Cer) + } else if file.rsync_uri.ends_with(".roa") { + (VcirArtifactRole::SignedObject, VcirArtifactKind::Roa) + } else if file.rsync_uri.ends_with(".asa") { + (VcirArtifactRole::SignedObject, VcirArtifactKind::Aspa) + } else if file.rsync_uri.ends_with(".gbr") { + (VcirArtifactRole::SignedObject, VcirArtifactKind::Gbr) + } else if file.rsync_uri.ends_with(".crl") { + (VcirArtifactRole::Other, VcirArtifactKind::Crl) + } else if file.rsync_uri.ends_with(".mft") { + (VcirArtifactRole::Manifest, VcirArtifactKind::Mft) + } else { + (VcirArtifactRole::Other, VcirArtifactKind::Other) + } +} + +fn object_type_from_uri(uri: &str) -> Option { + uri.rsplit_once('.') + .map(|(_, ext)| ext.to_ascii_lowercase()) +} + +fn audit_result_to_vcir_status(result: &AuditObjectResult) -> VcirArtifactValidationStatus { + match result { + AuditObjectResult::Ok => VcirArtifactValidationStatus::Accepted, + AuditObjectResult::Error => VcirArtifactValidationStatus::Rejected, + AuditObjectResult::Skipped => VcirArtifactValidationStatus::WarningOnly, + } +} + +fn roa_to_vrps_for_vcir(roa: &RoaObject) -> Vec { + let asn = roa.roa.as_id; + let mut out = Vec::new(); + for fam in &roa.roa.ip_addr_blocks { + for entry in &fam.addresses { + let max_length = entry.max_length.unwrap_or(entry.prefix.prefix_len); + out.push(Vrp { + asn, + prefix: entry.prefix.clone(), + max_length, + }); + } + } + out +} + +fn vrp_prefix_to_string(vrp: &Vrp) -> String { + match vrp.prefix.afi { + RoaAfi::Ipv4 => { + let addr = std::net::Ipv4Addr::new( + vrp.prefix.addr[0], + vrp.prefix.addr[1], + vrp.prefix.addr[2], + vrp.prefix.addr[3], + ); + format!("{addr}/{}", vrp.prefix.prefix_len) + } + RoaAfi::Ipv6 => { + let addr = std::net::Ipv6Addr::from(vrp.prefix.addr); + format!("{addr}/{}", vrp.prefix.prefix_len) + } + } +} + +#[cfg(test)] +#[path = "tree_runner/tests.rs"] +mod tests; diff --git a/crates/panda-rpki-validator/src/validation/tree_runner/tests.rs b/crates/panda-rpki-validator/src/validation/tree_runner/tests.rs new file mode 100644 index 0000000..8a7ecd9 --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/tree_runner/tests.rs @@ -0,0 +1,5869 @@ +use super::*; +use crate::data_model::oid::OID_AD_SIGNED_OBJECT; +use crate::data_model::rc::{ + AccessDescription, AsIdOrRange, AsIdentifierChoice, AsResourceSet, ResourceCertificate, +}; +use crate::data_model::roa::RoaAfi; +use crate::fetch::rsync::LocalDirRsyncFetcher; +use crate::fetch::rsync::{RsyncFetchError, RsyncFetcher}; +use crate::storage::{ + PackFile, PackTime, PublicationPointCacheProjection, + PublicationPointCacheProjectionWriteAction, RawByHashEntry, RepositoryViewEntry, + RepositoryViewState, RocksStore, ValidatedCaInstanceResult, ValidatedManifestMeta, + VcirArtifactKind, VcirArtifactRole, VcirArtifactValidationStatus, VcirAuditSummary, + VcirChildEntry, VcirInstanceGate, VcirLocalOutput, VcirLocalOutputPayload, VcirOutputType, + VcirRelatedArtifact, VcirSourceObjectType, VcirSummary, +}; +use crate::sync::rrdp::Fetcher; +use crate::validation::publication_point::PublicationPointSnapshot; +use crate::validation::tree::{DiscoveredChildEntryProjection, PublicationPointRunner}; + +use std::process::Command; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +fn sha256_32(input: &[u8]) -> [u8; 32] { + sha256_hex_to_32(&sha256_hex(input)) +} + +fn ipv4_addr(octets: [u8; 4]) -> [u8; 16] { + let mut addr = [0u8; 16]; + addr[..4].copy_from_slice(&octets); + addr +} + +#[test] +fn publication_point_cache_policy_fingerprint_includes_resource_validation_mode() { + let mut strict_policy = Policy::default(); + strict_policy.resource_validation_mode = ResourceValidationMode::Rfc6487; + let mut vrs_policy = Policy::default(); + vrs_policy.resource_validation_mode = ResourceValidationMode::ValidationUpdate03; + + assert_ne!( + publication_point_cache_policy_fingerprint(&strict_policy), + publication_point_cache_policy_fingerprint(&vrs_policy) + ); +} + +#[test] +fn router_asns_for_resource_mode_filters_vrs_and_rejects_strict_overclaim() { + let issuer_as = AsResourceSet { + asnum: Some(AsIdentifierChoice::AsIdsOrRanges(vec![ + AsIdOrRange::Range { + min: 64500, + max: 64510, + }, + ])), + rdi: None, + }; + + let strict = router_asns_for_resource_mode( + &[64505, 64520], + Some(&issuer_as), + ResourceValidationMode::Rfc6487, + ) + .unwrap_err(); + assert!(strict.contains("not a subset"), "{strict}"); + + let vrs = router_asns_for_resource_mode( + &[64505, 64520], + Some(&issuer_as), + ResourceValidationMode::ValidationUpdate03, + ) + .expect("vrs filters router asns"); + assert_eq!(vrs, vec![64505]); +} + +struct NeverHttpFetcher; +impl Fetcher for NeverHttpFetcher { + fn fetch(&self, _uri: &str) -> Result, String> { + Err("http fetch disabled in test".to_string()) + } +} + +struct FailingRsyncFetcher; +impl RsyncFetcher for FailingRsyncFetcher { + fn fetch_objects( + &self, + _rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + Err(RsyncFetchError::Fetch("rsync disabled in test".to_string())) + } +} + +fn sample_runner_with_ccr_accumulator<'a>( + store: &'a RocksStore, + policy: &'a Policy, +) -> Rpkiv1PublicationPointRunner<'a> { + Rpkiv1PublicationPointRunner { + store, + policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &FailingRsyncFetcher, + validation_time: time::OffsetDateTime::now_utc(), + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: Some(Mutex::new(CcrAccumulator::new(Vec::new()))), + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + } +} + +fn openssl_available() -> bool { + Command::new("openssl") + .arg("version") + .output() + .map(|o| o.status.success()) + .unwrap_or(false) +} + +struct Generated { + issuer_ca_der: Vec, + child_ca_der: Vec, + issuer_crl_der: Vec, + issuer_crl_der_next: Vec, +} + +fn run(cmd: &mut Command) { + let out = cmd.output().expect("run command"); + if !out.status.success() { + panic!( + "command failed: {:?}\nstdout={}\nstderr={}", + cmd, + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + } +} + +fn generate_chain_and_crl() -> Generated { + assert!(openssl_available(), "openssl is required for this test"); + + let td = tempfile::tempdir().expect("tempdir"); + let dir = td.path(); + + std::fs::create_dir_all(dir.join("newcerts")).expect("newcerts"); + std::fs::write(dir.join("index.txt"), b"").expect("index"); + std::fs::write(dir.join("serial"), b"1000\n").expect("serial"); + std::fs::write(dir.join("crlnumber"), b"1000\n").expect("crlnumber"); + + let cnf = format!( + r#" +[ ca ] +default_ca = CA_default + +[ CA_default ] +dir = {dir} +database = $dir/index.txt +new_certs_dir = $dir/newcerts +certificate = $dir/issuer.pem +private_key = $dir/issuer.key +serial = $dir/serial +crlnumber = $dir/crlnumber +default_md = sha256 +default_days = 365 +default_crl_days = 1 +policy = policy_any +x509_extensions = v3_issuer_ca +crl_extensions = crl_ext +unique_subject = no +copy_extensions = none + +[ policy_any ] +commonName = supplied + +[ req ] +prompt = no +distinguished_name = dn + +[ dn ] +CN = Test Issuer CA + +[ v3_issuer_ca ] +basicConstraints = critical,CA:true +keyUsage = critical, keyCertSign, cRLSign +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always +certificatePolicies = critical, 1.3.6.1.5.5.7.14.2 +subjectInfoAccess = caRepository;URI:rsync://example.test/repo/issuer/, rpkiManifest;URI:rsync://example.test/repo/issuer/issuer.mft, rpkiNotify;URI:https://example.test/notification.xml +sbgp-ipAddrBlock = critical, IPv4:10.0.0.0/8 +sbgp-autonomousSysNum = critical, AS:64496-64511 + +[ v3_child_ca ] +basicConstraints = critical,CA:true +keyUsage = critical, keyCertSign, cRLSign +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always +crlDistributionPoints = URI:rsync://example.test/repo/issuer/issuer.crl +authorityInfoAccess = caIssuers;URI:rsync://example.test/repo/issuer/issuer.cer +certificatePolicies = critical, 1.3.6.1.5.5.7.14.2 +subjectInfoAccess = caRepository;URI:rsync://example.test/repo/child/, rpkiManifest;URI:rsync://example.test/repo/child/child.mft, rpkiNotify;URI:https://example.test/notification.xml +sbgp-ipAddrBlock = critical, IPv4:10.0.0.0/16 +sbgp-autonomousSysNum = critical, AS:64496 + +[ crl_ext ] +authorityKeyIdentifier = keyid:always +"#, + dir = dir.display() + ); + std::fs::write(dir.join("openssl.cnf"), cnf.as_bytes()).expect("write cnf"); + + run(Command::new("openssl") + .arg("genrsa") + .arg("-out") + .arg(dir.join("issuer.key")) + .arg("2048")); + run(Command::new("openssl") + .arg("req") + .arg("-new") + .arg("-x509") + .arg("-sha256") + .arg("-days") + .arg("365") + .arg("-key") + .arg(dir.join("issuer.key")) + .arg("-config") + .arg(dir.join("openssl.cnf")) + .arg("-extensions") + .arg("v3_issuer_ca") + .arg("-out") + .arg(dir.join("issuer.pem"))); + + run(Command::new("openssl") + .arg("genrsa") + .arg("-out") + .arg(dir.join("child.key")) + .arg("2048")); + run(Command::new("openssl") + .arg("req") + .arg("-new") + .arg("-key") + .arg(dir.join("child.key")) + .arg("-subj") + .arg("/CN=Test Child CA") + .arg("-out") + .arg(dir.join("child.csr"))); + + run(Command::new("openssl") + .arg("ca") + .arg("-batch") + .arg("-config") + .arg(dir.join("openssl.cnf")) + .arg("-in") + .arg(dir.join("child.csr")) + .arg("-extensions") + .arg("v3_child_ca") + .arg("-out") + .arg(dir.join("child.pem"))); + + run(Command::new("openssl") + .arg("x509") + .arg("-in") + .arg(dir.join("issuer.pem")) + .arg("-outform") + .arg("DER") + .arg("-out") + .arg(dir.join("issuer.cer"))); + run(Command::new("openssl") + .arg("x509") + .arg("-in") + .arg(dir.join("child.pem")) + .arg("-outform") + .arg("DER") + .arg("-out") + .arg(dir.join("child.cer"))); + + run(Command::new("openssl") + .arg("ca") + .arg("-gencrl") + .arg("-config") + .arg(dir.join("openssl.cnf")) + .arg("-out") + .arg(dir.join("issuer.crl.pem"))); + run(Command::new("openssl") + .arg("crl") + .arg("-in") + .arg(dir.join("issuer.crl.pem")) + .arg("-outform") + .arg("DER") + .arg("-out") + .arg(dir.join("issuer.crl"))); + run(Command::new("openssl") + .arg("ca") + .arg("-gencrl") + .arg("-config") + .arg(dir.join("openssl.cnf")) + .arg("-out") + .arg(dir.join("issuer-next.crl.pem"))); + run(Command::new("openssl") + .arg("crl") + .arg("-in") + .arg(dir.join("issuer-next.crl.pem")) + .arg("-outform") + .arg("DER") + .arg("-out") + .arg(dir.join("issuer-next.crl"))); + + Generated { + issuer_ca_der: std::fs::read(dir.join("issuer.cer")).expect("read issuer der"), + child_ca_der: std::fs::read(dir.join("child.cer")).expect("read child der"), + issuer_crl_der: std::fs::read(dir.join("issuer.crl")).expect("read crl der"), + issuer_crl_der_next: std::fs::read(dir.join("issuer-next.crl")).expect("read next crl der"), + } +} + +struct GeneratedRouter { + issuer_ca_der: Vec, + router_der: Vec, + issuer_crl_der: Vec, +} + +fn generate_router_cert_with_variant(key_spec: &str, include_eku: bool) -> GeneratedRouter { + assert!(openssl_available(), "openssl is required for this test"); + + let td = tempfile::tempdir().expect("tempdir"); + let dir = td.path(); + + std::fs::create_dir_all(dir.join("newcerts")).expect("newcerts"); + std::fs::write(dir.join("index.txt"), b"").expect("index"); + std::fs::write(dir.join("serial"), b"1000\n").expect("serial"); + std::fs::write(dir.join("crlnumber"), b"1000\n").expect("crlnumber"); + + let eku_line = if include_eku { + "extendedKeyUsage = 1.3.6.1.5.5.7.3.30" + } else { + "" + }; + let cnf = format!( + r#" +[ ca ] +default_ca = CA_default + +[ CA_default ] +dir = {dir} +database = $dir/index.txt +new_certs_dir = $dir/newcerts +certificate = $dir/issuer.pem +private_key = $dir/issuer.key +serial = $dir/serial +crlnumber = $dir/crlnumber +default_md = sha256 +default_days = 365 +default_crl_days = 1 +policy = policy_any +x509_extensions = v3_issuer_ca +crl_extensions = crl_ext +unique_subject = no +copy_extensions = none + +[ policy_any ] +commonName = supplied + +[ req ] +prompt = no +distinguished_name = dn + +[ dn ] +CN = Test Issuer CA + +[ v3_issuer_ca ] +basicConstraints = critical,CA:true +keyUsage = critical, keyCertSign, cRLSign +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always +certificatePolicies = critical, 1.3.6.1.5.5.7.14.2 +subjectInfoAccess = caRepository;URI:rsync://example.test/repo/issuer/, rpkiManifest;URI:rsync://example.test/repo/issuer/issuer.mft, rpkiNotify;URI:https://example.test/notification.xml +sbgp-ipAddrBlock = critical, IPv4:10.0.0.0/8 +sbgp-autonomousSysNum = critical, AS:64496-64511 + +[ v3_router ] +keyUsage = critical, digitalSignature +{eku_line} +authorityKeyIdentifier = keyid:always +crlDistributionPoints = URI:rsync://example.test/repo/issuer/issuer.crl +authorityInfoAccess = caIssuers;URI:rsync://example.test/repo/issuer/issuer.cer +certificatePolicies = critical, 1.3.6.1.5.5.7.14.2 +sbgp-autonomousSysNum = critical, AS:64496 + +[ crl_ext ] +authorityKeyIdentifier = keyid:always +"#, + dir = dir.display(), + eku_line = eku_line, + ); + std::fs::write(dir.join("openssl.cnf"), cnf.as_bytes()).expect("write cnf"); + + run(Command::new("openssl") + .arg("genrsa") + .arg("-out") + .arg(dir.join("issuer.key")) + .arg("2048")); + run(Command::new("openssl") + .arg("req") + .arg("-new") + .arg("-x509") + .arg("-sha256") + .arg("-days") + .arg("365") + .arg("-key") + .arg(dir.join("issuer.key")) + .arg("-config") + .arg(dir.join("openssl.cnf")) + .arg("-extensions") + .arg("v3_issuer_ca") + .arg("-out") + .arg(dir.join("issuer.pem"))); + + match key_spec { + "ec-p256" => run(Command::new("openssl") + .arg("ecparam") + .arg("-name") + .arg("prime256v1") + .arg("-genkey") + .arg("-noout") + .arg("-out") + .arg(dir.join("router.key"))), + "ec-p384" => run(Command::new("openssl") + .arg("ecparam") + .arg("-name") + .arg("secp384r1") + .arg("-genkey") + .arg("-noout") + .arg("-out") + .arg(dir.join("router.key"))), + other => panic!("unsupported key_spec {other}"), + } + + run(Command::new("openssl") + .arg("req") + .arg("-new") + .arg("-key") + .arg(dir.join("router.key")) + .arg("-subj") + .arg("/CN=ROUTER-0000FC10/serialNumber=01020304") + .arg("-out") + .arg(dir.join("router.csr"))); + + run(Command::new("openssl") + .arg("ca") + .arg("-batch") + .arg("-config") + .arg(dir.join("openssl.cnf")) + .arg("-in") + .arg(dir.join("router.csr")) + .arg("-extensions") + .arg("v3_router") + .arg("-out") + .arg(dir.join("router.pem"))); + + run(Command::new("openssl") + .arg("x509") + .arg("-in") + .arg(dir.join("issuer.pem")) + .arg("-outform") + .arg("DER") + .arg("-out") + .arg(dir.join("issuer.cer"))); + run(Command::new("openssl") + .arg("x509") + .arg("-in") + .arg(dir.join("router.pem")) + .arg("-outform") + .arg("DER") + .arg("-out") + .arg(dir.join("router.cer"))); + + run(Command::new("openssl") + .arg("ca") + .arg("-gencrl") + .arg("-config") + .arg(dir.join("openssl.cnf")) + .arg("-out") + .arg(dir.join("issuer.crl.pem"))); + run(Command::new("openssl") + .arg("crl") + .arg("-in") + .arg(dir.join("issuer.crl.pem")) + .arg("-outform") + .arg("DER") + .arg("-out") + .arg(dir.join("issuer.crl"))); + + GeneratedRouter { + issuer_ca_der: std::fs::read(dir.join("issuer.cer")).expect("read issuer der"), + router_der: std::fs::read(dir.join("router.cer")).expect("read router der"), + issuer_crl_der: std::fs::read(dir.join("issuer.crl")).expect("read crl der"), + } +} +fn dummy_pack_with_files(files: Vec) -> PublicationPointSnapshot { + let now = time::OffsetDateTime::now_utc(); + PublicationPointSnapshot { + format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_number_be: vec![1], + this_update: PackTime::from_utc_offset_datetime(now), + next_update: PackTime::from_utc_offset_datetime(now + time::Duration::hours(1)), + verified_at: PackTime::from_utc_offset_datetime(now), + manifest_bytes: vec![0x01], + files, + } +} + +fn cernet_publication_point_snapshot_for_vcir_tests() +-> (PublicationPointSnapshot, Vec, time::OffsetDateTime) { + let dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/"; + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); + let manifest_bytes = std::fs::read(dir.join(manifest_file)).expect("read manifest fixture"); + let manifest = ManifestObject::decode_der(&manifest_bytes).expect("decode manifest fixture"); + let candidate = manifest.manifest.this_update + time::Duration::seconds(60); + let validation_time = if candidate < manifest.manifest.next_update { + candidate + } else { + manifest.manifest.this_update + }; + + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + + sync_publication_point( + &store, + &policy, + None, + rsync_base_uri, + &NeverHttpFetcher, + &LocalDirRsyncFetcher::new(&dir), + None, + None, + ) + .expect("sync cernet fixture"); + + let pp = crate::validation::manifest::process_manifest_publication_point( + &store, + &policy, + &manifest_rsync_uri, + rsync_base_uri, + issuer_ca_der.as_slice(), + Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + validation_time, + ) + .expect("process manifest publication point"); + + (pp.snapshot, issuer_ca_der, validation_time) +} + +fn sample_vcir_for_projection( + now: time::OffsetDateTime, + child_cert_hash: &str, +) -> ValidatedCaInstanceResult { + let manifest_uri = "rsync://example.test/repo/issuer/issuer.mft".to_string(); + let current_crl_uri = "rsync://example.test/repo/issuer/issuer.crl".to_string(); + let child_cert_uri = "rsync://example.test/repo/issuer/child.cer".to_string(); + let child_manifest_uri = "rsync://example.test/repo/child/child.mft".to_string(); + let roa_uri = "rsync://example.test/repo/issuer/a.roa".to_string(); + let aspa_uri = "rsync://example.test/repo/issuer/a.asa".to_string(); + let router_uri = "rsync://example.test/repo/issuer/router.cer".to_string(); + let manifest_hash = sha256_hex(b"manifest-bytes"); + let current_crl_hash = sha256_hex(b"current-crl-bytes"); + let roa_hash = sha256_hex(b"roa-bytes"); + let aspa_hash = sha256_hex(b"aspa-bytes"); + let router_hash = sha256_hex(b"router-bytes"); + let ee_hash = sha256_hex(b"ee-cert-bytes"); + let gate_until = PackTime::from_utc_offset_datetime(now + time::Duration::hours(1)); + let ccr_manifest_projection = VcirCcrManifestProjection { + manifest_rsync_uri: manifest_uri.clone(), + manifest_sha256: hex::decode(&manifest_hash).expect("decode manifest hash"), + manifest_size: 2048, + manifest_ee_aki: vec![0x11; 20], + manifest_number_be: vec![1], + manifest_this_update: PackTime::from_utc_offset_datetime(now), + manifest_sia_locations_der: vec![ + crate::ccr::manifest_location::encode_access_description_der(&AccessDescription { + access_method_oid: OID_AD_SIGNED_OBJECT.to_string(), + access_location: manifest_uri.clone(), + }) + .expect("encode signedObject"), + ], + subordinate_skis: vec![vec![0x33; 20]], + }; + ValidatedCaInstanceResult { + manifest_rsync_uri: manifest_uri.clone(), + parent_manifest_rsync_uri: None, + tal_id: "test-tal".to_string(), + ca_subject_name: "CN=Issuer".to_string(), + ca_ski: "11".repeat(20), + issuer_ski: "22".repeat(20), + last_successful_validation_time: PackTime::from_utc_offset_datetime(now), + current_manifest_rsync_uri: manifest_uri.clone(), + current_crl_rsync_uri: current_crl_uri.clone(), + validated_manifest_meta: ValidatedManifestMeta { + validated_manifest_number: vec![1], + validated_manifest_this_update: PackTime::from_utc_offset_datetime(now), + validated_manifest_next_update: gate_until.clone(), + }, + ccr_manifest_projection, + instance_gate: VcirInstanceGate { + manifest_next_update: gate_until.clone(), + current_crl_next_update: gate_until.clone(), + self_ca_not_after: PackTime::from_utc_offset_datetime(now + time::Duration::hours(2)), + instance_effective_until: gate_until.clone(), + }, + child_entries: vec![VcirChildEntry { + child_manifest_rsync_uri: child_manifest_uri, + child_cert_rsync_uri: child_cert_uri.clone(), + child_cert_hash: child_cert_hash.to_string(), + child_ski: "33".repeat(20), + child_rsync_base_uri: "rsync://example.test/repo/child/".to_string(), + child_publication_point_rsync_uri: "rsync://example.test/repo/child/".to_string(), + child_rrdp_notification_uri: Some("https://example.test/child-notify.xml".to_string()), + child_effective_ip_resources: None, + child_effective_as_resources: None, + accepted_at_validation_time: PackTime::from_utc_offset_datetime(now), + }], + local_outputs: vec![ + VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: PackTime::from_utc_offset_datetime( + now + time::Duration::minutes(30), + ), + source_object_uri: roa_uri.clone(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: sha256_hex_to_32(&roa_hash), + source_ee_cert_hash: sha256_hex_to_32(&ee_hash), + payload: VcirLocalOutputPayload::Vrp { + asn: 64496, + afi: RoaAfi::Ipv4, + prefix_len: 24, + addr: ipv4_addr([203, 0, 113, 0]), + max_length: 24, + }, + rule_hash: sha256_32(b"roa-rule"), + }, + VcirLocalOutput { + output_type: VcirOutputType::Aspa, + item_effective_until: PackTime::from_utc_offset_datetime( + now + time::Duration::minutes(30), + ), + source_object_uri: aspa_uri.clone(), + source_object_type: VcirSourceObjectType::Aspa, + source_object_hash: sha256_hex_to_32(&aspa_hash), + source_ee_cert_hash: sha256_hex_to_32(&ee_hash), + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: 64496, + provider_as_ids: vec![64497, 64498], + }, + rule_hash: sha256_32(b"aspa-rule"), + }, + VcirLocalOutput { + output_type: VcirOutputType::RouterKey, + item_effective_until: PackTime::from_utc_offset_datetime( + now + time::Duration::minutes(30), + ), + source_object_uri: router_uri.clone(), + source_object_type: VcirSourceObjectType::RouterKey, + source_object_hash: sha256_hex_to_32(&router_hash), + source_ee_cert_hash: sha256_hex_to_32(&router_hash), + payload: VcirLocalOutputPayload::RouterKey { + as_id: 64496, + ski: vec![0x11; 20], + spki_der: vec![0x30, 0x00], + }, + rule_hash: sha256_32(b"router-key-rule"), + }, + ], + related_artifacts: vec![ + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::Manifest, + artifact_kind: VcirArtifactKind::Mft, + uri: Some(manifest_uri.clone()), + sha256: manifest_hash, + object_type: Some("mft".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::CurrentCrl, + artifact_kind: VcirArtifactKind::Crl, + uri: Some(current_crl_uri), + sha256: current_crl_hash, + object_type: Some("crl".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::ChildCaCert, + artifact_kind: VcirArtifactKind::Cer, + uri: Some(child_cert_uri), + sha256: child_cert_hash.to_string(), + object_type: Some("cer".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: VcirArtifactKind::Roa, + uri: Some(roa_uri), + sha256: roa_hash, + object_type: Some("roa".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }, + VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: VcirArtifactKind::Aspa, + uri: Some(aspa_uri), + sha256: aspa_hash, + object_type: Some("aspa".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }, + ], + summary: VcirSummary { + local_vrp_count: 1, + local_aspa_count: 1, + local_router_key_count: 1, + child_count: 1, + accepted_object_count: 4, + rejected_object_count: 0, + }, + audit_summary: VcirAuditSummary { + failed_fetch_eligible: true, + last_failed_fetch_reason: None, + warning_count: 0, + audit_flags: Vec::new(), + }, + } +} + +fn put_vcir_for_failed_fetch_reuse( + store: &RocksStore, + ca: &CaInstanceHandle, + policy: &Policy, + vcir: &ValidatedCaInstanceResult, +) { + let validation_time = vcir + .last_successful_validation_time + .parse() + .expect("parse VCIR validation time"); + let identity = failed_fetch_reuse_identity_for_fresh_result( + ca, + policy, + validation_time, + vcir.instance_gate.instance_effective_until.clone(), + ) + .expect("build VCIR failed-fetch reuse identity"); + store + .put_vcir_with_failed_fetch_reuse_identity(vcir, &identity) + .expect("put reusable VCIR"); +} + +fn sample_ca_for_failed_fetch_reuse(vcir: &ValidatedCaInstanceResult) -> CaInstanceHandle { + CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), + } +} + +#[test] +fn never_http_fetcher_returns_error() { + let f = NeverHttpFetcher; + let err = f.fetch("https://example.test/").unwrap_err(); + assert!(err.contains("disabled"), "{err}"); +} + +#[test] +fn kind_from_rsync_uri_classifies_known_extensions() { + assert_eq!( + kind_from_rsync_uri("rsync://example.test/x.crl"), + AuditObjectKind::Crl + ); + assert_eq!( + kind_from_rsync_uri("rsync://example.test/x.cer"), + AuditObjectKind::Certificate + ); + assert_eq!( + kind_from_rsync_uri("rsync://example.test/x.roa"), + AuditObjectKind::Roa + ); + assert_eq!( + kind_from_rsync_uri("rsync://example.test/x.asa"), + AuditObjectKind::Aspa + ); + assert_eq!( + kind_from_rsync_uri("rsync://example.test/x.bin"), + AuditObjectKind::Other + ); +} + +#[test] +fn build_vcir_local_outputs_prefers_cached_outputs() { + let pack = dummy_pack_with_files(vec![]); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(vec![1]), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + let cached = vec![VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: pack.next_update.clone(), + source_object_uri: "rsync://example.test/repo/issuer/a.roa".to_string(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: sha256_32(b"cached-roa"), + source_ee_cert_hash: sha256_32(b"cached-ee"), + payload: VcirLocalOutputPayload::Vrp { + asn: 64500, + afi: RoaAfi::Ipv4, + prefix_len: 24, + addr: ipv4_addr([203, 0, 113, 0]), + max_length: 24, + }, + rule_hash: sha256_32(b"cached-rule"), + }]; + let mut objects = crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: cached.clone(), + warnings: Vec::new(), + stats: crate::validation::objects::ObjectsStats::default(), + audit: Vec::new(), + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + }; + let outputs = + take_or_build_vcir_local_outputs(&ca, &pack, &mut objects).expect("reuse cached outputs"); + assert_eq!(outputs, cached); + assert!(objects.local_outputs_cache.is_empty()); +} + +#[test] +fn persist_vcir_non_repository_evidence_stores_current_ca_cert_only() { + let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests(); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + let objects = crate::validation::objects::process_publication_point_snapshot_for_issuer( + &pack, + &Policy::default(), + issuer_ca_der.as_slice(), + Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + issuer_ca.tbs.extensions.ip_resources.as_ref(), + issuer_ca.tbs.extensions.as_resources.as_ref(), + validation_time, + None, + ); + assert!( + !objects.local_outputs_cache.is_empty(), + "expected local outputs from signed objects" + ); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der.clone()), + ca_certificate_rsync_uri: Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(), + ), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + persist_vcir_non_repository_evidence(&store, &ca).expect("persist embedded evidence"); + + let issuer_hash = sha256_hex(&issuer_ca_der); + let issuer_entry = store + .get_raw_by_hash_entry(&issuer_hash) + .expect("load issuer raw entry") + .expect("issuer raw entry present"); + assert!( + issuer_entry + .origin_uris + .iter() + .any(|uri| uri.ends_with("BfycW4hQb3wNP4YsiJW-1n6fjro.cer")) + ); + let first_output = objects + .local_outputs_cache + .first() + .expect("first local output"); + assert!( + store + .get_raw_by_hash_entry(&first_output.source_ee_cert_hash_hex()) + .expect("load source ee raw") + .is_none() + ); +} + +#[test] +fn build_router_key_local_outputs_encodes_router_key_payloads() { + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + let outputs = build_router_key_local_outputs( + &ca, + &[RouterKeyPayload { + as_id: 64496, + ski: vec![0x11; 20], + spki_der: vec![0x30, 0x00], + source_object_uri: "rsync://example.test/repo/issuer/router.cer".to_string(), + source_object_hash: "11".repeat(32), + source_ee_cert_hash: "11".repeat(32), + item_effective_until: PackTime { + rfc3339_utc: "2026-12-31T00:00:00Z".to_string(), + }, + }], + ); + assert_eq!(outputs.len(), 1); + assert_eq!(outputs[0].output_type, VcirOutputType::RouterKey); + assert_eq!( + outputs[0].source_object_type, + VcirSourceObjectType::RouterKey + ); + assert!(outputs[0].payload_json().contains("spki_der_base64")); +} + +#[test] +fn build_vcir_local_outputs_falls_back_to_decoding_accepted_objects_when_cache_is_empty() { + let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests(); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + let objects = crate::validation::objects::process_publication_point_snapshot_for_issuer( + &pack, + &Policy::default(), + issuer_ca_der.as_slice(), + Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + issuer_ca.tbs.extensions.ip_resources.as_ref(), + issuer_ca.tbs.extensions.as_resources.as_ref(), + validation_time, + None, + ); + let mut objects_without_cache = objects.clone(); + objects_without_cache.local_outputs_cache.clear(); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), + ca_certificate_rsync_uri: Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(), + ), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + let local_outputs = build_vcir_local_outputs(&ca, &pack, &objects_without_cache) + .expect("rebuild vcir local outputs"); + assert!(!local_outputs.is_empty()); + assert_eq!(local_outputs.len(), objects.vrps.len()); + assert!( + local_outputs + .iter() + .all(|output| output.output_type == VcirOutputType::Vrp) + ); +} + +#[test] +fn finalize_fresh_publication_point_releases_local_outputs_cache_after_persist() { + let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests(); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + let mut objects = crate::validation::objects::process_publication_point_snapshot_for_issuer( + &pack, + &Policy::default(), + issuer_ca_der.as_slice(), + Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + issuer_ca.tbs.extensions.ip_resources.as_ref(), + issuer_ca.tbs.extensions.as_resources.as_ref(), + validation_time, + None, + ); + assert!( + !objects.local_outputs_cache.is_empty(), + "expected local outputs from signed objects" + ); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &FailingRsyncFetcher, + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + }; + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der.clone()), + ca_certificate_rsync_uri: Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(), + ), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + let fresh_point = FreshValidatedPublicationPoint { + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + manifest_number_be: pack.manifest_number_be.clone(), + this_update: pack.this_update.clone(), + next_update: pack.next_update.clone(), + verified_at: pack.verified_at.clone(), + manifest_bytes: pack.manifest_bytes.clone(), + files: pack.files.clone(), + }; + + objects.local_outputs_cache.shrink_to_fit(); + let original_cache_capacity = objects.local_outputs_cache.capacity(); + let finalized = runner + .finalize_fresh_publication_point_from_reducer( + &ca, + &fresh_point, + Vec::new(), + objects, + Vec::new(), + Vec::new(), + None, + None, + 0, + None, + ) + .expect("finalize fresh publication point"); + + assert!( + finalized.result.objects.local_outputs_cache.is_empty(), + "local outputs cache should be released after VCIR persistence" + ); + assert_eq!( + finalized.result.objects.local_outputs_cache.capacity(), + 0, + "released cache should not keep its backing allocation" + ); + assert!(original_cache_capacity > 0); + + let persisted = store + .get_vcir(&pack.manifest_rsync_uri) + .expect("load persisted vcir") + .expect("persisted vcir"); + assert!( + !persisted.local_outputs.is_empty(), + "VCIR should still persist local outputs before cache release" + ); +} + +#[test] +fn persist_vcir_for_fresh_result_stores_vcir_and_replay_meta_for_real_snapshot() { + let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests(); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + let objects = crate::validation::objects::process_publication_point_snapshot_for_issuer( + &pack, + &Policy::default(), + issuer_ca_der.as_slice(), + Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + issuer_ca.tbs.extensions.ip_resources.as_ref(), + issuer_ca.tbs.extensions.as_resources.as_ref(), + validation_time, + None, + ); + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der.clone()), + ca_certificate_rsync_uri: Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(), + ), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + + let mut objects = objects; + persist_vcir_for_fresh_result_with_timing( + &store, + &Policy::default(), + &ca, + &pack, + &mut objects, + &[], + &[], + &[], + validation_time, + false, + ) + .map(|_timing| ()) + .expect("persist vcir for fresh result"); + + let vcir = store + .get_vcir(&pack.manifest_rsync_uri) + .expect("get vcir") + .expect("vcir exists"); + assert_eq!(vcir.manifest_rsync_uri, pack.manifest_rsync_uri); + assert_eq!(vcir.summary.local_vrp_count as usize, objects.vrps.len()); + assert_eq!( + vcir.ccr_manifest_projection.manifest_rsync_uri, + pack.manifest_rsync_uri + ); + assert_eq!( + vcir.ccr_manifest_projection.manifest_number_be, + pack.manifest_number_be + ); + assert_eq!( + vcir.ccr_manifest_projection.manifest_this_update, + pack.this_update + ); + assert_eq!( + vcir.ccr_manifest_projection.manifest_size, + pack.manifest_bytes.len() as u64 + ); + assert!(vcir.local_outputs.first().is_some(), "local outputs stored"); + let replay_meta = store + .get_manifest_replay_meta(&pack.manifest_rsync_uri) + .expect("get replay meta") + .expect("replay meta exists"); + assert_eq!(replay_meta.manifest_rsync_uri, pack.manifest_rsync_uri); + assert_eq!( + replay_meta.manifest_sha256, + sha2::Sha256::digest(&pack.manifest_bytes).to_vec() + ); +} + +#[test] +fn build_vcir_ccr_manifest_projection_from_fresh_real_snapshot_matches_manifest_contents() { + let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests(); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), + ca_certificate_rsync_uri: Some( + "rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string(), + ), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + let child_discovery = + discover_children_from_fresh_snapshot_with_audit(&ca, &pack, validation_time, None) + .expect("discover children"); + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let child_entries = + build_vcir_child_entries(&store, &child_discovery.children, validation_time) + .expect("build child entries"); + + let projection = build_vcir_ccr_manifest_projection_from_fresh(&ca, &pack, &child_entries) + .expect("build ccr manifest projection"); + let manifest = ManifestObject::decode_der(&pack.manifest_bytes).expect("decode manifest"); + let expected_locations = match manifest.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .extensions + .subject_info_access + .as_ref() + .expect("manifest sia") + { + SubjectInfoAccess::Ee(ee_sia) => vec![ + crate::ccr::manifest_location::select_manifest_signed_object_location( + &pack.manifest_rsync_uri, + &ee_sia.access_descriptions, + ) + .expect("select manifest signedObject"), + ], + SubjectInfoAccess::Ca(_) => panic!("manifest ee SIA should not be CA variant"), + }; + + assert_eq!(projection.manifest_rsync_uri, pack.manifest_rsync_uri); + assert_eq!( + projection.manifest_sha256, + sha2::Sha256::digest(&pack.manifest_bytes).to_vec() + ); + assert_eq!(projection.manifest_size, pack.manifest_bytes.len() as u64); + assert_eq!( + projection.manifest_ee_aki, + manifest.signed_object.signed_data.certificates[0] + .resource_cert + .tbs + .extensions + .authority_key_identifier + .clone() + .expect("manifest aki") + ); + assert_eq!( + projection.manifest_number_be, + manifest.manifest.manifest_number.bytes_be + ); + assert_eq!(projection.manifest_this_update, pack.this_update); + assert_eq!(projection.manifest_sia_locations_der, expected_locations); + let expected_subordinate_skis = child_entries + .iter() + .map(|child| hex::decode(&child.child_ski).expect("decode child ski")) + .collect::>(); + assert_eq!(projection.subordinate_skis, expected_subordinate_skis); +} + +#[test] +fn build_vcir_child_entries_uses_projection_without_repo_bytes() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let validation_time = time::OffsetDateTime::parse( + "2026-06-26T00:00:00Z", + &time::format_description::well_known::Rfc3339, + ) + .expect("parse time"); + let child = DiscoveredChildCaInstance { + handle: CaInstanceHandle { + depth: 1, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: Some("rsync://example.test/repo/root.mft".to_string()), + ca_certificate: CaCertificateRef::repo_bytes("aa".repeat(32)), + ca_certificate_rsync_uri: Some("rsync://example.test/repo/child.cer".to_string()), + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/child/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/child/child.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/child/".to_string(), + rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), + }, + discovered_from: crate::audit::DiscoveredFrom { + parent_manifest_rsync_uri: "rsync://example.test/repo/root.mft".to_string(), + child_ca_certificate_rsync_uri: "rsync://example.test/repo/child.cer".to_string(), + child_ca_certificate_sha256_hex: "aa".repeat(32), + }, + child_entry_projection: Some(DiscoveredChildEntryProjection { + child_ski: "11".repeat(20), + }), + }; + + let entries = build_vcir_child_entries(&store, &[child], validation_time) + .expect("projection should avoid repo-bytes load"); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].child_ski, "11".repeat(20)); +} + +#[test] +fn build_vcir_related_artifacts_classifies_snapshot_files_and_audit_statuses() { + let manifest_bytes = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft", + ), + ) + .expect("read manifest fixture"); + let crl_bytes = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.crl", + ), + ) + .expect("read crl fixture"); + let pack = PublicationPointSnapshot { + format_version: PublicationPointSnapshot::FORMAT_VERSION_V1, + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_number_be: vec![1], + this_update: PackTime::from_utc_offset_datetime(time::OffsetDateTime::now_utc()), + next_update: PackTime::from_utc_offset_datetime( + time::OffsetDateTime::now_utc() + time::Duration::hours(1), + ), + verified_at: PackTime::from_utc_offset_datetime(time::OffsetDateTime::now_utc()), + manifest_bytes, + files: vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + crl_bytes, + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/child.cer", + vec![1u8, 2], + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/a.roa", + vec![3u8, 4], + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/a.asa", + vec![5u8, 6], + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/a.gbr", + vec![7u8, 8], + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/extra.bin", + vec![9u8], + ), + ], + }; + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(vec![0x11, 0x22]), + ca_certificate_rsync_uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()), + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let objects = crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: Vec::new(), + stats: crate::validation::objects::ObjectsStats::default(), + audit: vec![ + ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/issuer/a.roa".to_string(), + sha256_hex: sha256_hex_from_32(&pack.files[2].sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Error, + detail: Some("bad roa".to_string()), + }, + ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/issuer/a.asa".to_string(), + sha256_hex: sha256_hex_from_32(&pack.files[3].sha256), + kind: AuditObjectKind::Aspa, + result: AuditObjectResult::Skipped, + detail: Some("skipped aspa".to_string()), + }, + ], + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + }; + let artifacts = build_vcir_related_artifacts( + &store, + &ca, + &pack, + "rsync://example.test/repo/issuer/issuer.crl", + &objects, + &[], + ); + assert!( + artifacts + .iter() + .any(|artifact| artifact.artifact_role == VcirArtifactRole::Manifest) + ); + assert!( + artifacts + .iter() + .any(|artifact| artifact.artifact_role == VcirArtifactRole::TrustAnchorCert) + ); + assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() + == Some("rsync://example.test/repo/issuer/issuer.crl") + && artifact.artifact_role == VcirArtifactRole::CurrentCrl)); + assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() + == Some("rsync://example.test/repo/issuer/child.cer") + && artifact.artifact_role == VcirArtifactRole::ChildCaCert)); + assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() + == Some("rsync://example.test/repo/issuer/a.roa") + && artifact.validation_status == VcirArtifactValidationStatus::Rejected + && artifact.reject_reason.as_deref() == Some("bad roa"))); + assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() + == Some("rsync://example.test/repo/issuer/a.asa") + && artifact.validation_status == VcirArtifactValidationStatus::WarningOnly + && artifact.reject_reason.is_none())); + assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() + == Some("rsync://example.test/repo/issuer/a.gbr") + && artifact.artifact_kind == VcirArtifactKind::Gbr)); + assert!(artifacts.iter().any(|artifact| artifact.uri.as_deref() + == Some("rsync://example.test/repo/issuer/extra.bin") + && artifact.artifact_kind == VcirArtifactKind::Other)); + assert!( + !artifacts + .iter() + .any(|artifact| artifact.uri.is_none() + && artifact.sha256 == sha256_hex(b"embedded-ee")), + "embedded EE cert artifacts should no longer be persisted separately" + ); +} + +#[test] +fn select_issuer_crl_from_snapshot_reports_missing_crldp_for_self_signed_cert() { + let ta_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/ta/apnic-ta.cer"), + ) + .expect("read TA fixture"); + + let pack = dummy_pack_with_files(vec![]); + let err = select_issuer_crl_from_snapshot(&ta_der, &pack).unwrap_err(); + assert!(err.contains("CRLDistributionPoints missing"), "{err}"); +} + +#[test] +fn select_issuer_crl_from_snapshot_finds_matching_crl() { + // Use real fixtures to ensure child cert has CRLDP rsync URI and CRL exists. + let child_cert_der = + std::fs::read(std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/R-lVU1XGsAeqzV1Fv0HjOD6ZFkE.cer", + )) + .expect("read child cert fixture"); + let crl_der = + std::fs::read(std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.crl", + )) + .expect("read crl fixture"); + + let pack = dummy_pack_with_files(vec![PackFile::from_bytes_compute_sha256( + "rsync://ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.crl", + crl_der.clone(), + )]); + + let (uri, found) = + select_issuer_crl_from_snapshot(child_cert_der.as_slice(), &pack).expect("find crl"); + assert_eq!( + uri, + "rsync://ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.crl" + ); + assert_eq!(found, crl_der.as_slice()); +} + +#[test] +fn discover_children_from_fresh_pack_discovers_child_ca() { + let g = generate_chain_and_crl(); + + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/child.cer", + g.child_ca_der.clone(), + ), + ]); + + let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), + ca_certificate_rsync_uri: None, + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let now = time::OffsetDateTime::now_utc(); + let children = discover_children_from_fresh_snapshot_with_audit(&issuer, &pack, now, None) + .expect("discover children") + .children; + assert_eq!(children.len(), 1); + assert_eq!( + children[0].discovered_from.parent_manifest_rsync_uri, + issuer.manifest_rsync_uri + ); + assert_eq!( + children[0].discovered_from.child_ca_certificate_rsync_uri, + "rsync://example.test/repo/issuer/child.cer" + ); + assert_eq!( + children[0].handle.rsync_base_uri, + "rsync://example.test/repo/child/".to_string() + ); + assert_eq!( + children[0].handle.manifest_rsync_uri, + "rsync://example.test/repo/child/child.mft".to_string() + ); + assert_eq!( + children[0].handle.publication_point_rsync_uri, + "rsync://example.test/repo/child/".to_string() + ); + assert_eq!( + children[0].handle.rrdp_notification_uri.as_deref(), + Some("https://example.test/notification.xml") + ); +} + +#[test] +fn discover_children_child_certificate_cache_reuses_successful_child_ca() { + let g = generate_chain_and_crl(); + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/child.cer", + g.child_ca_der.clone(), + ), + ]); + let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), + ca_certificate_rsync_uri: None, + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let cache_context = ChildCertificateValidationCacheContext { + store: &store, + issuer_ca_sha256: issuer.ca_certificate_sha256_32().unwrap(), + ca_validation_context_digest: ca_validation_context_digest_for_ca(&issuer), + policy_fingerprint: publication_point_cache_policy_fingerprint(&policy), + }; + let validation_time = time::OffsetDateTime::now_utc(); + + let first = discover_children_from_fresh_snapshot_with_audit_cached( + &issuer, + &pack, + validation_time, + None, + Some(cache_context), + ) + .expect("first discovery writes cache"); + assert_eq!(first.children.len(), 1); + + let child_file = pack + .files + .iter() + .find(|file| file.rsync_uri.ends_with("child.cer")) + .expect("child file"); + let cache_key = child_certificate_cache_key_sha256_hex( + &child_file.rsync_uri, + &child_file.sha256, + &cache_context.issuer_ca_sha256, + &cache_context.ca_validation_context_digest, + &cache_context.policy_fingerprint, + ); + assert!( + store + .get_child_certificate_cache_projection(&cache_key) + .expect("get projection") + .is_some() + ); + + let timing = TimingHandle::new(crate::analysis::timing::TimingMeta { + recorded_at_utc_rfc3339: "2026-06-24T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-06-24T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + let second = discover_children_from_fresh_snapshot_with_audit_cached( + &issuer, + &pack, + validation_time, + Some(&timing), + Some(cache_context), + ) + .expect("second discovery reuses cache"); + assert_eq!(second.children.len(), 1); + assert_eq!( + second.children[0].handle.manifest_rsync_uri, + first.children[0].handle.manifest_rsync_uri + ); + assert!(second.audits.iter().any(|audit| { + audit + .detail + .as_deref() + .unwrap_or("") + .contains("child certificate validation cache") + })); + let counts = timing.counts_snapshot(); + assert_eq!( + counts.get("child_certificate_cache_hit_ca").copied(), + Some(1) + ); + assert_eq!( + counts + .get("child_certificate_cache_batch_lookup_publication_points") + .copied(), + Some(1) + ); + assert_eq!( + counts + .get("child_certificate_cache_batch_lookup_entries") + .copied(), + Some(1) + ); + assert_eq!( + counts + .get("child_certificate_der_load_fresh_count") + .copied(), + Some(0) + ); +} + +#[test] +fn discover_children_child_certificate_cache_rechecks_changed_valid_crl() { + let g = generate_chain_and_crl(); + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/child.cer", + g.child_ca_der.clone(), + ), + ]); + let mut changed_pack = pack.clone(); + changed_pack.files[0] = PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der_next.clone(), + ); + let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), + ca_certificate_rsync_uri: None, + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let cache_context = ChildCertificateValidationCacheContext { + store: &store, + issuer_ca_sha256: issuer.ca_certificate_sha256_32().unwrap(), + ca_validation_context_digest: ca_validation_context_digest_for_ca(&issuer), + policy_fingerprint: publication_point_cache_policy_fingerprint(&policy), + }; + let validation_time = time::OffsetDateTime::now_utc(); + discover_children_from_fresh_snapshot_with_audit_cached( + &issuer, + &pack, + validation_time, + None, + Some(cache_context), + ) + .expect("populate cache"); + let child_file = pack + .files + .iter() + .find(|file| file.rsync_uri.ends_with("child.cer")) + .expect("child file"); + let cache_key = child_certificate_cache_key_sha256_hex( + &child_file.rsync_uri, + &child_file.sha256, + &cache_context.issuer_ca_sha256, + &cache_context.ca_validation_context_digest, + &cache_context.policy_fingerprint, + ); + let projection = store + .get_child_certificate_cache_projection(&cache_key) + .expect("get child certificate cache projection") + .expect("child certificate cache projection present"); + let projected_until = + parse_snapshot_time_value(&projection.effective_until).expect("parse projected until"); + let initial_crl = crate::data_model::crl::RpkixCrl::decode_der(&g.issuer_crl_der) + .expect("decode initial crl"); + assert!( + projected_until > initial_crl.next_update.utc, + "child certificate cache projection must not be hard-capped by the issuing CRL nextUpdate" + ); + + let timing = TimingHandle::new(crate::analysis::timing::TimingMeta { + recorded_at_utc_rfc3339: "2026-06-24T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-06-24T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + let out = discover_children_from_fresh_snapshot_with_audit_cached( + &issuer, + &changed_pack, + validation_time, + Some(&timing), + Some(cache_context), + ) + .expect("changed valid CRL should recheck revocation and reuse cache"); + assert_eq!(out.children.len(), 1); + assert!( + out.audits + .iter() + .any(|audit| audit.detail.as_deref().unwrap_or("").contains("cache")) + ); + let counts = timing.counts_snapshot(); + assert_eq!( + counts + .get("child_certificate_cache_crl_recheck_hit") + .copied(), + Some(1) + ); + assert_eq!( + counts.get("child_certificate_cache_hit_ca").copied(), + Some(1) + ); +} + +#[test] +fn discover_children_child_certificate_cache_misses_when_current_crl_invalid() { + let g = generate_chain_and_crl(); + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/child.cer", + g.child_ca_der.clone(), + ), + ]); + let mut changed_pack = pack.clone(); + changed_pack.files[0] = PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + b"not-a-valid-crl".to_vec(), + ); + let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), + ca_certificate_rsync_uri: None, + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let cache_context = ChildCertificateValidationCacheContext { + store: &store, + issuer_ca_sha256: issuer.ca_certificate_sha256_32().unwrap(), + ca_validation_context_digest: ca_validation_context_digest_for_ca(&issuer), + policy_fingerprint: publication_point_cache_policy_fingerprint(&policy), + }; + let validation_time = time::OffsetDateTime::now_utc(); + discover_children_from_fresh_snapshot_with_audit_cached( + &issuer, + &pack, + validation_time, + None, + Some(cache_context), + ) + .expect("populate cache"); + + let timing = TimingHandle::new(crate::analysis::timing::TimingMeta { + recorded_at_utc_rfc3339: "2026-06-24T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-06-24T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + let out = discover_children_from_fresh_snapshot_with_audit_cached( + &issuer, + &changed_pack, + validation_time, + Some(&timing), + Some(cache_context), + ) + .expect("invalid CRL should miss cache and continue with audit error"); + assert!(out.children.is_empty()); + assert!( + out.audits + .iter() + .all(|audit| !audit.detail.as_deref().unwrap_or("").contains("cache")) + ); + let counts = timing.counts_snapshot(); + assert_eq!( + counts + .get("child_certificate_cache_miss_crl_invalid") + .copied(), + Some(1) + ); +} + +#[test] +fn discover_children_child_certificate_cache_misses_when_unchanged_crl_expired() { + let g = generate_chain_and_crl(); + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/child.cer", + g.child_ca_der.clone(), + ), + ]); + let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), + ca_certificate_rsync_uri: None, + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let cache_context = ChildCertificateValidationCacheContext { + store: &store, + issuer_ca_sha256: issuer.ca_certificate_sha256_32().unwrap(), + ca_validation_context_digest: ca_validation_context_digest_for_ca(&issuer), + policy_fingerprint: publication_point_cache_policy_fingerprint(&policy), + }; + let validation_time = time::OffsetDateTime::now_utc(); + discover_children_from_fresh_snapshot_with_audit_cached( + &issuer, + &pack, + validation_time, + None, + Some(cache_context), + ) + .expect("populate cache"); + + let timing = TimingHandle::new(crate::analysis::timing::TimingMeta { + recorded_at_utc_rfc3339: "2026-06-24T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-06-24T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + let out = discover_children_from_fresh_snapshot_with_audit_cached( + &issuer, + &pack, + validation_time + time::Duration::days(2), + Some(&timing), + Some(cache_context), + ) + .expect("expired unchanged CRL should miss cache and continue with audit error"); + assert!(out.children.is_empty()); + assert!( + out.audits + .iter() + .all(|audit| !audit.detail.as_deref().unwrap_or("").contains("cache")) + ); + let counts = timing.counts_snapshot(); + assert_eq!( + counts + .get("child_certificate_cache_miss_crl_expired") + .copied(), + Some(1) + ); +} + +#[test] +fn discover_children_with_audit_records_missing_crl_for_child_certificate() { + let now = time::OffsetDateTime::now_utc(); + + let child_ca_der = + std::fs::read(std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/R-lVU1XGsAeqzV1Fv0HjOD6ZFkE.cer", + )) + .expect("read child ca fixture"); + + // Pack contains the child CA cert but does not contain the CRL referenced by the child + // certificate CRLDistributionPoints extension. + let pack = dummy_pack_with_files(vec![PackFile::from_bytes_compute_sha256( + "rsync://ca.rg.net/rpki/RGnet-OU/R-lVU1XGsAeqzV1Fv0HjOD6ZFkE.cer", + child_ca_der, + )]); + + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(vec![1]), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + + let out = discover_children_from_fresh_snapshot_with_audit(&issuer, &pack, now, None) + .expect("discovery should succeed with audit error"); + assert_eq!(out.children.len(), 0); + assert_eq!(out.audits.len(), 1); + assert_eq!( + out.audits[0].rsync_uri, + "rsync://ca.rg.net/rpki/RGnet-OU/R-lVU1XGsAeqzV1Fv0HjOD6ZFkE.cer" + ); + assert_eq!(out.audits[0].result, AuditObjectResult::Error); + assert!( + out.audits[0] + .detail + .as_deref() + .unwrap_or("") + .contains("cannot select issuer CRL"), + "expected deterministic CRL selection failure to be recorded" + ); +} + +#[test] +fn runner_offline_rsync_fixture_produces_pack_and_warnings() { + let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + assert!(fixture_dir.is_dir(), "fixture directory must exist"); + + let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); + + // Pick a validation_time inside the fixture manifest's validity window to keep this + // test stable across wall-clock time. + let fixture_manifest_bytes = + std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); + let fixture_manifest = + crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) + .expect("decode manifest fixture"); + let validation_time = { + let this_update = fixture_manifest.manifest.this_update; + let next_update = fixture_manifest.manifest.next_update; + let candidate = this_update + time::Duration::seconds(60); + if candidate < next_update { + candidate + } else { + this_update + } + }; + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + + let runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + }; + + // For this fixture-driven smoke, we provide the correct issuer CA certificate (the CA for + // this publication point) so ROA EE certificate paths can validate. + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + + let handle = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: rsync_base_uri.clone(), + manifest_rsync_uri: manifest_rsync_uri.clone(), + publication_point_rsync_uri: rsync_base_uri.clone(), + rrdp_notification_uri: None, + }; + + let out = runner + .run_publication_point(&handle) + .expect("run publication point"); + assert_eq!(out.source, PublicationPointSource::Fresh); + let pack = out.snapshot.expect("fresh run pack"); + assert_eq!(pack.manifest_rsync_uri, manifest_rsync_uri); + assert!(pack.files.len() > 1); + assert!( + out.objects.vrps.len() > 1, + "expected to extract VRPs from ROAs" + ); + + let vcir = store + .get_vcir(&manifest_rsync_uri) + .expect("get vcir") + .expect("vcir exists after fresh run"); + assert_eq!(vcir.manifest_rsync_uri, manifest_rsync_uri); + assert_eq!(vcir.tal_id, "test-tal"); + assert!( + vcir.local_outputs + .iter() + .any(|output| output.output_type == crate::storage::VcirOutputType::Vrp), + "expected VCIR local_outputs to contain VRP entries" + ); + let first_vrp = vcir + .local_outputs + .iter() + .find(|output| output.output_type == crate::storage::VcirOutputType::Vrp) + .expect("first VCIR VRP output"); + assert!(!first_vrp.rule_hash_hex().is_empty()); + assert!(!first_vrp.output_id().is_empty()); + let replay_meta = store + .get_manifest_replay_meta(&manifest_rsync_uri) + .expect("get replay meta") + .expect("replay meta exists"); + assert_eq!(replay_meta.manifest_rsync_uri, manifest_rsync_uri); +} + +#[test] +fn runner_roa_validation_cache_reuses_vcir_outputs_on_second_fixture_run() { + let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + assert!(fixture_dir.is_dir(), "fixture directory must exist"); + + let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); + + let fixture_manifest_bytes = + std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); + let fixture_manifest = + crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) + .expect("decode manifest fixture"); + let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + + let handle = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: rsync_base_uri.clone(), + manifest_rsync_uri: manifest_rsync_uri.clone(), + publication_point_rsync_uri: rsync_base_uri, + rrdp_notification_uri: None, + }; + + let first_runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + }; + let first = first_runner + .run_publication_point(&handle) + .expect("first fresh run"); + assert!(first.objects.vrps.len() > 1); + assert_eq!(first.objects.roa_cache_stats.hit_roas, 0); + + let second_runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: true, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + }; + let second = second_runner + .run_publication_point(&handle) + .expect("second cache-enabled run"); + + assert_eq!(second.objects.vrps, first.objects.vrps); + assert_eq!(second.objects.roa_cache_stats.enabled_publication_points, 1); + assert_eq!( + second.objects.roa_cache_stats.vcir_hit_publication_points, + 1 + ); + assert_eq!( + second.objects.roa_cache_stats.vcir_miss_publication_points, + 0 + ); + assert!(second.objects.roa_cache_stats.hit_roas > 1); + assert_eq!(second.objects.roa_cache_stats.miss_roas, 0); + assert_eq!(second.objects.roa_cache_stats.blocked_roas, 0); + assert_eq!(second.objects.roa_cache_stats.fresh_roas, 0); +} + +#[test] +fn runner_publication_point_cache_observe_and_reuse_path() { + let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + assert!(fixture_dir.is_dir(), "fixture directory must exist"); + + let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); + let fixture_manifest_bytes = + std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); + let fixture_manifest = + crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) + .expect("decode manifest fixture"); + let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + let handle = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: rsync_base_uri.clone(), + manifest_rsync_uri: manifest_rsync_uri.clone(), + publication_point_rsync_uri: rsync_base_uri, + rrdp_notification_uri: None, + }; + + let first_runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: true, + enable_publication_point_validation_cache: false, + }; + let first = first_runner + .run_publication_point(&handle) + .expect("first fresh run"); + assert_eq!(first.source, PublicationPointSource::Fresh); + assert!( + store + .get_publication_point_cache_projection(&manifest_rsync_uri) + .expect("load publication-point projection") + .is_some() + ); + + let timing = crate::analysis::timing::TimingHandle::new(crate::analysis::timing::TimingMeta { + recorded_at_utc_rfc3339: "2026-01-01T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-01-01T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + let observe_runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), + validation_time, + timing: Some(timing.clone()), + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: true, + enable_publication_point_validation_cache: false, + }; + let observed = observe_runner + .run_publication_point(&handle) + .expect("observe-only run"); + assert_eq!(observed.source, PublicationPointSource::Fresh); + assert_eq!(observed.objects.vrps, first.objects.vrps); + assert_eq!( + timing + .counts_snapshot() + .get("publication_point_cache_theoretical_hits") + .copied(), + Some(1) + ); + + let cache_runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: true, + }; + let cached = cache_runner + .run_publication_point(&handle) + .expect("publication-point cache run"); + assert_eq!(cached.source, PublicationPointSource::PublicationPointCache); + assert_eq!(cached.objects.vrps, first.objects.vrps); + assert_eq!(cached.objects.aspas, first.objects.aspas); + assert_eq!( + cached.discovered_children.len(), + first.discovered_children.len() + ); + assert!(!cached.cir_cached_objects.is_empty()); +} + +fn seed_publication_point_cache_projection( + store: &RocksStore, + policy: &Policy, + ca: &CaInstanceHandle, + validation_time: time::OffsetDateTime, +) -> ValidatedCaInstanceResult { + let child_bytes = b"child-cert".to_vec(); + let child_hash = sha256_hex(&child_bytes); + let vcir = sample_vcir_for_projection(validation_time, &child_hash); + store + .put_blob_bytes_batch(&[ + (child_hash, child_bytes), + (sha256_hex(b"manifest-bytes"), b"manifest-bytes".to_vec()), + ]) + .expect("put cache bytes"); + store + .put_repository_view_entry(&RepositoryViewEntry { + rsync_uri: ca.manifest_rsync_uri.clone(), + current_hash: Some(sha256_hex(b"manifest-bytes")), + repository_source: Some(ca.publication_point_rsync_uri.clone()), + object_type: Some("mft".to_string()), + state: RepositoryViewState::Present, + }) + .expect("put manifest current view"); + let projection = PublicationPointCacheProjection::from_vcir_with_context( + &vcir, + ca.publication_point_rsync_uri.clone(), + ca.ca_certificate_rsync_uri.clone(), + ca.ca_certificate_sha256_32().unwrap(), + sha256_32(b"manifest-bytes"), + ta_context_digest_for_ca(ca), + ca_validation_context_digest_for_ca(ca), + publication_point_cache_policy_fingerprint(policy), + ) + .expect("build publication point projection"); + store + .put_vcir_with_publication_point_cache_projection(&vcir, Some(&projection)) + .expect("put publication point projection"); + vcir +} + +#[test] +fn publication_point_cache_future_notbefore_guard_detects_future_roa_notbefore_only() { + let uri = "rsync://example.test/repo/issuer/future.roa"; + let bytes = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa"), + ) + .expect("read ROA fixture"); + let roa = RoaObject::decode_der(&bytes).expect("decode ROA fixture"); + let ee = &roa.signed_object.signed_data.certificates[0].resource_cert; + let file = PackFile::from_bytes_compute_sha256(uri, bytes); + let pack = dummy_pack_with_files(vec![file]); + let objects = crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: Vec::new(), + stats: Default::default(), + audit: vec![ObjectAuditEntry { + rsync_uri: uri.to_string(), + sha256_hex: sha256_hex_from_32(&pack.files[0].sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Error, + detail: Some( + "EE certificate path validation failed: certificate not valid at validation_time" + .to_string(), + ), + }], + roa_cache_stats: Default::default(), + roa_cache_object_meta: Vec::new(), + }; + + assert!(publication_point_cache_has_future_not_before_risk( + &pack, + &objects, + &[], + ee.tbs.validity_not_before - time::Duration::seconds(1), + &Policy::default(), + )); + assert!(!publication_point_cache_has_future_not_before_risk( + &pack, + &objects, + &[], + ee.tbs.validity_not_after + time::Duration::seconds(1), + &Policy::default(), + )); +} + +#[test] +fn publication_point_cache_delete_action_removes_existing_projection() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::minutes(1); + let ca = publication_point_cache_fixture_ca(); + let vcir = seed_publication_point_cache_projection(&store, &policy, &ca, validation_time); + + assert!( + store + .get_publication_point_cache_projection_cached(&ca.manifest_rsync_uri) + .expect("load projection") + .is_some() + ); + + store + .replace_vcir_manifest_replay_meta_and_projection_action( + &vcir, + None, + PublicationPointCacheProjectionWriteAction::Delete { + manifest_rsync_uri: &vcir.manifest_rsync_uri, + }, + ) + .expect("delete projection"); + + assert!( + store + .get_publication_point_cache_projection_cached(&ca.manifest_rsync_uri) + .expect("load projection after delete") + .is_none() + ); +} + +fn publication_point_cache_fixture_ca() -> CaInstanceHandle { + CaInstanceHandle { + depth: 1, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(b"ca-cert".to_vec()), + ca_certificate_rsync_uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()), + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: Some("https://example.test/notification.xml".to_string()), + } +} + +#[test] +fn runner_publication_point_cache_reuses_projection_outputs_children_and_ccr() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::minutes(1); + let ca = publication_point_cache_fixture_ca(); + let vcir = seed_publication_point_cache_projection(&store, &policy, &ca, validation_time); + let timing = crate::analysis::timing::TimingHandle::new(crate::analysis::timing::TimingMeta { + recorded_at_utc_rfc3339: "2026-01-01T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-01-01T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + let runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &FailingRsyncFetcher, + validation_time, + timing: Some(timing.clone()), + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: Some(Mutex::new(CcrAccumulator::new(Vec::new()))), + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: true, + }; + + let result = runner + .observe_or_reuse_publication_point_cache(&ca, Some("rrdp"), Some("delta"), 7, None, &[]) + .expect("cache result"); + + assert_eq!(result.source, PublicationPointSource::PublicationPointCache); + assert_eq!(result.objects.vrps.len(), 1); + assert_eq!(result.objects.aspas.len(), 1); + assert_eq!(result.objects.router_keys.len(), 1); + assert_eq!(result.discovered_children.len(), 1); + assert_eq!( + result.discovered_children[0].handle.manifest_rsync_uri, + vcir.child_entries[0].child_manifest_rsync_uri + ); + assert!(result.cir_fresh_objects.is_empty()); + assert!(!result.cir_cached_objects.is_empty()); + assert_eq!( + runner + .ccr_accumulator_snapshot() + .expect("ccr accumulator") + .manifest_count(), + 1 + ); + let counts = timing.counts_snapshot(); + assert_eq!( + counts.get("publication_point_cache_reuse_hits").copied(), + Some(1) + ); + assert_eq!( + counts + .get("publication_point_cache_outputs_reused") + .copied(), + Some(3) + ); + assert_eq!( + counts + .get("publication_point_cache_children_reused") + .copied(), + Some(1) + ); + assert_eq!( + counts + .get("publication_point_cache_related_objects_reused") + .copied(), + Some(vcir.related_artifacts.len() as u64) + ); + assert_eq!( + counts + .get("publication_point_cache_audit_objects_reused") + .copied(), + Some(result.cir_cached_objects.len() as u64) + ); + let timing_dir = tempfile::tempdir().expect("timing dir"); + let timing_path = timing_dir.path().join("timing.json"); + timing.write_json(&timing_path, 20).expect("write timing"); + let timing_json: serde_json::Value = + serde_json::from_slice(&std::fs::read(&timing_path).expect("read timing")) + .expect("parse timing"); + let phase_keys = timing_json["phases"] + .as_object() + .expect("phases") + .keys() + .map(|key| key.as_str()) + .collect::>(); + assert!(phase_keys.contains("publication_point_cache_lookup_hit_total")); + assert!(phase_keys.contains("publication_point_cache_reuse_build_total")); + assert!(phase_keys.contains("publication_point_cache_build_objects_total")); +} + +#[test] +fn publication_point_cache_restore_children_parallel_keeps_order_and_audit() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::minutes(1); + let ca = publication_point_cache_fixture_ca(); + seed_publication_point_cache_projection(&store, &policy, &ca, validation_time); + let mut projection = store + .get_publication_point_cache_projection(&ca.manifest_rsync_uri) + .expect("load projection") + .expect("projection"); + let template = projection.children[0].clone(); + let mut blobs = Vec::new(); + let mut children = Vec::new(); + for index in 0..300 { + let bytes = format!("child-cert-{index}").into_bytes(); + let child_hash = sha256_hex(&bytes); + let mut child = template.clone(); + child.child_cert_hash = child_hash.clone(); + child.child_cert_rsync_uri = format!("rsync://example.test/repo/issuer/child-{index}.cer"); + child.child_manifest_rsync_uri = + format!("rsync://example.test/repo/child-{index}/child.mft"); + child.child_publication_point_rsync_uri = + format!("rsync://example.test/repo/child-{index}/"); + child.child_rsync_base_uri = child.child_publication_point_rsync_uri.clone(); + blobs.push((child_hash, bytes)); + children.push(child); + } + projection.children = children; + store.put_blob_bytes_batch(&blobs).expect("put child blobs"); + let timing = crate::analysis::timing::TimingHandle::new(crate::analysis::timing::TimingMeta { + recorded_at_utc_rfc3339: "2026-01-01T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-01-01T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + let mut warnings = Vec::new(); + + let (restored_children, audits) = restore_children_from_publication_point_cache( + &store, + &ca, + &projection, + validation_time, + &mut warnings, + 4, + Some(&timing), + ); + + assert!(warnings.is_empty()); + assert_eq!(restored_children.len(), 300); + assert_eq!(audits.len(), 300); + assert_eq!( + restored_children[0].handle.ca_certificate_sha256_hex(), + Some(sha256_hex(b"child-cert-0").as_str()) + ); + assert_eq!( + restored_children[299] + .handle + .ca_certificate_der(&store) + .unwrap() + .as_ref(), + b"child-cert-299" + ); + assert_eq!( + restored_children[0].handle.manifest_rsync_uri, + "rsync://example.test/repo/child-0/child.mft" + ); + assert!( + audits + .iter() + .all(|audit| audit.result == AuditObjectResult::Ok) + ); + let counts = timing.counts_snapshot(); + assert_eq!( + counts + .get("publication_point_cache_restore_children_parallel_publication_points") + .copied(), + Some(1) + ); + assert_eq!( + counts + .get("publication_point_cache_restore_children_parallel_children") + .copied(), + Some(300) + ); + assert_eq!( + counts + .get("publication_point_cache_restore_children_workers_total") + .copied(), + Some(4) + ); +} + +#[test] +fn publication_point_cache_restore_children_does_not_require_child_der_bytes() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::minutes(1); + let ca = publication_point_cache_fixture_ca(); + seed_publication_point_cache_projection(&store, &policy, &ca, validation_time); + let mut projection = store + .get_publication_point_cache_projection(&ca.manifest_rsync_uri) + .expect("load projection") + .expect("projection"); + projection.children[0].child_cert_hash = "22".repeat(32); + let mut warnings = Vec::new(); + + let (restored_children, audits) = restore_children_from_publication_point_cache( + &store, + &ca, + &projection, + validation_time, + &mut warnings, + 1, + None, + ); + + assert!(warnings.is_empty()); + assert!(!restored_children.is_empty()); + assert_eq!(audits.len(), restored_children.len()); + assert_eq!( + restored_children[0].handle.ca_certificate_sha256_hex(), + Some(projection.children[0].child_cert_hash.as_str()) + ); + assert!( + restored_children[0] + .handle + .ca_certificate_der(&store) + .is_err(), + "lazy child handle should not require repo bytes until a fresh path asks for DER" + ); +} + +#[test] +fn runner_publication_point_cache_blocks_parent_policy_and_output_time_mismatch() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let validation_time = time::OffsetDateTime::UNIX_EPOCH + time::Duration::minutes(1); + let ca = publication_point_cache_fixture_ca(); + seed_publication_point_cache_projection(&store, &policy, &ca, validation_time); + + let timing = crate::analysis::timing::TimingHandle::new(crate::analysis::timing::TimingMeta { + recorded_at_utc_rfc3339: "2026-01-01T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-01-01T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + let mut parent_changed_ca = ca.clone(); + parent_changed_ca.parent_manifest_rsync_uri = + Some("rsync://example.test/repo/other-parent.mft".to_string()); + let parent_changed_runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &FailingRsyncFetcher, + validation_time, + timing: Some(timing.clone()), + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: true, + }; + assert!( + parent_changed_runner + .observe_or_reuse_publication_point_cache( + &parent_changed_ca, + Some("rrdp"), + Some("delta"), + 7, + None, + &[] + ) + .is_none() + ); + assert_eq!( + timing + .counts_snapshot() + .get("publication_point_cache_miss_parent_context_mismatch") + .copied(), + Some(1) + ); + + let strict_policy = Policy { + strict: crate::policy::StrictPolicy::all(), + ..Policy::default() + }; + let policy_changed_runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &strict_policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &FailingRsyncFetcher, + validation_time, + timing: Some(timing.clone()), + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: true, + }; + assert!( + policy_changed_runner + .observe_or_reuse_publication_point_cache( + &ca, + Some("rrdp"), + Some("delta"), + 7, + None, + &[] + ) + .is_none() + ); + assert_eq!( + timing + .counts_snapshot() + .get("publication_point_cache_miss_policy_mismatch") + .copied(), + Some(1) + ); + + let output_expired_runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &FailingRsyncFetcher, + validation_time: validation_time + time::Duration::minutes(40), + timing: Some(timing.clone()), + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: true, + }; + assert!( + output_expired_runner + .observe_or_reuse_publication_point_cache( + &ca, + Some("rrdp"), + Some("delta"), + 7, + None, + &[] + ) + .is_none() + ); + assert_eq!( + timing + .counts_snapshot() + .get("publication_point_cache_miss_output_time_gate") + .copied(), + Some(1) + ); +} + +#[test] +fn runner_rsync_dedup_skips_second_sync_for_same_base() { + let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + assert!(fixture_dir.is_dir(), "fixture directory must exist"); + + let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); + + let fixture_manifest_bytes = + std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); + let fixture_manifest = + crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) + .expect("decode manifest fixture"); + let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + + let handle = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: rsync_base_uri.clone(), + manifest_rsync_uri: manifest_rsync_uri.clone(), + publication_point_rsync_uri: rsync_base_uri.clone(), + rrdp_notification_uri: None, + }; + + struct CountingRsyncFetcher { + inner: LocalDirRsyncFetcher, + calls: Arc, + } + impl RsyncFetcher for CountingRsyncFetcher { + fn fetch_objects( + &self, + rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + self.calls.fetch_add(1, Ordering::SeqCst); + self.inner.fetch_objects(rsync_base_uri) + } + } + + let calls = Arc::new(AtomicUsize::new(0)); + let rsync = CountingRsyncFetcher { + inner: LocalDirRsyncFetcher::new(&fixture_dir), + calls: calls.clone(), + }; + + let runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &rsync, + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + }; + + let first = runner.run_publication_point(&handle).expect("first run ok"); + assert_eq!(first.source, PublicationPointSource::Fresh); + + let second = runner + .run_publication_point(&handle) + .expect("second run ok"); + assert_eq!(second.source, PublicationPointSource::Fresh); + + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "rsync should be called once" + ); +} + +#[test] +fn runner_rsync_dedup_skips_second_sync_for_same_module_scope() { + let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + assert!(fixture_dir.is_dir(), "fixture directory must exist"); + + let first_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); + let second_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/sub/".to_string(); + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{first_base_uri}{manifest_file}"); + + let fixture_manifest_bytes = + std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); + let fixture_manifest = + crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) + .expect("decode manifest fixture"); + let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + + let handle = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: first_base_uri.clone(), + manifest_rsync_uri: manifest_rsync_uri.clone(), + publication_point_rsync_uri: first_base_uri.clone(), + rrdp_notification_uri: None, + }; + let second_handle = CaInstanceHandle { + rsync_base_uri: second_base_uri.clone(), + publication_point_rsync_uri: second_base_uri.clone(), + ..handle.clone() + }; + + struct ModuleScopeRsyncFetcher { + inner: LocalDirRsyncFetcher, + calls: Arc, + } + impl RsyncFetcher for ModuleScopeRsyncFetcher { + fn fetch_objects( + &self, + rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + self.calls.fetch_add(1, Ordering::SeqCst); + self.inner.fetch_objects(rsync_base_uri) + } + + fn dedup_key(&self, _rsync_base_uri: &str) -> String { + "rsync://rpki.cernet.net/repo/".to_string() + } + } + + let calls = Arc::new(AtomicUsize::new(0)); + let rsync = ModuleScopeRsyncFetcher { + inner: LocalDirRsyncFetcher::new(&fixture_dir), + calls: calls.clone(), + }; + + let runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &rsync, + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + }; + + let first = runner.run_publication_point(&handle).expect("first run ok"); + assert_eq!(first.source, PublicationPointSource::Fresh); + + let second = runner + .run_publication_point(&second_handle) + .expect("second run ok"); + assert!(matches!( + second.source, + PublicationPointSource::Fresh | PublicationPointSource::VcirCurrentInstance + )); + + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "module-scope dedup should skip second sync" + ); +} + +#[test] +fn runner_rsync_dedup_works_in_rsync_only_mode_even_when_rrdp_notify_exists() { + let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + assert!(fixture_dir.is_dir(), "fixture directory must exist"); + + let first_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); + let second_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/sub/".to_string(); + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{first_base_uri}{manifest_file}"); + + let fixture_manifest_bytes = + std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); + let fixture_manifest = + crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) + .expect("decode manifest fixture"); + let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + + let handle = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: first_base_uri.clone(), + manifest_rsync_uri: manifest_rsync_uri.clone(), + publication_point_rsync_uri: first_base_uri.clone(), + rrdp_notification_uri: Some("https://rrdp.example.test/notification.xml".to_string()), + }; + let second_handle = CaInstanceHandle { + rsync_base_uri: second_base_uri.clone(), + publication_point_rsync_uri: second_base_uri.clone(), + ..handle.clone() + }; + + struct ModuleScopeRsyncFetcher { + inner: LocalDirRsyncFetcher, + calls: Arc, + } + impl RsyncFetcher for ModuleScopeRsyncFetcher { + fn fetch_objects( + &self, + rsync_base_uri: &str, + ) -> Result)>, RsyncFetchError> { + self.calls.fetch_add(1, Ordering::SeqCst); + self.inner.fetch_objects(rsync_base_uri) + } + + fn dedup_key(&self, _rsync_base_uri: &str) -> String { + "rsync://rpki.cernet.net/repo/".to_string() + } + } + + let calls = Arc::new(AtomicUsize::new(0)); + let rsync = ModuleScopeRsyncFetcher { + inner: LocalDirRsyncFetcher::new(&fixture_dir), + calls: calls.clone(), + }; + + let runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &rsync, + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + }; + + let first = runner.run_publication_point(&handle).expect("first run ok"); + assert_eq!(first.source, PublicationPointSource::Fresh); + + let second = runner + .run_publication_point(&second_handle) + .expect("second run ok"); + assert!(matches!( + second.source, + PublicationPointSource::Fresh | PublicationPointSource::VcirCurrentInstance + )); + + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "rsync-only mode must deduplicate by rsync scope even when RRDP notification is present" + ); +} + +#[test] +fn runner_when_repo_sync_fails_uses_current_instance_vcir_and_keeps_children_empty_for_fixture() { + let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + assert!(fixture_dir.is_dir(), "fixture directory must exist"); + + let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); + + let fixture_manifest_bytes = + std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); + let fixture_manifest = + crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) + .expect("decode manifest fixture"); + let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + + let handle = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: rsync_base_uri.clone(), + manifest_rsync_uri: manifest_rsync_uri.clone(), + publication_point_rsync_uri: rsync_base_uri.clone(), + rrdp_notification_uri: None, + }; + + // First: successful fresh run to populate the latest VCIR baseline. + let ok_runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + }; + let first = ok_runner + .run_publication_point(&handle) + .expect("first run ok"); + assert_eq!(first.source, PublicationPointSource::Fresh); + assert!( + first.discovered_children.is_empty(), + "fixture has no child .cer" + ); + + // Second: repo sync fails, but we can still reuse current-instance VCIR. + let bad_runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &FailingRsyncFetcher, + validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + }; + let second = bad_runner + .run_publication_point(&handle) + .expect("should reuse current-instance VCIR"); + assert_eq!(second.source, PublicationPointSource::VcirCurrentInstance); + assert!(second.discovered_children.is_empty()); + assert!( + second + .warnings + .iter() + .any(|w| w.message.contains("repo sync failed")), + "expected warning about repo sync failure" + ); +} + +#[test] +fn build_publication_point_audit_emits_no_audit_entry_for_duplicate_pack_uri() { + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256("rsync://example.test/repo/dup.roa", vec![1u8]), + PackFile::from_bytes_compute_sha256("rsync://example.test/repo/dup.roa", vec![2u8]), + ]); + let pp = crate::validation::manifest::PublicationPointResult { + source: crate::validation::manifest::PublicationPointSource::VcirCurrentInstance, + snapshot: pack.clone(), + warnings: Vec::new(), + }; + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(vec![1]), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + let objects = crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: Vec::new(), + stats: crate::validation::objects::ObjectsStats::default(), + audit: Vec::new(), + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + }; + + let audit = build_publication_point_audit_from_snapshot( + &ca, + pp.source, + None, + None, + None, + None, + &pp.snapshot, + &[], + &objects, + &[], + ); + assert_eq!(audit.source, "vcir_current_instance"); + assert_eq!(audit.repo_sync_phase, None); + assert_eq!(audit.repo_terminal_state, "fallback_current_instance"); + assert!( + audit + .objects + .iter() + .any(|e| e.detail.as_deref() == Some("skipped: no audit entry")), + "expected a duplicate key to produce a 'no audit entry' placeholder" + ); +} + +#[test] +fn build_publication_point_audit_marks_invalid_crl_as_error_and_overlays_roa_audit() { + let now = time::OffsetDateTime::now_utc(); + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256("rsync://example.test/repo/issuer/bad.crl", vec![0u8]), + PackFile::from_bytes_compute_sha256("rsync://example.test/repo/issuer/x.roa", vec![1u8]), + ]); + + let pp = crate::validation::manifest::PublicationPointResult { + source: crate::validation::manifest::PublicationPointSource::Fresh, + snapshot: pack.clone(), + warnings: Vec::new(), + }; + + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(vec![1]), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + + let objects = crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: Vec::new(), + stats: crate::validation::objects::ObjectsStats::default(), + audit: vec![ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/issuer/x.roa".to_string(), + sha256_hex: sha256_hex_from_32(&pack.files[1].sha256), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Ok, + detail: None, + }], + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + }; + + let audit = build_publication_point_audit_from_snapshot( + &issuer, + pp.source, + Some("rsync"), + Some("rsync_only_ok"), + Some(123), + Some("none"), + &pp.snapshot, + &[], + &objects, + &[], + ); + assert_eq!(audit.objects[0].kind, AuditObjectKind::Manifest); + assert_eq!(audit.repo_sync_source.as_deref(), Some("rsync")); + assert_eq!(audit.repo_sync_phase.as_deref(), Some("rsync_only_ok")); + assert_eq!(audit.repo_sync_duration_ms, Some(123)); + assert_eq!(audit.repo_sync_error.as_deref(), Some("none")); + assert_eq!(audit.repo_terminal_state, "fresh"); + + let crl = audit + .objects + .iter() + .find(|e| e.rsync_uri.ends_with("bad.crl")) + .expect("crl entry"); + assert!(matches!(crl.result, AuditObjectResult::Error)); + + let roa = audit + .objects + .iter() + .find(|e| e.rsync_uri.ends_with("x.roa")) + .expect("roa entry"); + assert!(matches!(roa.result, AuditObjectResult::Ok)); + + // Smoke that time fields are populated from pack. + assert!(audit.verified_at_rfc3339_utc.contains('T')); + assert!(audit.this_update_rfc3339_utc.contains('T')); + assert!(audit.next_update_rfc3339_utc.contains('T')); + let _ = now; +} + +#[test] +fn discover_children_with_router_certificate_records_ok_audit_and_no_child() { + let g = generate_router_cert_with_variant("ec-p256", true); + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/router.cer", + g.router_der.clone(), + ), + ]); + + let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), + ca_certificate_rsync_uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let out = discover_children_from_fresh_snapshot_with_audit( + &issuer, + &pack, + time::OffsetDateTime::now_utc(), + None, + ) + .expect("discover router cert"); + assert!(out.children.is_empty()); + assert_eq!(out.audits.len(), 1); + assert!(matches!(out.audits[0].result, AuditObjectResult::Ok)); + assert!( + out.audits[0] + .detail + .as_deref() + .unwrap_or("") + .contains("validated BGPsec router certificate") + ); +} + +#[test] +fn discover_children_with_non_router_ee_certificate_records_skipped_audit() { + let g = generate_router_cert_with_variant("ec-p256", false); + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/router-no-eku.cer", + g.router_der.clone(), + ), + ]); + + let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), + ca_certificate_rsync_uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let out = discover_children_from_fresh_snapshot_with_audit( + &issuer, + &pack, + time::OffsetDateTime::now_utc(), + None, + ) + .expect("discover non-router cert"); + assert!(out.children.is_empty()); + assert_eq!(out.audits.len(), 1); + assert!(matches!(out.audits[0].result, AuditObjectResult::Skipped)); + assert!( + out.audits[0] + .detail + .as_deref() + .unwrap_or("") + .contains("not a CA resource certificate or BGPsec router certificate") + ); +} + +#[test] +fn discover_children_with_invalid_router_certificate_records_error_audit() { + let g = generate_router_cert_with_variant("ec-p384", true); + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/router-invalid.cer", + g.router_der.clone(), + ), + ]); + + let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), + ca_certificate_rsync_uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let out = discover_children_from_fresh_snapshot_with_audit( + &issuer, + &pack, + time::OffsetDateTime::now_utc(), + None, + ) + .expect("discover invalid router cert"); + assert!(out.children.is_empty()); + assert_eq!(out.audits.len(), 1); + assert!(matches!(out.audits[0].result, AuditObjectResult::Error)); + assert!( + out.audits[0] + .detail + .as_deref() + .unwrap_or("") + .contains("router certificate validation failed") + ); +} + +#[test] +fn discover_children_with_audit_records_decode_error_for_corrupt_cer() { + let g = generate_chain_and_crl(); + + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/corrupt.cer", + vec![0u8], + ), + ]); + + let issuer_ca = ResourceCertificate::decode_der(&g.issuer_ca_der).expect("decode issuer"); + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(g.issuer_ca_der.clone()), + ca_certificate_rsync_uri: None, + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let now = time::OffsetDateTime::now_utc(); + let out = discover_children_from_fresh_snapshot_with_audit(&issuer, &pack, now, None) + .expect("discover children"); + assert!(out.children.is_empty()); + assert_eq!(out.audits.len(), 1); + assert!(matches!(out.audits[0].result, AuditObjectResult::Error)); +} + +#[test] +fn select_issuer_crl_uri_for_child_covers_missing_and_not_found_paths() { + let g = generate_chain_and_crl(); + let child = ResourceCertificate::decode_der(&g.child_ca_der).expect("decode child cert"); + + let empty: std::collections::HashMap = + std::collections::HashMap::new(); + let err = select_issuer_crl_uri_for_child(&child, &empty).unwrap_err(); + assert!(err.contains("no CRL available"), "{err}"); + + let ta_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/ta/apnic-ta.cer"), + ) + .expect("read TA fixture"); + let ta = ResourceCertificate::decode_der(&ta_der).expect("decode TA fixture"); + let mut cache = std::collections::HashMap::new(); + cache.insert( + "rsync://example.test/repo/issuer/issuer.crl".to_string(), + CachedIssuerCrl::Pending { + bytes: g.issuer_crl_der.clone(), + sha256_hex: None, + }, + ); + let err = select_issuer_crl_uri_for_child(&ta, &cache).unwrap_err(); + assert!(err.contains("CRLDistributionPoints missing"), "{err}"); + + let mut wrong = std::collections::HashMap::new(); + wrong.insert( + "rsync://example.test/repo/issuer/other.crl".to_string(), + CachedIssuerCrl::Pending { + bytes: g.issuer_crl_der, + sha256_hex: None, + }, + ); + let err = select_issuer_crl_uri_for_child(&child, &wrong).unwrap_err(); + assert!( + err.contains("not found in publication point snapshot"), + "{err}" + ); +} + +#[test] +fn ensure_issuer_crl_verified_promotes_pending_cache_entry() { + let g = generate_chain_and_crl(); + let mut cache = std::collections::HashMap::new(); + let crl_uri = "rsync://example.test/repo/issuer/issuer.crl".to_string(); + cache.insert( + crl_uri.clone(), + CachedIssuerCrl::Pending { + bytes: g.issuer_crl_der.clone(), + sha256_hex: None, + }, + ); + + let first = ensure_issuer_crl_verified(&crl_uri, &mut cache, &g.issuer_ca_der) + .expect("verify pending CRL"); + assert!(first.revoked_serials.is_empty()); + assert!(matches!(cache.get(&crl_uri), Some(CachedIssuerCrl::Ok(_)))); + + let second = ensure_issuer_crl_verified(&crl_uri, &mut cache, &g.issuer_ca_der) + .expect("reuse verified CRL"); + assert!(second.revoked_serials.is_empty()); +} + +#[test] +fn discover_children_with_invalid_issuer_der_records_error_audit() { + let g = generate_chain_and_crl(); + let pack = dummy_pack_with_files(vec![ + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/issuer.crl", + g.issuer_crl_der.clone(), + ), + PackFile::from_bytes_compute_sha256( + "rsync://example.test/repo/issuer/child.cer", + g.child_ca_der.clone(), + ), + ]); + + let issuer = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(vec![0u8]), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let out = discover_children_from_fresh_snapshot_with_audit( + &issuer, + &pack, + time::OffsetDateTime::now_utc(), + None, + ) + .expect("discover children with invalid issuer der"); + assert!(out.children.is_empty()); + assert_eq!(out.audits.len(), 1); + assert!(matches!(out.audits[0].result, AuditObjectResult::Error)); + assert!( + out.audits[0] + .detail + .as_deref() + .unwrap_or("") + .contains("issuer CA decode failed") + ); +} + +#[test] +fn project_current_instance_vcir_reuses_local_outputs_and_restores_children() { + let now = time::OffsetDateTime::now_utc(); + let g = generate_chain_and_crl(); + let child_cert_hash = sha256_hex(&g.child_ca_der); + let vcir = sample_vcir_for_projection(now, &child_cert_hash); + + let store_dir = tempfile::tempdir().expect("store dir"); + let main_db = store_dir.path().join("work-db"); + let repo_bytes_db = store_dir.path().join("repo-bytes.db"); + let store = RocksStore::open_with_external_repo_bytes(&main_db, &repo_bytes_db) + .expect("open rocksdb with external repo bytes"); + store + .put_blob_bytes_batch(&[(child_cert_hash.clone(), g.child_ca_der.clone())]) + .expect("put child cert repo bytes"); + assert!( + store + .get_raw_by_hash_entry(&child_cert_hash) + .expect("lookup child raw_by_hash") + .is_none(), + "child cert restoration should not require raw_by_hash entries" + ); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), + }; + let policy = Policy::default(); + put_vcir_for_failed_fetch_reuse(&store, &ca, &policy, &vcir); + + let projection = project_current_instance_vcir_on_failed_fetch( + &store, + &ca, + &ManifestFreshError::RepoSyncFailed { + detail: "synthetic".to_string(), + }, + &policy, + now, + ) + .expect("project vcir"); + + assert_eq!( + projection.source, + PublicationPointSource::VcirCurrentInstance + ); + assert_eq!(projection.objects.vrps.len(), 1); + assert_eq!(projection.objects.aspas.len(), 1); + assert_eq!(projection.objects.router_keys.len(), 1); + assert_eq!(projection.discovered_children.len(), 1); + assert_eq!( + projection.discovered_children[0].handle.manifest_rsync_uri, + "rsync://example.test/repo/child/child.mft" + ); + assert_eq!( + projection.ccr_manifest_projection.as_ref(), + Some(&vcir.ccr_manifest_projection) + ); + assert!( + projection.snapshot.is_none(), + "current-instance reuse should not reconstruct a byte-backed snapshot" + ); + assert!( + !projection + .warnings + .iter() + .any(|warning| warning.message.contains("manifest failed fetch")), + "successful current-instance reuse should not duplicate the fresh fetch error" + ); + assert!( + !projection + .warnings + .iter() + .any(|warning| warning.message.contains("using latest validated result")), + "successful current-instance reuse should be tracked by source, not warning" + ); + assert!( + !projection + .warnings + .iter() + .any(|warning| warning.message.contains("manifest raw bytes missing")), + "successful current-instance reuse should not load repo bytes for audit reconstruction" + ); + assert!( + !projection + .warnings + .iter() + .any(|warning| warning.message.contains("child certificate bytes missing")), + "child discovery restoration should read child certs from repo bytes" + ); +} + +#[test] +fn project_current_instance_vcir_does_not_reuse_without_identity() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let vcir = sample_vcir_for_projection(now, &child_cert_hash); + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + store.put_vcir(&vcir).expect("put legacy-style VCIR"); + let ca = sample_ca_for_failed_fetch_reuse(&vcir); + + let projection = project_current_instance_vcir_on_failed_fetch( + &store, + &ca, + &ManifestFreshError::RepoSyncFailed { + detail: "synthetic".to_string(), + }, + &Policy::default(), + now, + ) + .expect("project VCIR"); + + assert_eq!( + projection.source, + PublicationPointSource::FailedFetchNoCache + ); + assert!(projection.objects.vrps.is_empty()); + assert!(projection.discovered_children.is_empty()); + assert!( + projection + .warnings + .iter() + .any(|warning| warning.message.contains("reuse identity is missing")) + ); +} + +#[test] +fn project_current_instance_vcir_does_not_reuse_when_ta_context_changes() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let vcir = sample_vcir_for_projection(now, &child_cert_hash); + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let original_ca = sample_ca_for_failed_fetch_reuse(&vcir); + put_vcir_for_failed_fetch_reuse(&store, &original_ca, &policy, &vcir); + let mut changed_ca = original_ca; + changed_ca.tal_id = "different-tal".to_string(); + + let projection = project_current_instance_vcir_on_failed_fetch( + &store, + &changed_ca, + &ManifestFreshError::RepoSyncFailed { + detail: "synthetic".to_string(), + }, + &policy, + now, + ) + .expect("project VCIR"); + + assert_eq!( + projection.source, + PublicationPointSource::FailedFetchNoCache + ); + assert!(projection.objects.vrps.is_empty()); + assert!(projection.discovered_children.is_empty()); + assert!( + projection + .warnings + .iter() + .any(|warning| warning.message.contains("identity does not match")) + ); +} + +#[test] +fn project_current_instance_vcir_returns_no_output_when_instance_gate_expired() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); + let this_update = PackTime::from_utc_offset_datetime(now - time::Duration::minutes(2)); + let expired = PackTime::from_utc_offset_datetime(now - time::Duration::minutes(1)); + vcir.validated_manifest_meta.validated_manifest_this_update = this_update; + vcir.validated_manifest_meta.validated_manifest_next_update = expired.clone(); + vcir.instance_gate.manifest_next_update = expired.clone(); + vcir.instance_gate.current_crl_next_update = expired.clone(); + vcir.instance_gate.instance_effective_until = expired; + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + store.put_vcir(&vcir).expect("put vcir"); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let projection = project_current_instance_vcir_on_failed_fetch( + &store, + &ca, + &ManifestFreshError::RepoSyncFailed { + detail: "synthetic".to_string(), + }, + &Policy::default(), + now, + ) + .expect("project vcir"); + + assert_eq!( + projection.source, + PublicationPointSource::FailedFetchNoCache + ); + assert!(projection.ccr_manifest_projection.is_none()); + assert!(projection.objects.vrps.is_empty()); + assert!(projection.objects.aspas.is_empty()); + assert!(projection.discovered_children.is_empty()); +} + +#[test] +fn project_current_instance_vcir_keeps_real_fresh_validation_warning() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let vcir = sample_vcir_for_projection(now, &child_cert_hash); + + let store_dir = tempfile::tempdir().expect("store dir"); + let main_db = store_dir.path().join("work-db"); + let repo_bytes_db = store_dir.path().join("repo-bytes.db"); + let store = RocksStore::open_with_external_repo_bytes(&main_db, &repo_bytes_db) + .expect("open rocksdb with external repo bytes"); + store + .put_blob_bytes_batch(&[(child_cert_hash, b"child-cert".to_vec())]) + .expect("put child cert repo bytes"); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + let policy = Policy::default(); + put_vcir_for_failed_fetch_reuse(&store, &ca, &policy, &vcir); + + let projection = project_current_instance_vcir_on_failed_fetch( + &store, + &ca, + &ManifestFreshError::HashMismatch { + rsync_uri: "rsync://example.test/repo/issuer/a.roa".to_string(), + }, + &policy, + now, + ) + .expect("project vcir"); + + assert_eq!( + projection.source, + PublicationPointSource::VcirCurrentInstance + ); + assert!( + projection + .warnings + .iter() + .any(|warning| { warning.message.contains("manifest file hash mismatch") }) + ); + assert!( + !projection + .warnings + .iter() + .any(|warning| warning.message.contains("using latest validated result")), + "successful current-instance reuse should not emit bookkeeping warnings" + ); +} + +#[test] +fn project_current_instance_vcir_returns_no_output_when_latest_result_missing() { + let now = time::OffsetDateTime::now_utc(); + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let projection = project_current_instance_vcir_on_failed_fetch( + &store, + &ca, + &ManifestFreshError::RepoSyncFailed { + detail: "synthetic".to_string(), + }, + &Policy::default(), + now, + ) + .expect("project without cached vcir"); + + assert_eq!( + projection.source, + PublicationPointSource::FailedFetchNoCache + ); + assert!(projection.vcir.is_none()); + assert!(projection.ccr_manifest_projection.is_none()); + assert!(projection.snapshot.is_none()); + assert!(projection.objects.audit.is_empty()); + assert!(projection.discovered_children.is_empty()); + assert!(projection.warnings.iter().any(|warning| { + warning + .message + .contains("no latest validated result for current CA instance") + })); +} + +#[test] +fn project_current_instance_vcir_returns_no_output_when_latest_result_is_ineligible() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); + vcir.audit_summary.failed_fetch_eligible = false; + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + store.put_vcir(&vcir).expect("put vcir"); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let projection = project_current_instance_vcir_on_failed_fetch( + &store, + &ca, + &ManifestFreshError::RepoSyncFailed { + detail: "synthetic".to_string(), + }, + &Policy::default(), + now, + ) + .expect("project ineligible vcir"); + + assert_eq!( + projection.source, + PublicationPointSource::FailedFetchNoCache + ); + assert!(projection.vcir.is_some()); + assert!(projection.ccr_manifest_projection.is_none()); + assert!(projection.snapshot.is_none()); + assert!(projection.discovered_children.is_empty()); + assert!(projection.warnings.iter().any(|warning| { + warning + .message + .contains("latest VCIR is not marked failed-fetch eligible") + })); +} + +#[test] +fn project_current_instance_vcir_rejects_mismatched_ccr_projection_uri() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); + vcir.ccr_manifest_projection.manifest_rsync_uri = + "rsync://example.test/repo/issuer/other.mft".to_string(); + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + let policy = Policy::default(); + put_vcir_for_failed_fetch_reuse(&store, &ca, &policy, &vcir); + + let err = project_current_instance_vcir_on_failed_fetch( + &store, + &ca, + &ManifestFreshError::RepoSyncFailed { + detail: "synthetic".to_string(), + }, + &policy, + now, + ) + .unwrap_err(); + + assert!( + err.contains("vcir CCR manifest projection URI mismatch"), + "{err}" + ); +} + +#[test] +fn fresh_and_reuse_paths_produce_equivalent_ccr_manifest_projection() { + let (pack, issuer_ca_der, validation_time) = cernet_publication_point_snapshot_for_vcir_tests(); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: pack.publication_point_rsync_uri.clone(), + manifest_rsync_uri: pack.manifest_rsync_uri.clone(), + publication_point_rsync_uri: pack.publication_point_rsync_uri.clone(), + rrdp_notification_uri: None, + }; + let child_discovery = + discover_children_from_fresh_snapshot_with_audit(&ca, &pack, validation_time, None) + .expect("discover children"); + let mut objects = empty_objects_output(); + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let (fresh_vcir, _timing) = build_vcir_from_fresh_result_with_timing( + &store, + &ca, + &pack, + &mut objects, + &[], + &child_discovery.audits, + &child_discovery.children, + validation_time, + ) + .expect("build fresh vcir"); + + let reuse_projection = reuse_ccr_manifest_projection_from_vcir(&ca, &fresh_vcir) + .expect("reuse projection from vcir"); + + assert_eq!(fresh_vcir.ccr_manifest_projection, reuse_projection); +} + +#[test] +fn append_ccr_manifest_projection_from_reuse_requires_projection_for_current_instance() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let runner = sample_runner_with_ccr_accumulator(&store, &policy); + + let err = runner + .append_ccr_manifest_projection_from_reuse(&VcirReuseProjection { + source: PublicationPointSource::VcirCurrentInstance, + vcir: None, + ccr_manifest_projection: None, + snapshot: None, + objects: empty_objects_output(), + child_audits: Vec::new(), + discovered_children: Vec::new(), + warnings: Vec::new(), + }) + .unwrap_err(); + + assert!(err.contains("missing CCR manifest projection"), "{err}"); + assert_eq!( + runner + .ccr_accumulator_snapshot() + .expect("ccr accumulator snapshot") + .manifest_count(), + 0 + ); +} + +#[test] +fn append_ccr_manifest_projection_from_reuse_skips_failed_fetch_no_cache() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let runner = sample_runner_with_ccr_accumulator(&store, &policy); + + runner + .append_ccr_manifest_projection_from_reuse(&VcirReuseProjection { + source: PublicationPointSource::FailedFetchNoCache, + vcir: None, + ccr_manifest_projection: None, + snapshot: None, + objects: empty_objects_output(), + child_audits: Vec::new(), + discovered_children: Vec::new(), + warnings: Vec::new(), + }) + .expect("failed-fetch no-cache should not append"); + + assert_eq!( + runner + .ccr_accumulator_snapshot() + .expect("ccr accumulator snapshot") + .manifest_count(), + 0 + ); +} + +#[test] +fn parse_snapshot_time_value_reports_invalid_timestamp() { + let err = parse_snapshot_time_value(&PackTime { + rfc3339_utc: "not-a-time".to_string(), + }) + .unwrap_err(); + + assert!(err.contains("invalid RFC3339 time 'not-a-time'"), "{err}"); +} + +#[test] +fn runner_roa_validation_cache_uses_projection_not_full_vcir_fallback() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); + vcir.local_outputs + .retain(|output| output.source_object_type != VcirSourceObjectType::Roa); + vcir.summary.local_vrp_count = 0; + vcir.summary.local_aspa_count = 1; + vcir.summary.local_router_key_count = 1; + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + store.put_vcir(&vcir).expect("put vcir without projection"); + assert!( + store + .get_vcir(&vcir.manifest_rsync_uri) + .expect("get vcir") + .is_some() + ); + assert!( + store + .get_roa_cache_projection(&vcir.manifest_rsync_uri) + .expect("get projection") + .is_none() + ); + + let timing = crate::analysis::timing::TimingHandle::new(crate::analysis::timing::TimingMeta { + recorded_at_utc_rfc3339: "2026-06-07T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-06-07T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + let policy = Policy::default(); + let runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &FailingRsyncFetcher, + validation_time: now, + timing: Some(timing.clone()), + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: true, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + }; + + assert!( + runner + .roa_validation_cache_view_for_fresh_point(&vcir.manifest_rsync_uri) + .is_none() + ); + let dir = tempfile::tempdir().expect("timing dir"); + let path = dir.path().join("timing.json"); + timing.write_json(&path, 10).expect("write timing"); + let report: serde_json::Value = + serde_json::from_slice(&std::fs::read(path).expect("read timing")).expect("parse timing"); + assert_eq!( + report["counts"]["roa_validation_cache_projection_missing_publication_points"], + 1 + ); + assert!( + report["phases"]["roa_validation_cache_projection_load_total"]["count"] + .as_u64() + .unwrap_or_default() + >= 1 + ); +} + +#[test] +fn build_objects_output_from_vcir_tracks_expired_and_invalid_cached_outputs() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); + + let bad_time_uri = "rsync://example.test/repo/issuer/bad-time.roa".to_string(); + let expired_uri = "rsync://example.test/repo/issuer/expired.asa".to_string(); + let bad_json_uri = "rsync://example.test/repo/issuer/bad-json.roa".to_string(); + let bad_prefix_uri = "rsync://example.test/repo/issuer/bad-prefix.roa".to_string(); + let bad_aspa_uri = "rsync://example.test/repo/issuer/bad-aspa.asa".to_string(); + + for (uri, kind) in [ + (bad_time_uri.clone(), VcirArtifactKind::Roa), + (expired_uri.clone(), VcirArtifactKind::Aspa), + (bad_json_uri.clone(), VcirArtifactKind::Roa), + (bad_prefix_uri.clone(), VcirArtifactKind::Roa), + (bad_aspa_uri.clone(), VcirArtifactKind::Aspa), + ] { + vcir.related_artifacts.push(VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: kind, + uri: Some(uri.clone()), + sha256: sha256_hex(uri.as_bytes()), + object_type: Some( + match kind { + VcirArtifactKind::Aspa => "aspa", + _ => "roa", + } + .to_string(), + ), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }); + } + + vcir.local_outputs.push(VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: PackTime { + rfc3339_utc: "bad-time-value".to_string(), + }, + source_object_uri: bad_time_uri.clone(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: sha256_32(b"bad-time-src"), + source_ee_cert_hash: sha256_32(b"bad-time-ee"), + payload: VcirLocalOutputPayload::Vrp { + asn: 64496, + afi: RoaAfi::Ipv4, + prefix_len: 24, + addr: ipv4_addr([203, 0, 113, 0]), + max_length: 24, + }, + rule_hash: sha256_32(b"bad-time-rule"), + }); + vcir.local_outputs.push(VcirLocalOutput { + output_type: VcirOutputType::Aspa, + item_effective_until: PackTime::from_utc_offset_datetime(now - time::Duration::minutes(1)), + source_object_uri: expired_uri.clone(), + source_object_type: VcirSourceObjectType::Aspa, + source_object_hash: sha256_32(b"expired-src"), + source_ee_cert_hash: sha256_32(b"expired-ee"), + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: 64500, + provider_as_ids: vec![64501], + }, + rule_hash: sha256_32(b"expired-rule"), + }); + vcir.local_outputs.push(VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(5)), + source_object_uri: bad_json_uri.clone(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: sha256_32(b"bad-json-src"), + source_ee_cert_hash: sha256_32(b"bad-json-ee"), + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: 64510, + provider_as_ids: vec![64511], + }, + rule_hash: sha256_32(b"bad-json-rule"), + }); + vcir.local_outputs.push(VcirLocalOutput { + output_type: VcirOutputType::Vrp, + item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(5)), + source_object_uri: bad_prefix_uri.clone(), + source_object_type: VcirSourceObjectType::Roa, + source_object_hash: sha256_32(b"bad-prefix-src"), + source_ee_cert_hash: sha256_32(b"bad-prefix-ee"), + payload: VcirLocalOutputPayload::Aspa { + customer_as_id: 64512, + provider_as_ids: vec![64513], + }, + rule_hash: sha256_32(b"bad-prefix-rule"), + }); + vcir.local_outputs.push(VcirLocalOutput { + output_type: VcirOutputType::Aspa, + item_effective_until: PackTime::from_utc_offset_datetime(now + time::Duration::minutes(5)), + source_object_uri: bad_aspa_uri.clone(), + source_object_type: VcirSourceObjectType::Aspa, + source_object_hash: sha256_32(b"bad-aspa-src"), + source_ee_cert_hash: sha256_32(b"bad-aspa-ee"), + payload: VcirLocalOutputPayload::Vrp { + asn: 64520, + afi: RoaAfi::Ipv4, + prefix_len: 24, + addr: ipv4_addr([198, 51, 100, 0]), + max_length: 24, + }, + rule_hash: sha256_32(b"bad-aspa-rule"), + }); + + let mut warnings = Vec::new(); + let output = build_objects_output_from_vcir(&vcir, now, &mut warnings); + + assert_eq!(output.vrps.len(), 1); + assert_eq!(output.aspas.len(), 1); + assert_eq!(output.stats.roa_total, 4); + assert_eq!(output.stats.roa_ok, 1); + assert_eq!(output.stats.aspa_total, 3); + assert_eq!(output.stats.aspa_ok, 1); + assert!(warnings.iter().any(|warning| { + warning + .message + .contains("cached local output has invalid item_effective_until") + })); + assert!(warnings.iter().any(|warning| { + warning + .message + .contains("cached ROA local output parse failed") + })); + assert!(warnings.iter().any(|warning| { + warning + .message + .contains("cached ASPA local output parse failed") + })); + assert!(output.audit.iter().any(|entry| { + entry.rsync_uri == expired_uri + && matches!(entry.result, AuditObjectResult::Skipped) + && entry.detail.as_deref() == Some("skipped: cached local output expired") + })); + assert!(output.audit.iter().any(|entry| { + entry.rsync_uri == bad_time_uri && matches!(entry.result, AuditObjectResult::Error) + })); + assert!(output.audit.iter().any(|entry| { + entry.rsync_uri == bad_prefix_uri + && matches!(entry.result, AuditObjectResult::Error) + && entry + .detail + .as_deref() + .unwrap_or("") + .contains("cached ROA local output parse failed") + })); +} + +#[test] +fn build_publication_point_audit_from_vcir_uses_vcir_metadata_and_overlays_child_and_object_audits() +{ + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); + vcir.related_artifacts + .retain(|artifact| artifact.artifact_role != VcirArtifactRole::Manifest); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), + }; + let runner_warnings = vec![Warning::new("runner warning")]; + let objects = crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: vec![Warning::new("objects warning")], + stats: crate::validation::objects::ObjectsStats::default(), + audit: vec![ObjectAuditEntry { + rsync_uri: "rsync://example.test/repo/issuer/a.roa".to_string(), + sha256_hex: sha256_hex(b"override-roa"), + kind: AuditObjectKind::Roa, + result: AuditObjectResult::Error, + detail: Some("overridden from object audit".to_string()), + }], + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + }; + let child_audits = vec![ObjectAuditEntry { + rsync_uri: vcir.child_entries[0].child_cert_rsync_uri.clone(), + sha256_hex: vcir.child_entries[0].child_cert_hash.clone(), + kind: AuditObjectKind::Certificate, + result: AuditObjectResult::Ok, + detail: Some("restored child CA instance from VCIR".to_string()), + }]; + + let audit = build_publication_point_audit_from_vcir( + &ca, + PublicationPointSource::VcirCurrentInstance, + Some("rsync"), + Some("rrdp_failed_rsync_failed"), + Some(456), + Some("rsync failed"), + Some(&vcir), + None, + &runner_warnings, + &objects, + &child_audits, + &[], + ); + + assert_eq!(audit.source, "vcir_current_instance"); + assert_eq!(audit.repo_sync_source.as_deref(), Some("rsync")); + assert_eq!( + audit.repo_sync_phase.as_deref(), + Some("rrdp_failed_rsync_failed") + ); + assert_eq!(audit.repo_sync_duration_ms, Some(456)); + assert_eq!(audit.repo_sync_error.as_deref(), Some("rsync failed")); + assert_eq!(audit.repo_terminal_state, "fallback_current_instance"); + assert_eq!(audit.objects[0].rsync_uri, vcir.current_manifest_rsync_uri); + assert_eq!(audit.objects[0].kind, AuditObjectKind::Manifest); + assert_eq!( + audit.this_update_rfc3339_utc, + vcir.validated_manifest_meta + .validated_manifest_this_update + .rfc3339_utc + ); + assert_eq!( + audit.next_update_rfc3339_utc, + vcir.validated_manifest_meta + .validated_manifest_next_update + .rfc3339_utc + ); + assert_eq!( + audit.verified_at_rfc3339_utc, + vcir.last_successful_validation_time.rfc3339_utc + ); + assert_eq!(audit.warnings.len(), 2); + assert!(audit.objects.iter().any(|entry| { + entry.rsync_uri == "rsync://example.test/repo/issuer/a.roa" + && matches!(entry.result, AuditObjectResult::Error) + && entry.detail.as_deref() == Some("overridden from object audit") + })); + assert!(audit.objects.iter().any(|entry| { + entry.rsync_uri == vcir.child_entries[0].child_cert_rsync_uri + && matches!(entry.result, AuditObjectResult::Ok) + })); +} + +#[test] +fn build_publication_point_audit_from_vcir_restores_reject_reason_with_legacy_fallback() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); + vcir.related_artifacts.push(VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: VcirArtifactKind::Roa, + uri: Some("rsync://example.test/repo/issuer/rejected-with-reason.roa".to_string()), + sha256: sha256_hex(b"rejected-with-reason"), + object_type: Some("roa".to_string()), + validation_status: VcirArtifactValidationStatus::Rejected, + reject_reason: Some("EE certificate path validation failed: test".to_string()), + }); + vcir.related_artifacts.push(VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: VcirArtifactKind::Roa, + uri: Some("rsync://example.test/repo/issuer/rejected-legacy.roa".to_string()), + sha256: sha256_hex(b"rejected-legacy"), + object_type: Some("roa".to_string()), + validation_status: VcirArtifactValidationStatus::Rejected, + reject_reason: None, + }); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + let objects = crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: Vec::new(), + stats: crate::validation::objects::ObjectsStats::default(), + audit: Vec::new(), + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + }; + + let audit = build_publication_point_audit_from_vcir( + &ca, + PublicationPointSource::VcirCurrentInstance, + None, + None, + None, + None, + Some(&vcir), + None, + &[], + &objects, + &[], + &[], + ); + + assert!(audit.objects.iter().any(|entry| { + entry.rsync_uri == "rsync://example.test/repo/issuer/rejected-with-reason.roa" + && matches!(entry.result, AuditObjectResult::Error) + && entry.detail.as_deref() == Some("EE certificate path validation failed: test") + })); + assert!(audit.objects.iter().any(|entry| { + entry.rsync_uri == "rsync://example.test/repo/issuer/rejected-legacy.roa" + && matches!(entry.result, AuditObjectResult::Error) + && entry.detail.as_deref() == Some(CACHED_REJECT_REASON_NOT_RECORDED) + })); + assert!( + audit.objects.iter().all(|entry| { + !matches!(entry.result, AuditObjectResult::Ok) || entry.detail.is_none() + }) + ); +} + +#[test] +fn build_publication_point_audit_from_pp_cache_projection_restores_reject_reason() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); + vcir.related_artifacts.push(VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: VcirArtifactKind::Roa, + uri: Some("rsync://example.test/repo/issuer/rejected-with-reason.roa".to_string()), + sha256: sha256_hex(b"rejected-with-reason"), + object_type: Some("roa".to_string()), + validation_status: VcirArtifactValidationStatus::Rejected, + reject_reason: Some("EE certificate path validation failed: test".to_string()), + }); + vcir.related_artifacts.push(VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: VcirArtifactKind::Roa, + uri: Some("rsync://example.test/repo/issuer/rejected-legacy.roa".to_string()), + sha256: sha256_hex(b"rejected-legacy"), + object_type: Some("roa".to_string()), + validation_status: VcirArtifactValidationStatus::Rejected, + reject_reason: None, + }); + let projection = PublicationPointCacheProjection::from_vcir_with_context( + &vcir, + "rsync://example.test/repo/issuer/".to_string(), + None, + [0x11; 32], + [0x22; 32], + [0x33; 32], + [0x44; 32], + [0x55; 32], + ) + .expect("build publication point projection"); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + let objects = crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: Vec::new(), + stats: crate::validation::objects::ObjectsStats::default(), + audit: Vec::new(), + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + }; + + let audit = build_publication_point_audit_from_publication_point_cache_projection( + &ca, + PublicationPointSource::PublicationPointCache, + None, + None, + None, + None, + &projection, + now, + &[], + &objects, + &[], + ); + + assert!(audit.objects.iter().any(|entry| { + entry.rsync_uri == "rsync://example.test/repo/issuer/rejected-with-reason.roa" + && matches!(entry.result, AuditObjectResult::Error) + && entry.detail.as_deref() == Some("EE certificate path validation failed: test") + })); + assert!(audit.objects.iter().any(|entry| { + entry.rsync_uri == "rsync://example.test/repo/issuer/rejected-legacy.roa" + && matches!(entry.result, AuditObjectResult::Error) + && entry.detail.as_deref() == Some(CACHED_REJECT_REASON_NOT_RECORDED) + })); +} + +#[test] +fn build_publication_point_audit_from_vcir_failed_no_cache_keeps_current_reject_only() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let vcir = sample_vcir_for_projection(now, &child_cert_hash); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: Some("https://example.test/notify.xml".to_string()), + }; + let objects = crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: Vec::new(), + stats: crate::validation::objects::ObjectsStats::default(), + audit: vec![ObjectAuditEntry { + rsync_uri: vcir.current_manifest_rsync_uri.clone(), + sha256_hex: sha256_hex(b"current-manifest"), + kind: AuditObjectKind::Manifest, + result: AuditObjectResult::Error, + detail: Some("manifest is not valid at validation_time".to_string()), + }], + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + }; + + let audit = build_publication_point_audit_from_vcir( + &ca, + PublicationPointSource::FailedFetchNoCache, + Some("rsync"), + Some("rsync_only_ok"), + Some(123), + None, + Some(&vcir), + None, + &[Warning::new("latest VCIR instance_gate expired")], + &objects, + &[], + &[], + ); + + assert_eq!(audit.source, "failed_fetch_no_cache"); + assert_eq!(audit.repo_terminal_state, "failed_no_cache"); + assert_eq!( + audit.this_update_rfc3339_utc, + vcir.validated_manifest_meta + .validated_manifest_this_update + .rfc3339_utc + ); + assert_eq!(audit.objects.len(), 1); + assert_eq!(audit.objects[0].rsync_uri, vcir.current_manifest_rsync_uri); + assert!(matches!(audit.objects[0].result, AuditObjectResult::Error)); + assert!( + !audit + .objects + .iter() + .any(|entry| entry.rsync_uri == "rsync://example.test/repo/issuer/a.roa"), + "failed-no-cache must not expand old VCIR related artifacts into current-run audit", + ); + assert!( + !audit + .objects + .iter() + .any(|entry| entry.rsync_uri == "rsync://example.test/repo/issuer/issuer.crl"), + "failed-no-cache must not expose old CRL as current-run CIR input", + ); +} + +#[test] +fn rejected_manifest_audit_entry_for_failed_fetch_uses_current_repo_hash() { + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let policy = Policy::default(); + let runner = sample_runner_with_ccr_accumulator(&store, &policy); + let manifest_uri = "rsync://example.test/repo/issuer/issuer.mft"; + let manifest_hash = sha256_hex(b"manifest-bytes"); + store + .put_blob_bytes_batch(&[(manifest_hash.clone(), b"manifest-bytes".to_vec())]) + .expect("put manifest bytes"); + store + .put_repository_view_entry(&crate::storage::RepositoryViewEntry { + rsync_uri: manifest_uri.to_string(), + current_hash: Some(manifest_hash.clone()), + repository_source: Some("rsync://example.test/repo/issuer/".to_string()), + object_type: Some("mft".to_string()), + state: crate::storage::RepositoryViewState::Present, + }) + .expect("put repository view"); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: manifest_uri.to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let entry = runner + .rejected_manifest_audit_entry_for_failed_fetch( + &ca, + &ManifestFreshError::StaleOrEarly { + this_update_rfc3339_utc: "2026-05-27T08:37:07Z".to_string(), + next_update_rfc3339_utc: "2026-05-28T10:01:07Z".to_string(), + validation_time_rfc3339_utc: "2026-05-28T10:11:00Z".to_string(), + }, + ) + .expect("rejected manifest audit entry"); + + assert_eq!(entry.rsync_uri, manifest_uri); + assert_eq!(entry.sha256_hex, manifest_hash); + assert_eq!(entry.kind, AuditObjectKind::Manifest); + assert_eq!(entry.result, AuditObjectResult::Error); + assert!( + entry + .detail + .as_deref() + .unwrap_or("") + .contains("manifest is not valid at validation_time") + ); +} + +#[test] +fn build_publication_point_audit_from_vcir_without_cached_inputs_returns_empty_listing() { + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: "rsync://example.test/repo/issuer/issuer.mft".to_string(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let audit = build_publication_point_audit_from_vcir( + &ca, + PublicationPointSource::FailedFetchNoCache, + Some("rsync"), + Some("rsync_only_failed"), + Some(789), + Some("load from network failed, fallback to cache"), + None, + None, + &[Warning::new("runner warning")], + &crate::validation::objects::ObjectsOutput { + vrps: Vec::new(), + aspas: Vec::new(), + router_keys: Vec::new(), + local_outputs_cache: Vec::new(), + warnings: vec![Warning::new("object warning")], + stats: crate::validation::objects::ObjectsStats::default(), + audit: Vec::new(), + roa_cache_stats: crate::validation::objects::RoaValidationCacheStats::default(), + roa_cache_object_meta: Vec::new(), + }, + &[], + &[], + ); + + assert_eq!(audit.source, "failed_fetch_no_cache"); + assert_eq!(audit.repo_sync_source.as_deref(), Some("rsync")); + assert_eq!(audit.repo_sync_phase.as_deref(), Some("rsync_only_failed")); + assert_eq!(audit.repo_sync_duration_ms, Some(789)); + assert_eq!( + audit.repo_sync_error.as_deref(), + Some("load from network failed, fallback to cache") + ); + assert_eq!(audit.repo_terminal_state, "failed_no_cache"); + assert!(audit.this_update_rfc3339_utc.is_empty()); + assert!(audit.next_update_rfc3339_utc.is_empty()); + assert!(audit.verified_at_rfc3339_utc.is_empty()); + assert_eq!(audit.warnings.len(), 2); + assert!(audit.objects.is_empty()); +} + +#[test] +fn effective_repo_sync_duration_uses_runtime_duration_for_failures() { + assert_eq!(effective_repo_sync_duration_ms(0, Some(12), false), 12); + assert_eq!(effective_repo_sync_duration_ms(5, Some(12), false), 12); + assert_eq!(effective_repo_sync_duration_ms(20, Some(12), false), 20); + assert_eq!(effective_repo_sync_duration_ms(5, None, false), 5); + assert_eq!(effective_repo_sync_duration_ms(5, Some(12), true), 5); +} + +#[test] +fn reconstruct_snapshot_from_vcir_reports_missing_manifest_and_related_raw_bytes() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let mut vcir = sample_vcir_for_projection(now, &child_cert_hash); + let dup_uri = "rsync://example.test/repo/issuer/dup.roa".to_string(); + vcir.related_artifacts.push(VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: VcirArtifactKind::Roa, + uri: Some(dup_uri.clone()), + sha256: sha256_hex(b"dup-roa-1"), + object_type: Some("roa".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }); + vcir.related_artifacts.push(VcirRelatedArtifact { + artifact_role: VcirArtifactRole::SignedObject, + artifact_kind: VcirArtifactKind::Roa, + uri: Some(dup_uri.clone()), + sha256: sha256_hex(b"dup-roa-2"), + object_type: Some("roa".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }); + vcir.related_artifacts.push(VcirRelatedArtifact { + artifact_role: VcirArtifactRole::IssuerCert, + artifact_kind: VcirArtifactKind::Cer, + uri: Some("rsync://example.test/repo/issuer/issuer.cer".to_string()), + sha256: sha256_hex(b"issuer-cert"), + object_type: Some("cer".to_string()), + validation_status: VcirArtifactValidationStatus::Accepted, + reject_reason: None, + }); + + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let mut warnings = Vec::new(); + assert!(reconstruct_snapshot_from_vcir(&store, &ca, &vcir, &mut warnings).is_none()); + assert!(warnings.iter().any(|warning| { + warning + .message + .contains("manifest raw bytes missing for VCIR audit reconstruction") + })); + + let manifest_bytes = b"manifest-bytes".to_vec(); + let current_crl_bytes = b"current-crl-bytes".to_vec(); + let child_bytes = b"child-cert".to_vec(); + let roa_bytes = b"roa-bytes".to_vec(); + for (bytes, uri, object_type) in [ + ( + manifest_bytes.clone(), + Some(vcir.manifest_rsync_uri.clone()), + Some("mft".to_string()), + ), + ( + current_crl_bytes, + Some(vcir.current_crl_rsync_uri.clone()), + Some("crl".to_string()), + ), + ( + child_bytes, + Some(vcir.child_entries[0].child_cert_rsync_uri.clone()), + Some("cer".to_string()), + ), + ( + roa_bytes, + Some("rsync://example.test/repo/issuer/a.roa".to_string()), + Some("roa".to_string()), + ), + ] { + let mut entry = RawByHashEntry::from_bytes(sha256_hex(&bytes), bytes); + if let Some(uri) = uri { + entry.origin_uris.push(uri); + } + entry.object_type = object_type; + entry.encoding = Some("der".to_string()); + store.put_raw_by_hash_entry(&entry).expect("put raw entry"); + } + + warnings.clear(); + let pack = reconstruct_snapshot_from_vcir(&store, &ca, &vcir, &mut warnings) + .expect("reconstruct pack with partial related artifacts"); + assert_eq!(pack.manifest_bytes, manifest_bytes); + assert_eq!(pack.files.len(), 3, "crl + child cert + roa only"); + assert!( + pack.files + .iter() + .any(|file| file.rsync_uri.ends_with("issuer.crl")) + ); + assert!( + pack.files + .iter() + .any(|file| file.rsync_uri.ends_with("child.cer")) + ); + assert!( + pack.files + .iter() + .any(|file| file.rsync_uri.ends_with("a.roa")) + ); + assert!( + !pack + .files + .iter() + .any(|file| file.rsync_uri.ends_with("issuer.cer")) + ); + assert!(warnings.iter().any(|warning| { + warning + .message + .contains("related artifact raw bytes missing for VCIR audit reconstruction") + })); +} + +#[test] +fn reconstruct_snapshot_from_vcir_reads_repo_bytes_without_raw_entries() { + let now = time::OffsetDateTime::now_utc(); + let child_cert_hash = sha256_hex(b"child-cert"); + let vcir = sample_vcir_for_projection(now, &child_cert_hash); + let ca = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(Vec::new()), + ca_certificate_rsync_uri: None, + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: "rsync://example.test/repo/issuer/".to_string(), + manifest_rsync_uri: vcir.manifest_rsync_uri.clone(), + publication_point_rsync_uri: "rsync://example.test/repo/issuer/".to_string(), + rrdp_notification_uri: None, + }; + + let store_dir = tempfile::tempdir().expect("store dir"); + let main_db = store_dir.path().join("work-db"); + let repo_bytes_db = store_dir.path().join("repo-bytes.db"); + let store = RocksStore::open_with_external_repo_bytes(&main_db, &repo_bytes_db) + .expect("open rocksdb with external repo bytes"); + let repo_blobs = [ + b"manifest-bytes".to_vec(), + b"current-crl-bytes".to_vec(), + b"child-cert".to_vec(), + b"roa-bytes".to_vec(), + b"aspa-bytes".to_vec(), + ] + .into_iter() + .map(|bytes| (sha256_hex(&bytes), bytes)) + .collect::>(); + store + .put_blob_bytes_batch(&repo_blobs) + .expect("put external repo bytes"); + + let manifest_hash = vcir + .related_artifacts + .iter() + .find(|artifact| artifact.artifact_role == VcirArtifactRole::Manifest) + .expect("manifest artifact") + .sha256 + .clone(); + assert!( + store + .get_raw_by_hash_entry(&manifest_hash) + .expect("raw manifest lookup") + .is_none(), + "repo object bytes must not require raw_by_hash entries" + ); + + let mut warnings = Vec::new(); + let pack = reconstruct_snapshot_from_vcir(&store, &ca, &vcir, &mut warnings) + .expect("reconstruct pack from external repo bytes"); + assert_eq!(pack.manifest_bytes, b"manifest-bytes".to_vec()); + assert_eq!(pack.files.len(), 4, "crl + child cert + roa + aspa"); + assert!( + warnings.iter().all(|warning| { + !warning + .message + .contains("raw bytes missing for VCIR audit reconstruction") + }), + "external repo bytes should satisfy VCIR audit reconstruction without raw warnings" + ); +} + +#[test] +fn runner_dedup_paths_execute_with_timing_enabled() { + let fixture_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/repository/rpki.cernet.net/repo/cernet/0"); + let rsync_base_uri = "rsync://rpki.cernet.net/repo/cernet/0/".to_string(); + let manifest_file = "05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft"; + let manifest_rsync_uri = format!("{rsync_base_uri}{manifest_file}"); + let fixture_manifest_bytes = + std::fs::read(fixture_dir.join(manifest_file)).expect("read manifest fixture"); + let fixture_manifest = + crate::data_model::manifest::ManifestObject::decode_der(&fixture_manifest_bytes) + .expect("decode manifest fixture"); + let validation_time = fixture_manifest.manifest.this_update + time::Duration::seconds(60); + let store_dir = tempfile::tempdir().expect("store dir"); + let store = RocksStore::open(store_dir.path()).expect("open rocksdb"); + let issuer_ca_der = std::fs::read( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ), + ) + .expect("read issuer ca fixture"); + let issuer_ca = ResourceCertificate::decode_der(&issuer_ca_der).expect("decode issuer ca"); + let handle = CaInstanceHandle { + depth: 0, + tal_id: "test-tal".to_string(), + parent_manifest_rsync_uri: None, + ca_certificate: CaCertificateRef::inline_der(issuer_ca_der), + ca_certificate_rsync_uri: Some("rsync://rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer".to_string()), + effective_ip_resources: issuer_ca.tbs.extensions.ip_resources.clone(), + effective_as_resources: issuer_ca.tbs.extensions.as_resources.clone(), + rsync_base_uri: rsync_base_uri.clone(), + manifest_rsync_uri: manifest_rsync_uri.clone(), + publication_point_rsync_uri: rsync_base_uri.clone(), + rrdp_notification_uri: Some("https://example.test/notification.xml".to_string()), + }; + let timing = crate::analysis::timing::TimingHandle::new(crate::analysis::timing::TimingMeta { + recorded_at_utc_rfc3339: "2026-03-11T00:00:00Z".to_string(), + validation_time_utc_rfc3339: "2026-03-11T00:00:00Z".to_string(), + tal_url: None, + db_path: None, + }); + let policy_rrdp = Policy::default(); + let runner_rrdp = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy_rrdp, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), + validation_time, + timing: Some(timing.clone()), + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: true, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + }; + let first = runner_rrdp + .run_publication_point(&handle) + .expect("rrdp fallback to rsync"); + assert_eq!(first.source, PublicationPointSource::Fresh); + let second = runner_rrdp + .run_publication_point(&handle) + .expect("rrdp dedup skip"); + assert_eq!(second.source, PublicationPointSource::Fresh); + + let policy_rsync = Policy { + sync_preference: crate::policy::SyncPreference::RsyncOnly, + ..Policy::default() + }; + let runner_rsync = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy_rsync, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir), + validation_time, + timing: Some(timing), + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: true, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: None, + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: false, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: false, + }; + let third = runner_rsync + .run_publication_point(&handle) + .expect("rsync first run"); + assert_eq!(third.source, PublicationPointSource::Fresh); + let fourth = runner_rsync + .run_publication_point(&handle) + .expect("rsync dedup run"); + assert_eq!(fourth.source, PublicationPointSource::Fresh); + assert_eq!( + crate::fetch::rsync::normalize_rsync_base_uri("rsync://example.test/repo"), + "rsync://example.test/repo/" + ); +} + +#[derive(Debug, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +struct RipeRootFinalizeFixtureEvent { + event_type: String, + validation_time: String, + pp_manifest_uri: Option, + object_uri: Option, + sha256: Option, + object_type: Option, + result: Option, + reason: Option, +} + +#[derive(Debug)] +struct RipeRootFinalizeObject { + uri: String, + sha256_hex: String, + object_type: String, + result: String, + reason: Option, +} + +struct RipeRootFinalizeFixture { + manifest_uri: String, + publication_point_uri: String, + validation_time: time::OffsetDateTime, + manifest_sha256_hex: String, + objects: Vec, +} + +fn load_ripe_root_finalize_fixture( + fixture_root: &std::path::Path, + run_id: &str, +) -> RipeRootFinalizeFixture { + let events_path = fixture_root + .join(format!("run_{run_id}")) + .join("ripe-root-events.jsonl"); + let file = std::fs::File::open(&events_path) + .unwrap_or_else(|e| panic!("open fixture events {} failed: {e}", events_path.display())); + let reader = std::io::BufReader::new(file); + let mut manifest_uri = None; + let mut validation_time = None; + let mut manifest_sha256_hex = None; + let mut objects = Vec::new(); + + for line in std::io::BufRead::lines(reader) { + let line = line.expect("read fixture event line"); + let event: RipeRootFinalizeFixtureEvent = + serde_json::from_str(&line).expect("decode fixture event"); + if event.event_type == "publication_point" { + manifest_uri = event.pp_manifest_uri; + validation_time = Some( + time::OffsetDateTime::parse( + &event.validation_time, + &time::format_description::well_known::Rfc3339, + ) + .expect("parse validation_time"), + ); + continue; + } + if event.event_type != "object" { + continue; + } + let uri = event.object_uri.expect("fixture object uri"); + let sha256_hex = event.sha256.expect("fixture object sha256"); + let object_type = event.object_type.expect("fixture object type"); + let result = event.result.expect("fixture object result"); + if object_type == "manifest" { + manifest_sha256_hex = Some(sha256_hex.clone()); + } + objects.push(RipeRootFinalizeObject { + uri, + sha256_hex, + object_type, + result, + reason: event.reason, + }); + } + + let manifest_uri = manifest_uri.expect("fixture publication_point event"); + let publication_point_uri = manifest_uri + .rsplit_once('/') + .map(|(parent, _)| format!("{parent}/")) + .expect("manifest uri parent"); + RipeRootFinalizeFixture { + manifest_uri, + publication_point_uri, + validation_time: validation_time.expect("fixture validation_time"), + manifest_sha256_hex: manifest_sha256_hex.expect("fixture manifest object"), + objects, + } +} + +fn pack_file_from_fixture_object( + object: &RipeRootFinalizeObject, + repo_bytes: &Arc, +) -> PackFile { + PackFile::from_lazy_repo_bytes( + object.uri.clone(), + object.sha256_hex.clone(), + sha256_hex_to_32(&object.sha256_hex), + repo_bytes.clone(), + ) +} + +fn child_audit_from_fixture_object(object: &RipeRootFinalizeObject) -> ObjectAuditEntry { + ObjectAuditEntry { + rsync_uri: object.uri.clone(), + sha256_hex: object.sha256_hex.clone(), + kind: AuditObjectKind::Certificate, + result: match object.result.as_str() { + "ok" => AuditObjectResult::Ok, + "skipped" => AuditObjectResult::Skipped, + _ => AuditObjectResult::Error, + }, + detail: object.reason.clone(), + } +} + +fn discovered_child_from_fixture_object( + issuer: &CaInstanceHandle, + object: &RipeRootFinalizeObject, + child_entry_projection: Option, +) -> DiscoveredChildCaInstance { + let stem = object + .uri + .rsplit_once('/') + .map(|(_, file)| file.trim_end_matches(".cer")) + .unwrap_or("child"); + let child_publication_point = format!("{}synthetic-child-{stem}/", issuer.rsync_base_uri); + let child_manifest = format!("{child_publication_point}child.mft"); + DiscoveredChildCaInstance { + handle: CaInstanceHandle { + depth: issuer.depth + 1, + tal_id: issuer.tal_id.clone(), + parent_manifest_rsync_uri: Some(issuer.manifest_rsync_uri.clone()), + ca_certificate: CaCertificateRef::repo_bytes(object.sha256_hex.clone()), + ca_certificate_rsync_uri: Some(object.uri.clone()), + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: child_publication_point.clone(), + manifest_rsync_uri: child_manifest, + publication_point_rsync_uri: child_publication_point, + rrdp_notification_uri: issuer.rrdp_notification_uri.clone(), + }, + discovered_from: crate::audit::DiscoveredFrom { + parent_manifest_rsync_uri: issuer.manifest_rsync_uri.clone(), + child_ca_certificate_rsync_uri: object.uri.clone(), + child_ca_certificate_sha256_hex: object.sha256_hex.clone(), + }, + child_entry_projection, + } +} + +fn child_entry_projection_from_fixture_object( + store: &RocksStore, + object: &RipeRootFinalizeObject, +) -> DiscoveredChildEntryProjection { + let child_der = store + .get_blob_bytes(&object.sha256_hex) + .expect("load child certificate bytes for projection") + .expect("child certificate bytes exist for projection"); + let child_cert = + ResourceCertificate::decode_der(&child_der).expect("decode child certificate projection"); + let child_ski = child_cert + .tbs + .extensions + .subject_key_identifier + .as_ref() + .expect("child certificate projection SKI"); + DiscoveredChildEntryProjection { + child_ski: hex::encode(child_ski), + } +} + +struct RipeRootChildEntryProfile { + count: usize, + load_der_nanos: u128, + decode_cert_nanos: u128, + build_entry_nanos: u128, +} + +fn profile_ripe_root_child_entry_build( + store: &RocksStore, + discovered_children: &[DiscoveredChildCaInstance], + validation_time: time::OffsetDateTime, +) -> Result { + let mut out = Vec::with_capacity(discovered_children.len()); + let mut load_der_nanos = 0; + let mut decode_cert_nanos = 0; + let mut build_entry_nanos = 0; + for child in discovered_children { + let load_started = std::time::Instant::now(); + let child_der = child.handle.ca_certificate_der(store)?; + load_der_nanos += load_started.elapsed().as_nanos(); + + let decode_started = std::time::Instant::now(); + let child_cert = ResourceCertificate::decode_der(child_der.as_ref()) + .map_err(|e| format!("decode child certificate for VCIR failed: {e}"))?; + decode_cert_nanos += decode_started.elapsed().as_nanos(); + + let build_started = std::time::Instant::now(); + let child_ski = child_cert + .tbs + .extensions + .subject_key_identifier + .as_ref() + .ok_or_else(|| "child certificate missing SubjectKeyIdentifier".to_string())?; + out.push(VcirChildEntry { + child_manifest_rsync_uri: child.handle.manifest_rsync_uri.clone(), + child_cert_rsync_uri: child.discovered_from.child_ca_certificate_rsync_uri.clone(), + child_cert_hash: child + .discovered_from + .child_ca_certificate_sha256_hex + .clone(), + child_ski: hex::encode(child_ski), + child_rsync_base_uri: child.handle.rsync_base_uri.clone(), + child_publication_point_rsync_uri: child.handle.publication_point_rsync_uri.clone(), + child_rrdp_notification_uri: child.handle.rrdp_notification_uri.clone(), + child_effective_ip_resources: child.handle.effective_ip_resources.clone(), + child_effective_as_resources: child.handle.effective_as_resources.clone(), + accepted_at_validation_time: PackTime::from_utc_offset_datetime(validation_time), + }); + build_entry_nanos += build_started.elapsed().as_nanos(); + } + Ok(RipeRootChildEntryProfile { + count: out.len(), + load_der_nanos, + decode_cert_nanos, + build_entry_nanos, + }) +} + +#[test] +#[ignore = "manual performance repro: requires target/ripe-root-finalize-repro repo-bytes fixture"] +fn ripe_root_finalize_repro_from_remote_fixture() { + let fixture_root = std::env::var("RPKI_RIPE_ROOT_FINALIZE_FIXTURE") + .map(std::path::PathBuf::from) + .unwrap_or_else(|_| { + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("target/ripe-root-finalize-repro") + }); + let run_id = std::env::var("RPKI_RIPE_ROOT_FINALIZE_RUN").unwrap_or_else(|_| "0262".into()); + assert!( + fixture_root.exists(), + "fixture root missing: {}; expected copied remote fixture", + fixture_root.display() + ); + + let fixture = load_ripe_root_finalize_fixture(&fixture_root, &run_id); + let repo_bytes_db = fixture_root.join("db/repo-bytes.db"); + assert!( + repo_bytes_db.exists(), + "repo-bytes fixture missing: {}", + repo_bytes_db.display() + ); + + let store_dir = tempfile::tempdir().expect("store dir"); + let work_db = store_dir.path().join("work-db"); + let store = RocksStore::open_with_external_repo_bytes(&work_db, &repo_bytes_db) + .expect("open work-db with external repo-bytes"); + let repo_bytes = Arc::new( + store + .external_repo_bytes_ref() + .expect("external repo bytes") + .clone(), + ); + + let manifest_bytes = store + .get_blob_bytes(&fixture.manifest_sha256_hex) + .expect("load fixture manifest bytes") + .expect("fixture manifest bytes exist"); + let manifest = ManifestObject::decode_der(&manifest_bytes).expect("decode fixture manifest"); + let mut files = Vec::with_capacity(fixture.objects.len().saturating_sub(1)); + let mut child_audits = Vec::new(); + let current_ca_hash = "007ad0c291b01ede4bd60e1204074ce3f7192186a022c9577cef5d8e91d5171a"; + let current_ca_uri = "rsync://rpki.ripe.net/repository/aca/KpSo3VVK5wEHIJnHC2QHVV3d5mk.cer"; + let parent_manifest_uri = + "rsync://rpki.ripe.net/repository/aca/7DNNDzoYvgAht7joQih2Qayxcxo.mft"; + let ca = CaInstanceHandle { + depth: 1, + tal_id: "ripe-ncc".to_string(), + parent_manifest_rsync_uri: Some(parent_manifest_uri.to_string()), + ca_certificate: CaCertificateRef::repo_bytes(current_ca_hash.to_string()), + ca_certificate_rsync_uri: Some(current_ca_uri.to_string()), + effective_ip_resources: None, + effective_as_resources: None, + rsync_base_uri: fixture.publication_point_uri.clone(), + manifest_rsync_uri: fixture.manifest_uri.clone(), + publication_point_rsync_uri: fixture.publication_point_uri.clone(), + rrdp_notification_uri: Some("https://rrdp.ripe.net/notification.xml".to_string()), + }; + let mut discovered_children = Vec::new(); + let use_child_projection = std::env::var("RPKI_RIPE_ROOT_FINALIZE_USE_CHILD_PROJECTION") + .map(|value| value != "0" && value.to_ascii_lowercase() != "false") + .unwrap_or(false); + for object in &fixture.objects { + if object.uri == fixture.manifest_uri { + continue; + } + files.push(pack_file_from_fixture_object(object, &repo_bytes)); + if object.object_type == "certificate" { + child_audits.push(child_audit_from_fixture_object(object)); + if object.result == "ok" { + let child_entry_projection = use_child_projection + .then(|| child_entry_projection_from_fixture_object(&store, object)); + discovered_children.push(discovered_child_from_fixture_object( + &ca, + object, + child_entry_projection, + )); + } + } + } + if std::env::var("RPKI_RIPE_ROOT_FINALIZE_PROFILE_CHILD").is_ok() { + let profile_children = fixture + .objects + .iter() + .filter(|object| object.object_type == "certificate" && object.result == "ok") + .map(|object| discovered_child_from_fixture_object(&ca, object, None)) + .collect::>(); + let profile_started = std::time::Instant::now(); + let profile = + profile_ripe_root_child_entry_build(&store, &profile_children, fixture.validation_time) + .expect("profile child entry build"); + eprintln!( + "ripe root child-entry profile: run={} count={} total_ms={} load_der_ms={:.3} decode_cert_ms={:.3} build_entry_ms={:.3}", + run_id, + profile.count, + profile_started.elapsed().as_millis(), + profile.load_der_nanos as f64 / 1_000_000.0, + profile.decode_cert_nanos as f64 / 1_000_000.0, + profile.build_entry_nanos as f64 / 1_000_000.0, + ); + if std::env::var("RPKI_RIPE_ROOT_FINALIZE_ONLY_CHILD_PROFILE").is_ok() { + assert!(profile.count > 22_000); + return; + } + } + + let fresh_point = FreshValidatedPublicationPoint { + manifest_rsync_uri: fixture.manifest_uri.clone(), + publication_point_rsync_uri: fixture.publication_point_uri.clone(), + manifest_number_be: manifest.manifest.manifest_number.bytes_be.clone(), + this_update: PackTime::from_utc_offset_datetime(manifest.manifest.this_update), + next_update: PackTime::from_utc_offset_datetime(manifest.manifest.next_update), + verified_at: PackTime::from_utc_offset_datetime(fixture.validation_time), + manifest_bytes, + files, + }; + let policy = Policy::default(); + let enable_ccr_accumulator = std::env::var("RPKI_RIPE_ROOT_FINALIZE_CCR") + .map(|value| value != "0" && value.to_ascii_lowercase() != "false") + .unwrap_or(true); + let runner = Rpkiv1PublicationPointRunner { + store: &store, + policy: &policy, + http_fetcher: &NeverHttpFetcher, + rsync_fetcher: &FailingRsyncFetcher, + validation_time: fixture.validation_time, + timing: None, + download_log: None, + replay_archive_index: None, + replay_delta_index: None, + rrdp_dedup: false, + rrdp_repo_cache: Mutex::new(HashMap::new()), + rsync_dedup: false, + rsync_repo_cache: Mutex::new(HashMap::new()), + current_repo_index: None, + repo_sync_runtime: None, + parallel_phase2_config: None, + parallel_roa_worker_pool: None, + ccr_accumulator: enable_ccr_accumulator + .then(|| Mutex::new(CcrAccumulator::new(Vec::new()))), + persist_vcir: true, + enable_roa_validation_cache: false, + enable_child_certificate_validation_cache: true, + publication_point_cache_observe_only: false, + enable_publication_point_validation_cache: true, + }; + + eprintln!( + "ripe root finalize repro setup: run={} ccr_accumulator={} child_projection={} objects={} files={} child_audits={} discovered_children={}", + run_id, + enable_ccr_accumulator, + use_child_projection, + fixture.objects.len(), + fresh_point.files.len(), + child_audits.len(), + discovered_children.len() + ); + let started = std::time::Instant::now(); + let output = runner + .finalize_fresh_publication_point_from_reducer( + &ca, + &fresh_point, + Vec::new(), + empty_objects_output(), + child_audits, + discovered_children, + Some("rrdp"), + Some("rrdp_ok"), + 0, + None, + ) + .expect("finalize fixture publication point"); + let finalize_ms = started.elapsed().as_millis(); + eprintln!( + "ripe root finalize repro timing: run={} finalize_ms={} snapshot_pack_ms={} persist_vcir_ms={} build_vcir_ms={} child_entries_ms={} related_artifacts_ms={} replace_vcir_ms={} replace_vcir_encode_ms={} replace_vcir_write_batch_ms={} ccr_projection_build_ms={} audit_build_ms={}", + run_id, + finalize_ms, + output.snapshot_pack_ms, + output.persist_vcir_ms, + output.persist_vcir_timing.build_vcir_ms, + output.persist_vcir_timing.build_vcir.child_entries_ms, + output.persist_vcir_timing.build_vcir.related_artifacts_ms, + output.persist_vcir_timing.replace_vcir_ms, + output.persist_vcir_timing.replace_vcir.vcir_encode_ms, + output.persist_vcir_timing.replace_vcir.write_batch_ms, + output.ccr_projection_build_ms, + output.audit_build_ms, + ); + eprintln!( + "ripe root finalize repro result: audit_objects={} discovered_children={} ccr_manifest_count={}", + output.result.audit.objects.len(), + output.result.discovered_children.len(), + runner + .ccr_accumulator_snapshot() + .map(|snapshot| snapshot.manifest_count()) + .unwrap_or(0), + ); + + assert!(fresh_point.files.len() > 22_000); + assert!(output.result.discovered_children.len() > 22_000); + if enable_ccr_accumulator { + assert_eq!( + runner + .ccr_accumulator_snapshot() + .expect("ccr snapshot") + .manifest_count(), + 1 + ); + } +} diff --git a/crates/panda-rpki-validator/src/validation/x509_name.rs b/crates/panda-rpki-validator/src/validation/x509_name.rs new file mode 100644 index 0000000..6ef725b --- /dev/null +++ b/crates/panda-rpki-validator/src/validation/x509_name.rs @@ -0,0 +1,85 @@ +use crate::data_model::common::X509NameDer; +use x509_parser::prelude::FromDer; + +fn canonicalize(name: &X509NameDer) -> Option { + let (rem, parsed) = x509_parser::x509::X509Name::from_der(name.as_raw()).ok()?; + if !rem.is_empty() { + return None; + } + Some(parsed.to_string()) +} + +/// Compare two X.509 distinguished names using a tolerant semantic comparison. +/// +/// RPKI repositories in the wild sometimes encode the same name using different +/// ASN.1 string types (e.g., PrintableString vs UTF8String) while remaining +/// semantically equivalent. RFC 5280 path validation requires name matching, but +/// DER byte equality is too strict for interoperability. +pub fn x509_names_equivalent(a: &X509NameDer, b: &X509NameDer) -> bool { + let Some(ca) = canonicalize(a) else { + return a == b; + }; + let Some(cb) = canonicalize(b) else { + return a == b; + }; + ca == cb +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn x509_names_equivalent_falls_back_to_der_equality_when_parse_fails() { + // Invalid tag (not a SEQUENCE) makes x509-parser fail and forces DER equality fallback. + let a = X509NameDer(vec![0x01, 0x00]); + let b = X509NameDer(vec![0x01, 0x00]); + assert!(x509_names_equivalent(&a, &b)); + } + + #[test] + fn x509_names_equivalent_compares_semantic_names_when_parse_succeeds() { + let cert_der = std::fs::read( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ) + .expect("read certificate fixture"); + + let (_rem, cert) = + x509_parser::parse_x509_certificate(&cert_der).expect("parse certificate fixture"); + + let subject = X509NameDer(cert.tbs_certificate.subject.as_raw().to_vec()); + let issuer = X509NameDer(cert.tbs_certificate.issuer.as_raw().to_vec()); + + assert!(x509_names_equivalent(&subject, &subject)); + assert!(!x509_names_equivalent(&subject, &issuer)); + } + + #[test] + fn x509_names_equivalent_falls_back_when_name_has_trailing_bytes() { + // Use a real name DER and append a trailing byte so parsing yields leftover `rem`. + let cert_der = std::fs::read( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ) + .expect("read certificate fixture"); + let (_rem, cert) = + x509_parser::parse_x509_certificate(&cert_der).expect("parse certificate fixture"); + let mut name = cert.tbs_certificate.subject.as_raw().to_vec(); + name.push(0x00); + let a = X509NameDer(name.clone()); + let b = X509NameDer(name); + assert!(x509_names_equivalent(&a, &b)); + } + + #[test] + fn x509_names_equivalent_falls_back_when_one_side_fails_to_parse() { + let cert_der = std::fs::read( + "tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer", + ) + .expect("read certificate fixture"); + let (_rem, cert) = + x509_parser::parse_x509_certificate(&cert_der).expect("parse certificate fixture"); + let good = X509NameDer(cert.tbs_certificate.subject.as_raw().to_vec()); + let bad = X509NameDer(vec![0x01, 0x00]); + assert!(!x509_names_equivalent(&good, &bad)); + } +} diff --git a/crates/panda-rpki-validator/tests/fixtures/0099DEAB073EFD74C250C0A382B25012B5082AEE.crl b/crates/panda-rpki-validator/tests/fixtures/0099DEAB073EFD74C250C0A382B25012B5082AEE.crl new file mode 100644 index 0000000..e5b446a Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/0099DEAB073EFD74C250C0A382B25012B5082AEE.crl differ diff --git a/crates/panda-rpki-validator/tests/fixtures/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.crl b/crates/panda-rpki-validator/tests/fixtures/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.crl new file mode 100644 index 0000000..6b488ad Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.crl differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/E0AAy4jgV_BtY7GQELCxALV6Q5U.roa b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/E0AAy4jgV_BtY7GQELCxALV6Q5U.roa new file mode 100644 index 0000000..f6112d8 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/E0AAy4jgV_BtY7GQELCxALV6Q5U.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/IdgNYO8v_dEbdoPuHFTpsjA3l0U.roa b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/IdgNYO8v_dEbdoPuHFTpsjA3l0U.roa new file mode 100644 index 0000000..83ded42 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/IdgNYO8v_dEbdoPuHFTpsjA3l0U.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/R-lVU1XGsAeqzV1Fv0HjOD6ZFkE.cer b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/R-lVU1XGsAeqzV1Fv0HjOD6ZFkE.cer new file mode 100644 index 0000000..8f2d9a7 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/R-lVU1XGsAeqzV1Fv0HjOD6ZFkE.cer differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/UDn6-ZD0WTj18D8nHih3D--ZO_s.roa b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/UDn6-ZD0WTj18D8nHih3D--ZO_s.roa new file mode 100644 index 0000000..f4bd855 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/UDn6-ZD0WTj18D8nHih3D--ZO_s.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/Vvx1bC8uAflbQPltsuM_idhPJzQ.roa b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/Vvx1bC8uAflbQPltsuM_idhPJzQ.roa new file mode 100644 index 0000000..773266f Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/Vvx1bC8uAflbQPltsuM_idhPJzQ.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/ZW5EIqvxKWSSAOsBmoFfKxIjbpI.cer b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/ZW5EIqvxKWSSAOsBmoFfKxIjbpI.cer new file mode 100644 index 0000000..2795311 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/ZW5EIqvxKWSSAOsBmoFfKxIjbpI.cer differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.crl b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.crl new file mode 100644 index 0000000..72755f7 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.crl differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.mft b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.mft new file mode 100644 index 0000000..215819d Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/bW-_qXU9uNhGQz21NR2ansB8lr0.mft differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/fDvN8JDIeg8h7b5wWCFK_F7J_SY.roa b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/fDvN8JDIeg8h7b5wWCFK_F7J_SY.roa new file mode 100644 index 0000000..009e866 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/fDvN8JDIeg8h7b5wWCFK_F7J_SY.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/fykdvBDNLvaFDLOeKcvquSaaNlM.roa b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/fykdvBDNLvaFDLOeKcvquSaaNlM.roa new file mode 100644 index 0000000..14fb352 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/fykdvBDNLvaFDLOeKcvquSaaNlM.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/lAN537Pqyt3IsX0kCwc4qs0Cejk.roa b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/lAN537Pqyt3IsX0kCwc4qs0Cejk.roa new file mode 100644 index 0000000..3efb0d3 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/lAN537Pqyt3IsX0kCwc4qs0Cejk.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/laI1_vEYtlNzj-K0SXR_yGpd1TA.gbr b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/laI1_vEYtlNzj-K0SXR_yGpd1TA.gbr new file mode 100644 index 0000000..9a938f6 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/laI1_vEYtlNzj-K0SXR_yGpd1TA.gbr differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/ovsCA/IOUcOeBGM_Tb4dwfvswY4bnNZYY.crl b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/ovsCA/IOUcOeBGM_Tb4dwfvswY4bnNZYY.crl new file mode 100644 index 0000000..1a87a92 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/ovsCA/IOUcOeBGM_Tb4dwfvswY4bnNZYY.crl differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/ovsCA/IOUcOeBGM_Tb4dwfvswY4bnNZYY.mft b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/ovsCA/IOUcOeBGM_Tb4dwfvswY4bnNZYY.mft new file mode 100644 index 0000000..a08bcd9 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/ovsCA/IOUcOeBGM_Tb4dwfvswY4bnNZYY.mft differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/xF8fZFbI8NRA4qk_N5CLpw6Nmj8.roa b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/xF8fZFbI8NRA4qk_N5CLpw6Nmj8.roa new file mode 100644 index 0000000..65a21e3 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/xF8fZFbI8NRA4qk_N5CLpw6Nmj8.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/y9dGCqfvXd6vKRjYx3GJikg6pSw.roa b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/y9dGCqfvXd6vKRjYx3GJikg6pSw.roa new file mode 100644 index 0000000..770f12c Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/ca.rg.net/rpki/RGnet-OU/y9dGCqfvXd6vKRjYx3GJikg6pSw.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/111wEGLfsrCC1YXvRd90Mcgv3so.roa b/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/111wEGLfsrCC1YXvRd90Mcgv3so.roa new file mode 100644 index 0000000..736f0b4 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/111wEGLfsrCC1YXvRd90Mcgv3so.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/5m80fwYws_3FiFD7JiQjAqZ1RYQ.asa b/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/5m80fwYws_3FiFD7JiQjAqZ1RYQ.asa new file mode 100644 index 0000000..08d7661 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/5m80fwYws_3FiFD7JiQjAqZ1RYQ.asa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/9X0AhXWTJDl8lJhfOwvnac-42CA.spl b/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/9X0AhXWTJDl8lJhfOwvnac-42CA.spl new file mode 100644 index 0000000..f3540c9 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/9X0AhXWTJDl8lJhfOwvnac-42CA.spl differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/D53F-thyu-FB_Q-Qnu54dMG6kUc.roa b/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/D53F-thyu-FB_Q-Qnu54dMG6kUc.roa new file mode 100644 index 0000000..f64e9c8 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/D53F-thyu-FB_Q-Qnu54dMG6kUc.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/euv64En05073B5-r95s2Uu_UwJg.roa b/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/euv64En05073B5-r95s2Uu_UwJg.roa new file mode 100644 index 0000000..bee24c3 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/euv64En05073B5-r95s2Uu_UwJg.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/yqgF26w2R0m5sRVZCrbvD5cM29g.crl b/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/yqgF26w2R0m5sRVZCrbvD5cM29g.crl new file mode 100644 index 0000000..44dbd93 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/yqgF26w2R0m5sRVZCrbvD5cM29g.crl differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/yqgF26w2R0m5sRVZCrbvD5cM29g.mft b/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/yqgF26w2R0m5sRVZCrbvD5cM29g.mft new file mode 100644 index 0000000..dde9b49 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/yqgF26w2R0m5sRVZCrbvD5cM29g.mft differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/zIlYN-tDLuEj0BDwZSi5PS1eIDI.roa b/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/zIlYN-tDLuEj0BDwZSi5PS1eIDI.roa new file mode 100644 index 0000000..aa556c3 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nljobsnijders/zIlYN-tDLuEj0BDwZSi5PS1eIDI.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nlrssf/cdFOuyVdwFjUv6WlHJP3P4MKuI8.crl b/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nlrssf/cdFOuyVdwFjUv6WlHJP3P4MKuI8.crl new file mode 100644 index 0000000..b9b5bec Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nlrssf/cdFOuyVdwFjUv6WlHJP3P4MKuI8.crl differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nlrssf/cdFOuyVdwFjUv6WlHJP3P4MKuI8.mft b/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nlrssf/cdFOuyVdwFjUv6WlHJP3P4MKuI8.mft new file mode 100644 index 0000000..f629855 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/chloe.sobornost.net/rpki/RIPE-nlrssf/cdFOuyVdwFjUv6WlHJP3P4MKuI8.mft differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer new file mode 100644 index 0000000..c776da5 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.apnic.net/repository/B527EF581D6611E2BB468F7C72FD1FF2/BfycW4hQb3wNP4YsiJW-1n6fjro.cer differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.crl b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.crl new file mode 100644 index 0000000..6b488ad Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.crl differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft new file mode 100644 index 0000000..51cbb26 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/05FC9C5B88506F7C0D3F862C8895BED67E9F8EBA.mft differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142067.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142067.roa new file mode 100644 index 0000000..013a577 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142067.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142068.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142068.roa new file mode 100644 index 0000000..ef5fe69 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142068.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142069.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142069.roa new file mode 100644 index 0000000..abccbe2 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142069.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142070.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142070.roa new file mode 100644 index 0000000..30d1fd1 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142070.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142071.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142071.roa new file mode 100644 index 0000000..b3a9878 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142071.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142072.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142072.roa new file mode 100644 index 0000000..7a447f7 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142072.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142073.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142073.roa new file mode 100644 index 0000000..928dd08 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142073.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142074.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142074.roa new file mode 100644 index 0000000..abf8125 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142074.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142075.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142075.roa new file mode 100644 index 0000000..755181c Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142075.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142076.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142076.roa new file mode 100644 index 0000000..8aef0a5 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142076.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142077.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142077.roa new file mode 100644 index 0000000..73ef213 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142077.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142078.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142078.roa new file mode 100644 index 0000000..ab5ea8d Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142078.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142079.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142079.roa new file mode 100644 index 0000000..c86db63 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142079.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142080.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142080.roa new file mode 100644 index 0000000..c8cb6ec Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142080.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142081.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142081.roa new file mode 100644 index 0000000..c0c0f82 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142081.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142082.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142082.roa new file mode 100644 index 0000000..10f411d Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142082.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142083.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142083.roa new file mode 100644 index 0000000..da4b57c Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142083.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142084.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142084.roa new file mode 100644 index 0000000..069ccd7 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142084.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142085.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142085.roa new file mode 100644 index 0000000..bad4a99 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142085.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142086.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142086.roa new file mode 100644 index 0000000..52a6532 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142086.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142087.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142087.roa new file mode 100644 index 0000000..5816498 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142087.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142088.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142088.roa new file mode 100644 index 0000000..5548b91 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142088.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142089.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142089.roa new file mode 100644 index 0000000..fb9e080 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142089.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142090.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142090.roa new file mode 100644 index 0000000..87b4a0a Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142090.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142091.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142091.roa new file mode 100644 index 0000000..d5d7e5f Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142091.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142092.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142092.roa new file mode 100644 index 0000000..1429773 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142092.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142093.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142093.roa new file mode 100644 index 0000000..b51b082 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142093.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142094.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142094.roa new file mode 100644 index 0000000..07e1da2 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142094.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142095.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142095.roa new file mode 100644 index 0000000..d401660 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142095.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142096.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142096.roa new file mode 100644 index 0000000..f50bb33 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142096.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142097.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142097.roa new file mode 100644 index 0000000..e697357 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142097.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142098.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142098.roa new file mode 100644 index 0000000..e7e6a19 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142098.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142099.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142099.roa new file mode 100644 index 0000000..47ceb6b Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142099.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142100.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142100.roa new file mode 100644 index 0000000..2fe1220 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142100.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142101.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142101.roa new file mode 100644 index 0000000..8d5a596 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142101.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142102.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142102.roa new file mode 100644 index 0000000..1bcf013 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142102.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142103.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142103.roa new file mode 100644 index 0000000..f29251a Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142103.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142104.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142104.roa new file mode 100644 index 0000000..106402a Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142104.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142105.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142105.roa new file mode 100644 index 0000000..8397586 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142105.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142106.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142106.roa new file mode 100644 index 0000000..e1b7960 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142106.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142650.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142650.roa new file mode 100644 index 0000000..1e69010 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142650.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142651.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142651.roa new file mode 100644 index 0000000..824c1de Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142651.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142652.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142652.roa new file mode 100644 index 0000000..6c9c2e8 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142652.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142653.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142653.roa new file mode 100644 index 0000000..c79d7f5 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142653.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142654.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142654.roa new file mode 100644 index 0000000..ff5b171 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142654.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142655.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142655.roa new file mode 100644 index 0000000..d2952b8 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142655.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142656.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142656.roa new file mode 100644 index 0000000..7c832a7 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142656.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142657.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142657.roa new file mode 100644 index 0000000..071ef55 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142657.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142658.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142658.roa new file mode 100644 index 0000000..564add2 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142658.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142659.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142659.roa new file mode 100644 index 0000000..5f96ec2 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS142659.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144698.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144698.roa new file mode 100644 index 0000000..7ae6e02 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144698.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144699.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144699.roa new file mode 100644 index 0000000..6d8af43 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144699.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144700.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144700.roa new file mode 100644 index 0000000..5561289 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144700.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144701.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144701.roa new file mode 100644 index 0000000..3d9aff8 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144701.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144702.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144702.roa new file mode 100644 index 0000000..097885b Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144702.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144703.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144703.roa new file mode 100644 index 0000000..00e9899 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144703.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144704.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144704.roa new file mode 100644 index 0000000..87de888 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144704.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144705.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144705.roa new file mode 100644 index 0000000..f9733d6 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144705.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144706.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144706.roa new file mode 100644 index 0000000..0a99610 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144706.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144707.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144707.roa new file mode 100644 index 0000000..57dba74 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS144707.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS23910.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS23910.roa new file mode 100644 index 0000000..85f592e Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS23910.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa new file mode 100644 index 0000000..846e777 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/repository/rpki.cernet.net/repo/cernet/0/AS4538.roa differ diff --git a/crates/panda-rpki-validator/tests/fixtures/rrdp/notification.xml b/crates/panda-rpki-validator/tests/fixtures/rrdp/notification.xml new file mode 100644 index 0000000..b3e48ac --- /dev/null +++ b/crates/panda-rpki-validator/tests/fixtures/rrdp/notification.xml @@ -0,0 +1,7 @@ + + + diff --git a/crates/panda-rpki-validator/tests/fixtures/rrdp/snapshot.xml b/crates/panda-rpki-validator/tests/fixtures/rrdp/snapshot.xml new file mode 100644 index 0000000..f0cbe82 --- /dev/null +++ b/crates/panda-rpki-validator/tests/fixtures/rrdp/snapshot.xml @@ -0,0 +1,11 @@ + + + YWJj + + + ZGVm + + diff --git a/crates/panda-rpki-validator/tests/fixtures/ta/afrinic-ta.cer b/crates/panda-rpki-validator/tests/fixtures/ta/afrinic-ta.cer new file mode 100644 index 0000000..734586b Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/ta/afrinic-ta.cer differ diff --git a/crates/panda-rpki-validator/tests/fixtures/ta/apnic-ta.cer b/crates/panda-rpki-validator/tests/fixtures/ta/apnic-ta.cer new file mode 100644 index 0000000..83f98ce Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/ta/apnic-ta.cer differ diff --git a/crates/panda-rpki-validator/tests/fixtures/ta/arin-ta.cer b/crates/panda-rpki-validator/tests/fixtures/ta/arin-ta.cer new file mode 100644 index 0000000..573af6b Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/ta/arin-ta.cer differ diff --git a/crates/panda-rpki-validator/tests/fixtures/ta/lacnic-ta.cer b/crates/panda-rpki-validator/tests/fixtures/ta/lacnic-ta.cer new file mode 100644 index 0000000..d86e36f Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/ta/lacnic-ta.cer differ diff --git a/crates/panda-rpki-validator/tests/fixtures/ta/ripe-ncc-ta.cer b/crates/panda-rpki-validator/tests/fixtures/ta/ripe-ncc-ta.cer new file mode 100644 index 0000000..1364de7 Binary files /dev/null and b/crates/panda-rpki-validator/tests/fixtures/ta/ripe-ncc-ta.cer differ diff --git a/crates/panda-rpki-validator/tests/fixtures/ta_constraints/afrinic-current-ipv4-deny.constraints b/crates/panda-rpki-validator/tests/fixtures/ta_constraints/afrinic-current-ipv4-deny.constraints new file mode 100644 index 0000000..26ce0a9 --- /dev/null +++ b/crates/panda-rpki-validator/tests/fixtures/ta_constraints/afrinic-current-ipv4-deny.constraints @@ -0,0 +1,1842 @@ +# Feature #151 M7 current AFRINIC IPv4 holdings deny fixture. +# +# Source: AFRINIC delegated extended latest. +# URL: https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-latest +# Retrieved: 2026-08-27 +# Source SHA-256: 45ee1d198dc29640f04f3e858f05d2a357c08c93482243529f8f7864d2cd6708 +# +# Selection: IPv4 records with status allocated or assigned only. +# Excluded statuses: reserved and available (not current registry holdings). +# Source IPv4 records: 6051; allocated=3838; assigned=1651; reserved=547; available=15. +# Selected source records: 5489; selected addresses: 116118784. +# Overlap/adjacency union intervals: 815; exact CIDR deny rules: 1825. +# IPv6 and ASN remain allow-all to isolate the current AFRINIC IPv4 policy. +allow 0.0.0.0/0 +allow ::/0 +allow 0 - 4294967295 + +deny 41.0.0.0/11 +deny 41.32.0.0/12 +deny 41.48.0.0/13 +deny 41.56.0.0/16 +deny 41.57.0.0/18 +deny 41.57.64.0/19 +deny 41.57.96.0/20 +deny 41.57.120.0/22 +deny 41.57.128.0/18 +deny 41.58.0.0/15 +deny 41.60.0.0/14 +deny 41.64.0.0/15 +deny 41.66.0.0/16 +deny 41.67.0.0/18 +deny 41.67.64.0/20 +deny 41.67.88.0/21 +deny 41.67.96.0/19 +deny 41.67.128.0/18 +deny 41.68.0.0/14 +deny 41.72.96.0/19 +deny 41.72.128.0/18 +deny 41.72.192.0/19 +deny 41.73.0.0/18 +deny 41.73.96.0/19 +deny 41.73.128.0/17 +deny 41.74.0.0/19 +deny 41.74.48.0/20 +deny 41.74.64.0/18 +deny 41.74.128.0/18 +deny 41.74.192.0/19 +deny 41.74.240.0/20 +deny 41.75.0.0/17 +deny 41.75.128.0/18 +deny 41.75.192.0/19 +deny 41.76.0.0/20 +deny 41.76.16.0/21 +deny 41.76.32.0/20 +deny 41.76.56.0/21 +deny 41.76.64.0/20 +deny 41.76.80.0/21 +deny 41.76.96.0/20 +deny 41.76.112.0/21 +deny 41.76.128.0/17 +deny 41.77.0.0/18 +deny 41.77.72.0/21 +deny 41.77.80.0/20 +deny 41.77.96.0/20 +deny 41.77.112.0/21 +deny 41.77.128.0/19 +deny 41.77.160.0/20 +deny 41.77.184.0/21 +deny 41.77.200.0/21 +deny 41.77.216.0/21 +deny 41.77.232.0/21 +deny 41.77.248.0/21 +deny 41.78.4.0/22 +deny 41.78.8.0/21 +deny 41.78.16.0/20 +deny 41.78.36.0/22 +deny 41.78.44.0/22 +deny 41.78.48.0/21 +deny 41.78.56.0/22 +deny 41.78.64.0/19 +deny 41.78.100.0/22 +deny 41.78.104.0/21 +deny 41.78.116.0/22 +deny 41.78.120.0/21 +deny 41.78.128.0/19 +deny 41.78.160.0/22 +deny 41.78.168.0/21 +deny 41.78.176.0/20 +deny 41.78.192.0/21 +deny 41.78.200.0/22 +deny 41.78.208.0/20 +deny 41.78.224.0/19 +deny 41.79.0.0/19 +deny 41.79.32.0/21 +deny 41.79.44.0/22 +deny 41.79.48.0/22 +deny 41.79.56.0/21 +deny 41.79.64.0/18 +deny 41.79.132.0/22 +deny 41.79.136.0/21 +deny 41.79.148.0/22 +deny 41.79.152.0/21 +deny 41.79.160.0/22 +deny 41.79.168.0/21 +deny 41.79.180.0/22 +deny 41.79.184.0/21 +deny 41.79.192.0/18 +deny 41.80.0.0/14 +deny 41.84.0.0/15 +deny 41.86.0.0/18 +deny 41.86.96.0/19 +deny 41.86.128.0/18 +deny 41.86.224.0/19 +deny 41.87.0.0/16 +deny 41.89.0.0/16 +deny 41.90.0.0/15 +deny 41.92.0.0/17 +deny 41.92.128.0/21 +deny 41.92.152.0/21 +deny 41.92.180.0/22 +deny 41.92.184.0/22 +deny 41.92.196.0/22 +deny 41.92.200.0/22 +deny 41.92.208.0/21 +deny 41.92.216.0/22 +deny 41.92.224.0/24 +deny 41.92.248.0/21 +deny 41.93.0.0/17 +deny 41.94.0.0/15 +deny 41.96.0.0/11 +deny 41.128.0.0/13 +deny 41.136.0.0/15 +deny 41.138.32.0/19 +deny 41.138.64.0/18 +deny 41.138.128.0/18 +deny 41.138.224.0/19 +deny 41.139.0.0/16 +deny 41.140.0.0/14 +deny 41.144.0.0/13 +deny 41.152.0.0/15 +deny 41.155.0.0/16 +deny 41.156.0.0/14 +deny 41.160.0.0/12 +deny 41.176.0.0/13 +deny 41.184.0.0/14 +deny 41.188.0.0/15 +deny 41.190.0.0/18 +deny 41.190.68.0/22 +deny 41.190.80.0/20 +deny 41.190.96.0/19 +deny 41.190.128.0/18 +deny 41.190.192.0/19 +deny 41.190.224.0/22 +deny 41.190.232.0/21 +deny 41.190.240.0/20 +deny 41.191.0.0/17 +deny 41.191.128.0/19 +deny 41.191.192.0/19 +deny 41.191.224.0/20 +deny 41.191.248.0/22 +deny 41.192.0.0/13 +deny 41.200.0.0/15 +deny 41.202.32.0/19 +deny 41.202.64.0/18 +deny 41.202.160.0/19 +deny 41.202.192.0/18 +deny 41.203.0.0/16 +deny 41.204.0.0/17 +deny 41.204.128.0/18 +deny 41.204.192.0/19 +deny 41.205.0.0/17 +deny 41.205.128.0/18 +deny 41.205.192.0/19 +deny 41.206.0.0/16 +deny 41.207.0.0/17 +deny 41.207.128.0/18 +deny 41.207.192.0/19 +deny 41.207.224.0/22 +deny 41.207.232.0/21 +deny 41.207.240.0/20 +deny 41.208.0.0/16 +deny 41.209.0.0/17 +deny 41.209.128.0/18 +deny 41.210.0.0/18 +deny 41.210.128.0/17 +deny 41.211.0.0/16 +deny 41.212.0.0/14 +deny 41.216.0.0/16 +deny 41.217.0.0/17 +deny 41.217.148.0/22 +deny 41.217.192.0/18 +deny 41.218.0.0/15 +deny 41.220.0.0/19 +deny 41.220.32.0/20 +deny 41.220.64.0/18 +deny 41.220.128.0/17 +deny 41.221.0.0/17 +deny 41.221.144.0/20 +deny 41.221.160.0/19 +deny 41.221.192.0/18 +deny 41.222.0.0/20 +deny 41.222.16.0/21 +deny 41.222.32.0/19 +deny 41.222.64.0/18 +deny 41.222.128.0/18 +deny 41.222.192.0/20 +deny 41.222.208.0/21 +deny 41.222.216.0/22 +deny 41.222.224.0/20 +deny 41.222.244.0/22 +deny 41.222.248.0/21 +deny 41.223.0.0/20 +deny 41.223.20.0/22 +deny 41.223.24.0/21 +deny 41.223.32.0/19 +deny 41.223.64.0/20 +deny 41.223.80.0/21 +deny 41.223.96.0/20 +deny 41.223.116.0/22 +deny 41.223.120.0/21 +deny 41.223.128.0/19 +deny 41.223.160.0/21 +deny 41.223.172.0/22 +deny 41.223.180.0/22 +deny 41.223.184.0/21 +deny 41.223.192.0/18 +deny 41.224.0.0/12 +deny 41.240.0.0/15 +deny 41.242.0.0/21 +deny 41.242.16.0/20 +deny 41.242.40.0/21 +deny 41.242.48.0/20 +deny 41.242.64.0/20 +deny 41.242.80.0/22 +deny 41.242.88.0/22 +deny 41.242.96.0/20 +deny 41.242.112.0/21 +deny 41.242.124.0/22 +deny 41.242.132.0/22 +deny 41.242.140.0/22 +deny 41.242.144.0/20 +deny 41.242.160.0/19 +deny 41.242.192.0/18 +deny 41.243.0.0/16 +deny 41.245.0.0/17 +deny 41.246.0.0/15 +deny 41.248.0.0/13 +deny 45.96.0.0/12 +deny 45.192.0.0/12 +deny 45.208.0.0/13 +deny 45.216.0.0/14 +deny 45.220.0.0/19 +deny 45.220.48.0/23 +deny 45.220.52.0/22 +deny 45.220.56.0/21 +deny 45.220.128.0/17 +deny 45.221.0.0/20 +deny 45.221.16.0/22 +deny 45.221.24.0/22 +deny 45.221.32.0/19 +deny 45.221.64.0/18 +deny 45.221.128.0/17 +deny 45.222.0.0/16 +deny 45.240.0.0/13 +deny 62.8.64.0/19 +deny 62.12.96.0/19 +deny 62.24.96.0/19 +deny 62.61.192.0/18 +deny 62.68.32.0/19 +deny 62.68.224.0/19 +deny 62.114.0.0/16 +deny 62.117.32.0/19 +deny 62.135.0.0/17 +deny 62.139.0.0/16 +deny 62.140.64.0/18 +deny 62.173.32.0/19 +deny 62.193.64.0/18 +deny 62.193.160.0/19 +deny 62.240.32.0/19 +deny 62.240.96.0/19 +deny 62.241.128.0/19 +deny 62.251.128.0/17 +deny 64.57.112.0/20 +deny 66.8.0.0/17 +deny 66.18.64.0/19 +deny 66.251.128.0/18 +deny 69.63.64.0/20 +deny 69.67.32.0/20 +deny 77.220.0.0/19 +deny 80.67.128.0/20 +deny 80.72.96.0/20 +deny 80.75.160.0/19 +deny 80.87.64.0/19 +deny 80.88.0.0/22 +deny 80.88.4.0/23 +deny 80.88.7.0/24 +deny 80.88.8.0/21 +deny 80.95.0.0/20 +deny 80.240.192.0/20 +deny 80.246.0.0/20 +deny 80.248.0.0/20 +deny 80.248.64.0/20 +deny 80.249.64.0/20 +deny 80.250.32.0/20 +deny 81.10.0.0/17 +deny 81.21.96.0/20 +deny 81.22.64.0/19 +deny 81.26.72.0/21 +deny 81.29.96.0/20 +deny 81.91.224.0/20 +deny 81.192.0.0/16 +deny 82.101.128.0/18 +deny 82.129.128.0/17 +deny 82.151.64.0/19 +deny 82.201.128.0/17 +deny 83.143.24.0/21 +deny 84.36.0.0/16 +deny 84.205.96.0/19 +deny 84.233.0.0/17 +deny 87.255.96.0/19 +deny 102.0.0.0/12 +deny 102.16.0.0/14 +deny 102.20.0.0/15 +deny 102.22.0.0/18 +deny 102.22.64.0/20 +deny 102.22.80.0/22 +deny 102.22.88.0/21 +deny 102.22.100.0/22 +deny 102.22.104.0/21 +deny 102.22.112.0/20 +deny 102.22.128.0/17 +deny 102.23.0.0/18 +deny 102.23.80.0/20 +deny 102.23.96.0/22 +deny 102.23.112.0/21 +deny 102.23.120.0/22 +deny 102.23.132.0/22 +deny 102.23.136.0/21 +deny 102.23.144.0/20 +deny 102.23.160.0/19 +deny 102.23.192.0/20 +deny 102.23.208.0/21 +deny 102.23.218.0/23 +deny 102.23.220.0/22 +deny 102.23.224.0/19 +deny 102.24.0.0/13 +deny 102.32.0.0/14 +deny 102.36.0.0/17 +deny 102.36.132.0/22 +deny 102.36.136.0/21 +deny 102.36.144.0/20 +deny 102.36.160.0/20 +deny 102.36.176.0/21 +deny 102.36.184.0/22 +deny 102.36.192.0/23 +deny 102.36.196.0/22 +deny 102.36.200.0/21 +deny 102.36.208.0/20 +deny 102.36.224.0/21 +deny 102.36.236.0/22 +deny 102.36.240.0/22 +deny 102.36.248.0/21 +deny 102.37.0.0/16 +deny 102.38.0.0/19 +deny 102.38.32.0/20 +deny 102.38.48.0/22 +deny 102.38.56.0/21 +deny 102.38.64.0/18 +deny 102.38.128.0/17 +deny 102.39.0.0/16 +deny 102.40.0.0/13 +deny 102.48.0.0/12 +deny 102.64.0.0/18 +deny 102.64.64.0/20 +deny 102.64.92.0/22 +deny 102.64.96.0/19 +deny 102.64.128.0/17 +deny 102.65.0.0/16 +deny 102.66.0.0/15 +deny 102.68.0.0/20 +deny 102.68.16.0/22 +deny 102.68.20.0/23 +deny 102.68.22.0/24 +deny 102.68.24.0/21 +deny 102.68.32.0/20 +deny 102.68.48.0/22 +deny 102.68.56.0/21 +deny 102.68.64.0/20 +deny 102.68.80.0/21 +deny 102.68.96.0/20 +deny 102.68.112.0/21 +deny 102.68.120.0/22 +deny 102.68.124.0/23 +deny 102.68.126.0/24 +deny 102.68.128.0/21 +deny 102.68.136.0/22 +deny 102.68.141.0/24 +deny 102.68.142.0/23 +deny 102.68.144.0/20 +deny 102.68.160.0/20 +deny 102.68.176.0/21 +deny 102.68.188.0/22 +deny 102.68.192.0/18 +deny 102.69.0.0/16 +deny 102.70.0.0/15 +deny 102.72.0.0/13 +deny 102.80.0.0/12 +deny 102.96.0.0/11 +deny 102.128.0.0/18 +deny 102.128.76.0/22 +deny 102.128.80.0/20 +deny 102.128.96.0/19 +deny 102.128.128.0/19 +deny 102.128.160.0/20 +deny 102.128.176.0/21 +deny 102.128.184.0/22 +deny 102.128.192.0/18 +deny 102.129.32.0/22 +deny 102.129.40.0/21 +deny 102.129.48.0/20 +deny 102.129.64.0/18 +deny 102.129.128.0/17 +deny 102.130.0.0/21 +deny 102.130.12.0/22 +deny 102.130.16.0/20 +deny 102.130.32.0/20 +deny 102.130.48.0/21 +deny 102.130.64.0/19 +deny 102.130.96.0/23 +deny 102.130.100.0/22 +deny 102.130.104.0/21 +deny 102.130.112.0/20 +deny 102.130.128.0/17 +deny 102.131.16.0/24 +deny 102.131.18.0/23 +deny 102.131.20.0/22 +deny 102.131.28.0/22 +deny 102.131.32.0/21 +deny 102.131.48.0/20 +deny 102.131.64.0/18 +deny 102.131.128.0/17 +deny 102.132.0.0/18 +deny 102.132.64.0/20 +deny 102.132.80.0/22 +deny 102.132.96.0/19 +deny 102.132.128.0/17 +deny 102.133.0.0/16 +deny 102.134.0.0/18 +deny 102.134.64.0/21 +deny 102.134.72.0/22 +deny 102.134.80.0/20 +deny 102.134.96.0/19 +deny 102.134.128.0/21 +deny 102.134.140.0/22 +deny 102.134.144.0/20 +deny 102.134.160.0/19 +deny 102.134.192.0/18 +deny 102.135.0.0/17 +deny 102.135.128.0/19 +deny 102.135.160.0/22 +deny 102.135.168.0/21 +deny 102.135.176.0/20 +deny 102.135.192.0/18 +deny 102.136.0.0/14 +deny 102.140.0.0/18 +deny 102.140.64.0/19 +deny 102.140.96.0/21 +deny 102.140.112.0/21 +deny 102.140.128.0/17 +deny 102.141.0.0/17 +deny 102.141.128.0/18 +deny 102.141.192.0/20 +deny 102.141.208.0/21 +deny 102.141.224.0/22 +deny 102.141.232.0/21 +deny 102.141.240.0/20 +deny 102.142.0.0/15 +deny 102.144.0.0/12 +deny 102.160.0.0/14 +deny 102.164.0.0/19 +deny 102.164.32.0/21 +deny 102.164.44.0/22 +deny 102.164.52.0/22 +deny 102.164.56.0/21 +deny 102.164.64.0/18 +deny 102.164.128.0/17 +deny 102.165.0.0/17 +deny 102.165.128.0/19 +deny 102.165.160.0/20 +deny 102.165.176.0/21 +deny 102.165.184.0/22 +deny 102.165.188.0/23 +deny 102.165.191.0/24 +deny 102.165.192.0/18 +deny 102.166.0.0/15 +deny 102.168.0.0/13 +deny 102.176.0.0/16 +deny 102.177.0.0/18 +deny 102.177.64.0/21 +deny 102.177.72.0/22 +deny 102.177.80.0/21 +deny 102.177.100.0/22 +deny 102.177.112.0/20 +deny 102.177.128.0/18 +deny 102.177.192.0/21 +deny 102.178.0.0/15 +deny 102.180.0.0/16 +deny 102.181.0.0/17 +deny 102.181.128.0/18 +deny 102.181.192.0/19 +deny 102.182.0.0/16 +deny 102.184.0.0/13 +deny 102.201.116.0/22 +deny 102.201.120.0/21 +deny 102.201.128.0/18 +deny 102.201.192.0/20 +deny 102.201.208.0/21 +deny 102.201.228.0/22 +deny 102.201.232.0/21 +deny 102.201.240.0/20 +deny 102.202.0.0/16 +deny 102.203.0.0/20 +deny 102.203.16.0/22 +deny 102.203.32.0/20 +deny 102.203.48.0/22 +deny 102.203.56.0/21 +deny 102.203.64.0/21 +deny 102.203.72.0/22 +deny 102.203.80.0/20 +deny 102.203.96.0/19 +deny 102.203.128.0/17 +deny 102.204.0.0/17 +deny 102.204.144.0/20 +deny 102.204.160.0/19 +deny 102.204.192.0/18 +deny 102.205.0.0/16 +deny 102.206.0.0/15 +deny 102.208.0.0/17 +deny 102.208.128.0/20 +deny 102.208.144.0/21 +deny 102.208.152.0/22 +deny 102.208.160.0/19 +deny 102.208.192.0/18 +deny 102.209.0.0/16 +deny 102.210.0.0/16 +deny 102.211.0.0/17 +deny 102.211.128.0/20 +deny 102.211.144.0/22 +deny 102.211.152.0/21 +deny 102.211.160.0/19 +deny 102.211.192.0/18 +deny 102.212.0.0/16 +deny 102.213.0.0/17 +deny 102.213.128.0/18 +deny 102.213.192.0/19 +deny 102.213.228.0/22 +deny 102.213.232.0/21 +deny 102.213.240.0/20 +deny 102.214.0.0/21 +deny 102.214.8.0/22 +deny 102.214.12.0/23 +deny 102.214.14.0/24 +deny 102.214.16.0/20 +deny 102.214.32.0/20 +deny 102.214.48.0/21 +deny 102.214.56.0/22 +deny 102.214.64.0/18 +deny 102.214.128.0/18 +deny 102.214.192.0/19 +deny 102.214.224.0/20 +deny 102.214.240.0/22 +deny 102.214.244.0/23 +deny 102.214.247.0/24 +deny 102.214.248.0/21 +deny 102.215.0.0/18 +deny 102.215.64.0/22 +deny 102.215.72.0/21 +deny 102.215.84.0/22 +deny 102.215.88.0/21 +deny 102.215.96.0/19 +deny 102.215.128.0/20 +deny 102.215.148.0/22 +deny 102.215.152.0/22 +deny 102.215.160.0/21 +deny 102.215.168.0/22 +deny 102.215.176.0/20 +deny 102.215.192.0/20 +deny 102.215.208.0/21 +deny 102.215.218.0/23 +deny 102.215.222.0/23 +deny 102.215.224.0/21 +deny 102.215.236.0/22 +deny 102.215.240.0/20 +deny 102.216.0.0/20 +deny 102.216.16.0/21 +deny 102.216.25.0/24 +deny 102.216.27.0/24 +deny 102.216.28.0/22 +deny 102.216.32.0/19 +deny 102.216.64.0/20 +deny 102.216.80.0/21 +deny 102.216.92.0/22 +deny 102.216.96.0/19 +deny 102.216.128.0/19 +deny 102.216.160.0/22 +deny 102.216.168.0/21 +deny 102.216.176.0/20 +deny 102.216.192.0/19 +deny 102.216.224.0/22 +deny 102.216.232.0/21 +deny 102.216.240.0/20 +deny 102.217.0.0/19 +deny 102.217.32.0/20 +deny 102.217.48.0/21 +deny 102.217.56.0/22 +deny 102.217.64.0/20 +deny 102.217.80.0/21 +deny 102.217.88.0/22 +deny 102.217.96.0/19 +deny 102.217.128.0/18 +deny 102.217.192.0/20 +deny 102.217.209.0/24 +deny 102.217.210.0/23 +deny 102.217.212.0/22 +deny 102.217.216.0/21 +deny 102.217.224.0/19 +deny 102.218.0.0/22 +deny 102.218.8.0/21 +deny 102.218.16.0/20 +deny 102.218.32.0/19 +deny 102.218.64.0/19 +deny 102.218.98.0/23 +deny 102.218.100.0/22 +deny 102.218.104.0/21 +deny 102.218.112.0/20 +deny 102.218.128.0/19 +deny 102.218.160.0/21 +deny 102.218.172.0/22 +deny 102.218.176.0/20 +deny 102.218.192.0/19 +deny 102.218.224.0/20 +deny 102.218.240.0/21 +deny 102.218.252.0/22 +deny 102.219.0.0/22 +deny 102.219.8.0/21 +deny 102.219.16.0/20 +deny 102.219.32.0/22 +deny 102.219.40.0/21 +deny 102.219.48.0/20 +deny 102.219.64.0/18 +deny 102.219.132.0/22 +deny 102.219.136.0/21 +deny 102.219.144.0/20 +deny 102.219.160.0/19 +deny 102.219.192.0/20 +deny 102.219.208.0/22 +deny 102.219.216.0/21 +deny 102.219.224.0/20 +deny 102.219.240.0/22 +deny 102.219.248.0/21 +deny 102.220.8.0/21 +deny 102.220.16.0/20 +deny 102.220.36.0/22 +deny 102.220.40.0/21 +deny 102.220.48.0/20 +deny 102.220.64.0/18 +deny 102.220.128.0/21 +deny 102.220.140.0/22 +deny 102.220.144.0/20 +deny 102.220.160.0/20 +deny 102.220.176.0/22 +deny 102.220.184.0/21 +deny 102.220.192.0/19 +deny 102.220.224.0/20 +deny 102.220.244.0/22 +deny 102.220.248.0/21 +deny 102.221.0.0/18 +deny 102.221.64.0/19 +deny 102.221.96.0/21 +deny 102.221.108.0/22 +deny 102.221.112.0/20 +deny 102.221.128.0/20 +deny 102.221.152.0/24 +deny 102.221.154.0/23 +deny 102.221.156.0/22 +deny 102.221.160.0/19 +deny 102.221.192.0/20 +deny 102.221.208.0/22 +deny 102.221.216.0/21 +deny 102.221.224.0/19 +deny 102.222.0.0/20 +deny 102.222.16.0/21 +deny 102.222.25.0/24 +deny 102.222.27.0/24 +deny 102.222.28.0/22 +deny 102.222.36.0/22 +deny 102.222.40.0/21 +deny 102.222.48.0/22 +deny 102.222.56.0/22 +deny 102.222.61.0/24 +deny 102.222.62.0/23 +deny 102.222.64.0/18 +deny 102.222.128.0/18 +deny 102.222.192.0/19 +deny 102.222.228.0/22 +deny 102.222.232.0/22 +deny 102.222.236.0/23 +deny 102.222.240.0/22 +deny 102.222.248.0/22 +deny 102.222.252.0/23 +deny 102.222.255.0/24 +deny 102.223.0.0/22 +deny 102.223.4.0/24 +deny 102.223.6.0/23 +deny 102.223.8.0/21 +deny 102.223.16.0/20 +deny 102.223.32.0/20 +deny 102.223.48.0/21 +deny 102.223.56.0/22 +deny 102.223.62.0/23 +deny 102.223.64.0/22 +deny 102.223.72.0/21 +deny 102.223.80.0/20 +deny 102.223.96.0/22 +deny 102.223.100.0/23 +deny 102.223.103.0/24 +deny 102.223.104.0/22 +deny 102.223.110.0/23 +deny 102.223.114.0/23 +deny 102.223.116.0/22 +deny 102.223.120.0/21 +deny 102.223.128.0/21 +deny 102.223.140.0/22 +deny 102.223.144.0/21 +deny 102.223.154.0/23 +deny 102.223.156.0/22 +deny 102.223.164.0/22 +deny 102.223.168.0/22 +deny 102.223.173.0/24 +deny 102.223.174.0/23 +deny 102.223.176.0/20 +deny 102.223.192.0/19 +deny 102.223.224.0/20 +deny 102.223.240.0/22 +deny 102.223.248.0/21 +deny 102.240.0.0/12 +deny 105.0.0.0/9 +deny 105.128.0.0/11 +deny 105.160.0.0/12 +deny 105.176.0.0/15 +deny 105.178.0.0/17 +deny 105.179.0.0/19 +deny 105.180.0.0/14 +deny 105.184.0.0/13 +deny 105.192.0.0/10 +deny 129.0.0.0/16 +deny 129.18.0.0/16 +deny 129.45.0.0/16 +deny 129.56.0.0/16 +deny 129.122.0.0/16 +deny 129.140.0.0/16 +deny 129.205.0.0/18 +deny 129.205.64.0/20 +deny 129.205.96.0/19 +deny 129.205.128.0/17 +deny 129.232.0.0/16 +deny 137.63.0.0/16 +deny 137.64.0.0/16 +deny 137.115.0.0/16 +deny 137.158.0.0/16 +deny 137.196.0.0/16 +deny 137.214.0.0/15 +deny 137.255.0.0/16 +deny 139.26.0.0/16 +deny 139.53.0.0/16 +deny 143.128.0.0/16 +deny 143.160.0.0/16 +deny 146.64.0.0/16 +deny 146.141.0.0/16 +deny 146.182.0.0/16 +deny 146.196.128.0/17 +deny 146.230.0.0/15 +deny 146.232.0.0/16 +deny 147.110.0.0/16 +deny 152.106.0.0/15 +deny 152.108.0.0/14 +deny 152.112.0.0/16 +deny 154.0.0.0/18 +deny 154.0.128.0/17 +deny 154.16.0.0/16 +deny 154.65.0.0/19 +deny 154.65.32.0/21 +deny 154.65.52.0/22 +deny 154.65.56.0/21 +deny 154.65.64.0/18 +deny 154.65.128.0/17 +deny 154.66.0.0/18 +deny 154.66.64.0/20 +deny 154.66.80.0/22 +deny 154.66.88.0/21 +deny 154.66.96.0/22 +deny 154.66.104.0/21 +deny 154.66.112.0/20 +deny 154.66.128.0/18 +deny 154.66.192.0/19 +deny 154.66.224.0/22 +deny 154.66.240.0/20 +deny 154.67.0.0/16 +deny 154.68.0.0/15 +deny 154.70.0.0/18 +deny 154.70.80.0/20 +deny 154.70.96.0/19 +deny 154.70.128.0/20 +deny 154.70.144.0/21 +deny 154.70.152.0/22 +deny 154.70.160.0/19 +deny 154.70.192.0/22 +deny 154.70.200.0/21 +deny 154.70.208.0/20 +deny 154.70.224.0/19 +deny 154.71.0.0/16 +deny 154.72.0.0/20 +deny 154.72.16.0/21 +deny 154.72.24.0/22 +deny 154.72.32.0/19 +deny 154.72.64.0/18 +deny 154.72.128.0/17 +deny 154.73.0.0/21 +deny 154.73.8.0/22 +deny 154.73.16.0/20 +deny 154.73.32.0/19 +deny 154.73.64.0/22 +deny 154.73.72.0/22 +deny 154.73.80.0/20 +deny 154.73.96.0/20 +deny 154.73.112.0/22 +deny 154.73.120.0/21 +deny 154.73.128.0/20 +deny 154.73.152.0/21 +deny 154.73.160.0/20 +deny 154.73.180.0/22 +deny 154.73.184.0/21 +deny 154.73.192.0/21 +deny 154.73.208.0/20 +deny 154.73.224.0/21 +deny 154.73.232.0/22 +deny 154.73.240.0/20 +deny 154.74.0.0/15 +deny 154.76.0.0/14 +deny 154.80.0.0/12 +deny 154.96.0.0/12 +deny 154.112.0.0/15 +deny 154.114.0.0/17 +deny 154.115.0.0/17 +deny 154.115.128.0/20 +deny 154.115.156.0/22 +deny 154.115.160.0/19 +deny 154.115.192.0/18 +deny 154.116.0.0/16 +deny 154.117.64.0/18 +deny 154.117.128.0/17 +deny 154.118.0.0/17 +deny 154.118.128.0/18 +deny 154.118.192.0/19 +deny 154.118.224.0/21 +deny 154.118.240.0/21 +deny 154.118.252.0/22 +deny 154.119.0.0/16 +deny 154.120.0.0/14 +deny 154.124.0.0/15 +deny 154.126.0.0/16 +deny 154.127.0.0/18 +deny 154.127.64.0/19 +deny 154.127.112.0/20 +deny 154.127.128.0/17 +deny 154.128.0.0/9 +deny 155.0.0.0/16 +deny 155.12.0.0/17 +deny 155.12.128.0/18 +deny 155.12.224.0/19 +deny 155.89.0.0/16 +deny 155.93.0.0/16 +deny 155.159.0.0/16 +deny 155.196.0.0/16 +deny 155.232.0.0/13 +deny 155.240.0.0/16 +deny 155.251.0.0/16 +deny 156.0.0.0/18 +deny 156.0.64.0/20 +deny 156.0.80.0/22 +deny 156.0.88.0/21 +deny 156.0.96.0/24 +deny 156.0.128.0/18 +deny 156.0.192.0/22 +deny 156.0.196.0/23 +deny 156.0.198.0/24 +deny 156.0.200.0/21 +deny 156.0.208.0/21 +deny 156.0.216.0/22 +deny 156.0.224.0/20 +deny 156.0.244.0/22 +deny 156.0.248.0/22 +deny 156.0.252.0/23 +deny 156.0.255.0/24 +deny 156.8.0.0/16 +deny 156.38.4.0/22 +deny 156.38.8.0/21 +deny 156.38.16.0/20 +deny 156.38.32.0/19 +deny 156.38.64.0/18 +deny 156.38.128.0/17 +deny 156.155.0.0/16 +deny 156.156.0.0/14 +deny 156.160.0.0/11 +deny 156.192.0.0/10 +deny 160.0.0.0/17 +deny 160.0.128.0/18 +deny 160.0.192.0/19 +deny 160.19.36.0/22 +deny 160.19.96.0/21 +deny 160.19.112.0/20 +deny 160.19.128.0/20 +deny 160.19.152.0/22 +deny 160.19.188.0/22 +deny 160.19.196.0/22 +deny 160.19.232.0/22 +deny 160.20.24.0/21 +deny 160.20.112.0/22 +deny 160.20.213.0/24 +deny 160.20.217.0/24 +deny 160.20.221.0/24 +deny 160.20.226.0/23 +deny 160.77.0.0/16 +deny 160.89.0.0/16 +deny 160.90.0.0/16 +deny 160.105.0.0/16 +deny 160.113.0.0/16 +deny 160.115.0.0/16 +deny 160.116.0.0/15 +deny 160.118.0.0/16 +deny 160.119.0.0/18 +deny 160.119.64.0/19 +deny 160.119.96.0/21 +deny 160.119.108.0/22 +deny 160.119.112.0/20 +deny 160.119.128.0/18 +deny 160.119.192.0/22 +deny 160.119.196.0/23 +deny 160.119.200.0/21 +deny 160.119.212.0/22 +deny 160.119.216.0/21 +deny 160.119.224.0/19 +deny 160.120.0.0/14 +deny 160.124.0.0/16 +deny 160.152.0.0/16 +deny 160.154.0.0/15 +deny 160.156.0.0/14 +deny 160.160.0.0/12 +deny 160.176.0.0/14 +deny 160.182.0.0/15 +deny 160.224.0.0/15 +deny 160.226.0.0/17 +deny 160.226.128.0/19 +deny 160.226.160.0/20 +deny 160.226.176.0/21 +deny 160.226.184.0/22 +deny 160.226.192.0/18 +deny 160.238.11.0/24 +deny 160.238.50.0/24 +deny 160.238.57.0/24 +deny 160.238.101.0/24 +deny 160.242.0.0/17 +deny 160.242.192.0/18 +deny 161.123.0.0/16 +deny 163.121.0.0/16 +deny 163.195.0.0/16 +deny 163.196.0.0/14 +deny 163.200.0.0/14 +deny 164.88.0.0/16 +deny 164.146.0.0/15 +deny 164.148.0.0/14 +deny 164.155.0.0/16 +deny 164.160.0.0/21 +deny 164.160.8.0/22 +deny 164.160.16.0/21 +deny 164.160.24.0/22 +deny 164.160.32.0/20 +deny 164.160.48.0/21 +deny 164.160.60.0/22 +deny 164.160.64.0/19 +deny 164.160.100.0/22 +deny 164.160.104.0/21 +deny 164.160.116.0/22 +deny 164.160.120.0/21 +deny 164.160.132.0/22 +deny 164.160.136.0/21 +deny 164.160.144.0/22 +deny 164.160.176.0/21 +deny 164.160.184.0/22 +deny 164.160.224.0/19 +deny 165.0.0.0/16 +deny 165.3.0.0/16 +deny 165.4.0.0/15 +deny 165.8.0.0/14 +deny 165.16.0.0/16 +deny 165.25.0.0/16 +deny 165.49.0.0/16 +deny 165.50.0.0/15 +deny 165.56.0.0/13 +deny 165.73.0.0/17 +deny 165.73.128.0/20 +deny 165.73.160.0/19 +deny 165.73.192.0/18 +deny 165.90.0.0/17 +deny 165.90.192.0/18 +deny 165.143.0.0/16 +deny 165.144.0.0/14 +deny 165.148.0.0/15 +deny 165.165.0.0/16 +deny 165.169.0.0/16 +deny 165.180.0.0/16 +deny 165.210.0.0/15 +deny 165.231.0.0/16 +deny 165.233.0.0/16 +deny 165.255.0.0/16 +deny 166.85.0.0/16 +deny 168.76.0.0/16 +deny 168.80.0.0/15 +deny 168.89.0.0/16 +deny 168.128.0.0/16 +deny 168.142.0.0/16 +deny 168.155.0.0/16 +deny 168.164.0.0/16 +deny 168.167.0.0/16 +deny 168.172.0.0/16 +deny 168.206.0.0/16 +deny 168.209.0.0/16 +deny 168.210.0.0/16 +deny 168.253.0.0/16 +deny 169.0.0.0/15 +deny 169.129.0.0/16 +deny 169.159.0.0/16 +deny 169.202.0.0/16 +deny 169.239.0.0/19 +deny 169.239.32.0/20 +deny 169.239.48.0/22 +deny 169.239.60.0/22 +deny 169.239.64.0/20 +deny 169.239.80.0/21 +deny 169.239.92.0/22 +deny 169.239.96.0/19 +deny 169.239.128.0/20 +deny 169.239.148.0/22 +deny 169.239.156.0/22 +deny 169.239.160.0/21 +deny 169.239.168.0/22 +deny 169.239.176.0/20 +deny 169.239.192.0/19 +deny 169.239.224.0/20 +deny 169.239.240.0/21 +deny 169.239.248.0/22 +deny 169.255.0.0/21 +deny 169.255.9.0/24 +deny 169.255.12.0/22 +deny 169.255.16.0/20 +deny 169.255.32.0/19 +deny 169.255.64.0/20 +deny 169.255.80.0/21 +deny 169.255.88.0/22 +deny 169.255.104.0/22 +deny 169.255.112.0/20 +deny 169.255.128.0/21 +deny 169.255.136.0/22 +deny 169.255.144.0/20 +deny 169.255.160.0/22 +deny 169.255.168.0/21 +deny 169.255.176.0/20 +deny 169.255.192.0/21 +deny 169.255.204.0/22 +deny 169.255.208.0/22 +deny 169.255.216.0/21 +deny 169.255.228.0/22 +deny 169.255.232.0/21 +deny 169.255.240.0/20 +deny 192.12.110.0/23 +deny 192.12.116.0/23 +deny 192.33.10.0/24 +deny 192.42.99.0/24 +deny 192.47.36.0/24 +deny 192.48.253.0/24 +deny 192.51.240.0/24 +deny 192.52.232.0/24 +deny 192.68.138.0/24 +deny 192.70.200.0/23 +deny 192.70.237.0/24 +deny 192.75.236.0/24 +deny 192.82.142.0/24 +deny 192.83.208.0/22 +deny 192.84.244.0/24 +deny 192.91.200.0/24 +deny 192.94.61.0/24 +deny 192.94.210.0/24 +deny 192.94.240.0/23 +deny 192.94.246.0/24 +deny 192.96.1.0/24 +deny 192.96.2.0/23 +deny 192.96.4.0/22 +deny 192.96.8.0/21 +deny 192.96.16.0/24 +deny 192.96.19.0/24 +deny 192.96.20.0/23 +deny 192.96.22.0/24 +deny 192.96.24.0/21 +deny 192.96.32.0/22 +deny 192.96.37.0/24 +deny 192.96.38.0/23 +deny 192.96.40.0/24 +deny 192.96.43.0/24 +deny 192.96.45.0/24 +deny 192.96.46.0/23 +deny 192.96.48.0/21 +deny 192.96.56.0/23 +deny 192.96.58.0/24 +deny 192.96.60.0/23 +deny 192.96.68.0/22 +deny 192.96.72.0/24 +deny 192.96.74.0/23 +deny 192.96.76.0/23 +deny 192.96.79.0/24 +deny 192.96.80.0/20 +deny 192.96.96.0/22 +deny 192.96.100.0/23 +deny 192.96.102.0/24 +deny 192.96.104.0/21 +deny 192.96.112.0/20 +deny 192.96.128.0/22 +deny 192.96.132.0/23 +deny 192.96.134.0/24 +deny 192.96.138.0/23 +deny 192.96.140.0/23 +deny 192.96.142.0/24 +deny 192.96.144.0/24 +deny 192.96.146.0/23 +deny 192.96.148.0/22 +deny 192.96.152.0/22 +deny 192.96.158.0/24 +deny 192.96.164.0/22 +deny 192.96.177.0/24 +deny 192.96.178.0/23 +deny 192.96.180.0/22 +deny 192.96.184.0/21 +deny 192.96.193.0/24 +deny 192.96.194.0/23 +deny 192.96.196.0/22 +deny 192.96.224.0/21 +deny 192.96.232.0/24 +deny 192.96.234.0/23 +deny 192.96.236.0/22 +deny 192.96.240.0/22 +deny 192.96.245.0/24 +deny 192.96.246.0/23 +deny 192.96.248.0/22 +deny 192.96.252.0/23 +deny 192.96.254.0/24 +deny 192.100.1.0/24 +deny 192.101.142.0/24 +deny 192.102.9.0/24 +deny 192.109.242.0/24 +deny 192.133.250.0/24 +deny 192.136.55.0/24 +deny 192.136.56.0/23 +deny 192.142.0.0/15 +deny 192.145.128.0/19 +deny 192.145.160.0/22 +deny 192.145.168.0/21 +deny 192.145.176.0/20 +deny 192.145.230.0/24 +deny 192.157.190.0/24 +deny 192.188.164.0/22 +deny 192.189.75.0/24 +deny 192.189.139.0/24 +deny 192.189.140.0/24 +deny 192.231.237.0/24 +deny 192.231.254.0/24 +deny 192.245.148.0/24 +deny 192.251.202.0/24 +deny 193.17.215.0/24 +deny 193.19.232.0/22 +deny 193.41.146.0/23 +deny 193.95.0.0/17 +deny 193.108.23.0/24 +deny 193.108.28.0/24 +deny 193.108.214.0/24 +deny 193.108.252.0/22 +deny 193.109.66.0/23 +deny 193.110.104.0/23 +deny 193.189.0.0/18 +deny 193.194.32.0/19 +deny 193.194.64.0/19 +deny 193.194.128.0/24 +deny 193.194.160.0/19 +deny 193.221.218.0/24 +deny 193.227.0.0/18 +deny 193.227.128.0/24 +deny 194.9.64.0/23 +deny 194.9.82.0/23 +deny 194.79.96.0/19 +deny 194.204.192.0/18 +deny 195.24.80.0/21 +deny 195.24.192.0/19 +deny 195.39.218.0/23 +deny 195.43.0.0/19 +deny 195.166.224.0/19 +deny 195.202.64.0/19 +deny 195.234.120.0/22 +deny 195.234.168.0/24 +deny 195.234.185.0/24 +deny 195.234.252.0/22 +deny 195.246.32.0/19 +deny 196.0.0.0/16 +deny 196.1.0.0/24 +deny 196.1.4.0/24 +deny 196.1.7.0/24 +deny 196.1.8.0/21 +deny 196.1.16.0/21 +deny 196.1.24.0/23 +deny 196.1.28.0/22 +deny 196.1.32.0/20 +deny 196.1.48.0/22 +deny 196.1.56.0/21 +deny 196.1.74.0/23 +deny 196.1.76.0/22 +deny 196.1.80.0/20 +deny 196.1.96.0/21 +deny 196.1.115.0/24 +deny 196.1.116.0/22 +deny 196.1.120.0/21 +deny 196.1.128.0/23 +deny 196.1.130.0/24 +deny 196.1.132.0/23 +deny 196.1.137.0/24 +deny 196.1.143.0/24 +deny 196.1.144.0/20 +deny 196.1.176.0/20 +deny 196.1.192.0/18 +deny 196.2.2.0/23 +deny 196.2.8.0/21 +deny 196.2.16.0/20 +deny 196.2.32.0/19 +deny 196.2.64.0/18 +deny 196.2.128.0/18 +deny 196.2.192.0/19 +deny 196.3.14.0/23 +deny 196.3.58.0/23 +deny 196.3.60.0/22 +deny 196.3.64.0/24 +deny 196.3.90.0/24 +deny 196.3.92.0/23 +deny 196.3.94.0/24 +deny 196.3.96.0/21 +deny 196.3.105.0/24 +deny 196.3.107.0/24 +deny 196.3.108.0/22 +deny 196.3.112.0/20 +deny 196.3.128.0/22 +deny 196.3.148.0/22 +deny 196.3.154.0/23 +deny 196.3.156.0/22 +deny 196.3.160.0/20 +deny 196.3.176.0/21 +deny 196.3.224.0/19 +deny 196.4.0.0/19 +deny 196.4.32.0/21 +deny 196.4.40.0/22 +deny 196.4.44.0/23 +deny 196.4.71.0/24 +deny 196.4.72.0/21 +deny 196.4.80.0/20 +deny 196.4.96.0/19 +deny 196.4.128.0/17 +deny 196.5.0.0/16 +deny 196.6.0.0/18 +deny 196.6.64.0/19 +deny 196.6.96.0/20 +deny 196.6.112.0/21 +deny 196.6.121.0/24 +deny 196.6.122.0/23 +deny 196.6.124.0/22 +deny 196.6.128.0/19 +deny 196.6.160.0/20 +deny 196.6.184.0/23 +deny 196.6.188.0/22 +deny 196.6.198.0/24 +deny 196.6.200.0/21 +deny 196.6.208.0/20 +deny 196.6.224.0/23 +deny 196.6.227.0/24 +deny 196.6.228.0/23 +deny 196.6.231.0/24 +deny 196.6.232.0/21 +deny 196.6.240.0/20 +deny 196.7.0.0/16 +deny 196.8.0.0/15 +deny 196.10.0.0/19 +deny 196.10.32.0/20 +deny 196.10.48.0/23 +deny 196.10.50.0/24 +deny 196.10.52.0/22 +deny 196.10.61.0/24 +deny 196.10.62.0/23 +deny 196.10.64.0/19 +deny 196.10.96.0/22 +deny 196.10.101.0/24 +deny 196.10.102.0/23 +deny 196.10.104.0/21 +deny 196.10.112.0/20 +deny 196.10.128.0/21 +deny 196.10.136.0/22 +deny 196.10.141.0/24 +deny 196.10.142.0/23 +deny 196.10.144.0/23 +deny 196.10.146.0/24 +deny 196.10.148.0/22 +deny 196.10.152.0/21 +deny 196.10.160.0/19 +deny 196.10.192.0/19 +deny 196.10.224.0/20 +deny 196.10.240.0/21 +deny 196.10.248.0/22 +deny 196.11.0.0/20 +deny 196.11.16.0/21 +deny 196.11.24.0/22 +deny 196.11.28.0/23 +deny 196.11.30.0/24 +deny 196.11.32.0/19 +deny 196.11.64.0/21 +deny 196.11.72.0/23 +deny 196.11.74.0/24 +deny 196.11.76.0/22 +deny 196.11.80.0/20 +deny 196.11.96.0/19 +deny 196.11.128.0/18 +deny 196.11.192.0/19 +deny 196.11.224.0/20 +deny 196.11.240.0/21 +deny 196.11.248.0/23 +deny 196.11.252.0/22 +deny 196.12.10.0/23 +deny 196.12.12.0/22 +deny 196.12.16.0/20 +deny 196.12.128.0/19 +deny 196.12.192.0/18 +deny 196.13.0.0/18 +deny 196.13.64.0/19 +deny 196.13.96.0/22 +deny 196.13.100.0/23 +deny 196.13.103.0/24 +deny 196.13.104.0/23 +deny 196.13.107.0/24 +deny 196.13.108.0/22 +deny 196.13.112.0/20 +deny 196.13.128.0/19 +deny 196.13.160.0/20 +deny 196.13.184.0/21 +deny 196.13.192.0/21 +deny 196.13.200.0/24 +deny 196.13.202.0/24 +deny 196.13.204.0/22 +deny 196.13.208.0/20 +deny 196.13.224.0/19 +deny 196.14.0.0/16 +deny 196.15.0.0/20 +deny 196.15.64.0/18 +deny 196.15.128.0/17 +deny 196.16.0.0/14 +deny 196.20.0.0/19 +deny 196.20.64.0/18 +deny 196.20.128.0/17 +deny 196.21.0.0/16 +deny 196.22.0.0/21 +deny 196.22.8.0/22 +deny 196.22.16.0/20 +deny 196.22.32.0/19 +deny 196.22.64.0/18 +deny 196.22.128.0/17 +deny 196.23.0.0/16 +deny 196.24.0.0/15 +deny 196.26.0.0/16 +deny 196.27.64.0/18 +deny 196.27.128.0/17 +deny 196.28.0.0/21 +deny 196.28.8.0/24 +deny 196.28.10.0/23 +deny 196.28.16.0/20 +deny 196.28.32.0/20 +deny 196.28.64.0/18 +deny 196.28.128.0/17 +deny 196.29.0.0/18 +deny 196.29.128.0/17 +deny 196.30.0.0/15 +deny 196.32.8.0/21 +deny 196.32.96.0/19 +deny 196.32.160.0/19 +deny 196.32.192.0/21 +deny 196.32.208.0/20 +deny 196.32.224.0/19 +deny 196.33.0.0/16 +deny 196.34.0.0/15 +deny 196.36.0.0/14 +deny 196.40.96.0/19 +deny 196.40.128.0/17 +deny 196.41.0.0/18 +deny 196.41.65.0/24 +deny 196.41.67.0/24 +deny 196.41.68.0/24 +deny 196.41.72.0/23 +deny 196.41.75.0/24 +deny 196.41.76.0/23 +deny 196.41.80.0/23 +deny 196.41.83.0/24 +deny 196.41.84.0/22 +deny 196.41.88.0/23 +deny 196.41.94.0/23 +deny 196.41.96.0/19 +deny 196.41.128.0/17 +deny 196.42.64.0/18 +deny 196.43.0.0/17 +deny 196.43.128.0/18 +deny 196.43.192.0/24 +deny 196.43.194.0/23 +deny 196.43.196.0/23 +deny 196.43.199.0/24 +deny 196.43.201.0/24 +deny 196.43.202.0/24 +deny 196.43.205.0/24 +deny 196.43.206.0/23 +deny 196.43.208.0/21 +deny 196.43.217.0/24 +deny 196.43.219.0/24 +deny 196.43.220.0/23 +deny 196.43.223.0/24 +deny 196.43.225.0/24 +deny 196.43.226.0/23 +deny 196.43.228.0/22 +deny 196.43.232.0/22 +deny 196.43.236.0/23 +deny 196.43.239.0/24 +deny 196.43.240.0/21 +deny 196.43.248.0/23 +deny 196.43.251.0/24 +deny 196.43.253.0/24 +deny 196.44.0.0/16 +deny 196.45.0.0/18 +deny 196.45.64.0/19 +deny 196.45.96.0/20 +deny 196.45.128.0/17 +deny 196.46.0.0/20 +deny 196.46.16.0/23 +deny 196.46.20.0/22 +deny 196.46.24.0/23 +deny 196.46.27.0/24 +deny 196.46.28.0/24 +deny 196.46.30.0/23 +deny 196.46.32.0/19 +deny 196.46.64.0/19 +deny 196.46.96.0/20 +deny 196.46.116.0/22 +deny 196.46.120.0/21 +deny 196.46.128.0/20 +deny 196.46.144.0/21 +deny 196.46.160.0/19 +deny 196.46.192.0/19 +deny 196.46.224.0/20 +deny 196.46.240.0/21 +deny 196.47.0.0/16 +deny 196.48.0.0/16 +deny 196.49.0.0/20 +deny 196.49.16.0/22 +deny 196.49.20.0/24 +deny 196.49.22.0/24 +deny 196.49.24.0/24 +deny 196.49.26.0/24 +deny 196.49.28.0/24 +deny 196.49.30.0/24 +deny 196.49.32.0/24 +deny 196.49.34.0/24 +deny 196.49.38.0/24 +deny 196.49.40.0/24 +deny 196.49.42.0/24 +deny 196.49.44.0/24 +deny 196.49.46.0/24 +deny 196.49.48.0/24 +deny 196.49.50.0/24 +deny 196.49.52.0/24 +deny 196.49.54.0/24 +deny 196.49.56.0/24 +deny 196.49.58.0/24 +deny 196.49.60.0/24 +deny 196.49.62.0/24 +deny 196.49.64.0/24 +deny 196.49.66.0/24 +deny 196.49.68.0/23 +deny 196.49.72.0/24 +deny 196.49.74.0/24 +deny 196.49.76.0/23 +deny 196.49.82.0/24 +deny 196.49.84.0/24 +deny 196.49.86.0/24 +deny 196.49.88.0/24 +deny 196.49.90.0/24 +deny 196.49.92.0/24 +deny 196.49.96.0/24 +deny 196.49.98.0/24 +deny 196.49.100.0/24 +deny 196.49.102.0/24 +deny 196.49.104.0/24 +deny 196.49.106.0/24 +deny 196.49.108.0/24 +deny 196.49.110.0/24 +deny 196.49.112.0/24 +deny 196.49.114.0/23 +deny 196.49.116.0/24 +deny 196.49.118.0/24 +deny 196.49.120.0/23 +deny 196.49.122.0/24 +deny 196.49.124.0/24 +deny 196.49.126.0/23 +deny 196.50.0.0/20 +deny 196.50.16.0/22 +deny 196.50.20.0/24 +deny 196.50.22.0/23 +deny 196.50.24.0/21 +deny 196.50.32.0/23 +deny 196.50.64.0/18 +deny 196.50.128.0/17 +deny 196.51.0.0/16 +deny 196.56.0.0/14 +deny 196.60.0.0/24 +deny 196.60.2.0/24 +deny 196.60.4.0/24 +deny 196.60.6.0/23 +deny 196.60.8.0/22 +deny 196.60.12.0/24 +deny 196.60.14.0/24 +deny 196.60.36.0/24 +deny 196.60.38.0/24 +deny 196.60.40.0/24 +deny 196.60.42.0/24 +deny 196.60.44.0/24 +deny 196.60.46.0/24 +deny 196.60.48.0/24 +deny 196.60.50.0/24 +deny 196.60.52.0/24 +deny 196.60.54.0/24 +deny 196.60.56.0/24 +deny 196.60.58.0/24 +deny 196.60.61.0/24 +deny 196.60.64.0/24 +deny 196.60.66.0/24 +deny 196.60.68.0/24 +deny 196.60.70.0/23 +deny 196.60.74.0/24 +deny 196.60.76.0/24 +deny 196.60.78.0/24 +deny 196.60.80.0/24 +deny 196.60.82.0/24 +deny 196.60.84.0/24 +deny 196.60.86.0/24 +deny 196.60.88.0/24 +deny 196.60.90.0/24 +deny 196.60.92.0/24 +deny 196.60.94.0/24 +deny 196.60.96.0/23 +deny 196.60.104.0/24 +deny 196.60.106.0/24 +deny 196.60.110.0/24 +deny 196.60.112.0/24 +deny 196.60.114.0/24 +deny 196.60.116.0/24 +deny 196.60.118.0/24 +deny 196.60.120.0/24 +deny 196.60.122.0/24 +deny 196.60.124.0/24 +deny 196.60.126.0/24 +deny 196.60.160.0/24 +deny 196.60.162.0/23 +deny 196.60.164.0/24 +deny 196.60.166.0/24 +deny 196.60.252.0/24 +deny 196.60.254.0/24 +deny 196.61.8.0/21 +deny 196.61.16.0/20 +deny 196.61.32.0/19 +deny 196.61.64.0/20 +deny 196.61.96.0/19 +deny 196.61.128.0/18 +deny 196.61.208.0/20 +deny 196.61.224.0/19 +deny 196.64.0.0/10 +deny 196.128.0.0/11 +deny 196.168.0.0/13 +deny 196.176.0.0/12 +deny 196.192.0.0/17 +deny 196.192.128.0/20 +deny 196.192.160.0/19 +deny 196.196.0.0/14 +deny 196.200.0.0/15 +deny 196.202.0.0/17 +deny 196.202.128.0/18 +deny 196.202.192.0/19 +deny 196.202.232.0/21 +deny 196.202.240.0/20 +deny 196.203.0.0/16 +deny 196.204.0.0/15 +deny 196.206.0.0/16 +deny 196.207.0.0/18 +deny 196.207.128.0/17 +deny 196.208.0.0/13 +deny 196.216.2.0/23 +deny 196.216.4.0/22 +deny 196.216.8.0/21 +deny 196.216.24.0/21 +deny 196.216.32.0/19 +deny 196.216.64.0/18 +deny 196.216.132.0/22 +deny 196.216.136.0/21 +deny 196.216.144.0/20 +deny 196.216.161.0/24 +deny 196.216.162.0/23 +deny 196.216.164.0/22 +deny 196.216.168.0/21 +deny 196.216.176.0/20 +deny 196.216.192.0/22 +deny 196.216.200.0/21 +deny 196.216.208.0/20 +deny 196.216.224.0/21 +deny 196.216.234.0/23 +deny 196.216.236.0/22 +deny 196.216.240.0/20 +deny 196.217.0.0/16 +deny 196.218.0.0/15 +deny 196.220.0.0/17 +deny 196.220.128.0/18 +deny 196.220.192.0/20 +deny 196.220.224.0/19 +deny 196.221.0.0/16 +deny 196.222.0.0/16 +deny 196.223.0.0/23 +deny 196.223.2.0/24 +deny 196.223.4.0/22 +deny 196.223.8.0/22 +deny 196.223.12.0/23 +deny 196.223.15.0/24 +deny 196.223.16.0/20 +deny 196.223.32.0/19 +deny 196.223.64.0/18 +deny 196.223.128.0/17 +deny 196.224.0.0/12 +deny 196.240.0.0/14 +deny 196.244.0.0/15 +deny 196.247.0.0/16 +deny 196.248.0.0/15 +deny 196.250.0.0/17 +deny 196.250.128.0/19 +deny 196.250.176.0/20 +deny 196.250.192.0/19 +deny 196.250.224.0/20 +deny 196.250.240.0/21 +deny 196.250.252.0/22 +deny 196.251.0.0/16 +deny 196.252.0.0/14 +deny 197.0.0.0/9 +deny 197.128.0.0/12 +deny 197.144.0.0/14 +deny 197.148.0.0/17 +deny 197.148.128.0/18 +deny 197.148.192.0/19 +deny 197.149.0.0/17 +deny 197.149.128.0/20 +deny 197.149.144.0/21 +deny 197.149.152.0/22 +deny 197.149.160.0/20 +deny 197.149.176.0/21 +deny 197.149.184.0/22 +deny 197.149.192.0/18 +deny 197.150.0.0/15 +deny 197.152.0.0/14 +deny 197.156.0.0/16 +deny 197.157.0.0/17 +deny 197.157.128.0/18 +deny 197.157.192.0/21 +deny 197.157.204.0/22 +deny 197.157.208.0/20 +deny 197.157.228.0/22 +deny 197.157.232.0/21 +deny 197.157.240.0/20 +deny 197.158.0.0/17 +deny 197.158.192.0/18 +deny 197.159.0.0/18 +deny 197.159.64.0/20 +deny 197.159.88.0/21 +deny 197.159.96.0/20 +deny 197.159.128.0/17 +deny 197.160.0.0/11 +deny 197.192.0.0/12 +deny 197.208.0.0/13 +deny 197.216.0.0/14 +deny 197.220.0.0/17 +deny 197.220.128.0/19 +deny 197.221.0.0/16 +deny 197.222.0.0/15 +deny 197.224.0.0/14 +deny 197.228.0.0/15 +deny 197.230.0.0/16 +deny 197.231.0.0/17 +deny 197.231.128.0/21 +deny 197.231.144.0/20 +deny 197.231.160.0/19 +deny 197.231.192.0/20 +deny 197.231.212.0/22 +deny 197.231.216.0/21 +deny 197.231.224.0/20 +deny 197.231.240.0/21 +deny 197.231.252.0/22 +deny 197.232.0.0/15 +deny 197.234.0.0/17 +deny 197.234.128.0/18 +deny 197.234.192.0/20 +deny 197.234.216.0/21 +deny 197.234.224.0/19 +deny 197.235.0.0/16 +deny 197.236.0.0/14 +deny 197.240.0.0/13 +deny 197.248.0.0/14 +deny 197.252.0.0/15 +deny 197.254.0.0/16 +deny 197.255.0.0/17 +deny 197.255.128.0/18 +deny 197.255.192.0/19 +deny 197.255.224.0/20 +deny 197.255.240.0/21 +deny 197.255.252.0/22 +deny 198.54.1.0/24 +deny 198.54.2.0/23 +deny 198.54.4.0/22 +deny 198.54.8.0/23 +deny 198.54.10.0/24 +deny 198.54.15.0/24 +deny 198.54.16.0/20 +deny 198.54.32.0/20 +deny 198.54.48.0/21 +deny 198.54.56.0/22 +deny 198.54.60.0/23 +deny 198.54.64.0/20 +deny 198.54.80.0/24 +deny 198.54.83.0/24 +deny 198.54.84.0/22 +deny 198.54.91.0/24 +deny 198.54.98.0/23 +deny 198.54.141.0/24 +deny 198.54.142.0/23 +deny 198.54.144.0/21 +deny 198.54.154.0/23 +deny 198.54.156.0/22 +deny 198.54.160.0/22 +deny 198.54.165.0/24 +deny 198.54.170.0/24 +deny 198.54.173.0/24 +deny 198.54.174.0/23 +deny 198.54.176.0/22 +deny 198.54.180.0/23 +deny 198.54.183.0/24 +deny 198.54.184.0/21 +deny 198.54.192.0/21 +deny 198.54.202.0/23 +deny 198.54.204.0/22 +deny 198.54.208.0/23 +deny 198.54.210.0/24 +deny 198.54.219.0/24 +deny 198.54.220.0/24 +deny 198.54.223.0/24 +deny 198.54.225.0/24 +deny 198.54.226.0/23 +deny 198.54.228.0/23 +deny 198.54.231.0/24 +deny 198.54.232.0/24 +deny 198.54.234.0/24 +deny 198.54.247.0/24 +deny 198.54.248.0/22 +deny 198.54.252.0/23 +deny 198.54.254.0/24 +deny 200.16.8.0/21 +deny 202.123.0.0/19 +deny 204.8.204.0/22 +deny 204.12.128.0/20 +deny 204.87.179.0/24 +deny 204.152.14.0/23 +deny 204.235.32.0/21 +deny 204.235.40.0/22 +deny 205.159.79.0/24 +deny 206.223.136.0/24 +deny 208.85.156.0/22 +deny 209.203.0.0/18 +deny 209.212.96.0/19 +deny 212.0.128.0/19 +deny 212.22.160.0/19 +deny 212.49.64.0/19 +deny 212.52.128.0/19 +deny 212.60.64.0/19 +deny 212.85.192.0/19 +deny 212.88.96.0/19 +deny 212.96.0.0/19 +deny 212.100.64.0/19 +deny 212.103.160.0/19 +deny 212.217.0.0/17 +deny 213.55.64.0/18 +deny 213.131.64.0/19 +deny 213.136.96.0/19 +deny 213.150.160.0/19 +deny 213.150.192.0/19 +deny 213.152.64.0/19 +deny 213.154.32.0/19 +deny 213.154.64.0/19 +deny 213.158.160.0/19 +deny 213.172.128.0/19 +deny 213.179.160.0/19 +deny 213.181.224.0/19 +deny 213.193.32.0/19 +deny 213.212.192.0/18 +deny 213.255.128.0/19 +deny 217.14.80.0/20 +deny 217.20.224.0/20 +deny 217.21.112.0/20 +deny 217.29.128.0/20 +deny 217.29.208.0/20 +deny 217.52.0.0/14 +deny 217.64.96.0/20 +deny 217.77.64.0/20 +deny 217.117.0.0/20 +deny 217.139.0.0/16 +deny 217.170.144.0/20 +deny 217.199.144.0/20 diff --git a/crates/panda-rpki-validator/tests/fixtures/ta_constraints/afrinic-full-ipv4-deny.constraints b/crates/panda-rpki-validator/tests/fixtures/ta_constraints/afrinic-full-ipv4-deny.constraints new file mode 100644 index 0000000..463ba07 --- /dev/null +++ b/crates/panda-rpki-validator/tests/fixtures/ta_constraints/afrinic-full-ipv4-deny.constraints @@ -0,0 +1,394 @@ +# Feature #151 remote all5 performance fixture. +# +# Source: official rpki-client 9.8 afrinic.constraints +# ($OpenBSD: afrinic.constraints,v 1.4 2026/03/11 21:46:36 job Exp $). +# Every IPv4 allow entry from that file (including recovered and transfer +# ranges) is converted to deny. IPv6 and ASN entries are intentionally not +# included because this fixture measures the full AFRINIC IPv4 policy. +allow 0.0.0.0/0 +allow ::/0 +allow 0 - 4294967295 + +deny 41.0.0.0/8 +deny 102.0.0.0/8 +deny 105.0.0.0/8 +deny 154.0.0.0/16 +deny 154.16.0.0/16 +deny 154.65.0.0 - 154.255.255.255 +deny 196.0.0.0 - 196.1.0.255 +deny 196.1.4.0/24 +deny 196.1.7.0 - 196.1.63.255 +deny 196.1.71.0/24 +deny 196.1.74.0 - 196.1.103.255 +deny 196.1.115.0 - 196.1.133.255 +deny 196.1.137.0/24 +deny 196.1.143.0 - 196.1.159.255 +deny 196.1.176.0 - 196.1.255.255 +deny 196.2.2.0/23 +deny 196.2.8.0 - 196.2.255.255 +deny 196.3.14.0/23 +deny 196.3.57.0 - 196.3.64.255 +deny 196.3.90.0/24 +deny 196.3.92.0 - 196.3.94.255 +deny 196.3.96.0/21 +deny 196.3.105.0/24 +deny 196.3.107.0 - 196.3.131.255 +deny 196.3.148.0/22 +deny 196.3.154.0 - 196.3.183.255 +deny 196.3.224.0 - 196.4.45.255 +deny 196.4.71.0 - 196.11.171.255 +deny 196.11.174.0 - 196.11.239.255 +deny 196.11.248.0/21 +deny 196.12.10.0 - 196.12.31.255 +deny 196.12.128.0/19 +deny 196.12.192.0 - 196.15.15.255 +deny 196.15.64.0 - 196.26.255.255 +deny 196.27.64.0 - 196.28.47.255 +deny 196.28.64.0 - 196.29.63.255 +deny 196.29.96.0 - 196.31.255.255 +deny 196.32.8.0 - 196.32.31.255 +deny 196.32.96.0/19 +deny 196.32.160.0 - 196.39.255.255 +deny 196.40.96.0 - 196.41.255.255 +deny 196.42.64.0 - 196.216.0.255 +deny 196.216.2.0 - 197.255.255.255 +deny 45.96.0.0 - 45.111.255.255 +deny 45.192.0.0 - 45.222.255.255 +deny 45.240.0.0 - 45.247.255.255 +deny 66.251.128.0 - 66.251.191.255 +deny 139.26.0.0 - 139.26.255.255 +deny 146.196.128.0 - 146.196.255.255 +deny 160.19.36.0 - 160.19.39.255 +deny 160.19.60.0 - 160.19.63.255 +deny 160.19.96.0 - 160.19.103.255 +deny 160.19.112.0 - 160.19.143.255 +deny 160.19.152.0 - 160.19.155.255 +deny 160.19.188.0 - 160.19.191.255 +deny 160.19.192.0 - 160.19.199.255 +deny 160.19.232.0 - 160.19.239.255 +deny 160.20.24.0 - 160.20.31.255 +deny 160.20.112.0 - 160.20.115.255 +deny 160.20.213.0 - 160.20.213.255 +deny 160.20.217.0 - 160.20.217.255 +deny 160.20.221.0 - 160.20.221.255 +deny 160.20.226.0 - 160.20.227.255 +deny 160.20.252.0 - 160.20.255.255 +deny 160.238.11.0 - 160.238.11.255 +deny 160.238.48.0 - 160.238.49.255 +deny 160.238.50.0 - 160.238.50.255 +deny 160.238.57.0 - 160.238.57.255 +deny 160.238.101.0 - 160.238.101.255 +deny 161.123.0.0 - 161.123.255.255 +deny 164.160.0.0 - 164.160.255.255 +deny 192.12.110.0 - 192.12.111.255 +deny 192.12.116.0 - 192.12.117.255 +deny 192.47.36.0 - 192.47.36.255 +deny 192.51.240.0 - 192.51.240.255 +deny 192.70.200.0 - 192.70.201.255 +deny 192.75.236.0 - 192.75.236.255 +deny 192.83.208.0 - 192.83.215.255 +deny 192.91.200.0 - 192.91.200.255 +deny 192.142.0.0 - 192.143.255.255 +deny 192.145.128.0 - 192.145.191.255 +deny 192.145.230.0 - 192.145.230.255 +deny 204.8.204.0 - 204.8.207.255 +deny 208.85.156.0 - 208.85.159.255 +deny 83.143.24.0 - 83.143.31.255 +deny 84.205.96.0 - 84.205.127.255 +deny 131.176.0.0 - 131.176.255.255 +deny 163.121.0.0 - 163.121.255.255 +deny 165.231.0.0 - 165.231.255.255 +deny 192.52.232.0 - 192.52.232.255 +deny 193.17.215.0 - 193.17.215.255 +deny 193.19.232.0 - 193.19.235.255 +deny 193.41.146.0 - 193.41.147.255 +deny 193.108.23.0 - 193.108.23.255 +deny 193.108.28.0 - 193.108.28.255 +deny 193.109.66.0 - 193.109.67.255 +deny 193.110.104.0 - 193.110.105.255 +deny 193.194.128.0 - 193.194.128.255 +deny 193.227.128.0 - 193.227.128.255 +deny 194.9.64.0 - 194.9.65.255 +deny 194.9.82.0 - 194.9.83.255 +deny 195.24.80.0 - 195.24.87.255 +deny 195.39.218.0 - 195.39.219.255 +deny 195.234.120.0 - 195.234.123.255 +deny 195.234.168.0 - 195.234.168.255 +deny 195.234.185.0 - 195.234.185.255 +deny 195.234.252.0 - 195.234.255.255 +deny 193.188.7.0/24 +deny 193.189.0.0/18 +deny 193.189.128.0/24 +deny 193.194.160.0/19 +deny 193.221.218.0/24 +deny 64.57.112.0 - 64.57.127.255 +deny 66.8.0.0 - 66.8.127.255 +deny 66.18.64.0 - 66.18.95.255 +deny 69.63.64.0 - 69.63.79.255 +deny 69.67.32.0 - 69.67.47.255 +deny 137.158.0.0 - 137.158.255.255 +deny 137.214.0.0 - 137.214.255.255 +deny 137.215.0.0 - 137.215.255.255 +deny 139.53.0.0 - 139.53.255.255 +deny 143.128.0.0 - 143.128.255.255 +deny 143.160.0.0 - 143.160.255.255 +deny 146.64.0.0 - 146.64.255.255 +deny 146.141.0.0 - 146.141.255.255 +deny 146.182.0.0 - 146.182.255.255 +deny 146.230.0.0 - 146.230.255.255 +deny 146.231.0.0 - 146.231.255.255 +deny 146.232.0.0 - 146.232.255.255 +deny 147.110.0.0 - 147.110.255.255 +deny 152.106.0.0 - 152.106.255.255 +deny 152.107.0.0 - 152.107.255.255 +deny 152.108.0.0 - 152.108.255.255 +deny 152.109.0.0 - 152.109.255.255 +deny 152.110.0.0 - 152.110.255.255 +deny 152.111.0.0 - 152.111.255.255 +deny 152.112.0.0 - 152.112.255.255 +deny 155.159.0.0 - 155.159.255.255 +deny 155.232.0.0 - 155.232.255.255 +deny 155.233.0.0 - 155.233.255.255 +deny 155.234.0.0 - 155.234.255.255 +deny 155.235.0.0 - 155.235.255.255 +deny 155.236.0.0 - 155.236.255.255 +deny 155.237.0.0 - 155.237.255.255 +deny 155.238.0.0 - 155.238.255.255 +deny 155.239.0.0 - 155.239.255.255 +deny 155.240.0.0 - 155.240.255.255 +deny 156.8.0.0 - 156.8.255.255 +deny 160.115.0.0 - 160.115.255.255 +deny 160.116.0.0 - 160.116.255.255 +deny 160.117.0.0 - 160.117.255.255 +deny 160.118.0.0 - 160.118.255.255 +deny 160.119.0.0 - 160.119.255.255 +deny 160.120.0.0 - 160.120.255.255 +deny 160.121.0.0 - 160.121.255.255 +deny 160.122.0.0 - 160.122.255.255 +deny 160.123.0.0 - 160.123.255.255 +deny 160.124.0.0 - 160.124.255.255 +deny 163.195.0.0 - 163.195.255.255 +deny 163.196.0.0 - 163.196.255.255 +deny 163.197.0.0 - 163.197.255.255 +deny 163.198.0.0 - 163.198.255.255 +deny 163.199.0.0 - 163.199.255.255 +deny 163.200.0.0 - 163.200.255.255 +deny 163.201.0.0 - 163.201.255.255 +deny 163.202.0.0 - 163.202.255.255 +deny 163.203.0.0 - 163.203.255.255 +deny 164.88.0.0 - 164.88.255.255 +deny 164.146.0.0 - 164.151.255.255 +deny 164.155.0.0 - 164.155.255.255 +deny 165.3.0.0 - 165.5.255.255 +deny 165.8.0.0 - 165.11.255.255 +deny 165.25.0.0 - 165.25.255.255 +deny 165.143.0.0 - 165.149.255.255 +deny 165.165.0.0 - 165.165.255.255 +deny 165.180.0.0 - 165.180.255.255 +deny 165.233.0.0 - 165.233.255.255 +deny 166.85.0.0 - 166.85.255.255 +deny 168.76.0.0 - 168.76.255.255 +deny 168.80.0.0 - 168.81.255.255 +deny 168.89.0.0 - 168.89.255.255 +deny 168.128.0.0 - 168.128.255.255 +deny 168.142.0.0 - 168.142.255.255 +deny 168.155.0.0 - 168.155.255.255 +deny 168.164.0.0 - 168.164.255.255 +deny 168.167.0.0 - 168.167.255.255 +deny 168.172.0.0 - 168.172.255.255 +deny 168.206.0.0 - 168.206.255.255 +deny 168.209.0.0 - 168.210.255.255 +deny 169.129.0.0 - 169.129.255.255 +deny 169.202.0.0 - 169.202.255.255 +deny 192.33.10.0 - 192.33.10.255 +deny 192.42.99.0 - 192.42.99.255 +deny 192.48.253.0 - 192.48.253.255 +deny 192.68.138.0 - 192.68.138.255 +deny 192.70.237.0 - 192.70.237.255 +deny 192.82.142.0 - 192.82.142.255 +deny 192.84.244.0 - 192.84.244.255 +deny 192.94.61.0 - 192.94.61.255 +deny 192.94.210.0 - 192.94.210.255 +deny 192.94.240.0 - 192.94.240.255 +deny 192.94.241.0 - 192.94.241.255 +deny 192.94.246.0 - 192.94.246.255 +deny 192.96.0.0 - 192.96.255.255 +deny 192.100.1.0 - 192.100.1.255 +deny 192.101.142.0 - 192.101.142.255 +deny 192.102.9.0 - 192.102.9.255 +deny 192.133.250.0 - 192.133.250.255 +deny 192.136.55.0 - 192.136.55.255 +deny 192.136.56.0 - 192.136.56.255 +deny 192.136.57.0 - 192.136.57.255 +deny 192.157.190.0 - 192.157.190.255 +deny 192.188.164.0 - 192.188.167.255 +deny 192.189.75.0 - 192.189.75.255 +deny 192.189.139.0 - 192.189.140.255 +deny 192.231.237.0 - 192.231.237.255 +deny 192.231.254.0 - 192.231.254.255 +deny 192.245.148.0 - 192.245.148.255 +deny 192.251.202.0 - 192.251.202.255 +deny 198.54.0.0 - 198.54.255.255 +deny 200.16.8.0 - 200.16.15.255 +deny 204.12.128.0 - 204.12.143.255 +deny 204.87.179.0 - 204.87.179.255 +deny 204.152.14.0 - 204.152.15.255 +deny 204.235.32.0 - 204.235.43.255 +deny 205.159.79.0 - 205.159.79.255 +deny 206.223.136.0 - 206.223.136.255 +deny 209.203.0.0 - 209.203.63.255 +deny 209.212.96.0 - 209.212.127.255 +deny 216.236.176.0 - 216.236.191.255 +deny 202.123.0.0/19 +deny 62.8.64.0/19 +deny 62.12.96.0/19 +deny 62.24.96.0/19 +deny 62.61.192.0/18 +deny 62.68.32.0/19 +deny 62.68.224.0/19 +deny 62.114.0.0/16 +deny 62.117.32.0/19 +deny 62.135.0.0/17 +deny 62.139.0.0/16 +deny 62.140.64.0/18 +deny 62.173.32.0/19 +deny 62.193.64.0/18 +deny 62.193.160.0/19 +deny 62.240.32.0/19 +deny 62.240.96.0/19 +deny 62.241.128.0/19 +deny 62.251.128.0/17 +deny 77.220.0.0/19 +deny 80.67.128.0/20 +deny 80.72.96.0/20 +deny 80.75.160.0/19 +deny 80.87.64.0/19 +deny 80.88.0.0/20 +deny 80.95.0.0/20 +deny 80.240.192.0/20 +deny 80.246.0.0/20 +deny 80.248.0.0/20 +deny 80.248.64.0/20 +deny 80.249.64.0/20 +deny 80.250.32.0/20 +deny 81.4.0.0/18 +deny 81.10.0.0/17 +deny 81.21.96.0/20 +deny 81.22.64.0/19 +deny 81.26.64.0/20 +deny 81.29.96.0/20 +deny 81.91.224.0/20 +deny 81.192.0.0/16 +deny 82.101.128.0/18 +deny 82.128.0.0/17 +deny 82.129.128.0/17 +deny 82.151.64.0/19 +deny 82.201.128.0/17 +deny 84.36.0.0/16 +deny 84.233.0.0/17 +deny 87.255.96.0/19 +deny 193.95.0.0/17 +deny 193.108.214.0/24 +deny 193.108.252.0/22 +deny 193.189.64.0 - 193.189.65.255 +deny 193.194.1.0 - 193.194.5.255 +deny 193.194.32.0 - 193.194.95.255 +deny 193.227.0.0/18 +deny 194.6.224.0/24 +deny 194.79.96.0/19 +deny 194.204.192.0/18 +deny 195.24.192.0/19 +deny 195.43.0.0/19 +deny 195.166.224.0/19 +deny 195.202.64.0/19 +deny 195.246.32.0/19 +deny 212.0.128.0/19 +deny 212.12.224.0/19 +deny 212.22.160.0/19 +deny 212.49.64.0/19 +deny 212.52.128.0/19 +deny 212.60.64.0/19 +deny 212.85.192.0/19 +deny 212.88.96.0/19 +deny 212.96.0.0/19 +deny 212.100.64.0/19 +deny 212.103.160.0/19 +deny 212.122.224.0/19 +deny 212.217.0.0/17 +deny 213.55.64.0/18 +deny 213.131.64.0/19 +deny 213.136.96.0/19 +deny 213.147.64.0/19 +deny 213.150.96.0/19 +deny 213.150.160.0 - 213.150.223.255 +deny 213.152.64.0/19 +deny 213.154.32.0 - 213.154.95.255 +deny 213.158.160.0/19 +deny 213.172.128.0/19 +deny 213.179.160.0/19 +deny 213.181.224.0/19 +deny 213.193.32.0/19 +deny 213.212.192.0/18 +deny 213.247.0.0/19 +deny 213.255.128.0/19 +deny 217.14.80.0/20 +deny 217.20.224.0/20 +deny 217.21.112.0/20 +deny 217.29.128.0/20 +deny 217.29.208.0/20 +deny 217.52.0.0/14 +deny 217.64.96.0/20 +deny 217.77.64.0/20 +deny 217.78.64.0/20 +deny 217.117.0.0/20 +deny 217.139.0.0/16 +deny 217.170.144.0/20 +deny 217.199.144.0/20 +deny 129.0.0.0/16 +deny 129.18.0.0/16 +deny 129.45.0.0/16 +deny 129.56.0.0/16 +deny 129.122.0.0/16 +deny 129.140.0.0/16 +deny 129.205.0.0/16 +deny 129.232.0.0/16 +deny 137.63.0.0 - 137.64.255.255 +deny 137.115.0.0/16 +deny 137.171.0.0/16 +deny 137.196.0.0/16 +deny 137.255.0.0/16 +deny 155.0.0.0/16 +deny 155.11.0.0 - 155.12.255.255 +deny 155.89.0.0/16 +deny 155.93.0.0/16 +deny 155.196.0.0/16 +deny 155.251.0.0/16 +deny 155.255.0.0 - 156.0.255.255 +deny 156.38.0.0/16 +deny 156.155.0.0 - 156.255.255.255 +deny 160.0.0.0/16 +deny 160.77.0.0/16 +deny 160.89.0.0 - 160.90.255.255 +deny 160.105.0.0/16 +deny 160.113.0.0/16 +deny 160.152.0.0/16 +deny 160.154.0.0 - 160.179.255.255 +deny 160.181.0.0 - 160.184.255.255 +deny 160.224.0.0 - 160.226.255.255 +deny 160.242.0.0/16 +deny 160.255.0.0/16 +deny 165.0.0.0/16 +deny 165.16.0.0/16 +deny 165.49.0.0 - 165.63.255.255 +deny 165.73.0.0/16 +deny 165.90.0.0/16 +deny 165.169.0.0/16 +deny 165.210.0.0/15 +deny 165.255.0.0/16 +deny 168.211.0.0 - 168.211.255.255 +deny 168.253.0.0/16 +deny 169.0.0.0/15 +deny 169.159.0.0/16 +deny 169.239.0.0/16 +deny 169.255.0.0/16 +deny 192.109.242.0/24 diff --git a/crates/panda-rpki-validator/tests/fixtures/ta_constraints/local-custom-allow.constraints b/crates/panda-rpki-validator/tests/fixtures/ta_constraints/local-custom-allow.constraints new file mode 100644 index 0000000..5b64975 --- /dev/null +++ b/crates/panda-rpki-validator/tests/fixtures/ta_constraints/local-custom-allow.constraints @@ -0,0 +1,2 @@ +# local baseline-v1: the single valid ROA EE certificate has this IPv4 block +allow 203.0.113.0/24 diff --git a/crates/panda-rpki-validator/tests/fixtures/ta_constraints/local-custom-deny.constraints b/crates/panda-rpki-validator/tests/fixtures/ta_constraints/local-custom-deny.constraints new file mode 100644 index 0000000..9a4ac6c --- /dev/null +++ b/crates/panda-rpki-validator/tests/fixtures/ta_constraints/local-custom-deny.constraints @@ -0,0 +1,3 @@ +# local baseline-v1 negative control: deny wins over a covering allow rule +allow 0.0.0.0/0 +deny 203.0.113.0/24 diff --git a/crates/panda-rpki-validator/tests/fixtures/ta_constraints/ripe-ncc-afrinic-deny.constraints b/crates/panda-rpki-validator/tests/fixtures/ta_constraints/ripe-ncc-afrinic-deny.constraints new file mode 100644 index 0000000..1a9384b --- /dev/null +++ b/crates/panda-rpki-validator/tests/fixtures/ta_constraints/ripe-ncc-afrinic-deny.constraints @@ -0,0 +1,12 @@ +# Remote RIPE TA soak policy for feature #151. +# +# Start permissive for every INR kind, then carve out several /8 blocks +# allocated to AFRINIC. These denies should not match normal RIPE NCC +# EE resources, while exercising deny-overrides-allow in a live run. +allow 0.0.0.0/0 +allow ::/0 +allow 0 - 4294967295 + +deny 41.0.0.0/8 +deny 102.0.0.0/8 +deny 105.0.0.0/8 diff --git a/crates/panda-rpki-validator/tests/fixtures/tal/afrinic.tal b/crates/panda-rpki-validator/tests/fixtures/tal/afrinic.tal new file mode 100644 index 0000000..5ff40bf --- /dev/null +++ b/crates/panda-rpki-validator/tests/fixtures/tal/afrinic.tal @@ -0,0 +1,10 @@ +rsync://rpki.afrinic.net/repository/AfriNIC.cer +https://rpki.afrinic.net/repository/AfriNIC.cer + +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxsAqAhWIO+ON2Ef9oRDM +pKxv+AfmSLIdLWJtjrvUyDxJPBjgR+kVrOHUeTaujygFUp49tuN5H2C1rUuQavTH +vve6xNF5fU3OkTcqEzMOZy+ctkbde2SRMVdvbO22+TH9gNhKDc9l7Vu01qU4LeJH +k3X0f5uu5346YrGAOSv6AaYBXVgXxa0s9ZvgqFpim50pReQe/WI3QwFKNgpPzfQL +6Y7fDPYdYaVOXPXSKtx7P4s4KLA/ZWmRL/bobw/i2fFviAGhDrjqqqum+/9w1hEl +L/vqihVnV18saKTnLvkItA/Bf5i11Yhw2K7qv573YWxyuqCknO/iYLTR1DToBZcZ +UQIDAQAB diff --git a/crates/panda-rpki-validator/tests/fixtures/tal/apnic-rfc7730-https.tal b/crates/panda-rpki-validator/tests/fixtures/tal/apnic-rfc7730-https.tal new file mode 100644 index 0000000..803eb20 --- /dev/null +++ b/crates/panda-rpki-validator/tests/fixtures/tal/apnic-rfc7730-https.tal @@ -0,0 +1,10 @@ +https://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer +rsync://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer + +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx9RWSL61YAAYumEiU8z8 +qH2ETVIL01ilxZlzIL9JYSORMN5Cmtf8V2JblIealSqgOTGjvSjEsiV73s67zYQI +7C/iSOb96uf3/s86NqbxDiFQGN8qG7RNcdgVuUlAidl8WxvLNI8VhqbAB5uSg/Mr +LeSOvXRja041VptAxIhcGzDMvlAJRwkrYK/Mo8P4E2rSQgwqCgae0ebY1CsJ3Cjf +i67C1nw7oXqJJovvXJ4apGmEv8az23OLC6Ki54Ul/E6xk227BFttqFV3YMtKx42H +cCcDVZZy01n7JjzvO8ccaXmHIgR7utnqhBRNNq5Xc5ZhbkrUsNtiJmrZzVlgU6Ou +0wIDAQAB diff --git a/crates/panda-rpki-validator/tests/fixtures/tal/arin.tal b/crates/panda-rpki-validator/tests/fixtures/tal/arin.tal new file mode 100644 index 0000000..559099b --- /dev/null +++ b/crates/panda-rpki-validator/tests/fixtures/tal/arin.tal @@ -0,0 +1,19 @@ +# THIS TRUST ANCHOR LOCATOR IS PROVIDED BY THE AMERICAN REGISTRY FOR +# INTERNET NUMBERS (ARIN) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL ARIN BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS PUBLIC KEY, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +rsync://rpki.arin.net/repository/arin-rpki-ta.cer +https://rrdp.arin.net/arin-rpki-ta.cer + +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3lZPjbHvMRV5sDDqfLc/685th5FnreHMJjg8 +pEZUbG8Y8TQxSBsDebbsDpl3Ov3Cj1WtdrJ3CIfQODCPrrJdOBSrMATeUbPC+JlNf2SRP3UB+VJFgtTj +0RN8cEYIuhBW5t6AxQbHhdNQH+A1F/OJdw0q9da2U29Lx85nfFxvnC1EpK9CbLJS4m37+RlpNbT1cba+ +b+loXpx0Qcb1C4UpJCGDy7uNf5w6/+l7RpATAHqqsX4qCtwwDYlbHzp2xk9owF3mkCxzl0HwncO+sEHH +eaL3OjtwdIGrRGeHi2Mpt+mvWHhtQqVG+51MHTyg+nIjWFKKGx1Q9+KDx4wJStwveQIDAQAB diff --git a/crates/panda-rpki-validator/tests/fixtures/tal/lacnic.tal b/crates/panda-rpki-validator/tests/fixtures/tal/lacnic.tal new file mode 100644 index 0000000..f81af21 --- /dev/null +++ b/crates/panda-rpki-validator/tests/fixtures/tal/lacnic.tal @@ -0,0 +1,4 @@ +https://rrdp.lacnic.net/ta/rta-lacnic-rpki.cer +rsync://repository.lacnic.net/rpki/lacnic/rta-lacnic-rpki.cer + +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqZEzhYK0+PtDOPfub/KRc3MeWx3neXx4/wbnJWGbNAtbYqXg3uU5J4HFzPgk/VIppgSKAhlO0H60DRP48by9gr5/yDHu2KXhOmnMg46sYsUIpfgtBS9+VtrqWziJfb+pkGtuOWeTnj6zBmBNZKK+5AlMCW1WPhrylIcB+XSZx8tk9GS/3SMQ+YfMVwwAyYjsex14Uzto4GjONALE5oh1M3+glRQduD6vzSwOD+WahMbc9vCOTED+2McLHRKgNaQf0YJ9a1jG9oJIvDkKXEqdfqDRktwyoD74cV57bW3tBAexB7GglITbInyQAsmdngtfg2LUMrcROHHP86QPZINjDQIDAQAB \ No newline at end of file diff --git a/crates/panda-rpki-validator/tests/fixtures/tal/ripe-ncc.tal b/crates/panda-rpki-validator/tests/fixtures/tal/ripe-ncc.tal new file mode 100644 index 0000000..80002d9 --- /dev/null +++ b/crates/panda-rpki-validator/tests/fixtures/tal/ripe-ncc.tal @@ -0,0 +1,10 @@ +https://rpki.ripe.net/ta/ripe-ncc-ta.cer +rsync://rpki.ripe.net/ta/ripe-ncc-ta.cer + +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0URYSGqUz2myBsOzeW1j +Q6NsxNvlLMyhWknvnl8NiBCs/T/S2XuNKQNZ+wBZxIgPPV2pFBFeQAvoH/WK83Hw +A26V2siwm/MY2nKZ+Olw+wlpzlZ1p3Ipj2eNcKrmit8BwBC8xImzuCGaV0jkRB0G +Z0hoH6Ml03umLprRsn6v0xOP0+l6Qc1ZHMFVFb385IQ7FQQTcVIxrdeMsoyJq9eM +kE6DoclHhF/NlSllXubASQ9KUWqJ0+Ot3QCXr4LXECMfkpkVR2TZT+v5v658bHVs +6ZxRD1b6Uk1uQKAyHUbn/tXvP8lrjAibGzVsXDT2L0x4Edx+QdixPgOji3gBMyL2 +VwIDAQAB diff --git a/deploy/docker/.env.example b/deploy/docker/.env.example new file mode 100644 index 0000000..091f5c0 --- /dev/null +++ b/deploy/docker/.env.example @@ -0,0 +1,35 @@ +# Staging/runtime configuration. Copy to .env; never commit the copied file. +VALIDATOR_IMAGE=panda-rpki-validator:0.1.0-dirty-amd64 +VALIDATOR_PLATFORM=linux/amd64 +COMPOSE_PROJECT_NAME=panda-rpki-validator +HOST_DATA_DIR=/var/lib/panda-rpki-validator +RESTART_POLICY=no +VALIDATOR_COMMAND=run + +# Normal sync/validation contract (the defaults match the existing runtime). +MAX_RUNS=3 +INTERVAL_SECS=0 +RIRS=afrinic,apnic,arin,lacnic,ripe +TAL_INPUT_MODE=file-with-ta +RESOURCE_VALIDATION_MODE=validation-update-03 +RETAIN_RUNS=10 +OUTPUT_COMPACT_REPORT=1 +ALLOW_RSYNC_MIRROR_REUSE=1 +RSYNC_SCOPE=module-root +FAILURE_SNAPSHOT_RESET=1 +PERIODIC_SNAPSHOT_RESET=0 +PERIODIC_SNAPSHOT_MAX_DELTAS=100 +DB_STATS_EXACT_EVERY=3 +RPKI_PROGRESS_LOG=1 +ENABLE_CHILD_CERTIFICATE_VALIDATION_CACHE=0 +RPKI_ANALYZE=0 +RPKI_EXTRA_ARGS= + +# For a private/custom TAL, mount a directory at the package's +# /opt/panda-rpki-validator/custom-fixtures path and set these values in the +# compose environment; do not commit the mounted files. +# RIRS=custom +# TAL_INPUT_MODE=custom-file-with-ta +# CUSTOM_TAL_PATH=/opt/panda-rpki-validator/custom-fixtures/tal/custom.tal +# CUSTOM_TA_PATH=/opt/panda-rpki-validator/custom-fixtures/ta/custom-ta.cer +# CUSTOM_TAL_URI=rsync://example.invalid/module/ta.cer diff --git a/deploy/docker/compose.yml b/deploy/docker/compose.yml new file mode 100644 index 0000000..7a437f0 --- /dev/null +++ b/deploy/docker/compose.yml @@ -0,0 +1,37 @@ +services: + validator: + image: ${VALIDATOR_IMAGE:?VALIDATOR_IMAGE is required} + platform: ${VALIDATOR_PLATFORM:?VALIDATOR_PLATFORM is required} + container_name: ${COMPOSE_PROJECT_NAME:-panda-rpki-validator}-validator + environment: + PACKAGE_ROOT: /opt/panda-rpki-validator + RUN_ROOT: /var/lib/panda-rpki-validator + BIN_DIR: /opt/panda-rpki-validator/bin + FIXTURE_DIR: /opt/panda-rpki-validator/fixtures + MAX_RUNS: ${MAX_RUNS:-3} + INTERVAL_SECS: ${INTERVAL_SECS:-0} + RIRS: ${RIRS:-afrinic,apnic,arin,lacnic,ripe} + TAL_INPUT_MODE: ${TAL_INPUT_MODE:-file-with-ta} + RESOURCE_VALIDATION_MODE: ${RESOURCE_VALIDATION_MODE:-validation-update-03} + RETAIN_RUNS: ${RETAIN_RUNS:-10} + OUTPUT_COMPACT_REPORT: ${OUTPUT_COMPACT_REPORT:-1} + ALLOW_RSYNC_MIRROR_REUSE: ${ALLOW_RSYNC_MIRROR_REUSE:-1} + RSYNC_SCOPE: ${RSYNC_SCOPE:-module-root} + FAILURE_SNAPSHOT_RESET: ${FAILURE_SNAPSHOT_RESET:-1} + PERIODIC_SNAPSHOT_RESET: ${PERIODIC_SNAPSHOT_RESET:-0} + PERIODIC_SNAPSHOT_MAX_DELTAS: ${PERIODIC_SNAPSHOT_MAX_DELTAS:-100} + DB_STATS_EXACT_EVERY: ${DB_STATS_EXACT_EVERY:-3} + RPKI_PROGRESS_LOG: ${RPKI_PROGRESS_LOG:-1} + ENABLE_CHILD_CERTIFICATE_VALIDATION_CACHE: ${ENABLE_CHILD_CERTIFICATE_VALIDATION_CACHE:-0} + RPKI_ANALYZE: ${RPKI_ANALYZE:-0} + RPKI_EXTRA_ARGS: ${RPKI_EXTRA_ARGS:-} + CUSTOM_TAL_PATH: ${CUSTOM_TAL_PATH:-/opt/panda-rpki-validator/custom-fixtures/tal/custom.tal} + CUSTOM_TA_PATH: ${CUSTOM_TA_PATH:-/opt/panda-rpki-validator/custom-fixtures/ta/custom-ta.cer} + CUSTOM_TAL_URI: ${CUSTOM_TAL_URI:-} + volumes: + - ${HOST_DATA_DIR:-/var/lib/panda-rpki-validator}/state:/var/lib/panda-rpki-validator/state + - ${HOST_DATA_DIR:-/var/lib/panda-rpki-validator}/runs:/var/lib/panda-rpki-validator/runs + - ${HOST_DATA_DIR:-/var/lib/panda-rpki-validator}/logs:/var/lib/panda-rpki-validator/logs + - ${HOST_DATA_DIR:-/var/lib/panda-rpki-validator}/tmp:/var/lib/panda-rpki-validator/tmp + restart: ${RESTART_POLICY:-no} + command: ["${VALIDATOR_COMMAND:-run}"] diff --git a/deploy/docker/scripts/start.sh b/deploy/docker/scripts/start.sh new file mode 100755 index 0000000..b952d4d --- /dev/null +++ b/deploy/docker/scripts/start.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +COMPOSE_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +cd "$COMPOSE_ROOT" + +[[ -f .env ]] || { echo "missing $COMPOSE_ROOT/.env; copy .env.example first" >&2; exit 2; } +set -a +# shellcheck disable=SC1091 +source .env +set +a +mkdir -p "${HOST_DATA_DIR:-/var/lib/panda-rpki-validator}"/{state,runs,logs,tmp} +docker compose -f compose.yml up -d diff --git a/deploy/docker/scripts/status.sh b/deploy/docker/scripts/status.sh new file mode 100755 index 0000000..ad788e9 --- /dev/null +++ b/deploy/docker/scripts/status.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +COMPOSE_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +cd "$COMPOSE_ROOT" +docker compose -f compose.yml ps diff --git a/deploy/docker/scripts/stop.sh b/deploy/docker/scripts/stop.sh new file mode 100755 index 0000000..6d5ee73 --- /dev/null +++ b/deploy/docker/scripts/stop.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +COMPOSE_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +cd "$COMPOSE_ROOT" +docker compose -f compose.yml down diff --git a/docker/base-images.toml b/docker/base-images.toml new file mode 100644 index 0000000..25424e8 --- /dev/null +++ b/docker/base-images.toml @@ -0,0 +1,8 @@ +# M4 staging build inputs. Replace tags with immutable digest references before M7. +# The digest and verification date are intentionally empty until the release +# registry/base-image policy is approved. +builder_image = "rust:1-bookworm" +runtime_image = "debian:bookworm-slim" +builder_digest = "" +runtime_digest = "" +status = "pending-release-pinning" diff --git a/docker/validator-runtime.Dockerfile b/docker/validator-runtime.Dockerfile new file mode 100644 index 0000000..d3fbe80 --- /dev/null +++ b/docker/validator-runtime.Dockerfile @@ -0,0 +1,147 @@ +# syntax=docker/dockerfile:1.7 + +# Staging defaults. M7 release builds must replace these tags with pinned +# immutable references recorded in docker/base-images.toml. +ARG BUILDER_IMAGE=rust:1-bookworm +ARG RUNTIME_IMAGE=debian:bookworm-slim + +FROM --platform=$BUILDPLATFORM ${BUILDER_IMAGE} AS builder + +ARG BUILDARCH +ARG TARGETARCH +ARG TARGETPLATFORM + +WORKDIR /src + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + clang \ + cmake \ + git \ + libclang-dev \ + make \ + perl \ + pkg-config \ + python3 \ + && if [ "$TARGETARCH" != "$BUILDARCH" ]; then \ + case "$TARGETARCH" in \ + arm64) \ + apt-get install -y --no-install-recommends \ + g++-aarch64-linux-gnu \ + gcc-aarch64-linux-gnu \ + libc6-dev-arm64-cross \ + ;; \ + amd64) \ + apt-get install -y --no-install-recommends \ + g++-x86-64-linux-gnu \ + gcc-x86-64-linux-gnu \ + libc6-dev-amd64-cross \ + ;; \ + *) \ + echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 2 \ + ;; \ + esac; \ + fi \ + && rm -rf /var/lib/apt/lists/* + +COPY Cargo.toml Cargo.lock rust-toolchain.toml ./ +COPY crates/panda-rpki-validator/Cargo.toml crates/panda-rpki-validator/Cargo.toml +COPY crates/panda-rpki-validator/src crates/panda-rpki-validator/src + +RUN set -eux; \ + case "$TARGETARCH" in \ + amd64) target_triple=x86_64-unknown-linux-gnu ;; \ + arm64) target_triple=aarch64-unknown-linux-gnu ;; \ + *) echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 2 ;; \ + esac; \ + rustup target add "$target_triple"; \ + if [ "$TARGETARCH" != "$BUILDARCH" ]; then \ + case "$TARGETARCH" in \ + arm64) \ + export \ + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc \ + CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc \ + CXX_aarch64_unknown_linux_gnu=aarch64-linux-gnu-g++ \ + AR_aarch64_unknown_linux_gnu=aarch64-linux-gnu-ar \ + PKG_CONFIG_ALLOW_CROSS=1 \ + ;; \ + amd64) \ + export \ + CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc \ + CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc \ + CXX_x86_64_unknown_linux_gnu=x86_64-linux-gnu-g++ \ + AR_x86_64_unknown_linux_gnu=x86_64-linux-gnu-ar \ + PKG_CONFIG_ALLOW_CROSS=1 \ + ;; \ + *) \ + echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 2 \ + ;; \ + esac; \ + fi; \ + export CARGO_NET_RETRY=10 CARGO_HTTP_TIMEOUT=120 CARGO_HTTP_MULTIPLEXING=false; \ + cargo build --locked --release --target "$target_triple" \ + -p panda-rpki-validator --bins; \ + mkdir -p /build-out/bin; \ + cp \ + "target/$target_triple/release/panda-rpki-validator" \ + "target/$target_triple/release/panda-rpki-validator-daemon" \ + /build-out/bin/ + +FROM --platform=$TARGETPLATFORM ${RUNTIME_IMAGE} AS runtime + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + bash \ + ca-certificates \ + coreutils \ + curl \ + findutils \ + jq \ + python3 \ + procps \ + rsync \ + time \ + util-linux \ + tzdata \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /opt/panda-rpki-validator + +COPY --from=builder /build-out/bin/panda-rpki-validator /opt/panda-rpki-validator/bin/panda-rpki-validator +COPY --from=builder /build-out/bin/panda-rpki-validator-daemon /opt/panda-rpki-validator/bin/panda-rpki-validator-daemon +COPY scripts/runtime/entrypoint.sh /opt/panda-rpki-validator/entrypoint.sh +COPY scripts/runtime/run_validator.sh /opt/panda-rpki-validator/run_validator.sh +COPY fixtures/ /opt/panda-rpki-validator/fixtures/ + +ARG SOURCE_COMMIT=unknown +ARG SOURCE_DIRTY=unknown +ARG BUILD_TIMESTAMP_UTC=unknown + +LABEL org.opencontainers.image.title="panda-rpki-validator" \ + org.opencontainers.image.description="Panda RPKI synchronization and validation runtime" \ + org.opencontainers.image.revision="${SOURCE_COMMIT}" \ + org.opencontainers.image.created="${BUILD_TIMESTAMP_UTC}" \ + org.opencontainers.image.source-dirty="${SOURCE_DIRTY}" \ + org.opencontainers.image.vendor="panda-rpki" + +RUN chmod +x /opt/panda-rpki-validator/bin/panda-rpki-validator \ + /opt/panda-rpki-validator/bin/panda-rpki-validator-daemon \ + /opt/panda-rpki-validator/run_validator.sh \ + /opt/panda-rpki-validator/entrypoint.sh \ + && mkdir -p /var/lib/panda-rpki-validator/state \ + /var/lib/panda-rpki-validator/runs \ + /var/lib/panda-rpki-validator/logs \ + /var/lib/panda-rpki-validator/tmp + +ENV PACKAGE_ROOT=/opt/panda-rpki-validator \ + RUN_ROOT=/var/lib/panda-rpki-validator \ + BIN_DIR=/opt/panda-rpki-validator/bin \ + FIXTURE_DIR=/opt/panda-rpki-validator/fixtures \ + RUST_BACKTRACE=1 + +VOLUME ["/var/lib/panda-rpki-validator"] + +ENTRYPOINT ["/opt/panda-rpki-validator/entrypoint.sh"] +CMD ["--help"] diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..27406bc --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,21 @@ +# panda-rpki-validator M5 architecture + +The first public package is intentionally a single Cargo package. Its internal +boundaries are: + +1. `model-crypto`: TAL/TA, ASN.1/RPKI objects, signatures and resources. +2. `transport`: HTTP, RRDP, rsync and offline replay inputs. +3. `sync-validation-engine`: snapshot/delta orchestration, CA-tree validation, + policy/TA constraints and parallel scheduling. +4. `state-cache`: repository bytes, RocksDB state and cache lifecycle. +5. `runtime-lifecycle`: run numbering, retention, TA refresh, failure reset and + artifact materialization for native and Docker execution. + +M4 contains the extracted normal synchronization and validation implementation +across all five boundaries, plus the daemon and shell lifecycle wrapper used +by the Docker image. Metrics, query, UI, RTR, Prometheus/Grafana, old installer +control plane, remote scripts and `verification-only` are not dependencies of +this package. M5 has established the first canonical normal-run and +multi-architecture baseline; the remaining profile/performance gates against +the old runtime must pass before this repository can be considered +release-ready. diff --git a/docs/output-abi.md b/docs/output-abi.md new file mode 100644 index 0000000..29968f1 --- /dev/null +++ b/docs/output-abi.md @@ -0,0 +1,61 @@ +# Normal run output ABI + +This document defines the compatibility surface between the existing `rpki` +runtime and `panda-rpki-validator`. It covers normal snapshot and delta runs; +`verification-only` is deliberately outside this component. + +## Persistent volume + +The image exposes one data root, `/var/lib/panda-rpki-validator` by default. +The root may be changed by deployment configuration, but the relative layout +must remain: + +```text +/ +├── state/ +│ ├── db/ # work DB and repo-bytes state +│ ├── rsync-mirror/ # reusable rsync mirror when enabled +│ └── ... # lifecycle, TA refresh and failure-isolation state +├── runs/run_XXXX/ # retained normal run directories +├── logs/ +└── tmp/ +``` + +The component and the old runtime must never share a state root during A/B +comparison. + +## Required normal-run files + +Every successful snapshot/delta run must produce the following files under its +run directory: + +| File | Contract | +|---|---| +| `run-meta.json` | run id/sequence, status, sync mode, timing and lifecycle metadata | +| `run-summary.json` | status, counts, artifact index, stage timing and exit information | +| `daemon-status.json` | current runner status and last-run information | +| `report.json` | validation report and object/publication-point summaries | +| `input.cir` | canonical synchronized validation input | +| `result.ccr` | canonical validated state/cache output | +| `vrps.csv` | VRP payload with the existing header, order and canonical sorting | +| `vaps.csv` | VAP payload with the existing header, order and canonical sorting | +| `validation-contract.json` | effective validation configuration and binding | +| `stage-timing.json` | stage-level timing used by diagnostics and performance gates | +| `stdout.log` / `stderr.log` | child process logs | +| `process-time.txt` | resource/time measurement when enabled | + +`db-stats-estimate.txt` and lifecycle helper files remain part of the diagnostic +surface whenever the corresponding feature is enabled. Missing required files, +changed schema, changed CSV columns/order, or an undeclared semantic difference +fails compatibility even if a VRP count is produced. + +## Canonical comparison + +The compatibility harness compares exit code, sync mode, publication-point +outcome, accepted/rejected objects, VRP/VAP, report, CIR/CCR and state/cache +digests. It normalizes only lifecycle wall-clock timestamps, CCR `producedAt`, +absolute paths, fields explicitly marked as component/image version or +provenance, and runtime duration/resource telemetry (including digests derived +from timestamped audit events). The validation time is a fixed input and is +compared exactly. Every normalization is recorded in the baseline manifest; +output differences outside that list are failures. diff --git a/fixtures/manifest.toml b/fixtures/manifest.toml new file mode 100644 index 0000000..2c9fe26 --- /dev/null +++ b/fixtures/manifest.toml @@ -0,0 +1,10 @@ +schema_version = 1 +policy = "synthetic-or-explicitly-licensed-only" +redistribution_status = "pending-license-and-provenance-review" +contains_private_data = false +contains_live_rir_archive = false +verification_only = "not-included" +source_baseline = "provenance/source-baseline.toml" +expected_profiles = ["snapshot", "delta-warm", "rrdp-fallback-rsync", "payload-replay"] +runtime_fixture_policy = "image ships TAL/TA only; repository fixtures are explicit staging bind mounts" +native_fixture_path = "crates/panda-rpki-validator/tests/fixtures" diff --git a/fixtures/minimal/README.md b/fixtures/minimal/README.md new file mode 100644 index 0000000..8bac42f --- /dev/null +++ b/fixtures/minimal/README.md @@ -0,0 +1,11 @@ +# Minimal fixture policy + +This directory reserves the deterministic, locally generated RPKI fixture +contract. It intentionally contains no live RIR archive, private TAL/TA, key, +customer data, or source-repository bytes. The M4 Docker smoke uses standard +TAL/TA files from `fixtures/` and an explicit bind-mounted repository from the +native test fixture tree; that repository is not shipped in the image while +its redistribution and provenance are pending. M5 must add or approve the +smallest redistributable snapshot/delta fixture and record its generator, +hashes and license in `fixtures/manifest.toml` before using it as public +compatibility evidence. diff --git a/fixtures/ta/afrinic-ta.cer b/fixtures/ta/afrinic-ta.cer new file mode 100644 index 0000000..734586b Binary files /dev/null and b/fixtures/ta/afrinic-ta.cer differ diff --git a/fixtures/ta/apnic-ta.cer b/fixtures/ta/apnic-ta.cer new file mode 100644 index 0000000..83f98ce Binary files /dev/null and b/fixtures/ta/apnic-ta.cer differ diff --git a/fixtures/ta/arin-ta.cer b/fixtures/ta/arin-ta.cer new file mode 100644 index 0000000..573af6b Binary files /dev/null and b/fixtures/ta/arin-ta.cer differ diff --git a/fixtures/ta/lacnic-ta.cer b/fixtures/ta/lacnic-ta.cer new file mode 100644 index 0000000..d86e36f Binary files /dev/null and b/fixtures/ta/lacnic-ta.cer differ diff --git a/fixtures/ta/ripe-ncc-ta.cer b/fixtures/ta/ripe-ncc-ta.cer new file mode 100644 index 0000000..1364de7 Binary files /dev/null and b/fixtures/ta/ripe-ncc-ta.cer differ diff --git a/fixtures/tal/afrinic.tal b/fixtures/tal/afrinic.tal new file mode 100644 index 0000000..5ff40bf --- /dev/null +++ b/fixtures/tal/afrinic.tal @@ -0,0 +1,10 @@ +rsync://rpki.afrinic.net/repository/AfriNIC.cer +https://rpki.afrinic.net/repository/AfriNIC.cer + +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxsAqAhWIO+ON2Ef9oRDM +pKxv+AfmSLIdLWJtjrvUyDxJPBjgR+kVrOHUeTaujygFUp49tuN5H2C1rUuQavTH +vve6xNF5fU3OkTcqEzMOZy+ctkbde2SRMVdvbO22+TH9gNhKDc9l7Vu01qU4LeJH +k3X0f5uu5346YrGAOSv6AaYBXVgXxa0s9ZvgqFpim50pReQe/WI3QwFKNgpPzfQL +6Y7fDPYdYaVOXPXSKtx7P4s4KLA/ZWmRL/bobw/i2fFviAGhDrjqqqum+/9w1hEl +L/vqihVnV18saKTnLvkItA/Bf5i11Yhw2K7qv573YWxyuqCknO/iYLTR1DToBZcZ +UQIDAQAB diff --git a/fixtures/tal/apnic-rfc7730-https.tal b/fixtures/tal/apnic-rfc7730-https.tal new file mode 100644 index 0000000..803eb20 --- /dev/null +++ b/fixtures/tal/apnic-rfc7730-https.tal @@ -0,0 +1,10 @@ +https://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer +rsync://rpki.apnic.net/repository/apnic-rpki-root-iana-origin.cer + +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAx9RWSL61YAAYumEiU8z8 +qH2ETVIL01ilxZlzIL9JYSORMN5Cmtf8V2JblIealSqgOTGjvSjEsiV73s67zYQI +7C/iSOb96uf3/s86NqbxDiFQGN8qG7RNcdgVuUlAidl8WxvLNI8VhqbAB5uSg/Mr +LeSOvXRja041VptAxIhcGzDMvlAJRwkrYK/Mo8P4E2rSQgwqCgae0ebY1CsJ3Cjf +i67C1nw7oXqJJovvXJ4apGmEv8az23OLC6Ki54Ul/E6xk227BFttqFV3YMtKx42H +cCcDVZZy01n7JjzvO8ccaXmHIgR7utnqhBRNNq5Xc5ZhbkrUsNtiJmrZzVlgU6Ou +0wIDAQAB diff --git a/fixtures/tal/arin.tal b/fixtures/tal/arin.tal new file mode 100644 index 0000000..559099b --- /dev/null +++ b/fixtures/tal/arin.tal @@ -0,0 +1,19 @@ +# THIS TRUST ANCHOR LOCATOR IS PROVIDED BY THE AMERICAN REGISTRY FOR +# INTERNET NUMBERS (ARIN) "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL ARIN BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS PUBLIC KEY, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +rsync://rpki.arin.net/repository/arin-rpki-ta.cer +https://rrdp.arin.net/arin-rpki-ta.cer + +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3lZPjbHvMRV5sDDqfLc/685th5FnreHMJjg8 +pEZUbG8Y8TQxSBsDebbsDpl3Ov3Cj1WtdrJ3CIfQODCPrrJdOBSrMATeUbPC+JlNf2SRP3UB+VJFgtTj +0RN8cEYIuhBW5t6AxQbHhdNQH+A1F/OJdw0q9da2U29Lx85nfFxvnC1EpK9CbLJS4m37+RlpNbT1cba+ +b+loXpx0Qcb1C4UpJCGDy7uNf5w6/+l7RpATAHqqsX4qCtwwDYlbHzp2xk9owF3mkCxzl0HwncO+sEHH +eaL3OjtwdIGrRGeHi2Mpt+mvWHhtQqVG+51MHTyg+nIjWFKKGx1Q9+KDx4wJStwveQIDAQAB diff --git a/fixtures/tal/lacnic.tal b/fixtures/tal/lacnic.tal new file mode 100644 index 0000000..f81af21 --- /dev/null +++ b/fixtures/tal/lacnic.tal @@ -0,0 +1,4 @@ +https://rrdp.lacnic.net/ta/rta-lacnic-rpki.cer +rsync://repository.lacnic.net/rpki/lacnic/rta-lacnic-rpki.cer + +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqZEzhYK0+PtDOPfub/KRc3MeWx3neXx4/wbnJWGbNAtbYqXg3uU5J4HFzPgk/VIppgSKAhlO0H60DRP48by9gr5/yDHu2KXhOmnMg46sYsUIpfgtBS9+VtrqWziJfb+pkGtuOWeTnj6zBmBNZKK+5AlMCW1WPhrylIcB+XSZx8tk9GS/3SMQ+YfMVwwAyYjsex14Uzto4GjONALE5oh1M3+glRQduD6vzSwOD+WahMbc9vCOTED+2McLHRKgNaQf0YJ9a1jG9oJIvDkKXEqdfqDRktwyoD74cV57bW3tBAexB7GglITbInyQAsmdngtfg2LUMrcROHHP86QPZINjDQIDAQAB \ No newline at end of file diff --git a/fixtures/tal/ripe-ncc.tal b/fixtures/tal/ripe-ncc.tal new file mode 100644 index 0000000..80002d9 --- /dev/null +++ b/fixtures/tal/ripe-ncc.tal @@ -0,0 +1,10 @@ +https://rpki.ripe.net/ta/ripe-ncc-ta.cer +rsync://rpki.ripe.net/ta/ripe-ncc-ta.cer + +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0URYSGqUz2myBsOzeW1j +Q6NsxNvlLMyhWknvnl8NiBCs/T/S2XuNKQNZ+wBZxIgPPV2pFBFeQAvoH/WK83Hw +A26V2siwm/MY2nKZ+Olw+wlpzlZ1p3Ipj2eNcKrmit8BwBC8xImzuCGaV0jkRB0G +Z0hoH6Ml03umLprRsn6v0xOP0+l6Qc1ZHMFVFb385IQ7FQQTcVIxrdeMsoyJq9eM +kE6DoclHhF/NlSllXubASQ9KUWqJ0+Ot3QCXr4LXECMfkpkVR2TZT+v5v658bHVs +6ZxRD1b6Uk1uQKAyHUbn/tXvP8lrjAibGzVsXDT2L0x4Edx+QdixPgOji3gBMyL2 +VwIDAQAB diff --git a/migration/allowlist.toml b/migration/allowlist.toml new file mode 100644 index 0000000..b036e1a --- /dev/null +++ b/migration/allowlist.toml @@ -0,0 +1,77 @@ +schema_version = 1 +source_commit = "74cbebbd3334ac0063761c1a97a88ee000cc2a57" +source_history_policy = "do-not-copy" +license_status = "pending" +verification_only = "defer-to-follow-up-backlog" + +[[entry]] +source = "src/data_model/" +destination = "crates/panda-rpki-validator/src/data_model/" +classification = "replace" +reason = "RPKI object and resource model copied without private repository coupling" + +[[entry]] +source = "src/fetch/" +destination = "crates/panda-rpki-validator/src/fetch/" +classification = "replace" +reason = "RRDP/rsync transport and replay input" + +[[entry]] +source = "src/sync/" +destination = "crates/panda-rpki-validator/src/sync/" +classification = "replace" +reason = "snapshot/delta synchronization orchestration" + +[[entry]] +source = "src/validation/" +destination = "crates/panda-rpki-validator/src/validation/" +classification = "replace" +reason = "RPKI path and signed-object validation" + +[[entry]] +source = "src/parallel/" +destination = "crates/panda-rpki-validator/src/parallel/" +classification = "replace" +reason = "sync and validation scheduling" + +[[entry]] +source = "src/storage.rs and src/storage/" +destination = "crates/panda-rpki-validator/src/storage.rs and crates/panda-rpki-validator/src/storage/" +classification = "replace" +reason = "state, repo-bytes and cache backend" + +[[entry]] +source = "src/tools/rpki_daemon.rs and scripts/soak/run_soak.sh" +destination = "crates/panda-rpki-validator/src/daemon.rs and scripts/runtime/run_validator.sh" +classification = "replace" +reason = "preserve normal run lifecycle and artifacts without old daemon/private paths" + +[[entry]] +source = "docker/ours-rp-runtime.Dockerfile and scripts/docker/build_docker_runtime_image.sh" +destination = "docker/validator-runtime.Dockerfile and scripts/docker/build_image.sh" +classification = "replace" +reason = "retain complete runtime/multi-arch delivery with new public boundary" + +[[entry]] +source = "deploy/docker-installer/" +destination = "(none in this component)" +classification = "defer-private" +reason = "multi-service installer, monitor and upgrade control plane" + +[[entry]] +source = "src/verification_only.rs and verification-only scripts" +destination = "(none in this component)" +classification = "defer-private" +reason = "explicitly deferred by product decision" + +[[entry]] +source = "metrics, Prometheus/Grafana, Explorer, query, RTR, remote and experiment control" +destination = "(none in this component)" +classification = "drop" +reason = "not part of the validator runtime closure" + +[[entry]] +source = "(new) tests/compat/verify_run_abi.sh and tests/compat/compare_runs.py" +destination = "tests/compat/" +classification = "replace" +reason = "component-side normal-run ABI and original/component canonical comparison harness" diff --git a/provenance/source-baseline.toml b/provenance/source-baseline.toml new file mode 100644 index 0000000..5afa6ef --- /dev/null +++ b/provenance/source-baseline.toml @@ -0,0 +1,17 @@ +schema_version = 1 +component = "panda-rpki-validator" +staging_repository = "panda-rpki" +staging_remote = "https://git.nasp.fit/yuyr/panda-rpki.git" +staging_visibility = "private" +staging_freeze = "initial-main-freeze-2026-09-01" +source_workspace = "private rpki workspace (redacted)" +source_commit = "74cbebbd3334ac0063761c1a97a88ee000cc2a57" +source_branch = "dev_1.0.0_yuyr_parallel" +source_history_copied = false +source_remote_recorded = false +license_status = "pending" +verification_only_migrated = false +normal_output_abi = "docs/output-abi.md" +allowlist = "migration/allowlist.toml" +status = "m5-compatibility-multiarch-baseline" +compatibility_report = "../specs/develop/20260831/m5_compatibility_multiarch_baseline_milestone_report.md" diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000..fb5d1ee --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,5 @@ +[toolchain] +channel = "1.92.0" +profile = "minimal" +components = ["rustfmt", "clippy"] +targets = ["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu"] diff --git a/scripts/docker/build_image.sh b/scripts/docker/build_image.sh new file mode 100755 index 0000000..98dee0f --- /dev/null +++ b/scripts/docker/build_image.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +DOCKERFILE="${DOCKERFILE:-$REPO_ROOT/docker/validator-runtime.Dockerfile}" +TARGET_ARCH="" +IMAGE_TAG="" +OUT_DIR="${OUT_DIR:-$REPO_ROOT/docker-out}" +ALLOW_DIRTY=0 +SAVE_IMAGE=1 +LOAD_IMAGE=1 +INSTALL_BINFMT=1 +BUILDER_IMAGE="${BUILDER_IMAGE:-rust:1-bookworm}" +RUNTIME_IMAGE="${RUNTIME_IMAGE:-debian:bookworm-slim}" + +usage() { + cat <<'USAGE' +Usage: + scripts/docker/build_image.sh --arch amd64|arm64 [options] + +Options: + --arch Target architecture (required) + --image Image tag (default: panda-rpki-validator:-) + --out-dir Image archive/metadata directory (default: docker-out) + --dockerfile Dockerfile path + --builder-image Builder base image (M4 staging tag; pin before release) + --runtime-image Runtime base image (M4 staging tag; pin before release) + --allow-dirty Permit staging build and mark tag/metadata dirty + --no-binfmt Do not install binfmt/qemu for a cross-architecture build + --no-save Do not write docker save archive + --no-load Build without loading the image into the local daemon + -h, --help Show this help +USAGE +} + +normalize_arch() { + case "$1" in + amd64|x86_64) printf '%s\n' amd64 ;; + arm64|aarch64) printf '%s\n' arm64 ;; + *) return 1 ;; + esac +} + +safe_name() { + printf '%s' "$1" | tr '/:' '--' +} + +while [[ "$#" -gt 0 ]]; do + case "$1" in + --arch) + TARGET_ARCH="$(normalize_arch "$2")" || { echo "unsupported architecture: $2" >&2; exit 2; } + shift 2 + ;; + --image) + IMAGE_TAG="$2" + shift 2 + ;; + --out-dir) + OUT_DIR="$2" + shift 2 + ;; + --dockerfile) + DOCKERFILE="$2" + shift 2 + ;; + --builder-image) + BUILDER_IMAGE="$2" + shift 2 + ;; + --runtime-image) + RUNTIME_IMAGE="$2" + shift 2 + ;; + --allow-dirty) + ALLOW_DIRTY=1 + shift + ;; + --no-binfmt) + INSTALL_BINFMT=0 + shift + ;; + --no-save) + SAVE_IMAGE=0 + shift + ;; + --no-load) + LOAD_IMAGE=0 + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +[[ -n "$TARGET_ARCH" ]] || { echo "--arch is required" >&2; usage >&2; exit 2; } +[[ -f "$DOCKERFILE" ]] || { echo "missing Dockerfile: $DOCKERFILE" >&2; exit 2; } +command -v docker >/dev/null 2>&1 || { echo "missing required command: docker" >&2; exit 2; } + +if SOURCE_COMMIT_FULL="$(git -C "$REPO_ROOT" rev-parse --verify HEAD 2>/dev/null)"; then + : +else + SOURCE_COMMIT_FULL=unknown +fi +if SOURCE_COMMIT_SHORT="$(git -C "$REPO_ROOT" rev-parse --short=8 --verify HEAD 2>/dev/null)"; then + : +else + SOURCE_COMMIT_SHORT=unknown +fi +SOURCE_DIRTY=false +if [[ -n "$(git -C "$REPO_ROOT" status --short 2>/dev/null)" || "$SOURCE_COMMIT_FULL" == unknown ]]; then + SOURCE_DIRTY=true +fi +if [[ "$SOURCE_DIRTY" == true && "$ALLOW_DIRTY" != 1 ]]; then + echo "refusing dirty staging tree; pass --allow-dirty for M4 development" >&2 + exit 2 +fi + +VERSION="$(awk -F'"' '$1 == "version = " { print $2; exit }' "$REPO_ROOT/crates/panda-rpki-validator/Cargo.toml")" +REVISION="$VERSION" +if [[ "$SOURCE_DIRTY" == true ]]; then + REVISION="${REVISION}-dirty" +fi +if [[ -z "$IMAGE_TAG" ]]; then + IMAGE_TAG="panda-rpki-validator:${REVISION}-${TARGET_ARCH}" +fi + +mkdir -p "$OUT_DIR" +TARGET_PLATFORM="linux/$TARGET_ARCH" +HOST_ARCH="" +case "$(uname -m)" in + x86_64|amd64) HOST_ARCH=amd64 ;; + aarch64|arm64) HOST_ARCH=arm64 ;; + *) echo "unsupported host architecture: $(uname -m)" >&2; exit 2 ;; +esac +if [[ "$HOST_ARCH" != "$TARGET_ARCH" && "$INSTALL_BINFMT" == 1 ]]; then + echo "installing binfmt/qemu for $TARGET_ARCH" + docker run --rm --privileged tonistiigi/binfmt --install "$TARGET_ARCH" +fi +METADATA_PATH="$OUT_DIR/$(safe_name "$IMAGE_TAG").build.env" +IMAGE_TAR="$OUT_DIR/$(safe_name "$IMAGE_TAG").tar.gz" + +build_args=( + --platform "$TARGET_PLATFORM" + --file "$DOCKERFILE" + --build-arg "BUILDER_IMAGE=$BUILDER_IMAGE" + --build-arg "RUNTIME_IMAGE=$RUNTIME_IMAGE" + --build-arg "SOURCE_COMMIT=$SOURCE_COMMIT_FULL" + --build-arg "SOURCE_DIRTY=$SOURCE_DIRTY" + --build-arg "BUILD_TIMESTAMP_UTC=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + --tag "$IMAGE_TAG" +) +if [[ "$LOAD_IMAGE" == 1 ]]; then + docker buildx build --load "${build_args[@]}" "$REPO_ROOT" +else + docker buildx build "${build_args[@]}" --output "type=oci,dest=$OUT_DIR/$(safe_name "$IMAGE_TAG").oci.tar" "$REPO_ROOT" +fi + +if [[ "$SAVE_IMAGE" == 1 && "$LOAD_IMAGE" == 1 ]]; then + docker save "$IMAGE_TAG" | gzip -c > "$IMAGE_TAR" +fi + +IMAGE_DIGEST="" +IMAGE_ID="" +if [[ "$LOAD_IMAGE" == 1 ]]; then + IMAGE_DIGEST="$(docker image inspect --format '{{index .RepoDigests 0}}' "$IMAGE_TAG" 2>/dev/null || true)" + IMAGE_ID="$(docker image inspect --format '{{.Id}}' "$IMAGE_TAG")" +fi +BUILD_TIMESTAMP_UTC="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +{ + printf 'image=%s\n' "$IMAGE_TAG" + printf 'platform=%s\n' "$TARGET_PLATFORM" + printf 'arch=%s\n' "$TARGET_ARCH" + printf 'host_arch=%s\n' "$HOST_ARCH" + printf 'binfmt_installed=%s\n' "$([[ "$HOST_ARCH" != "$TARGET_ARCH" && "$INSTALL_BINFMT" == 1 ]] && printf true || printf false)" + printf 'source_commit=%s\n' "$SOURCE_COMMIT_FULL" + printf 'source_commit_short=%s\n' "$SOURCE_COMMIT_SHORT" + printf 'source_dirty=%s\n' "$SOURCE_DIRTY" + printf 'builder_image=%s\n' "$BUILDER_IMAGE" + printf 'runtime_image=%s\n' "$RUNTIME_IMAGE" + printf 'image_digest=%s\n' "$IMAGE_DIGEST" + printf 'image_id=%s\n' "$IMAGE_ID" + printf 'archive=%s\n' "$IMAGE_TAR" + printf 'archive_sha256=%s\n' "$(sha256sum "$IMAGE_TAR" 2>/dev/null | awk '{print $1}' || true)" + printf 'built_at_utc=%s\n' "$BUILD_TIMESTAMP_UTC" +} > "$METADATA_PATH" + +echo "built image=$IMAGE_TAG platform=$TARGET_PLATFORM archive=$IMAGE_TAR" diff --git a/scripts/docker/verify_image.sh b/scripts/docker/verify_image.sh new file mode 100755 index 0000000..283a4a0 --- /dev/null +++ b/scripts/docker/verify_image.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +set -euo pipefail + +IMAGE="" +ARCHIVE="" + +usage() { + cat <<'USAGE' +Usage: + scripts/docker/verify_image.sh --image [--archive ] +USAGE +} + +while [[ "$#" -gt 0 ]]; do + case "$1" in + --image) + IMAGE="$2" + shift 2 + ;; + --archive) + ARCHIVE="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +[[ -n "$IMAGE" ]] || { echo "--image is required" >&2; usage >&2; exit 2; } +command -v docker >/dev/null 2>&1 || { echo "missing required command: docker" >&2; exit 2; } + +expect_status() { + local expected="$1" + shift + local actual=0 + "$@" >/dev/null 2>&1 || actual=$? + if [[ "$actual" -ne "$expected" ]]; then + echo "unexpected exit status: expected=$expected actual=$actual command=$*" >&2 + return 1 + fi +} + +docker image inspect "$IMAGE" >/dev/null +IMAGE_ARCH="$(docker image inspect --format '{{.Architecture}}' "$IMAGE")" +case "$IMAGE_ARCH" in + amd64|arm64) IMAGE_PLATFORM="linux/$IMAGE_ARCH" ;; + *) echo "unsupported image architecture: $IMAGE_ARCH" >&2; exit 2 ;; +esac +expect_status 2 docker run --rm --platform "$IMAGE_PLATFORM" --entrypoint /opt/panda-rpki-validator/bin/panda-rpki-validator "$IMAGE" --help +expect_status 0 docker run --rm --platform "$IMAGE_PLATFORM" --entrypoint /opt/panda-rpki-validator/bin/panda-rpki-validator-daemon "$IMAGE" --help +expect_status 2 docker run --rm --platform "$IMAGE_PLATFORM" --entrypoint /opt/panda-rpki-validator/entrypoint.sh "$IMAGE" --help +expect_status 0 docker run --rm --platform "$IMAGE_PLATFORM" "$IMAGE" run --help +docker run --rm --platform "$IMAGE_PLATFORM" --entrypoint /bin/bash "$IMAGE" -ceu \ + 'test -x /opt/panda-rpki-validator/run_validator.sh; \ + test -s /opt/panda-rpki-validator/fixtures/manifest.toml; \ + test -s /opt/panda-rpki-validator/fixtures/tal/afrinic.tal; \ + test -s /opt/panda-rpki-validator/fixtures/ta/afrinic-ta.cer' + +if [[ -n "$ARCHIVE" ]]; then + [[ -f "$ARCHIVE" ]] || { echo "missing image archive: $ARCHIVE" >&2; exit 2; } + gzip -t "$ARCHIVE" + gunzip -c "$ARCHIVE" | docker load >/dev/null +fi + +echo "image smoke passed: $IMAGE" diff --git a/scripts/runtime/entrypoint.sh b/scripts/runtime/entrypoint.sh new file mode 100755 index 0000000..943b128 --- /dev/null +++ b/scripts/runtime/entrypoint.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +BIN_DIR="${BIN_DIR:-/opt/panda-rpki-validator/bin}" +VALIDATOR_BIN="$BIN_DIR/panda-rpki-validator" + +if [[ ! -x "$VALIDATOR_BIN" ]]; then + echo "panda-rpki-validator: missing executable: $VALIDATOR_BIN" >&2 + exit 127 +fi + +if [[ "${1:-}" == "run" || "${1:-}" == "run-validator" ]]; then + shift + exec "/opt/panda-rpki-validator/run_validator.sh" "$@" +fi + +if [[ "${1:-}" == "daemon" ]]; then + shift + exec "$BIN_DIR/panda-rpki-validator-daemon" "$@" +fi + +if [[ "$#" -eq 0 ]]; then + set -- --help +fi + +exec "$VALIDATOR_BIN" "$@" diff --git a/scripts/runtime/run_validator.sh b/scripts/runtime/run_validator.sh new file mode 100755 index 0000000..7452ef7 --- /dev/null +++ b/scripts/runtime/run_validator.sh @@ -0,0 +1,1621 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PACKAGE_ROOT="${PACKAGE_ROOT:-$SCRIPT_DIR}" +ENV_FILE="${ENV_FILE:-$PACKAGE_ROOT/.env}" + +if [[ -f "$ENV_FILE" ]]; then + # shellcheck disable=SC1090 + source "$ENV_FILE" +fi + +MAX_RUNS="${MAX_RUNS:-3}" +INTERVAL_SECS="${INTERVAL_SECS:-0}" +STOP_AFTER_SECS="${STOP_AFTER_SECS:-0}" +RIRS="${RIRS:-afrinic,apnic,arin,lacnic,ripe}" +TAL_INPUT_MODE="${TAL_INPUT_MODE:-file-with-ta}" +RESOURCE_VALIDATION_MODE="${RESOURCE_VALIDATION_MODE:-validation-update-03}" +RUN_ROOT="${RUN_ROOT:-$PACKAGE_ROOT}" +RETAIN_RUNS="${RETAIN_RUNS:-10}" +CLEAN_TMP_AFTER_RUN="${CLEAN_TMP_AFTER_RUN:-1}" +OUTPUT_COMPACT_REPORT="${OUTPUT_COMPACT_REPORT:-1}" +ALLOW_RSYNC_MIRROR_REUSE="${ALLOW_RSYNC_MIRROR_REUSE:-1}" +RSYNC_SCOPE="${RSYNC_SCOPE:-module-root}" +FAILURE_SNAPSHOT_RESET="${FAILURE_SNAPSHOT_RESET:-1}" +PERIODIC_SNAPSHOT_RESET="${PERIODIC_SNAPSHOT_RESET:-0}" +PERIODIC_SNAPSHOT_MAX_DELTAS="${PERIODIC_SNAPSHOT_MAX_DELTAS:-100}" +DB_STATS_EXACT_EVERY="${DB_STATS_EXACT_EVERY:-3}" +RPKI_PROGRESS_LOG="${RPKI_PROGRESS_LOG:-1}" +RPKI_PROGRESS_SLOW_SECS="${RPKI_PROGRESS_SLOW_SECS:-10}" +RPKI_PROGRESS_STAGE_FRESH_SLOW_MS="${RPKI_PROGRESS_STAGE_FRESH_SLOW_MS:-1000}" +RPKI_PROGRESS_PP_CONTROL_SLOW_MS="${RPKI_PROGRESS_PP_CONTROL_SLOW_MS:-100}" +RPKI_PROGRESS_PP_CACHE_SLOW_MS="${RPKI_PROGRESS_PP_CACHE_SLOW_MS:-50}" +RPKI_PROGRESS_CONTROL_LOOP_SLOW_MS="${RPKI_PROGRESS_CONTROL_LOOP_SLOW_MS:-1000}" +DISABLE_COMPETING_RPS="${DISABLE_COMPETING_RPS:-1}" +ENABLE_CHILD_CERTIFICATE_VALIDATION_CACHE="${ENABLE_CHILD_CERTIFICATE_VALIDATION_CACHE:-0}" +RPKI_EXTRA_ARGS="${RPKI_EXTRA_ARGS:-}" +RPKI_ANALYZE="${RPKI_ANALYZE:-0}" +CUSTOM_TAL_PATH="${CUSTOM_TAL_PATH:-$PACKAGE_ROOT/custom-fixtures/tal/custom.tal}" +CUSTOM_TA_PATH="${CUSTOM_TA_PATH:-$PACKAGE_ROOT/custom-fixtures/ta/custom-ta.cer}" +CUSTOM_TAL_URI="${CUSTOM_TAL_URI:-}" +HTTP_ROOT_CERT_PATHS="${HTTP_ROOT_CERT_PATHS:-}" + +BIN_DIR="${BIN_DIR:-$PACKAGE_ROOT/bin}" +FIXTURE_DIR="${FIXTURE_DIR:-$PACKAGE_ROOT/fixtures}" +STATE_ROOT="$RUN_ROOT/state" +RUNS_ROOT="$RUN_ROOT/runs" +LOG_ROOT="$RUN_ROOT/logs" +DB_DIR="${DB_DIR:-$STATE_ROOT/db}" +META_DIR="${META_DIR:-$STATE_ROOT/meta}" +TMP_DIR="${TMP_DIR:-$RUN_ROOT/tmp}" +RSYNC_MIRROR_ROOT="${RSYNC_MIRROR_ROOT:-$STATE_ROOT/rsync-mirror}" +INVALID_ROOT="$STATE_ROOT/invalid" +RESET_STAGING_ROOT="$STATE_ROOT/reset-staging" +STATE_EXECUTION_LOCK_PATH="$STATE_ROOT/locks/state-execution.lock" +LIVE_TA_REFRESH_DIR="${LIVE_TA_REFRESH_DIR:-$META_DIR/live-ta-refresh}" +LIVE_TA_REFRESH_CONNECT_TIMEOUT_SECS="${LIVE_TA_REFRESH_CONNECT_TIMEOUT_SECS:-15}" +LIVE_TA_REFRESH_MAX_TIME_SECS="${LIVE_TA_REFRESH_MAX_TIME_SECS:-120}" +LIVE_TA_REFRESH_BEFORE_SNAPSHOT="${LIVE_TA_REFRESH_BEFORE_SNAPSHOT:-1}" +RUN_LIFECYCLE_STATE_PATH="$STATE_ROOT/run-lifecycle-state.json" +RUN_LIFECYCLE_RECENT_RUNS_LIMIT=200 + +RPKI_BIN="${RPKI_BIN:-$BIN_DIR/panda-rpki-validator}" +RPKI_DAEMON_BIN="${RPKI_DAEMON_BIN:-$BIN_DIR/panda-rpki-validator-daemon}" +DB_STATS_BIN="${DB_STATS_BIN:-$BIN_DIR/db_stats}" + +usage() { + cat <<'USAGE' +Usage: + ./run_validator.sh + +配置来自 package 根目录下的 .env;也可以用 ENV_FILE=/path/to/.env 覆盖。 +USAGE +} + +die() { + echo "error: $*" >&2 + exit 2 +} + +warn() { + echo "warning: $*" >&2 +} + +is_true() { + case "${1:-}" in + 1|true|TRUE|yes|YES|on|ON) return 0 ;; + *) return 1 ;; + esac +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || die "missing required command: $1" +} + +validate_positive_int() { + local name="$1" + local value="$2" + [[ "$value" =~ ^[0-9]+$ ]] || die "$name must be an integer: $value" + [[ "$value" != "0" ]] || die "$name must be > 0" +} + +validate_non_negative_int() { + local name="$1" + local value="$2" + [[ "$value" =~ ^[0-9]+$ ]] || die "$name must be an integer: $value" +} + +validate_max_runs() { + [[ "$MAX_RUNS" =~ ^-?[0-9]+$ ]] || die "MAX_RUNS must be an integer: $MAX_RUNS" + [[ "$MAX_RUNS" != "0" ]] || die "MAX_RUNS must be non-zero; use a positive value for fixed runs or -1 for continuous mode" +} + +validate_rsync_scope() { + case "$RSYNC_SCOPE" in + host|publication-point|module-root) + ;; + *) + die "RSYNC_SCOPE must be host, publication-point, or module-root: $RSYNC_SCOPE" + ;; + esac +} + +validate_tal_input_mode() { + case "$TAL_INPUT_MODE" in + file-with-ta|file-live-ta|custom-file-with-ta|url) + ;; + *) + die "TAL_INPUT_MODE must be file-with-ta, file-live-ta, custom-file-with-ta or url: $TAL_INPUT_MODE" + ;; + esac +} + +validate_resource_validation_mode() { + case "$RESOURCE_VALIDATION_MODE" in + validation-update-03|rfc6487) + ;; + *) + die "RESOURCE_VALIDATION_MODE must be validation-update-03 or rfc6487: $RESOURCE_VALIDATION_MODE" + ;; + esac +} + +normalize_token() { + local token="$1" + token="${token#"${token%%[![:space:]]*}"}" + token="${token%"${token##*[![:space:]]}"}" + printf '%s' "$token" | tr '[:upper:]' '[:lower:]' +} + +parse_rirs() { + RIR_LIST=() + local raw_token + local normalized + IFS=',' read -r -a raw_rirs <<< "$RIRS" + for raw_token in "${raw_rirs[@]}"; do + normalized="$(normalize_token "$raw_token")" + [[ -n "$normalized" ]] || continue + case "$normalized" in + afrinic|apnic|arin|lacnic|ripe) + RIR_LIST+=("$normalized") + ;; + custom) + [[ "$TAL_INPUT_MODE" == "custom-file-with-ta" ]] \ + || die "RIRS=custom requires TAL_INPUT_MODE=custom-file-with-ta" + RIR_LIST+=("$normalized") + ;; + *) + die "invalid RIRS entry: $raw_token; allowed: afrinic,apnic,arin,lacnic,ripe,custom (custom mode only)" + ;; + esac + done + [[ "${#RIR_LIST[@]}" -gt 0 ]] || die "RIRS must contain at least one RIR" +} + +validate_custom_inputs() { + local root_cert_path + local -a root_cert_paths=() + + if [[ "$TAL_INPUT_MODE" == "custom-file-with-ta" ]]; then + [[ "${#RIR_LIST[@]}" -eq 1 && "${RIR_LIST[0]}" == "custom" ]] \ + || die "custom-file-with-ta requires exactly RIRS=custom" + [[ -s "$CUSTOM_TAL_PATH" ]] \ + || die "missing custom TAL: $CUSTOM_TAL_PATH" + [[ -s "$CUSTOM_TA_PATH" ]] \ + || die "missing custom TA DER: $CUSTOM_TA_PATH" + [[ -n "$CUSTOM_TAL_URI" ]] \ + || die "CUSTOM_TAL_URI is required in custom-file-with-ta mode" + elif [[ "$RIRS" == *custom* ]]; then + die "RIRS=custom requires TAL_INPUT_MODE=custom-file-with-ta" + fi + + if [[ -n "$HTTP_ROOT_CERT_PATHS" ]]; then + # shellcheck disable=SC2206 + root_cert_paths=( $HTTP_ROOT_CERT_PATHS ) + for root_cert_path in "${root_cert_paths[@]}"; do + [[ -s "$root_cert_path" ]] \ + || die "missing HTTP root certificate: $root_cert_path" + done + fi +} + +tal_file_for_rir() { + case "$1" in + afrinic) printf '%s' "$FIXTURE_DIR/tal/afrinic.tal" ;; + apnic) printf '%s' "$FIXTURE_DIR/tal/apnic-rfc7730-https.tal" ;; + arin) printf '%s' "$FIXTURE_DIR/tal/arin.tal" ;; + lacnic) printf '%s' "$FIXTURE_DIR/tal/lacnic.tal" ;; + ripe) printf '%s' "$FIXTURE_DIR/tal/ripe-ncc.tal" ;; + custom) printf '%s' "$CUSTOM_TAL_PATH" ;; + *) die "unknown RIR: $1" ;; + esac +} + +ta_file_for_rir() { + case "$1" in + afrinic) printf '%s' "$FIXTURE_DIR/ta/afrinic-ta.cer" ;; + apnic) printf '%s' "$FIXTURE_DIR/ta/apnic-ta.cer" ;; + arin) printf '%s' "$FIXTURE_DIR/ta/arin-ta.cer" ;; + lacnic) printf '%s' "$FIXTURE_DIR/ta/lacnic-ta.cer" ;; + ripe) printf '%s' "$FIXTURE_DIR/ta/ripe-ncc-ta.cer" ;; + custom) printf '%s' "$CUSTOM_TA_PATH" ;; + *) die "unknown RIR: $1" ;; + esac +} + +tal_url_for_rir() { + case "$1" in + afrinic) printf '%s' "https://rpki.afrinic.net/tal/afrinic.tal" ;; + apnic) printf '%s' "https://tal.apnic.net/apnic.tal" ;; + arin) printf '%s' "https://www.arin.net/resources/manage/rpki/arin.tal" ;; + lacnic) printf '%s' "https://www.lacnic.net/innovaportal/file/4983/1/lacnic.tal" ;; + ripe) printf '%s' "https://tal.rpki.ripe.net/ripe-ncc.tal" ;; + custom) printf '%s' "$CUSTOM_TAL_URI" ;; + *) die "unknown RIR: $1" ;; + esac +} + +cir_tal_uri_for_rir() { + tal_url_for_rir "$1" +} + +tal_https_uri_from_fixture() { + local tal_path="$1" + awk ' + /^[[:space:]]*#/ { next } + /^[[:space:]]*$/ { next } + { + gsub(/^[[:space:]]+|[[:space:]]+$/, "", $0) + } + /^https?:\/\// { + print + exit 0 + } + ' "$tal_path" +} + +live_ta_file_for_rir() { + printf '%s' "$STATE_ROOT/live-ta/$(basename "$(tal_file_for_rir "$1")" .tal).cer" +} + +live_ta_refresh_pid_file_for_rir() { + printf '%s' "$LIVE_TA_REFRESH_DIR/$1.pid" +} + +refresh_live_ta_for_rir() { + local rir_name="$1" + local run_id="${2:-manual}" + local log_path="${3:-}" + local tal_path + local ta_uri + local ta_file + local tmp_file + if [[ -n "$log_path" ]]; then + mkdir -p "$(dirname "$log_path")" + exec >> "$log_path" 2>&1 + fi + echo "live-ta-refresh start run=$run_id rir=$rir_name at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + tal_path="$(tal_file_for_rir "$rir_name")" + ta_uri="$(tal_https_uri_from_fixture "$tal_path")" + if [[ -z "$ta_uri" ]]; then + echo "live-ta-refresh failed rir=$rir_name reason=missing_https_uri tal=$tal_path" + return 1 + fi + ta_file="$(live_ta_file_for_rir "$rir_name")" + mkdir -p "$(dirname "$ta_file")" + tmp_file="${ta_file}.tmp.$$.$RANDOM" + if ! curl -fsSL --connect-timeout "$LIVE_TA_REFRESH_CONNECT_TIMEOUT_SECS" --max-time "$LIVE_TA_REFRESH_MAX_TIME_SECS" "$ta_uri" -o "$tmp_file"; then + rm -f "$tmp_file" + echo "live-ta-refresh failed rir=$rir_name reason=curl uri=$ta_uri" + return 1 + fi + if [[ ! -s "$tmp_file" ]]; then + rm -f "$tmp_file" + echo "live-ta-refresh failed rir=$rir_name reason=empty_download uri=$ta_uri" + return 1 + fi + mv "$tmp_file" "$ta_file" + echo "live-ta-refresh success rir=$rir_name uri=$ta_uri output=$ta_file bytes=$(wc -c < "$ta_file" | tr -d ' ')" +} + +ensure_live_ta_for_rir() { + local rir_name="$1" + local live_ta_file + local fixture_ta_file + live_ta_file="$(live_ta_file_for_rir "$rir_name")" + if [[ -s "$live_ta_file" ]]; then + return 0 + fi + fixture_ta_file="$(ta_file_for_rir "$rir_name")" + [[ -s "$fixture_ta_file" ]] || die "missing live TA and fixture TA for $rir_name: $live_ta_file / $fixture_ta_file" + mkdir -p "$(dirname "$live_ta_file")" + cp "$fixture_ta_file" "$live_ta_file" +} + +reap_finished_live_ta_refresh_for_rir() { + local rir_name="$1" + local pid_file + local pid + local pid_state + local pid_file_mtime + local now_epoch + local stale_after_secs + pid_file="$(live_ta_refresh_pid_file_for_rir "$rir_name")" + [[ -f "$pid_file" ]] || return 0 + pid="$(cat "$pid_file" 2>/dev/null || true)" + if [[ "$pid" =~ ^[0-9]+$ ]] && kill -0 "$pid" >/dev/null 2>&1; then + pid_state="" + if [[ -r "/proc/$pid/stat" ]]; then + pid_state="$(awk '{ print $3 }' "/proc/$pid/stat" 2>/dev/null || true)" + fi + if [[ "$pid_state" == "Z" ]]; then + wait "$pid" >/dev/null 2>&1 || true + rm -f "$pid_file" + return 0 + fi + pid_file_mtime="$(stat -c %Y "$pid_file" 2>/dev/null || date +%s)" + now_epoch="$(date +%s)" + stale_after_secs=$((LIVE_TA_REFRESH_MAX_TIME_SECS + 60)) + if (( now_epoch - pid_file_mtime > stale_after_secs )); then + rm -f "$pid_file" + return 0 + fi + return 1 + fi + if [[ "$pid" =~ ^[0-9]+$ ]]; then + wait "$pid" >/dev/null 2>&1 || true + fi + rm -f "$pid_file" + return 0 +} + +start_live_ta_refresh_for_rir() { + local rir_name="$1" + local run_id="$2" + local pid_file + local log_path + local pid + mkdir -p "$LIVE_TA_REFRESH_DIR" "$LOG_ROOT" + pid_file="$(live_ta_refresh_pid_file_for_rir "$rir_name")" + if ! reap_finished_live_ta_refresh_for_rir "$rir_name"; then + pid="$(cat "$pid_file" 2>/dev/null || true)" + echo "live-ta-refresh skip run=$run_id rir=$rir_name reason=previous_refresh_running pid=$pid" \ + >> "$LOG_ROOT/live-ta-refresh-$run_id-$rir_name.log" + return 0 + fi + log_path="$LOG_ROOT/live-ta-refresh-$run_id-$rir_name.log" + refresh_live_ta_for_rir "$rir_name" "$run_id" "$log_path" & + pid=$! + printf '%s\n' "$pid" > "$pid_file" +} + +wait_for_previous_live_ta_refresh_for_rir() { + local rir_name="$1" + local pid_file + local pid + local deadline_epoch + pid_file="$(live_ta_refresh_pid_file_for_rir "$rir_name")" + [[ -f "$pid_file" ]] || return 0 + if reap_finished_live_ta_refresh_for_rir "$rir_name"; then + return 0 + fi + pid="$(cat "$pid_file" 2>/dev/null || true)" + deadline_epoch=$(( $(date +%s) + LIVE_TA_REFRESH_MAX_TIME_SECS + 60 )) + echo "live-ta-refresh wait rir=$rir_name reason=previous_refresh_running pid=$pid" + while ! reap_finished_live_ta_refresh_for_rir "$rir_name"; do + if (( $(date +%s) > deadline_epoch )); then + die "timed out waiting for previous live TA refresh for $rir_name pid=$pid" + fi + sleep 1 + done +} + +refresh_live_ta_blocking_for_run() { + local run_id="$1" + local rir_name + local pid + local failed=0 + local pids=() + local names=() + local log_path + local pid_file + + for rir_name in "${RIR_LIST[@]}"; do + wait_for_previous_live_ta_refresh_for_rir "$rir_name" + done + + for rir_name in "${RIR_LIST[@]}"; do + mkdir -p "$LIVE_TA_REFRESH_DIR" "$LOG_ROOT" + pid_file="$(live_ta_refresh_pid_file_for_rir "$rir_name")" + log_path="$LOG_ROOT/live-ta-refresh-$run_id-$rir_name.log" + refresh_live_ta_for_rir "$rir_name" "$run_id" "$log_path" & + pid=$! + printf '%s\n' "$pid" > "$pid_file" + pids+=("$pid") + names+=("$rir_name") + done + + local index + for index in "${!pids[@]}"; do + pid="${pids[$index]}" + rir_name="${names[$index]}" + pid_file="$(live_ta_refresh_pid_file_for_rir "$rir_name")" + if wait "$pid"; then + rm -f "$pid_file" + else + failed=1 + rm -f "$pid_file" + echo "live-ta-refresh failed before snapshot rir=$rir_name log=$LOG_ROOT/live-ta-refresh-$run_id-$rir_name.log" >&2 + fi + done + + if (( failed != 0 )); then + die "live TA refresh failed before snapshot run=$run_id; see $LOG_ROOT/live-ta-refresh-$run_id-*.log" + fi + echo "live-ta-refresh completed before snapshot run=$run_id rirs=${#RIR_LIST[@]}" +} + +prepare_live_ta_inputs_for_run() { + local run_id="$1" + local sync_mode="$2" + local rir_name + if [[ "$TAL_INPUT_MODE" != "file-live-ta" ]]; then + return 0 + fi + if [[ "$sync_mode" == "snapshot" ]] && is_true "$LIVE_TA_REFRESH_BEFORE_SNAPSHOT"; then + refresh_live_ta_blocking_for_run "$run_id" + return 0 + fi + for rir_name in "${RIR_LIST[@]}"; do + ensure_live_ta_for_rir "$rir_name" + done + for rir_name in "${RIR_LIST[@]}"; do + start_live_ta_refresh_for_rir "$rir_name" "$run_id" + done +} + +compare_view_trust_anchor() { + if [[ "${#RIR_LIST[@]}" -eq 1 ]]; then + printf '%s' "${RIR_LIST[0]}" + else + printf '%s' "all5" + fi +} + +max_existing_run_index() { + local max_index=0 + local run_dir + local run_name + local numeric_part + shopt -s nullglob + for run_dir in "$RUNS_ROOT"/run_*; do + [[ -d "$run_dir" ]] || continue + run_name="$(basename "$run_dir")" + numeric_part="${run_name#run_}" + [[ "$numeric_part" =~ ^[0-9]+$ ]] || continue + if (( 10#$numeric_part > max_index )); then + max_index=$((10#$numeric_part)) + fi + done + shopt -u nullglob + printf '%s' "$max_index" +} + +json_status_is_success() { + local json_path="$1" + python3 - "$json_path" <<'PY' +import json +import sys +path = sys.argv[1] +try: + with open(path, "r", encoding="utf-8") as handle: + data = json.load(handle) +except Exception: + sys.exit(1) +sys.exit(0 if data.get("status") == "success" else 1) +PY +} + +previous_run_success() { + local run_dir="$1" + [[ -d "$run_dir" ]] || return 1 + [[ -f "$run_dir/run-meta.json" ]] || return 1 + [[ -f "$run_dir/run-summary.json" ]] || return 1 + json_status_is_success "$run_dir/run-meta.json" || return 1 + json_status_is_success "$run_dir/run-summary.json" || return 1 + for required_artifact in report.json result.ccr input.cir stage-timing.json process-time.txt stdout.log stderr.log; do + [[ -f "$run_dir/$required_artifact" ]] || return 1 + done + return 0 +} + +move_if_exists() { + local source_path="$1" + local target_dir="$2" + if [[ -e "$source_path" ]]; then + mkdir -p "$target_dir" + mv "$source_path" "$target_dir/" + fi +} + +clear_dir_contents() { + local dir_path="$1" + [[ -d "$dir_path" ]] || return 0 + find "$dir_path" -mindepth 1 -maxdepth 1 -exec rm -rf {} + +} + +db_state_exists() { + [[ -e "$DB_DIR/work-db" || -e "$DB_DIR/repo-bytes.db" ]] +} + +delta_state_available() { + [[ -e "$DB_DIR/work-db" ]] +} + +isolate_state_after_failure() { + local previous_run_id="$1" + local timestamp + timestamp="$(date -u +%Y%m%dT%H%M%SZ)" + local invalid_dir="$INVALID_ROOT/${previous_run_id}-${timestamp}" + mkdir -p "$invalid_dir" + move_if_exists "$DB_DIR" "$invalid_dir" + move_if_exists "$META_DIR" "$invalid_dir" + if [[ -e "$TMP_DIR" ]]; then + INVALID_TMP_PATH="$TMP_DIR" + TMP_CLEANUP_STATUS="pending" + TMP_CLEANUP_REASON="discard_after_failure" + clear_dir_contents "$TMP_DIR" + mkdir -p "$TMP_DIR" + TMP_CLEANUP_STATUS="discarded_recreated" + else + INVALID_TMP_PATH="" + TMP_CLEANUP_STATUS="not_present" + TMP_CLEANUP_REASON="discard_after_failure" + mkdir -p "$TMP_DIR" + fi + mkdir -p "$DB_DIR" "$META_DIR" "$TMP_DIR" + INVALID_DB_PATH="$invalid_dir/$(basename "$DB_DIR")" + INVALID_STATE_PATH="$invalid_dir/$(basename "$META_DIR")" +} + +periodic_snapshot_delta_scan() { + local command="$1" + shift + python3 - "$command" "$RUN_LIFECYCLE_STATE_PATH" "$RUNS_ROOT" "$RUN_LIFECYCLE_RECENT_RUNS_LIMIT" "$@" <<'PY' +import json +import os +import pathlib +import shlex +import sys +from datetime import datetime, timezone + +command = sys.argv[1] +state_path = pathlib.Path(sys.argv[2]) +runs_root = pathlib.Path(sys.argv[3]) +recent_limit = int(sys.argv[4]) +extra_args = sys.argv[5:] + + +def now_rfc3339() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def to_int(value): + if value is None or value == "": + return None + if isinstance(value, bool): + return None + return int(value) + + +def to_bool(value): + if isinstance(value, bool): + return value + if value in (None, ""): + return None + if isinstance(value, str): + lowered = value.strip().lower() + if lowered in {"1", "true", "yes", "on"}: + return True + if lowered in {"0", "false", "no", "off"}: + return False + raise ValueError(f"invalid bool value: {value!r}") + + +def normalize_run_entry(entry): + if entry is None: + return None + if not isinstance(entry, dict): + raise ValueError("run entry must be an object") + run_id = entry.get("run_id") or entry.get("runId") + run_index = to_int(entry.get("run_index", entry.get("runSeq"))) + if not run_id or run_index is None: + raise ValueError("run entry missing run_id/run_index") + return { + "run_id": run_id, + "run_index": run_index, + "status": entry.get("status") or "unknown", + "sync_mode": entry.get("sync_mode", entry.get("syncMode")), + "snapshot_reason": entry.get("snapshot_reason"), + "started_at_rfc3339_utc": entry.get("started_at_rfc3339_utc"), + "completed_at_rfc3339_utc": entry.get("completed_at_rfc3339_utc"), + "periodic_snapshot_reset_enabled": to_bool(entry.get("periodic_snapshot_reset_enabled")), + "periodic_snapshot_max_deltas": to_int(entry.get("periodic_snapshot_max_deltas")), + "periodic_snapshot_delta_count": to_int(entry.get("periodic_snapshot_delta_count")), + "periodic_snapshot_forced": to_bool(entry.get("periodic_snapshot_forced")), + } + + +def snapshot_ref(entry): + if entry is None: + return None + return { + "run_id": entry["run_id"], + "run_index": entry["run_index"], + "snapshot_reason": entry.get("snapshot_reason"), + "completed_at_rfc3339_utc": entry.get("completed_at_rfc3339_utc"), + } + + +def normalize_snapshot_ref(entry): + if entry is None: + return None + if not isinstance(entry, dict): + raise ValueError("snapshot ref must be an object") + run_id = entry.get("run_id") or entry.get("runId") + run_index = to_int(entry.get("run_index", entry.get("runSeq"))) + if not run_id or run_index is None: + raise ValueError("snapshot ref missing run_id/run_index") + return { + "run_id": run_id, + "run_index": run_index, + "snapshot_reason": entry.get("snapshot_reason"), + "completed_at_rfc3339_utc": entry.get("completed_at_rfc3339_utc"), + } + + +def state_health(last_run, last_success_snapshot): + if last_success_snapshot is not None: + return "ready", "ok" + if last_run is None: + return "empty", "no_runs" + return "bootstrap_incomplete", "no_successful_snapshot_in_retained_runs" + + +def default_state(): + health, detail = state_health(None, None) + return { + "version": 1, + "updated_at_rfc3339_utc": now_rfc3339(), + "state_health": health, + "state_detail": detail, + "recent_runs_limit": recent_limit, + "last_run": None, + "last_success_snapshot": None, + "successful_deltas_since_snapshot": None, + "recent_runs": [], + } + + +def finalize_state(data): + data = dict(data) + data["version"] = 1 + data["updated_at_rfc3339_utc"] = now_rfc3339() + data["recent_runs_limit"] = recent_limit + recent_runs = [] + for item in data.get("recent_runs", []): + recent_runs.append(normalize_run_entry(item)) + if len(recent_runs) > recent_limit: + recent_runs = recent_runs[-recent_limit:] + data["recent_runs"] = recent_runs + data["last_run"] = normalize_run_entry(data.get("last_run")) + data["last_success_snapshot"] = normalize_snapshot_ref(data.get("last_success_snapshot")) + delta_count = data.get("successful_deltas_since_snapshot") + data["successful_deltas_since_snapshot"] = to_int(delta_count) + if data["last_success_snapshot"] is None: + data["successful_deltas_since_snapshot"] = None + elif data["successful_deltas_since_snapshot"] is None: + data["successful_deltas_since_snapshot"] = 0 + health, detail = state_health(data["last_run"], data["last_success_snapshot"]) + if health == "ready": + detail = data.get("state_detail") or detail + data["state_health"] = health + data["state_detail"] = detail + return data + + +def sorted_run_dirs(): + candidates = [] + for path in runs_root.glob("run_*"): + if not path.is_dir(): + continue + suffix = path.name[4:] + if suffix.isdigit(): + candidates.append((int(suffix), path)) + candidates.sort() + return [path for _, path in candidates] + + +def parse_run_dir(run_dir): + meta_path = run_dir / "run-meta.json" + summary_path = run_dir / "run-summary.json" + if not meta_path.exists() or not summary_path.exists(): + return None + with meta_path.open("r", encoding="utf-8") as handle: + meta = json.load(handle) + with summary_path.open("r", encoding="utf-8") as handle: + summary = json.load(handle) + entry = normalize_run_entry(meta) + entry["status"] = meta.get("status") or summary.get("status") or entry["status"] + return entry + + +def bootstrap_state(exclude_run_id=None): + data = default_state() + last_success_snapshot = None + delta_count = None + recent_runs = [] + last_run = None + for run_dir in sorted_run_dirs(): + if exclude_run_id and run_dir.name == exclude_run_id: + continue + try: + entry = parse_run_dir(run_dir) + except Exception: + continue + if entry is None: + continue + last_run = entry + recent_runs.append(entry) + if entry["status"] != "success": + continue + if entry.get("sync_mode") == "snapshot": + last_success_snapshot = snapshot_ref(entry) + delta_count = 0 + elif entry.get("sync_mode") == "delta": + if last_success_snapshot is not None and delta_count is not None: + delta_count += 1 + data["last_run"] = last_run + data["last_success_snapshot"] = last_success_snapshot + data["successful_deltas_since_snapshot"] = delta_count + data["recent_runs"] = recent_runs[-recent_limit:] + return finalize_state(data) + + +def backup_corrupt_state(): + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + backup_path = state_path.with_name(f"{state_path.name}.corrupt.{timestamp}.{os.getpid()}") + backup_path.parent.mkdir(parents=True, exist_ok=True) + os.replace(state_path, backup_path) + return backup_path + + +def load_state_or_bootstrap(exclude_run_id=None): + if not state_path.exists(): + return bootstrap_state(exclude_run_id), "bootstrapped_from_runs_missing_file", "", "" + try: + with state_path.open("r", encoding="utf-8") as handle: + raw = json.load(handle) + if not isinstance(raw, dict): + raise ValueError("state root must be an object") + if to_int(raw.get("version")) != 1: + raise ValueError("unsupported version") + state = finalize_state(raw) + return state, "state_file", "", "" + except Exception as exc: + backup_path = backup_corrupt_state() + state = bootstrap_state(exclude_run_id) + return state, "bootstrapped_from_runs_after_corrupt_backup", exc.__class__.__name__, str(backup_path) + + +def atomic_write_json(path, payload): + path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = path.with_name(f"{path.name}.tmp.{os.getpid()}") + with tmp_path.open("w", encoding="utf-8") as handle: + json.dump(payload, handle, indent=2, sort_keys=True) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp_path, path) + try: + dir_fd = os.open(path.parent, os.O_DIRECTORY) + except OSError: + return + try: + os.fsync(dir_fd) + finally: + os.close(dir_fd) + + +def emit(name, value): + if value is None: + value = "" + elif isinstance(value, bool): + value = "true" if value else "false" + else: + value = str(value) + print(f"{name}={shlex.quote(value)}") + + +if command == "load": + max_deltas = int(extra_args[0]) + state, source, detail, backup_path = load_state_or_bootstrap() + if source != "state_file": + atomic_write_json(state_path, state) + delta_count = state.get("successful_deltas_since_snapshot") + force_needed = bool( + state.get("last_success_snapshot") is not None + and delta_count is not None + and delta_count >= max_deltas + ) + emit("PERIODIC_LIFECYCLE_SOURCE", source) + emit("PERIODIC_LIFECYCLE_DETAIL", detail or state.get("state_detail")) + emit("PERIODIC_LIFECYCLE_CORRUPT_BACKUP_PATH", backup_path) + emit("PERIODIC_LIFECYCLE_STATE_HEALTH", state.get("state_health")) + emit( + "PERIODIC_LIFECYCLE_LAST_SNAPSHOT_RUN_ID", + (state.get("last_success_snapshot") or {}).get("run_id"), + ) + emit("PERIODIC_LIFECYCLE_DELTA_COUNT", delta_count) + emit("PERIODIC_LIFECYCLE_FORCE_NEEDED", force_needed) +elif command == "update": + run_dir = pathlib.Path(extra_args[0]) + state, source, detail, backup_path = load_state_or_bootstrap(exclude_run_id=run_dir.name) + entry = parse_run_dir(run_dir) + if entry is None: + raise SystemExit(f"missing run metadata for lifecycle update: {run_dir}") + previous_snapshot = state.get("last_success_snapshot") + previous_delta_count = state.get("successful_deltas_since_snapshot") + state["last_run"] = entry + recent_runs = [item for item in state.get("recent_runs", []) if item.get("run_id") != entry["run_id"]] + recent_runs.append(entry) + state["recent_runs"] = recent_runs[-recent_limit:] + if entry["status"] == "success" and entry.get("sync_mode") == "snapshot": + state["last_success_snapshot"] = snapshot_ref(entry) + state["successful_deltas_since_snapshot"] = 0 + state["state_detail"] = "ok" + elif entry["status"] == "success" and entry.get("sync_mode") == "delta": + if previous_snapshot is not None and previous_delta_count is not None: + state["last_success_snapshot"] = previous_snapshot + state["successful_deltas_since_snapshot"] = previous_delta_count + 1 + state["state_detail"] = "ok" + else: + state["last_success_snapshot"] = previous_snapshot + state["successful_deltas_since_snapshot"] = None + state["state_detail"] = "success_delta_without_known_snapshot" + else: + state["last_success_snapshot"] = previous_snapshot + state["successful_deltas_since_snapshot"] = previous_delta_count + state = finalize_state(state) + atomic_write_json(state_path, state) + emit("RUN_LIFECYCLE_UPDATE_SOURCE", source) + emit("RUN_LIFECYCLE_UPDATE_DETAIL", detail or state.get("state_detail")) + emit("RUN_LIFECYCLE_UPDATE_CORRUPT_BACKUP_PATH", backup_path) + emit("RUN_LIFECYCLE_UPDATE_STATE_HEALTH", state.get("state_health")) + emit("RUN_LIFECYCLE_UPDATE_DELTA_COUNT", state.get("successful_deltas_since_snapshot")) + emit( + "RUN_LIFECYCLE_UPDATE_LAST_SNAPSHOT_RUN_ID", + (state.get("last_success_snapshot") or {}).get("run_id"), + ) +else: + raise SystemExit(f"unknown lifecycle helper command: {command}") +PY +} + +load_periodic_snapshot_lifecycle_context() { + eval "$(periodic_snapshot_delta_scan load "$PERIODIC_SNAPSHOT_MAX_DELTAS")" + if [[ -n "$PERIODIC_LIFECYCLE_CORRUPT_BACKUP_PATH" ]]; then + warn "run lifecycle state corrupt; backed up to $PERIODIC_LIFECYCLE_CORRUPT_BACKUP_PATH detail=${PERIODIC_LIFECYCLE_DETAIL:-unknown}" + fi + if [[ "$PERIODIC_LIFECYCLE_SOURCE" != "state_file" ]]; then + echo "run lifecycle state source=$PERIODIC_LIFECYCLE_SOURCE state_health=${PERIODIC_LIFECYCLE_STATE_HEALTH:-unknown} snapshot_run=${PERIODIC_LIFECYCLE_LAST_SNAPSHOT_RUN_ID:-none} delta_count=${PERIODIC_LIFECYCLE_DELTA_COUNT:-unknown}" + fi + if [[ "$PERIODIC_LIFECYCLE_STATE_HEALTH" == "bootstrap_incomplete" ]]; then + warn "run lifecycle state bootstrap incomplete detail=${PERIODIC_LIFECYCLE_DETAIL:-unknown}; forced snapshot counting resumes after the next successful snapshot" + fi +} + +update_run_lifecycle_state() { + local run_dir="$1" + eval "$(periodic_snapshot_delta_scan update "$run_dir")" + if [[ -n "$RUN_LIFECYCLE_UPDATE_CORRUPT_BACKUP_PATH" ]]; then + warn "run lifecycle state corrupt during update; backed up to $RUN_LIFECYCLE_UPDATE_CORRUPT_BACKUP_PATH detail=${RUN_LIFECYCLE_UPDATE_DETAIL:-unknown}" + fi +} + +prepare_periodic_reset_state_db() { + local run_id="$1" + RESET_DB_STAGING_PATH="" + RESET_DB_CLEANUP_STATUS="" + db_state_exists || return 0 + local timestamp + local staging_root + timestamp="$(date -u +%Y%m%dT%H%M%SZ)" + staging_root="$RESET_STAGING_ROOT/${run_id}-${timestamp}" + mkdir -p "$staging_root" + mv "$DB_DIR" "$staging_root/" + mkdir -p "$DB_DIR" + RESET_DB_STAGING_PATH="$staging_root/$(basename "$DB_DIR")" + RESET_DB_CLEANUP_STATUS="pending" +} + +finalize_periodic_reset_state_db() { + local final_status="$1" + local reset_db_staging_path="$2" + [[ -n "$reset_db_staging_path" ]] || { + printf '%s\n' "" + return 0 + } + local staging_root + staging_root="$(dirname "$reset_db_staging_path")" + if [[ "$final_status" == "success" ]]; then + if rm -rf "$staging_root"; then + printf '%s\n' "deleted" + return 0 + fi + warn "failed to delete periodic reset staging: $staging_root" + printf '%s\n' "cleanup_failed" + return 1 + fi + printf '%s\n' "retained_failure" +} + +write_run_meta() { + local output_path="$1" + local status="$2" + local run_index="$3" + local run_id="$4" + local sync_mode="$5" + local snapshot_reason="$6" + local previous_run_id="$7" + local previous_run_success_value="$8" + local started_at="$9" + local completed_at="${10}" + local invalid_db_path="${11}" + local invalid_state_path="${12}" + local invalid_tmp_path="${13}" + local daemon_exit_code="${14}" + local package_root="${15}" + local env_file="${16}" + local periodic_snapshot_reset_enabled="${17}" + local periodic_snapshot_max_deltas="${18}" + local periodic_snapshot_delta_count="${19}" + local periodic_snapshot_forced="${20}" + local reset_db_staging_path="${21}" + local reset_db_cleanup_status="${22}" + local tmp_cleanup_status="${23}" + local tmp_cleanup_reason="${24}" + python3 - "$output_path" "$status" "$run_index" "$run_id" "$sync_mode" "$snapshot_reason" \ + "$previous_run_id" "$previous_run_success_value" "$started_at" "$completed_at" \ + "$invalid_db_path" "$invalid_state_path" "$invalid_tmp_path" "$daemon_exit_code" \ + "$package_root" "$env_file" "$periodic_snapshot_reset_enabled" \ + "$periodic_snapshot_max_deltas" "$periodic_snapshot_delta_count" \ + "$periodic_snapshot_forced" "$reset_db_staging_path" "$reset_db_cleanup_status" \ + "$tmp_cleanup_status" "$tmp_cleanup_reason" <<'PY' +import json +import sys + +def nullable(value): + return None if value == "" else value + +def nullable_bool(value): + if value == "": + return None + return value == "true" + +def nullable_int(value): + if value == "": + return None + return int(value) + +def bool_value(value): + return value == "true" + +( + output_path, + status, + run_index, + run_id, + sync_mode, + snapshot_reason, + previous_run_id, + previous_run_success, + started_at, + completed_at, + invalid_db_path, + invalid_state_path, + invalid_tmp_path, + daemon_exit_code, + package_root, + env_file, + periodic_snapshot_reset_enabled, + periodic_snapshot_max_deltas, + periodic_snapshot_delta_count, + periodic_snapshot_forced, + reset_db_staging_path, + reset_db_cleanup_status, + tmp_cleanup_status, + tmp_cleanup_reason, +) = sys.argv[1:] + +data = { + "status": status, + "run_index": int(run_index), + "run_id": run_id, + "sync_mode": sync_mode, + "snapshot_reason": nullable(snapshot_reason), + "previous_run_id": nullable(previous_run_id), + "previous_run_success": nullable_bool(previous_run_success), + "started_at_rfc3339_utc": started_at, + "completed_at_rfc3339_utc": nullable(completed_at), + "invalid_db_path": nullable(invalid_db_path), + "invalid_state_path": nullable(invalid_state_path), + "invalid_tmp_path": nullable(invalid_tmp_path), + "daemon_exit_code": nullable_int(daemon_exit_code), + "package_root": package_root, + "env_file": env_file, + "periodic_snapshot_reset_enabled": bool_value(periodic_snapshot_reset_enabled), + "periodic_snapshot_max_deltas": int(periodic_snapshot_max_deltas), + "periodic_snapshot_delta_count": nullable_int(periodic_snapshot_delta_count), + "periodic_snapshot_forced": bool_value(periodic_snapshot_forced), + "reset_db_staging_path": nullable(reset_db_staging_path), + "reset_db_cleanup_status": nullable(reset_db_cleanup_status), + "tmp_cleanup_status": nullable(tmp_cleanup_status), + "tmp_cleanup_reason": nullable(tmp_cleanup_reason), +} +with open(output_path, "w", encoding="utf-8") as handle: + json.dump(data, handle, indent=2, sort_keys=True) + handle.write("\n") +PY +} + +summary_status() { + local summary_path="$1" + python3 - "$summary_path" <<'PY' +import json +import sys +try: + with open(sys.argv[1], "r", encoding="utf-8") as handle: + print(json.load(handle).get("status", "missing")) +except Exception: + print("missing") +PY +} + +prepare_competing_rp_state() { + if ! is_true "$DISABLE_COMPETING_RPS"; then + return 0 + fi + systemctl disable --now rpki-client.timer >/dev/null 2>&1 || true + systemctl stop rpki-client.service >/dev/null 2>&1 || true + pkill -x rpki-client >/dev/null 2>&1 || true + pkill -x routinator >/dev/null 2>&1 || true +} + +write_machine_snapshot() { + local suffix="$1" + df -h > "$LOG_ROOT/df-${suffix}.txt" 2>&1 || true + free -h > "$LOG_ROOT/free-${suffix}.txt" 2>&1 || true + ps -eo pid,ppid,stat,pcpu,pmem,rss,args --sort=-pcpu \ + | grep -E 'panda-rpki-validator|rpki-client|routinator' \ + | grep -v grep > "$LOG_ROOT/process-${suffix}.txt" || true + systemctl is-active rpki-client.timer > "$LOG_ROOT/rpki-client-timer-active-${suffix}.txt" 2>&1 || true + systemctl is-enabled rpki-client.timer > "$LOG_ROOT/rpki-client-timer-enabled-${suffix}.txt" 2>&1 || true +} + +build_child_args() { + CHILD_ARGS=( + --db "$DB_DIR/work-db" + --repo-bytes-db "$DB_DIR/repo-bytes.db" + --rsync-scope "$RSYNC_SCOPE" + --resource-validation-mode "$RESOURCE_VALIDATION_MODE" + ) + if is_true "$ALLOW_RSYNC_MIRROR_REUSE"; then + CHILD_ARGS+=(--rsync-mirror-root "$RSYNC_MIRROR_ROOT") + else + CHILD_ARGS+=(--rsync-mirror-root "$TMP_DIR/rsync-mirror-{run_id}") + fi + + local root_cert_path + local -a root_cert_paths=() + if [[ -n "$HTTP_ROOT_CERT_PATHS" ]]; then + # HTTPS trust roots for RRDP/TAL fetches (independent of the RPKI TA). + # shellcheck disable=SC2206 + root_cert_paths=( $HTTP_ROOT_CERT_PATHS ) + for root_cert_path in "${root_cert_paths[@]}"; do + CHILD_ARGS+=(--http-root-cert "$root_cert_path") + done + fi + + CHILD_ARGS+=( + --parallel-phase2-ready-batch-size 256 + --parallel-phase2-ready-batch-wall-time-budget-ms 100 + --parallel-phase2-result-drain-batch-size 2048 + --parallel-phase2-finalize-batch-size 256 + --parallel-phase2-finalize-batch-wall-time-budget-ms 100 + ) + + local rir_name + for rir_name in "${RIR_LIST[@]}"; do + if [[ "$TAL_INPUT_MODE" == "url" ]]; then + CHILD_ARGS+=(--tal-url "$(tal_url_for_rir "$rir_name")") + elif [[ "$TAL_INPUT_MODE" == "file-live-ta" ]]; then + CHILD_ARGS+=(--tal-path "$(tal_file_for_rir "$rir_name")") + CHILD_ARGS+=(--ta-path "$(live_ta_file_for_rir "$rir_name")") + else + CHILD_ARGS+=(--tal-path "$(tal_file_for_rir "$rir_name")") + CHILD_ARGS+=(--ta-path "$(ta_file_for_rir "$rir_name")") + fi + done + + CHILD_ARGS+=( + --report-json "{run_out}/report.json" + --validation-contract-out "{run_out}/validation-contract.json" + ) + if is_true "$OUTPUT_COMPACT_REPORT"; then + CHILD_ARGS+=(--report-json-compact) + fi + CHILD_ARGS+=( + --ccr-out "{run_out}/result.ccr" + --cir-enable + --cir-out "{run_out}/input.cir" + ) + + for rir_name in "${RIR_LIST[@]}"; do + CHILD_ARGS+=(--cir-tal-uri "$(cir_tal_uri_for_rir "$rir_name")") + done + + CHILD_ARGS+=( + --vrps-csv-out "{run_out}/vrps.csv" + --vaps-csv-out "{run_out}/vaps.csv" + --compare-view-trust-anchor "$(compare_view_trust_anchor)" + ) + if is_true "$ENABLE_CHILD_CERTIFICATE_VALIDATION_CACHE"; then + CHILD_ARGS+=(--enable-child-certificate-validation-cache) + fi + if [[ -n "$RPKI_EXTRA_ARGS" ]]; then + # shellcheck disable=SC2206 + local extra_args=( $RPKI_EXTRA_ARGS ) + CHILD_ARGS+=("${extra_args[@]}") + fi +} + +write_current_state_binding() { + local run_dir="$1" + local run_id="$2" + local output_path="$META_DIR/current-state-binding.json" + python3 - "$run_dir" "$run_id" "$output_path" <<'PY' +import hashlib +import json +import os +import sys +import tempfile +from datetime import datetime, timezone +from pathlib import Path + +run_dir = Path(sys.argv[1]) +run_id = sys.argv[2] +output_path = Path(sys.argv[3]) +summary = json.loads((run_dir / "run-summary.json").read_text(encoding="utf-8")) +if summary.get("status") != "success": + raise SystemExit(f"cannot bind unsuccessful source run: {run_id}") + +def digest(name): + path = run_dir / name + if not path.is_file(): + raise SystemExit(f"current state binding input is missing: {path}") + hasher = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + hasher.update(chunk) + return hasher.hexdigest() + +binding = { + "schemaVersion": 1, + "currentRunId": run_id, + "sourceCirSha256": digest("input.cir"), + "sourceCcrSha256": digest("result.ccr"), + "validationContractSha256": digest("validation-contract.json"), + "updatedAtRfc3339Utc": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), +} +output_path.parent.mkdir(parents=True, exist_ok=True) +fd, tmp_name = tempfile.mkstemp(prefix=output_path.name + ".", dir=output_path.parent) +try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + json.dump(binding, handle, indent=2, sort_keys=True) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp_name, output_path) +finally: + if os.path.exists(tmp_name): + os.unlink(tmp_name) +PY +} + +copy_inner_run_outputs() { + local daemon_state_root="$1" + local run_dir="$2" + local outer_run_index="$3" + local outer_run_id="$4" + local inner_run_dir + inner_run_dir="$(find "$daemon_state_root/runs" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort | tail -n 1 || true)" + if [[ -n "$inner_run_dir" && -d "$inner_run_dir" ]]; then + shopt -s dotglob nullglob + cp -a "$inner_run_dir"/. "$run_dir"/ + shopt -u dotglob nullglob + fi + [[ -f "$daemon_state_root/daemon-status.json" ]] && cp "$daemon_state_root/daemon-status.json" "$run_dir/daemon-status.inner.json" + [[ -f "$daemon_state_root/daemon-runs.jsonl" ]] && cp "$daemon_state_root/daemon-runs.jsonl" "$run_dir/daemon-runs.inner.jsonl" + normalize_outer_run_metadata "$run_dir" "$outer_run_index" "$outer_run_id" "$inner_run_dir" "$daemon_state_root" +} + +normalize_outer_run_metadata() { + local run_dir="$1" + local outer_run_index="$2" + local outer_run_id="$3" + local inner_run_dir="$4" + local daemon_state_root="$5" + python3 - "$run_dir" "$outer_run_index" "$outer_run_id" "$inner_run_dir" "$daemon_state_root" <<'PY' +import json +import pathlib +import sys + +run_dir = pathlib.Path(sys.argv[1]).resolve() +outer_run_index = int(sys.argv[2]) +outer_run_id = sys.argv[3] +inner_run_dir = sys.argv[4] +daemon_state_root = pathlib.Path(sys.argv[5]) + +def replace_paths(value): + if isinstance(value, dict): + return {key: replace_paths(item) for key, item in value.items()} + if isinstance(value, list): + return [replace_paths(item) for item in value] + if isinstance(value, str) and inner_run_dir: + return value.replace(inner_run_dir, str(run_dir)) + return value + +def normalize_summary(summary): + summary = dict(summary) + summary.setdefault("innerRunSeq", summary.get("runSeq")) + summary.setdefault("innerRunId", summary.get("runId")) + summary.setdefault("innerRunDir", summary.get("runDir")) + summary = replace_paths(summary) + summary["runSeq"] = outer_run_index + summary["runId"] = outer_run_id + summary["runDir"] = str(run_dir) + return summary + +summary_path = run_dir / "run-summary.json" +if summary_path.exists(): + summary = json.loads(summary_path.read_text(encoding="utf-8")) + summary_path.write_text( + json.dumps(normalize_summary(summary), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + +inner_status_path = run_dir / "daemon-status.inner.json" +if not inner_status_path.exists(): + raw_status_path = daemon_state_root / "daemon-status.json" + if raw_status_path.exists(): + inner_status_path.write_text(raw_status_path.read_text(encoding="utf-8"), encoding="utf-8") +if inner_status_path.exists(): + status = json.loads(inner_status_path.read_text(encoding="utf-8")) + status.setdefault("innerLastRunId", status.get("lastRunId")) + status["lastRunId"] = outer_run_id + status["outerRunId"] = outer_run_id + status["outerRunIndex"] = outer_run_index + (run_dir / "daemon-status.json").write_text( + json.dumps(status, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + +inner_runs_path = run_dir / "daemon-runs.inner.jsonl" +if not inner_runs_path.exists(): + raw_runs_path = daemon_state_root / "daemon-runs.jsonl" + if raw_runs_path.exists(): + inner_runs_path.write_text(raw_runs_path.read_text(encoding="utf-8"), encoding="utf-8") +if inner_runs_path.exists(): + lines = [] + for line in inner_runs_path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + lines.append(json.dumps(normalize_summary(json.loads(line)), sort_keys=True)) + (run_dir / "daemon-runs.jsonl").write_text("\n".join(lines) + ("\n" if lines else ""), encoding="utf-8") +PY +} + +apply_outer_retention() { + local dirs=() + local retain_limit="$RETAIN_RUNS" + local keep_run="${1:-}" + local run_dir + shopt -s nullglob + for run_dir in "$RUNS_ROOT"/run_[0-9][0-9][0-9][0-9]; do + [[ -d "$run_dir" ]] && dirs+=("$run_dir") + done + shopt -u nullglob + if (( ${#dirs[@]} <= retain_limit )); then + return 0 + fi + mapfile -t dirs < <(printf '%s\n' "${dirs[@]}" | sort) + local remove_count=$(( ${#dirs[@]} - retain_limit )) + local removed_count=0 + local candidate + for candidate in "${dirs[@]}"; do + if [[ -n "$keep_run" && "$(basename "$candidate")" == "$keep_run" ]]; then + continue + fi + rm -rf "$candidate" + removed_count=$((removed_count + 1)) + if (( removed_count >= remove_count )); then + break + fi + done +} + +run_one_round() { + local run_index="$1" + local run_id + run_id="$(printf 'run_%04d' "$run_index")" + local run_dir="$RUNS_ROOT/$run_id" + local previous_run_id="$2" + local previous_success_value="$3" + local sync_mode="$4" + local snapshot_reason="$5" + local daemon_state_root="$TMP_DIR/daemon-$run_id" + local started_at + local completed_at + local daemon_exit_code + local summary_state + + mkdir -p "$run_dir" "$daemon_state_root" + apply_outer_retention "$run_id" + started_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + write_run_meta "$run_dir/run-meta.json" "running" "$run_index" "$run_id" "$sync_mode" \ + "$snapshot_reason" "$previous_run_id" "$previous_success_value" "$started_at" "" \ + "$INVALID_DB_PATH" "$INVALID_STATE_PATH" "$INVALID_TMP_PATH" "" "$PACKAGE_ROOT" "$ENV_FILE" \ + "$RUN_META_PERIODIC_ENABLED" "$RUN_META_PERIODIC_MAX_DELTAS" "$RUN_META_PERIODIC_DELTA_COUNT" \ + "$RUN_META_PERIODIC_FORCED" "$RUN_META_RESET_DB_STAGING_PATH" "$RUN_META_RESET_DB_CLEANUP_STATUS" \ + "$RUN_META_TMP_CLEANUP_STATUS" "$RUN_META_TMP_CLEANUP_REASON" + + prepare_live_ta_inputs_for_run "$run_id" "$sync_mode" + build_child_args + if is_true "$RPKI_ANALYZE"; then + CHILD_ARGS+=(--analyze --analysis-out "$run_dir/analyze") + fi + local daemon_args=( + --state-root "$daemon_state_root" + --rpki-bin "$RPKI_BIN" + --interval-secs 0 + --max-runs 1 + --retain-runs "$RETAIN_RUNS" + --work-db "$DB_DIR/work-db" + --repo-bytes-db "$DB_DIR/repo-bytes.db" + ) + if [[ -x "$DB_STATS_BIN" ]]; then + daemon_args+=(--db-stats-bin "$DB_STATS_BIN") + if [[ -n "${DB_STATS_EXACT_EVERY:-}" && "$DB_STATS_EXACT_EVERY" != "0" ]]; then + daemon_args+=(--db-stats-exact-every "$DB_STATS_EXACT_EVERY") + fi + fi + # Dead-repo transport blacklist (#141): opt-in via env; the daemon forwards + # the enable flag and threshold to the child unless already present there. + if [[ -n "${DEAD_REPO_BLACKLIST_PATH:-}" ]]; then + daemon_args+=(--dead-repo-blacklist "$DEAD_REPO_BLACKLIST_PATH") + if [[ -n "${DEAD_REPO_BLACKLIST_FAIL_THRESHOLD:-}" ]]; then + daemon_args+=(--dead-repo-blacklist-fail-threshold "$DEAD_REPO_BLACKLIST_FAIL_THRESHOLD") + fi + if [[ -n "${DEAD_REPO_HEALTH_CHECK_INTERVAL_SECS:-}" ]]; then + daemon_args+=(--dead-repo-health-check-interval-secs "$DEAD_REPO_HEALTH_CHECK_INTERVAL_SECS") + fi + fi + + set +e + env \ + RPKI_PROGRESS_LOG="$RPKI_PROGRESS_LOG" \ + RPKI_PROGRESS_SLOW_SECS="$RPKI_PROGRESS_SLOW_SECS" \ + RPKI_PROGRESS_STAGE_FRESH_SLOW_MS="$RPKI_PROGRESS_STAGE_FRESH_SLOW_MS" \ + RPKI_PROGRESS_PP_CONTROL_SLOW_MS="$RPKI_PROGRESS_PP_CONTROL_SLOW_MS" \ + RPKI_PROGRESS_PP_CACHE_SLOW_MS="$RPKI_PROGRESS_PP_CACHE_SLOW_MS" \ + RPKI_PROGRESS_CONTROL_LOOP_SLOW_MS="$RPKI_PROGRESS_CONTROL_LOOP_SLOW_MS" \ + "$RPKI_DAEMON_BIN" "${daemon_args[@]}" -- "${CHILD_ARGS[@]}" \ + > "$run_dir/daemon-stdout.log" 2> "$run_dir/daemon-stderr.log" + daemon_exit_code=$? + set -e + + copy_inner_run_outputs "$daemon_state_root" "$run_dir" "$run_index" "$run_id" + completed_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + summary_state="$(summary_status "$run_dir/run-summary.json")" + local final_status="failed" + if [[ "$daemon_exit_code" -eq 0 && "$summary_state" == "success" ]]; then + final_status="success" + fi + if [[ -n "$RUN_META_RESET_DB_STAGING_PATH" ]]; then + if RUN_META_RESET_DB_CLEANUP_STATUS="$(finalize_periodic_reset_state_db "$final_status" "$RUN_META_RESET_DB_STAGING_PATH")"; then + : + else + final_status="failed" + fi + fi + if [[ "$final_status" == "success" ]]; then + if ! write_current_state_binding "$run_dir" "$run_id"; then + warn "failed to bind successful run to current state: $run_id" + final_status="failed" + fi + fi + write_run_meta "$run_dir/run-meta.json" "$final_status" "$run_index" "$run_id" "$sync_mode" \ + "$snapshot_reason" "$previous_run_id" "$previous_success_value" "$started_at" "$completed_at" \ + "$INVALID_DB_PATH" "$INVALID_STATE_PATH" "$INVALID_TMP_PATH" "$daemon_exit_code" "$PACKAGE_ROOT" "$ENV_FILE" \ + "$RUN_META_PERIODIC_ENABLED" "$RUN_META_PERIODIC_MAX_DELTAS" "$RUN_META_PERIODIC_DELTA_COUNT" \ + "$RUN_META_PERIODIC_FORCED" "$RUN_META_RESET_DB_STAGING_PATH" "$RUN_META_RESET_DB_CLEANUP_STATUS" \ + "$RUN_META_TMP_CLEANUP_STATUS" "$RUN_META_TMP_CLEANUP_REASON" + update_run_lifecycle_state "$run_dir" + printf '%s\n' "$run_id" > "$META_DIR/last-run-id" + if is_true "$CLEAN_TMP_AFTER_RUN"; then + rm -rf "$daemon_state_root" + fi + apply_outer_retention + [[ "$final_status" == "success" ]] +} + +main() { + if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then + usage + exit 0 + fi + require_command python3 + require_command date + require_command find + require_command flock + if [[ "$TAL_INPUT_MODE" == "file-live-ta" ]]; then + require_command curl + validate_positive_int "LIVE_TA_REFRESH_CONNECT_TIMEOUT_SECS" "$LIVE_TA_REFRESH_CONNECT_TIMEOUT_SECS" + validate_positive_int "LIVE_TA_REFRESH_MAX_TIME_SECS" "$LIVE_TA_REFRESH_MAX_TIME_SECS" + fi + validate_max_runs + validate_non_negative_int "INTERVAL_SECS" "$INTERVAL_SECS" + validate_non_negative_int "STOP_AFTER_SECS" "$STOP_AFTER_SECS" + validate_positive_int "RETAIN_RUNS" "$RETAIN_RUNS" + validate_rsync_scope + validate_tal_input_mode + validate_resource_validation_mode + validate_non_negative_int "PERIODIC_SNAPSHOT_MAX_DELTAS" "$PERIODIC_SNAPSHOT_MAX_DELTAS" + if [[ -n "${DB_STATS_EXACT_EVERY:-}" && "$DB_STATS_EXACT_EVERY" != "0" ]]; then + validate_positive_int "DB_STATS_EXACT_EVERY" "$DB_STATS_EXACT_EVERY" + fi + parse_rirs + validate_custom_inputs + [[ -x "$RPKI_BIN" ]] || die "missing executable: $RPKI_BIN" + [[ -x "$RPKI_DAEMON_BIN" ]] || die "missing executable: $RPKI_DAEMON_BIN" + + local rir_name + for rir_name in "${RIR_LIST[@]}"; do + if [[ "$TAL_INPUT_MODE" == "url" ]]; then + [[ -n "$(tal_url_for_rir "$rir_name")" ]] || die "missing TAL URL for $rir_name" + elif [[ "$TAL_INPUT_MODE" == "file-live-ta" ]]; then + [[ -f "$(tal_file_for_rir "$rir_name")" ]] || die "missing TAL fixture for $rir_name" + [[ -n "$(tal_https_uri_from_fixture "$(tal_file_for_rir "$rir_name")")" ]] || die "missing http(s) TA URI in TAL fixture for $rir_name" + else + [[ -f "$(tal_file_for_rir "$rir_name")" ]] || die "missing TAL fixture for $rir_name" + [[ -f "$(ta_file_for_rir "$rir_name")" ]] || die "missing TA fixture for $rir_name" + fi + done + + mkdir -p "$RUNS_ROOT" "$LOG_ROOT" "$DB_DIR" "$META_DIR" "$TMP_DIR" "$INVALID_ROOT" "$RESET_STAGING_ROOT" "$LIVE_TA_REFRESH_DIR" "$(dirname "$STATE_EXECUTION_LOCK_PATH")" + if is_true "$ALLOW_RSYNC_MIRROR_REUSE"; then + mkdir -p "$RSYNC_MIRROR_ROOT" + fi + prepare_competing_rp_state + write_machine_snapshot "before" + + local max_index + local next_index + local run_forever=0 + local stop_index=0 + local started_epoch + local elapsed_secs + started_epoch="$(date +%s)" + max_index="$(max_existing_run_index)" + next_index=$((max_index + 1)) + if (( MAX_RUNS < 0 )); then + run_forever=1 + echo "run_validator mode=continuous max_existing_run_index=$max_index next_run=$(printf 'run_%04d' "$next_index")" + else + stop_index=$((max_index + MAX_RUNS)) + echo "run_validator mode=fixed max_existing_run_index=$max_index next_run=$(printf 'run_%04d' "$next_index") stop_run=$(printf 'run_%04d' "$stop_index")" + fi + local any_failed=0 + + while (( run_forever == 1 || next_index <= stop_index )); do + local state_lock_fd + exec {state_lock_fd}>"$STATE_EXECUTION_LOCK_PATH" + flock "$state_lock_fd" + INVALID_DB_PATH="" + INVALID_STATE_PATH="" + INVALID_TMP_PATH="" + TMP_CLEANUP_STATUS="" + TMP_CLEANUP_REASON="" + PERIODIC_LIFECYCLE_SOURCE="" + PERIODIC_LIFECYCLE_DETAIL="" + PERIODIC_LIFECYCLE_CORRUPT_BACKUP_PATH="" + PERIODIC_LIFECYCLE_STATE_HEALTH="" + PERIODIC_LIFECYCLE_LAST_SNAPSHOT_RUN_ID="" + PERIODIC_LIFECYCLE_DELTA_COUNT="" + PERIODIC_LIFECYCLE_FORCE_NEEDED="false" + RESET_DB_STAGING_PATH="" + RESET_DB_CLEANUP_STATUS="" + RUN_META_PERIODIC_DELTA_COUNT="" + if is_true "$PERIODIC_SNAPSHOT_RESET"; then + RUN_META_PERIODIC_ENABLED="true" + load_periodic_snapshot_lifecycle_context + if [[ -n "$PERIODIC_LIFECYCLE_DELTA_COUNT" ]]; then + RUN_META_PERIODIC_DELTA_COUNT="$PERIODIC_LIFECYCLE_DELTA_COUNT" + fi + else + RUN_META_PERIODIC_ENABLED="false" + fi + RUN_META_PERIODIC_MAX_DELTAS="$PERIODIC_SNAPSHOT_MAX_DELTAS" + RUN_META_PERIODIC_FORCED="false" + RUN_META_RESET_DB_STAGING_PATH="" + RUN_META_RESET_DB_CLEANUP_STATUS="" + RUN_META_TMP_CLEANUP_STATUS="" + RUN_META_TMP_CLEANUP_REASON="" + local previous_run_id="" + local previous_success_value="" + local sync_mode="snapshot" + local snapshot_reason="" + if (( next_index > 1 )); then + previous_run_id="$(printf 'run_%04d' $((next_index - 1)))" + if previous_run_success "$RUNS_ROOT/$previous_run_id"; then + previous_success_value="true" + if delta_state_available; then + sync_mode="delta" + if is_true "$PERIODIC_SNAPSHOT_RESET"; then + if [[ "$PERIODIC_LIFECYCLE_FORCE_NEEDED" == "true" ]]; then + RUN_META_PERIODIC_FORCED="true" + sync_mode="snapshot" + snapshot_reason="periodic_snapshot_delta_limit" + prepare_periodic_reset_state_db "$(printf 'run_%04d' "$next_index")" + RUN_META_RESET_DB_STAGING_PATH="$RESET_DB_STAGING_PATH" + RUN_META_RESET_DB_CLEANUP_STATUS="$RESET_DB_CLEANUP_STATUS" + echo "periodic snapshot reset forcing snapshot run=$(printf 'run_%04d' "$next_index") delta_count=$PERIODIC_LIFECYCLE_DELTA_COUNT max_deltas=$PERIODIC_SNAPSHOT_MAX_DELTAS" + fi + fi + else + sync_mode="snapshot" + snapshot_reason="missing_db" + fi + else + previous_success_value="false" + if is_true "$FAILURE_SNAPSHOT_RESET"; then + isolate_state_after_failure "$previous_run_id" + RUN_META_TMP_CLEANUP_STATUS="$TMP_CLEANUP_STATUS" + RUN_META_TMP_CLEANUP_REASON="$TMP_CLEANUP_REASON" + sync_mode="snapshot" + snapshot_reason="previous_run_failed" + else + die "previous run is not successful: $previous_run_id" + fi + fi + else + sync_mode="snapshot" + if db_state_exists; then + isolate_state_after_failure "no_previous_run" + RUN_META_TMP_CLEANUP_STATUS="$TMP_CLEANUP_STATUS" + RUN_META_TMP_CLEANUP_REASON="$TMP_CLEANUP_REASON" + snapshot_reason="no_successful_previous_run" + else + snapshot_reason="first_run" + fi + fi + + echo "starting run $(printf 'run_%04d' "$next_index") sync_mode=$sync_mode" + if run_one_round "$next_index" "$previous_run_id" "$previous_success_value" "$sync_mode" "$snapshot_reason"; then + echo "completed run $(printf 'run_%04d' "$next_index") status=success" + else + echo "completed run $(printf 'run_%04d' "$next_index") status=failed" >&2 + any_failed=1 + fi + flock -u "$state_lock_fd" + exec {state_lock_fd}>&- + if (( STOP_AFTER_SECS > 0 )); then + elapsed_secs=$(( $(date +%s) - started_epoch )) + if (( elapsed_secs >= STOP_AFTER_SECS )); then + echo "run_validator stop_after_secs reached elapsed_secs=$elapsed_secs stop_after_secs=$STOP_AFTER_SECS" + break + fi + fi + if (( (run_forever == 1 || next_index < stop_index) && INTERVAL_SECS > 0 )); then + sleep "$INTERVAL_SECS" + fi + next_index=$((next_index + 1)) + done + + write_machine_snapshot "after" + exit "$any_failed" +} + +main "$@" diff --git a/tests/compat/README.md b/tests/compat/README.md new file mode 100644 index 0000000..13719e9 --- /dev/null +++ b/tests/compat/README.md @@ -0,0 +1,49 @@ +# Compatibility harness (M5) + +`verify_run_abi.sh` validates the required files and semantic fields for a +successful normal run. It is intentionally independent of absolute paths and +wall-clock timestamps, so it can be used for native and Docker run directories: + +```bash +tests/compat/verify_run_abi.sh /path/to/runs/run_0001 +``` + +`compare_runs.py` compares an original-runtime run with a component run. It +checks the normal-run status/count contract, report and validation contract, +and canonical CIR/CCR/VRP/VAP payloads. It normalises only the exceptions in +`baseline-manifest.toml` (including runtime telemetry that is measured +separately). Validation time is treated as a fixed input, while only lifecycle +timestamps and CCR `producedAt` are normalised; all other semantic payload +differences are failures: + +```bash +tests/compat/compare_runs.py \ + /path/to/original/runs/run_0001 \ + /path/to/panda/runs/run_0001 +``` + +The M5 harness will run the same frozen input against the original `rpki` +runtime, native `panda-rpki-validator`, and the Docker runtime. It will compare +the normal-run output ABI in `docs/output-abi.md`, then record stage timing, +CPU, RSS, state size and artifact size for five serial repetitions per profile. + +M4 provides the ABI verifier and a real snapshot/warm-delta staging run. The +canonical comparator is now available for M5; five-run performance comparison +and the complete profile matrix remain M5 deliverables. +`verification-only` is excluded. + +For live-RIR evidence, run both images with the same `RIRS` and fixed +`--validation-time`, then compare `input.cir` and the decoded `result.ccr` in +separate steps. `compare_runs.py` deliberately remains strict about report, +validation-event and CSV order, so a live RRDP/rsync source race can produce a +file-level mismatch even when the CCR state digest matches. Use +`ccr_state_compare` plus `triage_ccr_cir_pair` to distinguish source/input +drift from validator behavior; the 2026-09-01 APNIC/all5 evidence is recorded in +`specs/develop/20260901/m5_live_rir_image_artifact_comparison_milestone_report.md`. +The follow-up remote-231 serial all5 run (old image first, then new image; +one snapshot plus three deltas per side with cache/prefetch/parallel flags) is +recorded in +`specs/develop/20260901_2/m5_live_all5_cache_prefetch_serial_4run_milestone_report.md`. +Its strict file mismatches remain failures until the same frozen input is +replayed on both sides; live CCR/CIR differences must first be triaged with +the state comparator rather than treated as code parity. diff --git a/tests/compat/baseline-manifest.toml b/tests/compat/baseline-manifest.toml new file mode 100644 index 0000000..c5e3824 --- /dev/null +++ b/tests/compat/baseline-manifest.toml @@ -0,0 +1,95 @@ +schema_version = 1 +source_commit = "74cbebbd3334ac0063761c1a97a88ee000cc2a57" +component = "panda-rpki-validator" +comparison_status = "m5-canonical-snapshot-delta-baseline-in-progress" +verification_only = "excluded" +normal_profiles = [ + "snapshot", + "delta-warm", + "rrdp-fallback-rsync", + "policy-and-constraints", + "cache-profile", + "payload-replay", +] +container_profiles = ["snapshot", "delta-warm", "persistent-volume-restart"] +performance_repetitions = 5 +performance_gate_median_max_regression_percent = 5 +performance_gate_p95_max_regression_percent = 10 +canonicalization_exceptions = [ + "lifecycle wall-clock timestamps", + "CCR producedAt timestamp", + "absolute filesystem paths", + "declared component/image version", + "declared source/build provenance", + "runtime duration/resource telemetry and derived audit digests", +] + +[[artifact]] +relative_path = "runs/run_XXXX/run-meta.json" +kind = "json" +required = true + +[[artifact]] +relative_path = "runs/run_XXXX/run-summary.json" +kind = "json" +required = true + +[[artifact]] +relative_path = "runs/run_XXXX/daemon-status.json" +kind = "json" +required = true + +[[artifact]] +relative_path = "runs/run_XXXX/report.json" +kind = "json" +required = true + +[[artifact]] +relative_path = "runs/run_XXXX/input.cir" +kind = "cir" +required = true + +[[artifact]] +relative_path = "runs/run_XXXX/result.ccr" +kind = "ccr" +required = true + +[[artifact]] +relative_path = "runs/run_XXXX/vrps.csv" +kind = "csv" +required = true + +[[artifact]] +relative_path = "runs/run_XXXX/vaps.csv" +kind = "csv" +required = true + +[[artifact]] +relative_path = "runs/run_XXXX/validation-contract.json" +kind = "json" +required = true + +[[artifact]] +relative_path = "runs/run_XXXX/stage-timing.json" +kind = "json" +required = true + +[[artifact]] +relative_path = "runs/run_XXXX/stdout.log" +kind = "text" +required = true + +[[artifact]] +relative_path = "runs/run_XXXX/stderr.log" +kind = "text" +required = true + +[[artifact]] +relative_path = "runs/run_XXXX/process-time.txt" +kind = "text" +required = true + +[[artifact]] +relative_path = "runs/run_XXXX/db-stats-estimate.txt" +kind = "text" +required = false diff --git a/tests/compat/compare_runs.py b/tests/compat/compare_runs.py new file mode 100755 index 0000000..cca464e --- /dev/null +++ b/tests/compat/compare_runs.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +"""Compare two successful normal-run directories. + +The validator intentionally emits lifecycle and resource telemetry that is +expected to vary between processes. This comparator keeps those fields out +of the semantic comparison while normalising only the exceptions declared in +``baseline-manifest.toml``: lifecycle wall-clock values, absolute paths, and +binary identity. The validation time is an input and must match exactly. +Payloads (CIR/CCR/VRP/VAP) remain byte-for-byte checked, with only the CCR +``producedAt`` timestamp canonicalised. +""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import re +import sys +from pathlib import Path +from typing import Any + + +REQUIRED = ( + "run-meta.json", + "run-summary.json", + "daemon-status.json", + "report.json", + "input.cir", + "result.ccr", + "vrps.csv", + "vaps.csv", + "validation-contract.json", +) +PAYLOADS = ("input.cir", "result.ccr", "vrps.csv", "vaps.csv") +RFC3339_TEXT = re.compile( + r"20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d+)?(?:Z|[+-]\d\d:\d\d)" +) +GENERALIZED_TIME = re.compile(rb"20\d{12}Z") +INNER_RUN_ID = re.compile(r"\d{6}-20\d{6,8}T\d{6}Z") +ABSOLUTE_PATH = re.compile(r"(? Any: + try: + return json.loads((run_dir / name).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise ValueError(f"{run_dir / name}: {error}") from error + + +def normalise_scalar(value: Any, *, key: str = "") -> Any: + lowered_key = key.lower() + if ( + lowered_key.endswith("_ms") + or lowered_key.endswith("_nanos") + or "duration" in lowered_key + or lowered_key in {"eventssha256", "events_sha256"} + ): + return "" + if not isinstance(value, str): + return value + if key.lower() in { + "binarysha256", + "binary_sha256", + "sourcecommit", + "source_commit", + }: + return "" + wall_clock_keys = { + "started_at_rfc3339_utc", + "completed_at_rfc3339_utc", + "finished_at_rfc3339_utc", + "updated_at_rfc3339_utc", + "startedatrfc3339utc", + "completedatrfc3339utc", + "finishedatrfc3339utc", + "updatedatrfc3339utc", + "started_at", + "completed_at", + "finished_at", + "updated_at", + "timestamp", + } + if lowered_key in wall_clock_keys: + return "" + value = INNER_RUN_ID.sub("", value) + return ABSOLUTE_PATH.sub("", value) + + +def normalise_text(value: str) -> str: + """Normalise a free-form event/log stream without changing payload fields.""" + + value = RFC3339_TEXT.sub("", value) + value = INNER_RUN_ID.sub("", value) + return ABSOLUTE_PATH.sub("", value) + + +def normalise_json(value: Any, *, key: str = "") -> Any: + if isinstance(value, dict): + return {name: normalise_json(item, key=name) for name, item in value.items()} + if isinstance(value, list): + return [normalise_json(item, key=key) for item in value] + return normalise_scalar(value, key=key) + + +def canonical_payload(name: str, data: bytes) -> bytes: + if name in {"input.cir", "result.ccr"}: + # CCR's first GeneralizedTime is producedAt. CIR's first time is the + # validation time and therefore remains an input equality check. + if name == "result.ccr": + data = GENERALIZED_TIME.sub(b"", data, count=1) + if name in {"vrps.csv", "vaps.csv"}: + # Parse and re-emit CSV to avoid line-ending differences while keeping + # column order and row order observable. + rows = list(csv.reader(data.decode("utf-8").splitlines())) + data = "\n".join(",".join(row) for row in rows).encode("utf-8") + return data + + +def summary_projection(summary: dict[str, Any]) -> dict[str, Any]: + artifact_sizes: dict[str, int] = {} + for artifact in summary.get("artifacts", []): + path = Path(str(artifact.get("path", ""))) + if path.name in PAYLOADS: + artifact_sizes[path.name] = int(artifact.get("sizeBytes", -1)) + return { + "status": summary.get("status"), + "exitCode": summary.get("exitCode"), + "exitStatus": summary.get("exitStatus"), + "runId": summary.get("runId"), + "runSeq": summary.get("runSeq"), + "error": summary.get("error"), + "reportCounts": summary.get("reportCounts"), + "repoSyncStats": summary.get("repoSyncStats"), + "artifactSizes": artifact_sizes, + } + + +def daemon_projection(status: dict[str, Any]) -> dict[str, Any]: + return { + name: status.get(name) + for name in ( + "state", + "lastRunId", + "maxRuns", + "outerRunId", + "outerRunIndex", + "runsCompleted", + ) + } + + +def compare_json( + name: str, left: Path, right: Path, mismatches: list[dict[str, Any]] +) -> None: + left_value = load_json(left, name) + right_value = load_json(right, name) + if name == "run-summary.json": + left_value = summary_projection(left_value) + right_value = summary_projection(right_value) + elif name == "daemon-status.json": + left_value = daemon_projection(left_value) + right_value = daemon_projection(right_value) + else: + left_value = normalise_json(left_value) + right_value = normalise_json(right_value) + if left_value != right_value: + mismatches.append({"file": name, "left": left_value, "right": right_value}) + + +def compare_runs(left: Path, right: Path) -> list[dict[str, Any]]: + mismatches: list[dict[str, Any]] = [] + for run_dir in (left, right): + if not run_dir.is_dir(): + mismatches.append({"file": str(run_dir), "error": "run directory missing"}) + continue + for name in REQUIRED: + if not (run_dir / name).is_file(): + mismatches.append({"file": str(run_dir / name), "error": "required file missing"}) + if mismatches: + return mismatches + + for name in ( + "run-meta.json", + "run-summary.json", + "daemon-status.json", + "report.json", + "validation-contract.json", + ): + try: + compare_json(name, left, right, mismatches) + except ValueError as error: + mismatches.append({"file": name, "error": str(error)}) + + for name in PAYLOADS: + left_data = canonical_payload(name, (left / name).read_bytes()) + right_data = canonical_payload(name, (right / name).read_bytes()) + if left_data != right_data: + mismatches.append( + { + "file": name, + "leftSha256": hashlib.sha256(left_data).hexdigest(), + "rightSha256": hashlib.sha256(right_data).hexdigest(), + } + ) + + for name in ("validation-events.jsonl",): + left_file = left / name + right_file = right / name + if left_file.is_file() and right_file.is_file(): + left_data = normalise_text(left_file.read_text(encoding="utf-8")) + right_data = normalise_text(right_file.read_text(encoding="utf-8")) + if left_data != right_data: + mismatches.append({"file": name, "error": "canonical event stream differs"}) + return mismatches + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("left", type=Path, help="original rpki run directory") + parser.add_argument("right", type=Path, help="panda-rpki-validator run directory") + parser.add_argument("--json", action="store_true", help="emit machine-readable result") + args = parser.parse_args() + mismatches = compare_runs(args.left, args.right) + result = {"status": "mismatch" if mismatches else "match", "mismatches": mismatches} + if args.json: + print(json.dumps(result, indent=2, sort_keys=True)) + elif mismatches: + print("normal-run compatibility mismatch:") + for mismatch in mismatches: + print(json.dumps(mismatch, sort_keys=True)) + else: + print(f"normal-run compatibility match: {args.left} == {args.right}") + return 1 if mismatches else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/compat/verify_run_abi.sh b/tests/compat/verify_run_abi.sh new file mode 100755 index 0000000..163fbdd --- /dev/null +++ b/tests/compat/verify_run_abi.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +set -euo pipefail + +RUN_DIR="${1:-}" + +usage() { + cat <<'USAGE' +Usage: + tests/compat/verify_run_abi.sh +USAGE +} + +if [[ -z "$RUN_DIR" || "$RUN_DIR" == "-h" || "$RUN_DIR" == "--help" ]]; then + usage >&2 + exit 2 +fi +[[ -d "$RUN_DIR" ]] || { echo "missing run directory: $RUN_DIR" >&2; exit 2; } + +required_files=( + run-meta.json + run-summary.json + daemon-status.json + report.json + input.cir + result.ccr + vrps.csv + vaps.csv + validation-contract.json + stage-timing.json + stdout.log + stderr.log + process-time.txt +) +for name in "${required_files[@]}"; do + [[ -f "$RUN_DIR/$name" ]] || { echo "missing ABI file: $RUN_DIR/$name" >&2; exit 1; } +done + +python3 - "$RUN_DIR" <<'PY' +import csv +import json +import pathlib +import sys + +run_dir = pathlib.Path(sys.argv[1]) +meta = json.loads((run_dir / "run-meta.json").read_text(encoding="utf-8")) +summary = json.loads((run_dir / "run-summary.json").read_text(encoding="utf-8")) +daemon = json.loads((run_dir / "daemon-status.json").read_text(encoding="utf-8")) +contract = json.loads((run_dir / "validation-contract.json").read_text(encoding="utf-8")) +report = json.loads((run_dir / "report.json").read_text(encoding="utf-8")) + +checks = { + "run-meta status": meta.get("status") == "success", + "run-meta id": meta.get("run_id") == run_dir.name, + "run-summary status": summary.get("status") == "success", + "run-summary exit": summary.get("exitCode") == 0, + "run-summary id": summary.get("runId") == run_dir.name, + "daemon state": daemon.get("state") == "exited", + "daemon last run": daemon.get("lastRunId") == run_dir.name, + "contract schema": contract.get("schemaVersion") == 1, + "report format": report.get("format_version") == 2, +} +failed = [name for name, passed in checks.items() if not passed] +if failed: + raise SystemExit("ABI semantic checks failed: " + ", ".join(failed)) + +expected_headers = { + "vrps.csv": ["ASN", "IP Prefix", "Max Length", "Trust Anchor"], + "vaps.csv": ["Customer ASN", "Providers", "Trust Anchor"], +} +for name, expected in expected_headers.items(): + with (run_dir / name).open(newline="", encoding="utf-8") as handle: + actual = next(csv.reader(handle), []) + if actual != expected: + raise SystemExit(f"{name} header changed: expected={expected!r} actual={actual!r}") + +print(f"normal run ABI verified: {run_dir}") +PY