freeze initial panda-rpki staging

tests/compat/README.md

@@ -0,0 +1,49 @@
+# Compatibility harness (M5)
+
+`verify_run_abi.sh` validates the required files and semantic fields for a
+successful normal run. It is intentionally independent of absolute paths and
+wall-clock timestamps, so it can be used for native and Docker run directories:
+
+```bash
+tests/compat/verify_run_abi.sh /path/to/runs/run_0001
+```
+
+`compare_runs.py` compares an original-runtime run with a component run. It
+checks the normal-run status/count contract, report and validation contract,
+and canonical CIR/CCR/VRP/VAP payloads. It normalises only the exceptions in
+`baseline-manifest.toml` (including runtime telemetry that is measured
+separately). Validation time is treated as a fixed input, while only lifecycle
+timestamps and CCR `producedAt` are normalised; all other semantic payload
+differences are failures:
+
+```bash
+tests/compat/compare_runs.py \
+  /path/to/original/runs/run_0001 \
+  /path/to/panda/runs/run_0001
+```
+
+The M5 harness will run the same frozen input against the original `rpki`
+runtime, native `panda-rpki-validator`, and the Docker runtime. It will compare
+the normal-run output ABI in `docs/output-abi.md`, then record stage timing,
+CPU, RSS, state size and artifact size for five serial repetitions per profile.
+
+M4 provides the ABI verifier and a real snapshot/warm-delta staging run. The
+canonical comparator is now available for M5; five-run performance comparison
+and the complete profile matrix remain M5 deliverables.
+`verification-only` is excluded.
+
+For live-RIR evidence, run both images with the same `RIRS` and fixed
+`--validation-time`, then compare `input.cir` and the decoded `result.ccr` in
+separate steps. `compare_runs.py` deliberately remains strict about report,
+validation-event and CSV order, so a live RRDP/rsync source race can produce a
+file-level mismatch even when the CCR state digest matches. Use
+`ccr_state_compare` plus `triage_ccr_cir_pair` to distinguish source/input
+drift from validator behavior; the 2026-09-01 APNIC/all5 evidence is recorded in
+`specs/develop/20260901/m5_live_rir_image_artifact_comparison_milestone_report.md`.
+The follow-up remote-231 serial all5 run (old image first, then new image;
+one snapshot plus three deltas per side with cache/prefetch/parallel flags) is
+recorded in
+`specs/develop/20260901_2/m5_live_all5_cache_prefetch_serial_4run_milestone_report.md`.
+Its strict file mismatches remain failures until the same frozen input is
+replayed on both sides; live CCR/CIR differences must first be triaged with
+the state comparator rather than treated as code parity.

tests/compat/baseline-manifest.toml

@@ -0,0 +1,95 @@
+schema_version = 1
+source_commit = "74cbebbd3334ac0063761c1a97a88ee000cc2a57"
+component = "panda-rpki-validator"
+comparison_status = "m5-canonical-snapshot-delta-baseline-in-progress"
+verification_only = "excluded"
+normal_profiles = [
+  "snapshot",
+  "delta-warm",
+  "rrdp-fallback-rsync",
+  "policy-and-constraints",
+  "cache-profile",
+  "payload-replay",
+]
+container_profiles = ["snapshot", "delta-warm", "persistent-volume-restart"]
+performance_repetitions = 5
+performance_gate_median_max_regression_percent = 5
+performance_gate_p95_max_regression_percent = 10
+canonicalization_exceptions = [
+  "lifecycle wall-clock timestamps",
+  "CCR producedAt timestamp",
+  "absolute filesystem paths",
+  "declared component/image version",
+  "declared source/build provenance",
+  "runtime duration/resource telemetry and derived audit digests",
+]
+
+[[artifact]]
+relative_path = "runs/run_XXXX/run-meta.json"
+kind = "json"
+required = true
+
+[[artifact]]
+relative_path = "runs/run_XXXX/run-summary.json"
+kind = "json"
+required = true
+
+[[artifact]]
+relative_path = "runs/run_XXXX/daemon-status.json"
+kind = "json"
+required = true
+
+[[artifact]]
+relative_path = "runs/run_XXXX/report.json"
+kind = "json"
+required = true
+
+[[artifact]]
+relative_path = "runs/run_XXXX/input.cir"
+kind = "cir"
+required = true
+
+[[artifact]]
+relative_path = "runs/run_XXXX/result.ccr"
+kind = "ccr"
+required = true
+
+[[artifact]]
+relative_path = "runs/run_XXXX/vrps.csv"
+kind = "csv"
+required = true
+
+[[artifact]]
+relative_path = "runs/run_XXXX/vaps.csv"
+kind = "csv"
+required = true
+
+[[artifact]]
+relative_path = "runs/run_XXXX/validation-contract.json"
+kind = "json"
+required = true
+
+[[artifact]]
+relative_path = "runs/run_XXXX/stage-timing.json"
+kind = "json"
+required = true
+
+[[artifact]]
+relative_path = "runs/run_XXXX/stdout.log"
+kind = "text"
+required = true
+
+[[artifact]]
+relative_path = "runs/run_XXXX/stderr.log"
+kind = "text"
+required = true
+
+[[artifact]]
+relative_path = "runs/run_XXXX/process-time.txt"
+kind = "text"
+required = true
+
+[[artifact]]
+relative_path = "runs/run_XXXX/db-stats-estimate.txt"
+kind = "text"
+required = false

tests/compat/compare_runs.py

@@ -0,0 +1,239 @@
+#!/usr/bin/env python3
+"""Compare two successful normal-run directories.
+
+The validator intentionally emits lifecycle and resource telemetry that is
+expected to vary between processes.  This comparator keeps those fields out
+of the semantic comparison while normalising only the exceptions declared in
+``baseline-manifest.toml``: lifecycle wall-clock values, absolute paths, and
+binary identity.  The validation time is an input and must match exactly.
+Payloads (CIR/CCR/VRP/VAP) remain byte-for-byte checked, with only the CCR
+``producedAt`` timestamp canonicalised.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import hashlib
+import json
+import re
+import sys
+from pathlib import Path
+from typing import Any
+
+
+REQUIRED = (
+    "run-meta.json",
+    "run-summary.json",
+    "daemon-status.json",
+    "report.json",
+    "input.cir",
+    "result.ccr",
+    "vrps.csv",
+    "vaps.csv",
+    "validation-contract.json",
+)
+PAYLOADS = ("input.cir", "result.ccr", "vrps.csv", "vaps.csv")
+RFC3339_TEXT = re.compile(
+    r"20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d+)?(?:Z|[+-]\d\d:\d\d)"
+)
+GENERALIZED_TIME = re.compile(rb"20\d{12}Z")
+INNER_RUN_ID = re.compile(r"\d{6}-20\d{6,8}T\d{6}Z")
+ABSOLUTE_PATH = re.compile(r"(?<![A-Za-z0-9_:/])/(?:[^\s,\"']+/)*[^\s,\"']+")
+
+
+def load_json(run_dir: Path, name: str) -> Any:
+    try:
+        return json.loads((run_dir / name).read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError) as error:
+        raise ValueError(f"{run_dir / name}: {error}") from error
+
+
+def normalise_scalar(value: Any, *, key: str = "") -> Any:
+    lowered_key = key.lower()
+    if (
+        lowered_key.endswith("_ms")
+        or lowered_key.endswith("_nanos")
+        or "duration" in lowered_key
+        or lowered_key in {"eventssha256", "events_sha256"}
+    ):
+        return "<runtime-measurement>"
+    if not isinstance(value, str):
+        return value
+    if key.lower() in {
+        "binarysha256",
+        "binary_sha256",
+        "sourcecommit",
+        "source_commit",
+    }:
+        return "<declared-provenance>"
+    wall_clock_keys = {
+        "started_at_rfc3339_utc",
+        "completed_at_rfc3339_utc",
+        "finished_at_rfc3339_utc",
+        "updated_at_rfc3339_utc",
+        "startedatrfc3339utc",
+        "completedatrfc3339utc",
+        "finishedatrfc3339utc",
+        "updatedatrfc3339utc",
+        "started_at",
+        "completed_at",
+        "finished_at",
+        "updated_at",
+        "timestamp",
+    }
+    if lowered_key in wall_clock_keys:
+        return "<wall-clock>"
+    value = INNER_RUN_ID.sub("<inner-run-id>", value)
+    return ABSOLUTE_PATH.sub("<absolute-path>", value)
+
+
+def normalise_text(value: str) -> str:
+    """Normalise a free-form event/log stream without changing payload fields."""
+
+    value = RFC3339_TEXT.sub("<wall-clock>", value)
+    value = INNER_RUN_ID.sub("<inner-run-id>", value)
+    return ABSOLUTE_PATH.sub("<absolute-path>", value)
+
+
+def normalise_json(value: Any, *, key: str = "") -> Any:
+    if isinstance(value, dict):
+        return {name: normalise_json(item, key=name) for name, item in value.items()}
+    if isinstance(value, list):
+        return [normalise_json(item, key=key) for item in value]
+    return normalise_scalar(value, key=key)
+
+
+def canonical_payload(name: str, data: bytes) -> bytes:
+    if name in {"input.cir", "result.ccr"}:
+        # CCR's first GeneralizedTime is producedAt.  CIR's first time is the
+        # validation time and therefore remains an input equality check.
+        if name == "result.ccr":
+            data = GENERALIZED_TIME.sub(b"<wall-clock>", data, count=1)
+    if name in {"vrps.csv", "vaps.csv"}:
+        # Parse and re-emit CSV to avoid line-ending differences while keeping
+        # column order and row order observable.
+        rows = list(csv.reader(data.decode("utf-8").splitlines()))
+        data = "\n".join(",".join(row) for row in rows).encode("utf-8")
+    return data
+
+
+def summary_projection(summary: dict[str, Any]) -> dict[str, Any]:
+    artifact_sizes: dict[str, int] = {}
+    for artifact in summary.get("artifacts", []):
+        path = Path(str(artifact.get("path", "")))
+        if path.name in PAYLOADS:
+            artifact_sizes[path.name] = int(artifact.get("sizeBytes", -1))
+    return {
+        "status": summary.get("status"),
+        "exitCode": summary.get("exitCode"),
+        "exitStatus": summary.get("exitStatus"),
+        "runId": summary.get("runId"),
+        "runSeq": summary.get("runSeq"),
+        "error": summary.get("error"),
+        "reportCounts": summary.get("reportCounts"),
+        "repoSyncStats": summary.get("repoSyncStats"),
+        "artifactSizes": artifact_sizes,
+    }
+
+
+def daemon_projection(status: dict[str, Any]) -> dict[str, Any]:
+    return {
+        name: status.get(name)
+        for name in (
+            "state",
+            "lastRunId",
+            "maxRuns",
+            "outerRunId",
+            "outerRunIndex",
+            "runsCompleted",
+        )
+    }
+
+
+def compare_json(
+    name: str, left: Path, right: Path, mismatches: list[dict[str, Any]]
+) -> None:
+    left_value = load_json(left, name)
+    right_value = load_json(right, name)
+    if name == "run-summary.json":
+        left_value = summary_projection(left_value)
+        right_value = summary_projection(right_value)
+    elif name == "daemon-status.json":
+        left_value = daemon_projection(left_value)
+        right_value = daemon_projection(right_value)
+    else:
+        left_value = normalise_json(left_value)
+        right_value = normalise_json(right_value)
+    if left_value != right_value:
+        mismatches.append({"file": name, "left": left_value, "right": right_value})
+
+
+def compare_runs(left: Path, right: Path) -> list[dict[str, Any]]:
+    mismatches: list[dict[str, Any]] = []
+    for run_dir in (left, right):
+        if not run_dir.is_dir():
+            mismatches.append({"file": str(run_dir), "error": "run directory missing"})
+            continue
+        for name in REQUIRED:
+            if not (run_dir / name).is_file():
+                mismatches.append({"file": str(run_dir / name), "error": "required file missing"})
+    if mismatches:
+        return mismatches
+
+    for name in (
+        "run-meta.json",
+        "run-summary.json",
+        "daemon-status.json",
+        "report.json",
+        "validation-contract.json",
+    ):
+        try:
+            compare_json(name, left, right, mismatches)
+        except ValueError as error:
+            mismatches.append({"file": name, "error": str(error)})
+
+    for name in PAYLOADS:
+        left_data = canonical_payload(name, (left / name).read_bytes())
+        right_data = canonical_payload(name, (right / name).read_bytes())
+        if left_data != right_data:
+            mismatches.append(
+                {
+                    "file": name,
+                    "leftSha256": hashlib.sha256(left_data).hexdigest(),
+                    "rightSha256": hashlib.sha256(right_data).hexdigest(),
+                }
+            )
+
+    for name in ("validation-events.jsonl",):
+        left_file = left / name
+        right_file = right / name
+        if left_file.is_file() and right_file.is_file():
+            left_data = normalise_text(left_file.read_text(encoding="utf-8"))
+            right_data = normalise_text(right_file.read_text(encoding="utf-8"))
+            if left_data != right_data:
+                mismatches.append({"file": name, "error": "canonical event stream differs"})
+    return mismatches
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("left", type=Path, help="original rpki run directory")
+    parser.add_argument("right", type=Path, help="panda-rpki-validator run directory")
+    parser.add_argument("--json", action="store_true", help="emit machine-readable result")
+    args = parser.parse_args()
+    mismatches = compare_runs(args.left, args.right)
+    result = {"status": "mismatch" if mismatches else "match", "mismatches": mismatches}
+    if args.json:
+        print(json.dumps(result, indent=2, sort_keys=True))
+    elif mismatches:
+        print("normal-run compatibility mismatch:")
+        for mismatch in mismatches:
+            print(json.dumps(mismatch, sort_keys=True))
+    else:
+        print(f"normal-run compatibility match: {args.left} == {args.right}")
+    return 1 if mismatches else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/compat/verify_run_abi.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+RUN_DIR="${1:-}"
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  tests/compat/verify_run_abi.sh <successful-run-directory>
+USAGE
+}
+
+if [[ -z "$RUN_DIR" || "$RUN_DIR" == "-h" || "$RUN_DIR" == "--help" ]]; then
+  usage >&2
+  exit 2
+fi
+[[ -d "$RUN_DIR" ]] || { echo "missing run directory: $RUN_DIR" >&2; exit 2; }
+
+required_files=(
+  run-meta.json
+  run-summary.json
+  daemon-status.json
+  report.json
+  input.cir
+  result.ccr
+  vrps.csv
+  vaps.csv
+  validation-contract.json
+  stage-timing.json
+  stdout.log
+  stderr.log
+  process-time.txt
+)
+for name in "${required_files[@]}"; do
+  [[ -f "$RUN_DIR/$name" ]] || { echo "missing ABI file: $RUN_DIR/$name" >&2; exit 1; }
+done
+
+python3 - "$RUN_DIR" <<'PY'
+import csv
+import json
+import pathlib
+import sys
+
+run_dir = pathlib.Path(sys.argv[1])
+meta = json.loads((run_dir / "run-meta.json").read_text(encoding="utf-8"))
+summary = json.loads((run_dir / "run-summary.json").read_text(encoding="utf-8"))
+daemon = json.loads((run_dir / "daemon-status.json").read_text(encoding="utf-8"))
+contract = json.loads((run_dir / "validation-contract.json").read_text(encoding="utf-8"))
+report = json.loads((run_dir / "report.json").read_text(encoding="utf-8"))
+
+checks = {
+    "run-meta status": meta.get("status") == "success",
+    "run-meta id": meta.get("run_id") == run_dir.name,
+    "run-summary status": summary.get("status") == "success",
+    "run-summary exit": summary.get("exitCode") == 0,
+    "run-summary id": summary.get("runId") == run_dir.name,
+    "daemon state": daemon.get("state") == "exited",
+    "daemon last run": daemon.get("lastRunId") == run_dir.name,
+    "contract schema": contract.get("schemaVersion") == 1,
+    "report format": report.get("format_version") == 2,
+}
+failed = [name for name, passed in checks.items() if not passed]
+if failed:
+    raise SystemExit("ABI semantic checks failed: " + ", ".join(failed))
+
+expected_headers = {
+    "vrps.csv": ["ASN", "IP Prefix", "Max Length", "Trust Anchor"],
+    "vaps.csv": ["Customer ASN", "Providers", "Trust Anchor"],
+}
+for name, expected in expected_headers.items():
+    with (run_dir / name).open(newline="", encoding="utf-8") as handle:
+        actual = next(csv.reader(handle), [])
+    if actual != expected:
+        raise SystemExit(f"{name} header changed: expected={expected!r} actual={actual!r}")
+
+print(f"normal run ABI verified: {run_dir}")
+PY