240 lines
8.2 KiB
Python
Executable File
240 lines
8.2 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Compare two successful normal-run directories.
|
|
|
|
The validator intentionally emits lifecycle and resource telemetry that is
|
|
expected to vary between processes. This comparator keeps those fields out
|
|
of the semantic comparison while normalising only the exceptions declared in
|
|
``baseline-manifest.toml``: lifecycle wall-clock values, absolute paths, and
|
|
binary identity. The validation time is an input and must match exactly.
|
|
Payloads (CIR/CCR/VRP/VAP) remain byte-for-byte checked, with only the CCR
|
|
``producedAt`` timestamp canonicalised.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
REQUIRED = (
|
|
"run-meta.json",
|
|
"run-summary.json",
|
|
"daemon-status.json",
|
|
"report.json",
|
|
"input.cir",
|
|
"result.ccr",
|
|
"vrps.csv",
|
|
"vaps.csv",
|
|
"validation-contract.json",
|
|
)
|
|
PAYLOADS = ("input.cir", "result.ccr", "vrps.csv", "vaps.csv")
|
|
RFC3339_TEXT = re.compile(
|
|
r"20\d\d-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d+)?(?:Z|[+-]\d\d:\d\d)"
|
|
)
|
|
GENERALIZED_TIME = re.compile(rb"20\d{12}Z")
|
|
INNER_RUN_ID = re.compile(r"\d{6}-20\d{6,8}T\d{6}Z")
|
|
ABSOLUTE_PATH = re.compile(r"(?<![A-Za-z0-9_:/])/(?:[^\s,\"']+/)*[^\s,\"']+")
|
|
|
|
|
|
def load_json(run_dir: Path, name: str) -> Any:
|
|
try:
|
|
return json.loads((run_dir / name).read_text(encoding="utf-8"))
|
|
except (OSError, json.JSONDecodeError) as error:
|
|
raise ValueError(f"{run_dir / name}: {error}") from error
|
|
|
|
|
|
def normalise_scalar(value: Any, *, key: str = "") -> Any:
|
|
lowered_key = key.lower()
|
|
if (
|
|
lowered_key.endswith("_ms")
|
|
or lowered_key.endswith("_nanos")
|
|
or "duration" in lowered_key
|
|
or lowered_key in {"eventssha256", "events_sha256"}
|
|
):
|
|
return "<runtime-measurement>"
|
|
if not isinstance(value, str):
|
|
return value
|
|
if key.lower() in {
|
|
"binarysha256",
|
|
"binary_sha256",
|
|
"sourcecommit",
|
|
"source_commit",
|
|
}:
|
|
return "<declared-provenance>"
|
|
wall_clock_keys = {
|
|
"started_at_rfc3339_utc",
|
|
"completed_at_rfc3339_utc",
|
|
"finished_at_rfc3339_utc",
|
|
"updated_at_rfc3339_utc",
|
|
"startedatrfc3339utc",
|
|
"completedatrfc3339utc",
|
|
"finishedatrfc3339utc",
|
|
"updatedatrfc3339utc",
|
|
"started_at",
|
|
"completed_at",
|
|
"finished_at",
|
|
"updated_at",
|
|
"timestamp",
|
|
}
|
|
if lowered_key in wall_clock_keys:
|
|
return "<wall-clock>"
|
|
value = INNER_RUN_ID.sub("<inner-run-id>", value)
|
|
return ABSOLUTE_PATH.sub("<absolute-path>", value)
|
|
|
|
|
|
def normalise_text(value: str) -> str:
|
|
"""Normalise a free-form event/log stream without changing payload fields."""
|
|
|
|
value = RFC3339_TEXT.sub("<wall-clock>", value)
|
|
value = INNER_RUN_ID.sub("<inner-run-id>", value)
|
|
return ABSOLUTE_PATH.sub("<absolute-path>", value)
|
|
|
|
|
|
def normalise_json(value: Any, *, key: str = "") -> Any:
|
|
if isinstance(value, dict):
|
|
return {name: normalise_json(item, key=name) for name, item in value.items()}
|
|
if isinstance(value, list):
|
|
return [normalise_json(item, key=key) for item in value]
|
|
return normalise_scalar(value, key=key)
|
|
|
|
|
|
def canonical_payload(name: str, data: bytes) -> bytes:
|
|
if name in {"input.cir", "result.ccr"}:
|
|
# CCR's first GeneralizedTime is producedAt. CIR's first time is the
|
|
# validation time and therefore remains an input equality check.
|
|
if name == "result.ccr":
|
|
data = GENERALIZED_TIME.sub(b"<wall-clock>", data, count=1)
|
|
if name in {"vrps.csv", "vaps.csv"}:
|
|
# Parse and re-emit CSV to avoid line-ending differences while keeping
|
|
# column order and row order observable.
|
|
rows = list(csv.reader(data.decode("utf-8").splitlines()))
|
|
data = "\n".join(",".join(row) for row in rows).encode("utf-8")
|
|
return data
|
|
|
|
|
|
def summary_projection(summary: dict[str, Any]) -> dict[str, Any]:
|
|
artifact_sizes: dict[str, int] = {}
|
|
for artifact in summary.get("artifacts", []):
|
|
path = Path(str(artifact.get("path", "")))
|
|
if path.name in PAYLOADS:
|
|
artifact_sizes[path.name] = int(artifact.get("sizeBytes", -1))
|
|
return {
|
|
"status": summary.get("status"),
|
|
"exitCode": summary.get("exitCode"),
|
|
"exitStatus": summary.get("exitStatus"),
|
|
"runId": summary.get("runId"),
|
|
"runSeq": summary.get("runSeq"),
|
|
"error": summary.get("error"),
|
|
"reportCounts": summary.get("reportCounts"),
|
|
"repoSyncStats": summary.get("repoSyncStats"),
|
|
"artifactSizes": artifact_sizes,
|
|
}
|
|
|
|
|
|
def daemon_projection(status: dict[str, Any]) -> dict[str, Any]:
|
|
return {
|
|
name: status.get(name)
|
|
for name in (
|
|
"state",
|
|
"lastRunId",
|
|
"maxRuns",
|
|
"outerRunId",
|
|
"outerRunIndex",
|
|
"runsCompleted",
|
|
)
|
|
}
|
|
|
|
|
|
def compare_json(
|
|
name: str, left: Path, right: Path, mismatches: list[dict[str, Any]]
|
|
) -> None:
|
|
left_value = load_json(left, name)
|
|
right_value = load_json(right, name)
|
|
if name == "run-summary.json":
|
|
left_value = summary_projection(left_value)
|
|
right_value = summary_projection(right_value)
|
|
elif name == "daemon-status.json":
|
|
left_value = daemon_projection(left_value)
|
|
right_value = daemon_projection(right_value)
|
|
else:
|
|
left_value = normalise_json(left_value)
|
|
right_value = normalise_json(right_value)
|
|
if left_value != right_value:
|
|
mismatches.append({"file": name, "left": left_value, "right": right_value})
|
|
|
|
|
|
def compare_runs(left: Path, right: Path) -> list[dict[str, Any]]:
|
|
mismatches: list[dict[str, Any]] = []
|
|
for run_dir in (left, right):
|
|
if not run_dir.is_dir():
|
|
mismatches.append({"file": str(run_dir), "error": "run directory missing"})
|
|
continue
|
|
for name in REQUIRED:
|
|
if not (run_dir / name).is_file():
|
|
mismatches.append({"file": str(run_dir / name), "error": "required file missing"})
|
|
if mismatches:
|
|
return mismatches
|
|
|
|
for name in (
|
|
"run-meta.json",
|
|
"run-summary.json",
|
|
"daemon-status.json",
|
|
"report.json",
|
|
"validation-contract.json",
|
|
):
|
|
try:
|
|
compare_json(name, left, right, mismatches)
|
|
except ValueError as error:
|
|
mismatches.append({"file": name, "error": str(error)})
|
|
|
|
for name in PAYLOADS:
|
|
left_data = canonical_payload(name, (left / name).read_bytes())
|
|
right_data = canonical_payload(name, (right / name).read_bytes())
|
|
if left_data != right_data:
|
|
mismatches.append(
|
|
{
|
|
"file": name,
|
|
"leftSha256": hashlib.sha256(left_data).hexdigest(),
|
|
"rightSha256": hashlib.sha256(right_data).hexdigest(),
|
|
}
|
|
)
|
|
|
|
for name in ("validation-events.jsonl",):
|
|
left_file = left / name
|
|
right_file = right / name
|
|
if left_file.is_file() and right_file.is_file():
|
|
left_data = normalise_text(left_file.read_text(encoding="utf-8"))
|
|
right_data = normalise_text(right_file.read_text(encoding="utf-8"))
|
|
if left_data != right_data:
|
|
mismatches.append({"file": name, "error": "canonical event stream differs"})
|
|
return mismatches
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("left", type=Path, help="original rpki run directory")
|
|
parser.add_argument("right", type=Path, help="panda-rpki-validator run directory")
|
|
parser.add_argument("--json", action="store_true", help="emit machine-readable result")
|
|
args = parser.parse_args()
|
|
mismatches = compare_runs(args.left, args.right)
|
|
result = {"status": "mismatch" if mismatches else "match", "mismatches": mismatches}
|
|
if args.json:
|
|
print(json.dumps(result, indent=2, sort_keys=True))
|
|
elif mismatches:
|
|
print("normal-run compatibility mismatch:")
|
|
for mismatch in mismatches:
|
|
print(json.dumps(mismatch, sort_keys=True))
|
|
else:
|
|
print(f"normal-run compatibility match: {args.left} == {args.right}")
|
|
return 1 if mismatches else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|