159 lines
7.7 KiB
Python
159 lines
7.7 KiB
Python
"""Real CLI regression: HTTPS snapshot/delta, restart, interval, lock and signals."""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
|
|
|
|
def read(path):
|
|
return json.loads(path.read_text())
|
|
|
|
|
|
def wait_for(check, label, timeout=30):
|
|
start = time.monotonic()
|
|
while time.monotonic() - start < timeout:
|
|
if check():
|
|
return
|
|
time.sleep(0.05)
|
|
raise AssertionError(f"timed out waiting for {label}")
|
|
|
|
|
|
def state_is(root, state):
|
|
path = root / "daemon-status.json"
|
|
return path.exists() and read(path)["state"] == state
|
|
|
|
|
|
def run(binary):
|
|
support = Path(__file__).resolve().parent
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
base = Path(tmp)
|
|
fixture = base / "fixture"
|
|
with socket.socket() as sock:
|
|
sock.bind(("127.0.0.1", 0))
|
|
port = sock.getsockname()[1]
|
|
subprocess.run([sys.executable, str(support / "generate_repository.py"),
|
|
"--rrdp-host", "127.0.0.1", "--rrdp-port", str(port),
|
|
# Key generation is CPU-heavy under instrumented CI builds.
|
|
"--output", str(fixture)], check=True, timeout=120)
|
|
active = fixture / "active"
|
|
active.symlink_to("cases/baseline-v1/http")
|
|
server = subprocess.Popen([sys.executable, str(support / "serve_repository.py"),
|
|
str(active), str(port), str(fixture / "certs/rrdp-server.pem"),
|
|
str(fixture / "certs/rrdp-server.key")],
|
|
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True)
|
|
processes = []
|
|
logs = []
|
|
try:
|
|
assert server.stdout.readline().strip() == "ready"
|
|
child_args = ["--tal", str(fixture / "tal/custom.tal"), "--ta", str(fixture / "ta/custom-ta.cer"),
|
|
"--http-root-cert", str(fixture / "certs/rrdp-ca.pem"),
|
|
"--http-timeout-secs", "2", "--log-format", "json"]
|
|
|
|
def launch(root, options, inputs=None):
|
|
log = (base / f"controller-{len(logs)}.log").open("w")
|
|
logs.append(log)
|
|
proc = subprocess.Popen([binary, "daemon", "--state-root", str(root), *options,
|
|
"--", *(child_args if inputs is None else inputs)],
|
|
stdout=log, stderr=log)
|
|
processes.append(proc)
|
|
return proc
|
|
|
|
root = base / "normal"
|
|
proc = launch(root, ["--interval-secs", "2", "--max-runs", "2", "--retain-runs", "2"])
|
|
wait_for(lambda: state_is(root, "sleeping"), "snapshot completion")
|
|
first = read(root / "runs/run_000001/run-summary.json")
|
|
assert first["status"] == "success" and first["sync_mode"] == "snapshot", first
|
|
assert first["summary"]["vrps"] == 1, first
|
|
# Single controller even in the interval when RocksDB itself is closed.
|
|
rival = launch(root, ["--max-runs", "1"])
|
|
assert rival.wait(timeout=10) == 2
|
|
active.unlink()
|
|
active.symlink_to("cases/baseline-v2/http")
|
|
assert proc.wait(timeout=30) == 0
|
|
second = read(root / "runs/run_000002/run-summary.json")
|
|
assert second["status"] == "success" and second["sync_mode"] == "delta", second
|
|
assert second["summary"]["vrps"] == 2, second
|
|
timing = read(root / "runs/run_000002/stage-timing.json")
|
|
assert timing["analysis_counts"]["rrdp_delta_ops_applied_total"] > 0
|
|
from datetime import datetime
|
|
gap = (datetime.fromisoformat(second["started_at"].replace("Z", "+00:00")) -
|
|
datetime.fromisoformat(first["finished_at"].replace("Z", "+00:00"))).total_seconds()
|
|
assert gap >= 1.9, gap
|
|
for seq in (1, 2):
|
|
out = root / f"runs/run_{seq:06}"
|
|
assert len((out / "vrps.csv").read_text().splitlines()) - 1 == seq
|
|
subprocess.run(["openssl", "asn1parse", "-inform", "DER", "-in",
|
|
str(out / "result.ccr"), "-noout"], check=True)
|
|
# Restart appends sequence and keeps protocol state; retention only drops completed runs.
|
|
(root / "runs/operator-notes").mkdir()
|
|
(root / "runs/run_000003").mkdir() # Simulated interrupted run evidence.
|
|
restart = launch(root, ["--interval-secs", "0", "--max-runs", "1", "--retain-runs", "1"])
|
|
assert restart.wait(timeout=30) == 0
|
|
fourth = read(root / "runs/run_000004/run-summary.json")
|
|
assert fourth["sync_mode"] == "auto" and fourth["summary"]["vrps"] == 2
|
|
assert not (root / "runs/run_000001").exists()
|
|
assert (root / "runs/operator-notes").is_dir() and (root / "runs/run_000003").is_dir()
|
|
|
|
sleeper = launch(root, ["--interval-secs", "600"])
|
|
wait_for(lambda: state_is(root, "sleeping"), "daemon sleep")
|
|
sleeper.send_signal(signal.SIGTERM)
|
|
assert sleeper.wait(timeout=5) == 0
|
|
assert state_is(root, "exited")
|
|
|
|
# A real validator failure is recorded, retried, and returned as nonzero at max-runs.
|
|
failed = base / "failed"
|
|
bad_inputs = ["--tal", str(base / "missing.tal"), "--ta", str(base / "missing.cer")]
|
|
failure = launch(failed, ["--max-runs", "2", "--interval-secs", "0"], bad_inputs)
|
|
assert failure.wait(timeout=20) == 2
|
|
for seq in (1, 2):
|
|
assert read(failed / f"runs/run_{seq:06}/run-summary.json")["status"] == "failed"
|
|
|
|
# Suspend the HTTPS server so the actual validator cannot complete a request.
|
|
server.send_signal(signal.SIGSTOP)
|
|
hung = base / "timeout"
|
|
timeout = launch(hung, ["--max-runs", "1", "--run-timeout-secs", "1"])
|
|
assert timeout.wait(timeout=10) == 2
|
|
assert read(hung / "runs/run_000001/run-summary.json")["error"] == "timeout"
|
|
interrupted = base / "interrupted"
|
|
active_proc = launch(interrupted, ["--shutdown-grace-secs", "0"])
|
|
wait_for(lambda: state_is(interrupted, "running"), "active validator")
|
|
time.sleep(0.2)
|
|
active_proc.send_signal(signal.SIGTERM)
|
|
assert active_proc.wait(timeout=10) == 2
|
|
assert read(interrupted / "runs/run_000001/run-summary.json")["error"] == "interrupted"
|
|
server.send_signal(signal.SIGCONT)
|
|
recovered = launch(interrupted, ["--max-runs", "1", "--interval-secs", "0"])
|
|
assert recovered.wait(timeout=30) == 0
|
|
resumed = read(interrupted / "runs/run_000002/run-summary.json")
|
|
assert resumed["status"] == "success" and resumed["summary"]["vrps"] == 2
|
|
assert resumed["sync_mode"] in ("auto", "snapshot")
|
|
print("Real daemon E2E passed: snapshot/delta/restart/interval/retention/lock/failure/timeout/SIGTERM")
|
|
except BaseException:
|
|
for log in logs:
|
|
log.flush()
|
|
print(Path(log.name).read_text(), file=sys.stderr)
|
|
raise
|
|
finally:
|
|
server.send_signal(signal.SIGCONT)
|
|
for proc in processes:
|
|
if proc.poll() is None:
|
|
proc.terminate()
|
|
try:
|
|
proc.wait(timeout=35)
|
|
except subprocess.TimeoutExpired:
|
|
proc.kill()
|
|
proc.wait()
|
|
server.terminate()
|
|
server.wait(timeout=5)
|
|
for log in logs:
|
|
log.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(str(Path(sys.argv[1]).resolve()))
|