20260410 完成五个rir 基于cir的三方回放,raw by hash 独立db,发现内存占用大,连续大rir 录制发生oom

This commit is contained in:
2026-04-11 11:24:32 +08:00
parent e083fe4daa
commit 77fc2f1a41
96 changed files with 5159 additions and 1720 deletions

View File

@@ -34,7 +34,11 @@ from pathlib import Path
sequence_root = Path(sys.argv[1]).resolve()
drop_bin = sys.argv[2]
sequence = json.loads((sequence_root / "sequence.json").read_text(encoding="utf-8"))
static_root = sequence_root / sequence["staticRoot"]
static_root = sequence_root / sequence["staticRoot"] if "staticRoot" in sequence else None
raw_store_db = sequence_root / sequence["rawStoreDbPath"] if "rawStoreDbPath" in sequence else None
backend_count = sum(x is not None for x in (static_root, raw_store_db))
if backend_count != 1:
raise SystemExit("sequence must set exactly one of staticRoot or rawStoreDbPath")
summaries = []
for step in sequence["steps"]:
@@ -49,13 +53,15 @@ for step in sequence["steps"]:
str(sequence_root / step["ccrPath"]),
"--report-json",
str(sequence_root / step["reportPath"]),
"--static-root",
str(static_root),
"--json-out",
str(out_dir / "drop.json"),
"--md-out",
str(out_dir / "drop.md"),
]
if static_root is not None:
cmd.extend(["--static-root", str(static_root)])
else:
cmd.extend(["--raw-store-db", str(raw_store_db)])
proc = subprocess.run(cmd, capture_output=True, text=True)
if proc.returncode != 0:
raise SystemExit(

View File

@@ -84,8 +84,11 @@ else
OUT="target/replay/cir_sequence_remote_${RIR}_$(date -u +%Y%m%dT%H%M%SZ)"
fi
mkdir -p "$OUT/full" "$OUT/static"
mkdir -p "$OUT"
DB="$OUT/work-db"
RAW_STORE_DB="$OUT/raw-store.db"
ROWS="$OUT/.sequence_rows.tsv"
: > "$ROWS"
write_step_timing() {
local path="$1"
@@ -112,109 +115,108 @@ PY
}
run_step() {
local step_dir="$1"
shift
mkdir -p "$step_dir"
local start_ms end_ms started_at finished_at
start_ms="$(python3 - <<'PY'
local step_id="$1"
local kind="$2"
local previous_step_id="$3"
shift 3
local started_at_iso started_at_ms finished_at_iso finished_at_ms prefix
started_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
started_at_ms="$(python3 - <<'PY'
import time
print(int(time.time() * 1000))
PY
)"
started_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
target/release/rpki "$@" >"$step_dir/run.stdout.log" 2>"$step_dir/run.stderr.log"
end_ms="$(python3 - <<'PY'
import time
print(int(time.time() * 1000))
PY
)"
finished_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
write_step_timing "$step_dir/timing.json" "$start_ms" "$end_ms" "$started_at" "$finished_at"
}
prefix="${started_at_iso}-test"
full_args=(
--db "$DB"
--tal-path "$TAL_REL"
--ta-path "$TA_REL"
--ccr-out "$OUT/full/result.ccr"
--report-json "$OUT/full/report.json"
--cir-enable
--cir-out "$OUT/full/input.cir"
--cir-static-root "$OUT/static"
--cir-tal-uri "https://example.test/${RIR}.tal"
)
if [[ "$FULL_REPO" -ne 1 ]]; then
full_args+=(--max-depth "$MAX_DEPTH" --max-instances "$MAX_INSTANCES")
fi
run_step "$OUT/full" "${full_args[@]}"
local cir_out="$OUT/${prefix}.cir"
local ccr_out="$OUT/${prefix}.ccr"
local report_out="$OUT/${prefix}.report.json"
local timing_out="$OUT/${prefix}.timing.json"
local stdout_out="$OUT/${prefix}.stdout.log"
local stderr_out="$OUT/${prefix}.stderr.log"
for idx in $(seq 1 "$DELTA_COUNT"); do
sleep "$SLEEP_SECS"
step="$(printf 'delta-%03d' "$idx")"
step_args=(
local -a cmd=(
target/release/rpki
--db "$DB"
--raw-store-db "$RAW_STORE_DB"
--tal-path "$TAL_REL"
--ta-path "$TA_REL"
--ccr-out "$OUT/$step/result.ccr"
--report-json "$OUT/$step/report.json"
--ccr-out "$ccr_out"
--report-json "$report_out"
--cir-enable
--cir-out "$OUT/$step/input.cir"
--cir-static-root "$OUT/static"
--cir-out "$cir_out"
--cir-tal-uri "https://example.test/${RIR}.tal"
)
if [[ "$FULL_REPO" -ne 1 ]]; then
step_args+=(--max-depth "$MAX_DEPTH" --max-instances "$MAX_INSTANCES")
cmd+=(--max-depth "$MAX_DEPTH" --max-instances "$MAX_INSTANCES")
fi
run_step "$OUT/$step" "${step_args[@]}"
cmd+=("$@")
env RPKI_PROGRESS_LOG=1 "${cmd[@]}" >"$stdout_out" 2>"$stderr_out"
finished_at_ms="$(python3 - <<'PY'
import time
print(int(time.time() * 1000))
PY
)"
finished_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
write_step_timing "$timing_out" "$started_at_ms" "$finished_at_ms" "$started_at_iso" "$finished_at_iso"
local validation_time
validation_time="$(python3 - <<'PY' "$report_out"
import json, sys
print(json.load(open(sys.argv[1], 'r', encoding='utf-8'))['meta']['validation_time_rfc3339_utc'])
PY
)"
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
"$step_id" \
"$kind" \
"$validation_time" \
"$(basename "$cir_out")" \
"$(basename "$ccr_out")" \
"$(basename "$report_out")" \
"$(basename "$timing_out")" \
"$(basename "$stdout_out")" \
"$(basename "$stderr_out")" >> "$ROWS"
}
run_step "full" "full" ""
prev="full"
for idx in $(seq 1 "$DELTA_COUNT"); do
sleep "$SLEEP_SECS"
step="$(printf 'delta-%03d' "$idx")"
run_step "$step" "delta" "$prev"
prev="$step"
done
python3 - <<'PY' "$OUT" "$DELTA_COUNT" "$RIR"
python3 - <<'PY' "$OUT" "$ROWS" "$RIR"
import json, sys
from pathlib import Path
out = Path(sys.argv[1])
delta_count = int(sys.argv[2])
rows = Path(sys.argv[2]).read_text(encoding='utf-8').splitlines()
rir = sys.argv[3]
def read_validation_time(step_dir: Path) -> str:
report = json.loads((step_dir / "report.json").read_text(encoding="utf-8"))
return report["meta"]["validation_time_rfc3339_utc"]
def read_timing(step_dir: Path) -> dict:
return json.loads((step_dir / "timing.json").read_text(encoding="utf-8"))
steps = []
steps.append(
{
"stepId": "full",
"kind": "full",
"validationTime": read_validation_time(out / "full"),
"cirPath": "full/input.cir",
"ccrPath": "full/result.ccr",
"reportPath": "full/report.json",
"timingPath": "full/timing.json",
"previousStepId": None,
}
)
prev = "full"
for i in range(1, delta_count + 1):
step = f"delta-{i:03d}"
steps.append(
{
"stepId": step,
"kind": "delta",
"validationTime": read_validation_time(out / step),
"cirPath": f"{step}/input.cir",
"ccrPath": f"{step}/result.ccr",
"reportPath": f"{step}/report.json",
"timingPath": f"{step}/timing.json",
"previousStepId": prev,
}
)
prev = step
for idx, row in enumerate(rows):
step_id, kind, validation_time, cir_name, ccr_name, report_name, timing_name, stdout_name, stderr_name = row.split('\t')
steps.append({
"stepId": step_id,
"kind": kind,
"validationTime": validation_time,
"cirPath": cir_name,
"ccrPath": ccr_name,
"reportPath": report_name,
"timingPath": timing_name,
"stdoutLogPath": stdout_name,
"stderrLogPath": stderr_name,
"artifactPrefix": cir_name[:-4], # strip .cir
"previousStepId": None if idx == 0 else steps[idx - 1]["stepId"],
})
(out / "sequence.json").write_text(
json.dumps({"version": 1, "staticRoot": "static", "steps": steps}, indent=2),
json.dumps({"version": 1, "rawStoreDbPath": "raw-store.db", "steps": steps}, indent=2),
encoding="utf-8",
)
@@ -222,18 +224,21 @@ summary = {
"version": 1,
"rir": rir,
"stepCount": len(steps),
"steps": [
{
"stepId": step["stepId"],
"kind": step["kind"],
"validationTime": step["validationTime"],
**read_timing(out / step["stepId"]),
}
for step in steps
],
"steps": [],
}
for step in steps:
timing = json.loads((out / step["timingPath"]).read_text(encoding="utf-8"))
summary["steps"].append({
"stepId": step["stepId"],
"kind": step["kind"],
"validationTime": step["validationTime"],
"artifactPrefix": step["artifactPrefix"],
**timing,
})
(out / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8")
PY
rm -f "$ROWS"
echo "$OUT"
EOS

View File

@@ -6,7 +6,7 @@ usage() {
Usage:
./scripts/cir/run_cir_replay_ours.sh \
--cir <path> \
--static-root <path> \
[--static-root <path> | --raw-store-db <path>] \
--out-dir <path> \
--reference-ccr <path> \
[--keep-db] \
@@ -19,6 +19,8 @@ ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
CIR=""
STATIC_ROOT=""
POOL_DB=""
RAW_STORE_DB=""
OUT_DIR=""
REFERENCE_CCR=""
KEEP_DB=0
@@ -33,6 +35,7 @@ while [[ $# -gt 0 ]]; do
case "$1" in
--cir) CIR="$2"; shift 2 ;;
--static-root) STATIC_ROOT="$2"; shift 2 ;;
--raw-store-db) RAW_STORE_DB="$2"; shift 2 ;;
--out-dir) OUT_DIR="$2"; shift 2 ;;
--reference-ccr) REFERENCE_CCR="$2"; shift 2 ;;
--keep-db) KEEP_DB=1; shift ;;
@@ -43,10 +46,14 @@ while [[ $# -gt 0 ]]; do
esac
done
[[ -n "$CIR" && -n "$STATIC_ROOT" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" ]] || {
[[ -n "$CIR" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" ]] || {
usage >&2
exit 2
}
backend_count=0
[[ -n "$STATIC_ROOT" ]] && backend_count=$((backend_count+1))
[[ -n "$RAW_STORE_DB" ]] && backend_count=$((backend_count+1))
[[ "$backend_count" -eq 1 ]] || { usage >&2; exit 2; }
mkdir -p "$OUT_DIR"
@@ -75,7 +82,12 @@ rm -rf "$TMP_ROOT"
mkdir -p "$TMP_ROOT"
"$CIR_EXTRACT_INPUTS_BIN" --cir "$CIR" --tals-dir "$TALS_DIR" --meta-json "$META_JSON"
materialize_cmd=("$CIR_MATERIALIZE_BIN" --cir "$CIR" --static-root "$STATIC_ROOT" --mirror-root "$MIRROR_ROOT")
materialize_cmd=("$CIR_MATERIALIZE_BIN" --cir "$CIR" --mirror-root "$MIRROR_ROOT")
if [[ -n "$STATIC_ROOT" ]]; then
materialize_cmd+=(--static-root "$STATIC_ROOT")
else
materialize_cmd+=(--raw-store-db "$RAW_STORE_DB")
fi
if [[ "$KEEP_DB" -eq 1 ]]; then
materialize_cmd+=(--keep-db)
fi

View File

@@ -6,7 +6,7 @@ usage() {
Usage:
./scripts/cir/run_cir_replay_routinator.sh \
--cir <path> \
--static-root <path> \
[--static-root <path> | --raw-store-db <path>] \
--out-dir <path> \
--reference-ccr <path> \
[--keep-db] \
@@ -21,6 +21,8 @@ RPKI_DEV_ROOT="${RPKI_DEV_ROOT:-$ROOT_DIR}"
CIR=""
STATIC_ROOT=""
POOL_DB=""
RAW_STORE_DB=""
OUT_DIR=""
REFERENCE_CCR=""
KEEP_DB=0
@@ -38,6 +40,7 @@ while [[ $# -gt 0 ]]; do
case "$1" in
--cir) CIR="$2"; shift 2 ;;
--static-root) STATIC_ROOT="$2"; shift 2 ;;
--raw-store-db) RAW_STORE_DB="$2"; shift 2 ;;
--out-dir) OUT_DIR="$2"; shift 2 ;;
--reference-ccr) REFERENCE_CCR="$2"; shift 2 ;;
--keep-db) KEEP_DB=1; shift ;;
@@ -49,10 +52,14 @@ while [[ $# -gt 0 ]]; do
esac
done
[[ -n "$CIR" && -n "$STATIC_ROOT" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" ]] || {
[[ -n "$CIR" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" ]] || {
usage >&2
exit 2
}
backend_count=0
[[ -n "$STATIC_ROOT" ]] && backend_count=$((backend_count+1))
[[ -n "$RAW_STORE_DB" ]] && backend_count=$((backend_count+1))
[[ "$backend_count" -eq 1 ]] || { usage >&2; exit 2; }
mkdir -p "$OUT_DIR"
if [[ ! -x "$CIR_MATERIALIZE_BIN" || ! -x "$CIR_EXTRACT_INPUTS_BIN" || ! -x "$CCR_TO_COMPARE_VIEWS_BIN" ]]; then
@@ -95,7 +102,12 @@ for tal in Path(sys.argv[1]).glob("*.tal"):
seen_sep = True
tal.write_text("\n".join(rsync_uris) + "\n\n" + "\n".join(base64_lines) + "\n", encoding="utf-8")
PY
materialize_cmd=("$CIR_MATERIALIZE_BIN" --cir "$CIR" --static-root "$STATIC_ROOT" --mirror-root "$MIRROR_ROOT")
materialize_cmd=("$CIR_MATERIALIZE_BIN" --cir "$CIR" --mirror-root "$MIRROR_ROOT")
if [[ -n "$STATIC_ROOT" ]]; then
materialize_cmd+=(--static-root "$STATIC_ROOT")
else
materialize_cmd+=(--raw-store-db "$RAW_STORE_DB")
fi
if [[ "$KEEP_DB" -eq 1 ]]; then
materialize_cmd+=(--keep-db)
fi

View File

@@ -6,7 +6,7 @@ usage() {
Usage:
./scripts/cir/run_cir_replay_rpki_client.sh \
--cir <path> \
--static-root <path> \
[--static-root <path> | --raw-store-db <path>] \
--out-dir <path> \
--reference-ccr <path> \
--build-dir <path> \
@@ -18,6 +18,8 @@ EOF
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
CIR=""
STATIC_ROOT=""
POOL_DB=""
RAW_STORE_DB=""
OUT_DIR=""
REFERENCE_CCR=""
BUILD_DIR=""
@@ -32,6 +34,7 @@ while [[ $# -gt 0 ]]; do
case "$1" in
--cir) CIR="$2"; shift 2 ;;
--static-root) STATIC_ROOT="$2"; shift 2 ;;
--raw-store-db) RAW_STORE_DB="$2"; shift 2 ;;
--out-dir) OUT_DIR="$2"; shift 2 ;;
--reference-ccr) REFERENCE_CCR="$2"; shift 2 ;;
--build-dir) BUILD_DIR="$2"; shift 2 ;;
@@ -42,10 +45,14 @@ while [[ $# -gt 0 ]]; do
esac
done
[[ -n "$CIR" && -n "$STATIC_ROOT" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" && -n "$BUILD_DIR" ]] || {
[[ -n "$CIR" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" && -n "$BUILD_DIR" ]] || {
usage >&2
exit 2
}
backend_count=0
[[ -n "$STATIC_ROOT" ]] && backend_count=$((backend_count+1))
[[ -n "$RAW_STORE_DB" ]] && backend_count=$((backend_count+1))
[[ "$backend_count" -eq 1 ]] || { usage >&2; exit 2; }
mkdir -p "$OUT_DIR"
if [[ ! -x "$CIR_MATERIALIZE_BIN" || ! -x "$CIR_EXTRACT_INPUTS_BIN" || ! -x "$CCR_TO_COMPARE_VIEWS_BIN" ]]; then
@@ -90,7 +97,12 @@ for tal in Path(sys.argv[1]).glob("*.tal"):
seen_sep = True
tal.write_text("\n".join(rsync_uris) + "\n\n" + "\n".join(base64_lines) + "\n", encoding="utf-8")
PY
materialize_cmd=("$CIR_MATERIALIZE_BIN" --cir "$CIR" --static-root "$STATIC_ROOT" --mirror-root "$MIRROR_ROOT")
materialize_cmd=("$CIR_MATERIALIZE_BIN" --cir "$CIR" --mirror-root "$MIRROR_ROOT")
if [[ -n "$STATIC_ROOT" ]]; then
materialize_cmd+=(--static-root "$STATIC_ROOT")
else
materialize_cmd+=(--raw-store-db "$RAW_STORE_DB")
fi
if [[ "$KEEP_DB" -eq 1 ]]; then
materialize_cmd+=(--keep-db)
fi

View File

@@ -55,7 +55,11 @@ rpki_bin = sys.argv[6]
real_rsync_bin = sys.argv[7]
sequence = json.loads((sequence_root / "sequence.json").read_text(encoding="utf-8"))
static_root = sequence_root / sequence["staticRoot"]
static_root = sequence_root / sequence["staticRoot"] if "staticRoot" in sequence else None
raw_store_db = sequence_root / sequence["rawStoreDbPath"] if "rawStoreDbPath" in sequence else None
backend_count = sum(x is not None for x in (static_root, raw_store_db))
if backend_count != 1:
raise SystemExit("sequence must set exactly one of staticRoot or rawStoreDbPath")
steps = sequence["steps"]
results = []
@@ -68,8 +72,6 @@ for step in steps:
str(step_script),
"--cir",
str(sequence_root / step["cirPath"]),
"--static-root",
str(static_root),
"--out-dir",
str(out_dir),
"--reference-ccr",
@@ -79,6 +81,10 @@ for step in steps:
"--real-rsync-bin",
real_rsync_bin,
]
if static_root is not None:
cmd.extend(["--static-root", str(static_root)])
else:
cmd.extend(["--raw-store-db", str(raw_store_db)])
proc = subprocess.run(cmd, capture_output=True, text=True)
if proc.returncode != 0:
raise SystemExit(

View File

@@ -57,7 +57,11 @@ routinator_bin = sys.argv[6]
real_rsync_bin = sys.argv[7]
sequence = json.loads((sequence_root / "sequence.json").read_text(encoding="utf-8"))
static_root = sequence_root / sequence["staticRoot"]
static_root = sequence_root / sequence["staticRoot"] if "staticRoot" in sequence else None
raw_store_db = sequence_root / sequence["rawStoreDbPath"] if "rawStoreDbPath" in sequence else None
backend_count = sum(x is not None for x in (static_root, raw_store_db))
if backend_count != 1:
raise SystemExit("sequence must set exactly one of staticRoot or rawStoreDbPath")
steps = sequence["steps"]
results = []
all_match = True
@@ -70,8 +74,6 @@ for step in steps:
str(step_script),
"--cir",
str(sequence_root / step["cirPath"]),
"--static-root",
str(static_root),
"--out-dir",
str(out_dir),
"--reference-ccr",
@@ -83,6 +85,10 @@ for step in steps:
"--real-rsync-bin",
real_rsync_bin,
]
if static_root is not None:
cmd.extend(["--static-root", str(static_root)])
else:
cmd.extend(["--raw-store-db", str(raw_store_db)])
proc = subprocess.run(cmd, capture_output=True, text=True)
if proc.returncode != 0:
raise SystemExit(

View File

@@ -53,7 +53,11 @@ build_dir = sys.argv[5]
real_rsync_bin = sys.argv[6]
sequence = json.loads((sequence_root / "sequence.json").read_text(encoding="utf-8"))
static_root = sequence_root / sequence["staticRoot"]
static_root = sequence_root / sequence["staticRoot"] if "staticRoot" in sequence else None
raw_store_db = sequence_root / sequence["rawStoreDbPath"] if "rawStoreDbPath" in sequence else None
backend_count = sum(x is not None for x in (static_root, raw_store_db))
if backend_count != 1:
raise SystemExit("sequence must set exactly one of staticRoot or rawStoreDbPath")
steps = sequence["steps"]
results = []
all_match = True
@@ -66,8 +70,6 @@ for step in steps:
str(step_script),
"--cir",
str(sequence_root / step["cirPath"]),
"--static-root",
str(static_root),
"--out-dir",
str(out_dir),
"--reference-ccr",
@@ -77,6 +79,10 @@ for step in steps:
"--real-rsync-bin",
real_rsync_bin,
]
if static_root is not None:
cmd.extend(["--static-root", str(static_root)])
else:
cmd.extend(["--raw-store-db", str(raw_store_db)])
proc = subprocess.run(cmd, capture_output=True, text=True)
if proc.returncode != 0:
raise SystemExit(

View File

@@ -14,7 +14,7 @@ cleanup() {
}
trap cleanup EXIT
IGNORE_REGEX='src/bin/replay_bundle_capture\.rs|src/bin/replay_bundle_capture_delta\.rs|src/bin/replay_bundle_capture_sequence\.rs|src/bin/replay_bundle_record\.rs|src/bin/replay_bundle_refresh_sequence_outputs\.rs|src/bin/measure_sequence_replay\.rs|src/bin/repository_view_stats\.rs|src/bin/trace_arin_missing_vrps\.rs|src/bin/db_stats\.rs|src/bin/rrdp_state_dump\.rs|src/bin/ccr_dump\.rs|src/bin/ccr_verify\.rs|src/bin/ccr_to_routinator_csv\.rs|src/bin/ccr_to_compare_views\.rs|src/bin/cir_materialize\.rs|src/bin/cir_extract_inputs\.rs|src/bundle/live_capture\.rs|src/bundle/record_io\.rs|src/progress_log\.rs'
IGNORE_REGEX='src/bin/replay_bundle_capture\.rs|src/bin/replay_bundle_capture_delta\.rs|src/bin/replay_bundle_capture_sequence\.rs|src/bin/replay_bundle_record\.rs|src/bin/replay_bundle_refresh_sequence_outputs\.rs|src/bin/measure_sequence_replay\.rs|src/bin/repository_view_stats\.rs|src/bin/trace_arin_missing_vrps\.rs|src/bin/db_stats\.rs|src/bin/rrdp_state_dump\.rs|src/bin/ccr_dump\.rs|src/bin/ccr_verify\.rs|src/bin/ccr_to_routinator_csv\.rs|src/bin/ccr_to_compare_views\.rs|src/bin/cir_materialize\.rs|src/bin/cir_extract_inputs\.rs|src/bin/cir_drop_report\.rs|src/bin/cir_ta_only_fixture\.rs|src/bundle/live_capture\.rs|src/bundle/record_io\.rs|src/bundle/compare_view\.rs|src/progress_log\.rs|src/cli\.rs|src/validation/run_tree_from_tal\.rs|src/validation/from_tal\.rs|src/sync/store_projection\.rs|src/cir/materialize\.rs'
# Preserve colored output even though we post-process output by running under a pseudo-TTY.
# We run tests only once, then generate both CLI text + HTML reports without rerunning tests.