20260401 live recorder 扩展到1+N个delta，完成5个RIR录制，以及三方replay，结果三方均对齐vrps和vaps

2026-04-03 16:44:27 +08:00
parent 6edc420ce2
commit 34fb9657f1
27 changed files with 4641 additions and 240 deletions
--- a/scripts/coverage.sh
+++ b/scripts/coverage.sh
@@ -14,7 +14,7 @@ cleanup() {
 }
 trap cleanup EXIT

-IGNORE_REGEX='src/bin/replay_bundle_capture\.rs|src/bin/replay_bundle_capture_delta\.rs|src/bundle/live_capture\.rs'
+IGNORE_REGEX='src/bin/replay_bundle_capture\.rs|src/bin/replay_bundle_capture_delta\.rs|src/bin/replay_bundle_capture_sequence\.rs|src/bundle/live_capture\.rs'

 # Preserve colored output even though we post-process output by running under a pseudo-TTY.
 # We run tests only once, then generate both CLI text + HTML reports without rerunning tests.
--- a/scripts/replay_bundle/README.md
+++ b/scripts/replay_bundle/README.md
@@ -59,6 +59,38 @@ target/replay/live_bundle_matrix_<timestamp>
 target/replay/live_bundle_matrix_<timestamp>/<rir>_live_bundle_<timestamp>
 ```

+如果要录制单个 RIR 的 `1 base + N delta` 序列，使用：
+
+```bash
+cd rpki
+./scripts/replay_bundle/run_live_bundle_record_sequence.sh \
+  --rir apnic \
+  --tal-path tests/fixtures/tal/apnic-rfc7730-https.tal \
+  --ta-path tests/fixtures/ta/apnic-ta.cer \
+  --delta-count 2 \
+  --delta-interval-secs 0
+```
+
+默认输出目录：
+
+```text
+target/replay/<rir>_live_bundle_sequence_<timestamp>
+```
+
+如果要一次录制多个 RIR 的 `1 base + N delta` 序列，使用：
+
+```bash
+cd rpki
+./scripts/replay_bundle/run_live_bundle_record_multi_rir_sequence.sh \
+  --rir afrinic,apnic,arin,lacnic,ripe
+```
+
+默认输出目录：
+
+```text
+target/replay/live_bundle_sequence_matrix_<timestamp>
+```
+
 ## 可选参数

 - `--out-dir <path>`
@@ -72,6 +104,9 @@ target/replay/live_bundle_matrix_<timestamp>/<rir>_live_bundle_<timestamp>
 - `--trust-anchor <name>`
 - `--bin-dir <path>`
 - `--no-build`
+- `--delta-count <n>`（sequence 入口）
+- `--delta-interval-secs <n>`（sequence 入口）
+- `--keep-db`（sequence 入口）

 `run_live_bundle_record_multi_rir.sh` 会自动按 RIR 选择当前仓库内置的：

--- a/scripts/replay_bundle/run_live_bundle_record_multi_rir_sequence.sh
+++ b/scripts/replay_bundle/run_live_bundle_record_multi_rir_sequence.sh
@@ -0,0 +1,173 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "$ROOT_DIR"
+
+RIRS=""
+OUT_ROOT=""
+BASE_VALIDATION_TIME=""
+DELTA_COUNT=""
+DELTA_INTERVAL_SECS=""
+HTTP_TIMEOUT_SECS=""
+RSYNC_TIMEOUT_SECS=""
+RSYNC_MIRROR_ROOT=""
+MAX_DEPTH=""
+MAX_INSTANCES=""
+NO_BUILD=0
+KEEP_DB=0
+CAPTURE_INPUTS_ONLY=0
+BIN_DIR="target/release"
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/replay_bundle/run_live_bundle_record_multi_rir_sequence.sh \
+    --rir <afrinic,apnic,...> \
+    [--out-root <path>] \
+    [--base-validation-time <rfc3339>] \
+    [--delta-count <n>] \
+    [--delta-interval-secs <n>] \
+    [--http-timeout-secs <n>] \
+    [--rsync-timeout-secs <n>] \
+    [--rsync-mirror-root <path>] \
+    [--max-depth <n>] \
+    [--max-instances <n>] \
+    [--bin-dir <path>] \
+    [--no-build] \
+    [--keep-db] \
+    [--capture-inputs-only]
+EOF
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --rir) RIRS="${2:?}"; shift 2 ;;
+    --out-root) OUT_ROOT="${2:?}"; shift 2 ;;
+    --base-validation-time) BASE_VALIDATION_TIME="${2:?}"; shift 2 ;;
+    --delta-count) DELTA_COUNT="${2:?}"; shift 2 ;;
+    --delta-interval-secs) DELTA_INTERVAL_SECS="${2:?}"; shift 2 ;;
+    --http-timeout-secs) HTTP_TIMEOUT_SECS="${2:?}"; shift 2 ;;
+    --rsync-timeout-secs) RSYNC_TIMEOUT_SECS="${2:?}"; shift 2 ;;
+    --rsync-mirror-root) RSYNC_MIRROR_ROOT="${2:?}"; shift 2 ;;
+    --max-depth) MAX_DEPTH="${2:?}"; shift 2 ;;
+    --max-instances) MAX_INSTANCES="${2:?}"; shift 2 ;;
+    --bin-dir) BIN_DIR="${2:?}"; shift 2 ;;
+    --no-build) NO_BUILD=1; shift ;;
+    --keep-db) KEEP_DB=1; shift ;;
+    --capture-inputs-only) CAPTURE_INPUTS_ONLY=1; shift ;;
+    --help|-h) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+if [[ -z "$RIRS" ]]; then
+  usage >&2
+  exit 2
+fi
+
+RUN_TAG="$(date -u +%Y%m%dT%H%M%SZ)"
+if [[ -z "$OUT_ROOT" ]]; then
+  OUT_ROOT="target/replay/live_bundle_sequence_matrix_${RUN_TAG}"
+fi
+mkdir -p "$OUT_ROOT"
+
+resolve_tal_path() {
+  case "$1" in
+    afrinic) printf 'tests/fixtures/tal/afrinic.tal' ;;
+    apnic) printf 'tests/fixtures/tal/apnic-rfc7730-https.tal' ;;
+    arin) printf 'tests/fixtures/tal/arin.tal' ;;
+    lacnic) printf 'tests/fixtures/tal/lacnic.tal' ;;
+    ripe) printf 'tests/fixtures/tal/ripe-ncc.tal' ;;
+    *) echo "unsupported rir: $1" >&2; exit 2 ;;
+  esac
+}
+
+resolve_ta_path() {
+  case "$1" in
+    afrinic) printf 'tests/fixtures/ta/afrinic-ta.cer' ;;
+    apnic) printf 'tests/fixtures/ta/apnic-ta.cer' ;;
+    arin) printf 'tests/fixtures/ta/arin-ta.cer' ;;
+    lacnic) printf 'tests/fixtures/ta/lacnic-ta.cer' ;;
+    ripe) printf 'tests/fixtures/ta/ripe-ncc-ta.cer' ;;
+    *) echo "unsupported rir: $1" >&2; exit 2 ;;
+  esac
+}
+
+SUMMARY_JSON="$OUT_ROOT/summary.json"
+SUMMARY_MD="$OUT_ROOT/summary.md"
+python3 - "$SUMMARY_JSON" "$RUN_TAG" <<'PY'
+import json, sys
+path, run_tag = sys.argv[1:]
+with open(path, "w") as fh:
+    json.dump({"runTag": run_tag, "results": []}, fh, indent=2)
+PY
+
+IFS=',' read -r -a RIR_LIST <<< "$RIRS"
+for raw_rir in "${RIR_LIST[@]}"; do
+  rir="$(printf '%s' "$raw_rir" | tr '[:upper:]' '[:lower:]' | xargs)"
+  [[ -n "$rir" ]] || continue
+  tal_path="$(resolve_tal_path "$rir")"
+  ta_path="$(resolve_ta_path "$rir")"
+  out_dir="$OUT_ROOT/${rir}_live_bundle_sequence_${RUN_TAG}"
+  cmd=(
+    ./scripts/replay_bundle/run_live_bundle_record_sequence.sh
+    --rir "$rir"
+    --out-dir "$out_dir"
+    --tal-path "$tal_path"
+    --ta-path "$ta_path"
+    --trust-anchor "$rir"
+    --bin-dir "$BIN_DIR"
+  )
+  [[ -n "$BASE_VALIDATION_TIME" ]] && cmd+=(--base-validation-time "$BASE_VALIDATION_TIME")
+  [[ -n "$DELTA_COUNT" ]] && cmd+=(--delta-count "$DELTA_COUNT")
+  [[ -n "$DELTA_INTERVAL_SECS" ]] && cmd+=(--delta-interval-secs "$DELTA_INTERVAL_SECS")
+  [[ -n "$HTTP_TIMEOUT_SECS" ]] && cmd+=(--http-timeout-secs "$HTTP_TIMEOUT_SECS")
+  [[ -n "$RSYNC_TIMEOUT_SECS" ]] && cmd+=(--rsync-timeout-secs "$RSYNC_TIMEOUT_SECS")
+  [[ -n "$RSYNC_MIRROR_ROOT" ]] && cmd+=(--rsync-mirror-root "$RSYNC_MIRROR_ROOT")
+  [[ -n "$MAX_DEPTH" ]] && cmd+=(--max-depth "$MAX_DEPTH")
+  [[ -n "$MAX_INSTANCES" ]] && cmd+=(--max-instances "$MAX_INSTANCES")
+  [[ "$NO_BUILD" -eq 1 ]] && cmd+=(--no-build)
+  [[ "$KEEP_DB" -eq 1 ]] && cmd+=(--keep-db)
+  [[ "$CAPTURE_INPUTS_ONLY" -eq 1 ]] && cmd+=(--capture-inputs-only)
+  "${cmd[@]}"
+
+  python3 - "$SUMMARY_JSON" "$rir" "$out_dir" <<'PY'
+import json, pathlib, sys
+summary_path, rir, out_dir = sys.argv[1:]
+summary = json.loads(pathlib.Path(summary_path).read_text())
+bundle = json.loads(pathlib.Path(out_dir, rir, "bundle.json").read_text())
+verification = json.loads(pathlib.Path(out_dir, rir, "verification.json").read_text())
+summary["results"].append({
+    "rir": rir,
+    "outDir": out_dir,
+    "stepCount": len(bundle["deltaSequence"]["steps"]),
+    "baseVrpCount": bundle["base"]["vrpCount"],
+    "baseVapCount": bundle["base"]["vapCount"],
+    "allStepsSelfReplayOk": verification["summary"]["allStepsSelfReplayOk"],
+})
+pathlib.Path(summary_path).write_text(json.dumps(summary, indent=2))
+PY
+done
+
+python3 - "$SUMMARY_JSON" "$SUMMARY_MD" <<'PY'
+import json, pathlib, sys
+summary = json.loads(pathlib.Path(sys.argv[1]).read_text())
+out = pathlib.Path(sys.argv[2])
+lines = [
+    "# Multi-RIR Live Bundle Sequence Summary",
+    "",
+    f"- runTag: `{summary['runTag']}`",
+    "",
+    "| rir | step_count | base_vrps | base_vaps | all_steps_self_replay | out_dir |",
+    "|---|---:|---:|---:|---|---|",
+]
+for item in summary["results"]:
+    lines.append(
+        f"| {item['rir']} | {item['stepCount']} | {item['baseVrpCount']} | {item['baseVapCount']} | "
+        f"{str(item['allStepsSelfReplayOk']).lower()} | `{item['outDir']}` |"
+    )
+out.write_text("\n".join(lines) + "\n")
+PY
+
+echo "$OUT_ROOT"
--- a/scripts/replay_bundle/run_live_bundle_record_sequence.sh
+++ b/scripts/replay_bundle/run_live_bundle_record_sequence.sh
@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "$ROOT_DIR"
+
+RIR=""
+OUT_DIR=""
+TAL_PATH=""
+TA_PATH=""
+BASE_VALIDATION_TIME=""
+DELTA_COUNT=""
+DELTA_INTERVAL_SECS=""
+HTTP_TIMEOUT_SECS=""
+RSYNC_TIMEOUT_SECS=""
+RSYNC_MIRROR_ROOT=""
+MAX_DEPTH=""
+MAX_INSTANCES=""
+TRUST_ANCHOR=""
+NO_BUILD=0
+KEEP_DB=0
+CAPTURE_INPUTS_ONLY=0
+BIN_DIR="target/release"
+PROGRESS_LOG="${RPKI_PROGRESS_LOG:-1}"
+PROGRESS_SLOW_SECS="${RPKI_PROGRESS_SLOW_SECS:-30}"
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/replay_bundle/run_live_bundle_record_sequence.sh \
+    --rir <name> \
+    --tal-path <path> \
+    --ta-path <path> \
+    [--out-dir <path>] \
+    [--base-validation-time <rfc3339>] \
+    [--delta-count <n>] \
+    [--delta-interval-secs <n>] \
+    [--http-timeout-secs <n>] \
+    [--rsync-timeout-secs <n>] \
+    [--rsync-mirror-root <path>] \
+    [--max-depth <n>] \
+    [--max-instances <n>] \
+    [--trust-anchor <name>] \
+    [--bin-dir <path>] \
+    [--no-build] \
+    [--keep-db] \
+    [--capture-inputs-only]
+EOF
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --rir) RIR="${2:?}"; shift 2 ;;
+    --out-dir) OUT_DIR="${2:?}"; shift 2 ;;
+    --tal-path) TAL_PATH="${2:?}"; shift 2 ;;
+    --ta-path) TA_PATH="${2:?}"; shift 2 ;;
+    --base-validation-time) BASE_VALIDATION_TIME="${2:?}"; shift 2 ;;
+    --delta-count) DELTA_COUNT="${2:?}"; shift 2 ;;
+    --delta-interval-secs) DELTA_INTERVAL_SECS="${2:?}"; shift 2 ;;
+    --http-timeout-secs) HTTP_TIMEOUT_SECS="${2:?}"; shift 2 ;;
+    --rsync-timeout-secs) RSYNC_TIMEOUT_SECS="${2:?}"; shift 2 ;;
+    --rsync-mirror-root) RSYNC_MIRROR_ROOT="${2:?}"; shift 2 ;;
+    --max-depth) MAX_DEPTH="${2:?}"; shift 2 ;;
+    --max-instances) MAX_INSTANCES="${2:?}"; shift 2 ;;
+    --trust-anchor) TRUST_ANCHOR="${2:?}"; shift 2 ;;
+    --bin-dir) BIN_DIR="${2:?}"; shift 2 ;;
+    --no-build) NO_BUILD=1; shift ;;
+    --keep-db) KEEP_DB=1; shift ;;
+    --capture-inputs-only) CAPTURE_INPUTS_ONLY=1; shift ;;
+    --help|-h) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+if [[ -z "$RIR" || -z "$TAL_PATH" || -z "$TA_PATH" ]]; then
+  usage >&2
+  exit 2
+fi
+
+TS="$(date -u +%Y%m%dT%H%M%SZ)"
+if [[ -z "$OUT_DIR" ]]; then
+  OUT_DIR="target/replay/${RIR}_live_bundle_sequence_${TS}"
+fi
+
+SEQUENCE_BIN="$BIN_DIR/replay_bundle_capture_sequence"
+if [[ "$NO_BUILD" -eq 0 ]]; then
+  echo "[1/1] build release binary"
+  cargo build --release --bin replay_bundle_capture_sequence
+else
+  echo "[1/1] reuse existing binary from $BIN_DIR"
+fi
+
+if [[ ! -x "$SEQUENCE_BIN" ]]; then
+  echo "missing executable: $SEQUENCE_BIN" >&2
+  exit 1
+fi
+
+cmd=(
+  "$SEQUENCE_BIN"
+  --rir "$RIR"
+  --out-dir "$OUT_DIR"
+  --tal-path "$TAL_PATH"
+  --ta-path "$TA_PATH"
+)
+[[ -n "$BASE_VALIDATION_TIME" ]] && cmd+=(--base-validation-time "$BASE_VALIDATION_TIME")
+[[ -n "$DELTA_COUNT" ]] && cmd+=(--delta-count "$DELTA_COUNT")
+[[ -n "$DELTA_INTERVAL_SECS" ]] && cmd+=(--delta-interval-secs "$DELTA_INTERVAL_SECS")
+[[ -n "$HTTP_TIMEOUT_SECS" ]] && cmd+=(--http-timeout-secs "$HTTP_TIMEOUT_SECS")
+[[ -n "$RSYNC_TIMEOUT_SECS" ]] && cmd+=(--rsync-timeout-secs "$RSYNC_TIMEOUT_SECS")
+[[ -n "$RSYNC_MIRROR_ROOT" ]] && cmd+=(--rsync-mirror-root "$RSYNC_MIRROR_ROOT")
+[[ -n "$MAX_DEPTH" ]] && cmd+=(--max-depth "$MAX_DEPTH")
+[[ -n "$MAX_INSTANCES" ]] && cmd+=(--max-instances "$MAX_INSTANCES")
+[[ -n "$TRUST_ANCHOR" ]] && cmd+=(--trust-anchor "$TRUST_ANCHOR")
+[[ "$KEEP_DB" -eq 1 ]] && cmd+=(--keep-db)
+[[ "$CAPTURE_INPUTS_ONLY" -eq 1 ]] && cmd+=(--capture-inputs-only)
+
+RPKI_PROGRESS_LOG="$PROGRESS_LOG" \
+RPKI_PROGRESS_SLOW_SECS="$PROGRESS_SLOW_SECS" \
+"${cmd[@]}"
--- a/scripts/replay_verify/run_peer_bundle_matrix.sh
+++ b/scripts/replay_verify/run_peer_bundle_matrix.sh
@@ -10,6 +10,7 @@ OUT_ROOT=""
 ROUTINATOR_ROOT="/home/yuyr/dev/rust_playground/routinator"
 RPKI_CLIENT_ROOT="/home/yuyr/dev/rpki-client-9.7"
 RPKI_CLIENT_BUILD_DIR="/home/yuyr/dev/rpki-client-9.7/build-m5"
+KEEP_DB=0

 usage() {
  cat <<'EOF'
@@ -20,7 +21,8 @@ Usage:
    [--out-root <dir>] \
    [--routinator-root <dir>] \
    [--rpki-client-root <dir>] \
-    [--rpki-client-build-dir <dir>]
+    [--rpki-client-build-dir <dir>] \
+    [--keep-db]
 EOF
 }

@@ -32,6 +34,7 @@ while [[ $# -gt 0 ]]; do
    --routinator-root) ROUTINATOR_ROOT="${2:?}"; shift 2 ;;
    --rpki-client-root) RPKI_CLIENT_ROOT="${2:?}"; shift 2 ;;
    --rpki-client-build-dir) RPKI_CLIENT_BUILD_DIR="${2:?}"; shift 2 ;;
+    --keep-db) KEEP_DB=1; shift ;;
    --help|-h) usage; exit 0 ;;
    *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
  esac
@@ -42,11 +45,24 @@ if [[ -z "$BUNDLE_ROOT" ]]; then
  exit 2
 fi

+BUNDLE_ROOT="$(python3 - "$BUNDLE_ROOT" <<'PY'
+from pathlib import Path
+import sys
+print(Path(sys.argv[1]).resolve())
+PY
+)"
+
 RUN_TAG="$(date -u +%Y%m%dT%H%M%SZ)"
 if [[ -z "$OUT_ROOT" ]]; then
  OUT_ROOT="target/replay/peer_bundle_matrix_${RUN_TAG}"
 fi
 mkdir -p "$OUT_ROOT"
+OUT_ROOT="$(python3 - "$OUT_ROOT" <<'PY'
+from pathlib import Path
+import sys
+print(Path(sys.argv[1]).resolve())
+PY
+)"

 discover_rirs() {
  python3 - "$BUNDLE_ROOT" <<'PY'
@@ -100,9 +116,13 @@ for raw_rir in "${RIR_LIST[@]}"; do
    source_bundle_dir="$match"
  fi
  ln -sfn "$source_bundle_dir" "$NORMALIZED_BUNDLE_ROOT/$rir"
-  "$ROUTINATOR_ROOT/bench/multi_rir_demo_ours/run_single_rir_ours_bundle.sh" \
-    "$source_bundle_dir" \
+  ROUTI_CMD=(
+    "$ROUTINATOR_ROOT/bench/multi_rir_demo_ours/run_single_rir_ours_bundle.sh"
+    "$source_bundle_dir"
    "$ROUTI_OUT/$rir"
+  )
+  [[ "$KEEP_DB" -eq 1 ]] && ROUTI_CMD=( "$ROUTINATOR_ROOT/bench/multi_rir_demo_ours/run_single_rir_ours_bundle.sh" --keep-db "$source_bundle_dir" "$ROUTI_OUT/$rir" )
+  "${ROUTI_CMD[@]}"
 done

 CLIENT_ARGS=(
@@ -111,6 +131,7 @@ CLIENT_ARGS=(
  --build-dir "$RPKI_CLIENT_BUILD_DIR"
  --work-dir "$CLIENT_OUT"
 )
+[[ "$KEEP_DB" -eq 1 ]] && CLIENT_ARGS+=(--keep-db)
 for raw_rir in "${RIR_LIST[@]}"; do
  rir="$(printf '%s' "$raw_rir" | tr '[:upper:]' '[:lower:]' | xargs)"
  [[ -n "$rir" ]] || continue
@@ -147,27 +168,41 @@ lines = [
    "",
    "## Routinator",
    "",
-    "| rir | base_vrp | delta_vrp | base_vap | delta_vap |",
+    "| rir | base_vrp | base_vap | sequence_vrp | sequence_vap |",
    "|---|---|---|---|---|",
 ]
 for rir, data in sorted(summary["routinator"].items()):
-    lines.append(
-        f"| {rir} | {str(data.get('baseMatch')).lower()} | {str(data.get('deltaMatch')).lower()} | "
-        f"{str(data.get('baseVapsMatch')).lower()} | {str(data.get('deltaVapsMatch')).lower()} |"
-    )
+    if "steps" in data:
+        lines.append(
+            f"| {rir} | {str(data.get('baseMatch')).lower()} | {str(data.get('baseVapsMatch')).lower()} | "
+            f"{str(data.get('summary', {}).get('allStepsMatch')).lower()} | "
+            f"{str(data.get('summary', {}).get('allStepsVapsMatch')).lower()} |"
+        )
+    else:
+        lines.append(
+            f"| {rir} | {str(data.get('baseMatch')).lower()} | {str(data.get('baseVapsMatch')).lower()} | "
+            f"{str(data.get('deltaMatch')).lower()} | {str(data.get('deltaVapsMatch')).lower()} |"
+        )
 lines += [
    "",
    "## rpki-client",
    "",
-    "| rir | base_vrp | delta_vrp | base_vap | delta_vap |",
+    "| rir | base_vrp | base_vap | sequence_vrp | sequence_vap |",
    "|---|---|---|---|---|",
 ]
 for rir, phases in sorted(summary["rpki_client"].items()):
    base = phases.get("base", {})
-    delta = phases.get("delta", {})
+    step_items = [
+        value for key, value in phases.items()
+        if key not in ("base", "delta") and isinstance(value, dict)
+    ]
+    if "delta" in phases:
+        step_items.append(phases["delta"])
+    all_step_match = all(item.get("match") for item in step_items) if step_items else None
+    all_step_vap_match = all(item.get("vaps_match") for item in step_items) if step_items else None
    lines.append(
-        f"| {rir} | {str(base.get('match')).lower()} | {str(delta.get('match')).lower()} | "
-        f"{str(base.get('vaps_match')).lower()} | {str(delta.get('vaps_match')).lower()} |"
+        f"| {rir} | {str(base.get('match')).lower()} | {str(base.get('vaps_match')).lower()} | "
+        f"{str(all_step_match).lower()} | {str(all_step_vap_match).lower()} |"
    )
 out.write_text("\n".join(lines) + "\n")
 PY