将fetch pp cache改成使用vcir结构,跑通apnic全量同步

This commit is contained in:
2026-03-13 14:45:41 +08:00
parent e3339533b8
commit cf764c35bb
53 changed files with 8931 additions and 2201 deletions

View File

@@ -24,7 +24,7 @@ They are meant for **hands-on validation / acceptance runs**, not for CI.
- run meta JSON (includes durations + download_stats)
- short summary Markdown (includes durations + download_stats)
- RocksDB key statistics (`db_stats --exact`)
- RRDP repo state dump (`rrdp_state_dump`)
- RRDP legacy session/serial dump (`rrdp_state_dump --view legacy-state`)
### `delta_sync.sh`
@@ -33,7 +33,7 @@ They are meant for **hands-on validation / acceptance runs**, not for CI.
- Produces the same artifacts as `full_sync.sh`
- Additionally generates a Markdown **delta analysis** report by comparing:
- base vs delta report JSON
- base vs delta `rrdp_state_dump` TSV
- base vs delta `rrdp_state_dump --view legacy-state` TSV
- and includes a **duration comparison** (base vs delta) if the base meta JSON is available
- delta meta JSON includes download_stats copied from delta report JSON
@@ -47,6 +47,9 @@ The `rpki` binary writes an audit report JSON with:
These are useful for diagnosing why a run is slow (e.g. RRDP snapshot vs delta vs rsync fallback).
The standalone `rrdp_state_dump` tool also supports `source`, `members`, `owners`, and `all` views.
The manual sync scripts intentionally call `--view legacy-state` so delta analysis keeps using a stable session/serial TSV format.
## Meta fields (meta.json)
The scripts generate `*_meta.json` next to `*_report.json` and include:

View File

@@ -16,7 +16,7 @@ set -euo pipefail
# - *_delta_report_*.json audit report
# - *_delta_run_*.log stdout/stderr log (includes summary)
# - *_delta_db_stats_*.txt db_stats --exact output
# - *_delta_rrdp_state_*.tsv rrdp_state_dump output
# - *_delta_rrdp_state_*.tsv rrdp_state_dump --view legacy-state output
# - *_delta_analysis_*.md base vs delta comparison report
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
@@ -113,10 +113,10 @@ cargo run --release --bin db_stats -- --db "$DELTA_DB_DIR" --exact 2>&1 | tee "$
db_stats_end_s="$(date +%s)"
db_stats_duration_s="$((db_stats_end_s - db_stats_start_s))"
echo "== rrdp_state_dump ==" >&2
echo "== rrdp_state_dump (legacy-state) ==" >&2
state_start_s="$(date +%s)"
cargo run --release --bin rrdp_state_dump -- --db "$BASE_DB_DIR" >"$BASE_RRDP_STATE_TSV"
cargo run --release --bin rrdp_state_dump -- --db "$DELTA_DB_DIR" >"$DELTA_RRDP_STATE_TSV"
cargo run --release --bin rrdp_state_dump -- --db "$BASE_DB_DIR" --view legacy-state >"$BASE_RRDP_STATE_TSV"
cargo run --release --bin rrdp_state_dump -- --db "$DELTA_DB_DIR" --view legacy-state >"$DELTA_RRDP_STATE_TSV"
state_end_s="$(date +%s)"
state_duration_s="$((state_end_s - state_start_s))"
@@ -174,12 +174,18 @@ def load_optional_json(path_s: str):
return load_json(p)
def parse_rrdp_state_tsv(p: Path):
# format: "<notify_uri>\t<serial>\t<session_id>"
# format from `rrdp_state_dump --view legacy-state`:
# [legacy-state]
# notify_uri serial session_id
# <notify_uri> <serial> <session_id>
out = {}
for line in p.read_text(encoding="utf-8").splitlines():
if not line.strip():
line = line.strip()
if not line or line.startswith("["):
continue
parts = line.split("\t")
if line == "notify_uri serial session_id":
continue
parts = line.split(" ")
if len(parts) != 3:
raise SystemExit(f"invalid rrdp_state_dump line in {p}: {line!r}")
uri, serial, session = parts
@@ -339,7 +345,7 @@ delta_repo_sync_failed = count_repo_sync_failed(delta)
def cache_reason_counts(rep: dict) -> Counter:
c = Counter()
for pp in rep.get("publication_points", []):
if pp.get("source") != "fetch_cache_pp":
if pp.get("source") != "vcir_current_instance":
continue
# Use warning messages as "reason". If missing, emit a fallback bucket.
ws = pp.get("warnings", [])
@@ -378,11 +384,33 @@ added_v = delta_v - base_v
removed_v = base_v - delta_v
def fmt_db_stats(db: dict) -> str:
keys = ["raw_objects","rrdp_object_index","fetch_cache_pp","rrdp_state","total"]
ordered = [
"mode",
"repository_view",
"raw_by_hash",
"vcir",
"audit_rule_index",
"rrdp_source",
"rrdp_source_member",
"rrdp_uri_owner",
"rrdp_state",
"raw_objects",
"rrdp_object_index",
"group_current_repository_view",
"group_current_validation_state",
"group_current_rrdp_state",
"group_legacy_compatibility",
"total",
"sst_files",
]
out = []
for k in keys:
seen = set()
for k in ordered:
if k in db:
out.append(f"- `{k}={db[k]}`")
seen.add(k)
for k in sorted(set(db) - seen):
out.append(f"- `{k}={db[k]}`")
return "\n".join(out) if out else "_(missing db_stats keys)_"
lines = []
@@ -459,7 +487,7 @@ lines.append(f"- `updated_pp={updated_pp}`\n")
lines.append(f"- `unchanged_pp={unchanged_pp}`\n\n")
lines.append("> 注:`new_pp/missing_pp/updated_pp` 会混入“遍历范围变化”的影响(例如 validation_time 不同、或 base 中存在更多失败 PP)。\n\n")
lines.append("## fail fetch / cache 使用情况\n\n")
lines.append("## fail fetch / VCIR 当前实例缓存复用情况\n\n")
lines.append(f"- repo sync failed(启发式:warning contains 'repo sync failed'/'rrdp fetch failed'/'rsync fetch failed')\n")
lines.append(f" - base:`{base_repo_sync_failed}`\n")
lines.append(f" - delta:`{delta_repo_sync_failed}`\n\n")
@@ -470,7 +498,7 @@ lines.append(f" - delta:`{dict(delta_sources)}`\n\n")
def render_cache_reasons(title: str, c: Counter) -> str:
if not c:
return f"{title}:`0`(未使用 fetch_cache_pp)\n\n"
return f"{title}:`0`(未使用 VCIR 当前实例缓存复用)\n\n"
lines = []
total = sum(c.values())
lines.append(f"{title}:`{total}`\n\n")
@@ -480,8 +508,8 @@ def render_cache_reasons(title: str, c: Counter) -> str:
lines.append("\n")
return "".join(lines)
lines.append(render_cache_reasons("- base `source=fetch_cache_pp`", base_cache_reasons))
lines.append(render_cache_reasons("- delta `source=fetch_cache_pp`", delta_cache_reasons))
lines.append(render_cache_reasons("- base `source=vcir_current_instance`", base_cache_reasons))
lines.append(render_cache_reasons("- delta `source=vcir_current_instance`", delta_cache_reasons))
lines.append("## 文件变更统计(按对象类型)\n\n")
lines.append("按 `ObjectAuditEntry.sha256_hex` 对比(同一 rsync URI 前后 hash 变化记为 `~changed`):\n\n")

View File

@@ -8,7 +8,7 @@ set -euo pipefail
# - *_report_*.json audit report
# - *_run_*.log stdout/stderr log (includes summary)
# - *_db_stats_*.txt db_stats --exact output
# - *_rrdp_state_*.tsv rrdp_state_dump output
# - *_rrdp_state_*.tsv rrdp_state_dump --view legacy-state output
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
cd "$ROOT_DIR"
@@ -71,9 +71,9 @@ cargo run --release --bin db_stats -- --db "$DB_DIR" --exact 2>&1 | tee "$DB_STA
db_stats_end_s="$(date +%s)"
db_stats_duration_s="$((db_stats_end_s - db_stats_start_s))"
echo "== rrdp_state_dump ==" >&2
echo "== rrdp_state_dump (legacy-state) ==" >&2
state_start_s="$(date +%s)"
cargo run --release --bin rrdp_state_dump -- --db "$DB_DIR" >"$RRDP_STATE_TSV"
cargo run --release --bin rrdp_state_dump -- --db "$DB_DIR" --view legacy-state >"$RRDP_STATE_TSV"
state_end_s="$(date +%s)"
state_duration_s="$((state_end_s - state_start_s))"