将fetch pp cache改成使用vcir结构,跑通apnic全量同步
This commit is contained in:
@@ -24,7 +24,7 @@ They are meant for **hands-on validation / acceptance runs**, not for CI.
|
||||
- run meta JSON (includes durations + download_stats)
|
||||
- short summary Markdown (includes durations + download_stats)
|
||||
- RocksDB key statistics (`db_stats --exact`)
|
||||
- RRDP repo state dump (`rrdp_state_dump`)
|
||||
- RRDP legacy session/serial dump (`rrdp_state_dump --view legacy-state`)
|
||||
|
||||
### `delta_sync.sh`
|
||||
|
||||
@@ -33,7 +33,7 @@ They are meant for **hands-on validation / acceptance runs**, not for CI.
|
||||
- Produces the same artifacts as `full_sync.sh`
|
||||
- Additionally generates a Markdown **delta analysis** report by comparing:
|
||||
- base vs delta report JSON
|
||||
- base vs delta `rrdp_state_dump` TSV
|
||||
- base vs delta `rrdp_state_dump --view legacy-state` TSV
|
||||
- and includes a **duration comparison** (base vs delta) if the base meta JSON is available
|
||||
- delta meta JSON includes download_stats copied from delta report JSON
|
||||
|
||||
@@ -47,6 +47,9 @@ The `rpki` binary writes an audit report JSON with:
|
||||
|
||||
These are useful for diagnosing why a run is slow (e.g. RRDP snapshot vs delta vs rsync fallback).
|
||||
|
||||
The standalone `rrdp_state_dump` tool also supports `source`, `members`, `owners`, and `all` views.
|
||||
The manual sync scripts intentionally call `--view legacy-state` so delta analysis keeps using a stable session/serial TSV format.
|
||||
|
||||
## Meta fields (meta.json)
|
||||
|
||||
The scripts generate `*_meta.json` next to `*_report.json` and include:
|
||||
|
||||
@@ -16,7 +16,7 @@ set -euo pipefail
|
||||
# - *_delta_report_*.json audit report
|
||||
# - *_delta_run_*.log stdout/stderr log (includes summary)
|
||||
# - *_delta_db_stats_*.txt db_stats --exact output
|
||||
# - *_delta_rrdp_state_*.tsv rrdp_state_dump output
|
||||
# - *_delta_rrdp_state_*.tsv rrdp_state_dump --view legacy-state output
|
||||
# - *_delta_analysis_*.md base vs delta comparison report
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
@@ -113,10 +113,10 @@ cargo run --release --bin db_stats -- --db "$DELTA_DB_DIR" --exact 2>&1 | tee "$
|
||||
db_stats_end_s="$(date +%s)"
|
||||
db_stats_duration_s="$((db_stats_end_s - db_stats_start_s))"
|
||||
|
||||
echo "== rrdp_state_dump ==" >&2
|
||||
echo "== rrdp_state_dump (legacy-state) ==" >&2
|
||||
state_start_s="$(date +%s)"
|
||||
cargo run --release --bin rrdp_state_dump -- --db "$BASE_DB_DIR" >"$BASE_RRDP_STATE_TSV"
|
||||
cargo run --release --bin rrdp_state_dump -- --db "$DELTA_DB_DIR" >"$DELTA_RRDP_STATE_TSV"
|
||||
cargo run --release --bin rrdp_state_dump -- --db "$BASE_DB_DIR" --view legacy-state >"$BASE_RRDP_STATE_TSV"
|
||||
cargo run --release --bin rrdp_state_dump -- --db "$DELTA_DB_DIR" --view legacy-state >"$DELTA_RRDP_STATE_TSV"
|
||||
state_end_s="$(date +%s)"
|
||||
state_duration_s="$((state_end_s - state_start_s))"
|
||||
|
||||
@@ -174,12 +174,18 @@ def load_optional_json(path_s: str):
|
||||
return load_json(p)
|
||||
|
||||
def parse_rrdp_state_tsv(p: Path):
|
||||
# format: "<notify_uri>\t<serial>\t<session_id>"
|
||||
# format from `rrdp_state_dump --view legacy-state`:
|
||||
# [legacy-state]
|
||||
# notify_uri serial session_id
|
||||
# <notify_uri> <serial> <session_id>
|
||||
out = {}
|
||||
for line in p.read_text(encoding="utf-8").splitlines():
|
||||
if not line.strip():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("["):
|
||||
continue
|
||||
parts = line.split("\t")
|
||||
if line == "notify_uri serial session_id":
|
||||
continue
|
||||
parts = line.split(" ")
|
||||
if len(parts) != 3:
|
||||
raise SystemExit(f"invalid rrdp_state_dump line in {p}: {line!r}")
|
||||
uri, serial, session = parts
|
||||
@@ -339,7 +345,7 @@ delta_repo_sync_failed = count_repo_sync_failed(delta)
|
||||
def cache_reason_counts(rep: dict) -> Counter:
|
||||
c = Counter()
|
||||
for pp in rep.get("publication_points", []):
|
||||
if pp.get("source") != "fetch_cache_pp":
|
||||
if pp.get("source") != "vcir_current_instance":
|
||||
continue
|
||||
# Use warning messages as "reason". If missing, emit a fallback bucket.
|
||||
ws = pp.get("warnings", [])
|
||||
@@ -378,11 +384,33 @@ added_v = delta_v - base_v
|
||||
removed_v = base_v - delta_v
|
||||
|
||||
def fmt_db_stats(db: dict) -> str:
|
||||
keys = ["raw_objects","rrdp_object_index","fetch_cache_pp","rrdp_state","total"]
|
||||
ordered = [
|
||||
"mode",
|
||||
"repository_view",
|
||||
"raw_by_hash",
|
||||
"vcir",
|
||||
"audit_rule_index",
|
||||
"rrdp_source",
|
||||
"rrdp_source_member",
|
||||
"rrdp_uri_owner",
|
||||
"rrdp_state",
|
||||
"raw_objects",
|
||||
"rrdp_object_index",
|
||||
"group_current_repository_view",
|
||||
"group_current_validation_state",
|
||||
"group_current_rrdp_state",
|
||||
"group_legacy_compatibility",
|
||||
"total",
|
||||
"sst_files",
|
||||
]
|
||||
out = []
|
||||
for k in keys:
|
||||
seen = set()
|
||||
for k in ordered:
|
||||
if k in db:
|
||||
out.append(f"- `{k}={db[k]}`")
|
||||
seen.add(k)
|
||||
for k in sorted(set(db) - seen):
|
||||
out.append(f"- `{k}={db[k]}`")
|
||||
return "\n".join(out) if out else "_(missing db_stats keys)_"
|
||||
|
||||
lines = []
|
||||
@@ -459,7 +487,7 @@ lines.append(f"- `updated_pp={updated_pp}`\n")
|
||||
lines.append(f"- `unchanged_pp={unchanged_pp}`\n\n")
|
||||
lines.append("> 注:`new_pp/missing_pp/updated_pp` 会混入“遍历范围变化”的影响(例如 validation_time 不同、或 base 中存在更多失败 PP)。\n\n")
|
||||
|
||||
lines.append("## fail fetch / cache 使用情况\n\n")
|
||||
lines.append("## fail fetch / VCIR 当前实例缓存复用情况\n\n")
|
||||
lines.append(f"- repo sync failed(启发式:warning contains 'repo sync failed'/'rrdp fetch failed'/'rsync fetch failed')\n")
|
||||
lines.append(f" - base:`{base_repo_sync_failed}`\n")
|
||||
lines.append(f" - delta:`{delta_repo_sync_failed}`\n\n")
|
||||
@@ -470,7 +498,7 @@ lines.append(f" - delta:`{dict(delta_sources)}`\n\n")
|
||||
|
||||
def render_cache_reasons(title: str, c: Counter) -> str:
|
||||
if not c:
|
||||
return f"{title}:`0`(未使用 fetch_cache_pp)\n\n"
|
||||
return f"{title}:`0`(未使用 VCIR 当前实例缓存复用)\n\n"
|
||||
lines = []
|
||||
total = sum(c.values())
|
||||
lines.append(f"{title}:`{total}`\n\n")
|
||||
@@ -480,8 +508,8 @@ def render_cache_reasons(title: str, c: Counter) -> str:
|
||||
lines.append("\n")
|
||||
return "".join(lines)
|
||||
|
||||
lines.append(render_cache_reasons("- base `source=fetch_cache_pp`", base_cache_reasons))
|
||||
lines.append(render_cache_reasons("- delta `source=fetch_cache_pp`", delta_cache_reasons))
|
||||
lines.append(render_cache_reasons("- base `source=vcir_current_instance`", base_cache_reasons))
|
||||
lines.append(render_cache_reasons("- delta `source=vcir_current_instance`", delta_cache_reasons))
|
||||
|
||||
lines.append("## 文件变更统计(按对象类型)\n\n")
|
||||
lines.append("按 `ObjectAuditEntry.sha256_hex` 对比(同一 rsync URI 前后 hash 变化记为 `~changed`):\n\n")
|
||||
|
||||
@@ -8,7 +8,7 @@ set -euo pipefail
|
||||
# - *_report_*.json audit report
|
||||
# - *_run_*.log stdout/stderr log (includes summary)
|
||||
# - *_db_stats_*.txt db_stats --exact output
|
||||
# - *_rrdp_state_*.tsv rrdp_state_dump output
|
||||
# - *_rrdp_state_*.tsv rrdp_state_dump --view legacy-state output
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
cd "$ROOT_DIR"
|
||||
@@ -71,9 +71,9 @@ cargo run --release --bin db_stats -- --db "$DB_DIR" --exact 2>&1 | tee "$DB_STA
|
||||
db_stats_end_s="$(date +%s)"
|
||||
db_stats_duration_s="$((db_stats_end_s - db_stats_start_s))"
|
||||
|
||||
echo "== rrdp_state_dump ==" >&2
|
||||
echo "== rrdp_state_dump (legacy-state) ==" >&2
|
||||
state_start_s="$(date +%s)"
|
||||
cargo run --release --bin rrdp_state_dump -- --db "$DB_DIR" >"$RRDP_STATE_TSV"
|
||||
cargo run --release --bin rrdp_state_dump -- --db "$DB_DIR" --view legacy-state >"$RRDP_STATE_TSV"
|
||||
state_end_s="$(date +%s)"
|
||||
state_duration_s="$((state_end_s - state_start_s))"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user