20260526 增加持续soak监控与本地回放工具

This commit is contained in:
2026-05-26 18:02:40 +08:00
parent cda7fdb135
commit 7e1c24fcc3
30 changed files with 4541 additions and 12 deletions

View File

@@ -0,0 +1,632 @@
{
"annotations": {
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_publication_points",
"legendFormat": "publication points",
"refId": "A"
}
],
"title": "Publication Points",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 6,
"y": 0
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_ok\"}",
"legendFormat": "rrdp ok",
"refId": "A"
}
],
"title": "RRDP OK Points",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 12,
"y": 0
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_ok\"}",
"legendFormat": "fallback",
"refId": "A"
}
],
"title": "Rsync Fallback Points",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 6,
"x": 18,
"y": 0
},
"id": 4,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_repo_terminal_state_count{terminal_state=\"failed_no_cache\"}",
"legendFormat": "failed no cache",
"refId": "A"
}
],
"title": "Failed No Cache Points",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 4
},
"id": 5,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_run_stage_duration_seconds{stage=\"repo_sync_total\"}",
"legendFormat": "repo sync total",
"refId": "A"
},
{
"expr": "ours_rp_run_stage_duration_seconds{stage=\"rrdp_download_total\"}",
"legendFormat": "rrdp download",
"refId": "B"
},
{
"expr": "ours_rp_run_stage_duration_seconds{stage=\"rsync_download_total\"}",
"legendFormat": "rsync download",
"refId": "C"
}
],
"title": "Repo Sync Download Durations",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 12,
"w": 12,
"h": 8
},
"id": 6,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_repo_sync_phase_count",
"legendFormat": "{{phase}}",
"refId": "A"
}
],
"title": "Repo Sync Phase Counts",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"x": 12,
"y": 12,
"w": 12,
"h": 8
},
"id": 7,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_ok\"}",
"legendFormat": "rrdp failed, rsync ok",
"refId": "A"
},
{
"expr": "ours_rp_repo_sync_phase_count{phase=\"rrdp_failed_rsync_failed\"}",
"legendFormat": "rrdp failed, rsync failed",
"refId": "B"
},
{
"expr": "ours_rp_repo_terminal_state_count{terminal_state=\"failed_no_cache\"}",
"legendFormat": "failed no cache",
"refId": "C"
},
{
"expr": "ours_rp_tree_instances{state=\"failed\"}",
"legendFormat": "tree failed",
"refId": "D"
}
],
"title": "Repo Failure / Fallback Counts",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 20,
"w": 12,
"h": 8
},
"id": 8,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_repo_sync_phase_duration_seconds_total{phase=\"rrdp_failed_rsync_ok\"}",
"legendFormat": "rsync fallback duration",
"refId": "A"
},
{
"expr": "ours_rp_repo_sync_phase_duration_seconds_total{phase=\"rrdp_failed_rsync_failed\"}",
"legendFormat": "failed duration",
"refId": "B"
},
{
"expr": "ours_rp_repo_terminal_state_duration_seconds_total{terminal_state=\"failed_no_cache\"}",
"legendFormat": "failed no cache duration",
"refId": "C"
}
],
"title": "Repo Failure / Fallback Durations",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "none"
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 29,
"w": 12,
"h": 9
},
"id": 9,
"options": {
"showHeader": true,
"cellHeight": "sm",
"footer": {
"show": false,
"reducer": [
"sum"
],
"countRows": false,
"fields": ""
}
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_rrdp_rsync_failed_repository_duration_seconds",
"format": "table",
"instant": true,
"legendFormat": "",
"refId": "A"
}
],
"title": "RRDP + Rsync Failed Repositories",
"type": "table",
"transformations": [
{
"id": "organize",
"options": {
"excludeByName": {
"job": true,
"terminal_state": true,
"rank": true,
"transport": true,
"__name__": true,
"publication_points": true,
"instance": true,
"repo_id": true,
"pp_id": true,
"exported_instance": true,
"rp": true,
"source": true
},
"indexByName": {
"Time": 0,
"host": 1,
"phase": 2,
"uri": 3,
"Value": 4
},
"renameByName": {
"Value": "duration"
}
}
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "none"
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 38,
"w": 24,
"h": 9
},
"id": 10,
"options": {
"showHeader": true,
"cellHeight": "sm",
"footer": {
"show": false,
"reducer": [
"sum"
],
"countRows": false,
"fields": ""
}
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "topk(20, ours_rp_top_publication_point_object_count)",
"format": "table",
"instant": true,
"legendFormat": "",
"refId": "A"
}
],
"title": "Top Publication Points by Objects",
"type": "table",
"transformations": [
{
"id": "organize",
"options": {
"excludeByName": {
"job": true,
"__name__": true,
"publication_points": true,
"instance": true,
"repo_id": true,
"phase": true,
"pp_id": true,
"exported_instance": true,
"rp": true,
"source": true
},
"indexByName": {
"Time": 0,
"host": 1,
"rank": 2,
"terminal_state": 3,
"transport": 4,
"uri": 5,
"Value": 6
},
"renameByName": {}
}
}
]
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "none"
},
"overrides": []
},
"gridPos": {
"x": 12,
"y": 29,
"w": 12,
"h": 9
},
"id": 11,
"options": {
"showHeader": true,
"cellHeight": "sm",
"footer": {
"show": false,
"reducer": [
"sum"
],
"countRows": false,
"fields": ""
}
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "topk(20, ours_rp_top_repository_sync_duration_seconds)",
"format": "table",
"instant": true,
"legendFormat": "",
"refId": "A"
}
],
"title": "Top 20 Repositories by Sync Duration",
"type": "table",
"transformations": [
{
"id": "organize",
"options": {
"excludeByName": {
"job": true,
"terminal_state": true,
"__name__": true,
"publication_points": true,
"instance": true,
"repo_id": true,
"phase": true,
"pp_id": true,
"exported_instance": true,
"rp": true,
"source": true
},
"indexByName": {
"Time": 0,
"host": 1,
"rank": 2,
"transport": 3,
"uri": 4,
"Value": 5
},
"renameByName": {
"Value": "value"
}
}
}
]
}
],
"refresh": "5s",
"schemaVersion": 40,
"tags": [
"ours-rp",
"rpki",
"soak",
"repo-sync"
],
"templating": {
"list": []
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Ours RP Repo Sync",
"uid": "ours-rp-repo-sync",
"version": 3,
"weekStart": ""
}

View File

@@ -0,0 +1,582 @@
{
"annotations": {
"list": []
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 0,
"w": 6,
"h": 4
},
"id": 1,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_cir_trust_anchors",
"legendFormat": "RIRs",
"refId": "A"
}
],
"title": "Current Run RIRs",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"gridPos": {
"x": 6,
"y": 0,
"w": 6,
"h": 4
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_run_duration_seconds",
"legendFormat": "wall",
"refId": "A"
}
],
"title": "Latest Wall Time",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"x": 12,
"y": 0,
"w": 6,
"h": 4
},
"id": 3,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_run_max_rss_bytes",
"legendFormat": "rss",
"refId": "A"
}
],
"title": "Latest Max RSS",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"x": 18,
"y": 0,
"w": 6,
"h": 4
},
"id": 4,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_publication_points",
"legendFormat": "publication points",
"refId": "A"
}
],
"title": "Publication Points",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 8,
"w": 12,
"h": 8
},
"id": 5,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_run_duration_seconds",
"legendFormat": "wall",
"refId": "A"
},
{
"expr": "ours_rp_stage_duration_seconds{stage=\"validation\"}",
"legendFormat": "validation",
"refId": "B"
}
],
"title": "Run / Validation Duration",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"x": 12,
"y": 8,
"w": 12,
"h": 8
},
"id": 6,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_vrps",
"legendFormat": "VRPs",
"refId": "A"
},
{
"expr": "ours_rp_vaps",
"legendFormat": "VAPs",
"refId": "B"
},
{
"expr": "ours_rp_cir_objects",
"legendFormat": "CIR objects",
"refId": "C"
}
],
"title": "Output and Input Sizes",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 16,
"w": 12,
"h": 8
},
"id": 8,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_large_publication_points",
"legendFormat": "> {{object_count_gt}} objects",
"refId": "A"
}
],
"title": "Large Publication Points by Object Count",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 4,
"w": 6,
"h": 4
},
"id": 9,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_run_sequence",
"legendFormat": "seq",
"refId": "A"
}
],
"title": "Latest Run Sequence",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"x": 6,
"y": 4,
"w": 6,
"h": 4
},
"id": 10,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_run_success",
"legendFormat": "success",
"refId": "A"
}
],
"title": "Latest Run Success",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"x": 12,
"y": 4,
"w": 6,
"h": 4
},
"id": 11,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_vrps",
"legendFormat": "VRPs",
"refId": "A"
}
],
"title": "VRPs",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
},
"gridPos": {
"x": 18,
"y": 4,
"w": 6,
"h": 4
},
"id": 12,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "ours_rp_vaps",
"legendFormat": "VAPs",
"refId": "A"
}
],
"title": "VAPs",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
},
"gridPos": {
"x": 12,
"y": 16,
"w": 12,
"h": 8
},
"id": 13,
"options": {
"legend": {
"calcs": [
"lastNotNull"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "ours_rp_run_stage_duration_seconds{stage=\"validation\"}",
"legendFormat": "validation",
"refId": "A"
},
{
"expr": "ours_rp_run_stage_duration_seconds{stage=\"report_write\"}",
"legendFormat": "report write",
"refId": "E"
},
{
"expr": "ours_rp_run_stage_duration_seconds{stage=\"ccr_write\"}",
"legendFormat": "ccr write",
"refId": "F"
},
{
"expr": "ours_rp_run_stage_duration_seconds{stage=\"cir_write\"}",
"legendFormat": "cir write",
"refId": "G"
}
],
"title": "Output Stage Durations",
"type": "timeseries"
}
],
"refresh": "5s",
"schemaVersion": 40,
"tags": [
"ours-rp",
"rpki",
"soak"
],
"templating": {
"list": []
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Ours RP Soak Overview",
"uid": "ours-rp-soak-overview",
"version": 4,
"weekStart": ""
}

View File

@@ -0,0 +1,12 @@
apiVersion: 1
providers:
- name: ours-rp
orgId: 1
folder: Ours RP
type: file
disableDeletion: false
updateIntervalSeconds: 10
allowUiUpdates: true
options:
path: /var/lib/grafana/dashboards

View File

@@ -0,0 +1,10 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: Prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: true