diff --git a/monitor/grafana/dashboards/ours-rp-inter-rp.json b/monitor/grafana/dashboards/ours-rp-inter-rp.json index 474df02..4c97ef3 100644 --- a/monitor/grafana/dashboards/ours-rp-inter-rp.json +++ b/monitor/grafana/dashboards/ours-rp-inter-rp.json @@ -452,6 +452,114 @@ } ] }, + { + "id": 15, + "title": "VRP Diff Trend by Class", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 28 + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "min": 0, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "expr": "inter_rp_vrps_diff_by_class{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"total\"}", + "legendFormat": "total diff", + "refId": "A" + }, + { + "expr": "inter_rp_vrps_diff_by_class{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"only_ours\"}", + "legendFormat": "only ours", + "refId": "B" + }, + { + "expr": "inter_rp_vrps_diff_by_class{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"only_routinator\"}", + "legendFormat": "only routinator", + "refId": "C" + } + ] + }, + { + "id": 16, + "title": "VAP Diff Trend by Class", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 28 + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "min": 0, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "expr": "inter_rp_vaps_diff_by_class{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"total\"}", + "legendFormat": "total diff", + "refId": "A" + }, + { + "expr": "inter_rp_vaps_diff_by_class{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"only_ours\"}", + "legendFormat": "only ours", + "refId": "B" + }, + { + "expr": "inter_rp_vaps_diff_by_class{exported_instance=~\".*inter-rp\",left=\"ours-rp\",right=\"routinator\",class=\"only_routinator\"}", + "legendFormat": "only routinator", + "refId": "C" + } + ] + }, { "id": 11, "title": "Artifact Age by RP", @@ -464,7 +572,7 @@ "h": 8, "w": 24, "x": 0, - "y": 28 + "y": 36 }, "fieldConfig": { "defaults": { @@ -507,7 +615,7 @@ "h": 8, "w": 12, "x": 0, - "y": 36 + "y": 44 }, "fieldConfig": { "defaults": { @@ -551,7 +659,7 @@ "h": 8, "w": 12, "x": 12, - "y": 36 + "y": 44 }, "fieldConfig": { "defaults": { @@ -595,7 +703,7 @@ "h": 9, "w": 24, "x": 0, - "y": 44 + "y": 52 }, "fieldConfig": { "defaults": { diff --git a/scripts/inter_rp/inter_rp_ours_routinator_exporter.py b/scripts/inter_rp/inter_rp_ours_routinator_exporter.py index 2d6d757..ef8d696 100755 --- a/scripts/inter_rp/inter_rp_ours_routinator_exporter.py +++ b/scripts/inter_rp/inter_rp_ours_routinator_exporter.py @@ -20,6 +20,7 @@ SCAN_TTL = float(os.environ.get("INTER_RP_SCAN_TTL_SECONDS", "10")) _cache_lock = threading.Lock() _cache = {"deadline": 0.0, "metrics": "", "status": {}} _count_cache = {} +_set_cache = {} def unix_now(): return time.time() @@ -42,13 +43,13 @@ def latest_ours_run(): candidates = sorted(p for p in runs.glob("run_*") if (p / "run-summary.json").exists()) return candidates[-1] if candidates else None -def count_unique_csv(path, cols): +def unique_csv_set(path, cols): if not path.exists(): return None stat = path.stat() key = (str(path), stat.st_mtime_ns, stat.st_size, cols) - if key in _count_cache: - return _count_cache[key] + if key in _set_cache: + return _set_cache[key] seen = set() with open(path, "r", encoding="utf-8", newline="") as handle: reader = csv.reader(handle) @@ -62,10 +63,15 @@ def count_unique_csv(path, cols): if len(row) < cols: continue seen.add(tuple(cell.strip() for cell in row[:cols])) - value = len(seen) - _count_cache.clear() - _count_cache[key] = value - return value + _set_cache.clear() + _set_cache[key] = seen + return seen + +def count_unique_csv(path, cols): + values = unique_csv_set(path, cols) + if values is None: + return None + return len(values) def metric_line(name, labels, value): label_text = ",".join(f'{k}="{str(v).replace(chr(92), chr(92)+chr(92)).replace(chr(34), chr(92)+chr(34))}"' for k, v in labels.items()) @@ -102,8 +108,10 @@ def sample_ours(now, errors): if rss is not None: sample["max"]["parent"] = int(rss) * 1024 sample["max"]["aggregate_peak"] = int(rss) * 1024 - sample["vrps"] = count_unique_csv(run_dir / "vrps.csv", 3) - sample["vaps"] = count_unique_csv(run_dir / "vaps.csv", 2) + sample["vrp_set"] = unique_csv_set(run_dir / "vrps.csv", 3) + sample["vap_set"] = unique_csv_set(run_dir / "vaps.csv", 2) + sample["vrps"] = len(sample["vrp_set"]) if sample["vrp_set"] is not None else None + sample["vaps"] = len(sample["vap_set"]) if sample["vap_set"] is not None else None return sample def sample_routinator(now, errors): @@ -135,8 +143,10 @@ def sample_routinator(now, errors): for label, key in [("parent", "parent"), ("child_max", "childMax"), ("aggregate_peak", "aggregatePeak")]: if max_rss.get(key) is not None: sample["max"][label] = int(max_rss[key]) * 1024 - sample["vrps"] = count_unique_csv(latest / "vrps.csv", 3) - sample["vaps"] = count_unique_csv(latest / "vaps.csv", 2) + sample["vrp_set"] = unique_csv_set(latest / "vrps.csv", 3) + sample["vap_set"] = unique_csv_set(latest / "vaps.csv", 2) + sample["vrps"] = len(sample["vrp_set"]) if sample["vrp_set"] is not None else None + sample["vaps"] = len(sample["vap_set"]) if sample["vap_set"] is not None else None return sample @@ -384,10 +394,34 @@ def build_metrics(): labels2["kind"] = kind out.append(metric_line("inter_rp_run_max_rss_bytes", labels2, value)) ours, rout = by_rp.get("ours-rp", {}), by_rp.get("routinator", {}) - if ours.get("vrps") is not None and rout.get("vrps") is not None: - out.append(metric_line("inter_rp_vrps_diff", {"instance": INSTANCE, "left": "ours-rp", "right": "routinator"}, abs(int(ours["vrps"]) - int(rout["vrps"])))) - if ours.get("vaps") is not None and rout.get("vaps") is not None: - out.append(metric_line("inter_rp_vaps_diff", {"instance": INSTANCE, "left": "ours-rp", "right": "routinator"}, abs(int(ours["vaps"]) - int(rout["vaps"])))) + if ours.get("vrp_set") is not None and rout.get("vrp_set") is not None: + only_ours = ours["vrp_set"] - rout["vrp_set"] + only_rout = rout["vrp_set"] - ours["vrp_set"] + total_diff = len(only_ours) + len(only_rout) + labels = {"instance": INSTANCE, "left": "ours-rp", "right": "routinator"} + out.append(metric_line("inter_rp_vrps_diff", labels, total_diff)) + out.append(metric_line("inter_rp_vrps_diff_by_class", {**labels, "class": "total"}, total_diff)) + out.append(metric_line("inter_rp_vrps_diff_by_class", {**labels, "class": "only_ours"}, len(only_ours))) + out.append(metric_line("inter_rp_vrps_diff_by_class", {**labels, "class": "only_routinator"}, len(only_rout))) + elif ours.get("vrps") is not None and rout.get("vrps") is not None: + labels = {"instance": INSTANCE, "left": "ours-rp", "right": "routinator"} + total_diff = abs(int(ours["vrps"]) - int(rout["vrps"])) + out.append(metric_line("inter_rp_vrps_diff", labels, total_diff)) + out.append(metric_line("inter_rp_vrps_diff_by_class", {**labels, "class": "total"}, total_diff)) + if ours.get("vap_set") is not None and rout.get("vap_set") is not None: + only_ours = ours["vap_set"] - rout["vap_set"] + only_rout = rout["vap_set"] - ours["vap_set"] + total_diff = len(only_ours) + len(only_rout) + labels = {"instance": INSTANCE, "left": "ours-rp", "right": "routinator"} + out.append(metric_line("inter_rp_vaps_diff", labels, total_diff)) + out.append(metric_line("inter_rp_vaps_diff_by_class", {**labels, "class": "total"}, total_diff)) + out.append(metric_line("inter_rp_vaps_diff_by_class", {**labels, "class": "only_ours"}, len(only_ours))) + out.append(metric_line("inter_rp_vaps_diff_by_class", {**labels, "class": "only_routinator"}, len(only_rout))) + elif ours.get("vaps") is not None and rout.get("vaps") is not None: + labels = {"instance": INSTANCE, "left": "ours-rp", "right": "routinator"} + total_diff = abs(int(ours["vaps"]) - int(rout["vaps"])) + out.append(metric_line("inter_rp_vaps_diff", labels, total_diff)) + out.append(metric_line("inter_rp_vaps_diff_by_class", {**labels, "class": "total"}, total_diff)) repo_diff = emit_repo_diff_metrics(out, errors) return "".join(out), {"errors": errors, "samples": samples, "sync": sync, "repoDiff": repo_diff}