全量同步测试增加download过程审计输出
This commit is contained in:
parent
6276d13814
commit
afc50364f8
@ -21,8 +21,8 @@ They are meant for **hands-on validation / acceptance runs**, not for CI.
|
|||||||
- Writes:
|
- Writes:
|
||||||
- run log
|
- run log
|
||||||
- audit report JSON
|
- audit report JSON
|
||||||
- run meta JSON (includes durations)
|
- run meta JSON (includes durations + download_stats)
|
||||||
- short summary Markdown (includes durations)
|
- short summary Markdown (includes durations + download_stats)
|
||||||
- RocksDB key statistics (`db_stats --exact`)
|
- RocksDB key statistics (`db_stats --exact`)
|
||||||
- RRDP repo state dump (`rrdp_state_dump`)
|
- RRDP repo state dump (`rrdp_state_dump`)
|
||||||
|
|
||||||
@ -35,6 +35,24 @@ They are meant for **hands-on validation / acceptance runs**, not for CI.
|
|||||||
- base vs delta report JSON
|
- base vs delta report JSON
|
||||||
- base vs delta `rrdp_state_dump` TSV
|
- base vs delta `rrdp_state_dump` TSV
|
||||||
- and includes a **duration comparison** (base vs delta) if the base meta JSON is available
|
- and includes a **duration comparison** (base vs delta) if the base meta JSON is available
|
||||||
|
- delta meta JSON includes download_stats copied from delta report JSON
|
||||||
|
|
||||||
|
## Audit report fields (report.json)
|
||||||
|
|
||||||
|
The `rpki` binary writes an audit report JSON with:
|
||||||
|
|
||||||
|
- `format_version: 2`
|
||||||
|
- `downloads`: per-download RRDP/rsync events (URI, timestamps, duration, ok/fail, error, bytes, objects stats)
|
||||||
|
- `download_stats`: aggregate counters (by kind)
|
||||||
|
|
||||||
|
These are useful for diagnosing why a run is slow (e.g. RRDP snapshot vs delta vs rsync fallback).
|
||||||
|
|
||||||
|
## Meta fields (meta.json)
|
||||||
|
|
||||||
|
The scripts generate `*_meta.json` next to `*_report.json` and include:
|
||||||
|
|
||||||
|
- `durations_secs`: wall-clock duration breakdown for the script steps
|
||||||
|
- `download_stats`: copied from `report_json.download_stats`
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
|
|||||||
@ -266,6 +266,7 @@ base_state = parse_rrdp_state_tsv(base_state_path)
|
|||||||
delta_state = parse_rrdp_state_tsv(delta_state_path)
|
delta_state = parse_rrdp_state_tsv(delta_state_path)
|
||||||
|
|
||||||
base_meta = load_optional_json(base_meta_path_s)
|
base_meta = load_optional_json(base_meta_path_s)
|
||||||
|
download_stats = delta.get("download_stats") or {}
|
||||||
delta_meta = {
|
delta_meta = {
|
||||||
"recorded_at_utc": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
"recorded_at_utc": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||||
"tal_url": os.environ["TAL_URL"],
|
"tal_url": os.environ["TAL_URL"],
|
||||||
@ -283,6 +284,7 @@ delta_meta = {
|
|||||||
"rrdp_state_dump": int(os.environ["STATE_DURATION_S"]),
|
"rrdp_state_dump": int(os.environ["STATE_DURATION_S"]),
|
||||||
"total_script": int(os.environ["TOTAL_DURATION_S"]),
|
"total_script": int(os.environ["TOTAL_DURATION_S"]),
|
||||||
},
|
},
|
||||||
|
"download_stats": download_stats,
|
||||||
}
|
}
|
||||||
delta_meta_path.write_text(json.dumps(delta_meta, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
delta_meta_path.write_text(json.dumps(delta_meta, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||||
|
|
||||||
|
|||||||
@ -106,6 +106,7 @@ summary_path = Path(sys.argv[3])
|
|||||||
rep = json.loads(report_path.read_text(encoding="utf-8"))
|
rep = json.loads(report_path.read_text(encoding="utf-8"))
|
||||||
|
|
||||||
now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
now = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||||
|
download_stats = rep.get("download_stats") or {}
|
||||||
meta = {
|
meta = {
|
||||||
"recorded_at_utc": now,
|
"recorded_at_utc": now,
|
||||||
"tal_url": os.environ["TAL_URL"],
|
"tal_url": os.environ["TAL_URL"],
|
||||||
@ -129,6 +130,7 @@ meta = {
|
|||||||
"aspas": len(rep["aspas"]),
|
"aspas": len(rep["aspas"]),
|
||||||
"audit_publication_points": len(rep["publication_points"]),
|
"audit_publication_points": len(rep["publication_points"]),
|
||||||
},
|
},
|
||||||
|
"download_stats": download_stats,
|
||||||
}
|
}
|
||||||
|
|
||||||
meta_path.write_text(json.dumps(meta, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
meta_path.write_text(json.dumps(meta, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||||
@ -153,6 +155,25 @@ for k,v in meta["durations_secs"].items():
|
|||||||
lines.append(f"| {k} | {v} |\\n")
|
lines.append(f"| {k} | {v} |\\n")
|
||||||
lines.append("\\n")
|
lines.append("\\n")
|
||||||
|
|
||||||
|
lines.append("## Download Stats\\n\\n")
|
||||||
|
lines.append("- raw events: `report_json.downloads`\\n")
|
||||||
|
lines.append("- aggregated: `report_json.download_stats` (copied into meta.json)\\n\\n")
|
||||||
|
|
||||||
|
def fmt_u(v):
|
||||||
|
if v is None:
|
||||||
|
return ""
|
||||||
|
return str(v)
|
||||||
|
|
||||||
|
by_kind = download_stats.get("by_kind") or {}
|
||||||
|
lines.append("| kind | ok | fail | duration_ms_total | bytes_total | objects_count_total | objects_bytes_total |\\n")
|
||||||
|
lines.append("|---|---:|---:|---:|---:|---:|---:|\\n")
|
||||||
|
for kind in sorted(by_kind.keys()):
|
||||||
|
st = by_kind[kind] or {}
|
||||||
|
lines.append(
|
||||||
|
f"| {kind} | {st.get('ok_total',0)} | {st.get('fail_total',0)} | {st.get('duration_ms_total',0)} | {fmt_u(st.get('bytes_total'))} | {fmt_u(st.get('objects_count_total'))} | {fmt_u(st.get('objects_bytes_total'))} |\\n"
|
||||||
|
)
|
||||||
|
lines.append("\\n")
|
||||||
|
|
||||||
summary_path.write_text("".join(lines), encoding="utf-8")
|
summary_path.write_text("".join(lines), encoding="utf-8")
|
||||||
print(summary_path)
|
print(summary_path)
|
||||||
PY
|
PY
|
||||||
|
|||||||
66
src/audit.rs
66
src/audit.rs
@ -98,6 +98,57 @@ pub struct AuditRunMeta {
|
|||||||
pub validation_time_rfc3339_utc: String,
|
pub validation_time_rfc3339_utc: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum AuditDownloadKind {
|
||||||
|
RrdpNotification,
|
||||||
|
RrdpSnapshot,
|
||||||
|
RrdpDelta,
|
||||||
|
Rsync,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
|
||||||
|
pub struct AuditDownloadObjectsStat {
|
||||||
|
pub objects_count: u64,
|
||||||
|
pub objects_bytes_total: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||||
|
pub struct AuditDownloadEvent {
|
||||||
|
pub kind: AuditDownloadKind,
|
||||||
|
pub uri: String,
|
||||||
|
pub started_at_rfc3339_utc: String,
|
||||||
|
pub finished_at_rfc3339_utc: String,
|
||||||
|
pub duration_ms: u64,
|
||||||
|
pub success: bool,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub error: Option<String>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub bytes: Option<u64>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub objects: Option<AuditDownloadObjectsStat>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
|
||||||
|
pub struct AuditDownloadKindStats {
|
||||||
|
pub ok_total: u64,
|
||||||
|
pub fail_total: u64,
|
||||||
|
pub duration_ms_total: u64,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub bytes_total: Option<u64>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub objects_count_total: Option<u64>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
pub objects_bytes_total: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize)]
|
||||||
|
pub struct AuditDownloadStats {
|
||||||
|
pub events_total: u64,
|
||||||
|
/// Statistics keyed by serialized `AuditDownloadKind` string (e.g. "rrdp_snapshot").
|
||||||
|
pub by_kind: std::collections::BTreeMap<String, AuditDownloadKindStats>,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||||
pub struct AuditReportV1 {
|
pub struct AuditReportV1 {
|
||||||
pub format_version: u32,
|
pub format_version: u32,
|
||||||
@ -110,6 +161,21 @@ pub struct AuditReportV1 {
|
|||||||
pub aspas: Vec<AspaOutput>,
|
pub aspas: Vec<AspaOutput>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||||
|
pub struct AuditReportV2 {
|
||||||
|
pub format_version: u32,
|
||||||
|
pub meta: AuditRunMeta,
|
||||||
|
pub policy: Policy,
|
||||||
|
pub tree: TreeSummary,
|
||||||
|
pub publication_points: Vec<PublicationPointAudit>,
|
||||||
|
|
||||||
|
pub vrps: Vec<VrpOutput>,
|
||||||
|
pub aspas: Vec<AspaOutput>,
|
||||||
|
|
||||||
|
pub downloads: Vec<AuditDownloadEvent>,
|
||||||
|
pub download_stats: AuditDownloadStats,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
|
||||||
pub struct VrpOutput {
|
pub struct VrpOutput {
|
||||||
pub asn: u32,
|
pub asn: u32,
|
||||||
|
|||||||
168
src/audit_downloads.rs
Normal file
168
src/audit_downloads.rs
Normal file
@ -0,0 +1,168 @@
|
|||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use crate::audit::{
|
||||||
|
AuditDownloadEvent, AuditDownloadKind, AuditDownloadKindStats, AuditDownloadObjectsStat,
|
||||||
|
AuditDownloadStats,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Default)]
|
||||||
|
pub struct DownloadLogHandle {
|
||||||
|
inner: Arc<Mutex<Vec<AuditDownloadEvent>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl DownloadLogHandle {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self::default()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn record_event(&self, event: AuditDownloadEvent) {
|
||||||
|
self.inner.lock().expect("download log lock").push(event);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn snapshot_events(&self) -> Vec<AuditDownloadEvent> {
|
||||||
|
self.inner.lock().expect("download log lock").clone()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn stats_from_events(events: &[AuditDownloadEvent]) -> AuditDownloadStats {
|
||||||
|
let mut out = AuditDownloadStats {
|
||||||
|
events_total: events.len() as u64,
|
||||||
|
by_kind: BTreeMap::new(),
|
||||||
|
};
|
||||||
|
for e in events {
|
||||||
|
let kind_key = match e.kind {
|
||||||
|
AuditDownloadKind::RrdpNotification => "rrdp_notification",
|
||||||
|
AuditDownloadKind::RrdpSnapshot => "rrdp_snapshot",
|
||||||
|
AuditDownloadKind::RrdpDelta => "rrdp_delta",
|
||||||
|
AuditDownloadKind::Rsync => "rsync",
|
||||||
|
}
|
||||||
|
.to_string();
|
||||||
|
|
||||||
|
let st = out.by_kind.entry(kind_key).or_insert_with(|| AuditDownloadKindStats {
|
||||||
|
ok_total: 0,
|
||||||
|
fail_total: 0,
|
||||||
|
duration_ms_total: 0,
|
||||||
|
bytes_total: None,
|
||||||
|
objects_count_total: None,
|
||||||
|
objects_bytes_total: None,
|
||||||
|
});
|
||||||
|
if e.success {
|
||||||
|
st.ok_total = st.ok_total.saturating_add(1);
|
||||||
|
} else {
|
||||||
|
st.fail_total = st.fail_total.saturating_add(1);
|
||||||
|
}
|
||||||
|
st.duration_ms_total = st.duration_ms_total.saturating_add(e.duration_ms);
|
||||||
|
if let Some(b) = e.bytes {
|
||||||
|
st.bytes_total = Some(st.bytes_total.unwrap_or(0).saturating_add(b));
|
||||||
|
}
|
||||||
|
if let Some(objects) = &e.objects {
|
||||||
|
st.objects_count_total = Some(
|
||||||
|
st.objects_count_total
|
||||||
|
.unwrap_or(0)
|
||||||
|
.saturating_add(objects.objects_count),
|
||||||
|
);
|
||||||
|
st.objects_bytes_total = Some(
|
||||||
|
st.objects_bytes_total
|
||||||
|
.unwrap_or(0)
|
||||||
|
.saturating_add(objects.objects_bytes_total),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn stats(&self) -> AuditDownloadStats {
|
||||||
|
let events = self.snapshot_events();
|
||||||
|
Self::stats_from_events(&events)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn span_download<'a>(
|
||||||
|
&'a self,
|
||||||
|
kind: AuditDownloadKind,
|
||||||
|
uri: &'a str,
|
||||||
|
) -> DownloadSpanGuard<'a> {
|
||||||
|
DownloadSpanGuard {
|
||||||
|
handle: self,
|
||||||
|
kind,
|
||||||
|
uri,
|
||||||
|
start_instant: Instant::now(),
|
||||||
|
started_at: time::OffsetDateTime::now_utc(),
|
||||||
|
bytes: None,
|
||||||
|
objects: None,
|
||||||
|
error: None,
|
||||||
|
success: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct DownloadSpanGuard<'a> {
|
||||||
|
handle: &'a DownloadLogHandle,
|
||||||
|
kind: AuditDownloadKind,
|
||||||
|
uri: &'a str,
|
||||||
|
start_instant: Instant,
|
||||||
|
started_at: time::OffsetDateTime,
|
||||||
|
bytes: Option<u64>,
|
||||||
|
objects: Option<AuditDownloadObjectsStat>,
|
||||||
|
error: Option<String>,
|
||||||
|
success: Option<bool>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl DownloadSpanGuard<'_> {
|
||||||
|
pub fn set_bytes(&mut self, bytes: u64) {
|
||||||
|
self.bytes = Some(bytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn set_objects(&mut self, objects_count: u64, objects_bytes_total: u64) {
|
||||||
|
self.objects = Some(AuditDownloadObjectsStat {
|
||||||
|
objects_count,
|
||||||
|
objects_bytes_total,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn set_ok(&mut self) {
|
||||||
|
self.success = Some(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn set_err(&mut self, msg: impl Into<String>) {
|
||||||
|
self.success = Some(false);
|
||||||
|
self.error = Some(msg.into());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for DownloadSpanGuard<'_> {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
use time::format_description::well_known::Rfc3339;
|
||||||
|
let finished_at = time::OffsetDateTime::now_utc();
|
||||||
|
let dur = self.start_instant.elapsed();
|
||||||
|
let duration_ms = duration_to_ms(dur);
|
||||||
|
let started_at_rfc3339_utc = self
|
||||||
|
.started_at
|
||||||
|
.to_offset(time::UtcOffset::UTC)
|
||||||
|
.format(&Rfc3339)
|
||||||
|
.unwrap_or_else(|_| "<format-error>".to_string());
|
||||||
|
let finished_at_rfc3339_utc = finished_at
|
||||||
|
.to_offset(time::UtcOffset::UTC)
|
||||||
|
.format(&Rfc3339)
|
||||||
|
.unwrap_or_else(|_| "<format-error>".to_string());
|
||||||
|
let success = self.success.unwrap_or(false);
|
||||||
|
let event = AuditDownloadEvent {
|
||||||
|
kind: self.kind.clone(),
|
||||||
|
uri: self.uri.to_string(),
|
||||||
|
started_at_rfc3339_utc,
|
||||||
|
finished_at_rfc3339_utc,
|
||||||
|
duration_ms,
|
||||||
|
success,
|
||||||
|
error: if success { None } else { self.error.clone() },
|
||||||
|
bytes: self.bytes,
|
||||||
|
objects: self.objects.clone(),
|
||||||
|
};
|
||||||
|
self.handle.record_event(event);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn duration_to_ms(d: Duration) -> u64 {
|
||||||
|
let ms = d.as_millis();
|
||||||
|
ms.min(u128::from(u64::MAX)) as u64
|
||||||
|
}
|
||||||
|
|
||||||
24
src/cli.rs
24
src/cli.rs
@ -2,7 +2,7 @@ use std::path::{Path, PathBuf};
|
|||||||
|
|
||||||
use crate::analysis::timing::{TimingHandle, TimingMeta, TimingMetaUpdate};
|
use crate::analysis::timing::{TimingHandle, TimingMeta, TimingMetaUpdate};
|
||||||
use crate::audit::{
|
use crate::audit::{
|
||||||
AspaOutput, AuditReportV1, AuditRunMeta, AuditWarning, TreeSummary, VrpOutput,
|
AspaOutput, AuditReportV2, AuditRunMeta, AuditWarning, TreeSummary, VrpOutput,
|
||||||
format_roa_ip_prefix,
|
format_roa_ip_prefix,
|
||||||
};
|
};
|
||||||
use crate::fetch::http::{BlockingHttpFetcher, HttpFetcherConfig};
|
use crate::fetch::http::{BlockingHttpFetcher, HttpFetcherConfig};
|
||||||
@ -239,7 +239,7 @@ fn read_policy(path: Option<&Path>) -> Result<Policy, String> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn write_json(path: &Path, report: &AuditReportV1) -> Result<(), String> {
|
fn write_json(path: &Path, report: &AuditReportV2) -> Result<(), String> {
|
||||||
let f = std::fs::File::create(path)
|
let f = std::fs::File::create(path)
|
||||||
.map_err(|e| format!("create report file failed: {}: {e}", path.display()))?;
|
.map_err(|e| format!("create report file failed: {}: {e}", path.display()))?;
|
||||||
serde_json::to_writer_pretty(f, report)
|
serde_json::to_writer_pretty(f, report)
|
||||||
@ -247,7 +247,7 @@ fn write_json(path: &Path, report: &AuditReportV1) -> Result<(), String> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn unique_rrdp_repos(report: &AuditReportV1) -> usize {
|
fn unique_rrdp_repos(report: &AuditReportV2) -> usize {
|
||||||
use std::collections::HashSet;
|
use std::collections::HashSet;
|
||||||
let mut set: HashSet<&str> = HashSet::new();
|
let mut set: HashSet<&str> = HashSet::new();
|
||||||
for pp in &report.publication_points {
|
for pp in &report.publication_points {
|
||||||
@ -258,7 +258,7 @@ fn unique_rrdp_repos(report: &AuditReportV1) -> usize {
|
|||||||
set.len()
|
set.len()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn print_summary(report: &AuditReportV1) {
|
fn print_summary(report: &AuditReportV2) {
|
||||||
let rrdp_repos = unique_rrdp_repos(report);
|
let rrdp_repos = unique_rrdp_repos(report);
|
||||||
println!("RPKI stage2 serial run summary");
|
println!("RPKI stage2 serial run summary");
|
||||||
println!(
|
println!(
|
||||||
@ -291,7 +291,7 @@ fn build_report(
|
|||||||
policy: &Policy,
|
policy: &Policy,
|
||||||
validation_time: time::OffsetDateTime,
|
validation_time: time::OffsetDateTime,
|
||||||
out: RunTreeFromTalAuditOutput,
|
out: RunTreeFromTalAuditOutput,
|
||||||
) -> AuditReportV1 {
|
) -> AuditReportV2 {
|
||||||
use time::format_description::well_known::Rfc3339;
|
use time::format_description::well_known::Rfc3339;
|
||||||
let validation_time_rfc3339_utc = validation_time
|
let validation_time_rfc3339_utc = validation_time
|
||||||
.to_offset(time::UtcOffset::UTC)
|
.to_offset(time::UtcOffset::UTC)
|
||||||
@ -319,8 +319,8 @@ fn build_report(
|
|||||||
})
|
})
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
AuditReportV1 {
|
AuditReportV2 {
|
||||||
format_version: 1,
|
format_version: 2,
|
||||||
meta: AuditRunMeta {
|
meta: AuditRunMeta {
|
||||||
validation_time_rfc3339_utc,
|
validation_time_rfc3339_utc,
|
||||||
},
|
},
|
||||||
@ -333,6 +333,8 @@ fn build_report(
|
|||||||
publication_points: out.publication_points,
|
publication_points: out.publication_points,
|
||||||
vrps,
|
vrps,
|
||||||
aspas,
|
aspas,
|
||||||
|
downloads: out.downloads,
|
||||||
|
download_stats: out.download_stats,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -879,6 +881,8 @@ mod tests {
|
|||||||
discovery,
|
discovery,
|
||||||
tree,
|
tree,
|
||||||
publication_points: vec![pp1, pp2, pp3],
|
publication_points: vec![pp1, pp2, pp3],
|
||||||
|
downloads: Vec::new(),
|
||||||
|
download_stats: crate::audit::AuditDownloadStats::default(),
|
||||||
};
|
};
|
||||||
|
|
||||||
let policy = Policy::default();
|
let policy = Policy::default();
|
||||||
@ -894,8 +898,8 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn write_json_writes_report() {
|
fn write_json_writes_report() {
|
||||||
let report = AuditReportV1 {
|
let report = AuditReportV2 {
|
||||||
format_version: 1,
|
format_version: 2,
|
||||||
meta: AuditRunMeta {
|
meta: AuditRunMeta {
|
||||||
validation_time_rfc3339_utc: "2026-01-01T00:00:00Z".to_string(),
|
validation_time_rfc3339_utc: "2026-01-01T00:00:00Z".to_string(),
|
||||||
},
|
},
|
||||||
@ -908,6 +912,8 @@ mod tests {
|
|||||||
publication_points: Vec::new(),
|
publication_points: Vec::new(),
|
||||||
vrps: Vec::new(),
|
vrps: Vec::new(),
|
||||||
aspas: Vec::new(),
|
aspas: Vec::new(),
|
||||||
|
downloads: Vec::new(),
|
||||||
|
download_stats: crate::audit::AuditDownloadStats::default(),
|
||||||
};
|
};
|
||||||
|
|
||||||
let dir = tempfile::tempdir().expect("tmpdir");
|
let dir = tempfile::tempdir().expect("tmpdir");
|
||||||
|
|||||||
@ -5,6 +5,8 @@ pub mod analysis;
|
|||||||
#[cfg(feature = "full")]
|
#[cfg(feature = "full")]
|
||||||
pub mod audit;
|
pub mod audit;
|
||||||
#[cfg(feature = "full")]
|
#[cfg(feature = "full")]
|
||||||
|
pub mod audit_downloads;
|
||||||
|
#[cfg(feature = "full")]
|
||||||
pub mod cli;
|
pub mod cli;
|
||||||
#[cfg(feature = "full")]
|
#[cfg(feature = "full")]
|
||||||
pub mod fetch;
|
pub mod fetch;
|
||||||
|
|||||||
229
src/sync/repo.rs
229
src/sync/repo.rs
@ -1,9 +1,11 @@
|
|||||||
use crate::analysis::timing::TimingHandle;
|
use crate::analysis::timing::TimingHandle;
|
||||||
|
use crate::audit_downloads::DownloadLogHandle;
|
||||||
|
use crate::audit::{AuditDownloadKind};
|
||||||
use crate::fetch::rsync::{RsyncFetchError, RsyncFetcher};
|
use crate::fetch::rsync::{RsyncFetchError, RsyncFetcher};
|
||||||
use crate::policy::{Policy, SyncPreference};
|
use crate::policy::{Policy, SyncPreference};
|
||||||
use crate::report::{RfcRef, Warning};
|
use crate::report::{RfcRef, Warning};
|
||||||
use crate::storage::RocksStore;
|
use crate::storage::RocksStore;
|
||||||
use crate::sync::rrdp::sync_from_notification_with_timing;
|
use crate::sync::rrdp::sync_from_notification_with_timing_and_download_log;
|
||||||
use crate::sync::rrdp::{Fetcher as HttpFetcher, RrdpSyncError};
|
use crate::sync::rrdp::{Fetcher as HttpFetcher, RrdpSyncError};
|
||||||
use std::thread;
|
use std::thread;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
@ -56,10 +58,17 @@ pub fn sync_publication_point(
|
|||||||
http_fetcher: &dyn HttpFetcher,
|
http_fetcher: &dyn HttpFetcher,
|
||||||
rsync_fetcher: &dyn RsyncFetcher,
|
rsync_fetcher: &dyn RsyncFetcher,
|
||||||
timing: Option<&TimingHandle>,
|
timing: Option<&TimingHandle>,
|
||||||
|
download_log: Option<&DownloadLogHandle>,
|
||||||
) -> Result<RepoSyncResult, RepoSyncError> {
|
) -> Result<RepoSyncResult, RepoSyncError> {
|
||||||
match (policy.sync_preference, rrdp_notification_uri) {
|
match (policy.sync_preference, rrdp_notification_uri) {
|
||||||
(SyncPreference::RrdpThenRsync, Some(notification_uri)) => {
|
(SyncPreference::RrdpThenRsync, Some(notification_uri)) => {
|
||||||
match try_rrdp_sync_with_retry(store, notification_uri, http_fetcher, timing) {
|
match try_rrdp_sync_with_retry(
|
||||||
|
store,
|
||||||
|
notification_uri,
|
||||||
|
http_fetcher,
|
||||||
|
timing,
|
||||||
|
download_log,
|
||||||
|
) {
|
||||||
Ok(written) => {
|
Ok(written) => {
|
||||||
if let Some(t) = timing.as_ref() {
|
if let Some(t) = timing.as_ref() {
|
||||||
t.record_count("repo_sync_rrdp_ok_total", 1);
|
t.record_count("repo_sync_rrdp_ok_total", 1);
|
||||||
@ -80,8 +89,13 @@ pub fn sync_publication_point(
|
|||||||
.with_rfc_refs(&[RfcRef("RFC 8182 §3.4.5")])
|
.with_rfc_refs(&[RfcRef("RFC 8182 §3.4.5")])
|
||||||
.with_context(notification_uri),
|
.with_context(notification_uri),
|
||||||
];
|
];
|
||||||
let written =
|
let written = rsync_sync_into_raw_objects(
|
||||||
rsync_sync_into_raw_objects(store, rsync_base_uri, rsync_fetcher, timing)?;
|
store,
|
||||||
|
rsync_base_uri,
|
||||||
|
rsync_fetcher,
|
||||||
|
timing,
|
||||||
|
download_log,
|
||||||
|
)?;
|
||||||
if let Some(t) = timing.as_ref() {
|
if let Some(t) = timing.as_ref() {
|
||||||
t.record_count("repo_sync_rsync_fallback_ok_total", 1);
|
t.record_count("repo_sync_rsync_fallback_ok_total", 1);
|
||||||
t.record_count("repo_sync_rsync_objects_written_total", written as u64);
|
t.record_count("repo_sync_rsync_objects_written_total", written as u64);
|
||||||
@ -95,8 +109,13 @@ pub fn sync_publication_point(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
_ => {
|
_ => {
|
||||||
let written =
|
let written = rsync_sync_into_raw_objects(
|
||||||
rsync_sync_into_raw_objects(store, rsync_base_uri, rsync_fetcher, timing)?;
|
store,
|
||||||
|
rsync_base_uri,
|
||||||
|
rsync_fetcher,
|
||||||
|
timing,
|
||||||
|
download_log,
|
||||||
|
)?;
|
||||||
if let Some(t) = timing.as_ref() {
|
if let Some(t) = timing.as_ref() {
|
||||||
t.record_count("repo_sync_rsync_direct_total", 1);
|
t.record_count("repo_sync_rsync_direct_total", 1);
|
||||||
t.record_count("repo_sync_rsync_objects_written_total", written as u64);
|
t.record_count("repo_sync_rsync_objects_written_total", written as u64);
|
||||||
@ -115,6 +134,7 @@ fn try_rrdp_sync(
|
|||||||
notification_uri: &str,
|
notification_uri: &str,
|
||||||
http_fetcher: &dyn HttpFetcher,
|
http_fetcher: &dyn HttpFetcher,
|
||||||
timing: Option<&TimingHandle>,
|
timing: Option<&TimingHandle>,
|
||||||
|
download_log: Option<&DownloadLogHandle>,
|
||||||
) -> Result<usize, RrdpSyncError> {
|
) -> Result<usize, RrdpSyncError> {
|
||||||
let notification_xml = {
|
let notification_xml = {
|
||||||
let _step = timing
|
let _step = timing
|
||||||
@ -123,17 +143,27 @@ fn try_rrdp_sync(
|
|||||||
let _total = timing
|
let _total = timing
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|t| t.span_phase("rrdp_fetch_notification_total"));
|
.map(|t| t.span_phase("rrdp_fetch_notification_total"));
|
||||||
|
let mut dl_span = download_log.map(|dl| {
|
||||||
|
dl.span_download(AuditDownloadKind::RrdpNotification, notification_uri)
|
||||||
|
});
|
||||||
match http_fetcher.fetch(notification_uri) {
|
match http_fetcher.fetch(notification_uri) {
|
||||||
Ok(v) => {
|
Ok(v) => {
|
||||||
if let Some(t) = timing.as_ref() {
|
if let Some(t) = timing.as_ref() {
|
||||||
t.record_count("rrdp_notification_fetch_ok_total", 1);
|
t.record_count("rrdp_notification_fetch_ok_total", 1);
|
||||||
}
|
}
|
||||||
|
if let Some(s) = dl_span.as_mut() {
|
||||||
|
s.set_bytes(v.len() as u64);
|
||||||
|
s.set_ok();
|
||||||
|
}
|
||||||
v
|
v
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
if let Some(t) = timing.as_ref() {
|
if let Some(t) = timing.as_ref() {
|
||||||
t.record_count("rrdp_notification_fetch_fail_total", 1);
|
t.record_count("rrdp_notification_fetch_fail_total", 1);
|
||||||
}
|
}
|
||||||
|
if let Some(s) = dl_span.as_mut() {
|
||||||
|
s.set_err(e.clone());
|
||||||
|
}
|
||||||
return Err(RrdpSyncError::Fetch(e));
|
return Err(RrdpSyncError::Fetch(e));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -145,12 +175,13 @@ fn try_rrdp_sync(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
sync_from_notification_with_timing(
|
sync_from_notification_with_timing_and_download_log(
|
||||||
store,
|
store,
|
||||||
notification_uri,
|
notification_uri,
|
||||||
¬ification_xml,
|
¬ification_xml,
|
||||||
http_fetcher,
|
http_fetcher,
|
||||||
timing,
|
timing,
|
||||||
|
download_log,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -183,6 +214,7 @@ fn try_rrdp_sync_with_retry(
|
|||||||
notification_uri: &str,
|
notification_uri: &str,
|
||||||
http_fetcher: &dyn HttpFetcher,
|
http_fetcher: &dyn HttpFetcher,
|
||||||
timing: Option<&TimingHandle>,
|
timing: Option<&TimingHandle>,
|
||||||
|
download_log: Option<&DownloadLogHandle>,
|
||||||
) -> Result<usize, RrdpSyncError> {
|
) -> Result<usize, RrdpSyncError> {
|
||||||
let backoffs = rrdp_retry_backoffs();
|
let backoffs = rrdp_retry_backoffs();
|
||||||
let max_attempts = backoffs.len().saturating_add(1).max(1);
|
let max_attempts = backoffs.len().saturating_add(1).max(1);
|
||||||
@ -194,7 +226,7 @@ fn try_rrdp_sync_with_retry(
|
|||||||
t.record_count("rrdp_retry_attempt_total", 1);
|
t.record_count("rrdp_retry_attempt_total", 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
match try_rrdp_sync(store, notification_uri, http_fetcher, timing) {
|
match try_rrdp_sync(store, notification_uri, http_fetcher, timing, download_log) {
|
||||||
Ok(written) => {
|
Ok(written) => {
|
||||||
if attempt > 1 {
|
if attempt > 1 {
|
||||||
if let Some(t) = timing.as_ref() {
|
if let Some(t) = timing.as_ref() {
|
||||||
@ -244,12 +276,31 @@ fn rsync_sync_into_raw_objects(
|
|||||||
rsync_base_uri: &str,
|
rsync_base_uri: &str,
|
||||||
rsync_fetcher: &dyn RsyncFetcher,
|
rsync_fetcher: &dyn RsyncFetcher,
|
||||||
timing: Option<&TimingHandle>,
|
timing: Option<&TimingHandle>,
|
||||||
|
download_log: Option<&DownloadLogHandle>,
|
||||||
) -> Result<usize, RepoSyncError> {
|
) -> Result<usize, RepoSyncError> {
|
||||||
let _s = timing
|
let _s = timing
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|t| t.span_rrdp_repo_step(rsync_base_uri, "rsync_fetch_objects"));
|
.map(|t| t.span_rrdp_repo_step(rsync_base_uri, "rsync_fetch_objects"));
|
||||||
let _p = timing.as_ref().map(|t| t.span_phase("rsync_fetch_total"));
|
let _p = timing.as_ref().map(|t| t.span_phase("rsync_fetch_total"));
|
||||||
let objects = rsync_fetcher.fetch_objects(rsync_base_uri)?;
|
let mut dl_span =
|
||||||
|
download_log.map(|dl| dl.span_download(AuditDownloadKind::Rsync, rsync_base_uri));
|
||||||
|
let objects = match rsync_fetcher.fetch_objects(rsync_base_uri) {
|
||||||
|
Ok(v) => {
|
||||||
|
let bytes_total: u64 = v.iter().map(|(_u, b)| b.len() as u64).sum::<u64>();
|
||||||
|
if let Some(s) = dl_span.as_mut() {
|
||||||
|
s.set_objects(v.len() as u64, bytes_total);
|
||||||
|
s.set_bytes(bytes_total);
|
||||||
|
s.set_ok();
|
||||||
|
}
|
||||||
|
v
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
if let Some(s) = dl_span.as_mut() {
|
||||||
|
s.set_err(e.to_string());
|
||||||
|
}
|
||||||
|
return Err(e.into());
|
||||||
|
}
|
||||||
|
};
|
||||||
if let Some(t) = timing.as_ref() {
|
if let Some(t) = timing.as_ref() {
|
||||||
t.record_count("rsync_objects_fetched_total", objects.len() as u64);
|
t.record_count("rsync_objects_fetched_total", objects.len() as u64);
|
||||||
let bytes_total: u64 = objects.iter().map(|(_u, b)| b.len() as u64).sum::<u64>();
|
let bytes_total: u64 = objects.iter().map(|(_u, b)| b.len() as u64).sum::<u64>();
|
||||||
@ -270,6 +321,7 @@ mod tests {
|
|||||||
use crate::analysis::timing::{TimingHandle, TimingMeta};
|
use crate::analysis::timing::{TimingHandle, TimingMeta};
|
||||||
use crate::fetch::rsync::LocalDirRsyncFetcher;
|
use crate::fetch::rsync::LocalDirRsyncFetcher;
|
||||||
use crate::sync::rrdp::Fetcher as HttpFetcher;
|
use crate::sync::rrdp::Fetcher as HttpFetcher;
|
||||||
|
use crate::sync::rrdp::RrdpState;
|
||||||
use base64::Engine;
|
use base64::Engine;
|
||||||
use sha2::Digest;
|
use sha2::Digest;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
@ -364,6 +416,7 @@ mod tests {
|
|||||||
let http = DummyHttpFetcher;
|
let http = DummyHttpFetcher;
|
||||||
let rsync = LocalDirRsyncFetcher::new(&repo_dir);
|
let rsync = LocalDirRsyncFetcher::new(&repo_dir);
|
||||||
|
|
||||||
|
let download_log = DownloadLogHandle::new();
|
||||||
let out = sync_publication_point(
|
let out = sync_publication_point(
|
||||||
&store,
|
&store,
|
||||||
&policy,
|
&policy,
|
||||||
@ -372,12 +425,22 @@ mod tests {
|
|||||||
&http,
|
&http,
|
||||||
&rsync,
|
&rsync,
|
||||||
Some(&timing),
|
Some(&timing),
|
||||||
|
Some(&download_log),
|
||||||
)
|
)
|
||||||
.expect("sync ok");
|
.expect("sync ok");
|
||||||
|
|
||||||
assert_eq!(out.source, RepoSyncSource::Rsync);
|
assert_eq!(out.source, RepoSyncSource::Rsync);
|
||||||
assert_eq!(out.objects_written, 3);
|
assert_eq!(out.objects_written, 3);
|
||||||
|
|
||||||
|
let events = download_log.snapshot_events();
|
||||||
|
assert_eq!(events.len(), 1);
|
||||||
|
assert_eq!(events[0].kind, AuditDownloadKind::Rsync);
|
||||||
|
assert!(events[0].success);
|
||||||
|
assert_eq!(events[0].bytes, Some(9));
|
||||||
|
let objects = events[0].objects.as_ref().expect("objects stat");
|
||||||
|
assert_eq!(objects.objects_count, 3);
|
||||||
|
assert_eq!(objects.objects_bytes_total, 9);
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
store.get_raw("rsync://example.test/repo/a.mft").unwrap(),
|
store.get_raw("rsync://example.test/repo/a.mft").unwrap(),
|
||||||
Some(b"mft".to_vec())
|
Some(b"mft".to_vec())
|
||||||
@ -481,6 +544,7 @@ mod tests {
|
|||||||
..Policy::default()
|
..Policy::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let download_log = DownloadLogHandle::new();
|
||||||
let out = sync_publication_point(
|
let out = sync_publication_point(
|
||||||
&store,
|
&store,
|
||||||
&policy,
|
&policy,
|
||||||
@ -489,6 +553,7 @@ mod tests {
|
|||||||
&http,
|
&http,
|
||||||
&PanicRsyncFetcher,
|
&PanicRsyncFetcher,
|
||||||
Some(&timing),
|
Some(&timing),
|
||||||
|
Some(&download_log),
|
||||||
)
|
)
|
||||||
.expect("sync ok");
|
.expect("sync ok");
|
||||||
|
|
||||||
@ -498,6 +563,30 @@ mod tests {
|
|||||||
Some(published_bytes.to_vec())
|
Some(published_bytes.to_vec())
|
||||||
);
|
);
|
||||||
|
|
||||||
|
let events = download_log.snapshot_events();
|
||||||
|
assert_eq!(events.len(), 4, "expected 3x notification + 1x snapshot");
|
||||||
|
assert_eq!(
|
||||||
|
events
|
||||||
|
.iter()
|
||||||
|
.filter(|e| e.kind == AuditDownloadKind::RrdpNotification)
|
||||||
|
.count(),
|
||||||
|
3
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
events
|
||||||
|
.iter()
|
||||||
|
.filter(|e| e.kind == AuditDownloadKind::RrdpSnapshot)
|
||||||
|
.count(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
events
|
||||||
|
.iter()
|
||||||
|
.filter(|e| e.kind == AuditDownloadKind::RrdpNotification && !e.success)
|
||||||
|
.count(),
|
||||||
|
2
|
||||||
|
);
|
||||||
|
|
||||||
let v = timing_to_json(temp.path(), &timing);
|
let v = timing_to_json(temp.path(), &timing);
|
||||||
let counts = v.get("counts").expect("counts");
|
let counts = v.get("counts").expect("counts");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@ -542,11 +631,12 @@ mod tests {
|
|||||||
&[(published_uri, published_bytes)],
|
&[(published_uri, published_bytes)],
|
||||||
);
|
);
|
||||||
// Intentionally wrong hash to trigger protocol error (SnapshotHashMismatch).
|
// Intentionally wrong hash to trigger protocol error (SnapshotHashMismatch).
|
||||||
|
let wrong_hash = "00".repeat(32);
|
||||||
let notif = notification_xml(
|
let notif = notification_xml(
|
||||||
"9df4b597-af9e-4dca-bdda-719cce2c4e28",
|
"9df4b597-af9e-4dca-bdda-719cce2c4e28",
|
||||||
1,
|
1,
|
||||||
snapshot_uri,
|
snapshot_uri,
|
||||||
"00",
|
&wrong_hash,
|
||||||
);
|
);
|
||||||
|
|
||||||
let mut map = HashMap::new();
|
let mut map = HashMap::new();
|
||||||
@ -569,6 +659,7 @@ mod tests {
|
|||||||
..Policy::default()
|
..Policy::default()
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let download_log = DownloadLogHandle::new();
|
||||||
let out = sync_publication_point(
|
let out = sync_publication_point(
|
||||||
&store,
|
&store,
|
||||||
&policy,
|
&policy,
|
||||||
@ -577,6 +668,7 @@ mod tests {
|
|||||||
&http,
|
&http,
|
||||||
&EmptyRsyncFetcher,
|
&EmptyRsyncFetcher,
|
||||||
Some(&timing),
|
Some(&timing),
|
||||||
|
Some(&download_log),
|
||||||
)
|
)
|
||||||
.expect("sync ok");
|
.expect("sync ok");
|
||||||
|
|
||||||
@ -588,6 +680,15 @@ mod tests {
|
|||||||
"expected RRDP fallback warning"
|
"expected RRDP fallback warning"
|
||||||
);
|
);
|
||||||
|
|
||||||
|
let events = download_log.snapshot_events();
|
||||||
|
assert_eq!(events.len(), 3, "expected notification + snapshot + rsync fallback");
|
||||||
|
assert_eq!(events[0].kind, AuditDownloadKind::RrdpNotification);
|
||||||
|
assert!(events[0].success);
|
||||||
|
assert_eq!(events[1].kind, AuditDownloadKind::RrdpSnapshot);
|
||||||
|
assert!(events[1].success);
|
||||||
|
assert_eq!(events[2].kind, AuditDownloadKind::Rsync);
|
||||||
|
assert!(events[2].success);
|
||||||
|
|
||||||
let v = timing_to_json(temp.path(), &timing);
|
let v = timing_to_json(temp.path(), &timing);
|
||||||
let counts = v.get("counts").expect("counts");
|
let counts = v.get("counts").expect("counts");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@ -615,4 +716,112 @@ mod tests {
|
|||||||
Some(1)
|
Some(1)
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rrdp_delta_fetches_are_logged_even_if_snapshot_fallback_is_used() {
|
||||||
|
let temp = tempfile::tempdir().expect("tempdir");
|
||||||
|
let store_dir = temp.path().join("db");
|
||||||
|
let store = RocksStore::open(&store_dir).expect("open rocksdb");
|
||||||
|
|
||||||
|
let timing = TimingHandle::new(TimingMeta {
|
||||||
|
recorded_at_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(),
|
||||||
|
validation_time_utc_rfc3339: "2026-02-28T00:00:00Z".to_string(),
|
||||||
|
tal_url: None,
|
||||||
|
db_path: Some(store_dir.to_string_lossy().into_owned()),
|
||||||
|
});
|
||||||
|
|
||||||
|
let notification_uri = "https://example.test/notification.xml";
|
||||||
|
let snapshot_uri = "https://example.test/snapshot.xml";
|
||||||
|
let delta_2_uri = "https://example.test/delta_2.xml";
|
||||||
|
let delta_3_uri = "https://example.test/delta_3.xml";
|
||||||
|
let published_uri = "rsync://example.test/repo/a.mft";
|
||||||
|
let published_bytes = b"x";
|
||||||
|
|
||||||
|
let sid = "9df4b597-af9e-4dca-bdda-719cce2c4e28";
|
||||||
|
|
||||||
|
// Seed old RRDP state so sync_from_notification tries deltas (RFC 8182 §3.4.1).
|
||||||
|
let state = RrdpState {
|
||||||
|
session_id: sid.to_string(),
|
||||||
|
serial: 1,
|
||||||
|
};
|
||||||
|
let state_bytes = state.encode().expect("encode state");
|
||||||
|
store
|
||||||
|
.put_rrdp_state(notification_uri, &state_bytes)
|
||||||
|
.expect("seed state");
|
||||||
|
|
||||||
|
let delta_2 = format!(
|
||||||
|
r#"<delta xmlns="http://www.ripe.net/rpki/rrdp" version="1" session_id="{sid}" serial="2"></delta>"#
|
||||||
|
)
|
||||||
|
.into_bytes();
|
||||||
|
let delta_3 = format!(
|
||||||
|
r#"<delta xmlns="http://www.ripe.net/rpki/rrdp" version="1" session_id="{sid}" serial="3"></delta>"#
|
||||||
|
)
|
||||||
|
.into_bytes();
|
||||||
|
let delta_2_hash = hex::encode(sha2::Sha256::digest(&delta_2));
|
||||||
|
let delta_3_hash = hex::encode(sha2::Sha256::digest(&delta_3));
|
||||||
|
|
||||||
|
let snapshot = snapshot_xml(sid, 3, &[(published_uri, published_bytes)]);
|
||||||
|
let snapshot_hash = hex::encode(sha2::Sha256::digest(&snapshot));
|
||||||
|
let notif = format!(
|
||||||
|
r#"<notification xmlns="http://www.ripe.net/rpki/rrdp" version="1" session_id="{sid}" serial="3"><snapshot uri="{snapshot_uri}" hash="{snapshot_hash}"/><delta serial="2" uri="{delta_2_uri}" hash="{delta_2_hash}"/><delta serial="3" uri="{delta_3_uri}" hash="{delta_3_hash}"/></notification>"#
|
||||||
|
)
|
||||||
|
.into_bytes();
|
||||||
|
|
||||||
|
let mut map = HashMap::new();
|
||||||
|
map.insert(notification_uri.to_string(), notif);
|
||||||
|
map.insert(snapshot_uri.to_string(), snapshot);
|
||||||
|
map.insert(delta_2_uri.to_string(), delta_2);
|
||||||
|
map.insert(delta_3_uri.to_string(), delta_3);
|
||||||
|
let http = MapFetcher { map };
|
||||||
|
|
||||||
|
let policy = Policy {
|
||||||
|
sync_preference: SyncPreference::RrdpThenRsync,
|
||||||
|
..Policy::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
let download_log = DownloadLogHandle::new();
|
||||||
|
let out = sync_publication_point(
|
||||||
|
&store,
|
||||||
|
&policy,
|
||||||
|
Some(notification_uri),
|
||||||
|
"rsync://example.test/repo/",
|
||||||
|
&http,
|
||||||
|
&PanicRsyncFetcher,
|
||||||
|
Some(&timing),
|
||||||
|
Some(&download_log),
|
||||||
|
)
|
||||||
|
.expect("sync ok");
|
||||||
|
|
||||||
|
assert_eq!(out.source, RepoSyncSource::Rrdp);
|
||||||
|
assert_eq!(out.objects_written, 1);
|
||||||
|
assert_eq!(
|
||||||
|
store.get_raw(published_uri).unwrap(),
|
||||||
|
Some(published_bytes.to_vec())
|
||||||
|
);
|
||||||
|
|
||||||
|
let events = download_log.snapshot_events();
|
||||||
|
assert_eq!(events.len(), 4);
|
||||||
|
assert_eq!(
|
||||||
|
events
|
||||||
|
.iter()
|
||||||
|
.filter(|e| e.kind == AuditDownloadKind::RrdpNotification)
|
||||||
|
.count(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
events
|
||||||
|
.iter()
|
||||||
|
.filter(|e| e.kind == AuditDownloadKind::RrdpDelta)
|
||||||
|
.count(),
|
||||||
|
2
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
events
|
||||||
|
.iter()
|
||||||
|
.filter(|e| e.kind == AuditDownloadKind::RrdpSnapshot)
|
||||||
|
.count(),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
assert!(events.iter().all(|e| e.success));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
117
src/sync/rrdp.rs
117
src/sync/rrdp.rs
@ -1,4 +1,6 @@
|
|||||||
use crate::analysis::timing::TimingHandle;
|
use crate::analysis::timing::TimingHandle;
|
||||||
|
use crate::audit::AuditDownloadKind;
|
||||||
|
use crate::audit_downloads::DownloadLogHandle;
|
||||||
use crate::storage::RocksStore;
|
use crate::storage::RocksStore;
|
||||||
use crate::storage::RrdpDeltaOp;
|
use crate::storage::RrdpDeltaOp;
|
||||||
use base64::Engine;
|
use base64::Engine;
|
||||||
@ -432,7 +434,14 @@ pub fn sync_from_notification_snapshot(
|
|||||||
notification_xml: &[u8],
|
notification_xml: &[u8],
|
||||||
fetcher: &dyn Fetcher,
|
fetcher: &dyn Fetcher,
|
||||||
) -> RrdpSyncResult<usize> {
|
) -> RrdpSyncResult<usize> {
|
||||||
sync_from_notification_snapshot_inner(store, notification_uri, notification_xml, fetcher, None)
|
sync_from_notification_snapshot_inner(
|
||||||
|
store,
|
||||||
|
notification_uri,
|
||||||
|
notification_xml,
|
||||||
|
fetcher,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn sync_from_notification_snapshot_with_timing(
|
pub fn sync_from_notification_snapshot_with_timing(
|
||||||
@ -448,6 +457,25 @@ pub fn sync_from_notification_snapshot_with_timing(
|
|||||||
notification_xml,
|
notification_xml,
|
||||||
fetcher,
|
fetcher,
|
||||||
timing,
|
timing,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn sync_from_notification_snapshot_with_timing_and_download_log(
|
||||||
|
store: &RocksStore,
|
||||||
|
notification_uri: &str,
|
||||||
|
notification_xml: &[u8],
|
||||||
|
fetcher: &dyn Fetcher,
|
||||||
|
timing: Option<&TimingHandle>,
|
||||||
|
download_log: Option<&DownloadLogHandle>,
|
||||||
|
) -> RrdpSyncResult<usize> {
|
||||||
|
sync_from_notification_snapshot_inner(
|
||||||
|
store,
|
||||||
|
notification_uri,
|
||||||
|
notification_xml,
|
||||||
|
fetcher,
|
||||||
|
timing,
|
||||||
|
download_log,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -457,6 +485,7 @@ fn sync_from_notification_snapshot_inner(
|
|||||||
notification_xml: &[u8],
|
notification_xml: &[u8],
|
||||||
fetcher: &dyn Fetcher,
|
fetcher: &dyn Fetcher,
|
||||||
timing: Option<&TimingHandle>,
|
timing: Option<&TimingHandle>,
|
||||||
|
download_log: Option<&DownloadLogHandle>,
|
||||||
) -> RrdpSyncResult<usize> {
|
) -> RrdpSyncResult<usize> {
|
||||||
let _parse_step = timing
|
let _parse_step = timing
|
||||||
.as_ref()
|
.as_ref()
|
||||||
@ -470,16 +499,30 @@ fn sync_from_notification_snapshot_inner(
|
|||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|t| t.span_rrdp_repo_step(notification_uri, "fetch_snapshot"));
|
.map(|t| t.span_rrdp_repo_step(notification_uri, "fetch_snapshot"));
|
||||||
let _fetch_total = timing.as_ref().map(|t| t.span_phase("rrdp_fetch_snapshot_total"));
|
let _fetch_total = timing.as_ref().map(|t| t.span_phase("rrdp_fetch_snapshot_total"));
|
||||||
let snapshot_xml = fetcher.fetch(¬if.snapshot_uri).map_err(|e| {
|
let mut dl_span = download_log
|
||||||
|
.map(|dl| dl.span_download(AuditDownloadKind::RrdpSnapshot, ¬if.snapshot_uri));
|
||||||
|
let snapshot_xml = match fetcher.fetch(¬if.snapshot_uri) {
|
||||||
|
Ok(v) => {
|
||||||
|
if let Some(t) = timing.as_ref() {
|
||||||
|
t.record_count("rrdp_snapshot_fetch_ok_total", 1);
|
||||||
|
t.record_count("rrdp_snapshot_bytes_total", v.len() as u64);
|
||||||
|
}
|
||||||
|
if let Some(s) = dl_span.as_mut() {
|
||||||
|
s.set_bytes(v.len() as u64);
|
||||||
|
s.set_ok();
|
||||||
|
}
|
||||||
|
v
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
if let Some(t) = timing.as_ref() {
|
if let Some(t) = timing.as_ref() {
|
||||||
t.record_count("rrdp_snapshot_fetch_fail_total", 1);
|
t.record_count("rrdp_snapshot_fetch_fail_total", 1);
|
||||||
}
|
}
|
||||||
RrdpSyncError::Fetch(e)
|
if let Some(s) = dl_span.as_mut() {
|
||||||
})?;
|
s.set_err(e.clone());
|
||||||
if let Some(t) = timing.as_ref() {
|
|
||||||
t.record_count("rrdp_snapshot_fetch_ok_total", 1);
|
|
||||||
t.record_count("rrdp_snapshot_bytes_total", snapshot_xml.len() as u64);
|
|
||||||
}
|
}
|
||||||
|
return Err(RrdpSyncError::Fetch(e));
|
||||||
|
}
|
||||||
|
};
|
||||||
drop(_fetch_step);
|
drop(_fetch_step);
|
||||||
drop(_fetch_total);
|
drop(_fetch_total);
|
||||||
|
|
||||||
@ -535,7 +578,7 @@ pub fn sync_from_notification(
|
|||||||
notification_xml: &[u8],
|
notification_xml: &[u8],
|
||||||
fetcher: &dyn Fetcher,
|
fetcher: &dyn Fetcher,
|
||||||
) -> RrdpSyncResult<usize> {
|
) -> RrdpSyncResult<usize> {
|
||||||
sync_from_notification_inner(store, notification_uri, notification_xml, fetcher, None)
|
sync_from_notification_inner(store, notification_uri, notification_xml, fetcher, None, None)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn sync_from_notification_with_timing(
|
pub fn sync_from_notification_with_timing(
|
||||||
@ -545,7 +588,25 @@ pub fn sync_from_notification_with_timing(
|
|||||||
fetcher: &dyn Fetcher,
|
fetcher: &dyn Fetcher,
|
||||||
timing: Option<&TimingHandle>,
|
timing: Option<&TimingHandle>,
|
||||||
) -> RrdpSyncResult<usize> {
|
) -> RrdpSyncResult<usize> {
|
||||||
sync_from_notification_inner(store, notification_uri, notification_xml, fetcher, timing)
|
sync_from_notification_inner(store, notification_uri, notification_xml, fetcher, timing, None)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn sync_from_notification_with_timing_and_download_log(
|
||||||
|
store: &RocksStore,
|
||||||
|
notification_uri: &str,
|
||||||
|
notification_xml: &[u8],
|
||||||
|
fetcher: &dyn Fetcher,
|
||||||
|
timing: Option<&TimingHandle>,
|
||||||
|
download_log: Option<&DownloadLogHandle>,
|
||||||
|
) -> RrdpSyncResult<usize> {
|
||||||
|
sync_from_notification_inner(
|
||||||
|
store,
|
||||||
|
notification_uri,
|
||||||
|
notification_xml,
|
||||||
|
fetcher,
|
||||||
|
timing,
|
||||||
|
download_log,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sync_from_notification_inner(
|
fn sync_from_notification_inner(
|
||||||
@ -554,6 +615,7 @@ fn sync_from_notification_inner(
|
|||||||
notification_xml: &[u8],
|
notification_xml: &[u8],
|
||||||
fetcher: &dyn Fetcher,
|
fetcher: &dyn Fetcher,
|
||||||
timing: Option<&TimingHandle>,
|
timing: Option<&TimingHandle>,
|
||||||
|
download_log: Option<&DownloadLogHandle>,
|
||||||
) -> RrdpSyncResult<usize> {
|
) -> RrdpSyncResult<usize> {
|
||||||
let _parse_step = timing
|
let _parse_step = timing
|
||||||
.as_ref()
|
.as_ref()
|
||||||
@ -632,18 +694,27 @@ fn sync_from_notification_inner(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let mut dl_span = download_log
|
||||||
|
.map(|dl| dl.span_download(AuditDownloadKind::RrdpDelta, &dref.uri));
|
||||||
match fetcher.fetch(&dref.uri) {
|
match fetcher.fetch(&dref.uri) {
|
||||||
Ok(bytes) => {
|
Ok(bytes) => {
|
||||||
if let Some(t) = timing.as_ref() {
|
if let Some(t) = timing.as_ref() {
|
||||||
t.record_count("rrdp_delta_fetch_ok_total", 1);
|
t.record_count("rrdp_delta_fetch_ok_total", 1);
|
||||||
t.record_count("rrdp_delta_bytes_total", bytes.len() as u64);
|
t.record_count("rrdp_delta_bytes_total", bytes.len() as u64);
|
||||||
}
|
}
|
||||||
|
if let Some(s) = dl_span.as_mut() {
|
||||||
|
s.set_bytes(bytes.len() as u64);
|
||||||
|
s.set_ok();
|
||||||
|
}
|
||||||
fetched.push((serial, dref.hash_sha256, bytes))
|
fetched.push((serial, dref.hash_sha256, bytes))
|
||||||
}
|
}
|
||||||
Err(_) => {
|
Err(e) => {
|
||||||
if let Some(t) = timing.as_ref() {
|
if let Some(t) = timing.as_ref() {
|
||||||
t.record_count("rrdp_delta_fetch_fail_total", 1);
|
t.record_count("rrdp_delta_fetch_fail_total", 1);
|
||||||
}
|
}
|
||||||
|
if let Some(s) = dl_span.as_mut() {
|
||||||
|
s.set_err(e.clone());
|
||||||
|
}
|
||||||
fetch_ok = false;
|
fetch_ok = false;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@ -710,16 +781,30 @@ fn sync_from_notification_inner(
|
|||||||
.as_ref()
|
.as_ref()
|
||||||
.map(|t| t.span_rrdp_repo_step(notification_uri, "fetch_snapshot"));
|
.map(|t| t.span_rrdp_repo_step(notification_uri, "fetch_snapshot"));
|
||||||
let _fetch_total = timing.as_ref().map(|t| t.span_phase("rrdp_fetch_snapshot_total"));
|
let _fetch_total = timing.as_ref().map(|t| t.span_phase("rrdp_fetch_snapshot_total"));
|
||||||
let snapshot_xml = fetcher.fetch(¬if.snapshot_uri).map_err(|e| {
|
let mut dl_span = download_log
|
||||||
|
.map(|dl| dl.span_download(AuditDownloadKind::RrdpSnapshot, ¬if.snapshot_uri));
|
||||||
|
let snapshot_xml = match fetcher.fetch(¬if.snapshot_uri) {
|
||||||
|
Ok(v) => {
|
||||||
|
if let Some(t) = timing.as_ref() {
|
||||||
|
t.record_count("rrdp_snapshot_fetch_ok_total", 1);
|
||||||
|
t.record_count("rrdp_snapshot_bytes_total", v.len() as u64);
|
||||||
|
}
|
||||||
|
if let Some(s) = dl_span.as_mut() {
|
||||||
|
s.set_bytes(v.len() as u64);
|
||||||
|
s.set_ok();
|
||||||
|
}
|
||||||
|
v
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
if let Some(t) = timing.as_ref() {
|
if let Some(t) = timing.as_ref() {
|
||||||
t.record_count("rrdp_snapshot_fetch_fail_total", 1);
|
t.record_count("rrdp_snapshot_fetch_fail_total", 1);
|
||||||
}
|
}
|
||||||
RrdpSyncError::Fetch(e)
|
if let Some(s) = dl_span.as_mut() {
|
||||||
})?;
|
s.set_err(e.clone());
|
||||||
if let Some(t) = timing.as_ref() {
|
|
||||||
t.record_count("rrdp_snapshot_fetch_ok_total", 1);
|
|
||||||
t.record_count("rrdp_snapshot_bytes_total", snapshot_xml.len() as u64);
|
|
||||||
}
|
}
|
||||||
|
return Err(RrdpSyncError::Fetch(e));
|
||||||
|
}
|
||||||
|
};
|
||||||
drop(_fetch_step);
|
drop(_fetch_step);
|
||||||
drop(_fetch_total);
|
drop(_fetch_total);
|
||||||
|
|
||||||
|
|||||||
@ -52,6 +52,7 @@ pub fn run_publication_point_once(
|
|||||||
http_fetcher,
|
http_fetcher,
|
||||||
rsync_fetcher,
|
rsync_fetcher,
|
||||||
None,
|
None,
|
||||||
|
None,
|
||||||
)?;
|
)?;
|
||||||
|
|
||||||
let publication_point = process_manifest_publication_point(
|
let publication_point = process_manifest_publication_point(
|
||||||
|
|||||||
@ -2,6 +2,7 @@ use url::Url;
|
|||||||
|
|
||||||
use crate::analysis::timing::TimingHandle;
|
use crate::analysis::timing::TimingHandle;
|
||||||
use crate::audit::PublicationPointAudit;
|
use crate::audit::PublicationPointAudit;
|
||||||
|
use crate::audit_downloads::DownloadLogHandle;
|
||||||
use crate::data_model::ta::TrustAnchor;
|
use crate::data_model::ta::TrustAnchor;
|
||||||
use crate::sync::rrdp::Fetcher;
|
use crate::sync::rrdp::Fetcher;
|
||||||
use crate::validation::from_tal::{
|
use crate::validation::from_tal::{
|
||||||
@ -27,6 +28,8 @@ pub struct RunTreeFromTalAuditOutput {
|
|||||||
pub discovery: DiscoveredRootCaInstance,
|
pub discovery: DiscoveredRootCaInstance,
|
||||||
pub tree: TreeRunOutput,
|
pub tree: TreeRunOutput,
|
||||||
pub publication_points: Vec<PublicationPointAudit>,
|
pub publication_points: Vec<PublicationPointAudit>,
|
||||||
|
pub downloads: Vec<crate::audit::AuditDownloadEvent>,
|
||||||
|
pub download_stats: crate::audit::AuditDownloadStats,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, thiserror::Error)]
|
#[derive(Debug, thiserror::Error)]
|
||||||
@ -75,6 +78,7 @@ pub fn run_tree_from_tal_url_serial(
|
|||||||
rsync_fetcher,
|
rsync_fetcher,
|
||||||
validation_time,
|
validation_time,
|
||||||
timing: None,
|
timing: None,
|
||||||
|
download_log: None,
|
||||||
revalidate_only: config.revalidate_only,
|
revalidate_only: config.revalidate_only,
|
||||||
rrdp_dedup: true,
|
rrdp_dedup: true,
|
||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
@ -99,6 +103,7 @@ pub fn run_tree_from_tal_url_serial_audit(
|
|||||||
) -> Result<RunTreeFromTalAuditOutput, RunTreeFromTalError> {
|
) -> Result<RunTreeFromTalAuditOutput, RunTreeFromTalError> {
|
||||||
let discovery = discover_root_ca_instance_from_tal_url(http_fetcher, tal_url)?;
|
let discovery = discover_root_ca_instance_from_tal_url(http_fetcher, tal_url)?;
|
||||||
|
|
||||||
|
let download_log = DownloadLogHandle::new();
|
||||||
let runner = Rpkiv1PublicationPointRunner {
|
let runner = Rpkiv1PublicationPointRunner {
|
||||||
store,
|
store,
|
||||||
policy,
|
policy,
|
||||||
@ -106,6 +111,7 @@ pub fn run_tree_from_tal_url_serial_audit(
|
|||||||
rsync_fetcher,
|
rsync_fetcher,
|
||||||
validation_time,
|
validation_time,
|
||||||
timing: None,
|
timing: None,
|
||||||
|
download_log: Some(download_log.clone()),
|
||||||
revalidate_only: config.revalidate_only,
|
revalidate_only: config.revalidate_only,
|
||||||
rrdp_dedup: true,
|
rrdp_dedup: true,
|
||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
@ -119,10 +125,14 @@ pub fn run_tree_from_tal_url_serial_audit(
|
|||||||
publication_points,
|
publication_points,
|
||||||
} = run_tree_serial_audit(root, &runner, config)?;
|
} = run_tree_serial_audit(root, &runner, config)?;
|
||||||
|
|
||||||
|
let downloads = download_log.snapshot_events();
|
||||||
|
let download_stats = DownloadLogHandle::stats_from_events(&downloads);
|
||||||
Ok(RunTreeFromTalAuditOutput {
|
Ok(RunTreeFromTalAuditOutput {
|
||||||
discovery,
|
discovery,
|
||||||
tree,
|
tree,
|
||||||
publication_points,
|
publication_points,
|
||||||
|
downloads,
|
||||||
|
download_stats,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -140,6 +150,7 @@ pub fn run_tree_from_tal_url_serial_audit_with_timing(
|
|||||||
let discovery = discover_root_ca_instance_from_tal_url(http_fetcher, tal_url)?;
|
let discovery = discover_root_ca_instance_from_tal_url(http_fetcher, tal_url)?;
|
||||||
drop(_tal);
|
drop(_tal);
|
||||||
|
|
||||||
|
let download_log = DownloadLogHandle::new();
|
||||||
let runner = Rpkiv1PublicationPointRunner {
|
let runner = Rpkiv1PublicationPointRunner {
|
||||||
store,
|
store,
|
||||||
policy,
|
policy,
|
||||||
@ -147,6 +158,7 @@ pub fn run_tree_from_tal_url_serial_audit_with_timing(
|
|||||||
rsync_fetcher,
|
rsync_fetcher,
|
||||||
validation_time,
|
validation_time,
|
||||||
timing: Some(timing.clone()),
|
timing: Some(timing.clone()),
|
||||||
|
download_log: Some(download_log.clone()),
|
||||||
revalidate_only: config.revalidate_only,
|
revalidate_only: config.revalidate_only,
|
||||||
rrdp_dedup: true,
|
rrdp_dedup: true,
|
||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
@ -161,10 +173,14 @@ pub fn run_tree_from_tal_url_serial_audit_with_timing(
|
|||||||
publication_points,
|
publication_points,
|
||||||
} = run_tree_serial_audit(root, &runner, config)?;
|
} = run_tree_serial_audit(root, &runner, config)?;
|
||||||
|
|
||||||
|
let downloads = download_log.snapshot_events();
|
||||||
|
let download_stats = DownloadLogHandle::stats_from_events(&downloads);
|
||||||
Ok(RunTreeFromTalAuditOutput {
|
Ok(RunTreeFromTalAuditOutput {
|
||||||
discovery,
|
discovery,
|
||||||
tree,
|
tree,
|
||||||
publication_points,
|
publication_points,
|
||||||
|
downloads,
|
||||||
|
download_stats,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -189,6 +205,7 @@ pub fn run_tree_from_tal_and_ta_der_serial(
|
|||||||
rsync_fetcher,
|
rsync_fetcher,
|
||||||
validation_time,
|
validation_time,
|
||||||
timing: None,
|
timing: None,
|
||||||
|
download_log: None,
|
||||||
revalidate_only: config.revalidate_only,
|
revalidate_only: config.revalidate_only,
|
||||||
rrdp_dedup: true,
|
rrdp_dedup: true,
|
||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
@ -216,6 +233,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit(
|
|||||||
let discovery =
|
let discovery =
|
||||||
discover_root_ca_instance_from_tal_and_ta_der(tal_bytes, ta_der, resolved_ta_uri)?;
|
discover_root_ca_instance_from_tal_and_ta_der(tal_bytes, ta_der, resolved_ta_uri)?;
|
||||||
|
|
||||||
|
let download_log = DownloadLogHandle::new();
|
||||||
let runner = Rpkiv1PublicationPointRunner {
|
let runner = Rpkiv1PublicationPointRunner {
|
||||||
store,
|
store,
|
||||||
policy,
|
policy,
|
||||||
@ -223,6 +241,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit(
|
|||||||
rsync_fetcher,
|
rsync_fetcher,
|
||||||
validation_time,
|
validation_time,
|
||||||
timing: None,
|
timing: None,
|
||||||
|
download_log: Some(download_log.clone()),
|
||||||
revalidate_only: config.revalidate_only,
|
revalidate_only: config.revalidate_only,
|
||||||
rrdp_dedup: true,
|
rrdp_dedup: true,
|
||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
@ -236,10 +255,14 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit(
|
|||||||
publication_points,
|
publication_points,
|
||||||
} = run_tree_serial_audit(root, &runner, config)?;
|
} = run_tree_serial_audit(root, &runner, config)?;
|
||||||
|
|
||||||
|
let downloads = download_log.snapshot_events();
|
||||||
|
let download_stats = DownloadLogHandle::stats_from_events(&downloads);
|
||||||
Ok(RunTreeFromTalAuditOutput {
|
Ok(RunTreeFromTalAuditOutput {
|
||||||
discovery,
|
discovery,
|
||||||
tree,
|
tree,
|
||||||
publication_points,
|
publication_points,
|
||||||
|
downloads,
|
||||||
|
download_stats,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -260,6 +283,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit_with_timing(
|
|||||||
discover_root_ca_instance_from_tal_and_ta_der(tal_bytes, ta_der, resolved_ta_uri)?;
|
discover_root_ca_instance_from_tal_and_ta_der(tal_bytes, ta_der, resolved_ta_uri)?;
|
||||||
drop(_tal);
|
drop(_tal);
|
||||||
|
|
||||||
|
let download_log = DownloadLogHandle::new();
|
||||||
let runner = Rpkiv1PublicationPointRunner {
|
let runner = Rpkiv1PublicationPointRunner {
|
||||||
store,
|
store,
|
||||||
policy,
|
policy,
|
||||||
@ -267,6 +291,7 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit_with_timing(
|
|||||||
rsync_fetcher,
|
rsync_fetcher,
|
||||||
validation_time,
|
validation_time,
|
||||||
timing: Some(timing.clone()),
|
timing: Some(timing.clone()),
|
||||||
|
download_log: Some(download_log.clone()),
|
||||||
revalidate_only: config.revalidate_only,
|
revalidate_only: config.revalidate_only,
|
||||||
rrdp_dedup: true,
|
rrdp_dedup: true,
|
||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
@ -281,9 +306,13 @@ pub fn run_tree_from_tal_and_ta_der_serial_audit_with_timing(
|
|||||||
publication_points,
|
publication_points,
|
||||||
} = run_tree_serial_audit(root, &runner, config)?;
|
} = run_tree_serial_audit(root, &runner, config)?;
|
||||||
|
|
||||||
|
let downloads = download_log.snapshot_events();
|
||||||
|
let download_stats = DownloadLogHandle::stats_from_events(&downloads);
|
||||||
Ok(RunTreeFromTalAuditOutput {
|
Ok(RunTreeFromTalAuditOutput {
|
||||||
discovery,
|
discovery,
|
||||||
tree,
|
tree,
|
||||||
publication_points,
|
publication_points,
|
||||||
|
downloads,
|
||||||
|
download_stats,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@ -3,6 +3,7 @@ use crate::audit::{
|
|||||||
AuditObjectKind, AuditObjectResult, AuditWarning, ObjectAuditEntry, PublicationPointAudit,
|
AuditObjectKind, AuditObjectResult, AuditWarning, ObjectAuditEntry, PublicationPointAudit,
|
||||||
sha256_hex, sha256_hex_from_32,
|
sha256_hex, sha256_hex_from_32,
|
||||||
};
|
};
|
||||||
|
use crate::audit_downloads::DownloadLogHandle;
|
||||||
use crate::fetch::rsync::RsyncFetcher;
|
use crate::fetch::rsync::RsyncFetcher;
|
||||||
use crate::policy::Policy;
|
use crate::policy::Policy;
|
||||||
use crate::report::{RfcRef, Warning};
|
use crate::report::{RfcRef, Warning};
|
||||||
@ -33,6 +34,7 @@ pub struct Rpkiv1PublicationPointRunner<'a> {
|
|||||||
pub rsync_fetcher: &'a dyn RsyncFetcher,
|
pub rsync_fetcher: &'a dyn RsyncFetcher,
|
||||||
pub validation_time: time::OffsetDateTime,
|
pub validation_time: time::OffsetDateTime,
|
||||||
pub timing: Option<TimingHandle>,
|
pub timing: Option<TimingHandle>,
|
||||||
|
pub download_log: Option<DownloadLogHandle>,
|
||||||
pub revalidate_only: bool,
|
pub revalidate_only: bool,
|
||||||
/// In-run RRDP dedup: when RRDP is enabled, only sync each `rrdp_notification_uri` once per run.
|
/// In-run RRDP dedup: when RRDP is enabled, only sync each `rrdp_notification_uri` once per run.
|
||||||
///
|
///
|
||||||
@ -157,6 +159,7 @@ impl<'a> PublicationPointRunner for Rpkiv1PublicationPointRunner<'a> {
|
|||||||
self.http_fetcher,
|
self.http_fetcher,
|
||||||
self.rsync_fetcher,
|
self.rsync_fetcher,
|
||||||
self.timing.as_ref(),
|
self.timing.as_ref(),
|
||||||
|
self.download_log.as_ref(),
|
||||||
) {
|
) {
|
||||||
Ok(res) => {
|
Ok(res) => {
|
||||||
// Populate rsync dedup cache when we actually used rsync.
|
// Populate rsync dedup cache when we actually used rsync.
|
||||||
@ -1390,6 +1393,7 @@ authorityKeyIdentifier = keyid:always
|
|||||||
rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir),
|
rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir),
|
||||||
validation_time,
|
validation_time,
|
||||||
timing: None,
|
timing: None,
|
||||||
|
download_log: None,
|
||||||
revalidate_only: false,
|
revalidate_only: false,
|
||||||
rrdp_dedup: false,
|
rrdp_dedup: false,
|
||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
@ -1502,6 +1506,7 @@ authorityKeyIdentifier = keyid:always
|
|||||||
rsync_fetcher: &rsync,
|
rsync_fetcher: &rsync,
|
||||||
validation_time,
|
validation_time,
|
||||||
timing: None,
|
timing: None,
|
||||||
|
download_log: None,
|
||||||
revalidate_only: false,
|
revalidate_only: false,
|
||||||
rrdp_dedup: false,
|
rrdp_dedup: false,
|
||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
@ -1576,6 +1581,7 @@ authorityKeyIdentifier = keyid:always
|
|||||||
rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir),
|
rsync_fetcher: &LocalDirRsyncFetcher::new(&fixture_dir),
|
||||||
validation_time,
|
validation_time,
|
||||||
timing: None,
|
timing: None,
|
||||||
|
download_log: None,
|
||||||
revalidate_only: false,
|
revalidate_only: false,
|
||||||
rrdp_dedup: false,
|
rrdp_dedup: false,
|
||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
@ -1599,6 +1605,7 @@ authorityKeyIdentifier = keyid:always
|
|||||||
rsync_fetcher: &FailingRsyncFetcher,
|
rsync_fetcher: &FailingRsyncFetcher,
|
||||||
validation_time,
|
validation_time,
|
||||||
timing: None,
|
timing: None,
|
||||||
|
download_log: None,
|
||||||
revalidate_only: false,
|
revalidate_only: false,
|
||||||
rrdp_dedup: false,
|
rrdp_dedup: false,
|
||||||
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
rrdp_repo_cache: Mutex::new(HashMap::new()),
|
||||||
|
|||||||
@ -114,6 +114,7 @@ fn apnic_live_bootstrap_snapshot_and_fetch_cache_pp_pack_to_persistent_db() {
|
|||||||
&http,
|
&http,
|
||||||
&rsync,
|
&rsync,
|
||||||
None,
|
None,
|
||||||
|
None,
|
||||||
)
|
)
|
||||||
.expect("repo sync");
|
.expect("repo sync");
|
||||||
|
|
||||||
|
|||||||
@ -167,6 +167,7 @@ fn apnic_tree_full_stats_serial() {
|
|||||||
rsync_fetcher: &rsync,
|
rsync_fetcher: &rsync,
|
||||||
validation_time,
|
validation_time,
|
||||||
timing: None,
|
timing: None,
|
||||||
|
download_log: None,
|
||||||
revalidate_only: false,
|
revalidate_only: false,
|
||||||
rrdp_dedup: true,
|
rrdp_dedup: true,
|
||||||
rrdp_repo_cache: std::sync::Mutex::new(std::collections::HashMap::new()),
|
rrdp_repo_cache: std::sync::Mutex::new(std::collections::HashMap::new()),
|
||||||
|
|||||||
@ -37,5 +37,5 @@ fn cli_run_offline_mode_executes_and_writes_json() {
|
|||||||
|
|
||||||
let bytes = std::fs::read(&report_path).expect("read report json");
|
let bytes = std::fs::read(&report_path).expect("read report json");
|
||||||
let v: serde_json::Value = serde_json::from_slice(&bytes).expect("parse report json");
|
let v: serde_json::Value = serde_json::from_slice(&bytes).expect("parse report json");
|
||||||
assert_eq!(v["format_version"], 1);
|
assert_eq!(v["format_version"], 2);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -44,10 +44,12 @@ fn cli_offline_smoke_writes_report_json() {
|
|||||||
|
|
||||||
let bytes = std::fs::read(&report_path).expect("read report json");
|
let bytes = std::fs::read(&report_path).expect("read report json");
|
||||||
let v: serde_json::Value = serde_json::from_slice(&bytes).expect("parse report json");
|
let v: serde_json::Value = serde_json::from_slice(&bytes).expect("parse report json");
|
||||||
assert_eq!(v["format_version"], 1);
|
assert_eq!(v["format_version"], 2);
|
||||||
assert!(v.get("policy").is_some());
|
assert!(v.get("policy").is_some());
|
||||||
assert!(v.get("tree").is_some());
|
assert!(v.get("tree").is_some());
|
||||||
assert!(v.get("publication_points").is_some());
|
assert!(v.get("publication_points").is_some());
|
||||||
assert!(v.get("vrps").is_some());
|
assert!(v.get("vrps").is_some());
|
||||||
assert!(v.get("aspas").is_some());
|
assert!(v.get("aspas").is_some());
|
||||||
|
assert!(v.get("downloads").is_some());
|
||||||
|
assert!(v.get("download_stats").is_some());
|
||||||
}
|
}
|
||||||
|
|||||||
@ -92,6 +92,7 @@ fn build_fetch_cache_pp_from_local_rsync_fixture(
|
|||||||
&NoopHttpFetcher,
|
&NoopHttpFetcher,
|
||||||
&LocalDirRsyncFetcher::new(dir),
|
&LocalDirRsyncFetcher::new(dir),
|
||||||
None,
|
None,
|
||||||
|
None,
|
||||||
)
|
)
|
||||||
.expect("sync into raw_objects");
|
.expect("sync into raw_objects");
|
||||||
|
|
||||||
|
|||||||
@ -84,6 +84,7 @@ fn repo_sync_uses_rrdp_when_available() {
|
|||||||
&http_fetcher,
|
&http_fetcher,
|
||||||
&rsync_fetcher,
|
&rsync_fetcher,
|
||||||
None,
|
None,
|
||||||
|
None,
|
||||||
)
|
)
|
||||||
.expect("sync");
|
.expect("sync");
|
||||||
|
|
||||||
@ -137,6 +138,7 @@ fn repo_sync_skips_snapshot_when_state_unchanged() {
|
|||||||
&http_fetcher,
|
&http_fetcher,
|
||||||
&rsync_fetcher,
|
&rsync_fetcher,
|
||||||
None,
|
None,
|
||||||
|
None,
|
||||||
)
|
)
|
||||||
.expect("sync 1");
|
.expect("sync 1");
|
||||||
assert_eq!(out1.source, RepoSyncSource::Rrdp);
|
assert_eq!(out1.source, RepoSyncSource::Rrdp);
|
||||||
@ -150,6 +152,7 @@ fn repo_sync_skips_snapshot_when_state_unchanged() {
|
|||||||
&http_fetcher,
|
&http_fetcher,
|
||||||
&rsync_fetcher,
|
&rsync_fetcher,
|
||||||
None,
|
None,
|
||||||
|
None,
|
||||||
)
|
)
|
||||||
.expect("sync 2");
|
.expect("sync 2");
|
||||||
assert_eq!(out2.source, RepoSyncSource::Rrdp);
|
assert_eq!(out2.source, RepoSyncSource::Rrdp);
|
||||||
@ -203,6 +206,7 @@ fn repo_sync_falls_back_to_rsync_on_rrdp_failure() {
|
|||||||
&http_fetcher,
|
&http_fetcher,
|
||||||
&rsync_fetcher,
|
&rsync_fetcher,
|
||||||
None,
|
None,
|
||||||
|
None,
|
||||||
)
|
)
|
||||||
.expect("fallback sync");
|
.expect("fallback sync");
|
||||||
|
|
||||||
@ -252,6 +256,7 @@ fn repo_sync_rsync_populates_raw_objects() {
|
|||||||
&http_fetcher,
|
&http_fetcher,
|
||||||
&rsync_fetcher,
|
&rsync_fetcher,
|
||||||
None,
|
None,
|
||||||
|
None,
|
||||||
)
|
)
|
||||||
.expect("rsync-only sync");
|
.expect("rsync-only sync");
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user