129 lines
4.1 KiB
Rust
129 lines
4.1 KiB
Rust
use std::path::{Path, PathBuf};
|
|
use std::process::Command;
|
|
use std::time::{Duration, Instant};
|
|
|
|
use rpki::storage::RocksStore;
|
|
|
|
const RSYNC_BASE_URI: &str = "rsync://rpki.luys.cloud/repo/LY-RPKI/1/";
|
|
|
|
fn run_rsync(src: &str, dst: &Path, timeout: Duration) -> Result<(), String> {
|
|
let timeout_secs = timeout.as_secs().max(1).to_string();
|
|
let out = Command::new("rsync")
|
|
.arg("-rt")
|
|
.arg("--delete")
|
|
.arg("--timeout")
|
|
.arg(timeout_secs)
|
|
.arg(src)
|
|
.arg(dst)
|
|
.output()
|
|
.map_err(|e| format!("rsync spawn failed: {e}"))?;
|
|
if !out.status.success() {
|
|
return Err(format!(
|
|
"rsync failed: status={} stdout={} stderr={}",
|
|
out.status,
|
|
String::from_utf8_lossy(&out.stdout).trim(),
|
|
String::from_utf8_lossy(&out.stderr).trim()
|
|
));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn normalize_rsync_base_uri(s: &str) -> String {
|
|
if s.ends_with('/') {
|
|
s.to_string()
|
|
} else {
|
|
format!("{s}/")
|
|
}
|
|
}
|
|
|
|
fn walk_files(root: &Path, current: &Path, out: &mut Vec<PathBuf>) -> Result<(), String> {
|
|
for entry in std::fs::read_dir(current).map_err(|e| e.to_string())? {
|
|
let entry = entry.map_err(|e| e.to_string())?;
|
|
let path = entry.path();
|
|
let meta = entry.metadata().map_err(|e| e.to_string())?;
|
|
if meta.is_dir() {
|
|
walk_files(root, &path, out)?;
|
|
} else if meta.is_file() {
|
|
out.push(path);
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "live network rsync breakdown; prints timing table"]
|
|
fn rsync_fallback_breakdown_luys_cloud() {
|
|
let rsync_base_uri = normalize_rsync_base_uri(RSYNC_BASE_URI);
|
|
let timeout = Duration::from_secs(30 * 60);
|
|
|
|
let temp_dir = tempfile::tempdir().expect("tempdir");
|
|
let dst = temp_dir.path().join("rsync");
|
|
std::fs::create_dir_all(&dst).expect("create dst dir");
|
|
|
|
let t0 = Instant::now();
|
|
run_rsync(&rsync_base_uri, &dst, timeout).expect("rsync fetch");
|
|
let rsync_wall = t0.elapsed();
|
|
|
|
let t1 = Instant::now();
|
|
let mut files: Vec<PathBuf> = Vec::new();
|
|
walk_files(&dst, &dst, &mut files).expect("walk files");
|
|
let walk_wall = t1.elapsed();
|
|
|
|
let t2 = Instant::now();
|
|
let mut objects: Vec<(String, Vec<u8>)> = Vec::with_capacity(files.len());
|
|
let mut total_bytes: u64 = 0;
|
|
for path in &files {
|
|
let rel = path
|
|
.strip_prefix(&dst)
|
|
.expect("strip prefix")
|
|
.to_string_lossy()
|
|
.replace('\\', "/");
|
|
let uri = format!("{rsync_base_uri}{rel}");
|
|
let bytes = std::fs::read(path).expect("read file");
|
|
total_bytes = total_bytes.saturating_add(bytes.len() as u64);
|
|
objects.push((uri, bytes));
|
|
}
|
|
let read_wall = t2.elapsed();
|
|
|
|
let db_dir = tempfile::tempdir().expect("db tempdir");
|
|
let store = RocksStore::open(db_dir.path()).expect("open rocksdb");
|
|
|
|
let t3 = Instant::now();
|
|
for (uri, bytes) in &objects {
|
|
store.put_raw(uri, bytes).expect("put_raw");
|
|
}
|
|
let write_wall = t3.elapsed();
|
|
|
|
println!("rsync_base_uri: {rsync_base_uri}");
|
|
println!("| phase | duration(ms) | pct | notes |\n|---|---:|---:|---|");
|
|
let total = rsync_wall + walk_wall + read_wall + write_wall;
|
|
let pct = |d: Duration| (d.as_secs_f64() / total.as_secs_f64()) * 100.0;
|
|
println!(
|
|
"| rsync_network | {:>11.3} | {:>5.1}% | rsync command wall time |",
|
|
rsync_wall.as_secs_f64() * 1000.0,
|
|
pct(rsync_wall)
|
|
);
|
|
println!(
|
|
"| walk_files | {:>11.3} | {:>5.1}% | enumerate local files |",
|
|
walk_wall.as_secs_f64() * 1000.0,
|
|
pct(walk_wall)
|
|
);
|
|
println!(
|
|
"| read_files | {:>11.3} | {:>5.1}% | read all bytes into memory |",
|
|
read_wall.as_secs_f64() * 1000.0,
|
|
pct(read_wall)
|
|
);
|
|
println!(
|
|
"| rocksdb_put_raw | {:>11.3} | {:>5.1}% | write raw_objects ({} keys) |",
|
|
write_wall.as_secs_f64() * 1000.0,
|
|
pct(write_wall),
|
|
objects.len()
|
|
);
|
|
println!(
|
|
"| TOTAL | {:>11.3} | 100.0% | files={} bytes={} |",
|
|
total.as_secs_f64() * 1000.0,
|
|
objects.len(),
|
|
total_bytes
|
|
);
|
|
}
|