add delta sync and fail fetch process

src/bin/db_stats.rs

@@ -0,0 +1,125 @@
+use std::path::PathBuf;
+
+use rocksdb::{ColumnFamilyDescriptor, DB, IteratorMode, Options};
+
+const CF_RAW_OBJECTS: &str = "raw_objects";
+const CF_FETCH_CACHE_PP: &str = "fetch_cache_pp";
+const CF_RRDP_STATE: &str = "rrdp_state";
+const CF_RRDP_OBJECT_INDEX: &str = "rrdp_object_index";
+
+fn enable_blobdb_if_supported(opts: &mut Options) {
+    // Keep this in sync with `rpki::storage`:
+    // blob files are CF-level options; readers should open CFs with blob enabled too.
+    #[allow(dead_code)]
+    fn _set(opts: &mut Options) {
+        opts.set_enable_blob_files(true);
+        opts.set_min_blob_size(1024);
+    }
+    _set(opts);
+}
+
+fn usage() -> String {
+    let bin = "db_stats";
+    format!(
+        "\
+Usage:
+  {bin} --db <path> [--exact]
+
+Options:
+  --db <path>     RocksDB directory
+  --exact         Iterate to count keys (slower; default uses RocksDB estimates)
+  --help          Show this help
+"
+    )
+}
+
+fn cf_descriptors() -> Vec<ColumnFamilyDescriptor> {
+    let mut cf_opts = Options::default();
+    enable_blobdb_if_supported(&mut cf_opts);
+    vec![
+        ColumnFamilyDescriptor::new(CF_RAW_OBJECTS, cf_opts.clone()),
+        ColumnFamilyDescriptor::new(CF_FETCH_CACHE_PP, cf_opts.clone()),
+        ColumnFamilyDescriptor::new(CF_RRDP_STATE, cf_opts.clone()),
+        ColumnFamilyDescriptor::new(CF_RRDP_OBJECT_INDEX, cf_opts),
+    ]
+}
+
+fn estimate_keys(db: &DB, cf_name: &str) -> Result<Option<u64>, Box<dyn std::error::Error>> {
+    let cf = db
+        .cf_handle(cf_name)
+        .ok_or_else(|| format!("missing column family: {cf_name}"))?;
+    Ok(db.property_int_value_cf(cf, "rocksdb.estimate-num-keys")?)
+}
+
+fn exact_keys(db: &DB, cf_name: &str) -> Result<u64, Box<dyn std::error::Error>> {
+    let cf = db
+        .cf_handle(cf_name)
+        .ok_or_else(|| format!("missing column family: {cf_name}"))?;
+    let mode = IteratorMode::Start;
+    let mut count = 0u64;
+    for res in db.iterator_cf(cf, mode) {
+        res?;
+        count += 1;
+    }
+    Ok(count)
+}
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let argv: Vec<String> = std::env::args().collect();
+    if argv.iter().any(|a| a == "--help" || a == "-h") {
+        print!("{}", usage());
+        return Ok(());
+    }
+
+    let mut db_path: Option<PathBuf> = None;
+    let mut exact = false;
+    let mut i = 1usize;
+    while i < argv.len() {
+        match argv[i].as_str() {
+            "--db" => {
+                i += 1;
+                let v = argv.get(i).ok_or("--db requires a value")?;
+                db_path = Some(PathBuf::from(v));
+            }
+            "--exact" => exact = true,
+            other => return Err(format!("unknown argument: {other}\n\n{}", usage()).into()),
+        }
+        i += 1;
+    }
+
+    let db_path = db_path.ok_or_else(|| format!("--db is required\n\n{}", usage()))?;
+
+    let mut opts = Options::default();
+    opts.create_if_missing(false);
+    opts.create_missing_column_families(false);
+
+    let db = DB::open_cf_descriptors(&opts, &db_path, cf_descriptors())?;
+
+    let cf_names = [
+        CF_RAW_OBJECTS,
+        CF_FETCH_CACHE_PP,
+        CF_RRDP_STATE,
+        CF_RRDP_OBJECT_INDEX,
+    ];
+
+    println!("db={}", db_path.display());
+    println!("mode={}", if exact { "exact" } else { "estimate" });
+
+    let mut total: u64 = 0;
+    for name in cf_names {
+        let n = if exact {
+            exact_keys(&db, name)?
+        } else {
+            estimate_keys(&db, name)?.unwrap_or(0)
+        };
+        total = total.saturating_add(n);
+        println!("{name}={n}");
+    }
+    println!("total={total}");
+
+    // Also print # of SST files (useful sanity signal).
+    let live = db.live_files()?;
+    println!("sst_files={}", live.len());
+
+    Ok(())
+}

src/bin/rrdp_state_dump.rs

@@ -0,0 +1,85 @@
+use std::path::PathBuf;
+
+use rocksdb::{ColumnFamilyDescriptor, DB, IteratorMode, Options};
+
+fn enable_blobdb_if_supported(opts: &mut Options) {
+    // Keep this in sync with `rpki::storage`:
+    // blob files are CF-level options; readers should open CFs with blob enabled too.
+    #[allow(dead_code)]
+    fn _set(opts: &mut Options) {
+        opts.set_enable_blob_files(true);
+        opts.set_min_blob_size(1024);
+    }
+    _set(opts);
+}
+
+fn usage() -> String {
+    let bin = "rrdp_state_dump";
+    format!(
+        "\
+Usage:
+  {bin} --db <path>
+
+Options:
+  --db <path>     RocksDB directory
+  --help          Show this help
+"
+    )
+}
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let argv: Vec<String> = std::env::args().collect();
+    if argv.iter().any(|a| a == "--help" || a == "-h") {
+        print!("{}", usage());
+        return Ok(());
+    }
+
+    let mut db_path: Option<PathBuf> = None;
+    let mut i = 1usize;
+    while i < argv.len() {
+        match argv[i].as_str() {
+            "--db" => {
+                i += 1;
+                let v = argv.get(i).ok_or("--db requires a value")?;
+                db_path = Some(PathBuf::from(v));
+            }
+            other => return Err(format!("unknown argument: {other}\n\n{}", usage()).into()),
+        }
+        i += 1;
+    }
+
+    let db_path = db_path.ok_or_else(|| format!("--db is required\n\n{}", usage()))?;
+
+    let mut opts = Options::default();
+    opts.create_if_missing(false);
+    opts.create_missing_column_families(false);
+
+    // Open only the column families we need.
+    let mut cf_opts = Options::default();
+    enable_blobdb_if_supported(&mut cf_opts);
+    let cfs = vec![
+        ColumnFamilyDescriptor::new("raw_objects", cf_opts.clone()),
+        ColumnFamilyDescriptor::new("fetch_cache_pp", cf_opts.clone()),
+        ColumnFamilyDescriptor::new("rrdp_state", cf_opts.clone()),
+        ColumnFamilyDescriptor::new("rrdp_object_index", cf_opts),
+    ];
+    let db = DB::open_cf_descriptors(&opts, &db_path, cfs)?;
+    let cf = db
+        .cf_handle("rrdp_state")
+        .ok_or("missing column family: rrdp_state")?;
+
+    let mut out: Vec<(String, u64, String)> = Vec::new();
+    for res in db.iterator_cf(cf, IteratorMode::Start) {
+        let (k, v) = res?;
+        let k = String::from_utf8_lossy(&k).to_string();
+        let st = rpki::sync::rrdp::RrdpState::decode(&v)
+            .map_err(|e| format!("decode rrdp_state failed for {k}: {e}"))?;
+        out.push((k, st.serial, st.session_id));
+    }
+
+    out.sort_by(|a, b| a.0.cmp(&b.0));
+    for (k, serial, session) in out {
+        println!("{k}\t{serial}\t{session}");
+    }
+    Ok(())
+}